From d459aa88ff233afb168b9f64a66a5cd6367b9b8b Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Sat, 21 Jan 2023 13:55:55 +0100 Subject: [PATCH 1/8] Add support for KCP remediation during cluster provisioning --- .../kubeadm/api/v1alpha3/conversion.go | 7 + .../api/v1alpha3/zz_generated.conversion.go | 2 + .../kubeadm/api/v1alpha4/conversion.go | 13 + .../api/v1alpha4/zz_generated.conversion.go | 17 +- .../v1beta1/kubeadm_control_plane_types.go | 76 ++ .../api/v1beta1/zz_generated.deepcopy.go | 52 + ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 76 ++ .../kubeadm/internal/controllers/helpers.go | 12 + .../internal/controllers/remediation.go | 248 +++- .../internal/controllers/remediation_test.go | 1141 ++++++++++++++--- .../machinehealthcheck_controller.go | 20 +- .../machinehealthcheck_targets.go | 24 +- test/e2e/config/docker.yaml | 3 + .../cluster-with-kcp.yaml | 98 ++ .../kustomization.yaml | 2 +- .../cluster-template-kcp-remediation/mhc.yaml | 7 +- test/e2e/kcp_remediations.go | 693 ++++++++++ ...tions_test.go => kcp_remediations_test.go} | 6 +- ...mhc_remediations.go => md_remediations.go} | 62 +- test/e2e/md_remediations_test.go | 36 + test/infrastructure/container/docker.go | 3 +- .../controllers/dockermachine_controller.go | 31 +- 22 files changed, 2315 insertions(+), 314 deletions(-) create mode 100644 test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/cluster-with-kcp.yaml create mode 100644 test/e2e/kcp_remediations.go rename test/e2e/{mhc_remediations_test.go => kcp_remediations_test.go} (83%) rename test/e2e/{mhc_remediations.go => md_remediations.go} (61%) create mode 100644 test/e2e/md_remediations_test.go diff --git a/controlplane/kubeadm/api/v1alpha3/conversion.go b/controlplane/kubeadm/api/v1alpha3/conversion.go index 0061e6331a8a..e8fb091e02d0 100644 --- a/controlplane/kubeadm/api/v1alpha3/conversion.go +++ b/controlplane/kubeadm/api/v1alpha3/conversion.go @@ -99,6 +99,13 @@ func (src *KubeadmControlPlane) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.ImagePullPolicy = restored.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.ImagePullPolicy } + if restored.Spec.RemediationStrategy != nil { + dst.Spec.RemediationStrategy = restored.Spec.RemediationStrategy + } + if restored.Status.LastRemediation != nil { + dst.Status.LastRemediation = restored.Status.LastRemediation + } + return nil } diff --git a/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go b/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go index 94b5206584e7..ce3672490458 100644 --- a/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go +++ b/controlplane/kubeadm/api/v1alpha3/zz_generated.conversion.go @@ -201,6 +201,7 @@ func autoConvert_v1beta1_KubeadmControlPlaneSpec_To_v1alpha3_KubeadmControlPlane // WARNING: in.RolloutBefore requires manual conversion: does not exist in peer-type // WARNING: in.RolloutAfter requires manual conversion: does not exist in peer-type out.RolloutStrategy = (*RolloutStrategy)(unsafe.Pointer(in.RolloutStrategy)) + // WARNING: in.RemediationStrategy requires manual conversion: does not exist in peer-type return nil } @@ -257,6 +258,7 @@ func autoConvert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha3_KubeadmControlPla } else { out.Conditions = nil } + // WARNING: in.LastRemediation requires manual conversion: does not exist in peer-type return nil } diff --git a/controlplane/kubeadm/api/v1alpha4/conversion.go b/controlplane/kubeadm/api/v1alpha4/conversion.go index 9fae7f390022..9da266d9213d 100644 --- a/controlplane/kubeadm/api/v1alpha4/conversion.go +++ b/controlplane/kubeadm/api/v1alpha4/conversion.go @@ -84,6 +84,13 @@ func (src *KubeadmControlPlane) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.ImagePullPolicy = restored.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.ImagePullPolicy } + if restored.Spec.RemediationStrategy != nil { + dst.Spec.RemediationStrategy = restored.Spec.RemediationStrategy + } + if restored.Status.LastRemediation != nil { + dst.Status.LastRemediation = restored.Status.LastRemediation + } + return nil } @@ -262,5 +269,11 @@ func Convert_v1beta1_KubeadmControlPlaneMachineTemplate_To_v1alpha4_KubeadmContr func Convert_v1beta1_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in *controlplanev1.KubeadmControlPlaneSpec, out *KubeadmControlPlaneSpec, scope apiconversion.Scope) error { // .RolloutBefore was added in v1beta1. + // .RemediationStrategy was added in v1beta1. return autoConvert_v1beta1_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlaneSpec(in, out, scope) } + +func Convert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(in *controlplanev1.KubeadmControlPlaneStatus, out *KubeadmControlPlaneStatus, scope apiconversion.Scope) error { + // .LastRemediation was added in v1beta1. + return autoConvert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(in, out, scope) +} diff --git a/controlplane/kubeadm/api/v1alpha4/zz_generated.conversion.go b/controlplane/kubeadm/api/v1alpha4/zz_generated.conversion.go index 83cdb2e4718d..efe50a58635e 100644 --- a/controlplane/kubeadm/api/v1alpha4/zz_generated.conversion.go +++ b/controlplane/kubeadm/api/v1alpha4/zz_generated.conversion.go @@ -77,11 +77,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeadmControlPlaneStatus)(nil), (*KubeadmControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(a.(*v1beta1.KubeadmControlPlaneStatus), b.(*KubeadmControlPlaneStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*KubeadmControlPlaneTemplate)(nil), (*v1beta1.KubeadmControlPlaneTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_KubeadmControlPlaneTemplate_To_v1beta1_KubeadmControlPlaneTemplate(a.(*KubeadmControlPlaneTemplate), b.(*v1beta1.KubeadmControlPlaneTemplate), scope) }); err != nil { @@ -157,6 +152,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.KubeadmControlPlaneStatus)(nil), (*KubeadmControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(a.(*v1beta1.KubeadmControlPlaneStatus), b.(*KubeadmControlPlaneStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.KubeadmControlPlaneTemplateResourceSpec)(nil), (*KubeadmControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_KubeadmControlPlaneTemplateResourceSpec_To_v1alpha4_KubeadmControlPlaneSpec(a.(*v1beta1.KubeadmControlPlaneTemplateResourceSpec), b.(*KubeadmControlPlaneSpec), scope) }); err != nil { @@ -295,6 +295,7 @@ func autoConvert_v1beta1_KubeadmControlPlaneSpec_To_v1alpha4_KubeadmControlPlane // WARNING: in.RolloutBefore requires manual conversion: does not exist in peer-type out.RolloutAfter = (*v1.Time)(unsafe.Pointer(in.RolloutAfter)) out.RolloutStrategy = (*RolloutStrategy)(unsafe.Pointer(in.RolloutStrategy)) + // WARNING: in.RemediationStrategy requires manual conversion: does not exist in peer-type return nil } @@ -352,14 +353,10 @@ func autoConvert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPla } else { out.Conditions = nil } + // WARNING: in.LastRemediation requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus is an autogenerated conversion function. -func Convert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(in *v1beta1.KubeadmControlPlaneStatus, out *KubeadmControlPlaneStatus, s conversion.Scope) error { - return autoConvert_v1beta1_KubeadmControlPlaneStatus_To_v1alpha4_KubeadmControlPlaneStatus(in, out, s) -} - func autoConvert_v1alpha4_KubeadmControlPlaneTemplate_To_v1beta1_KubeadmControlPlaneTemplate(in *KubeadmControlPlaneTemplate, out *v1beta1.KubeadmControlPlaneTemplate, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha4_KubeadmControlPlaneTemplateSpec_To_v1beta1_KubeadmControlPlaneTemplateSpec(&in.Spec, &out.Spec, s); err != nil { diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go index 823fb123f679..f4661380ccd3 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1beta1 import ( + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -49,6 +51,23 @@ const ( // KubeadmClusterConfigurationAnnotation is a machine annotation that stores the json-marshalled string of KCP ClusterConfiguration. // This annotation is used to detect any changes in ClusterConfiguration and trigger machine rollout in KCP. KubeadmClusterConfigurationAnnotation = "controlplane.cluster.x-k8s.io/kubeadm-cluster-configuration" + + // RemediatingInProgressAnnotation is used to keep track that a KCP remediation is in progress, and more + // specifically it tracks that the system is in between having deleted an unhealthy machine and recreating its replacement. + // NOTE: if something external to CAPI removes this annotation the system cannot detect the above situation; this can lead to + // failures in updating remediation retry or remediation count (both counters restart from zero). + RemediatingInProgressAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-in-progress" + + // MachineRemediationForAnnotation is used to link a new machine to the unhealthy machine it is replacing; + // please note that in case of retry, when also the remediating machine fails, the system keep tracks + // the first machine of the sequence only. + // NOTE: if something external to CAPI removes this annotation the system this can lead to + // failures in updating remediation retry (the counter restarts from zero). + MachineRemediationForAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-for" + + // DefaultMinHealthyPeriod defines the default minimum number of seconds before we consider a remediation on a + // machine unrelated from the previous remediation. + DefaultMinHealthyPeriod = 1 * time.Hour ) // KubeadmControlPlaneSpec defines the desired state of KubeadmControlPlane. @@ -91,6 +110,10 @@ type KubeadmControlPlaneSpec struct { // +optional // +kubebuilder:default={type: "RollingUpdate", rollingUpdate: {maxSurge: 1}} RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"` + + // The RemediationStrategy that controls how control plane machines remediation happens. + // +optional + RemediationStrategy *RemediationStrategy `json:"remediationStrategy,omitempty"` } // KubeadmControlPlaneMachineTemplate defines the template for Machines @@ -158,6 +181,40 @@ type RollingUpdate struct { MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` } +// RemediationStrategy allows to define how control plane machines remediation happens. +type RemediationStrategy struct { + // MaxRetry is the Max number of retry while attempting to remediate an unhealthy machine. + // A retry happens when a machine that was created as a replacement for an unhealthy machine also fails. + // For example, given a control plane with three machines M1, M2, M3: + // + // M1 become unhealthy; remediation happens, and M1bis is created as a replacement. + // If M1-1 (replacement of M1) have problems while bootstrapping it will become unhealthy, and then be + // remediated; such operation is considered a retry, remediation-retry #1. + // If M1-2 (replacement of M1-2) becomes unhealthy, remediation-retry #2 will happen, etc. + // + // A retry could happen only after RetryPeriod from the previous retry. + // If a machine is marked as unhealthy after MinHealthyPeriod from the previous remediation expired, + // this is not considered anymore a retry because the new issue is assumed unrelated from the previous one. + // + // If not set, infinite retry will be attempted. + // +optional + MaxRetry *int32 `json:"maxRetry,omitempty"` + + // RetryPeriod is the duration that KCP should wait before remediating a machine being created as a replacement + // for an unhealthy machine (a retry). + // + // If not set, a retry will happen immediately. + // +optional + RetryPeriod metav1.Duration `json:"retryDelaySeconds,omitempty"` + + // MinHealthyPeriod defines the duration after which KCP will consider any failure to a machine unrelated + // from the previous one, and thus a new remediation is not considered a retry anymore. + // + // If not set, this value is defaulted to 1h. + // +optional + MinHealthyPeriod *metav1.Duration `json:"minHealthySeconds,omitempty"` +} + // KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane. type KubeadmControlPlaneStatus struct { // Selector is the label selector in string format to avoid introspection @@ -223,6 +280,25 @@ type KubeadmControlPlaneStatus struct { // Conditions defines current service state of the KubeadmControlPlane. // +optional Conditions clusterv1.Conditions `json:"conditions,omitempty"` + + // LastRemediation stores info about last remediation performed. + // +optional + LastRemediation *LastRemediationStatus `json:"lastRemediation,omitempty"` +} + +// LastRemediationStatus stores info about last remediation performed. +// NOTE: if for any reason information about last remediation are lost, RetryCount is going to restarts from 0 and thus +// more remediation than expected might happen. +type LastRemediationStatus struct { + // Machine is the machine name of the latest machine being remediated. + Machine string `json:"machine"` + + // Timestamp is RFC 3339 date and time at which last remediation happened. + Timestamp metav1.Timestamp `json:"timestamp"` + + // RetryCount used to keep track of remediation retry for the last remediated machine. + // A retry happens when a machine that was created as a replacement for an unhealthy machine also fails. + RetryCount int32 `json:"retryCount"` } // +kubebuilder:object:root=true diff --git a/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go index 27f9d2ecd98a..010dd455c1ea 100644 --- a/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go +++ b/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go @@ -143,6 +143,11 @@ func (in *KubeadmControlPlaneSpec) DeepCopyInto(out *KubeadmControlPlaneSpec) { *out = new(RolloutStrategy) (*in).DeepCopyInto(*out) } + if in.RemediationStrategy != nil { + in, out := &in.RemediationStrategy, &out.RemediationStrategy + *out = new(RemediationStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmControlPlaneSpec. @@ -175,6 +180,11 @@ func (in *KubeadmControlPlaneStatus) DeepCopyInto(out *KubeadmControlPlaneStatus (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.LastRemediation != nil { + in, out := &in.LastRemediation, &out.LastRemediation + *out = new(LastRemediationStatus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmControlPlaneStatus. @@ -342,6 +352,48 @@ func (in *KubeadmControlPlaneTemplateSpec) DeepCopy() *KubeadmControlPlaneTempla return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LastRemediationStatus) DeepCopyInto(out *LastRemediationStatus) { + *out = *in + out.Timestamp = in.Timestamp +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LastRemediationStatus. +func (in *LastRemediationStatus) DeepCopy() *LastRemediationStatus { + if in == nil { + return nil + } + out := new(LastRemediationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemediationStrategy) DeepCopyInto(out *RemediationStrategy) { + *out = *in + if in.MaxRetry != nil { + in, out := &in.MaxRetry, &out.MaxRetry + *out = new(int32) + **out = **in + } + out.RetryPeriod = in.RetryPeriod + if in.MinHealthyPeriod != nil { + in, out := &in.MinHealthyPeriod, &out.MinHealthyPeriod + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemediationStrategy. +func (in *RemediationStrategy) DeepCopy() *RemediationStrategy { + if in == nil { + return nil + } + out := new(RemediationStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RollingUpdate) DeepCopyInto(out *RollingUpdate) { *out = *in diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index da79fa45d92d..c36b809eeb56 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -3620,6 +3620,41 @@ spec: required: - infrastructureRef type: object + remediationStrategy: + description: The RemediationStrategy that controls how control plane + machines remediation happens. + properties: + maxRetry: + description: "MaxRetry is the Max number of retry while attempting + to remediate an unhealthy machine. A retry happens when a machine + that was created as a replacement for an unhealthy machine also + fails. For example, given a control plane with three machines + M1, M2, M3: \n M1 become unhealthy; remediation happens, and + M1bis is created as a replacement. If M1-1 (replacement of M1) + have problems while bootstrapping it will become unhealthy, + and then be remediated; such operation is considered a retry, + remediation-retry #1. If M1-2 (replacement of M1-2) becomes + unhealthy, remediation-retry #2 will happen, etc. \n A retry + could happen only after RetryPeriod from the previous retry. + If a machine is marked as unhealthy after MinHealthyPeriod from + the previous remediation expired, this is not considered anymore + a retry because the new issue is assumed unrelated from the + previous one. \n If not set, infinite retry will be attempted." + format: int32 + type: integer + minHealthySeconds: + description: "MinHealthyPeriod defines the duration after which + KCP will consider any failure to a machine unrelated from the + previous one, and thus a new remediation is not considered a + retry anymore. \n If not set, this value is defaulted to 1h." + type: string + retryDelaySeconds: + description: "RetryPeriod is the duration that KCP should wait + before remediating a machine being created as a replacement + for an unhealthy machine (a retry). \n If not set, a retry will + happen immediately." + type: string + type: object replicas: description: Number of desired machines. Defaults to 1. When stacked etcd is used only odd numbers are permitted, as per [etcd best practice](https://etcd.io/docs/v3.3.12/faq/#why-an-odd-number-of-cluster-members). @@ -3746,6 +3781,47 @@ spec: description: Initialized denotes whether or not the control plane has the uploaded kubeadm-config configmap. type: boolean + lastRemediation: + description: LastRemediation stores info about last remediation performed. + properties: + machine: + description: Machine is the machine name of the latest machine + being remediated. + type: string + retryCount: + description: RetryCount used to keep track of remediation retry + for the last remediated machine. A retry happens when a machine + that was created as a replacement for an unhealthy machine also + fails. + format: int32 + type: integer + timestamp: + description: Timestamp is RFC 3339 date and time at which last + remediation happened. + properties: + nanos: + description: Non-negative fractions of a second at nanosecond + resolution. Negative second values with fractions must still + have non-negative nanos values that count forward in time. + Must be from 0 to 999,999,999 inclusive. This field may + be limited in precision depending on context. + format: int32 + type: integer + seconds: + description: Represents seconds of UTC time since Unix epoch + 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z + to 9999-12-31T23:59:59Z inclusive. + format: int64 + type: integer + required: + - nanos + - seconds + type: object + required: + - machine + - retryCount + - timestamp + type: object observedGeneration: description: ObservedGeneration is the latest generation observed by the controller. diff --git a/controlplane/kubeadm/internal/controllers/helpers.go b/controlplane/kubeadm/internal/controllers/helpers.go index 30bfba3058a5..02fbd7bf240e 100644 --- a/controlplane/kubeadm/internal/controllers/helpers.go +++ b/controlplane/kubeadm/internal/controllers/helpers.go @@ -315,6 +315,13 @@ func (r *KubeadmControlPlaneReconciler) generateMachine(ctx context.Context, kcp machine.Spec.NodeDeletionTimeout = kcp.Spec.MachineTemplate.NodeDeletionTimeout } + // In case this machine is being created as a consequence of a remediation, then add an annotation + // tracking the name of the machine we are remediating for. + // NOTE: This is required in order to track remediation retries. + if v, ok := kcp.Annotations[controlplanev1.RemediatingInProgressAnnotation]; ok && v == "true" { + machine.Annotations[controlplanev1.MachineRemediationForAnnotation] = kcp.Status.LastRemediation.Machine + } + // Machine's bootstrap config may be missing ClusterConfiguration if it is not the first machine in the control plane. // We store ClusterConfiguration as annotation here to detect any changes in KCP ClusterConfiguration and rollout the machine if any. clusterConfig, err := json.Marshal(kcp.Spec.KubeadmConfigSpec.ClusterConfiguration) @@ -332,5 +339,10 @@ func (r *KubeadmControlPlaneReconciler) generateMachine(ctx context.Context, kcp if err := r.Client.Create(ctx, machine); err != nil { return errors.Wrap(err, "failed to create machine") } + + // Remove the annotation tracking that a remediation is in progress (the remediation completed when + // the replacement machine have been created above). + delete(kcp.Annotations, controlplanev1.RemediatingInProgressAnnotation) + return nil } diff --git a/controlplane/kubeadm/internal/controllers/remediation.go b/controlplane/kubeadm/internal/controllers/remediation.go index 429f8a2e954a..c7341b72ef49 100644 --- a/controlplane/kubeadm/internal/controllers/remediation.go +++ b/controlplane/kubeadm/internal/controllers/remediation.go @@ -19,9 +19,12 @@ package controllers import ( "context" "fmt" + "time" "github.com/blang/semver" + "github.com/go-logr/logr" "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" @@ -31,6 +34,7 @@ import ( controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/controlplane/kubeadm/internal" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/patch" ) @@ -67,6 +71,11 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C return ctrl.Result{}, kerrors.NewAggregate(errList) } + // Returns if another remediation is in progress but the new machine is not yet created. + if _, ok := controlPlane.KCP.Annotations[controlplanev1.RemediatingInProgressAnnotation]; ok { + return ctrl.Result{}, nil + } + // Gets all machines that have `MachineHealthCheckSucceeded=False` (indicating a problem was detected on the machine) // and `MachineOwnerRemediated` present, indicating that this controller is responsible for performing remediation. unhealthyMachines := controlPlane.UnhealthyMachines() @@ -107,81 +116,92 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // Before starting remediation, run preflight checks in order to verify it is safe to remediate. // If any of the following checks fails, we'll surface the reason in the MachineOwnerRemediated condition. - log.WithValues("Machine", klog.KObj(machineToBeRemediated)) - desiredReplicas := int(*controlPlane.KCP.Spec.Replicas) - - // The cluster MUST have more than one replica, because this is the smallest cluster size that allows any etcd failure tolerance. - if controlPlane.Machines.Len() <= 1 { - log.Info("A control plane machine needs remediation, but the number of current replicas is less or equal to 1. Skipping remediation", "Replicas", controlPlane.Machines.Len()) - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") - return ctrl.Result{}, nil - } - - // The number of replicas MUST be equal to or greater than the desired replicas. This rule ensures that when the cluster - // is missing replicas, we skip remediation and instead perform regular scale up/rollout operations first. - if controlPlane.Machines.Len() < desiredReplicas { - log.Info("A control plane machine needs remediation, but the current number of replicas is lower that expected. Skipping remediation", "Replicas", desiredReplicas, "CurrentReplicas", controlPlane.Machines.Len()) - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for having at least %d control plane machines before triggering remediation", desiredReplicas) - return ctrl.Result{}, nil - } + log.WithValues("Machine", klog.KObj(machineToBeRemediated), "Initialized", controlPlane.KCP.Status.Initialized) - // The cluster MUST have no machines with a deletion timestamp. This rule prevents KCP taking actions while the cluster is in a transitional state. - if controlPlane.HasDeletingMachine() { - log.Info("A control plane machine needs remediation, but there are other control-plane machines being deleted. Skipping remediation") - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") + // Check if KCP is allowed to remediate considering retry limits: + // - Remediation cannot happen because retryPeriod is not yet expired. + // - KCP already reached MaxRetries limit. + hasRetries, retryCount := r.checkRetryLimits(log, machineToBeRemediated, controlPlane) + if !hasRetries { return ctrl.Result{}, nil } - // Remediation MUST preserve etcd quorum. This rule ensures that we will not remove a member that would result in etcd - // losing a majority of members and thus become unable to field new requests. - if controlPlane.IsEtcdManaged() { - canSafelyRemediate, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, machineToBeRemediated) - if err != nil { - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) - return ctrl.Result{}, err + // Executes checks that applies only if the control plane is already initialized; in this case KCP can + // remediate only if it can safely assume that the operation preserves the operation state of the existing cluster (or at least it doesn't make it worst). + if controlPlane.KCP.Status.Initialized { + // The cluster MUST have more than one replica, because this is the smallest cluster size that allows any etcd failure tolerance. + if controlPlane.Machines.Len() <= 1 { + log.Info("A control plane machine needs remediation, but the number of current replicas is less or equal to 1. Skipping remediation", "Replicas", controlPlane.Machines.Len()) + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") + return ctrl.Result{}, nil } - if !canSafelyRemediate { - log.Info("A control plane machine needs remediation, but removing this machine could result in etcd quorum loss. Skipping remediation") - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + + // The cluster MUST have no machines with a deletion timestamp. This rule prevents KCP taking actions while the cluster is in a transitional state. + if controlPlane.HasDeletingMachine() { + log.Info("A control plane machine needs remediation, but there are other control-plane machines being deleted. Skipping remediation") + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") return ctrl.Result{}, nil } - } - workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(controlPlane.Cluster)) - if err != nil { - log.Error(err, "Failed to create client to workload cluster") - return ctrl.Result{}, errors.Wrapf(err, "failed to create client to workload cluster") + // Remediation MUST preserve etcd quorum. This rule ensures that KCP will not remove a member that would result in etcd + // losing a majority of members and thus become unable to field new requests. + if controlPlane.IsEtcdManaged() { + canSafelyRemediate, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, machineToBeRemediated) + if err != nil { + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) + return ctrl.Result{}, err + } + if !canSafelyRemediate { + log.Info("A control plane machine needs remediation, but removing this machine could result in etcd quorum loss. Skipping remediation") + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + return ctrl.Result{}, nil + } + } } - // If the machine that is about to be deleted is the etcd leader, move it to the newest member available. - if controlPlane.IsEtcdManaged() { - etcdLeaderCandidate := controlPlane.HealthyMachines().Newest() - if etcdLeaderCandidate == nil { - log.Info("A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to") - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, - "A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to. Skipping remediation") - return ctrl.Result{}, nil - } - if err := workloadCluster.ForwardEtcdLeadership(ctx, machineToBeRemediated, etcdLeaderCandidate); err != nil { - log.Error(err, "Failed to move etcd leadership to candidate machine", "candidate", klog.KObj(etcdLeaderCandidate)) - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) - return ctrl.Result{}, err + // Remediate the unhealthy control plane machine by deleting it. + + // If the control plane is initialized, before deleting the machine: + // - if the machine hosts the etcd leader, forward etcd leadership to another machine. + // - delete the etcd member hosted on the machine being deleted. + // - remove the etcd member from the kubeadm config map (only for kubernetes version older than v1.22.0) + if controlPlane.KCP.Status.Initialized { + workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(controlPlane.Cluster)) + if err != nil { + log.Error(err, "Failed to create client to workload cluster") + return ctrl.Result{}, errors.Wrapf(err, "failed to create client to workload cluster") } - if err := workloadCluster.RemoveEtcdMemberForMachine(ctx, machineToBeRemediated); err != nil { - log.Error(err, "Failed to remove etcd member for machine") - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) - return ctrl.Result{}, err + + // If the machine that is about to be deleted is the etcd leader, move it to the newest member available. + if controlPlane.IsEtcdManaged() { + etcdLeaderCandidate := controlPlane.HealthyMachines().Newest() + if etcdLeaderCandidate == nil { + log.Info("A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to") + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, + "A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to. Skipping remediation") + return ctrl.Result{}, nil + } + if err := workloadCluster.ForwardEtcdLeadership(ctx, machineToBeRemediated, etcdLeaderCandidate); err != nil { + log.Error(err, "Failed to move etcd leadership to candidate machine", "candidate", klog.KObj(etcdLeaderCandidate)) + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) + return ctrl.Result{}, err + } + if err := workloadCluster.RemoveEtcdMemberForMachine(ctx, machineToBeRemediated); err != nil { + log.Error(err, "Failed to remove etcd member for machine") + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) + return ctrl.Result{}, err + } } - } - parsedVersion, err := semver.ParseTolerant(controlPlane.KCP.Spec.Version) - if err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version) - } + parsedVersion, err := semver.ParseTolerant(controlPlane.KCP.Spec.Version) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version) + } - if err := workloadCluster.RemoveMachineFromKubeadmConfigMap(ctx, machineToBeRemediated, parsedVersion); err != nil { - log.Error(err, "Failed to remove machine from kubeadm ConfigMap") - return ctrl.Result{}, err + if err := workloadCluster.RemoveMachineFromKubeadmConfigMap(ctx, machineToBeRemediated, parsedVersion); err != nil { + log.Error(err, "Failed to remove machine from kubeadm ConfigMap") + return ctrl.Result{}, err + } } if err := r.Client.Delete(ctx, machineToBeRemediated); err != nil { @@ -191,9 +211,107 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C log.Info("Remediating unhealthy machine") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + // Set annotations tracking remediation is in progress (remediation will complete when a replacement machine is created). + annotations.AddAnnotations(controlPlane.KCP, map[string]string{ + controlplanev1.RemediatingInProgressAnnotation: "", + }) + + // Stores info about last remediation. + // NOTE: Some of those info have been computed above, but they must surface on the object only here, after machine has been deleted. + controlPlane.KCP.Status.LastRemediation = &controlplanev1.LastRemediationStatus{ + Machine: machineToBeRemediated.Name, + Timestamp: metav1.Timestamp{Seconds: time.Now().Unix()}, + RetryCount: retryCount, + } + return ctrl.Result{Requeue: true}, nil } +// checkRetryLimits checks if KCP is allowed to remediate considering retry limits: +// - Remediation cannot happen because retryDelay is not yet expired. +// - KCP already reached the maximum number of retries for a machine. +// NOTE: Counting the number of retries is required In order to prevent infinite remediation e.g. in case the +// first Control Plane machine is failing due to quota issue. +func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machineToBeRemediated *clusterv1.Machine, controlPlane *internal.ControlPlane) (hasRetries bool, retryCount int32) { + // If there is no last remediation, this is the first try of a new retry sequence. + if controlPlane.KCP.Status.LastRemediation == nil { + return true, 0 + } + + // Gets MinHealthySeconds and RetryDelaySeconds from the remediation strategy, or use defaults. + minHealthyPeriod := controlplanev1.DefaultMinHealthyPeriod + if controlPlane.KCP.Spec.RemediationStrategy != nil && controlPlane.KCP.Spec.RemediationStrategy.MinHealthyPeriod != nil { + minHealthyPeriod = controlPlane.KCP.Spec.RemediationStrategy.MinHealthyPeriod.Duration + } + + retryDelay := time.Duration(0) + if controlPlane.KCP.Spec.RemediationStrategy != nil { + retryDelay = controlPlane.KCP.Spec.RemediationStrategy.RetryPeriod.Duration + } + + // Gets the timestamp of the last remediation; if missing, default to a value + // that ensures both MinHealthySeconds and RetryDelaySeconds are expired. + // NOTE: this could potentially lead to executing more retries than expected or to executing retries before than + // expected, but this is considered acceptable. + // when the system recovers from someone/something manually removing the LastRemediatingTimeStampAnnotation. + max := func(x, y time.Duration) time.Duration { + if x < y { + return y + } + return x + } + + lastRemediationTimestamp := time.Now().Add(-2 * max(minHealthyPeriod, retryDelay)).UTC() + if controlPlane.KCP.Status.LastRemediation != nil { + lastRemediationTimestamp = time.Unix(controlPlane.KCP.Status.LastRemediation.Timestamp.Seconds, int64(controlPlane.KCP.Status.LastRemediation.Timestamp.Nanos)) + } + + // Check if the machine being remediated has been created as a remediation for a previous unhealthy machine. + // NOTE: if someone/something manually removing the MachineRemediationForAnnotation on Machines or one of the LastRemediatedMachineAnnotation + // and LastRemediatedMachineRetryAnnotation on KCP, this could potentially lead to executing more retries than expected, + // but this is considered acceptable in such a case. + machineRemediatingFor := machineToBeRemediated.Name + if remediationFor, ok := machineToBeRemediated.Annotations[controlplanev1.MachineRemediationForAnnotation]; ok { + // If the remediation is happening before minHealthyPeriod is expired, then KCP considers this + // as a remediation for the same previously unhealthy machine. + // TODO: add example + if lastRemediationTimestamp.Add(minHealthyPeriod).After(time.Now()) { + machineRemediatingFor = remediationFor + log.WithValues("RemediationRetryFor", klog.KRef(machineToBeRemediated.Namespace, machineRemediatingFor)) + } + } + + // If remediation is happening for a different machine, this is the first try of a new retry sequence. + if controlPlane.KCP.Status.LastRemediation.Machine != machineRemediatingFor { + return true, 0 + } + + // Check if remediation can happen because retryDelay is passed. + if lastRemediationTimestamp.Add(retryDelay).After(time.Now().UTC()) { + log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed in the latest %s. Skipping remediation", retryDelay)) + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryDelay)", retryDelay) + return false, 0 + } + + // Check if remediation can happen because of maxRetry is not reached yet, if defined. + retry := controlPlane.KCP.Status.LastRemediation.RetryCount + + if controlPlane.KCP.Spec.RemediationStrategy != nil && controlPlane.KCP.Spec.RemediationStrategy.MaxRetry != nil { + maxRetry := *controlPlane.KCP.Spec.RemediationStrategy.MaxRetry + if retry >= maxRetry { + log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed %d times (MaxRetry %d). Skipping remediation", retry, maxRetry)) + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed %d times (MaxRetry)", maxRetry) + return false, 0 + } + } + + retryCount = retry + 1 + + log.WithValues("RetryCount", retryCount) + return true, retryCount +} + // canSafelyRemoveEtcdMember assess if it is possible to remove the member hosted on the machine to be remediated // without loosing etcd quorum. // @@ -208,7 +326,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // - etc. // // NOTE: this func assumes the list of members in sync with the list of machines/nodes, it is required to call reconcileEtcdMembers -// ans well as reconcileControlPlaneConditions before this. +// as well as reconcileControlPlaneConditions before this. func (r *KubeadmControlPlaneReconciler) canSafelyRemoveEtcdMember(ctx context.Context, controlPlane *internal.ControlPlane, machineToBeRemediated *clusterv1.Machine) (bool, error) { log := ctrl.LoggerFrom(ctx) @@ -258,10 +376,10 @@ func (r *KubeadmControlPlaneReconciler) canSafelyRemoveEtcdMember(ctx context.Co } } - // If an etcd member does not have a corresponding machine, it is not possible to retrieve etcd member health - // so we are assuming the worst scenario and considering the member unhealthy. + // If an etcd member does not have a corresponding machine it is not possible to retrieve etcd member health, + // so KCP is assuming the worst scenario and considering the member unhealthy. // - // NOTE: This should not happen given that we are running reconcileEtcdMembers before calling this method. + // NOTE: This should not happen given that KCP is running reconcileEtcdMembers before calling this method. if machine == nil { log.Info("An etcd member does not have a corresponding machine, assuming this member is unhealthy", "MemberName", etcdMember) targetUnhealthyMembers++ diff --git a/controlplane/kubeadm/internal/controllers/remediation_test.go b/controlplane/kubeadm/internal/controllers/remediation_test.go index e6c13e1691b3..9ce9e72acf0a 100644 --- a/controlplane/kubeadm/internal/controllers/remediation_test.go +++ b/controlplane/kubeadm/internal/controllers/remediation_test.go @@ -42,7 +42,7 @@ import ( func TestReconcileUnhealthyMachines(t *testing.T) { g := NewWithT(t) - ctx := context.TODO() + r := &KubeadmControlPlaneReconciler{ Client: env.GetClient(), recorder: record.NewFakeRecorder(32), @@ -53,7 +53,14 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(env.Cleanup(ctx, ns)).To(Succeed()) }() - t.Run("Remediation cleans up stuck remediation on previously unhealthy machines", func(t *testing.T) { + var removeFinalizer = func(g *WithT, m *clusterv1.Machine) { + patchHelper, err := patch.NewHelper(m, env.GetClient()) + g.Expect(err).ToNot(HaveOccurred()) + m.ObjectMeta.Finalizers = nil + g.Expect(patchHelper.Patch(ctx, m)) + } + + t.Run("It cleans up stuck remediation on previously unhealthy machines", func(t *testing.T) { g := NewWithT(t) m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withStuckRemediation()) @@ -63,7 +70,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m), } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) @@ -79,6 +86,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { return errors.Errorf("condition %s still exists", clusterv1.MachineOwnerRemediatedCondition) }, 10*time.Second).Should(Succeed()) }) + + // Generic preflight checks + // Those are ore flight checks that happen no matter if the control plane has been already initialized or not. + t.Run("Remediation does not happen if there are no unhealthy machines", func(t *testing.T) { g := NewWithT(t) @@ -87,12 +98,34 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Cluster: &clusterv1.Cluster{}, Machines: collections.New(), } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) }) - t.Run("reconcileUnhealthyMachines return early if the machine to be remediated is marked for deletion", func(t *testing.T) { + t.Run("reconcileUnhealthyMachines return early if another remediation is in progress", func(t *testing.T) { + g := NewWithT(t) + + m := getDeletingMachine(ns.Name, "m1-unhealthy-deleting-", withMachineHealthCheckFailed()) + conditions.MarkFalse(m, clusterv1.MachineHealthCheckSucceededCondition, clusterv1.MachineHasFailureReason, clusterv1.ConditionSeverityWarning, "") + conditions.MarkFalse(m, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controlplanev1.RemediatingInProgressAnnotation: "true", + }, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m), + } + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + }) + t.Run("reconcileUnhealthyMachines return early if the machine to be remediated is already being deleted", func(t *testing.T) { g := NewWithT(t) m := getDeletingMachine(ns.Name, "m1-unhealthy-deleting-", withMachineHealthCheckFailed()) @@ -103,92 +136,655 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m), } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) }) + t.Run("Remediation does not happen if MaxRetry is reached", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + RemediationStrategy: &controlplanev1.RemediationStrategy{ + MaxRetry: utilpointer.Int32(3), + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + LastRemediation: &controlplanev1.LastRemediationStatus{ + Machine: "m0", + Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).Unix()}, // minHealthy not expired yet. + RetryCount: 3, + }, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2, m3), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(3))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal("m0")) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed 3 times (MaxRetry)") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeTrue()) + + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) + }) + t.Run("Retry history is ignored if min healthy period is expired, default min healthy period", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + RemediationStrategy: &controlplanev1.RemediationStrategy{ + MaxRetry: utilpointer.Int32(3), + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + LastRemediation: &controlplanev1.LastRemediationStatus{ + Machine: "m0", + Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-2 * controlplanev1.DefaultMinHealthyPeriod).Unix()}, // minHealthyPeriod already expired. + RetryCount: 3, + }, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2, m3), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) + }) + t.Run("Retry history is ignored if min healthy period is expired", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) + + minHealthyPeriod := 4 * controlplanev1.DefaultMinHealthyPeriod // big min healthy period, so we are user that we are not using DefaultMinHealthyPeriod. + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + RemediationStrategy: &controlplanev1.RemediationStrategy{ + MaxRetry: utilpointer.Int32(3), + MinHealthyPeriod: &metav1.Duration{Duration: minHealthyPeriod}, + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + LastRemediation: &controlplanev1.LastRemediationStatus{ + Machine: "m0", + Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-2 * minHealthyPeriod).Unix()}, // minHealthyPeriod already expired. + RetryCount: 3, + }, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2, m3), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) + }) + t.Run("Remediation does not happen if RetryDelay is not yet passed", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + RemediationStrategy: &controlplanev1.RemediationStrategy{ + MaxRetry: utilpointer.Int32(3), + RetryPeriod: metav1.Duration{Duration: controlplanev1.DefaultMinHealthyPeriod}, // RetryDelaySeconds not yet expired. + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + LastRemediation: &controlplanev1.LastRemediationStatus{ + Machine: "m0", + Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).Unix()}, // minHealthyPeriod not yet expired. + RetryCount: 2, + }, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2, m3), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(2))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal("m0")) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest 1h0m0s (RetryDelay)") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeTrue()) + + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) + }) + + // There are no preflight checks for when control plane is not yet initialized + // (it is the first CP, we can nuke it). + + // Preflight checks for when control plane is already initialized. + t.Run("Remediation does not happen if desired replicas <= 1", func(t *testing.T) { g := NewWithT(t) m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(1), - RolloutStrategy: &controlplanev1.RolloutStrategy{ - RollingUpdate: &controlplanev1.RollingUpdate{ - MaxSurge: &intstr.IntOrString{ - IntVal: 1, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(1), + RolloutStrategy: &controlplanev1.RolloutStrategy{ + RollingUpdate: &controlplanev1.RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, }, }, }, - }}, - Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m), + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m), + } + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) + + assertMachineCondition(ctx, g, m, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") + + g.Expect(env.Cleanup(ctx, m)).To(Succeed()) + }) + t.Run("Remediation does not happen if there is a deleting machine", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-") + m3 := getDeletingMachine(ns.Name, "m3-deleting") // NB. This machine is not created, it gets only added to control plane + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2, m3), + } + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") + + g.Expect(env.Cleanup(ctx, m1, m2)).To(Succeed()) + }) + t.Run("Remediation does not happen if there is at least one additional unhealthy etcd member on a 3 machine CP", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-mhc-unhealthy-", withMachineHealthCheckFailed()) + m2 := createMachine(ctx, g, ns.Name, "m2-etcd-unhealthy-", withUnhealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-etcd-healthy-", withHealthyEtcdMember()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2, m3), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + + g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) + }) + t.Run("Remediation does not happen if there are at least two additional unhealthy etcd member on a 5 machine CP", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-mhc-unhealthy-", withMachineHealthCheckFailed()) + m2 := createMachine(ctx, g, ns.Name, "m2-etcd-unhealthy-", withUnhealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-etcd-unhealthy-", withUnhealthyEtcdMember()) + m4 := createMachine(ctx, g, ns.Name, "m4-etcd-healthy-", withHealthyEtcdMember()) + m5 := createMachine(ctx, g, ns.Name, "m5-etcd-healthy-", withHealthyEtcdMember()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(5), + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2, m3, m4, m5), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + + g.Expect(env.Cleanup(ctx, m1, m2, m3, m4, m5)).To(Succeed()) + }) + + // Remediation for when control plane is not yet initialized + + t.Run("Remediation deletes unhealthy machine - 1 CP not initialized", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(1), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: false, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1)).To(Succeed()) + }) + t.Run("Subsequent remediation of the same machine increase retry count - 1 CP not initialized", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(1), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: false, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1), + } + + // First reconcile, remediate machine m1 for the first time + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m1) + g.Expect(env.CleanupAndWait(ctx, m1)).To(Succeed()) + + machineRemediatingFor := m1.Name + for i := 2; i < 4; i++ { + // Simulate the creation of a replacement for 0. + mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(machineRemediatingFor)) + + // Simulate KCP dropping RemediationInProgressAnnotation after creating the replacement machine. + delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + + controlPlane.Machines = collections.FromMachines(mi) + + // Reconcile unhealthy replacements for m1. + r.managementCluster = &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(collections.FromMachines(mi)), + }, + } + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(i - 1))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(mi.Name)) + + assertMachineCondition(ctx, g, mi, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: mi.Namespace, Name: mi.Name}, mi) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(mi.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, mi) + g.Expect(env.CleanupAndWait(ctx, mi)).To(Succeed()) + + machineRemediatingFor = mi.Name + } + }) + + // Remediation for when control plane is already initialized + + t.Run("Remediation deletes unhealthy machine - 2 CP (during 1 CP rolling upgrade)", func(t *testing.T) { + g := NewWithT(t) + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + + controlPlane := &internal.ControlPlane{ + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(2), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, + Cluster: &clusterv1.Cluster{}, + Machines: collections.FromMachines(m1, m2), + } + + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) - g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - assertMachineCondition(ctx, g, m, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") - g.Expect(env.Cleanup(ctx, m)).To(Succeed()) - }) - t.Run("Remediation does not happen if number of machines lower than desired", func(t *testing.T) { - g := NewWithT(t) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) - m2 := createMachine(ctx, g, ns.Name, "m2-healthy-") - controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(3), - RolloutStrategy: &controlplanev1.RolloutStrategy{}, - }}, - Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m1, m2), - } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") - g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) - assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for having at least 3 control plane machines before triggering remediation") + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + removeFinalizer(g, m1) g.Expect(env.Cleanup(ctx, m1, m2)).To(Succeed()) }) - t.Run("Remediation does not happen if there is a deleting machine", func(t *testing.T) { + t.Run("Remediation deletes unhealthy machine - 3 CP", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) - m2 := createMachine(ctx, g, ns.Name, "m2-healthy-") - m3 := getDeletingMachine(ns.Name, "m3-deleting") // NB. This machine is not created, it gets only added to control plane + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) + controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(3), - }}, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m1, m2, m3), } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) - g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + r := &KubeadmControlPlaneReconciler{ + Client: env.GetClient(), + recorder: record.NewFakeRecorder(32), + managementCluster: &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") - g.Expect(env.Cleanup(ctx, m1, m2)).To(Succeed()) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) }) - t.Run("Remediation does not happen if there is at least one additional unhealthy etcd member on a 3 machine CP", func(t *testing.T) { + t.Run("Remediation deletes unhealthy machine - 4 CP (during 3 CP rolling upgrade)", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-mhc-unhealthy-", withMachineHealthCheckFailed()) - m2 := createMachine(ctx, g, ns.Name, "m2-etcd-unhealthy-", withUnhealthyEtcdMember()) - m3 := createMachine(ctx, g, ns.Name, "m3-etcd-healthy-", withHealthyEtcdMember()) + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) + m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withHealthyEtcdMember()) controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(3), - }}, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(4), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m1, m2, m3), + Machines: collections.FromMachines(m1, m2, m3, m4), } r := &KubeadmControlPlaneReconciler{ @@ -201,29 +797,44 @@ func TestReconcileUnhealthyMachines(t *testing.T) { }, } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) - g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") - g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1, m2, m3, m4)).To(Succeed()) }) - t.Run("Remediation does not happen if there is at least two additional unhealthy etcd member on a 5 machine CP", func(t *testing.T) { + t.Run("Remediation fails gracefully if no healthy Control Planes are available to become etcd leader", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-mhc-unhealthy-", withMachineHealthCheckFailed()) - m2 := createMachine(ctx, g, ns.Name, "m2-etcd-unhealthy-", withUnhealthyEtcdMember()) - m3 := createMachine(ctx, g, ns.Name, "m3-etcd-unhealthy-", withUnhealthyEtcdMember()) - m4 := createMachine(ctx, g, ns.Name, "m4-etcd-healthy-", withHealthyEtcdMember()) - m5 := createMachine(ctx, g, ns.Name, "m5-etcd-healthy-", withHealthyEtcdMember()) + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) + m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withMachineHealthCheckFailed(), withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withMachineHealthCheckFailed(), withHealthyEtcdMember()) + m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withMachineHealthCheckFailed(), withHealthyEtcdMember()) controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(5), - }}, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(4), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m1, m2, m3, m4, m5), + Machines: collections.FromMachines(m1, m2, m3, m4), } r := &KubeadmControlPlaneReconciler{ @@ -235,35 +846,40 @@ func TestReconcileUnhealthyMachines(t *testing.T) { }, }, } + _, err = r.reconcileUnhealthyMachines(ctx, controlPlane) + g.Expect(err).ToNot(HaveOccurred()) - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) - g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped - g.Expect(err).ToNot(HaveOccurred()) - assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, + "A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to. Skipping remediation") - g.Expect(env.Cleanup(ctx, m1, m2, m3, m4, m5)).To(Succeed()) + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1, m2, m3, m4)).To(Succeed()) }) - t.Run("Remediation deletes unhealthy machine - 2 CP (during 1 CP rolling upgrade)", func(t *testing.T) { + t.Run("Subsequent remediation of the same machine increase retry count - 3 CP", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) - patchHelper, err := patch.NewHelper(m1, env.GetClient()) - g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = []string{"wait-before-delete"} - g.Expect(patchHelper.Patch(ctx, m1)) - + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(2), - Version: "v1.19.1", - }}, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(1), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: false, + }, + }, Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m1, m2), + Machines: collections.FromMachines(m1, m2, m3), } + // First reconcile, remediate machine m1 for the first time r := &KubeadmControlPlaneReconciler{ Client: env.GetClient(), recorder: record.NewFakeRecorder(32), @@ -274,43 +890,98 @@ func TestReconcileUnhealthyMachines(t *testing.T) { }, } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) - patchHelper, err = patch.NewHelper(m1, env.GetClient()) - g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = nil - g.Expect(patchHelper.Patch(ctx, m1)) + removeFinalizer(g, m1) + g.Expect(env.CleanupAndWait(ctx, m1)).To(Succeed()) - g.Expect(env.Cleanup(ctx, m1, m2)).To(Succeed()) + machineRemediatingFor := m1.Name + for i := 5; i < 6; i++ { + // Simulate the creation of a replacement for m1. + mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(machineRemediatingFor)) + + // Simulate KCP dropping RemediationInProgressAnnotation after creating the replacement machine. + delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + controlPlane.Machines = collections.FromMachines(mi, m2, m3) + + // Reconcile unhealthy replacements for m1. + r.managementCluster = &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(collections.FromMachines(mi, m2, m3)), + }, + } + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(i - 4))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(mi.Name)) + + assertMachineCondition(ctx, g, mi, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: mi.Namespace, Name: mi.Name}, mi) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(mi.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, mi) + g.Expect(env.CleanupAndWait(ctx, mi)).To(Succeed()) + + machineRemediatingFor = mi.Name + } + + g.Expect(env.CleanupAndWait(ctx, m2, m3)).To(Succeed()) }) - t.Run("Remediation deletes unhealthy machine - 3 CP", func(t *testing.T) { +} + +func TestReconcileUnhealthyMachinesSequences(t *testing.T) { + var removeFinalizer = func(g *WithT, m *clusterv1.Machine) { + patchHelper, err := patch.NewHelper(m, env.GetClient()) + g.Expect(err).ToNot(HaveOccurred()) + m.ObjectMeta.Finalizers = nil + g.Expect(patchHelper.Patch(ctx, m)) + } + + t.Run("Remediates the first CP machine having problems to come up", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) - patchHelper, err := patch.NewHelper(m1, env.GetClient()) + ns, err := env.CreateNamespace(ctx, "ns1") g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = []string{"wait-before-delete"} - g.Expect(patchHelper.Patch(ctx, m1)) + defer func() { + g.Expect(env.Cleanup(ctx, ns)).To(Succeed()) + }() - m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) - m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) + // Control plane not initialized yet, First CP is unhealthy and gets remediated: + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(3), - Version: "v1.19.1", - }}, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: false, + }, + }, Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m1, m2, m3), + Machines: collections.FromMachines(m1), } r := &KubeadmControlPlaneReconciler{ @@ -323,44 +994,99 @@ func TestReconcileUnhealthyMachines(t *testing.T) { }, } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) - patchHelper, err = patch.NewHelper(m1, env.GetClient()) + removeFinalizer(g, m1) + g.Expect(env.Cleanup(ctx, m1)).To(Succeed()) + + // Fake scaling up, which creates a remediation machine, fast forwards to when also the replacement machine is marked unhealthy. + // NOTE: scale up also resets remediation in progress and remediation counts. + + m2 := createMachine(ctx, g, ns.Name, "m2-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(m1.Name)) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + + // Control plane not initialized yet, Second CP is unhealthy and gets remediated (retry 2) + + controlPlane.Machines = collections.FromMachines(m2) + r.managementCluster = &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + } + + ret, err = r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = nil - g.Expect(patchHelper.Patch(ctx, m1)) - g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(1))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) + + assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m2.Namespace, Name: m2.Name}, m1) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m2.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m2) + g.Expect(env.Cleanup(ctx, m2)).To(Succeed()) + + // Fake scaling up, which creates a remediation machine, which is healthy. + // NOTE: scale up also resets remediation in progress and remediation counts. + + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(m1.Name)) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + + g.Expect(env.Cleanup(ctx, m3)).To(Succeed()) }) - t.Run("Remediation deletes unhealthy machine - 4 CP (during 3 CP rolling upgrade)", func(t *testing.T) { + + t.Run("Remediates the second CP machine having problems to come up", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) - patchHelper, err := patch.NewHelper(m1, env.GetClient()) + ns, err := env.CreateNamespace(ctx, "ns1") g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = []string{"wait-before-delete"} - g.Expect(patchHelper.Patch(ctx, m1)) + defer func() { + g.Expect(env.Cleanup(ctx, ns)).To(Succeed()) + }() - m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) - m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) - m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withHealthyEtcdMember()) + // Control plane initialized yet, First CP healthy, second CP is unhealthy and gets remediated: + + m1 := createMachine(ctx, g, ns.Name, "m1-healthy-", withHealthyEtcdMember()) + m2 := createMachine(ctx, g, ns.Name, "m2-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(4), - Version: "v1.19.1", - }}, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + RolloutStrategy: &controlplanev1.RolloutStrategy{ + RollingUpdate: &controlplanev1.RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m1, m2, m3, m4), + Machines: collections.FromMachines(m1, m2), } r := &KubeadmControlPlaneReconciler{ @@ -373,44 +1099,100 @@ func TestReconcileUnhealthyMachines(t *testing.T) { }, } - ret, err := r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) - err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) + assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m2.Namespace, Name: m2.Name}, m2) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(m1.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + g.Expect(m2.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m2) + g.Expect(env.Cleanup(ctx, m2)).To(Succeed()) + + // Fake scaling up, which creates a remediation machine, fast forwards to when also the replacement machine is marked unhealthy. + // NOTE: scale up also resets remediation in progress and remediation counts. - patchHelper, err = patch.NewHelper(m1, env.GetClient()) + m3 := createMachine(ctx, g, ns.Name, "m3-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(m2.Name)) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + + // Control plane not initialized yet, Second CP is unhealthy and gets remediated (retry 2) + + controlPlane.Machines = collections.FromMachines(m1, m3) + r.managementCluster = &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + } + + ret, err = r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = nil - g.Expect(patchHelper.Patch(ctx, m1)) - g.Expect(env.Cleanup(ctx, m1, m2, m3, m4)).To(Succeed()) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(1))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m3.Name)) + + assertMachineCondition(ctx, g, m3, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + + err = env.Get(ctx, client.ObjectKey{Namespace: m3.Namespace, Name: m3.Name}, m3) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(m2.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + + removeFinalizer(g, m3) + g.Expect(env.Cleanup(ctx, m3)).To(Succeed()) + + // Fake scaling up, which creates a remediation machine, which is healthy. + // NOTE: scale up also resets remediation in progress and remediation counts. + + m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(m1.Name)) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + + g.Expect(env.Cleanup(ctx, m1, m4)).To(Succeed()) }) - t.Run("Remediation does not happen if no healthy Control Planes are available to become etcd leader", func(t *testing.T) { + + t.Run("Remediates only one CP machine in case of multiple failures", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) - patchHelper, err := patch.NewHelper(m1, env.GetClient()) + ns, err := env.CreateNamespace(ctx, "ns1") g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = []string{"wait-before-delete"} - g.Expect(patchHelper.Patch(ctx, m1)) + defer func() { + g.Expect(env.Cleanup(ctx, ns)).To(Succeed()) + }() - m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withMachineHealthCheckFailed(), withHealthyEtcdMember()) - m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withMachineHealthCheckFailed(), withHealthyEtcdMember()) - m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withMachineHealthCheckFailed(), withHealthyEtcdMember()) + // Control plane initialized yet, First CP healthy, second and third CP are unhealthy. second gets remediated: + + m1 := createMachine(ctx, g, ns.Name, "m1-healthy-", withHealthyEtcdMember()) + m2 := createMachine(ctx, g, ns.Name, "m2-unhealthy-", withHealthyEtcdMember(), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) + m3 := createMachine(ctx, g, ns.Name, "m3-unhealthy-", withHealthyEtcdMember(), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer()) controlPlane := &internal.ControlPlane{ - KCP: &controlplanev1.KubeadmControlPlane{Spec: controlplanev1.KubeadmControlPlaneSpec{ - Replicas: utilpointer.Int32(4), - Version: "v1.19.1", - }}, + KCP: &controlplanev1.KubeadmControlPlane{ + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Replicas: utilpointer.Int32(3), + Version: "v1.19.1", + RolloutStrategy: &controlplanev1.RolloutStrategy{ + RollingUpdate: &controlplanev1.RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, + }, + Status: controlplanev1.KubeadmControlPlaneStatus{ + Initialized: true, + }, + }, Cluster: &clusterv1.Cluster{}, - Machines: collections.FromMachines(m1, m2, m3, m4), + Machines: collections.FromMachines(m1, m2, m3), } r := &KubeadmControlPlaneReconciler{ @@ -422,24 +1204,46 @@ func TestReconcileUnhealthyMachines(t *testing.T) { }, }, } - _, err = r.reconcileUnhealthyMachines(context.TODO(), controlPlane) + + ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, - "A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to. Skipping remediation") + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) + g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) + + assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + assertMachineCondition(ctx, g, m3, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") - patchHelper, err = patch.NewHelper(m1, env.GetClient()) + err = env.Get(ctx, client.ObjectKey{Namespace: m2.Namespace, Name: m2.Name}, m2) g.Expect(err).ToNot(HaveOccurred()) - m1.ObjectMeta.Finalizers = nil - g.Expect(patchHelper.Patch(ctx, m1)) + g.Expect(m2.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) - g.Expect(env.Cleanup(ctx, m1, m2, m3, m4)).To(Succeed()) + removeFinalizer(g, m2) + g.Expect(env.Cleanup(ctx, m2)).To(Succeed()) + + // Check next reconcile does not further remediate + + controlPlane.Machines = collections.FromMachines(m1, m3) + r.managementCluster = &fakeManagementCluster{ + Workload: fakeWorkloadCluster{ + EtcdMembersResult: nodes(controlPlane.Machines), + }, + } + + ret, err = r.reconcileUnhealthyMachines(ctx, controlPlane) + + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(env.Cleanup(ctx, m1)).To(Succeed()) }) } func TestCanSafelyRemoveEtcdMember(t *testing.T) { g := NewWithT(t) - ctx := context.TODO() ns, err := env.CreateNamespace(ctx, "ns1") g.Expect(err).ToNot(HaveOccurred()) @@ -470,7 +1274,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeFalse()) g.Expect(err).ToNot(HaveOccurred()) @@ -501,7 +1305,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeTrue()) g.Expect(err).ToNot(HaveOccurred()) @@ -538,7 +1342,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeTrue()) g.Expect(err).ToNot(HaveOccurred()) @@ -568,7 +1372,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeFalse()) g.Expect(err).ToNot(HaveOccurred()) @@ -599,7 +1403,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeTrue()) g.Expect(err).ToNot(HaveOccurred()) @@ -637,7 +1441,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeTrue()) g.Expect(err).ToNot(HaveOccurred()) @@ -668,7 +1472,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeFalse()) g.Expect(err).ToNot(HaveOccurred()) @@ -701,7 +1505,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeTrue()) g.Expect(err).ToNot(HaveOccurred()) @@ -734,7 +1538,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeFalse()) g.Expect(err).ToNot(HaveOccurred()) @@ -769,7 +1573,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeTrue()) g.Expect(err).ToNot(HaveOccurred()) @@ -804,7 +1608,7 @@ func TestCanSafelyRemoveEtcdMember(t *testing.T) { }, } - ret, err := r.canSafelyRemoveEtcdMember(context.TODO(), controlPlane, m1) + ret, err := r.canSafelyRemoveEtcdMember(ctx, controlPlane, m1) g.Expect(ret).To(BeFalse()) g.Expect(err).ToNot(HaveOccurred()) @@ -859,6 +1663,21 @@ func withNodeRef(ref string) machineOption { } } +func withRemediateForAnnotation(remediatedFor string) machineOption { + return func(machine *clusterv1.Machine) { + if machine.Annotations == nil { + machine.Annotations = map[string]string{} + } + machine.Annotations[controlplanev1.MachineRemediationForAnnotation] = remediatedFor + } +} + +func withWaitBeforeDeleteFinalizer() machineOption { + return func(machine *clusterv1.Machine) { + machine.Finalizers = []string{"wait-before-delete"} + } +} + func createMachine(ctx context.Context, g *WithT, namespace, name string, options ...machineOption) *clusterv1.Machine { m := &clusterv1.Machine{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/controllers/machinehealthcheck/machinehealthcheck_controller.go b/internal/controllers/machinehealthcheck/machinehealthcheck_controller.go index f2b98f34b581..2d0557132958 100644 --- a/internal/controllers/machinehealthcheck/machinehealthcheck_controller.go +++ b/internal/controllers/machinehealthcheck/machinehealthcheck_controller.go @@ -199,15 +199,19 @@ func (r *Reconciler) reconcile(ctx context.Context, logger logr.Logger, cluster UID: cluster.UID, }) - // Get the remote cluster cache to use as a client.Reader. - remoteClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster)) - if err != nil { - logger.Error(err, "error creating remote cluster cache") - return ctrl.Result{}, err - } + // If the cluster is already initialized, get the remote cluster cache to use as a client.Reader. + var remoteClient client.Client + if conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { + var err error + remoteClient, err = r.Tracker.GetClient(ctx, util.ObjectKey(cluster)) + if err != nil { + logger.Error(err, "error creating remote cluster cache") + return ctrl.Result{}, err + } - if err := r.watchClusterNodes(ctx, cluster); err != nil { - return ctrl.Result{}, err + if err := r.watchClusterNodes(ctx, cluster); err != nil { + return ctrl.Result{}, err + } } // fetch all targets diff --git a/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go b/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go index 97eb0659bf0c..bdd9c5861f2a 100644 --- a/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go +++ b/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/patch" @@ -111,8 +112,9 @@ func (t *healthCheckTarget) needsRemediation(logger logr.Logger, timeoutForMachi return true, time.Duration(0) } - // Don't penalize any Machine/Node if the control plane has not been initialized. - if !conditions.IsTrue(t.Cluster, clusterv1.ControlPlaneInitializedCondition) { + // Don't penalize any Machine/Node if the control plane has not been initialized + // Exception of this rule are control plane machine itself, so the first control plane machine can be remediated. + if !conditions.IsTrue(t.Cluster, clusterv1.ControlPlaneInitializedCondition) && !util.IsControlPlaneMachine(t.Machine) { logger.V(3).Info("Not evaluating target health because the control plane has not yet been initialized") // Return a nextCheck time of 0 because we'll get requeued when the Cluster is updated. return false, 0 @@ -218,16 +220,18 @@ func (r *Reconciler) getTargetsFromMHC(ctx context.Context, logger logr.Logger, Machine: &machines[k], patchHelper: patchHelper, } - node, err := r.getNodeFromMachine(ctx, clusterClient, target.Machine) - if err != nil { - if !apierrors.IsNotFound(err) { - return nil, errors.Wrap(err, "error getting node") + if clusterClient != nil { + node, err := r.getNodeFromMachine(ctx, clusterClient, target.Machine) + if err != nil { + if !apierrors.IsNotFound(err) { + return nil, errors.Wrap(err, "error getting node") + } + + // A node has been seen for this machine, but it no longer exists + target.nodeMissing = true } - - // A node has been seen for this machine, but it no longer exists - target.nodeMissing = true + target.Node = node } - target.Node = node targets = append(targets, target) } return targets, nil diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index 708ecef3b2e0..74a52cd3647e 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -290,3 +290,6 @@ intervals: node-drain/wait-deployment-available: ["3m", "10s"] node-drain/wait-control-plane: ["15m", "10s"] node-drain/wait-machine-deleted: ["2m", "10s"] + kcp-remediation/wait-machines: ["5m", "10s"] + kcp-remediation/check-machines-stable: ["30s", "5s"] + kcp-remediation/wait-machine-provisioned: ["5m", "10s"] diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/cluster-with-kcp.yaml new file mode 100644 index 000000000000..12516bab5ccd --- /dev/null +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/cluster-with-kcp.yaml @@ -0,0 +1,98 @@ +--- +# DockerCluster object referenced by the Cluster object +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: '${CLUSTER_NAME}' +--- +# Cluster object with +# - Reference to the KubeadmControlPlane object +# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet. +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: '${CLUSTER_NAME}' + labels: + cni: "${CLUSTER_NAME}-crs-0" +spec: + clusterNetwork: + services: + cidrBlocks: ['${DOCKER_SERVICE_CIDRS}'] + pods: + cidrBlocks: ['${DOCKER_POD_CIDRS}'] + serviceDomain: '${DOCKER_SERVICE_DOMAIN}' + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: '${CLUSTER_NAME}' + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +# DockerMachineTemplate object referenced by the KubeadmControlPlane object +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: {} +--- +# KubeadmControlPlane referenced by the Cluster +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: {enable-hostpath-provisioner: 'true'} + apiServer: + # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied. + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + files: + - path: /wait-signal.sh + content: | + #!/bin/bash + + set -o errexit + set -o pipefail + + echo "Waiting for signal..." + + TOKEN=$1 + SERVER=$2 + NAMESPACE=$3 + + while true; + do + sleep 1s + + signal=$(curl -k -s --header "Authorization: Bearer $TOKEN" $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test | jq -r .data.signal?) + echo "signal $signal" + + if [ "$signal" == "pass" ]; then + curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test + exit 0 + fi + done + permissions: "0777" + preKubeadmCommands: + - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}" + version: "${KUBERNETES_VERSION}" diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/kustomization.yaml index e234e37be1b2..92761f83d820 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/kustomization.yaml @@ -1,5 +1,5 @@ bases: - - ../bases/cluster-with-kcp.yaml + - cluster-with-kcp.yaml - ../bases/md.yaml - ../bases/crs.yaml - mhc.yaml diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/mhc.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/mhc.yaml index 3ed3e0a9473a..39187cec0a40 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/mhc.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-kcp-remediation/mhc.yaml @@ -1,7 +1,8 @@ --- # MachineHealthCheck object with -# - a selector that targets all the machines with label cluster.x-k8s.io/control-plane="" -# - unhealthyConditions triggering remediation after 10s the condition is set +# - a selector that targets all the machines with label cluster.x-k8s.io/control-plane="" and the mhc-test: "fail" (the label is used to trigger remediation in a controlled way - by adding CP under MHC control intentionally -) +# - nodeStartupTimeout: 30s (to force remediation on nodes still provisioning) +# - unhealthyConditions triggering remediation after 10s the e2e.remediation.condition condition is set to false (to force remediation on nodes already provisioned) apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: @@ -12,6 +13,8 @@ spec: selector: matchLabels: cluster.x-k8s.io/control-plane: "" + mhc-test: "fail" + nodeStartupTimeout: 30s unhealthyConditions: - type: e2e.remediation.condition status: "False" diff --git a/test/e2e/kcp_remediations.go b/test/e2e/kcp_remediations.go new file mode 100644 index 000000000000..6d16633dfbd8 --- /dev/null +++ b/test/e2e/kcp_remediations.go @@ -0,0 +1,693 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/e2e/internal/log" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +const ( + configMapName = "mhc-test" + configMapDataKey = "signal" + + failLabelValue = "fail" +) + +// KCPRemediationSpecInput is the input for KCPRemediationSpec. +type KCPRemediationSpecInput struct { + // This spec requires following intervals to be defined in order to work: + // - wait-cluster, used when waiting for the cluster infrastructure to be provisioned. + // - wait-machines, used when waiting for an old machine to be remediated and a new one provisioned. + // - check-machines-stable, used when checking that the current list of machines in stable. + // - wait-machine-provisioned, used when waiting for a machine to be provisioned after unblocking bootstrap. + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool + + // Flavor, if specified, must refer to a template that has a MachineHealthCheck + // - 3 node CP, no workers + // - Control plane machines having a pre-kubeadm command that queries for a well-known ConfigMap on the management cluster, + // holding up bootstrap until a signal is passed via the config map. + // NOTE: In order for this to work communications from workload cluster to management cluster must be enabled. + // - An MHC targeting control plane machines with the mhc-test=fail labels and + // nodeStartupTimeout: 30s + // unhealthyConditions: + // - type: e2e.remediation.condition + // status: "False" + // timeout: 10s + // If not specified, "kcp-remediation" is used. + Flavor *string +} + +// KCPRemediationSpec implements a test that verifies that Machines are remediated by MHC during unhealthy conditions. +func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSpecInput) { + var ( + specName = "kcp-remediation" + input KCPRemediationSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + }) + + It("Should replace unhealthy machines", func() { + By("Creating a workload cluster") + + // NOTE: This test is quite different from other tests, because it has to trigger failures on machines in a controlled ways. + + // creates the mhc-test ConfigMap that will be used to control machines bootstrap during the remediation tests. + createConfigMapForMachinesBootstrapSignal(ctx, input.BootstrapClusterProxy.GetClient(), namespace.Name) + + // Creates the workload cluster. + clusterResources = createWorkloadClusterAndWait(ctx, createWorkloadClusterAndWaitInput{ + E2EConfig: input.E2EConfig, + clusterctlConfigPath: input.ClusterctlConfigPath, + proxy: input.BootstrapClusterProxy, + artifactFolder: input.ArtifactFolder, + specName: specName, + flavor: input.Flavor, + + // values to be injected in the template + + namespace: namespace.Name, + // Token with credentials to use for accessing the ConfigMap on managements cluster from the workload cluster. + // NOTE: this func also setups credentials/RBAC rules and everything necessary to get the authentication authenticationToken. + authenticationToken: getAuthenticationToken(ctx, input.BootstrapClusterProxy, namespace.Name), + // Address to be used for accessing the management cluster from a workload cluster. + serverAddr: getServerAddr(input.BootstrapClusterProxy.GetKubeconfigPath()), + }) + + // The first CP machine comes up but it does not complete bootstrap + + By("FIRST CONTROL PLANE MACHINE") + + By("Wait for the cluster to get stuck with the first CP machine not completing the bootstrap") + allMachines, newMachines := waitForMachines(ctx, waitForMachinesInput{ + lister: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + clusterName: clusterResources.Cluster.Name, + expectedReplicas: 1, + waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + }) + Expect(allMachines).To(HaveLen(1)) + Expect(newMachines).To(HaveLen(1)) + firstMachineName := newMachines[0] + firstMachine := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: firstMachineName, + Namespace: namespace.Name, + }, + } + Expect(input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(firstMachine), firstMachine)).To(Succeed(), "Failed to get machine %d", firstMachineName) + Expect(firstMachine.Status.NodeRef).To(BeNil()) + log.Logf("Machine %s is up but still bootstrapping", firstMachineName) + + // Intentionally trigger remediation on the first CP, and validate the first machine is deleted and a replacement should come up. + + By("REMEDIATING FIRST CONTROL PLANE MACHINE") + + Byf("Add mhc-test:fail label to machine %s so it will be immediately remediated", firstMachineName) + firstMachineWithLabel := firstMachine.DeepCopy() + firstMachineWithLabel.Labels["mhc-test"] = failLabelValue + Expect(input.BootstrapClusterProxy.GetClient().Patch(ctx, firstMachineWithLabel, client.MergeFrom(firstMachine))).To(Succeed(), "Failed to patch machine %d", firstMachineName) + + log.Logf("Wait for the first CP machine to be remediated, and the replacement machine to come up, but again get stuck with the Machine not completing the bootstrap") + allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ + lister: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + clusterName: clusterResources.Cluster.Name, + expectedReplicas: 1, + expectedDeletedMachines: []string{firstMachineName}, + waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + }) + Expect(allMachines).To(HaveLen(1)) + Expect(newMachines).To(HaveLen(1)) + firstMachineReplacementName := newMachines[0] + firstMachineReplacement := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: firstMachineReplacementName, + Namespace: namespace.Name, + }, + } + Expect(input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(firstMachineReplacement), firstMachineReplacement)).To(Succeed(), "Failed to get machine %d", firstMachineReplacementName) + Expect(firstMachineReplacement.Status.NodeRef).To(BeNil()) + log.Logf("Machine %s is up but still bootstrapping", firstMachineReplacementName) + + // The firstMachine replacement is up, meaning that the test validated that remediation of the first CP machine works (note: first CP is a special case because the cluster is not initialized yet). + // In order to test remediation of other machine while provisioning we unblock bootstrap of the first CP replacement + // and wait for the second cp machine to come up. + + By("FIRST CONTROL PLANE MACHINE SUCCESSFULLY REMEDIATED!") + + Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", firstMachineReplacementName) + sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ + client: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + machine: firstMachineReplacementName, + signal: "pass", + }) + log.Logf("Waiting for Machine %s to be provisioned", firstMachineReplacementName) + Eventually(func() bool { + if err := input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(firstMachineReplacement), firstMachineReplacement); err != nil { + return false + } + return firstMachineReplacement.Status.NodeRef != nil + }, input.E2EConfig.GetIntervals(specName, "wait-machine-provisioned")...).Should(BeTrue(), "Machine %s failed to be provisioned", firstMachineReplacementName) + + By("FIRST CONTROL PLANE MACHINE UP AND RUNNING!") + By("START PROVISIONING OF SECOND CONTROL PLANE MACHINE!") + + By("Wait for the cluster to get stuck with the second CP machine not completing the bootstrap") + allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ + lister: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + clusterName: clusterResources.Cluster.Name, + expectedReplicas: 2, + expectedDeletedMachines: []string{}, + expectedOldMachines: []string{firstMachineReplacementName}, + waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + }) + Expect(allMachines).To(HaveLen(2)) + Expect(newMachines).To(HaveLen(1)) + secondMachineName := newMachines[0] + secondMachine := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: secondMachineName, + Namespace: namespace.Name, + }, + } + Expect(input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(secondMachine), secondMachine)).To(Succeed(), "Failed to get machine %d", secondMachineName) + Expect(secondMachine.Status.NodeRef).To(BeNil()) + log.Logf("Machine %s is up but still bootstrapping", secondMachineName) + + // Intentionally trigger remediation on the second CP, and validate and validate also this one is deleted and a replacement should come up. + + By("REMEDIATING SECOND CONTROL PLANE MACHINE") + + Byf("Add mhc-test:fail label to machine %s so it will be immediately remediated", firstMachineName) + secondMachineWithLabel := secondMachine.DeepCopy() + secondMachineWithLabel.Labels["mhc-test"] = failLabelValue + Expect(input.BootstrapClusterProxy.GetClient().Patch(ctx, secondMachineWithLabel, client.MergeFrom(secondMachine))).To(Succeed(), "Failed to patch machine %d", secondMachineName) + + log.Logf("Wait for the second CP machine to be remediated, and the replacement machine to come up, but again get stuck with the Machine not completing the bootstrap") + allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ + lister: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + clusterName: clusterResources.Cluster.Name, + expectedReplicas: 2, + expectedDeletedMachines: []string{secondMachineName}, + expectedOldMachines: []string{firstMachineReplacementName}, + waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + }) + Expect(allMachines).To(HaveLen(2)) + Expect(newMachines).To(HaveLen(1)) + secondMachineReplacementName := newMachines[0] + secondMachineReplacement := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: secondMachineReplacementName, + Namespace: namespace.Name, + }, + } + Expect(input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(secondMachineReplacement), secondMachineReplacement)).To(Succeed(), "Failed to get machine %d", secondMachineReplacementName) + Expect(secondMachineReplacement.Status.NodeRef).To(BeNil()) + log.Logf("Machine %s is up but still bootstrapping", secondMachineReplacementName) + + // The secondMachine replacement is up, meaning that the test validated that remediation of the second CP machine works (note: this test remediation after the cluster is initialized, but not yet fully provisioned). + // In order to test remediation after provisioning we unblock bootstrap of the second CP replacement as well as for the third CP machine. + // and wait for the second cp machine to come up. + + By("SECOND CONTROL PLANE MACHINE SUCCESSFULLY REMEDIATED!") + + Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", secondMachineReplacementName) + sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ + client: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + machine: secondMachineReplacementName, + signal: "pass", + }) + log.Logf("Waiting for Machine %s to be provisioned", secondMachineReplacementName) + Eventually(func() bool { + if err := input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(secondMachineReplacement), secondMachineReplacement); err != nil { + return false + } + return secondMachineReplacement.Status.NodeRef != nil + }, input.E2EConfig.GetIntervals(specName, "wait-machine-provisioned")...).Should(BeTrue(), "Machine %s failed to be provisioned", secondMachineReplacementName) + + By("SECOND CONTROL PLANE MACHINE UP AND RUNNING!") + By("START PROVISIONING OF THIRD CONTROL PLANE MACHINE!") + + By("Wait for the cluster to get stuck with the third CP machine not completing the bootstrap") + allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ + lister: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + clusterName: clusterResources.Cluster.Name, + expectedReplicas: 3, + expectedDeletedMachines: []string{}, + expectedOldMachines: []string{firstMachineReplacementName, secondMachineReplacementName}, + waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + }) + Expect(allMachines).To(HaveLen(3)) + Expect(newMachines).To(HaveLen(1)) + thirdMachineName := newMachines[0] + thirdMachine := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: thirdMachineName, + Namespace: namespace.Name, + }, + } + Expect(input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(thirdMachine), thirdMachine)).To(Succeed(), "Failed to get machine %d", thirdMachineName) + Expect(thirdMachine.Status.NodeRef).To(BeNil()) + log.Logf("Machine %s is up but still bootstrapping", thirdMachineName) + + Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", thirdMachineName) + sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ + client: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + machine: thirdMachineName, + signal: "pass", + }) + log.Logf("Waiting for Machine %s to be provisioned", thirdMachineName) + Eventually(func() bool { + if err := input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(thirdMachine), thirdMachine); err != nil { + return false + } + return thirdMachine.Status.NodeRef != nil + }, input.E2EConfig.GetIntervals(specName, "wait-machine-provisioned")...).Should(BeTrue(), "Machine %s failed to be provisioned", thirdMachineName) + + // All three CP machines are up. + + By("ALL THE CONTROL PLANE MACHINES SUCCESSFULLY PROVISIONED!") + + // We now want to test remediation of a CP machine already provisioned. + // In order to do so we need to apply both mhc-test:fail as well as setting an unhealthy condition in order to trigger remediation + + By("REMEDIATING THIRD CP") + + Byf("Add mhc-test:fail label to machine %s and set an unhealthy condition on the node so it will be immediately remediated", thirdMachineName) + thirdMachineWithLabel := thirdMachine.DeepCopy() + thirdMachineWithLabel.Labels["mhc-test"] = failLabelValue + Expect(input.BootstrapClusterProxy.GetClient().Patch(ctx, thirdMachineWithLabel, client.MergeFrom(thirdMachine))).To(Succeed(), "Failed to patch machine %d", thirdMachineName) + + unhealthyNodeCondition := corev1.NodeCondition{ + Type: "e2e.remediation.condition", + Status: "False", + LastTransitionTime: metav1.Time{Time: time.Now()}, + } + framework.PatchNodeCondition(ctx, framework.PatchNodeConditionInput{ + ClusterProxy: input.BootstrapClusterProxy, + Cluster: clusterResources.Cluster, + NodeCondition: unhealthyNodeCondition, + Machine: *thirdMachine, // TODO: make this a pointer. + }) + + log.Logf("Wait for the third CP machine to be remediated, and the replacement machine to come up, but again get stuck with the Machine not completing the bootstrap") + allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ + lister: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + clusterName: clusterResources.Cluster.Name, + expectedReplicas: 3, + expectedDeletedMachines: []string{thirdMachineName}, + expectedOldMachines: []string{firstMachineReplacementName, secondMachineReplacementName}, + waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + }) + Expect(allMachines).To(HaveLen(3)) + Expect(newMachines).To(HaveLen(1)) + thirdMachineReplacementName := newMachines[0] + thirdMachineReplacement := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: thirdMachineReplacementName, + Namespace: namespace.Name, + }, + } + Expect(input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(thirdMachineReplacement), thirdMachineReplacement)).To(Succeed(), "Failed to get machine %d", thirdMachineReplacementName) + Expect(thirdMachineReplacement.Status.NodeRef).To(BeNil()) + log.Logf("Machine %s is up but still bootstrapping", thirdMachineReplacementName) + + // The thirdMachine replacement is up, meaning that the test validated that remediation of the third CP machine works (note: this test remediation after the cluster is fully provisioned). + + By("THIRD CP SUCCESSFULLY REMEDIATED!") + + Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", thirdMachineReplacementName) + sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ + client: input.BootstrapClusterProxy.GetClient(), + namespace: namespace.Name, + machine: thirdMachineReplacementName, + signal: "pass", + }) + log.Logf("Waiting for Machine %s to be provisioned", thirdMachineReplacementName) + Eventually(func() bool { + if err := input.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(thirdMachineReplacement), thirdMachineReplacement); err != nil { + return false + } + return thirdMachineReplacement.Status.NodeRef != nil + }, input.E2EConfig.GetIntervals(specName, "wait-machine-provisioned")...).Should(BeTrue(), "Machine %s failed to be provisioned", thirdMachineReplacementName) + + // All three CP machines are up again. + + By("CP BACK TO FULL OPERATIONAL STATE!") + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} + +func createConfigMapForMachinesBootstrapSignal(ctx context.Context, writer client.Writer, namespace string) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: namespace, + }, + Data: map[string]string{ + configMapDataKey: "hold", + }, + } + Expect(writer.Create(ctx, cm)).To(Succeed(), "failed to create mhc-test config map") +} + +type createWorkloadClusterAndWaitInput struct { + E2EConfig *clusterctl.E2EConfig + clusterctlConfigPath string + proxy framework.ClusterProxy + artifactFolder string + specName string + flavor *string + namespace string + authenticationToken []byte + serverAddr string +} + +// createWorkloadClusterAndWait creates a workload cluster ard return ass soon as the cluster infrastructure is ready. +// NOTE: clusterResources is filled only partially. +func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClusterAndWaitInput) (clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) { + clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + + // gets the cluster template + log.Logf("Getting the cluster template yaml") + clusterName := fmt.Sprintf("%s-%s", input.specName, util.RandomString(6)) + workloadClusterTemplate := clusterctl.ConfigCluster(ctx, clusterctl.ConfigClusterInput{ + // pass the clusterctl config file that points to the local provider repository created for this test, + ClusterctlConfigPath: input.clusterctlConfigPath, + // pass reference to the management cluster hosting this test + KubeconfigPath: input.proxy.GetKubeconfigPath(), + + // select template + Flavor: pointer.StringDeref(input.flavor, "kcp-remediation"), + // define template variables + Namespace: input.namespace, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64(3), + WorkerMachineCount: pointer.Int64(0), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + // setup clusterctl logs folder + LogFolder: filepath.Join(input.artifactFolder, "clusters", input.proxy.GetName()), + // Adds authenticationToken, server address and namespace variables to be injected in the cluster template. + ClusterctlVariables: map[string]string{ + "TOKEN": string(input.authenticationToken), + "SERVER": input.serverAddr, + "NAMESPACE": input.namespace, + }, + }) + Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") + + Eventually(func() error { + return input.proxy.Apply(ctx, workloadClusterTemplate) + }, 10*time.Second).Should(Succeed(), "Failed to apply the cluster template") + + log.Logf("Waiting for the cluster infrastructure to be provisioned") + clusterResources.Cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ + Getter: input.proxy.GetClient(), + Namespace: input.namespace, + Name: clusterName, + }, input.E2EConfig.GetIntervals(input.specName, "wait-cluster")...) + + return clusterResources +} + +type sendSignalToBootstrappingMachineInput struct { + client client.Client + namespace string + machine string + signal string +} + +// sendSignalToBootstrappingMachine sends a signal to a machine stuck during bootstrap. +func sendSignalToBootstrappingMachine(ctx context.Context, input sendSignalToBootstrappingMachineInput) { + log.Logf("Sending bootstrap signal %s to Machine %s", input.signal, input.machine) + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: input.namespace, + }, + } + Expect(input.client.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed(), "failed to get mhc-test config map") + + cmWithSignal := cm.DeepCopy() + cmWithSignal.Data[configMapDataKey] = input.signal + Expect(input.client.Patch(ctx, cmWithSignal, client.MergeFrom(cm))).To(Succeed(), "failed to patch mhc-test config map") + + log.Logf("Waiting for Machine %s to acknowledge signal %s has been received", input.machine, input.signal) + Eventually(func() string { + _ = input.client.Get(ctx, client.ObjectKeyFromObject(cmWithSignal), cmWithSignal) + return cmWithSignal.Data[configMapDataKey] + }, "1m", "10s").Should(Equal(fmt.Sprintf("ack-%s", input.signal)), "Failed to get ack signal from machine %s", input.machine) + + machine := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: input.machine, + Namespace: input.namespace, + }, + } + Expect(input.client.Get(ctx, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) + + // Resetting the signal in the config map + cmWithSignal.Data[configMapDataKey] = "hold" + Expect(input.client.Patch(ctx, cmWithSignal, client.MergeFrom(cm))).To(Succeed(), "failed to patch mhc-test config map") +} + +type waitForMachinesInput struct { + lister framework.Lister + namespace string + clusterName string + expectedReplicas int + expectedOldMachines []string + expectedDeletedMachines []string + waitForMachinesIntervals []interface{} + checkMachineListStableIntervals []interface{} +} + +// waitForMachines waits for machines to reach a well known state defined by number of replicas, a list of machines to exists, +// a list of machines to not exists anymore. The func also check that the state is stable for some time before +// returning the list of new machines. +func waitForMachines(ctx context.Context, input waitForMachinesInput) (allMachineNames, newMachineNames []string) { + inClustersNamespaceListOption := client.InNamespace(input.namespace) + matchClusterListOption := client.MatchingLabels{ + clusterv1.ClusterNameLabel: input.clusterName, + clusterv1.MachineControlPlaneLabel: "", + } + + expectedOldMachines := sets.NewString(input.expectedOldMachines...) + expectedDeletedMachines := sets.NewString(input.expectedDeletedMachines...) + allMachines := sets.NewString() + newMachines := sets.NewString() + machineList := &clusterv1.MachineList{} + + // Waits for the desired set of machines to exist. + log.Logf("Waiting for %d machines, must have %s, must not have %s", input.expectedReplicas, expectedOldMachines.List(), expectedDeletedMachines.List()) + Eventually(func() bool { + // Gets the list of machines + if err := input.lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { + return false + } + allMachines = sets.NewString() + for i := range machineList.Items { + allMachines.Insert(machineList.Items[i].Name) + } + + // Compute new machines (all - old - to be deleted) + newMachines = allMachines.Clone() + newMachines.Delete(expectedOldMachines.List()...) + newMachines.Delete(expectedDeletedMachines.List()...) + + log.Logf(" - expected %d, got %d: %s, of which new %s, must have check: %t, must not have check: %t", input.expectedReplicas, allMachines.Len(), allMachines.List(), newMachines.List(), allMachines.HasAll(expectedOldMachines.List()...), !allMachines.HasAny(expectedDeletedMachines.List()...)) + + // Ensures all the expected old machines are still there. + if !allMachines.HasAll(expectedOldMachines.List()...) { + return false + } + + // Ensures none of the machines to be deleted is still there. + if allMachines.HasAny(expectedDeletedMachines.List()...) { + return false + } + + return allMachines.Len() == input.expectedReplicas + }, input.waitForMachinesIntervals...).Should(BeTrue(), "Failed to get the expected list of machines: got %s (expected %d machines, must have %s, must not have %s)", allMachines.List(), input.expectedReplicas, expectedOldMachines.List(), expectedDeletedMachines.List()) + log.Logf("Got %d machines: %s", input.expectedReplicas, allMachines.List()) + + // Ensures the desired set of machines is stable (no further machines are created or deleted). + log.Logf("Checking the list of machines is stable") + allMachinesNow := sets.NewString() + Consistently(func() bool { + // Gets the list of machines + if err := input.lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { + return false + } + allMachinesNow = sets.NewString() + for i := range machineList.Items { + allMachinesNow.Insert(machineList.Items[i].Name) + } + + return allMachines.Len() == allMachinesNow.Len() && allMachines.HasAll(allMachinesNow.List()...) + }, input.checkMachineListStableIntervals...).Should(BeTrue(), "Expected list of machines is not stable: got %s, expected %s", allMachinesNow.List(), allMachines.List()) + + return allMachines.List(), newMachines.List() +} + +// getServerAddr returns the address to be used for accessing the management cluster from a workload cluster. +func getServerAddr(kubeconfigPath string) string { + kubeConfig, err := clientcmd.LoadFromFile(kubeconfigPath) + Expect(err).ToNot(HaveOccurred(), "failed to load management cluster's kubeconfig file") + + clusterName := kubeConfig.Contexts[kubeConfig.CurrentContext].Cluster + Expect(clusterName).ToNot(BeEmpty(), "failed to identify current cluster name in management cluster's kubeconfig file") + + serverAddr := kubeConfig.Clusters[clusterName].Server + Expect(serverAddr).ToNot(BeEmpty(), "failed to identify current server address in management cluster's kubeconfig file") + + // On CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host + // to the CAPD cluster. Moby on Linux doesn't use the host.docker.internal DNS name. + if runtime.GOOS != "linux" { + serverAddr = strings.ReplaceAll(serverAddr, "127.0.0.1", "host.docker.internal") + } + return serverAddr +} + +// getAuthenticationToken returns a bearer authenticationToken with minimal RBAC permissions to access the mhc-test ConfigMap that will be used +// to control machines bootstrap during the remediation tests. +func getAuthenticationToken(ctx context.Context, managementClusterProxy framework.ClusterProxy, namespace string) []byte { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mhc-test", + Namespace: namespace, + }, + } + Expect(managementClusterProxy.GetClient().Create(ctx, sa)).To(Succeed(), "failed to create mhc-test service account") + + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mhc-test", + Namespace: namespace, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "patch"}, + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"mhc-test"}, + }, + }, + } + Expect(managementClusterProxy.GetClient().Create(ctx, role)).To(Succeed(), "failed to create mhc-test role") + + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mhc-test", + Namespace: namespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: "", + Name: "mhc-test", + Namespace: namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "Role", + Name: "mhc-test", + }, + } + Expect(managementClusterProxy.GetClient().Create(ctx, roleBinding)).To(Succeed(), "failed to create mhc-test role binding") + + cmd := exec.CommandContext(ctx, "kubectl", fmt.Sprintf("--kubeconfig=%s", managementClusterProxy.GetKubeconfigPath()), fmt.Sprintf("--namespace=%s", namespace), "create", "token", "mhc-test") //nolint:gosec + stdout, err := cmd.StdoutPipe() + Expect(err).ToNot(HaveOccurred(), "failed to get stdout for kubectl create authenticationToken") + stderr, err := cmd.StderrPipe() + Expect(err).ToNot(HaveOccurred(), "failed to get stderr for kubectl create authenticationToken") + + Expect(cmd.Start()).To(Succeed(), "failed to run kubectl create authenticationToken") + + output, err := io.ReadAll(stdout) + Expect(err).ToNot(HaveOccurred(), "failed to read stdout from kubectl create authenticationToken") + errout, err := io.ReadAll(stderr) + Expect(err).ToNot(HaveOccurred(), "failed to read stderr from kubectl create authenticationToken") + + Expect(cmd.Wait()).To(Succeed(), "failed to wait kubectl create authenticationToken") + Expect(errout).To(BeEmpty()) + + return output +} diff --git a/test/e2e/mhc_remediations_test.go b/test/e2e/kcp_remediations_test.go similarity index 83% rename from test/e2e/mhc_remediations_test.go rename to test/e2e/kcp_remediations_test.go index 66471901d939..cdd9fd50efc4 100644 --- a/test/e2e/mhc_remediations_test.go +++ b/test/e2e/kcp_remediations_test.go @@ -23,9 +23,9 @@ import ( . "github.com/onsi/ginkgo/v2" ) -var _ = Describe("When testing unhealthy machines remediation", func() { - MachineRemediationSpec(ctx, func() MachineRemediationSpecInput { - return MachineRemediationSpecInput{ +var _ = Describe("When testing KCP remediation", func() { + KCPRemediationSpec(ctx, func() KCPRemediationSpecInput { + return KCPRemediationSpecInput{ E2EConfig: e2eConfig, ClusterctlConfigPath: clusterctlConfigPath, BootstrapClusterProxy: bootstrapClusterProxy, diff --git a/test/e2e/mhc_remediations.go b/test/e2e/md_remediations.go similarity index 61% rename from test/e2e/mhc_remediations.go rename to test/e2e/md_remediations.go index b1c3989d4f61..7d95a032518f 100644 --- a/test/e2e/mhc_remediations.go +++ b/test/e2e/md_remediations.go @@ -32,8 +32,8 @@ import ( "sigs.k8s.io/cluster-api/util" ) -// MachineRemediationSpecInput is the input for MachineRemediationSpec. -type MachineRemediationSpecInput struct { +// MachineDeploymentRemediationSpecInput is the input for MachineDeploymentRemediationSpec. +type MachineDeploymentRemediationSpecInput struct { E2EConfig *clusterctl.E2EConfig ClusterctlConfigPath string BootstrapClusterProxy framework.ClusterProxy @@ -41,26 +41,19 @@ type MachineRemediationSpecInput struct { SkipCleanup bool ControlPlaneWaiters clusterctl.ControlPlaneWaiters - // KCPFlavor, if specified, must refer to a template that has a MachineHealthCheck - // resource configured to match the control plane Machines (cluster.x-k8s.io/controlplane: "" label) - // and be configured to treat "e2e.remediation.condition" "False" as an unhealthy - // condition with a short timeout. - // If not specified, "kcp-remediation" is used. - KCPFlavor *string - - // MDFlavor, if specified, must refer to a template that has a MachineHealthCheck + // Flavor, if specified, must refer to a template that has a MachineHealthCheck // resource configured to match the MachineDeployment managed Machines and be // configured to treat "e2e.remediation.condition" "False" as an unhealthy // condition with a short timeout. // If not specified, "md-remediation" is used. - MDFlavor *string + Flavor *string } -// MachineRemediationSpec implements a test that verifies that Machines are remediated by MHC during unhealthy conditions. -func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemediationSpecInput) { +// MachineDeploymentRemediationSpec implements a test that verifies that Machines are remediated by MHC during unhealthy conditions. +func MachineDeploymentRemediationSpec(ctx context.Context, inputGetter func() MachineDeploymentRemediationSpecInput) { var ( - specName = "mhc-remediation" - input MachineRemediationSpecInput + specName = "md-remediation" + input MachineDeploymentRemediationSpecInput namespace *corev1.Namespace cancelWatches context.CancelFunc clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult @@ -80,7 +73,7 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) }) - It("Should successfully trigger machine deployment remediation", func() { + It("Should replace unhealthy machines", func() { By("Creating a workload cluster") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -90,7 +83,7 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed ClusterctlConfigPath: input.ClusterctlConfigPath, KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: pointer.StringDeref(input.MDFlavor, "md-remediation"), + Flavor: pointer.StringDeref(input.Flavor, "md-remediation"), Namespace: namespace.Name, ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), @@ -103,6 +96,8 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), }, clusterResources) + // TODO: this should be re-written like the KCP remediation test, because the current implementation + // only tests that MHC applies the unhealthy condition but it doesn't test that the unhealthy machine is delete and a replacement machine comes up. By("Setting a machine unhealthy and wait for MachineDeployment remediation") framework.DiscoverMachineHealthChecksAndWaitForRemediation(ctx, framework.DiscoverMachineHealthCheckAndWaitForRemediationInput{ ClusterProxy: input.BootstrapClusterProxy, @@ -113,39 +108,6 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed By("PASSED!") }) - It("Should successfully trigger KCP remediation", func() { - By("Creating a workload cluster") - - clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: input.BootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), - ClusterctlConfigPath: input.ClusterctlConfigPath, - KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: pointer.StringDeref(input.KCPFlavor, "kcp-remediation"), - Namespace: namespace.Name, - ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), - KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64(3), - WorkerMachineCount: pointer.Int64(1), - }, - ControlPlaneWaiters: input.ControlPlaneWaiters, - WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), - }, clusterResources) - - By("Setting a machine unhealthy and wait for KubeadmControlPlane remediation") - framework.DiscoverMachineHealthChecksAndWaitForRemediation(ctx, framework.DiscoverMachineHealthCheckAndWaitForRemediationInput{ - ClusterProxy: input.BootstrapClusterProxy, - Cluster: clusterResources.Cluster, - WaitForMachineRemediation: input.E2EConfig.GetIntervals(specName, "wait-machine-remediation"), - }) - - By("PASSED!") - }) - AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) diff --git a/test/e2e/md_remediations_test.go b/test/e2e/md_remediations_test.go new file mode 100644 index 000000000000..c35b60938d82 --- /dev/null +++ b/test/e2e/md_remediations_test.go @@ -0,0 +1,36 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("When testing MachineDeployment remediation", func() { + MachineDeploymentRemediationSpec(ctx, func() MachineDeploymentRemediationSpecInput { + return MachineDeploymentRemediationSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) +}) diff --git a/test/infrastructure/container/docker.go b/test/infrastructure/container/docker.go index 204ec75ee7a3..a790f881b782 100644 --- a/test/infrastructure/container/docker.go +++ b/test/infrastructure/container/docker.go @@ -161,8 +161,7 @@ func (d *dockerRuntime) GetHostPort(ctx context.Context, containerName, portAndP } // ExecContainer executes a command in a running container and writes any output to the provided writer. -func (d *dockerRuntime) ExecContainer(_ context.Context, containerName string, config *ExecContainerInput, command string, args ...string) error { - ctx := context.Background() // Let the command finish, even if it takes longer than the default timeout +func (d *dockerRuntime) ExecContainer(ctx context.Context, containerName string, config *ExecContainerInput, command string, args ...string) error { execConfig := types.ExecConfig{ // Run with privileges so we can remount etc.. // This might not make sense in the most general sense, but it is diff --git a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go index edc90b4232ee..73e1906fb865 100644 --- a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go +++ b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go @@ -295,7 +295,8 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * // if the machine isn't bootstrapped, only then run bootstrap scripts if !dockerMachine.Spec.Bootstrapped { - timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) + // TODO: make this timeout configurable via DockerMachine API if it is required by the KCP remediation test. + timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() // Check for bootstrap success @@ -305,10 +306,29 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * if err := externalMachine.CheckForBootstrapSuccess(timeoutCtx, false); err != nil { bootstrapData, format, err := r.getBootstrapData(timeoutCtx, machine) if err != nil { - log.Error(err, "failed to get bootstrap data") return ctrl.Result{}, err } + // Setup a go routing to check for the machine being deleted while running bootstrap as a + // synchronous process, e.g. due to remediation. The routine stops when timeoutCtx is Done + // (either because canceled intentionally due to machine deletion or canceled by the defer cancel() + // call when exiting from this func). + go func() { + for { + select { + case <-timeoutCtx.Done(): + return + default: + updatedDockerMachine := &infrav1.DockerMachine{} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(dockerMachine), updatedDockerMachine); err == nil && !updatedDockerMachine.DeletionTimestamp.IsZero() { + cancel() + return + } + time.Sleep(5 * time.Second) + } + } + }() + // Run the bootstrap script. Simulates cloud-init/Ignition. if err := externalMachine.ExecBootstrap(timeoutCtx, bootstrapData, format); err != nil { conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrapFailedReason, clusterv1.ConditionSeverityWarning, "Repeating bootstrap") @@ -331,6 +351,13 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } + // If the control plane is not yet initialized, there is no API server to contact to get the ProviderID for the Node + // hosted on this machine, so return early. + // NOTE: we are using RequeueAfter with a short interval in order to make test execution time more stable. + if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { + return ctrl.Result{RequeueAfter: 15 * time.Second}, nil + } + // Usually a cloud provider will do this, but there is no docker-cloud provider. // Requeue if there is an error, as this is likely momentary load balancer // state changes during control plane provisioning. From 23209f7cdc55003de14136a2ed58d5953d275c50 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Mon, 23 Jan 2023 18:35:27 +0100 Subject: [PATCH 2/8] address comments --- .../v1beta1/kubeadm_control_plane_types.go | 34 +++++------ .../kubeadm/internal/controllers/helpers.go | 6 +- .../internal/controllers/remediation.go | 8 +-- .../internal/controllers/remediation_test.go | 60 +++++++++---------- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go index f4661380ccd3..57d391f8ad5f 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go @@ -52,20 +52,20 @@ const ( // This annotation is used to detect any changes in ClusterConfiguration and trigger machine rollout in KCP. KubeadmClusterConfigurationAnnotation = "controlplane.cluster.x-k8s.io/kubeadm-cluster-configuration" - // RemediatingInProgressAnnotation is used to keep track that a KCP remediation is in progress, and more + // RemediationInProgressAnnotation is used to keep track that a KCP remediation is in progress, and more // specifically it tracks that the system is in between having deleted an unhealthy machine and recreating its replacement. // NOTE: if something external to CAPI removes this annotation the system cannot detect the above situation; this can lead to // failures in updating remediation retry or remediation count (both counters restart from zero). - RemediatingInProgressAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-in-progress" + RemediationInProgressAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-in-progress" - // MachineRemediationForAnnotation is used to link a new machine to the unhealthy machine it is replacing; - // please note that in case of retry, when also the remediating machine fails, the system keep tracks + // RemediationForAnnotation is used to link a new machine to the unhealthy machine it is replacing; + // please note that in case of retry, when also the remediating machine fails, the system keeps track of // the first machine of the sequence only. // NOTE: if something external to CAPI removes this annotation the system this can lead to // failures in updating remediation retry (the counter restarts from zero). - MachineRemediationForAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-for" + RemediationForAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-for" - // DefaultMinHealthyPeriod defines the default minimum number of seconds before we consider a remediation on a + // DefaultMinHealthyPeriod defines the default minimum period before we consider a remediation on a // machine unrelated from the previous remediation. DefaultMinHealthyPeriod = 1 * time.Hour ) @@ -111,7 +111,7 @@ type KubeadmControlPlaneSpec struct { // +kubebuilder:default={type: "RollingUpdate", rollingUpdate: {maxSurge: 1}} RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"` - // The RemediationStrategy that controls how control plane machines remediation happens. + // The RemediationStrategy that controls how control plane machine remediation happens. // +optional RemediationStrategy *RemediationStrategy `json:"remediationStrategy,omitempty"` } @@ -181,22 +181,22 @@ type RollingUpdate struct { MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` } -// RemediationStrategy allows to define how control plane machines remediation happens. +// RemediationStrategy allows to define how control plane machine remediation happens. type RemediationStrategy struct { - // MaxRetry is the Max number of retry while attempting to remediate an unhealthy machine. + // MaxRetry is the Max number of retries while attempting to remediate an unhealthy machine. // A retry happens when a machine that was created as a replacement for an unhealthy machine also fails. // For example, given a control plane with three machines M1, M2, M3: // - // M1 become unhealthy; remediation happens, and M1bis is created as a replacement. - // If M1-1 (replacement of M1) have problems while bootstrapping it will become unhealthy, and then be + // M1 become unhealthy; remediation happens, and M1-1 is created as a replacement. + // If M1-1 (replacement of M1) has problems while bootstrapping it will become unhealthy, and then be // remediated; such operation is considered a retry, remediation-retry #1. // If M1-2 (replacement of M1-2) becomes unhealthy, remediation-retry #2 will happen, etc. // // A retry could happen only after RetryPeriod from the previous retry. // If a machine is marked as unhealthy after MinHealthyPeriod from the previous remediation expired, - // this is not considered anymore a retry because the new issue is assumed unrelated from the previous one. + // this is not considered a retry anymore because the new issue is assumed unrelated from the previous one. // - // If not set, infinite retry will be attempted. + // If not set, the remedation will be retried infinitely. // +optional MaxRetry *int32 `json:"maxRetry,omitempty"` @@ -205,14 +205,14 @@ type RemediationStrategy struct { // // If not set, a retry will happen immediately. // +optional - RetryPeriod metav1.Duration `json:"retryDelaySeconds,omitempty"` + RetryPeriod metav1.Duration `json:"retryPeriod,omitempty"` // MinHealthyPeriod defines the duration after which KCP will consider any failure to a machine unrelated // from the previous one, and thus a new remediation is not considered a retry anymore. // // If not set, this value is defaulted to 1h. // +optional - MinHealthyPeriod *metav1.Duration `json:"minHealthySeconds,omitempty"` + MinHealthyPeriod *metav1.Duration `json:"minHealthyPeriod,omitempty"` } // KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane. @@ -287,8 +287,8 @@ type KubeadmControlPlaneStatus struct { } // LastRemediationStatus stores info about last remediation performed. -// NOTE: if for any reason information about last remediation are lost, RetryCount is going to restarts from 0 and thus -// more remediation than expected might happen. +// NOTE: if for any reason information about last remediation are lost, RetryCount is going to restart from 0 and thus +// more remediations than expected might happen. type LastRemediationStatus struct { // Machine is the machine name of the latest machine being remediated. Machine string `json:"machine"` diff --git a/controlplane/kubeadm/internal/controllers/helpers.go b/controlplane/kubeadm/internal/controllers/helpers.go index 02fbd7bf240e..3093408cd607 100644 --- a/controlplane/kubeadm/internal/controllers/helpers.go +++ b/controlplane/kubeadm/internal/controllers/helpers.go @@ -318,8 +318,8 @@ func (r *KubeadmControlPlaneReconciler) generateMachine(ctx context.Context, kcp // In case this machine is being created as a consequence of a remediation, then add an annotation // tracking the name of the machine we are remediating for. // NOTE: This is required in order to track remediation retries. - if v, ok := kcp.Annotations[controlplanev1.RemediatingInProgressAnnotation]; ok && v == "true" { - machine.Annotations[controlplanev1.MachineRemediationForAnnotation] = kcp.Status.LastRemediation.Machine + if v, ok := kcp.Annotations[controlplanev1.RemediationInProgressAnnotation]; ok && v == "true" { + machine.Annotations[controlplanev1.RemediationForAnnotation] = kcp.Status.LastRemediation.Machine } // Machine's bootstrap config may be missing ClusterConfiguration if it is not the first machine in the control plane. @@ -342,7 +342,7 @@ func (r *KubeadmControlPlaneReconciler) generateMachine(ctx context.Context, kcp // Remove the annotation tracking that a remediation is in progress (the remediation completed when // the replacement machine have been created above). - delete(kcp.Annotations, controlplanev1.RemediatingInProgressAnnotation) + delete(kcp.Annotations, controlplanev1.RemediationInProgressAnnotation) return nil } diff --git a/controlplane/kubeadm/internal/controllers/remediation.go b/controlplane/kubeadm/internal/controllers/remediation.go index c7341b72ef49..81248f571aeb 100644 --- a/controlplane/kubeadm/internal/controllers/remediation.go +++ b/controlplane/kubeadm/internal/controllers/remediation.go @@ -72,7 +72,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C } // Returns if another remediation is in progress but the new machine is not yet created. - if _, ok := controlPlane.KCP.Annotations[controlplanev1.RemediatingInProgressAnnotation]; ok { + if _, ok := controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]; ok { return ctrl.Result{}, nil } @@ -214,7 +214,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // Set annotations tracking remediation is in progress (remediation will complete when a replacement machine is created). annotations.AddAnnotations(controlPlane.KCP, map[string]string{ - controlplanev1.RemediatingInProgressAnnotation: "", + controlplanev1.RemediationInProgressAnnotation: "", }) // Stores info about last remediation. @@ -268,11 +268,11 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin } // Check if the machine being remediated has been created as a remediation for a previous unhealthy machine. - // NOTE: if someone/something manually removing the MachineRemediationForAnnotation on Machines or one of the LastRemediatedMachineAnnotation + // NOTE: if someone/something manually removing the RemediationForAnnotation on Machines or one of the LastRemediatedMachineAnnotation // and LastRemediatedMachineRetryAnnotation on KCP, this could potentially lead to executing more retries than expected, // but this is considered acceptable in such a case. machineRemediatingFor := machineToBeRemediated.Name - if remediationFor, ok := machineToBeRemediated.Annotations[controlplanev1.MachineRemediationForAnnotation]; ok { + if remediationFor, ok := machineToBeRemediated.Annotations[controlplanev1.RemediationForAnnotation]; ok { // If the remediation is happening before minHealthyPeriod is expired, then KCP considers this // as a remediation for the same previously unhealthy machine. // TODO: add example diff --git a/controlplane/kubeadm/internal/controllers/remediation_test.go b/controlplane/kubeadm/internal/controllers/remediation_test.go index 9ce9e72acf0a..9e82066ca9e4 100644 --- a/controlplane/kubeadm/internal/controllers/remediation_test.go +++ b/controlplane/kubeadm/internal/controllers/remediation_test.go @@ -113,7 +113,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { KCP: &controlplanev1.KubeadmControlPlane{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - controlplanev1.RemediatingInProgressAnnotation: "true", + controlplanev1.RemediationInProgressAnnotation: "true", }, }, }, @@ -185,7 +185,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(3))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal("m0")) @@ -242,7 +242,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -302,7 +302,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -360,7 +360,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(2))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal("m0")) @@ -407,7 +407,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") @@ -437,7 +437,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") @@ -479,7 +479,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") @@ -523,7 +523,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") @@ -567,7 +567,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -615,7 +615,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -634,7 +634,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(machineRemediatingFor)) // Simulate KCP dropping RemediationInProgressAnnotation after creating the replacement machine. - delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) controlPlane.Machines = collections.FromMachines(mi) @@ -649,7 +649,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(i - 1))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(mi.Name)) @@ -703,7 +703,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -752,7 +752,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -802,7 +802,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -849,7 +849,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { _, err = r.reconcileUnhealthyMachines(ctx, controlPlane) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, @@ -895,7 +895,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -914,7 +914,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(machineRemediatingFor)) // Simulate KCP dropping RemediationInProgressAnnotation after creating the replacement machine. - delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) controlPlane.Machines = collections.FromMachines(mi, m2, m3) // Reconcile unhealthy replacements for m1. @@ -929,7 +929,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(i - 4))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(mi.Name)) @@ -999,7 +999,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) @@ -1016,7 +1016,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // NOTE: scale up also resets remediation in progress and remediation counts. m2 := createMachine(ctx, g, ns.Name, "m2-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(m1.Name)) - delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) // Control plane not initialized yet, Second CP is unhealthy and gets remediated (retry 2) @@ -1032,7 +1032,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(1))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) @@ -1049,7 +1049,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // NOTE: scale up also resets remediation in progress and remediation counts. m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(m1.Name)) - delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) g.Expect(env.Cleanup(ctx, m3)).To(Succeed()) }) @@ -1104,7 +1104,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) @@ -1121,7 +1121,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // NOTE: scale up also resets remediation in progress and remediation counts. m3 := createMachine(ctx, g, ns.Name, "m3-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(m2.Name)) - delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) // Control plane not initialized yet, Second CP is unhealthy and gets remediated (retry 2) @@ -1137,7 +1137,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(1))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m3.Name)) @@ -1154,7 +1154,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // NOTE: scale up also resets remediation in progress and remediation counts. m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(m1.Name)) - delete(controlPlane.KCP.Annotations, controlplanev1.RemediatingInProgressAnnotation) + delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) g.Expect(env.Cleanup(ctx, m1, m4)).To(Succeed()) }) @@ -1210,7 +1210,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(ret.IsZero()).To(BeFalse()) // Remediation completed, requeue g.Expect(err).ToNot(HaveOccurred()) - g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediatingInProgressAnnotation)) + g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) @@ -1668,7 +1668,7 @@ func withRemediateForAnnotation(remediatedFor string) machineOption { if machine.Annotations == nil { machine.Annotations = map[string]string{} } - machine.Annotations[controlplanev1.MachineRemediationForAnnotation] = remediatedFor + machine.Annotations[controlplanev1.RemediationForAnnotation] = remediatedFor } } From 0000b8ebe3c349dac1a072bb2a00fbda604416ba Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Mon, 30 Jan 2023 11:43:35 +0100 Subject: [PATCH 3/8] improve minHealthyPeriod comment --- .../api/v1beta1/kubeadm_control_plane_types.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go index 57d391f8ad5f..d4eef2b1113c 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go @@ -208,7 +208,17 @@ type RemediationStrategy struct { RetryPeriod metav1.Duration `json:"retryPeriod,omitempty"` // MinHealthyPeriod defines the duration after which KCP will consider any failure to a machine unrelated - // from the previous one, and thus a new remediation is not considered a retry anymore. + // from the previous one. In this case the remediation is not considered a retry anymore, and thus the retry + // counter restarts from 0. For example, assuming MinHealthyPeriod is set to 1h (default) + // + // M1 become unhealthy; remediation happens, and M1-1 is created as a replacement. + // If M1-1 (replacement of M1) has problems within the 1hr after the creation, also + // this machine will be remediated and this operation is considered a retry - a problem related + // to the original issue happened to M1 -. + // + // If instead the problem on M1-1 is happening after MinHealthyPeriod expired, e.g. four days after + // m1-1 has been created as a remediation of M1, the problem on M1-1 is considered unrelated to + // the original issue happened to M1. // // If not set, this value is defaulted to 1h. // +optional From 7b22911a4b40833cc498bf485e86d86675634826 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Wed, 1 Feb 2023 18:15:31 +0100 Subject: [PATCH 4/8] more comments --- .../internal/controllers/remediation.go | 46 +-- .../src/developer/providers/v1.3-to-v1.4.md | 2 + test/e2e/kcp_remediations.go | 284 +++++++++--------- .../controllers/dockermachine_controller.go | 8 +- 4 files changed, 174 insertions(+), 166 deletions(-) diff --git a/controlplane/kubeadm/internal/controllers/remediation.go b/controlplane/kubeadm/internal/controllers/remediation.go index 81248f571aeb..71843e9799ad 100644 --- a/controlplane/kubeadm/internal/controllers/remediation.go +++ b/controlplane/kubeadm/internal/controllers/remediation.go @@ -43,6 +43,7 @@ import ( // based on the process described in https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20191017-kubeadm-based-control-plane.md#remediation-using-delete-and-recreate func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.Context, controlPlane *internal.ControlPlane) (ret ctrl.Result, retErr error) { log := ctrl.LoggerFrom(ctx) + reconciliationTime := time.Now() // Cleanup pending remediation actions not completed for any reasons (e.g. number of current replicas is less or equal to 1) // if the underlying machine is now back to healthy / not deleting. @@ -116,15 +117,16 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // Before starting remediation, run preflight checks in order to verify it is safe to remediate. // If any of the following checks fails, we'll surface the reason in the MachineOwnerRemediated condition. - log.WithValues("Machine", klog.KObj(machineToBeRemediated), "Initialized", controlPlane.KCP.Status.Initialized) + log = log.WithValues("Machine", klog.KObj(machineToBeRemediated), "initialized", controlPlane.KCP.Status.Initialized) // Check if KCP is allowed to remediate considering retry limits: // - Remediation cannot happen because retryPeriod is not yet expired. // - KCP already reached MaxRetries limit. - hasRetries, retryCount := r.checkRetryLimits(log, machineToBeRemediated, controlPlane) - if !hasRetries { + machineRemediatingFor, canRemediate, retryCount := r.checkRetryLimits(log, machineToBeRemediated, controlPlane, reconciliationTime) + if !canRemediate { return ctrl.Result{}, nil } + log = log.WithValues("machineRemediatingFor", klog.KRef(controlPlane.KCP.Namespace, machineRemediatingFor), "retryCount", retryCount) // Executes checks that applies only if the control plane is already initialized; in this case KCP can // remediate only if it can safely assume that the operation preserves the operation state of the existing cluster (or at least it doesn't make it worst). @@ -221,7 +223,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // NOTE: Some of those info have been computed above, but they must surface on the object only here, after machine has been deleted. controlPlane.KCP.Status.LastRemediation = &controlplanev1.LastRemediationStatus{ Machine: machineToBeRemediated.Name, - Timestamp: metav1.Timestamp{Seconds: time.Now().Unix()}, + Timestamp: metav1.Timestamp{Seconds: reconciliationTime.Unix()}, RetryCount: retryCount, } @@ -229,14 +231,14 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C } // checkRetryLimits checks if KCP is allowed to remediate considering retry limits: -// - Remediation cannot happen because retryDelay is not yet expired. +// - Remediation cannot happen because retryPeriod is not yet expired. // - KCP already reached the maximum number of retries for a machine. // NOTE: Counting the number of retries is required In order to prevent infinite remediation e.g. in case the // first Control Plane machine is failing due to quota issue. -func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machineToBeRemediated *clusterv1.Machine, controlPlane *internal.ControlPlane) (hasRetries bool, retryCount int32) { +func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machineToBeRemediated *clusterv1.Machine, controlPlane *internal.ControlPlane, reconciliationTime time.Time) (machineRemediatingFor string, canRemediate bool, retryCount int32) { // If there is no last remediation, this is the first try of a new retry sequence. if controlPlane.KCP.Status.LastRemediation == nil { - return true, 0 + return "", true, 0 } // Gets MinHealthySeconds and RetryDelaySeconds from the remediation strategy, or use defaults. @@ -245,9 +247,9 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin minHealthyPeriod = controlPlane.KCP.Spec.RemediationStrategy.MinHealthyPeriod.Duration } - retryDelay := time.Duration(0) + retryPeriod := time.Duration(0) if controlPlane.KCP.Spec.RemediationStrategy != nil { - retryDelay = controlPlane.KCP.Spec.RemediationStrategy.RetryPeriod.Duration + retryPeriod = controlPlane.KCP.Spec.RemediationStrategy.RetryPeriod.Duration } // Gets the timestamp of the last remediation; if missing, default to a value @@ -262,7 +264,7 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin return x } - lastRemediationTimestamp := time.Now().Add(-2 * max(minHealthyPeriod, retryDelay)).UTC() + lastRemediationTimestamp := reconciliationTime.Add(-2 * max(minHealthyPeriod, retryPeriod)).UTC() if controlPlane.KCP.Status.LastRemediation != nil { lastRemediationTimestamp = time.Unix(controlPlane.KCP.Status.LastRemediation.Timestamp.Seconds, int64(controlPlane.KCP.Status.LastRemediation.Timestamp.Nanos)) } @@ -271,27 +273,27 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin // NOTE: if someone/something manually removing the RemediationForAnnotation on Machines or one of the LastRemediatedMachineAnnotation // and LastRemediatedMachineRetryAnnotation on KCP, this could potentially lead to executing more retries than expected, // but this is considered acceptable in such a case. - machineRemediatingFor := machineToBeRemediated.Name + machineRemediatingFor = machineToBeRemediated.Name if remediationFor, ok := machineToBeRemediated.Annotations[controlplanev1.RemediationForAnnotation]; ok { // If the remediation is happening before minHealthyPeriod is expired, then KCP considers this // as a remediation for the same previously unhealthy machine. // TODO: add example - if lastRemediationTimestamp.Add(minHealthyPeriod).After(time.Now()) { + if lastRemediationTimestamp.Add(minHealthyPeriod).After(reconciliationTime) { machineRemediatingFor = remediationFor - log.WithValues("RemediationRetryFor", klog.KRef(machineToBeRemediated.Namespace, machineRemediatingFor)) + log = log.WithValues("RemediationRetryFor", klog.KRef(machineToBeRemediated.Namespace, machineRemediatingFor)) } } // If remediation is happening for a different machine, this is the first try of a new retry sequence. if controlPlane.KCP.Status.LastRemediation.Machine != machineRemediatingFor { - return true, 0 + return machineRemediatingFor, true, 0 } - // Check if remediation can happen because retryDelay is passed. - if lastRemediationTimestamp.Add(retryDelay).After(time.Now().UTC()) { - log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed in the latest %s. Skipping remediation", retryDelay)) - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryDelay)", retryDelay) - return false, 0 + // Check if remediation can happen because retryPeriod is passed. + if lastRemediationTimestamp.Add(retryPeriod).After(reconciliationTime.UTC()) { + log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed in the latest %s. Skipping remediation", retryPeriod)) + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryDelay)", retryPeriod) + return machineRemediatingFor, false, 0 } // Check if remediation can happen because of maxRetry is not reached yet, if defined. @@ -302,14 +304,12 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin if retry >= maxRetry { log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed %d times (MaxRetry %d). Skipping remediation", retry, maxRetry)) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed %d times (MaxRetry)", maxRetry) - return false, 0 + return machineRemediatingFor, false, 0 } } retryCount = retry + 1 - - log.WithValues("RetryCount", retryCount) - return true, retryCount + return machineRemediatingFor, true, retryCount } // canSafelyRemoveEtcdMember assess if it is possible to remove the member hosted on the machine to be remediated diff --git a/docs/book/src/developer/providers/v1.3-to-v1.4.md b/docs/book/src/developer/providers/v1.3-to-v1.4.md index ef339f3ecba2..4e8b31b3ec3c 100644 --- a/docs/book/src/developer/providers/v1.3-to-v1.4.md +++ b/docs/book/src/developer/providers/v1.3-to-v1.4.md @@ -61,6 +61,8 @@ maintainers of providers and consumers of our Go API. Please note that the following logging flags have been removed: (in `component-base`, but this affects all CAPI controllers): `--add-dir-header`, `--alsologtostderr`, `--log-backtrace-at`, `--log-dir`, `--log-file`, `--log-file-max-size`, `--logtostderr`, `--one-output`, `--skip-headers`, `--skip-log-headers` and `--stderrthreshold`. For more information, please see: https://github.com/kubernetes/enhancements/issues/2845 +- A new `KCPRemediationSpec` test has been added providing better test coverage for KCP remediation most common use cases. As a consequence `MachineRemediationSpec` now only tests remediation of + worker machines (NOTE: we plan to improve this test as well in a future iteration). ### Suggested changes for providers diff --git a/test/e2e/kcp_remediations.go b/test/e2e/kcp_remediations.go index 6d16633dfbd8..c0e69cd0a0bc 100644 --- a/test/e2e/kcp_remediations.go +++ b/test/e2e/kcp_remediations.go @@ -113,20 +113,20 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp // Creates the workload cluster. clusterResources = createWorkloadClusterAndWait(ctx, createWorkloadClusterAndWaitInput{ E2EConfig: input.E2EConfig, - clusterctlConfigPath: input.ClusterctlConfigPath, - proxy: input.BootstrapClusterProxy, - artifactFolder: input.ArtifactFolder, - specName: specName, - flavor: input.Flavor, + ClusterctlConfigPath: input.ClusterctlConfigPath, + Proxy: input.BootstrapClusterProxy, + ArtifactFolder: input.ArtifactFolder, + SpecName: specName, + Flavor: input.Flavor, // values to be injected in the template - namespace: namespace.Name, + Namespace: namespace.Name, // Token with credentials to use for accessing the ConfigMap on managements cluster from the workload cluster. - // NOTE: this func also setups credentials/RBAC rules and everything necessary to get the authentication authenticationToken. - authenticationToken: getAuthenticationToken(ctx, input.BootstrapClusterProxy, namespace.Name), + // NOTE: this func also setups credentials/RBAC rules and everything necessary to get the authenticationToken. + AuthenticationToken: getAuthenticationToken(ctx, input.BootstrapClusterProxy, namespace.Name), // Address to be used for accessing the management cluster from a workload cluster. - serverAddr: getServerAddr(input.BootstrapClusterProxy.GetKubeconfigPath()), + ServerAddr: getServerAddr(input.BootstrapClusterProxy.GetKubeconfigPath()), }) // The first CP machine comes up but it does not complete bootstrap @@ -135,12 +135,12 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp By("Wait for the cluster to get stuck with the first CP machine not completing the bootstrap") allMachines, newMachines := waitForMachines(ctx, waitForMachinesInput{ - lister: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - clusterName: clusterResources.Cluster.Name, - expectedReplicas: 1, - waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), - checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + Lister: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + ClusterName: clusterResources.Cluster.Name, + ExpectedReplicas: 1, + WaitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + CheckMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), }) Expect(allMachines).To(HaveLen(1)) Expect(newMachines).To(HaveLen(1)) @@ -166,13 +166,13 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp log.Logf("Wait for the first CP machine to be remediated, and the replacement machine to come up, but again get stuck with the Machine not completing the bootstrap") allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ - lister: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - clusterName: clusterResources.Cluster.Name, - expectedReplicas: 1, - expectedDeletedMachines: []string{firstMachineName}, - waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), - checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + Lister: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + ClusterName: clusterResources.Cluster.Name, + ExpectedReplicas: 1, + ExpectedDeletedMachines: []string{firstMachineName}, + WaitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + CheckMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), }) Expect(allMachines).To(HaveLen(1)) Expect(newMachines).To(HaveLen(1)) @@ -195,10 +195,10 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", firstMachineReplacementName) sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ - client: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - machine: firstMachineReplacementName, - signal: "pass", + Client: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + Machine: firstMachineReplacementName, + Signal: "pass", }) log.Logf("Waiting for Machine %s to be provisioned", firstMachineReplacementName) Eventually(func() bool { @@ -213,14 +213,14 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp By("Wait for the cluster to get stuck with the second CP machine not completing the bootstrap") allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ - lister: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - clusterName: clusterResources.Cluster.Name, - expectedReplicas: 2, - expectedDeletedMachines: []string{}, - expectedOldMachines: []string{firstMachineReplacementName}, - waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), - checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + Lister: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + ClusterName: clusterResources.Cluster.Name, + ExpectedReplicas: 2, + ExpectedDeletedMachines: []string{}, + ExpectedOldMachines: []string{firstMachineReplacementName}, + WaitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + CheckMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), }) Expect(allMachines).To(HaveLen(2)) Expect(newMachines).To(HaveLen(1)) @@ -246,14 +246,14 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp log.Logf("Wait for the second CP machine to be remediated, and the replacement machine to come up, but again get stuck with the Machine not completing the bootstrap") allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ - lister: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - clusterName: clusterResources.Cluster.Name, - expectedReplicas: 2, - expectedDeletedMachines: []string{secondMachineName}, - expectedOldMachines: []string{firstMachineReplacementName}, - waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), - checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + Lister: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + ClusterName: clusterResources.Cluster.Name, + ExpectedReplicas: 2, + ExpectedDeletedMachines: []string{secondMachineName}, + ExpectedOldMachines: []string{firstMachineReplacementName}, + WaitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + CheckMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), }) Expect(allMachines).To(HaveLen(2)) Expect(newMachines).To(HaveLen(1)) @@ -276,10 +276,10 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", secondMachineReplacementName) sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ - client: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - machine: secondMachineReplacementName, - signal: "pass", + Client: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + Machine: secondMachineReplacementName, + Signal: "pass", }) log.Logf("Waiting for Machine %s to be provisioned", secondMachineReplacementName) Eventually(func() bool { @@ -294,14 +294,14 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp By("Wait for the cluster to get stuck with the third CP machine not completing the bootstrap") allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ - lister: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - clusterName: clusterResources.Cluster.Name, - expectedReplicas: 3, - expectedDeletedMachines: []string{}, - expectedOldMachines: []string{firstMachineReplacementName, secondMachineReplacementName}, - waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), - checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + Lister: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + ClusterName: clusterResources.Cluster.Name, + ExpectedReplicas: 3, + ExpectedDeletedMachines: []string{}, + ExpectedOldMachines: []string{firstMachineReplacementName, secondMachineReplacementName}, + WaitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + CheckMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), }) Expect(allMachines).To(HaveLen(3)) Expect(newMachines).To(HaveLen(1)) @@ -318,10 +318,10 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", thirdMachineName) sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ - client: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - machine: thirdMachineName, - signal: "pass", + Client: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + Machine: thirdMachineName, + Signal: "pass", }) log.Logf("Waiting for Machine %s to be provisioned", thirdMachineName) Eventually(func() bool { @@ -359,14 +359,14 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp log.Logf("Wait for the third CP machine to be remediated, and the replacement machine to come up, but again get stuck with the Machine not completing the bootstrap") allMachines, newMachines = waitForMachines(ctx, waitForMachinesInput{ - lister: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - clusterName: clusterResources.Cluster.Name, - expectedReplicas: 3, - expectedDeletedMachines: []string{thirdMachineName}, - expectedOldMachines: []string{firstMachineReplacementName, secondMachineReplacementName}, - waitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), - checkMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), + Lister: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + ClusterName: clusterResources.Cluster.Name, + ExpectedReplicas: 3, + ExpectedDeletedMachines: []string{thirdMachineName}, + ExpectedOldMachines: []string{firstMachineReplacementName, secondMachineReplacementName}, + WaitForMachinesIntervals: input.E2EConfig.GetIntervals(specName, "wait-machines"), + CheckMachineListStableIntervals: input.E2EConfig.GetIntervals(specName, "check-machines-stable"), }) Expect(allMachines).To(HaveLen(3)) Expect(newMachines).To(HaveLen(1)) @@ -387,10 +387,10 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp Byf("Unblock bootstrap for Machine %s and wait for it to be provisioned", thirdMachineReplacementName) sendSignalToBootstrappingMachine(ctx, sendSignalToBootstrappingMachineInput{ - client: input.BootstrapClusterProxy.GetClient(), - namespace: namespace.Name, - machine: thirdMachineReplacementName, - signal: "pass", + Client: input.BootstrapClusterProxy.GetClient(), + Namespace: namespace.Name, + Machine: thirdMachineReplacementName, + Signal: "pass", }) log.Logf("Waiting for Machine %s to be provisioned", thirdMachineReplacementName) Eventually(func() bool { @@ -428,182 +428,186 @@ func createConfigMapForMachinesBootstrapSignal(ctx context.Context, writer clien type createWorkloadClusterAndWaitInput struct { E2EConfig *clusterctl.E2EConfig - clusterctlConfigPath string - proxy framework.ClusterProxy - artifactFolder string - specName string - flavor *string - namespace string - authenticationToken []byte - serverAddr string + ClusterctlConfigPath string + Proxy framework.ClusterProxy + ArtifactFolder string + SpecName string + Flavor *string + Namespace string + AuthenticationToken []byte + ServerAddr string } -// createWorkloadClusterAndWait creates a workload cluster ard return ass soon as the cluster infrastructure is ready. +// createWorkloadClusterAndWait creates a workload cluster ard return as soon as the cluster infrastructure is ready. +// NOTE: we are not using the same func used by other tests because it would fail if the control plane doesn't come up, +// +// which instead is expected in this case. +// // NOTE: clusterResources is filled only partially. func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClusterAndWaitInput) (clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) { clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) // gets the cluster template log.Logf("Getting the cluster template yaml") - clusterName := fmt.Sprintf("%s-%s", input.specName, util.RandomString(6)) + clusterName := fmt.Sprintf("%s-%s", input.SpecName, util.RandomString(6)) workloadClusterTemplate := clusterctl.ConfigCluster(ctx, clusterctl.ConfigClusterInput{ // pass the clusterctl config file that points to the local provider repository created for this test, - ClusterctlConfigPath: input.clusterctlConfigPath, + ClusterctlConfigPath: input.ClusterctlConfigPath, // pass reference to the management cluster hosting this test - KubeconfigPath: input.proxy.GetKubeconfigPath(), + KubeconfigPath: input.Proxy.GetKubeconfigPath(), // select template - Flavor: pointer.StringDeref(input.flavor, "kcp-remediation"), + Flavor: pointer.StringDeref(input.Flavor, "kcp-remediation"), // define template variables - Namespace: input.namespace, + Namespace: input.Namespace, ClusterName: clusterName, KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), ControlPlaneMachineCount: pointer.Int64(3), WorkerMachineCount: pointer.Int64(0), InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, // setup clusterctl logs folder - LogFolder: filepath.Join(input.artifactFolder, "clusters", input.proxy.GetName()), + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.Proxy.GetName()), // Adds authenticationToken, server address and namespace variables to be injected in the cluster template. ClusterctlVariables: map[string]string{ - "TOKEN": string(input.authenticationToken), - "SERVER": input.serverAddr, - "NAMESPACE": input.namespace, + "TOKEN": string(input.AuthenticationToken), + "SERVER": input.ServerAddr, + "NAMESPACE": input.Namespace, }, }) Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") Eventually(func() error { - return input.proxy.Apply(ctx, workloadClusterTemplate) + return input.Proxy.Apply(ctx, workloadClusterTemplate) }, 10*time.Second).Should(Succeed(), "Failed to apply the cluster template") log.Logf("Waiting for the cluster infrastructure to be provisioned") clusterResources.Cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ - Getter: input.proxy.GetClient(), - Namespace: input.namespace, + Getter: input.Proxy.GetClient(), + Namespace: input.Namespace, Name: clusterName, - }, input.E2EConfig.GetIntervals(input.specName, "wait-cluster")...) + }, input.E2EConfig.GetIntervals(input.SpecName, "wait-cluster")...) return clusterResources } type sendSignalToBootstrappingMachineInput struct { - client client.Client - namespace string - machine string - signal string + Client client.Client + Namespace string + Machine string + Signal string } // sendSignalToBootstrappingMachine sends a signal to a machine stuck during bootstrap. func sendSignalToBootstrappingMachine(ctx context.Context, input sendSignalToBootstrappingMachineInput) { - log.Logf("Sending bootstrap signal %s to Machine %s", input.signal, input.machine) + log.Logf("Sending bootstrap signal %s to Machine %s", input.Signal, input.Machine) cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: configMapName, - Namespace: input.namespace, + Namespace: input.Namespace, }, } - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed(), "failed to get mhc-test config map") + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed(), "failed to get mhc-test config map") cmWithSignal := cm.DeepCopy() - cmWithSignal.Data[configMapDataKey] = input.signal - Expect(input.client.Patch(ctx, cmWithSignal, client.MergeFrom(cm))).To(Succeed(), "failed to patch mhc-test config map") + cmWithSignal.Data[configMapDataKey] = input.Signal + Expect(input.Client.Patch(ctx, cmWithSignal, client.MergeFrom(cm))).To(Succeed(), "failed to patch mhc-test config map") - log.Logf("Waiting for Machine %s to acknowledge signal %s has been received", input.machine, input.signal) + log.Logf("Waiting for Machine %s to acknowledge signal %s has been received", input.Machine, input.Signal) Eventually(func() string { - _ = input.client.Get(ctx, client.ObjectKeyFromObject(cmWithSignal), cmWithSignal) + _ = input.Client.Get(ctx, client.ObjectKeyFromObject(cmWithSignal), cmWithSignal) return cmWithSignal.Data[configMapDataKey] - }, "1m", "10s").Should(Equal(fmt.Sprintf("ack-%s", input.signal)), "Failed to get ack signal from machine %s", input.machine) + }, "1m", "10s").Should(Equal(fmt.Sprintf("ack-%s", input.Signal)), "Failed to get ack signal from machine %s", input.Machine) machine := &clusterv1.Machine{ ObjectMeta: metav1.ObjectMeta{ - Name: input.machine, - Namespace: input.namespace, + Name: input.Machine, + Namespace: input.Namespace, }, } - Expect(input.client.Get(ctx, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) + Expect(input.Client.Get(ctx, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) // Resetting the signal in the config map cmWithSignal.Data[configMapDataKey] = "hold" - Expect(input.client.Patch(ctx, cmWithSignal, client.MergeFrom(cm))).To(Succeed(), "failed to patch mhc-test config map") + Expect(input.Client.Patch(ctx, cmWithSignal, client.MergeFrom(cm))).To(Succeed(), "failed to patch mhc-test config map") } type waitForMachinesInput struct { - lister framework.Lister - namespace string - clusterName string - expectedReplicas int - expectedOldMachines []string - expectedDeletedMachines []string - waitForMachinesIntervals []interface{} - checkMachineListStableIntervals []interface{} + Lister framework.Lister + Namespace string + ClusterName string + ExpectedReplicas int + ExpectedOldMachines []string + ExpectedDeletedMachines []string + WaitForMachinesIntervals []interface{} + CheckMachineListStableIntervals []interface{} } // waitForMachines waits for machines to reach a well known state defined by number of replicas, a list of machines to exists, -// a list of machines to not exists anymore. The func also check that the state is stable for some time before +// a list of machines to not exist anymore. The func also check that the state is stable for some time before // returning the list of new machines. func waitForMachines(ctx context.Context, input waitForMachinesInput) (allMachineNames, newMachineNames []string) { - inClustersNamespaceListOption := client.InNamespace(input.namespace) + inClustersNamespaceListOption := client.InNamespace(input.Namespace) matchClusterListOption := client.MatchingLabels{ - clusterv1.ClusterNameLabel: input.clusterName, + clusterv1.ClusterNameLabel: input.ClusterName, clusterv1.MachineControlPlaneLabel: "", } - expectedOldMachines := sets.NewString(input.expectedOldMachines...) - expectedDeletedMachines := sets.NewString(input.expectedDeletedMachines...) - allMachines := sets.NewString() - newMachines := sets.NewString() + expectedOldMachines := sets.Set[string]{}.Insert(input.ExpectedOldMachines...) + expectedDeletedMachines := sets.Set[string]{}.Insert(input.ExpectedDeletedMachines...) + allMachines := sets.Set[string]{} + newMachines := sets.Set[string]{} machineList := &clusterv1.MachineList{} // Waits for the desired set of machines to exist. - log.Logf("Waiting for %d machines, must have %s, must not have %s", input.expectedReplicas, expectedOldMachines.List(), expectedDeletedMachines.List()) + log.Logf("Waiting for %d machines, must have %s, must not have %s", input.ExpectedReplicas, expectedOldMachines.UnsortedList(), expectedDeletedMachines.UnsortedList()) Eventually(func() bool { // Gets the list of machines - if err := input.lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { + if err := input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { return false } - allMachines = sets.NewString() + allMachines = sets.Set[string]{} for i := range machineList.Items { allMachines.Insert(machineList.Items[i].Name) } // Compute new machines (all - old - to be deleted) newMachines = allMachines.Clone() - newMachines.Delete(expectedOldMachines.List()...) - newMachines.Delete(expectedDeletedMachines.List()...) + newMachines.Delete(expectedOldMachines.UnsortedList()...) + newMachines.Delete(expectedDeletedMachines.UnsortedList()...) - log.Logf(" - expected %d, got %d: %s, of which new %s, must have check: %t, must not have check: %t", input.expectedReplicas, allMachines.Len(), allMachines.List(), newMachines.List(), allMachines.HasAll(expectedOldMachines.List()...), !allMachines.HasAny(expectedDeletedMachines.List()...)) + log.Logf(" - expected %d, got %d: %s, of which new %s, must have check: %t, must not have check: %t", input.ExpectedReplicas, allMachines.Len(), allMachines.UnsortedList(), newMachines.UnsortedList(), allMachines.HasAll(expectedOldMachines.UnsortedList()...), !allMachines.HasAny(expectedDeletedMachines.UnsortedList()...)) // Ensures all the expected old machines are still there. - if !allMachines.HasAll(expectedOldMachines.List()...) { + if !allMachines.HasAll(expectedOldMachines.UnsortedList()...) { return false } // Ensures none of the machines to be deleted is still there. - if allMachines.HasAny(expectedDeletedMachines.List()...) { + if allMachines.HasAny(expectedDeletedMachines.UnsortedList()...) { return false } - return allMachines.Len() == input.expectedReplicas - }, input.waitForMachinesIntervals...).Should(BeTrue(), "Failed to get the expected list of machines: got %s (expected %d machines, must have %s, must not have %s)", allMachines.List(), input.expectedReplicas, expectedOldMachines.List(), expectedDeletedMachines.List()) - log.Logf("Got %d machines: %s", input.expectedReplicas, allMachines.List()) + return allMachines.Len() == input.ExpectedReplicas + }, input.WaitForMachinesIntervals...).Should(BeTrue(), "Failed to get the expected list of machines: got %s (expected %d machines, must have %s, must not have %s)", allMachines.UnsortedList(), input.ExpectedReplicas, expectedOldMachines.UnsortedList(), expectedDeletedMachines.UnsortedList()) + log.Logf("Got %d machines: %s", allMachines.Len(), allMachines.UnsortedList()) // Ensures the desired set of machines is stable (no further machines are created or deleted). log.Logf("Checking the list of machines is stable") - allMachinesNow := sets.NewString() + allMachinesNow := sets.Set[string]{} Consistently(func() bool { // Gets the list of machines - if err := input.lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { + if err := input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { return false } - allMachinesNow = sets.NewString() + allMachinesNow = sets.Set[string]{} for i := range machineList.Items { allMachinesNow.Insert(machineList.Items[i].Name) } - return allMachines.Len() == allMachinesNow.Len() && allMachines.HasAll(allMachinesNow.List()...) - }, input.checkMachineListStableIntervals...).Should(BeTrue(), "Expected list of machines is not stable: got %s, expected %s", allMachinesNow.List(), allMachines.List()) + return allMachines.Equal(allMachinesNow) + }, input.CheckMachineListStableIntervals...).Should(BeTrue(), "Expected list of machines is not stable: got %s, expected %s", allMachinesNow.UnsortedList(), allMachines.UnsortedList()) - return allMachines.List(), newMachines.List() + return allMachines.UnsortedList(), newMachines.UnsortedList() } // getServerAddr returns the address to be used for accessing the management cluster from a workload cluster. diff --git a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go index 73e1906fb865..7dd5737d1197 100644 --- a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go +++ b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go @@ -295,8 +295,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * // if the machine isn't bootstrapped, only then run bootstrap scripts if !dockerMachine.Spec.Bootstrapped { - // TODO: make this timeout configurable via DockerMachine API if it is required by the KCP remediation test. - timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() // Check for bootstrap success @@ -317,10 +316,13 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * for { select { case <-timeoutCtx.Done(): + log.Info("Cancelling Bootstrap due to timeout") return default: updatedDockerMachine := &infrav1.DockerMachine{} - if err := r.Client.Get(ctx, client.ObjectKeyFromObject(dockerMachine), updatedDockerMachine); err == nil && !updatedDockerMachine.DeletionTimestamp.IsZero() { + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(dockerMachine), updatedDockerMachine); err == nil && + !updatedDockerMachine.DeletionTimestamp.IsZero() { + log.Info("Cancelling Bootstrap because the underlying machine has been deleted") cancel() return } From 5c888faf436fc848373885a51b066161d01e7f1a Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 2 Feb 2023 14:00:58 +0100 Subject: [PATCH 5/8] use metav1.Time instead of metav1.TimeStamp --- .../v1beta1/kubeadm_control_plane_types.go | 4 +- .../api/v1beta1/zz_generated.deepcopy.go | 4 +- ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 67 +++++++++---------- .../internal/controllers/remediation.go | 10 +-- .../internal/controllers/remediation_test.go | 8 +-- 5 files changed, 43 insertions(+), 50 deletions(-) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go index d4eef2b1113c..94e3946387f8 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go @@ -303,8 +303,8 @@ type LastRemediationStatus struct { // Machine is the machine name of the latest machine being remediated. Machine string `json:"machine"` - // Timestamp is RFC 3339 date and time at which last remediation happened. - Timestamp metav1.Timestamp `json:"timestamp"` + // Timestamp is when last remediation happened. It is represented in RFC3339 form and is in UTC. + Timestamp metav1.Time `json:"timestamp"` // RetryCount used to keep track of remediation retry for the last remediated machine. // A retry happens when a machine that was created as a replacement for an unhealthy machine also fails. diff --git a/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go index 010dd455c1ea..88d4bca9de22 100644 --- a/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go +++ b/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go @@ -183,7 +183,7 @@ func (in *KubeadmControlPlaneStatus) DeepCopyInto(out *KubeadmControlPlaneStatus if in.LastRemediation != nil { in, out := &in.LastRemediation, &out.LastRemediation *out = new(LastRemediationStatus) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -355,7 +355,7 @@ func (in *KubeadmControlPlaneTemplateSpec) DeepCopy() *KubeadmControlPlaneTempla // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LastRemediationStatus) DeepCopyInto(out *LastRemediationStatus) { *out = *in - out.Timestamp = in.Timestamp + in.Timestamp.DeepCopyInto(&out.Timestamp) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LastRemediationStatus. diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index c36b809eeb56..6eb1490ceb71 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -3622,33 +3622,43 @@ spec: type: object remediationStrategy: description: The RemediationStrategy that controls how control plane - machines remediation happens. + machine remediation happens. properties: maxRetry: - description: "MaxRetry is the Max number of retry while attempting + description: "MaxRetry is the Max number of retries while attempting to remediate an unhealthy machine. A retry happens when a machine that was created as a replacement for an unhealthy machine also fails. For example, given a control plane with three machines M1, M2, M3: \n M1 become unhealthy; remediation happens, and - M1bis is created as a replacement. If M1-1 (replacement of M1) - have problems while bootstrapping it will become unhealthy, - and then be remediated; such operation is considered a retry, - remediation-retry #1. If M1-2 (replacement of M1-2) becomes - unhealthy, remediation-retry #2 will happen, etc. \n A retry - could happen only after RetryPeriod from the previous retry. - If a machine is marked as unhealthy after MinHealthyPeriod from - the previous remediation expired, this is not considered anymore - a retry because the new issue is assumed unrelated from the - previous one. \n If not set, infinite retry will be attempted." + M1-1 is created as a replacement. If M1-1 (replacement of M1) + has problems while bootstrapping it will become unhealthy, and + then be remediated; such operation is considered a retry, remediation-retry + #1. If M1-2 (replacement of M1-2) becomes unhealthy, remediation-retry + #2 will happen, etc. \n A retry could happen only after RetryPeriod + from the previous retry. If a machine is marked as unhealthy + after MinHealthyPeriod from the previous remediation expired, + this is not considered a retry anymore because the new issue + is assumed unrelated from the previous one. \n If not set, the + remedation will be retried infinitely." format: int32 type: integer - minHealthySeconds: + minHealthyPeriod: description: "MinHealthyPeriod defines the duration after which KCP will consider any failure to a machine unrelated from the - previous one, and thus a new remediation is not considered a - retry anymore. \n If not set, this value is defaulted to 1h." + previous one. In this case the remediation is not considered + a retry anymore, and thus the retry counter restarts from 0. + For example, assuming MinHealthyPeriod is set to 1h (default) + \n M1 become unhealthy; remediation happens, and M1-1 is created + as a replacement. If M1-1 (replacement of M1) has problems within + the 1hr after the creation, also this machine will be remediated + and this operation is considered a retry - a problem related + to the original issue happened to M1 -. \n If instead the problem + on M1-1 is happening after MinHealthyPeriod expired, e.g. four + days after m1-1 has been created as a remediation of M1, the + problem on M1-1 is considered unrelated to the original issue + happened to M1. \n If not set, this value is defaulted to 1h." type: string - retryDelaySeconds: + retryPeriod: description: "RetryPeriod is the duration that KCP should wait before remediating a machine being created as a replacement for an unhealthy machine (a retry). \n If not set, a retry will @@ -3796,27 +3806,10 @@ spec: format: int32 type: integer timestamp: - description: Timestamp is RFC 3339 date and time at which last - remediation happened. - properties: - nanos: - description: Non-negative fractions of a second at nanosecond - resolution. Negative second values with fractions must still - have non-negative nanos values that count forward in time. - Must be from 0 to 999,999,999 inclusive. This field may - be limited in precision depending on context. - format: int32 - type: integer - seconds: - description: Represents seconds of UTC time since Unix epoch - 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z - to 9999-12-31T23:59:59Z inclusive. - format: int64 - type: integer - required: - - nanos - - seconds - type: object + description: Timestamp is when last remediation happened. It is + represented in RFC3339 form and is in UTC. + format: date-time + type: string required: - machine - retryCount diff --git a/controlplane/kubeadm/internal/controllers/remediation.go b/controlplane/kubeadm/internal/controllers/remediation.go index 71843e9799ad..3e9ec20bc782 100644 --- a/controlplane/kubeadm/internal/controllers/remediation.go +++ b/controlplane/kubeadm/internal/controllers/remediation.go @@ -43,7 +43,7 @@ import ( // based on the process described in https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20191017-kubeadm-based-control-plane.md#remediation-using-delete-and-recreate func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.Context, controlPlane *internal.ControlPlane) (ret ctrl.Result, retErr error) { log := ctrl.LoggerFrom(ctx) - reconciliationTime := time.Now() + reconciliationTime := time.Now().UTC() // Cleanup pending remediation actions not completed for any reasons (e.g. number of current replicas is less or equal to 1) // if the underlying machine is now back to healthy / not deleting. @@ -223,7 +223,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // NOTE: Some of those info have been computed above, but they must surface on the object only here, after machine has been deleted. controlPlane.KCP.Status.LastRemediation = &controlplanev1.LastRemediationStatus{ Machine: machineToBeRemediated.Name, - Timestamp: metav1.Timestamp{Seconds: reconciliationTime.Unix()}, + Timestamp: metav1.Time{Time: reconciliationTime}, RetryCount: retryCount, } @@ -264,9 +264,9 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin return x } - lastRemediationTimestamp := reconciliationTime.Add(-2 * max(minHealthyPeriod, retryPeriod)).UTC() + lastRemediationTimestamp := reconciliationTime.Add(-2 * max(minHealthyPeriod, retryPeriod)) if controlPlane.KCP.Status.LastRemediation != nil { - lastRemediationTimestamp = time.Unix(controlPlane.KCP.Status.LastRemediation.Timestamp.Seconds, int64(controlPlane.KCP.Status.LastRemediation.Timestamp.Nanos)) + lastRemediationTimestamp = controlPlane.KCP.Status.LastRemediation.Timestamp.Time } // Check if the machine being remediated has been created as a remediation for a previous unhealthy machine. @@ -290,7 +290,7 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin } // Check if remediation can happen because retryPeriod is passed. - if lastRemediationTimestamp.Add(retryPeriod).After(reconciliationTime.UTC()) { + if lastRemediationTimestamp.Add(retryPeriod).After(reconciliationTime) { log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed in the latest %s. Skipping remediation", retryPeriod)) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryDelay)", retryPeriod) return machineRemediatingFor, false, 0 diff --git a/controlplane/kubeadm/internal/controllers/remediation_test.go b/controlplane/kubeadm/internal/controllers/remediation_test.go index 9e82066ca9e4..24c046c479c3 100644 --- a/controlplane/kubeadm/internal/controllers/remediation_test.go +++ b/controlplane/kubeadm/internal/controllers/remediation_test.go @@ -161,7 +161,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Initialized: true, LastRemediation: &controlplanev1.LastRemediationStatus{ Machine: "m0", - Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).Unix()}, // minHealthy not expired yet. + Timestamp: metav1.Time{Time: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).UTC()}, // minHealthy not expired yet. RetryCount: 3, }, }, @@ -218,7 +218,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Initialized: true, LastRemediation: &controlplanev1.LastRemediationStatus{ Machine: "m0", - Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-2 * controlplanev1.DefaultMinHealthyPeriod).Unix()}, // minHealthyPeriod already expired. + Timestamp: metav1.Time{Time: time.Now().Add(-2 * controlplanev1.DefaultMinHealthyPeriod).UTC()}, // minHealthyPeriod already expired. RetryCount: 3, }, }, @@ -278,7 +278,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Initialized: true, LastRemediation: &controlplanev1.LastRemediationStatus{ Machine: "m0", - Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-2 * minHealthyPeriod).Unix()}, // minHealthyPeriod already expired. + Timestamp: metav1.Time{Time: time.Now().Add(-2 * minHealthyPeriod).UTC()}, // minHealthyPeriod already expired. RetryCount: 3, }, }, @@ -336,7 +336,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Initialized: true, LastRemediation: &controlplanev1.LastRemediationStatus{ Machine: "m0", - Timestamp: metav1.Timestamp{Seconds: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).Unix()}, // minHealthyPeriod not yet expired. + Timestamp: metav1.Time{Time: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).UTC()}, // minHealthyPeriod not yet expired. RetryCount: 2, }, }, From 17e26764672212fef6d7d1317da4feebe25dbfd2 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Fri, 3 Feb 2023 11:41:52 +0100 Subject: [PATCH 6/8] store last remediation data at machine level and use this as a source of truth --- .../v1beta1/kubeadm_control_plane_types.go | 4 +- .../kubeadm/internal/controllers/helpers.go | 6 +- .../internal/controllers/helpers_test.go | 5 + .../internal/controllers/remediation.go | 148 ++++++++++---- .../internal/controllers/remediation_test.go | 191 +++++++++--------- .../kubeadm/internal/controllers/status.go | 28 +++ .../src/reference/labels_and_annotations.md | 2 + 7 files changed, 247 insertions(+), 137 deletions(-) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go index 94e3946387f8..272ffd12a4a1 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go @@ -56,14 +56,14 @@ const ( // specifically it tracks that the system is in between having deleted an unhealthy machine and recreating its replacement. // NOTE: if something external to CAPI removes this annotation the system cannot detect the above situation; this can lead to // failures in updating remediation retry or remediation count (both counters restart from zero). - RemediationInProgressAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-in-progress" + RemediationInProgressAnnotation = "controlplane.cluster.x-k8s.io/remediation-in-progress" // RemediationForAnnotation is used to link a new machine to the unhealthy machine it is replacing; // please note that in case of retry, when also the remediating machine fails, the system keeps track of // the first machine of the sequence only. // NOTE: if something external to CAPI removes this annotation the system this can lead to // failures in updating remediation retry (the counter restarts from zero). - RemediationForAnnotation = "kubeadm.controlplane.cluster.x-k8s.io/remediation-for" + RemediationForAnnotation = "controlplane.cluster.x-k8s.io/remediation-for" // DefaultMinHealthyPeriod defines the default minimum period before we consider a remediation on a // machine unrelated from the previous remediation. diff --git a/controlplane/kubeadm/internal/controllers/helpers.go b/controlplane/kubeadm/internal/controllers/helpers.go index 3093408cd607..f6ef0dc13b48 100644 --- a/controlplane/kubeadm/internal/controllers/helpers.go +++ b/controlplane/kubeadm/internal/controllers/helpers.go @@ -316,10 +316,10 @@ func (r *KubeadmControlPlaneReconciler) generateMachine(ctx context.Context, kcp } // In case this machine is being created as a consequence of a remediation, then add an annotation - // tracking the name of the machine we are remediating for. + // tracking remediating data. // NOTE: This is required in order to track remediation retries. - if v, ok := kcp.Annotations[controlplanev1.RemediationInProgressAnnotation]; ok && v == "true" { - machine.Annotations[controlplanev1.RemediationForAnnotation] = kcp.Status.LastRemediation.Machine + if remediationData, ok := kcp.Annotations[controlplanev1.RemediationInProgressAnnotation]; ok { + machine.Annotations[controlplanev1.RemediationForAnnotation] = remediationData } // Machine's bootstrap config may be missing ClusterConfiguration if it is not the first machine in the control plane. diff --git a/controlplane/kubeadm/internal/controllers/helpers_test.go b/controlplane/kubeadm/internal/controllers/helpers_test.go index 1886271ece8c..72ec13fba0d6 100644 --- a/controlplane/kubeadm/internal/controllers/helpers_test.go +++ b/controlplane/kubeadm/internal/controllers/helpers_test.go @@ -511,6 +511,9 @@ func TestKubeadmControlPlaneReconciler_generateMachine(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "testControlPlane", Namespace: cluster.Namespace, + Annotations: map[string]string{ + controlplanev1.RemediationInProgressAnnotation: "foo", + }, }, Spec: controlplanev1.KubeadmControlPlaneSpec{ Version: "v1.16.6", @@ -555,6 +558,8 @@ func TestKubeadmControlPlaneReconciler_generateMachine(t *testing.T) { g.Expect(machine.Namespace).To(Equal(kcp.Namespace)) g.Expect(machine.OwnerReferences).To(HaveLen(1)) g.Expect(machine.OwnerReferences).To(ContainElement(*metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("KubeadmControlPlane")))) + g.Expect(machine.Annotations).To(HaveKeyWithValue(controlplanev1.RemediationForAnnotation, "foo")) + g.Expect(kcp.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) g.Expect(machine.Spec).To(Equal(expectedMachineSpec)) // Verify that the machineTemplate.ObjectMeta has been propagated to the Machine. diff --git a/controlplane/kubeadm/internal/controllers/remediation.go b/controlplane/kubeadm/internal/controllers/remediation.go index 3e9ec20bc782..c0a6b6f9479d 100644 --- a/controlplane/kubeadm/internal/controllers/remediation.go +++ b/controlplane/kubeadm/internal/controllers/remediation.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "encoding/json" "fmt" "time" @@ -122,11 +123,14 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // Check if KCP is allowed to remediate considering retry limits: // - Remediation cannot happen because retryPeriod is not yet expired. // - KCP already reached MaxRetries limit. - machineRemediatingFor, canRemediate, retryCount := r.checkRetryLimits(log, machineToBeRemediated, controlPlane, reconciliationTime) + remediationInProgressData, canRemediate, err := r.checkRetryLimits(log, machineToBeRemediated, controlPlane, reconciliationTime) + if err != nil { + return ctrl.Result{}, err + } if !canRemediate { + // NOTE: log lines and conditions surfacing why it is not possible to remediate are set by checkRetryLimits. return ctrl.Result{}, nil } - log = log.WithValues("machineRemediatingFor", klog.KRef(controlPlane.KCP.Namespace, machineRemediatingFor), "retryCount", retryCount) // Executes checks that applies only if the control plane is already initialized; in this case KCP can // remediate only if it can safely assume that the operation preserves the operation state of the existing cluster (or at least it doesn't make it worst). @@ -161,7 +165,14 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C } } - // Remediate the unhealthy control plane machine by deleting it. + // Prepare the info for tracking the remediation progress into the RemediationInProgressAnnotation. + remediationInProgressValue, err := remediationInProgressData.Marshal() + if err != nil { + return ctrl.Result{}, err + } + + // Start remediating the unhealthy control plane machine by deleting it. + // A new machine will come up completing the operation as part of the regular // If the control plane is initialized, before deleting the machine: // - if the machine hosts the etcd leader, forward etcd leadership to another machine. @@ -206,27 +217,22 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C } } + // Delete the machine if err := r.Client.Delete(ctx, machineToBeRemediated); err != nil { conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityError, err.Error()) return ctrl.Result{}, errors.Wrapf(err, "failed to delete unhealthy machine %s", machineToBeRemediated.Name) } + // Surface the operation is in progress. log.Info("Remediating unhealthy machine") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") - // Set annotations tracking remediation is in progress (remediation will complete when a replacement machine is created). + // Set annotations tracking remediation details so they can be picked up by the machine + // that will be created as part of the scale up action that completes the remediation. annotations.AddAnnotations(controlPlane.KCP, map[string]string{ - controlplanev1.RemediationInProgressAnnotation: "", + controlplanev1.RemediationInProgressAnnotation: remediationInProgressValue, }) - // Stores info about last remediation. - // NOTE: Some of those info have been computed above, but they must surface on the object only here, after machine has been deleted. - controlPlane.KCP.Status.LastRemediation = &controlplanev1.LastRemediationStatus{ - Machine: machineToBeRemediated.Name, - Timestamp: metav1.Time{Time: reconciliationTime}, - RetryCount: retryCount, - } - return ctrl.Result{Requeue: true}, nil } @@ -235,10 +241,26 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // - KCP already reached the maximum number of retries for a machine. // NOTE: Counting the number of retries is required In order to prevent infinite remediation e.g. in case the // first Control Plane machine is failing due to quota issue. -func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machineToBeRemediated *clusterv1.Machine, controlPlane *internal.ControlPlane, reconciliationTime time.Time) (machineRemediatingFor string, canRemediate bool, retryCount int32) { +func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machineToBeRemediated *clusterv1.Machine, controlPlane *internal.ControlPlane, reconciliationTime time.Time) (*RemediationData, bool, error) { + // Get last remediation info from the machine. + var lastRemediationData *RemediationData + if value, ok := machineToBeRemediated.Annotations[controlplanev1.RemediationForAnnotation]; ok { + l, err := RemediationDataFromAnnotation(value) + if err != nil { + return nil, false, err + } + lastRemediationData = l + } + + remediationInProgressData := &RemediationData{ + Machine: machineToBeRemediated.Name, + Timestamp: metav1.Time{Time: reconciliationTime}, + RetryCount: 0, + } + // If there is no last remediation, this is the first try of a new retry sequence. - if controlPlane.KCP.Status.LastRemediation == nil { - return "", true, 0 + if lastRemediationData == nil { + return remediationInProgressData, true, nil } // Gets MinHealthySeconds and RetryDelaySeconds from the remediation strategy, or use defaults. @@ -255,8 +277,8 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin // Gets the timestamp of the last remediation; if missing, default to a value // that ensures both MinHealthySeconds and RetryDelaySeconds are expired. // NOTE: this could potentially lead to executing more retries than expected or to executing retries before than - // expected, but this is considered acceptable. - // when the system recovers from someone/something manually removing the LastRemediatingTimeStampAnnotation. + // expected, but this is considered acceptable when the system recovers from someone/something changes or deletes + // the RemediationForAnnotation on Machines. max := func(x, y time.Duration) time.Duration { if x < y { return y @@ -264,52 +286,53 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin return x } - lastRemediationTimestamp := reconciliationTime.Add(-2 * max(minHealthyPeriod, retryPeriod)) - if controlPlane.KCP.Status.LastRemediation != nil { - lastRemediationTimestamp = controlPlane.KCP.Status.LastRemediation.Timestamp.Time + lastRemediationTime := reconciliationTime.Add(-2 * max(minHealthyPeriod, retryPeriod)) + if !lastRemediationData.Timestamp.IsZero() { + lastRemediationTime = lastRemediationData.Timestamp.Time } // Check if the machine being remediated has been created as a remediation for a previous unhealthy machine. - // NOTE: if someone/something manually removing the RemediationForAnnotation on Machines or one of the LastRemediatedMachineAnnotation - // and LastRemediatedMachineRetryAnnotation on KCP, this could potentially lead to executing more retries than expected, - // but this is considered acceptable in such a case. - machineRemediatingFor = machineToBeRemediated.Name - if remediationFor, ok := machineToBeRemediated.Annotations[controlplanev1.RemediationForAnnotation]; ok { + // NOTE: if someone/something changes or deletes the RemediationForAnnotation on Machines, this could potentially + // lead to executing more retries than expected, but this is considered acceptable in such a case. + machineRemediationFor := remediationInProgressData.Machine + if lastRemediationData.Machine != "" { // If the remediation is happening before minHealthyPeriod is expired, then KCP considers this // as a remediation for the same previously unhealthy machine. - // TODO: add example - if lastRemediationTimestamp.Add(minHealthyPeriod).After(reconciliationTime) { - machineRemediatingFor = remediationFor - log = log.WithValues("RemediationRetryFor", klog.KRef(machineToBeRemediated.Namespace, machineRemediatingFor)) + if lastRemediationTime.Add(minHealthyPeriod).After(reconciliationTime) { + machineRemediationFor = lastRemediationData.Machine + log = log.WithValues("RemediationRetryFor", klog.KRef(machineToBeRemediated.Namespace, machineRemediationFor)) } } // If remediation is happening for a different machine, this is the first try of a new retry sequence. - if controlPlane.KCP.Status.LastRemediation.Machine != machineRemediatingFor { - return machineRemediatingFor, true, 0 + if lastRemediationData.Machine != machineRemediationFor { + return remediationInProgressData, true, nil } + // If the remediation is for the same machine, carry over the retry count. + remediationInProgressData.RetryCount = lastRemediationData.RetryCount + // Check if remediation can happen because retryPeriod is passed. - if lastRemediationTimestamp.Add(retryPeriod).After(reconciliationTime) { + if lastRemediationTime.Add(retryPeriod).After(reconciliationTime) { log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed in the latest %s. Skipping remediation", retryPeriod)) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryDelay)", retryPeriod) - return machineRemediatingFor, false, 0 + return remediationInProgressData, false, nil } // Check if remediation can happen because of maxRetry is not reached yet, if defined. - retry := controlPlane.KCP.Status.LastRemediation.RetryCount - if controlPlane.KCP.Spec.RemediationStrategy != nil && controlPlane.KCP.Spec.RemediationStrategy.MaxRetry != nil { - maxRetry := *controlPlane.KCP.Spec.RemediationStrategy.MaxRetry - if retry >= maxRetry { - log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed %d times (MaxRetry %d). Skipping remediation", retry, maxRetry)) + maxRetry := int(*controlPlane.KCP.Spec.RemediationStrategy.MaxRetry) + if remediationInProgressData.RetryCount >= maxRetry { + log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed %d times (MaxRetry %d). Skipping remediation", remediationInProgressData.RetryCount, maxRetry)) conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed %d times (MaxRetry)", maxRetry) - return machineRemediatingFor, false, 0 + return remediationInProgressData, false, nil } } - retryCount = retry + 1 - return machineRemediatingFor, true, retryCount + // All the check passed, increase the remediation number. + remediationInProgressData.RetryCount++ + + return remediationInProgressData, true, nil } // canSafelyRemoveEtcdMember assess if it is possible to remove the member hosted on the machine to be remediated @@ -411,3 +434,44 @@ func (r *KubeadmControlPlaneReconciler) canSafelyRemoveEtcdMember(ctx context.Co return canSafelyRemediate, nil } + +// RemediationData struct is used to keep track of information stored in the RemediationInProgressAnnotation in KCP +// during remediation and then into the RemediationForAnnotation on the replacement machine once it is created. +type RemediationData struct { + // Machine is the machine name of the latest machine being remediated. + Machine string `json:"machine"` + + // Timestamp is when last remediation happened. It is represented in RFC3339 form and is in UTC. + Timestamp metav1.Time `json:"timestamp"` + + // RetryCount used to keep track of remediation retry for the last remediated machine. + // A retry happens when a machine that was created as a replacement for an unhealthy machine also fails. + RetryCount int `json:"retryCount"` +} + +// RemediationDataFromAnnotation gets RemediationData from an annotation value. +func RemediationDataFromAnnotation(value string) (*RemediationData, error) { + ret := &RemediationData{} + if err := json.Unmarshal([]byte(value), ret); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal value %s for %s annotation", value, clusterv1.RemediationInProgressReason) + } + return ret, nil +} + +// Marshal an RemediationData into an annotation value. +func (r *RemediationData) Marshal() (string, error) { + b, err := json.Marshal(r) + if err != nil { + return "", errors.Wrapf(err, "failed to marshal value for %s annotation", clusterv1.RemediationInProgressReason) + } + return string(b), nil +} + +// ToStatus converts a RemediationData into a LastRemediationStatus struct. +func (r *RemediationData) ToStatus() *controlplanev1.LastRemediationStatus { + return &controlplanev1.LastRemediationStatus{ + Machine: r.Machine, + Timestamp: r.Timestamp, + RetryCount: int32(r.RetryCount), + } +} diff --git a/controlplane/kubeadm/internal/controllers/remediation_test.go b/controlplane/kubeadm/internal/controllers/remediation_test.go index 24c046c479c3..3766cbacbf56 100644 --- a/controlplane/kubeadm/internal/controllers/remediation_test.go +++ b/controlplane/kubeadm/internal/controllers/remediation_test.go @@ -113,7 +113,11 @@ func TestReconcileUnhealthyMachines(t *testing.T) { KCP: &controlplanev1.KubeadmControlPlane{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - controlplanev1.RemediationInProgressAnnotation: "true", + controlplanev1.RemediationInProgressAnnotation: MustMarshalRemediationData(&RemediationData{ + Machine: "foo", + Timestamp: metav1.Time{Time: time.Now().UTC()}, + RetryCount: 0, + }), }, }, }, @@ -144,7 +148,11 @@ func TestReconcileUnhealthyMachines(t *testing.T) { t.Run("Remediation does not happen if MaxRetry is reached", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(&RemediationData{ + Machine: "m0", + Timestamp: metav1.Time{Time: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).UTC()}, // minHealthy not expired yet. + RetryCount: 3, + }))) m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) @@ -157,14 +165,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { MaxRetry: utilpointer.Int32(3), }, }, - Status: controlplanev1.KubeadmControlPlaneStatus{ - Initialized: true, - LastRemediation: &controlplanev1.LastRemediationStatus{ - Machine: "m0", - Timestamp: metav1.Time{Time: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).UTC()}, // minHealthy not expired yet. - RetryCount: 3, - }, - }, }, Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m1, m2, m3), @@ -186,8 +186,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(3))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal("m0")) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed 3 times (MaxRetry)") @@ -201,7 +199,11 @@ func TestReconcileUnhealthyMachines(t *testing.T) { t.Run("Retry history is ignored if min healthy period is expired, default min healthy period", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(&RemediationData{ + Machine: "m0", + Timestamp: metav1.Time{Time: time.Now().Add(-2 * controlplanev1.DefaultMinHealthyPeriod).UTC()}, // minHealthyPeriod already expired. + RetryCount: 3, + }))) m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) @@ -214,14 +216,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { MaxRetry: utilpointer.Int32(3), }, }, - Status: controlplanev1.KubeadmControlPlaneStatus{ - Initialized: true, - LastRemediation: &controlplanev1.LastRemediationStatus{ - Machine: "m0", - Timestamp: metav1.Time{Time: time.Now().Add(-2 * controlplanev1.DefaultMinHealthyPeriod).UTC()}, // minHealthyPeriod already expired. - RetryCount: 3, - }, - }, }, Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m1, m2, m3), @@ -243,8 +237,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -258,12 +254,16 @@ func TestReconcileUnhealthyMachines(t *testing.T) { t.Run("Retry history is ignored if min healthy period is expired", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + minHealthyPeriod := 4 * controlplanev1.DefaultMinHealthyPeriod // big min healthy period, so we are user that we are not using DefaultMinHealthyPeriod. + + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(&RemediationData{ + Machine: "m0", + Timestamp: metav1.Time{Time: time.Now().Add(-2 * minHealthyPeriod).UTC()}, // minHealthyPeriod already expired. + RetryCount: 3, + }))) m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) - minHealthyPeriod := 4 * controlplanev1.DefaultMinHealthyPeriod // big min healthy period, so we are user that we are not using DefaultMinHealthyPeriod. - controlPlane := &internal.ControlPlane{ KCP: &controlplanev1.KubeadmControlPlane{ Spec: controlplanev1.KubeadmControlPlaneSpec{ @@ -274,14 +274,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { MinHealthyPeriod: &metav1.Duration{Duration: minHealthyPeriod}, }, }, - Status: controlplanev1.KubeadmControlPlaneStatus{ - Initialized: true, - LastRemediation: &controlplanev1.LastRemediationStatus{ - Machine: "m0", - Timestamp: metav1.Time{Time: time.Now().Add(-2 * minHealthyPeriod).UTC()}, // minHealthyPeriod already expired. - RetryCount: 3, - }, - }, }, Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m1, m2, m3), @@ -303,8 +295,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -318,7 +312,11 @@ func TestReconcileUnhealthyMachines(t *testing.T) { t.Run("Remediation does not happen if RetryDelay is not yet passed", func(t *testing.T) { g := NewWithT(t) - m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation("m0")) + m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(&RemediationData{ + Machine: "m0", + Timestamp: metav1.Time{Time: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).UTC()}, // minHealthyPeriod not yet expired. + RetryCount: 2, + }))) m2 := createMachine(ctx, g, ns.Name, "m2-healthy-", withHealthyEtcdMember()) m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember()) @@ -332,14 +330,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { RetryPeriod: metav1.Duration{Duration: controlplanev1.DefaultMinHealthyPeriod}, // RetryDelaySeconds not yet expired. }, }, - Status: controlplanev1.KubeadmControlPlaneStatus{ - Initialized: true, - LastRemediation: &controlplanev1.LastRemediationStatus{ - Machine: "m0", - Timestamp: metav1.Time{Time: time.Now().Add(-controlplanev1.DefaultMinHealthyPeriod / 2).UTC()}, // minHealthyPeriod not yet expired. - RetryCount: 2, - }, - }, }, Cluster: &clusterv1.Cluster{}, Machines: collections.FromMachines(m1, m2, m3), @@ -361,8 +351,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(2))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal("m0")) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest 1h0m0s (RetryDelay)") @@ -408,7 +396,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate if current replicas are less or equal to 1") @@ -438,7 +425,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP waiting for control plane machine deletion to complete before triggering remediation") @@ -480,7 +466,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") @@ -524,7 +509,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because this could result in etcd loosing quorum") @@ -568,8 +552,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -616,8 +602,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -628,10 +616,9 @@ func TestReconcileUnhealthyMachines(t *testing.T) { removeFinalizer(g, m1) g.Expect(env.CleanupAndWait(ctx, m1)).To(Succeed()) - machineRemediatingFor := m1.Name for i := 2; i < 4; i++ { // Simulate the creation of a replacement for 0. - mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(machineRemediatingFor)) + mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(remediationData))) // Simulate KCP dropping RemediationInProgressAnnotation after creating the replacement machine. delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) @@ -650,8 +637,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(i - 1))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(mi.Name)) + remediationData, err = RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(mi.Name)) + g.Expect(remediationData.RetryCount).To(Equal(i - 1)) assertMachineCondition(ctx, g, mi, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -661,8 +650,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { removeFinalizer(g, mi) g.Expect(env.CleanupAndWait(ctx, mi)).To(Succeed()) - - machineRemediatingFor = mi.Name } }) @@ -704,8 +691,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -753,8 +742,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -803,8 +794,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -850,7 +843,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation).To(BeNil()) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationFailedReason, clusterv1.ConditionSeverityWarning, "A control plane machine needs remediation, but there is no healthy machine to forward etcd leadership to. Skipping remediation") @@ -896,8 +888,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -908,10 +902,9 @@ func TestReconcileUnhealthyMachines(t *testing.T) { removeFinalizer(g, m1) g.Expect(env.CleanupAndWait(ctx, m1)).To(Succeed()) - machineRemediatingFor := m1.Name for i := 5; i < 6; i++ { // Simulate the creation of a replacement for m1. - mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(machineRemediatingFor)) + mi := createMachine(ctx, g, ns.Name, fmt.Sprintf("m%d-unhealthy-", i), withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(remediationData))) // Simulate KCP dropping RemediationInProgressAnnotation after creating the replacement machine. delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) @@ -930,8 +923,10 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(i - 4))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(mi.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(mi.Name)) + g.Expect(remediationData.RetryCount).To(Equal(i - 4)) assertMachineCondition(ctx, g, mi, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -941,8 +936,6 @@ func TestReconcileUnhealthyMachines(t *testing.T) { removeFinalizer(g, mi) g.Expect(env.CleanupAndWait(ctx, mi)).To(Succeed()) - - machineRemediatingFor = mi.Name } g.Expect(env.CleanupAndWait(ctx, m2, m3)).To(Succeed()) @@ -1000,8 +993,10 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m1.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m1.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -1015,7 +1010,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // Fake scaling up, which creates a remediation machine, fast forwards to when also the replacement machine is marked unhealthy. // NOTE: scale up also resets remediation in progress and remediation counts. - m2 := createMachine(ctx, g, ns.Name, "m2-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(m1.Name)) + m2 := createMachine(ctx, g, ns.Name, "m2-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(remediationData))) delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) // Control plane not initialized yet, Second CP is unhealthy and gets remediated (retry 2) @@ -1033,8 +1028,10 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(1))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) + remediationData, err = RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m2.Name)) + g.Expect(remediationData.RetryCount).To(Equal(1)) assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -1048,7 +1045,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // Fake scaling up, which creates a remediation machine, which is healthy. // NOTE: scale up also resets remediation in progress and remediation counts. - m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(m1.Name)) + m3 := createMachine(ctx, g, ns.Name, "m3-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(MustMarshalRemediationData(remediationData))) delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) g.Expect(env.Cleanup(ctx, m3)).To(Succeed()) @@ -1105,8 +1102,10 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m2.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -1120,7 +1119,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // Fake scaling up, which creates a remediation machine, fast forwards to when also the replacement machine is marked unhealthy. // NOTE: scale up also resets remediation in progress and remediation counts. - m3 := createMachine(ctx, g, ns.Name, "m3-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(m2.Name)) + m3 := createMachine(ctx, g, ns.Name, "m3-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(remediationData))) delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) // Control plane not initialized yet, Second CP is unhealthy and gets remediated (retry 2) @@ -1138,8 +1137,10 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(1))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m3.Name)) + remediationData, err = RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m3.Name)) + g.Expect(remediationData.RetryCount).To(Equal(1)) assertMachineCondition(ctx, g, m3, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") @@ -1153,7 +1154,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { // Fake scaling up, which creates a remediation machine, which is healthy. // NOTE: scale up also resets remediation in progress and remediation counts. - m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(m1.Name)) + m4 := createMachine(ctx, g, ns.Name, "m4-healthy-", withHealthyEtcdMember(), withRemediateForAnnotation(MustMarshalRemediationData(remediationData))) delete(controlPlane.KCP.Annotations, controlplanev1.RemediationInProgressAnnotation) g.Expect(env.Cleanup(ctx, m1, m4)).To(Succeed()) @@ -1211,8 +1212,10 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(controlPlane.KCP.Annotations).To(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - g.Expect(controlPlane.KCP.Status.LastRemediation.RetryCount).To(Equal(int32(0))) - g.Expect(controlPlane.KCP.Status.LastRemediation.Machine).To(Equal(m2.Name)) + remediationData, err := RemediationDataFromAnnotation(controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(remediationData.Machine).To(Equal(m2.Name)) + g.Expect(remediationData.RetryCount).To(Equal(0)) assertMachineCondition(ctx, g, m2, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") assertMachineCondition(ctx, g, m3, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") @@ -1750,3 +1753,11 @@ func assertMachineCondition(ctx context.Context, g *WithT, m *clusterv1.Machine, return nil }, 10*time.Second).Should(Succeed()) } + +func MustMarshalRemediationData(r *RemediationData) string { + s, err := r.Marshal() + if err != nil { + panic("failed to marshal remediation data") + } + return s +} diff --git a/controlplane/kubeadm/internal/controllers/status.go b/controlplane/kubeadm/internal/controllers/status.go index c17d62c0cf7c..3d742bdac7f4 100644 --- a/controlplane/kubeadm/internal/controllers/status.go +++ b/controlplane/kubeadm/internal/controllers/status.go @@ -116,5 +116,33 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c kcp.Status.Ready = true } + // Surface lastRemediation data in status. + // LastRemediation is the remediation currently in progress, in any, or the + // most recent of the remediation we are keeping track on machines. + var lastRemediation *RemediationData + + if v, ok := kcp.Annotations[controlplanev1.RemediationInProgressAnnotation]; ok { + remediationData, err := RemediationDataFromAnnotation(v) + if err != nil { + return err + } + lastRemediation = remediationData + } else { + for _, m := range ownedMachines.UnsortedList() { + if v, ok := m.Annotations[controlplanev1.RemediationForAnnotation]; ok { + remediationData, err := RemediationDataFromAnnotation(v) + if err != nil { + return err + } + if lastRemediation == nil || lastRemediation.Timestamp.Time.Before(remediationData.Timestamp.Time) { + lastRemediation = remediationData + } + } + } + } + + if lastRemediation != nil { + kcp.Status.LastRemediation = lastRemediation.ToStatus() + } return nil } diff --git a/docs/book/src/reference/labels_and_annotations.md b/docs/book/src/reference/labels_and_annotations.md index e426f2d8821a..be03817863d3 100644 --- a/docs/book/src/reference/labels_and_annotations.md +++ b/docs/book/src/reference/labels_and_annotations.md @@ -49,3 +49,5 @@ | controlplane.cluster.x-k8s.io/skip-coredns | It explicitly skips reconciling CoreDNS if set. | | controlplane.cluster.x-k8s.io/skip-kube-proxy | It explicitly skips reconciling kube-proxy if set. | | controlplane.cluster.x-k8s.io/kubeadm-cluster-configuration | It is a machine annotation that stores the json-marshalled string of KCP ClusterConfiguration. This annotation is used to detect any changes in ClusterConfiguration and trigger machine rollout in KCP. | +| controlplane.cluster.x-k8s.io/remediation-in-progress | It is a KCP remediation is that tracks that the system is in between having deleted an unhealthy machine and recreating its replacement. | +| controlplane.cluster.x-k8s.io/remediation-for | It is a machine annotation that links a new machine to the unhealthy machine it is replacing. | From 980dde55c5118675872e31150ce718d4166a46a0 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Fri, 3 Feb 2023 15:10:27 +0100 Subject: [PATCH 7/8] fix mhc flaky test --- .../machinehealthcheck_targets.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go b/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go index bdd9c5861f2a..5d174fb63155 100644 --- a/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go +++ b/internal/controllers/machinehealthcheck/machinehealthcheck_targets.go @@ -135,18 +135,18 @@ func (t *healthCheckTarget) needsRemediation(logger logr.Logger, timeoutForMachi return false, 0 } - controlPlaneInitializedTime := conditions.GetLastTransitionTime(t.Cluster, clusterv1.ControlPlaneInitializedCondition).Time - clusterInfraReadyTime := conditions.GetLastTransitionTime(t.Cluster, clusterv1.InfrastructureReadyCondition).Time + controlPlaneInitialized := conditions.GetLastTransitionTime(t.Cluster, clusterv1.ControlPlaneInitializedCondition) + clusterInfraReady := conditions.GetLastTransitionTime(t.Cluster, clusterv1.InfrastructureReadyCondition) machineCreationTime := t.Machine.CreationTimestamp.Time // Use the latest of the 3 times comparisonTime := machineCreationTime - logger.V(3).Info("Determining comparison time", "machineCreationTime", machineCreationTime, "clusterInfraReadyTime", clusterInfraReadyTime, "controlPlaneInitializedTime", controlPlaneInitializedTime) - if controlPlaneInitializedTime.After(comparisonTime) { - comparisonTime = controlPlaneInitializedTime + logger.V(3).Info("Determining comparison time", "machineCreationTime", machineCreationTime, "clusterInfraReadyTime", clusterInfraReady, "controlPlaneInitializedTime", controlPlaneInitialized) + if conditions.IsTrue(t.Cluster, clusterv1.ControlPlaneInitializedCondition) && controlPlaneInitialized != nil && controlPlaneInitialized.Time.After(comparisonTime) { + comparisonTime = controlPlaneInitialized.Time } - if clusterInfraReadyTime.After(comparisonTime) { - comparisonTime = clusterInfraReadyTime + if conditions.IsTrue(t.Cluster, clusterv1.InfrastructureReadyCondition) && clusterInfraReady != nil && clusterInfraReady.Time.After(comparisonTime) { + comparisonTime = clusterInfraReady.Time } logger.V(3).Info("Using comparison time", "time", comparisonTime) From 1528532d8f6f8c61194eeed7a6497a132a04a9a6 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Tue, 7 Feb 2023 17:14:24 +0100 Subject: [PATCH 8/8] Fixups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com --- .../kubeadm/api/v1alpha4/conversion.go | 4 + .../v1beta1/kubeadm_control_plane_webhook.go | 2 + .../kubeadm_control_plane_webhook_test.go | 5 + .../kubeadmcontrolplanetemplate_types.go | 4 + .../api/v1beta1/zz_generated.deepcopy.go | 5 + ...x-k8s.io_kubeadmcontrolplanetemplates.yaml | 49 +++++++++ .../kubeadm/internal/controllers/helpers.go | 2 +- .../internal/controllers/remediation.go | 96 ++++++++-------- .../internal/controllers/remediation_test.go | 14 ++- .../src/reference/labels_and_annotations.md | 2 +- test/e2e/kcp_remediations.go | 104 ++++++++---------- .../controllers/dockermachine_controller.go | 8 -- 12 files changed, 175 insertions(+), 120 deletions(-) diff --git a/controlplane/kubeadm/api/v1alpha4/conversion.go b/controlplane/kubeadm/api/v1alpha4/conversion.go index 9da266d9213d..1637b272a82c 100644 --- a/controlplane/kubeadm/api/v1alpha4/conversion.go +++ b/controlplane/kubeadm/api/v1alpha4/conversion.go @@ -180,6 +180,10 @@ func (src *KubeadmControlPlaneTemplate) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.ImagePullPolicy = restored.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.ImagePullPolicy } + if restored.Spec.Template.Spec.RemediationStrategy != nil { + dst.Spec.Template.Spec.RemediationStrategy = restored.Spec.Template.Spec.RemediationStrategy + } + return nil } diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go index dd512fe26967..52a6276ba132 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go @@ -171,6 +171,8 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error { {spec, "machineTemplate", "nodeDeletionTimeout"}, {spec, "replicas"}, {spec, "version"}, + {spec, "remediationStrategy"}, + {spec, "remediationStrategy", "*"}, {spec, "rolloutAfter"}, {spec, "rolloutBefore", "*"}, {spec, "rolloutStrategy", "*"}, diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go index 3ead11553c28..360e345ec187 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go @@ -407,6 +407,11 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { validUpdate.Spec.RolloutBefore = &RolloutBefore{ CertificatesExpiryDays: pointer.Int32(14), } + validUpdate.Spec.RemediationStrategy = &RemediationStrategy{ + MaxRetry: pointer.Int32(50), + MinHealthyPeriod: &metav1.Duration{Duration: 10 * time.Hour}, + RetryPeriod: metav1.Duration{Duration: 10 * time.Minute}, + } validUpdate.Spec.KubeadmConfigSpec.Format = bootstrapv1.CloudConfig scaleToZero := before.DeepCopy() diff --git a/controlplane/kubeadm/api/v1beta1/kubeadmcontrolplanetemplate_types.go b/controlplane/kubeadm/api/v1beta1/kubeadmcontrolplanetemplate_types.go index 4817b71b18fb..9fab688e84f7 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadmcontrolplanetemplate_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadmcontrolplanetemplate_types.go @@ -91,6 +91,10 @@ type KubeadmControlPlaneTemplateResourceSpec struct { // +optional // +kubebuilder:default={type: "RollingUpdate", rollingUpdate: {maxSurge: 1}} RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"` + + // The RemediationStrategy that controls how control plane machine remediation happens. + // +optional + RemediationStrategy *RemediationStrategy `json:"remediationStrategy,omitempty"` } // KubeadmControlPlaneTemplateMachineTemplate defines the template for Machines diff --git a/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go index 88d4bca9de22..5d6d56bccd09 100644 --- a/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go +++ b/controlplane/kubeadm/api/v1beta1/zz_generated.deepcopy.go @@ -324,6 +324,11 @@ func (in *KubeadmControlPlaneTemplateResourceSpec) DeepCopyInto(out *KubeadmCont *out = new(RolloutStrategy) (*in).DeepCopyInto(*out) } + if in.RemediationStrategy != nil { + in, out := &in.RemediationStrategy, &out.RemediationStrategy + *out = new(RemediationStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmControlPlaneTemplateResourceSpec. diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml index a70a55892c40..92fcc11ab3d7 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml @@ -2382,6 +2382,55 @@ spec: time limitations. type: string type: object + remediationStrategy: + description: The RemediationStrategy that controls how control + plane machine remediation happens. + properties: + maxRetry: + description: "MaxRetry is the Max number of retries while + attempting to remediate an unhealthy machine. A retry + happens when a machine that was created as a replacement + for an unhealthy machine also fails. For example, given + a control plane with three machines M1, M2, M3: \n M1 + become unhealthy; remediation happens, and M1-1 is created + as a replacement. If M1-1 (replacement of M1) has problems + while bootstrapping it will become unhealthy, and then + be remediated; such operation is considered a retry, + remediation-retry #1. If M1-2 (replacement of M1-2) + becomes unhealthy, remediation-retry #2 will happen, + etc. \n A retry could happen only after RetryPeriod + from the previous retry. If a machine is marked as unhealthy + after MinHealthyPeriod from the previous remediation + expired, this is not considered a retry anymore because + the new issue is assumed unrelated from the previous + one. \n If not set, the remedation will be retried infinitely." + format: int32 + type: integer + minHealthyPeriod: + description: "MinHealthyPeriod defines the duration after + which KCP will consider any failure to a machine unrelated + from the previous one. In this case the remediation + is not considered a retry anymore, and thus the retry + counter restarts from 0. For example, assuming MinHealthyPeriod + is set to 1h (default) \n M1 become unhealthy; remediation + happens, and M1-1 is created as a replacement. If M1-1 + (replacement of M1) has problems within the 1hr after + the creation, also this machine will be remediated and + this operation is considered a retry - a problem related + to the original issue happened to M1 -. \n If instead + the problem on M1-1 is happening after MinHealthyPeriod + expired, e.g. four days after m1-1 has been created + as a remediation of M1, the problem on M1-1 is considered + unrelated to the original issue happened to M1. \n If + not set, this value is defaulted to 1h." + type: string + retryPeriod: + description: "RetryPeriod is the duration that KCP should + wait before remediating a machine being created as a + replacement for an unhealthy machine (a retry). \n If + not set, a retry will happen immediately." + type: string + type: object rolloutAfter: description: RolloutAfter is a field to indicate a rollout should be performed after the specified time even if no diff --git a/controlplane/kubeadm/internal/controllers/helpers.go b/controlplane/kubeadm/internal/controllers/helpers.go index f6ef0dc13b48..a94370537cd2 100644 --- a/controlplane/kubeadm/internal/controllers/helpers.go +++ b/controlplane/kubeadm/internal/controllers/helpers.go @@ -341,7 +341,7 @@ func (r *KubeadmControlPlaneReconciler) generateMachine(ctx context.Context, kcp } // Remove the annotation tracking that a remediation is in progress (the remediation completed when - // the replacement machine have been created above). + // the replacement machine has been created above). delete(kcp.Annotations, controlplanev1.RemediationInProgressAnnotation) return nil diff --git a/controlplane/kubeadm/internal/controllers/remediation.go b/controlplane/kubeadm/internal/controllers/remediation.go index c0a6b6f9479d..72a0eb8236aa 100644 --- a/controlplane/kubeadm/internal/controllers/remediation.go +++ b/controlplane/kubeadm/internal/controllers/remediation.go @@ -73,11 +73,6 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C return ctrl.Result{}, kerrors.NewAggregate(errList) } - // Returns if another remediation is in progress but the new machine is not yet created. - if _, ok := controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]; ok { - return ctrl.Result{}, nil - } - // Gets all machines that have `MachineHealthCheckSucceeded=False` (indicating a problem was detected on the machine) // and `MachineOwnerRemediated` present, indicating that this controller is responsible for performing remediation. unhealthyMachines := controlPlane.UnhealthyMachines() @@ -99,6 +94,16 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C return ctrl.Result{}, nil } + log = log.WithValues("Machine", klog.KObj(machineToBeRemediated), "initialized", controlPlane.KCP.Status.Initialized) + + // Returns if another remediation is in progress but the new Machine is not yet created. + // Note: This condition is checked after we check for unhealthy Machines and if machineToBeRemediated + // is being deleted to avoid unnecessary logs if no further remediation should be done. + if _, ok := controlPlane.KCP.Annotations[controlplanev1.RemediationInProgressAnnotation]; ok { + log.Info("Another remediation is already in progress. Skipping remediation.") + return ctrl.Result{}, nil + } + patchHelper, err := patch.NewHelper(machineToBeRemediated, r.Client) if err != nil { return ctrl.Result{}, err @@ -118,7 +123,6 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C // Before starting remediation, run preflight checks in order to verify it is safe to remediate. // If any of the following checks fails, we'll surface the reason in the MachineOwnerRemediated condition. - log = log.WithValues("Machine", klog.KObj(machineToBeRemediated), "initialized", controlPlane.KCP.Status.Initialized) // Check if KCP is allowed to remediate considering retry limits: // - Remediation cannot happen because retryPeriod is not yet expired. @@ -132,9 +136,11 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C return ctrl.Result{}, nil } - // Executes checks that applies only if the control plane is already initialized; in this case KCP can - // remediate only if it can safely assume that the operation preserves the operation state of the existing cluster (or at least it doesn't make it worst). if controlPlane.KCP.Status.Initialized { + // Executes checks that apply only if the control plane is already initialized; in this case KCP can + // remediate only if it can safely assume that the operation preserves the operation state of the + // existing cluster (or at least it doesn't make it worse). + // The cluster MUST have more than one replica, because this is the smallest cluster size that allows any etcd failure tolerance. if controlPlane.Machines.Len() <= 1 { log.Info("A control plane machine needs remediation, but the number of current replicas is less or equal to 1. Skipping remediation", "Replicas", controlPlane.Machines.Len()) @@ -163,22 +169,14 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C return ctrl.Result{}, nil } } - } - - // Prepare the info for tracking the remediation progress into the RemediationInProgressAnnotation. - remediationInProgressValue, err := remediationInProgressData.Marshal() - if err != nil { - return ctrl.Result{}, err - } - // Start remediating the unhealthy control plane machine by deleting it. - // A new machine will come up completing the operation as part of the regular + // Start remediating the unhealthy control plane machine by deleting it. + // A new machine will come up completing the operation as part of the regular reconcile. - // If the control plane is initialized, before deleting the machine: - // - if the machine hosts the etcd leader, forward etcd leadership to another machine. - // - delete the etcd member hosted on the machine being deleted. - // - remove the etcd member from the kubeadm config map (only for kubernetes version older than v1.22.0) - if controlPlane.KCP.Status.Initialized { + // If the control plane is initialized, before deleting the machine: + // - if the machine hosts the etcd leader, forward etcd leadership to another machine. + // - delete the etcd member hosted on the machine being deleted. + // - remove the etcd member from the kubeadm config map (only for kubernetes version older than v1.22.0) workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(controlPlane.Cluster)) if err != nil { log.Error(err, "Failed to create client to workload cluster") @@ -227,6 +225,12 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C log.Info("Remediating unhealthy machine") conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "") + // Prepare the info for tracking the remediation progress into the RemediationInProgressAnnotation. + remediationInProgressValue, err := remediationInProgressData.Marshal() + if err != nil { + return ctrl.Result{}, err + } + // Set annotations tracking remediation details so they can be picked up by the machine // that will be created as part of the scale up action that completes the remediation. annotations.AddAnnotations(controlPlane.KCP, map[string]string{ @@ -263,49 +267,39 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin return remediationInProgressData, true, nil } - // Gets MinHealthySeconds and RetryDelaySeconds from the remediation strategy, or use defaults. + // Gets MinHealthyPeriod and RetryPeriod from the remediation strategy, or use defaults. minHealthyPeriod := controlplanev1.DefaultMinHealthyPeriod if controlPlane.KCP.Spec.RemediationStrategy != nil && controlPlane.KCP.Spec.RemediationStrategy.MinHealthyPeriod != nil { minHealthyPeriod = controlPlane.KCP.Spec.RemediationStrategy.MinHealthyPeriod.Duration } - retryPeriod := time.Duration(0) if controlPlane.KCP.Spec.RemediationStrategy != nil { retryPeriod = controlPlane.KCP.Spec.RemediationStrategy.RetryPeriod.Duration } // Gets the timestamp of the last remediation; if missing, default to a value - // that ensures both MinHealthySeconds and RetryDelaySeconds are expired. + // that ensures both MinHealthyPeriod and RetryPeriod are expired. // NOTE: this could potentially lead to executing more retries than expected or to executing retries before than // expected, but this is considered acceptable when the system recovers from someone/something changes or deletes // the RemediationForAnnotation on Machines. - max := func(x, y time.Duration) time.Duration { - if x < y { - return y - } - return x - } - lastRemediationTime := reconciliationTime.Add(-2 * max(minHealthyPeriod, retryPeriod)) if !lastRemediationData.Timestamp.IsZero() { lastRemediationTime = lastRemediationData.Timestamp.Time } - // Check if the machine being remediated has been created as a remediation for a previous unhealthy machine. - // NOTE: if someone/something changes or deletes the RemediationForAnnotation on Machines, this could potentially - // lead to executing more retries than expected, but this is considered acceptable in such a case. - machineRemediationFor := remediationInProgressData.Machine - if lastRemediationData.Machine != "" { - // If the remediation is happening before minHealthyPeriod is expired, then KCP considers this - // as a remediation for the same previously unhealthy machine. - if lastRemediationTime.Add(minHealthyPeriod).After(reconciliationTime) { - machineRemediationFor = lastRemediationData.Machine - log = log.WithValues("RemediationRetryFor", klog.KRef(machineToBeRemediated.Namespace, machineRemediationFor)) - } + // Once we get here we already know that there was a last remediation for the Machine. + // If the current remediation is happening before minHealthyPeriod is expired, then KCP considers this + // as a remediation for the same previously unhealthy machine. + // NOTE: If someone/something changes the RemediationForAnnotation on Machines (e.g. changes the Timestamp), + // this could potentially lead to executing more retries than expected, but this is considered acceptable in such a case. + var retryForSameMachineInProgress bool + if lastRemediationTime.Add(minHealthyPeriod).After(reconciliationTime) { + retryForSameMachineInProgress = true + log = log.WithValues("RemediationRetryFor", klog.KRef(machineToBeRemediated.Namespace, lastRemediationData.Machine)) } - // If remediation is happening for a different machine, this is the first try of a new retry sequence. - if lastRemediationData.Machine != machineRemediationFor { + // If the retry for the same machine is not in progress, this is the first try of a new retry sequence. + if !retryForSameMachineInProgress { return remediationInProgressData, true, nil } @@ -315,7 +309,7 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin // Check if remediation can happen because retryPeriod is passed. if lastRemediationTime.Add(retryPeriod).After(reconciliationTime) { log.Info(fmt.Sprintf("A control plane machine needs remediation, but the operation already failed in the latest %s. Skipping remediation", retryPeriod)) - conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryDelay)", retryPeriod) + conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest %s (RetryPeriod)", retryPeriod) return remediationInProgressData, false, nil } @@ -329,12 +323,20 @@ func (r *KubeadmControlPlaneReconciler) checkRetryLimits(log logr.Logger, machin } } - // All the check passed, increase the remediation number. + // All the check passed, increase the remediation retry count. remediationInProgressData.RetryCount++ return remediationInProgressData, true, nil } +// max calculates the maximum duration. +func max(x, y time.Duration) time.Duration { + if x < y { + return y + } + return x +} + // canSafelyRemoveEtcdMember assess if it is possible to remove the member hosted on the machine to be remediated // without loosing etcd quorum. // diff --git a/controlplane/kubeadm/internal/controllers/remediation_test.go b/controlplane/kubeadm/internal/controllers/remediation_test.go index 3766cbacbf56..26183baa1cc7 100644 --- a/controlplane/kubeadm/internal/controllers/remediation_test.go +++ b/controlplane/kubeadm/internal/controllers/remediation_test.go @@ -106,7 +106,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { t.Run("reconcileUnhealthyMachines return early if another remediation is in progress", func(t *testing.T) { g := NewWithT(t) - m := getDeletingMachine(ns.Name, "m1-unhealthy-deleting-", withMachineHealthCheckFailed()) + m := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withStuckRemediation()) conditions.MarkFalse(m, clusterv1.MachineHealthCheckSucceededCondition, clusterv1.MachineHasFailureReason, clusterv1.ConditionSeverityWarning, "") conditions.MarkFalse(m, clusterv1.MachineOwnerRemediatedCondition, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "") controlPlane := &internal.ControlPlane{ @@ -142,6 +142,8 @@ func TestReconcileUnhealthyMachines(t *testing.T) { } ret, err := r.reconcileUnhealthyMachines(ctx, controlPlane) + g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) + g.Expect(ret.IsZero()).To(BeTrue()) // Remediation skipped g.Expect(err).ToNot(HaveOccurred()) }) @@ -309,7 +311,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { removeFinalizer(g, m1) g.Expect(env.Cleanup(ctx, m1, m2, m3)).To(Succeed()) }) - t.Run("Remediation does not happen if RetryDelay is not yet passed", func(t *testing.T) { + t.Run("Remediation does not happen if RetryPeriod is not yet passed", func(t *testing.T) { g := NewWithT(t) m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed(), withWaitBeforeDeleteFinalizer(), withRemediateForAnnotation(MustMarshalRemediationData(&RemediationData{ @@ -327,7 +329,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { Version: "v1.19.1", RemediationStrategy: &controlplanev1.RemediationStrategy{ MaxRetry: utilpointer.Int32(3), - RetryPeriod: metav1.Duration{Duration: controlplanev1.DefaultMinHealthyPeriod}, // RetryDelaySeconds not yet expired. + RetryPeriod: metav1.Duration{Duration: controlplanev1.DefaultMinHealthyPeriod}, // RetryPeriod not yet expired. }, }, }, @@ -352,7 +354,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(controlPlane.KCP.Annotations).ToNot(HaveKey(controlplanev1.RemediationInProgressAnnotation)) - assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest 1h0m0s (RetryDelay)") + assertMachineCondition(ctx, g, m1, clusterv1.MachineOwnerRemediatedCondition, corev1.ConditionFalse, clusterv1.WaitingForRemediationReason, clusterv1.ConditionSeverityWarning, "KCP can't remediate this machine because the operation already failed in the latest 1h0m0s (RetryPeriod)") err = env.Get(ctx, client.ObjectKey{Namespace: m1.Namespace, Name: m1.Name}, m1) g.Expect(err).ToNot(HaveOccurred()) @@ -401,7 +403,7 @@ func TestReconcileUnhealthyMachines(t *testing.T) { g.Expect(env.Cleanup(ctx, m)).To(Succeed()) }) - t.Run("Remediation does not happen if there is a deleting machine", func(t *testing.T) { + t.Run("Remediation does not happen if there is another machine being deleted (not the one to be remediated)", func(t *testing.T) { g := NewWithT(t) m1 := createMachine(ctx, g, ns.Name, "m1-unhealthy-", withMachineHealthCheckFailed()) @@ -1146,7 +1148,7 @@ func TestReconcileUnhealthyMachinesSequences(t *testing.T) { err = env.Get(ctx, client.ObjectKey{Namespace: m3.Namespace, Name: m3.Name}, m3) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(m2.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) + g.Expect(m3.ObjectMeta.DeletionTimestamp.IsZero()).To(BeFalse()) removeFinalizer(g, m3) g.Expect(env.Cleanup(ctx, m3)).To(Succeed()) diff --git a/docs/book/src/reference/labels_and_annotations.md b/docs/book/src/reference/labels_and_annotations.md index be03817863d3..5a6e5089bf24 100644 --- a/docs/book/src/reference/labels_and_annotations.md +++ b/docs/book/src/reference/labels_and_annotations.md @@ -49,5 +49,5 @@ | controlplane.cluster.x-k8s.io/skip-coredns | It explicitly skips reconciling CoreDNS if set. | | controlplane.cluster.x-k8s.io/skip-kube-proxy | It explicitly skips reconciling kube-proxy if set. | | controlplane.cluster.x-k8s.io/kubeadm-cluster-configuration | It is a machine annotation that stores the json-marshalled string of KCP ClusterConfiguration. This annotation is used to detect any changes in ClusterConfiguration and trigger machine rollout in KCP. | -| controlplane.cluster.x-k8s.io/remediation-in-progress | It is a KCP remediation is that tracks that the system is in between having deleted an unhealthy machine and recreating its replacement. | +| controlplane.cluster.x-k8s.io/remediation-in-progress | It is a KCP annotation that tracks that the system is in between having deleted an unhealthy machine and recreating its replacement. | | controlplane.cluster.x-k8s.io/remediation-for | It is a machine annotation that links a new machine to the unhealthy machine it is replacing. | diff --git a/test/e2e/kcp_remediations.go b/test/e2e/kcp_remediations.go index c0e69cd0a0bc..64fc229b0b59 100644 --- a/test/e2e/kcp_remediations.go +++ b/test/e2e/kcp_remediations.go @@ -19,16 +19,13 @@ package e2e import ( "context" "fmt" - "io" "os" - "os/exec" "path/filepath" - "runtime" - "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -117,7 +114,7 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp Proxy: input.BootstrapClusterProxy, ArtifactFolder: input.ArtifactFolder, SpecName: specName, - Flavor: input.Flavor, + Flavor: pointer.StringDeref(input.Flavor, "kcp-remediation"), // values to be injected in the template @@ -126,7 +123,7 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp // NOTE: this func also setups credentials/RBAC rules and everything necessary to get the authenticationToken. AuthenticationToken: getAuthenticationToken(ctx, input.BootstrapClusterProxy, namespace.Name), // Address to be used for accessing the management cluster from a workload cluster. - ServerAddr: getServerAddr(input.BootstrapClusterProxy.GetKubeconfigPath()), + ServerAddr: getServerAddr(ctx, input.BootstrapClusterProxy), }) // The first CP machine comes up but it does not complete bootstrap @@ -235,7 +232,7 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp Expect(secondMachine.Status.NodeRef).To(BeNil()) log.Logf("Machine %s is up but still bootstrapping", secondMachineName) - // Intentionally trigger remediation on the second CP, and validate and validate also this one is deleted and a replacement should come up. + // Intentionally trigger remediation on the second CP and validate that also this one is deleted and a replacement should come up. By("REMEDIATING SECOND CONTROL PLANE MACHINE") @@ -432,17 +429,15 @@ type createWorkloadClusterAndWaitInput struct { Proxy framework.ClusterProxy ArtifactFolder string SpecName string - Flavor *string + Flavor string Namespace string - AuthenticationToken []byte + AuthenticationToken string ServerAddr string } -// createWorkloadClusterAndWait creates a workload cluster ard return as soon as the cluster infrastructure is ready. +// createWorkloadClusterAndWait creates a workload cluster and return as soon as the cluster infrastructure is ready. // NOTE: we are not using the same func used by other tests because it would fail if the control plane doesn't come up, -// -// which instead is expected in this case. -// +// which instead is expected in this case. // NOTE: clusterResources is filled only partially. func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClusterAndWaitInput) (clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) { clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) @@ -457,7 +452,7 @@ func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClust KubeconfigPath: input.Proxy.GetKubeconfigPath(), // select template - Flavor: pointer.StringDeref(input.Flavor, "kcp-remediation"), + Flavor: input.Flavor, // define template variables Namespace: input.Namespace, ClusterName: clusterName, @@ -469,7 +464,7 @@ func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClust LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.Proxy.GetName()), // Adds authenticationToken, server address and namespace variables to be injected in the cluster template. ClusterctlVariables: map[string]string{ - "TOKEN": string(input.AuthenticationToken), + "TOKEN": input.AuthenticationToken, "SERVER": input.ServerAddr, "NAMESPACE": input.Namespace, }, @@ -560,11 +555,10 @@ func waitForMachines(ctx context.Context, input waitForMachinesInput) (allMachin // Waits for the desired set of machines to exist. log.Logf("Waiting for %d machines, must have %s, must not have %s", input.ExpectedReplicas, expectedOldMachines.UnsortedList(), expectedDeletedMachines.UnsortedList()) - Eventually(func() bool { + Eventually(func(g Gomega) { // Gets the list of machines - if err := input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { - return false - } + g.Expect(input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption)).To(Succeed()) + allMachines = sets.Set[string]{} for i := range machineList.Items { allMachines.Insert(machineList.Items[i].Name) @@ -578,17 +572,17 @@ func waitForMachines(ctx context.Context, input waitForMachinesInput) (allMachin log.Logf(" - expected %d, got %d: %s, of which new %s, must have check: %t, must not have check: %t", input.ExpectedReplicas, allMachines.Len(), allMachines.UnsortedList(), newMachines.UnsortedList(), allMachines.HasAll(expectedOldMachines.UnsortedList()...), !allMachines.HasAny(expectedDeletedMachines.UnsortedList()...)) // Ensures all the expected old machines are still there. - if !allMachines.HasAll(expectedOldMachines.UnsortedList()...) { - return false - } + g.Expect(allMachines.HasAll(expectedOldMachines.UnsortedList()...)).To(BeTrue(), + "Got machines: %s, must contain all of: %s", allMachines.UnsortedList(), expectedOldMachines.UnsortedList()) // Ensures none of the machines to be deleted is still there. - if allMachines.HasAny(expectedDeletedMachines.UnsortedList()...) { - return false - } + g.Expect(!allMachines.HasAny(expectedDeletedMachines.UnsortedList()...)).To(BeTrue(), + "Got machines: %s, must not contain any of: %s", allMachines.UnsortedList(), expectedDeletedMachines.UnsortedList()) - return allMachines.Len() == input.ExpectedReplicas - }, input.WaitForMachinesIntervals...).Should(BeTrue(), "Failed to get the expected list of machines: got %s (expected %d machines, must have %s, must not have %s)", allMachines.UnsortedList(), input.ExpectedReplicas, expectedOldMachines.UnsortedList(), expectedDeletedMachines.UnsortedList()) + g.Expect(allMachines).To(HaveLen(input.ExpectedReplicas), "Got %d machines, must be %d", len(allMachines), input.ExpectedReplicas) + }, input.WaitForMachinesIntervals...).Should(Succeed(), + "Failed to get the expected list of machines: got %s (expected %d machines, must have %s, must not have %s)", + allMachines.UnsortedList(), input.ExpectedReplicas, expectedOldMachines.UnsortedList(), expectedDeletedMachines.UnsortedList()) log.Logf("Got %d machines: %s", allMachines.Len(), allMachines.UnsortedList()) // Ensures the desired set of machines is stable (no further machines are created or deleted). @@ -611,27 +605,31 @@ func waitForMachines(ctx context.Context, input waitForMachinesInput) (allMachin } // getServerAddr returns the address to be used for accessing the management cluster from a workload cluster. -func getServerAddr(kubeconfigPath string) string { - kubeConfig, err := clientcmd.LoadFromFile(kubeconfigPath) - Expect(err).ToNot(HaveOccurred(), "failed to load management cluster's kubeconfig file") +func getServerAddr(ctx context.Context, clusterProxy framework.ClusterProxy) string { + // With CAPD, we can't just access the bootstrap cluster via 127.0.0.1: from the + // workload cluster. Instead we retrieve the server name from the cluster-info ConfigMap in the bootstrap + // cluster (e.g. "https://test-z45p9k-control-plane:6443") + // Note: This has been tested with MacOS,Linux and Prow. + clusterInfoCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-info", + Namespace: metav1.NamespacePublic, + }, + } + Expect(clusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(clusterInfoCM), clusterInfoCM)).To(Succeed()) + Expect(clusterInfoCM.Data).To(HaveKey("kubeconfig")) - clusterName := kubeConfig.Contexts[kubeConfig.CurrentContext].Cluster - Expect(clusterName).ToNot(BeEmpty(), "failed to identify current cluster name in management cluster's kubeconfig file") + kubeConfigString := clusterInfoCM.Data["kubeconfig"] - serverAddr := kubeConfig.Clusters[clusterName].Server - Expect(serverAddr).ToNot(BeEmpty(), "failed to identify current server address in management cluster's kubeconfig file") + kubeConfig, err := clientcmd.Load([]byte(kubeConfigString)) + Expect(err).ToNot(HaveOccurred()) - // On CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host - // to the CAPD cluster. Moby on Linux doesn't use the host.docker.internal DNS name. - if runtime.GOOS != "linux" { - serverAddr = strings.ReplaceAll(serverAddr, "127.0.0.1", "host.docker.internal") - } - return serverAddr + return kubeConfig.Clusters[""].Server } // getAuthenticationToken returns a bearer authenticationToken with minimal RBAC permissions to access the mhc-test ConfigMap that will be used // to control machines bootstrap during the remediation tests. -func getAuthenticationToken(ctx context.Context, managementClusterProxy framework.ClusterProxy, namespace string) []byte { +func getAuthenticationToken(ctx context.Context, managementClusterProxy framework.ClusterProxy, namespace string) string { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: "mhc-test", @@ -677,21 +675,13 @@ func getAuthenticationToken(ctx context.Context, managementClusterProxy framewor } Expect(managementClusterProxy.GetClient().Create(ctx, roleBinding)).To(Succeed(), "failed to create mhc-test role binding") - cmd := exec.CommandContext(ctx, "kubectl", fmt.Sprintf("--kubeconfig=%s", managementClusterProxy.GetKubeconfigPath()), fmt.Sprintf("--namespace=%s", namespace), "create", "token", "mhc-test") //nolint:gosec - stdout, err := cmd.StdoutPipe() - Expect(err).ToNot(HaveOccurred(), "failed to get stdout for kubectl create authenticationToken") - stderr, err := cmd.StderrPipe() - Expect(err).ToNot(HaveOccurred(), "failed to get stderr for kubectl create authenticationToken") - - Expect(cmd.Start()).To(Succeed(), "failed to run kubectl create authenticationToken") - - output, err := io.ReadAll(stdout) - Expect(err).ToNot(HaveOccurred(), "failed to read stdout from kubectl create authenticationToken") - errout, err := io.ReadAll(stderr) - Expect(err).ToNot(HaveOccurred(), "failed to read stderr from kubectl create authenticationToken") - - Expect(cmd.Wait()).To(Succeed(), "failed to wait kubectl create authenticationToken") - Expect(errout).To(BeEmpty()) + tokenRequest := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: pointer.Int64(2 * 60 * 60), // 2 hours. + }, + } + tokenRequest, err := managementClusterProxy.GetClientSet().CoreV1().ServiceAccounts(namespace).CreateToken(ctx, "mhc-test", tokenRequest, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) - return output + return tokenRequest.Status.Token } diff --git a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go index 7dd5737d1197..7920065ae40f 100644 --- a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go +++ b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go @@ -316,7 +316,6 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * for { select { case <-timeoutCtx.Done(): - log.Info("Cancelling Bootstrap due to timeout") return default: updatedDockerMachine := &infrav1.DockerMachine{} @@ -353,13 +352,6 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - // If the control plane is not yet initialized, there is no API server to contact to get the ProviderID for the Node - // hosted on this machine, so return early. - // NOTE: we are using RequeueAfter with a short interval in order to make test execution time more stable. - if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { - return ctrl.Result{RequeueAfter: 15 * time.Second}, nil - } - // Usually a cloud provider will do this, but there is no docker-cloud provider. // Requeue if there is an error, as this is likely momentary load balancer // state changes during control plane provisioning.