diff --git a/go.mod b/go.mod index 02987a94..b32458ca 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,8 @@ require ( k8s.io/client-go v0.26.1 k8s.io/component-base v0.26.1 k8s.io/kube-aggregator v0.26.1 - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 + k8s.io/utils v0.0.0-20230115233650-391b47cb4029 + sigs.k8s.io/cli-utils v0.35.0 sigs.k8s.io/controller-runtime v0.14.4 sigs.k8s.io/yaml v1.3.0 ) @@ -76,14 +77,14 @@ require ( github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-errors/errors v1.0.1 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-gorp/gorp/v3 v3.0.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -115,7 +116,7 @@ require ( github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.7 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect diff --git a/go.sum b/go.sum index 4c4e2957..a5822aa4 100644 --- a/go.sum +++ b/go.sum @@ -390,8 +390,8 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= @@ -438,8 +438,8 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= @@ -704,8 +704,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= @@ -1706,8 +1706,8 @@ k8s.io/kubectl v0.26.1 h1:K8A0Jjlwg8GqrxOXxAbjY5xtmXYeYjLU96cHp2WMQ7s= k8s.io/kubectl v0.26.1/go.mod h1:miYFVzldVbdIiXMrHZYmL/EDWwJKM+F0sSsdxsATFPo= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230115233650-391b47cb4029 h1:L8zDtT4jrxj+TaQYD0k8KNlr556WaVQylDXswKmX+dE= +k8s.io/utils v0.0.0-20230115233650-391b47cb4029/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.2 h1:0E9tOHUfrNH7TCDk5KU0jVBEzCqbfdyuVfGmJ7ZeRPE= oras.land/oras-go v1.2.2/go.mod h1:Apa81sKoZPpP7CDciE006tSZ0x3Q3+dOoBcMZ/aNxvw= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1719,6 +1719,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyz sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 h1:+xBL5uTc+BkPBwmMi3vYfUJjq+N3K+H6PXeETwf5cPI= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= +sigs.k8s.io/cli-utils v0.35.0 h1:dfSJaF1W0frW74PtjwiyoB4cwdRygbHnC7qe7HF0g/Y= +sigs.k8s.io/cli-utils v0.35.0/go.mod h1:ITitykCJxP1vaj1Cew/FZEaVJ2YsTN9Q71m02jebkoE= sigs.k8s.io/controller-runtime v0.14.4 h1:Kd/Qgx5pd2XUL08eOV2vwIq3L9GhIbJ5Nxengbd4/0M= sigs.k8s.io/controller-runtime v0.14.4/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= diff --git a/internal/healthchecks/builtin.go b/internal/healthchecks/builtin.go index e31b781a..6c823527 100644 --- a/internal/healthchecks/builtin.go +++ b/internal/healthchecks/builtin.go @@ -5,13 +5,11 @@ import ( "errors" "fmt" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "sigs.k8s.io/cli-utils/pkg/kstatus/status" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -27,6 +25,10 @@ import ( // - Pods // - APIServices // - CustomResourceDefinitions +// - Jobs +// - Services +// - PersistentVolumeClaims +// - PodDisruptionBudgets // // If the resource is not supported, it is assumed to be healthy. func AreObjectsHealthy(ctx context.Context, client client.Client, objects []client.Object) error { @@ -45,147 +47,38 @@ func AreObjectsHealthy(ctx context.Context, client client.Client, objects []clie continue } - switch u.GroupVersionKind() { - case appsv1.SchemeGroupVersion.WithKind("Deployment"): - // Check if the deployment is available. - obj := &appsv1.Deployment{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { - gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) - continue - } - conditionExists := false - for _, condition := range obj.Status.Conditions { - if condition.Type == appsv1.DeploymentAvailable { - if condition.Status != "True" { - gvkErrors = appendResourceError(gvkErrors, obj, condition.Message) - } - conditionExists = true - break - } - } - if conditionExists { - continue - } - gvkErrors = appendResourceError(gvkErrors, obj, "DeploymentAvailable condition not found") - case appsv1.SchemeGroupVersion.WithKind("StatefulSet"): - // This logic has been adapted from the helm codebase. - obj := &appsv1.StatefulSet{} + if object.GetObjectKind().GroupVersionKind() == apiregistrationv1.SchemeGroupVersion.WithKind("APIService") { + obj := &apiregistrationv1.APIService{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) continue } - // This logic has been adapted from the helm codebase. - // - https://github.com/helm/helm/blob/e7bb860d9a32e8739c944b8e7b7f7031d752411a/pkg/kube/ready.go#L357-L410 - - // If the statefulset is not using the RollingUpdate strategy, we assume it's healthy. - if obj.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType { - continue - } - if obj.Status.ObservedGeneration < obj.Generation { - gvkErrors = appendResourceError(gvkErrors, obj, "StatefulSet is not ready (update has not yet been observed)") - } - - var partition int - var replicas = 1 - if obj.Spec.UpdateStrategy.RollingUpdate != nil && obj.Spec.UpdateStrategy.RollingUpdate.Partition != nil { - partition = int(*obj.Spec.UpdateStrategy.RollingUpdate.Partition) - } - if obj.Spec.Replicas != nil { - replicas = int(*obj.Spec.Replicas) - } - expectedReplicas := replicas - partition - if obj.Status.UpdatedReplicas < int32(expectedReplicas) { - gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected %d replicas, got %d)", expectedReplicas, obj.Status.UpdatedReplicas)) - continue - } - if int(obj.Status.ReadyReplicas) != replicas { - gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected %d replicas, got %d)", replicas, obj.Status.ReadyReplicas)) - continue - } - if partition == 0 && obj.Status.CurrentRevision != obj.Status.UpdateRevision { - gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected revision %s, got %s)", obj.Status.CurrentRevision, obj.Status.UpdateRevision)) - continue - } - case appsv1.SchemeGroupVersion.WithKind("DaemonSet"): - // Check if the daemonset is ready. - obj := &appsv1.DaemonSet{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { - gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) - continue - } - if obj.Status.NumberAvailable != obj.Status.DesiredNumberScheduled { - gvkErrors = appendResourceError(gvkErrors, obj, "DaemonSet is not ready") - } - case appsv1.SchemeGroupVersion.WithKind("ReplicaSet"): - // Check if the replicaset is ready. - obj := &appsv1.ReplicaSet{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { - gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) - continue - } - if obj.Status.AvailableReplicas != obj.Status.Replicas { - gvkErrors = appendResourceError(gvkErrors, obj, "ReplicaSet is not ready") - } - case corev1.SchemeGroupVersion.WithKind("Pod"): - // Check if the pod is running or succeeded. - obj := &corev1.Pod{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { - gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) - continue - } - if obj.Status.Phase != corev1.PodRunning && obj.Status.Phase != corev1.PodSucceeded { - gvkErrors = appendResourceError(gvkErrors, obj, "Pod is not Running or Succeeded") - } - case apiregistrationv1.SchemeGroupVersion.WithKind("APIService"): // Check if the APIService is available. - obj := &apiregistrationv1.APIService{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { - gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) - continue - } - conditionExists := false - for _, condition := range obj.Status.Conditions { + var isAvailable *apiregistrationv1.APIServiceCondition + for i, condition := range obj.Status.Conditions { if condition.Type == apiregistrationv1.Available { - if condition.Status != "True" { - gvkErrors = appendResourceError(gvkErrors, obj, condition.Message) - } - conditionExists = true + isAvailable = &obj.Status.Conditions[i] break } } - if conditionExists { - continue + if isAvailable == nil { + gvkErrors = appendResourceError(gvkErrors, obj, "Available condition not found") + } else if isAvailable.Status == apiregistrationv1.ConditionFalse { + gvkErrors = appendResourceError(gvkErrors, obj, isAvailable.Message) } - // If we are here we didn't find the "Available" condition, so we assume the APIService is non healthy. - gvkErrors = appendResourceError(gvkErrors, obj, "Available condition not found") - case apiextensionsv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"): - // Check if the CRD is established. - obj := &apiextensionsv1.CustomResourceDefinition{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { - gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) - continue - } - conditionExists := false - for _, condition := range obj.Status.Conditions { - if condition.Type == apiextensionsv1.Established { - if condition.Status != "True" { - gvkErrors = appendResourceError(gvkErrors, obj, condition.Message) - } - conditionExists = true - break - } - } - if conditionExists { - continue - } - gvkErrors = appendResourceError(gvkErrors, obj, "Established condition not found") - default: - // If we don't know how to check the health of the object, we assume it's healthy. continue } - } + result, err := status.Compute(u) + if err != nil { + gvkErrors = appendResourceError(gvkErrors, object, err.Error()) + } + + if result.Status != status.CurrentStatus { + gvkErrors = appendResourceError(gvkErrors, object, fmt.Sprintf("object %s: %s", result.Status, result.Message)) + } + } return errors.Join(gvkErrors...) } diff --git a/internal/healthchecks/builtin_test.go b/internal/healthchecks/builtin_test.go index edac3a02..c4197944 100644 --- a/internal/healthchecks/builtin_test.go +++ b/internal/healthchecks/builtin_test.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -20,7 +21,7 @@ func TestAreObjectsHealthy(t *testing.T) { for _, tt := range []struct { name string resources []client.Object - expectedErr bool + expectedErr string }{ { name: "Return true, all resources are healthy", @@ -40,6 +41,10 @@ func TestAreObjectsHealthy(t *testing.T) { Status: "True", }, }, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + ReadyReplicas: 1, }, }, &appsv1.StatefulSet{ @@ -51,8 +56,9 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyStatefulSet", }, Status: appsv1.StatefulSetStatus{ - ReadyReplicas: 1, - Replicas: 1, + ReadyReplicas: 1, + Replicas: 1, + CurrentReplicas: 1, }, }, &appsv1.DaemonSet{ @@ -61,11 +67,16 @@ func TestAreObjectsHealthy(t *testing.T) { APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "MyDaemonSet", + Name: "MyDaemonSet", + Generation: 1, }, Status: appsv1.DaemonSetStatus{ NumberAvailable: 1, DesiredNumberScheduled: 1, + ObservedGeneration: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 1, + NumberReady: 1, }, }, &appsv1.ReplicaSet{ @@ -77,8 +88,10 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyReplicatSet", }, Status: appsv1.ReplicaSetStatus{ - AvailableReplicas: 1, - Replicas: 1, + AvailableReplicas: 1, + Replicas: 1, + FullyLabeledReplicas: 1, + ReadyReplicas: 1, }, }, &corev1.Pod{ @@ -135,7 +148,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: false, + expectedErr: "", }, { name: "multiple resources are healthy, only one is not, return error", @@ -155,6 +168,10 @@ func TestAreObjectsHealthy(t *testing.T) { Status: "True", }, }, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + ReadyReplicas: 1, }, }, &appsv1.StatefulSet{ @@ -166,8 +183,9 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyStatefulSet", }, Status: appsv1.StatefulSetStatus{ - ReadyReplicas: 1, - Replicas: 1, + ReadyReplicas: 1, + Replicas: 1, + CurrentReplicas: 1, }, }, &appsv1.DaemonSet{ @@ -176,15 +194,19 @@ func TestAreObjectsHealthy(t *testing.T) { APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "MyDaemonSet", + Name: "MyDaemonSet", + Generation: 1, }, Status: appsv1.DaemonSetStatus{ NumberAvailable: 0, DesiredNumberScheduled: 1, + ObservedGeneration: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 1, }, }, }, - expectedErr: true, + expectedErr: "(apps/v1, Kind=DaemonSet)(MyDaemonSet): object InProgress: Available: 0/1", }, { name: "All resources are unhealthy, return error", @@ -205,6 +227,10 @@ func TestAreObjectsHealthy(t *testing.T) { Message: "Something went wrong", }, }, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + ReadyReplicas: 1, }, }, &appsv1.StatefulSet{ @@ -216,8 +242,9 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyStatefulSet", }, Status: appsv1.StatefulSetStatus{ - ReadyReplicas: 0, - Replicas: 1, + ReadyReplicas: 0, + Replicas: 1, + CurrentReplicas: 1, }, }, &appsv1.DaemonSet{ @@ -226,11 +253,15 @@ func TestAreObjectsHealthy(t *testing.T) { APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "MyDaemonSet", + Name: "MyDaemonSet", + Generation: 1, }, Status: appsv1.DaemonSetStatus{ NumberAvailable: 0, DesiredNumberScheduled: 1, + ObservedGeneration: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 1, }, }, &appsv1.ReplicaSet{ @@ -242,8 +273,9 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyReplicatSet", }, Status: appsv1.ReplicaSetStatus{ - AvailableReplicas: 0, - Replicas: 1, + AvailableReplicas: 0, + Replicas: 1, + FullyLabeledReplicas: 1, }, }, &corev1.Pod{ @@ -255,6 +287,7 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyPod", }, Status: corev1.PodStatus{ + Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{ { Type: corev1.PodReady, @@ -300,7 +333,13 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: `(apps/v1, Kind=Deployment)(MyDeployment): object InProgress: Deployment not Available +(apps/v1, Kind=StatefulSet)(MyStatefulSet): object InProgress: Ready: 0/1 +(apps/v1, Kind=DaemonSet)(MyDaemonSet): object InProgress: Available: 0/1 +(apps/v1, Kind=ReplicaSet)(MyReplicatSet): object InProgress: Available: 0/1 +(/v1, Kind=Pod)(MyPod): object InProgress: Pod is running but is not Ready +(apiregistration.k8s.io/v1, Kind=APIService)(MyAPIService): Something went wrong +(apiextensions.k8s.io/v1, Kind=CustomResourceDefinition)(MyCustomResourceDefinition): object Failed: CustomResourceDefinition is not established`, }, { name: "unknown resource, no error", @@ -315,7 +354,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: false, + expectedErr: "", }, { name: "Pod: valid resource with no conditions doesn't return error", @@ -329,12 +368,17 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyPod", }, Status: corev1.PodStatus{ - Conditions: nil, - Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + Phase: corev1.PodRunning, }, }, }, - expectedErr: false, + expectedErr: "", }, { name: "APIService: resource with no conditions, return error", @@ -352,7 +396,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(apiregistration.k8s.io/v1, Kind=APIService)(MyAPIService): Available condition not found", }, { name: "CustomResourceDefinition: resource with no conditions return error", @@ -370,7 +414,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(apiextensions.k8s.io/v1, Kind=CustomResourceDefinition)(MyCustomResourceDefinition): object InProgress: Install in progress", }, { name: "Deployment: resource with no conditions return error", @@ -384,11 +428,15 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyDeployment", }, Status: appsv1.DeploymentStatus{ - Conditions: nil, + Conditions: nil, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + ReadyReplicas: 1, }, }, }, - expectedErr: true, + expectedErr: "(apps/v1, Kind=Deployment)(MyDeployment): object InProgress: Deployment not Available", }, { name: "StatefulSet: valid resource with no conditions, doesn't return error", @@ -413,10 +461,11 @@ func TestAreObjectsHealthy(t *testing.T) { ObservedGeneration: 1, ReadyReplicas: 1, Replicas: 1, + CurrentReplicas: 1, }, }, }, - expectedErr: true, + expectedErr: "", }, { name: "DaemonSet: valid resource with no conditions, doesn't return error", @@ -427,16 +476,21 @@ func TestAreObjectsHealthy(t *testing.T) { APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "MyDaemonSet", + Name: "MyDaemonSet", + Generation: 1, }, Status: appsv1.DaemonSetStatus{ DesiredNumberScheduled: 1, NumberAvailable: 1, Conditions: nil, + ObservedGeneration: 1, + CurrentNumberScheduled: 1, + UpdatedNumberScheduled: 1, + NumberReady: 1, }, }, }, - expectedErr: false, + expectedErr: "", }, { name: "ReplicaSet: valid resource with no conditions, doesn't return error", @@ -450,13 +504,15 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyReplicaSet", }, Status: appsv1.ReplicaSetStatus{ - Replicas: 1, - AvailableReplicas: 1, - Conditions: nil, + Replicas: 1, + AvailableReplicas: 1, + Conditions: nil, + FullyLabeledReplicas: 1, + ReadyReplicas: 1, }, }, }, - expectedErr: false, + expectedErr: "", }, { name: "APIService: resource with conditions but not the one we are looking for, return error", @@ -480,7 +536,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(apiregistration.k8s.io/v1, Kind=APIService)(MyAPIService): Available condition not found", }, { name: "CustomResourceDefinition: resource with conditions but not the one we are looking for, return error", @@ -504,7 +560,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(apiextensions.k8s.io/v1, Kind=CustomResourceDefinition)(MyCustomResourceDefinition): object InProgress: Install in progress", }, { name: "Deployment: resource with conditions but not the one we are looking for, return error", @@ -524,38 +580,14 @@ func TestAreObjectsHealthy(t *testing.T) { Status: "True", }, }, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + ReadyReplicas: 1, }, }, }, - expectedErr: true, - }, - { - name: "StatefulSet: resource with conditions but not the one we are looking for, return error", - resources: []client.Object{ - &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "StatefulSet", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "MyStatefulSet", - Generation: 1, - }, - Spec: appsv1.StatefulSetSpec{ - UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ - Type: appsv1.RollingUpdateStatefulSetStrategyType, - }, - Replicas: func() *int32 { i := int32(1); return &i }(), - }, - Status: appsv1.StatefulSetStatus{ - Conditions: []appsv1.StatefulSetCondition{}, - ObservedGeneration: 1, - ReadyReplicas: 1, - Replicas: 1, - }, - }, - }, - expectedErr: true, + expectedErr: "(apps/v1, Kind=Deployment)(MyDeployment): object InProgress: Deployment not Available", }, { name: "Pod: resource with conditions but not the one we are looking for, return error", @@ -569,6 +601,7 @@ func TestAreObjectsHealthy(t *testing.T) { Name: "MyPod", }, Status: corev1.PodStatus{ + Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{ { Type: corev1.PodInitialized, @@ -578,7 +611,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(/v1, Kind=Pod)(MyPod): object InProgress: Pod is running but is not Ready", }, { name: "APIService: resource with conditions but not the one we are looking for, return error", @@ -601,7 +634,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(apiregistration.k8s.io/v1, Kind=APIService)(MyAPIService): Available condition not found", }, { name: "StatefulSet is not ready as observedGeneration doesn't match, return error", @@ -628,7 +661,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(apps/v1, Kind=StatefulSet)(MyStatefulSet): object InProgress: StatefulSet generation is 2, but latest observed generation is 1", }, { name: "StatefulSet is not ready as replicas are not ready, return error", @@ -656,7 +689,7 @@ func TestAreObjectsHealthy(t *testing.T) { }, }, }, - expectedErr: true, + expectedErr: "(apps/v1, Kind=StatefulSet)(MyStatefulSet): object InProgress: Ready: 0/1", }, { name: "StatefulSet is not ready as Revisions don't match, return error", @@ -674,7 +707,7 @@ func TestAreObjectsHealthy(t *testing.T) { UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ Type: appsv1.RollingUpdateStatefulSetStrategyType, }, - Replicas: func() *int32 { i := int32(1); return &i }(), + Replicas: pointer.Int32(1), }, Status: appsv1.StatefulSetStatus{ CurrentRevision: "revision2", @@ -683,10 +716,11 @@ func TestAreObjectsHealthy(t *testing.T) { ReadyReplicas: 1, UpdatedReplicas: 1, Replicas: 1, + CurrentReplicas: 1, }, }, }, - expectedErr: true, + expectedErr: "(apps/v1, Kind=StatefulSet)(MyStatefulSet): object InProgress: Waiting for updated revision to match current", }, { name: "StatefulSet is not using the RollingUpdate strategy, return no error", @@ -707,7 +741,7 @@ func TestAreObjectsHealthy(t *testing.T) { Status: appsv1.StatefulSetStatus{}, }, }, - expectedErr: false, + expectedErr: "", }, } { ctx := context.Background() @@ -716,7 +750,10 @@ func TestAreObjectsHealthy(t *testing.T) { // Instantiate a fake client. client.setResources(tt.resources) err := AreObjectsHealthy(ctx, client, tt.resources) - if (err != nil) != tt.expectedErr { + if (err != nil) != (len(tt.expectedErr) != 0) { + t.Errorf("AreRelObjectsHealthy() testName=%q error = %v, expectedErr %v", tt.name, err, tt.expectedErr) + } + if err != nil && err.Error() != tt.expectedErr { t.Errorf("AreRelObjectsHealthy() testName=%q error = %v, expectedErr %v", tt.name, err, tt.expectedErr) } })