diff --git a/CHANGELOG.md b/CHANGELOG.md index 51a84c47b0..6b684ebf18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add `global.podSecurityStandards.enforced` value for PSS migration. +- Emit event when an unhealthy node is terminated. +- Bump `badnodedetector` to be able to use `node-problem-detector` app for unhealthy node termination. ## [14.23.0] - 2023-10-04 diff --git a/go.mod b/go.mod index 718fd7c15a..98b39c1a40 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/ghodss/yaml v1.0.1-0.20220118164431-d8423dcdf344 github.com/giantswarm/apiextensions/v6 v6.6.0 github.com/giantswarm/backoff v1.0.0 - github.com/giantswarm/badnodedetector/v2 v2.0.0 + github.com/giantswarm/badnodedetector/v3 v3.0.0 github.com/giantswarm/certs/v4 v4.0.0 github.com/giantswarm/ipam v0.3.0 github.com/giantswarm/k8sclient/v7 v7.0.1 diff --git a/go.sum b/go.sum index c14f7a78ff..fca2db5f76 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/giantswarm/apiextensions/v6 v6.6.0/go.mod h1:Wgc2Rx8YAYF2HidjabEvhsj5 github.com/giantswarm/backoff v0.2.0/go.mod h1:Z3WRsFilSJ5H5VlFa4XhraoPr+9pmZgYasoY2OSfNOk= github.com/giantswarm/backoff v1.0.0 h1:1oeTvyPsm1tJrHlSmfxbIWuoCNWPOkWJCb8kfLvE2T0= github.com/giantswarm/backoff v1.0.0/go.mod h1:l/WqbggvG5Ndxxws0LUgVEvP5E82Qj5/PF8SMip/1QM= -github.com/giantswarm/badnodedetector/v2 v2.0.0 h1:o7db58Ndga62dPjXouUMzDZxGikpb/ut7csbHPXMWy8= -github.com/giantswarm/badnodedetector/v2 v2.0.0/go.mod h1:5GOyKly0ywEYs9se7lcuho98jK6Jgmq+vg4InGVEucU= +github.com/giantswarm/badnodedetector/v3 v3.0.0 h1:rhC+xkeM6E1+71E/S1b3DVOD+EtAuzaJiXeH67FS2HA= +github.com/giantswarm/badnodedetector/v3 v3.0.0/go.mod h1:bXi5UHMDDz2svvkcyo9fVvWXOApwRdAPiGw1ivkqJ1A= github.com/giantswarm/certs/v4 v4.0.0 h1:kaxovkDF2fiXxkLNB8WZ/RLyYdNctSfFM7VhNqka9t4= github.com/giantswarm/certs/v4 v4.0.0/go.mod h1:9hGD/dtmE8f/T9M70zaW8QeXHsawevSv7Xy70bMG51A= github.com/giantswarm/exporterkit v1.0.0 h1:6TwzJ5UklUp/xajBK3sx4gArFfMAAIEDs4/F1FvOCaE= diff --git a/service/controller/resource/terminateunhealthynode/create.go b/service/controller/resource/terminateunhealthynode/create.go index 60228f99da..36e46a644c 100644 --- a/service/controller/resource/terminateunhealthynode/create.go +++ b/service/controller/resource/terminateunhealthynode/create.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" - "github.com/giantswarm/badnodedetector/v2/pkg/detector" + "github.com/giantswarm/badnodedetector/v3/pkg/detector" "github.com/giantswarm/k8smetadata/pkg/annotation" "github.com/giantswarm/microerror" corev1 "k8s.io/api/core/v1" @@ -112,6 +112,9 @@ func (r *Resource) terminateNode(ctx context.Context, node corev1.Node, clusterI } // expose metric about node termination reportNodeTermination(clusterID, node.Name, instanceID) + + // emit event about node termination + r.event.Emit(ctx, &node, "UnhealthyNodeTerminated", fmt.Sprintf("node %q (instance %q) was unhealthy so it was terminated", node.Name, instanceID)) return nil } diff --git a/service/controller/resource/terminateunhealthynode/resource.go b/service/controller/resource/terminateunhealthynode/resource.go index 5faac897a6..e40e88c4f0 100644 --- a/service/controller/resource/terminateunhealthynode/resource.go +++ b/service/controller/resource/terminateunhealthynode/resource.go @@ -3,6 +3,8 @@ package terminateunhealthynode import ( "github.com/giantswarm/microerror" "github.com/giantswarm/micrologger" + + event "github.com/giantswarm/aws-operator/v14/service/internal/recorder" ) const ( @@ -10,19 +12,25 @@ const ( ) type Config struct { + Event event.Interface Logger micrologger.Logger } type Resource struct { + event event.Interface logger micrologger.Logger } func New(config Config) (*Resource, error) { + if config.Event == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.Event must not be empty", config) + } if config.Logger == nil { return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config) } r := &Resource{ + event: config.Event, logger: config.Logger, } diff --git a/service/controller/terminate_unhealthy_node.go b/service/controller/terminate_unhealthy_node.go index 73b2726332..ded6bb7f53 100644 --- a/service/controller/terminate_unhealthy_node.go +++ b/service/controller/terminate_unhealthy_node.go @@ -26,12 +26,14 @@ import ( "github.com/giantswarm/aws-operator/v14/service/controller/resource/tenantclients" "github.com/giantswarm/aws-operator/v14/service/controller/resource/terminateunhealthynode" "github.com/giantswarm/aws-operator/v14/service/internal/locker" + event "github.com/giantswarm/aws-operator/v14/service/internal/recorder" ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" ) type TerminateUnhealthyNodeConfig struct { K8sClient k8sclient.Interface + Event event.Interface Locker locker.Interface Logger micrologger.Logger @@ -45,6 +47,9 @@ type TerminateUnhealthyNode struct { func NewTerminateUnhealthyNode(config TerminateUnhealthyNodeConfig) (*TerminateUnhealthyNode, error) { var err error + if config.Event == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.Event must not be empty", config) + } if config.K8sClient == nil { return nil, microerror.Maskf(invalidConfigError, "%T.K8sClient must not be empty", config) } @@ -181,6 +186,7 @@ func newTerminateUnhealthyNodeResources(config TerminateUnhealthyNodeConfig) ([] var terminateUnhealthyNodeResource resource.Interface { c := terminateunhealthynode.Config{ + Event: config.Event, Logger: config.Logger, } diff --git a/service/service.go b/service/service.go index d0728d548f..18fd12a286 100644 --- a/service/service.go +++ b/service/service.go @@ -384,6 +384,7 @@ func New(config Config) (*Service, error) { { c := controller.TerminateUnhealthyNodeConfig{ K8sClient: k8sClient, + Event: event, Locker: kubeLockLocker, Logger: config.Logger,