Diskfull (#3643)

* Bump `badnodedetector` to be able to use `node-problem-detector` app for unhealthy node termination

* Bump `badnodedetector` to be able to use `node-problem-detector` app for unhealthy node termination

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Add `global.podSecurityStandards.enforced` value for PSS migration.
+- Emit event when an unhealthy node is terminated.
+- Bump `badnodedetector` to be able to use `node-problem-detector` app for unhealthy node termination.
 
 ## [14.23.0] - 2023-10-04
 

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/ghodss/yaml v1.0.1-0.20220118164431-d8423dcdf344
 	github.com/giantswarm/apiextensions/v6 v6.6.0
 	github.com/giantswarm/backoff v1.0.0
-	github.com/giantswarm/badnodedetector/v2 v2.0.0
+	github.com/giantswarm/badnodedetector/v3 v3.0.0
 	github.com/giantswarm/certs/v4 v4.0.0
 	github.com/giantswarm/ipam v0.3.0
 	github.com/giantswarm/k8sclient/v7 v7.0.1

go.sum

@@ -223,8 +223,8 @@ github.com/giantswarm/apiextensions/v6 v6.6.0/go.mod h1:Wgc2Rx8YAYF2HidjabEvhsj5
 github.com/giantswarm/backoff v0.2.0/go.mod h1:Z3WRsFilSJ5H5VlFa4XhraoPr+9pmZgYasoY2OSfNOk=
 github.com/giantswarm/backoff v1.0.0 h1:1oeTvyPsm1tJrHlSmfxbIWuoCNWPOkWJCb8kfLvE2T0=
 github.com/giantswarm/backoff v1.0.0/go.mod h1:l/WqbggvG5Ndxxws0LUgVEvP5E82Qj5/PF8SMip/1QM=
-github.com/giantswarm/badnodedetector/v2 v2.0.0 h1:o7db58Ndga62dPjXouUMzDZxGikpb/ut7csbHPXMWy8=
-github.com/giantswarm/badnodedetector/v2 v2.0.0/go.mod h1:5GOyKly0ywEYs9se7lcuho98jK6Jgmq+vg4InGVEucU=
+github.com/giantswarm/badnodedetector/v3 v3.0.0 h1:rhC+xkeM6E1+71E/S1b3DVOD+EtAuzaJiXeH67FS2HA=
+github.com/giantswarm/badnodedetector/v3 v3.0.0/go.mod h1:bXi5UHMDDz2svvkcyo9fVvWXOApwRdAPiGw1ivkqJ1A=
 github.com/giantswarm/certs/v4 v4.0.0 h1:kaxovkDF2fiXxkLNB8WZ/RLyYdNctSfFM7VhNqka9t4=
 github.com/giantswarm/certs/v4 v4.0.0/go.mod h1:9hGD/dtmE8f/T9M70zaW8QeXHsawevSv7Xy70bMG51A=
 github.com/giantswarm/exporterkit v1.0.0 h1:6TwzJ5UklUp/xajBK3sx4gArFfMAAIEDs4/F1FvOCaE=

service/controller/resource/terminateunhealthynode/create.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
-	"github.com/giantswarm/badnodedetector/v2/pkg/detector"
+	"github.com/giantswarm/badnodedetector/v3/pkg/detector"
 	"github.com/giantswarm/k8smetadata/pkg/annotation"
 	"github.com/giantswarm/microerror"
 	corev1 "k8s.io/api/core/v1"
@@ -112,6 +112,9 @@ func (r *Resource) terminateNode(ctx context.Context, node corev1.Node, clusterI
 	}
 	// expose metric about node termination
 	reportNodeTermination(clusterID, node.Name, instanceID)
+
+	// emit event about node termination
+	r.event.Emit(ctx, &node, "UnhealthyNodeTerminated", fmt.Sprintf("node %q (instance %q) was unhealthy so it was terminated", node.Name, instanceID))
 	return nil
 }
 

service/controller/resource/terminateunhealthynode/resource.go

@@ -3,26 +3,34 @@ package terminateunhealthynode
 import (
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/micrologger"
+
+	event "github.com/giantswarm/aws-operator/v14/service/internal/recorder"
 )
 
 const (
 	Name = "terminateunhealthynode"
 )
 
 type Config struct {
+	Event  event.Interface
 	Logger micrologger.Logger
 }
 
 type Resource struct {
+	event  event.Interface
 	logger micrologger.Logger
 }
 
 func New(config Config) (*Resource, error) {
+	if config.Event == nil {
+		return nil, microerror.Maskf(invalidConfigError, "%T.Event must not be empty", config)
+	}
 	if config.Logger == nil {
 		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
 	}
 
 	r := &Resource{
+		event:  config.Event,
 		logger: config.Logger,
 	}
 

service/controller/terminate_unhealthy_node.go

@@ -26,12 +26,14 @@ import (
 	"github.com/giantswarm/aws-operator/v14/service/controller/resource/tenantclients"
 	"github.com/giantswarm/aws-operator/v14/service/controller/resource/terminateunhealthynode"
 	"github.com/giantswarm/aws-operator/v14/service/internal/locker"
+	event "github.com/giantswarm/aws-operator/v14/service/internal/recorder"
 
 	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type TerminateUnhealthyNodeConfig struct {
 	K8sClient k8sclient.Interface
+	Event     event.Interface
 	Locker    locker.Interface
 	Logger    micrologger.Logger
 
@@ -45,6 +47,9 @@ type TerminateUnhealthyNode struct {
 func NewTerminateUnhealthyNode(config TerminateUnhealthyNodeConfig) (*TerminateUnhealthyNode, error) {
 	var err error
 
+	if config.Event == nil {
+		return nil, microerror.Maskf(invalidConfigError, "%T.Event must not be empty", config)
+	}
 	if config.K8sClient == nil {
 		return nil, microerror.Maskf(invalidConfigError, "%T.K8sClient must not be empty", config)
 	}
@@ -181,6 +186,7 @@ func newTerminateUnhealthyNodeResources(config TerminateUnhealthyNodeConfig) ([]
 	var terminateUnhealthyNodeResource resource.Interface
 	{
 		c := terminateunhealthynode.Config{
+			Event:  config.Event,
 			Logger: config.Logger,
 		}
 

service/service.go

@@ -384,6 +384,7 @@ func New(config Config) (*Service, error) {
 	{
 		c := controller.TerminateUnhealthyNodeConfig{
 			K8sClient: k8sClient,
+			Event:     event,
 			Locker:    kubeLockLocker,
 			Logger:    config.Logger,