[CHORE] recreating alert if not found in remote (#111)

Signed-off-by: Nicolas Takashi <[email protected]>

apis/coralogix/v1alpha1/alert_types.go

@@ -31,6 +31,7 @@ import (
 	"google.golang.org/protobuf/types/known/wrapperspb"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
@@ -3094,6 +3095,12 @@ type AlertStatus struct {
 	AlertType AlertType `json:"alertType,omitempty"`
 }
 
+func NewDefaultAlertStatus() *AlertStatus {
+	return &AlertStatus{
+		ID: ptr.To(""),
+	}
+}
+
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 //+kubebuilder:storageversion

controllers/alphacontrollers/alert_controller.go

@@ -30,6 +30,7 @@ import (
 	"google.golang.org/protobuf/types/known/wrapperspb"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -67,9 +68,7 @@ type AlertReconciler struct {
 
 func (r *AlertReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	var (
-		resultError ctrl.Result = ctrl.Result{RequeueAfter: defaultErrRequeuePeriod}
-		resultOk    ctrl.Result = ctrl.Result{RequeueAfter: defaultRequeuePeriod}
-		err         error
+		err error
 	)
 
 	log := log.FromContext(ctx).WithValues(
@@ -87,34 +86,34 @@ func (r *AlertReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 			// Return and don't requeue
 			return ctrl.Result{}, nil
 		}
-		return resultError, err
+		return ctrl.Result{RequeueAfter: defaultErrRequeuePeriod}, err
 	}
 
-	if alert.Status.ID == nil {
+	if ptr.Deref(alert.Status.ID, "") == "" {
 		err = r.create(ctx, log, alert)
 		if err != nil {
 			log.Error(err, "Error on creating alert")
-			return resultError, err
+			return ctrl.Result{RequeueAfter: defaultErrRequeuePeriod}, err
 		}
-		return resultOk, nil
+		return ctrl.Result{}, nil
 	}
 
 	if !alert.ObjectMeta.DeletionTimestamp.IsZero() {
 		err = r.delete(ctx, log, alert)
 		if err != nil {
 			log.Error(err, "Error on deleting alert")
-			return resultError, err
+			return ctrl.Result{RequeueAfter: defaultErrRequeuePeriod}, err
 		}
-		return resultOk, nil
+		return ctrl.Result{}, nil
 	}
 
 	err = r.update(ctx, log, alert)
 	if err != nil {
 		log.Error(err, "Error on updating alert")
-		return resultError, err
+		return ctrl.Result{RequeueAfter: defaultErrRequeuePeriod}, err
 	}
 
-	return resultOk, nil
+	return ctrl.Result{}, nil
 }
 
 func (r *AlertReconciler) update(ctx context.Context,
@@ -125,6 +124,15 @@ func (r *AlertReconciler) update(ctx context.Context,
 	})
 
 	if err != nil {
+		if status.Code(err) == codes.NotFound {
+			log.Info("alert not found on remote, recreating it")
+			alert.Status = *coralogixv1alpha1.NewDefaultAlertStatus()
+			if err := r.Status().Update(ctx, alert); err != nil {
+				log.Error(err, "Error on updating alert status")
+				return err
+			}
+			return err
+		}
 		log.Error(err, "Error on getting alert")
 		return err
 	}

controllers/alphacontrollers/alert_controller_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/coralogix/coralogix-operator/controllers/mock_clientset"
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/mock/gomock"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -232,7 +234,6 @@ func TestAlertCreation(t *testing.T) {
 				assert.Equal(t, defaultErrRequeuePeriod, result.RequeueAfter)
 			} else {
 				assert.NoError(t, err)
-				assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 			}
 		})
 	}
@@ -370,6 +371,65 @@ func TestAlertUpdate(t *testing.T) {
 					MinTimes(1).MaxTimes(1)
 			},
 		},
+		{
+			name:       "Alert update clean status if not found in remote",
+			shouldFail: true,
+			alert: &coralogixv1alpha1.Alert{
+				TypeMeta:   metav1.TypeMeta{Kind: "Alert", APIVersion: "coralogix.com/v1alpha1"},
+				ObjectMeta: metav1.ObjectMeta{Name: "alert-update-clean-status", Namespace: "default"},
+				Spec: coralogixv1alpha1.AlertSpec{
+					Name:               "AlertUpdateCleanStatus",
+					Description:        "AlertUpdateCleanStatus",
+					Active:             true,
+					Severity:           alertProtoSeverityToSchemaSeverity[alerts.AlertSeverity_ALERT_SEVERITY_CRITICAL],
+					NotificationGroups: defaultNotificationGroups,
+					PayloadFilters:     []string{"filter"},
+					AlertType:          defaultAlertType,
+				},
+				Status: coralogixv1alpha1.AlertStatus{
+					ID:          pointer.String("AlertUpdateCleanStatus"),
+					Name:        "AlertUpdateCleanStatus",
+					Description: "AlertUpdateCleanStatus",
+					Active:      true,
+					Severity:    "Critical",
+				},
+			},
+			remoteAlert: &alerts.Alert{
+				UniqueIdentifier: wrapperspb.String("AlertUpdateCleanStatus"),
+				Name:             wrapperspb.String("AlertUpdateCleanStatus"),
+				Description:      wrapperspb.String("AlertUpdateCleanStatus"),
+				IsActive:         wrapperspb.Bool(true),
+				Severity:         alerts.AlertSeverity_ALERT_SEVERITY_CRITICAL,
+				MetaLabels: []*alerts.MetaLabel{
+					{Key: wrapperspb.String("key"), Value: wrapperspb.String("value")},
+					{Key: wrapperspb.String("managed-by"), Value: wrapperspb.String("coralogix-operator")},
+				},
+				Condition:          defaultRemoteCondition,
+				NotificationGroups: defaultRemoteNotificationGroups,
+				Filters: &alerts.AlertFilters{
+					FilterType: alerts.AlertFilters_FILTER_TYPE_METRIC,
+				},
+				NotificationPayloadFilters: []*wrapperspb.StringValue{wrapperspb.String("filter")},
+			},
+			prepare: func(params PrepareParams) {
+				params.alertsClient.EXPECT().
+					GetAlert(params.alert.Namespace, coralogixv1alpha1.NewAlert()).
+					Return(&alerts.GetAlertByUniqueIdResponse{Alert: params.remoteAlert}, nil).
+					MinTimes(1).MaxTimes(1)
+
+				params.alertsClient.EXPECT().CreateAlert(params.ctx, gomock.Any()).
+					Return(&alerts.CreateAlertResponse{Alert: params.remoteAlert}, nil).
+					MinTimes(1).MaxTimes(1)
+
+				params.alertsClient.EXPECT().UpdateAlert(params.ctx, gomock.Any()).
+					Return(&alerts.UpdateAlertByUniqueIdResponse{Alert: params.remoteAlert}, nil).
+					MinTimes(1).MaxTimes(1)
+
+				params.alertsClient.EXPECT().GetAlert(params.ctx, gomock.Any()).
+					Return(&alerts.GetAlertByUniqueIdResponse{Alert: params.remoteAlert}, status.Error(codes.NotFound, "")).
+					MinTimes(1).MaxTimes(1)
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -443,7 +503,6 @@ func TestAlertUpdate(t *testing.T) {
 				assert.Equal(t, defaultErrRequeuePeriod, result.RequeueAfter)
 			} else {
 				assert.NoError(t, err)
-				assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 			}
 		})
 	}
@@ -654,7 +713,6 @@ func TestAlertDelete(t *testing.T) {
 				assert.Equal(t, defaultErrRequeuePeriod, result.RequeueAfter)
 			} else {
 				assert.NoError(t, err)
-				assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 			}
 		})
 	}

controllers/alphacontrollers/recordingrulegroupset_controller.go

@@ -167,7 +167,7 @@ func (r *RecordingRuleGroupSetReconciler) Reconcile(ctx context.Context, req ctr
 			if err := r.Status().Update(ctx, ruleGroupSetCRD); err != nil {
 				log.V(1).Error(err, "updating crd")
 			}
-			return ctrl.Result{RequeueAfter: defaultRequeuePeriod}, nil
+			return ctrl.Result{}, nil
 		} else {
 			log.Error(err, "Received an error while creating a RecordingRuleGroupSet", "recordingRuleGroupSet", createRuleGroupReq)
 			return ctrl.Result{RequeueAfter: defaultErrRequeuePeriod}, err
@@ -203,7 +203,7 @@ func (r *RecordingRuleGroupSetReconciler) Reconcile(ctx context.Context, req ctr
 		}
 	}
 
-	return ctrl.Result{RequeueAfter: defaultRequeuePeriod}, nil
+	return ctrl.Result{}, nil
 }
 
 func flattenRecordingRuleGroupSet(set *rrg.OutRuleGroupSet) coralogixv1alpha1.RecordingRuleGroupSetStatus {

controllers/alphacontrollers/recordingrulegroupset_controller_test.go

@@ -123,7 +123,6 @@ func TestRecordingRuleGroupSetReconciler_Reconcile(t *testing.T) {
 
 	result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 	assert.NoError(t, err)
-	assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 
 	namespacedName := types.NamespacedName{Namespace: "default", Name: "test"}
 	actualRecordingRuleGroupSetCRD := &coralogixv1alpha1.RecordingRuleGroupSet{}
@@ -191,7 +190,6 @@ func TestRecordingRuleGroupSetReconciler_Reconcile_5XX_StatusError(t *testing.T)
 
 	result, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 	assert.NoError(t, err)
-	assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 
 	namespacedName := types.NamespacedName{Namespace: "default", Name: "test"}
 	actualRecordingRuleGroupSetCRD := &coralogixv1alpha1.RecordingRuleGroupSet{}

controllers/alphacontrollers/rulegroup_controller.go

@@ -180,7 +180,7 @@ func (r *RuleGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			if err := r.Status().Update(ctx, ruleGroupCRD); err != nil {
 				log.V(1).Error(err, "updating crd")
 			}
-			return ctrl.Result{RequeueAfter: defaultRequeuePeriod}, nil
+			return ctrl.Result{}, nil
 		} else {
 			log.Error(err, "Received an error while creating a Rule-Group", "ruleGroup", createRuleGroupReq)
 			return ctrl.Result{RequeueAfter: defaultErrRequeuePeriod}, err
@@ -202,7 +202,7 @@ func (r *RuleGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		log.V(1).Info("Rule-Group was updated", "ruleGroup", jstr)
 	}
 
-	return ctrl.Result{RequeueAfter: defaultRequeuePeriod}, nil
+	return ctrl.Result{}, nil
 }
 
 func flattenRuleGroup(ruleGroup *rulesgroups.RuleGroup) (*coralogixv1alpha1.RuleGroupStatus, error) {

controllers/alphacontrollers/rulegroup_controller_test.go

@@ -202,7 +202,6 @@ func TestRuleGroupReconciler_Reconcile(t *testing.T) {
 
 	result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 	assert.NoError(t, err)
-	assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 
 	namespacedName := types.NamespacedName{Namespace: "default", Name: "test"}
 	actualRuleGroupCRD := &coralogixv1alpha1.RuleGroup{}
@@ -270,7 +269,6 @@ func TestRuleGroupReconciler_Reconcile_5XX_StatusError(t *testing.T) {
 
 	result, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 	assert.NoError(t, err)
-	assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 
 	namespacedName := types.NamespacedName{Namespace: "default", Name: "test"}
 	actualRuleGroupCRD := &coralogixv1alpha1.RuleGroup{}

controllers/alphacontrollers/utils.go

@@ -5,6 +5,5 @@ import (
 )
 
 const (
-	defaultRequeuePeriod    = 30 * time.Second
-	defaultErrRequeuePeriod = 20 * time.Second
+	defaultErrRequeuePeriod = 30 * time.Second
 )

controllers/prometheusrule_controller_test.go

@@ -87,14 +87,13 @@ func TestPrometheusRuleReconciler_Reconcile(t *testing.T) {
 	<-prometheusRulesWatcher.ResultChan()
 	prometheusRuleCrd := &prometheus.PrometheusRule{}
 	prometheusRuleReconciler.Client.Get(ctx, types.NamespacedName{Namespace: "default", Name: "test"}, prometheusRuleCrd)
-	result, err := prometheusRuleReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
+	_, err = prometheusRuleReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 	assert.NoError(t, err)
-	assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 
 	<-recordingRuleGroupWatcher.ResultChan()
-	result, err = recordingRuleGroupReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
+	_, err = recordingRuleGroupReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 	assert.NoError(t, err)
-	assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
+
 	actualRecordingRuleGroupSetCRD := &coralogixv1alpha1.RecordingRuleGroupSet{}
 	err = recordingRuleGroupReconciler.Client.Get(ctx, types.NamespacedName{Namespace: "default", Name: "test"}, actualRecordingRuleGroupSetCRD)
 	assert.NoError(t, err)
@@ -117,15 +116,14 @@ func TestPrometheusRuleReconciler_Reconcile(t *testing.T) {
 	err = prometheusRuleReconciler.Client.Delete(ctx, prometheusRuleCrd)
 	assert.NoError(t, err)
 	<-prometheusRulesWatcher.ResultChan()
-	result, err = prometheusRuleReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
+	_, err = prometheusRuleReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 	assert.NoError(t, err)
 
-	assert.Equal(t, defaultRequeuePeriod, result.RequeueAfter)
 	err = prometheusRuleReconciler.Client.Get(ctx, types.NamespacedName{Namespace: "default", Name: "test"}, prometheusRuleCrd)
 	assert.Error(t, err)
 
 	for event := <-recordingRuleGroupWatcher.ResultChan(); event.Type != watch.Deleted; event = <-recordingRuleGroupWatcher.ResultChan() {
-		result, err = recordingRuleGroupReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
+		_, err = recordingRuleGroupReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "test"}})
 		assert.NoError(t, err)
 	}
 
@@ -135,7 +133,7 @@ func TestPrometheusRuleReconciler_Reconcile(t *testing.T) {
 
 	for deletedAlertsEvents := 0; deletedAlertsEvents < 2; {
 		event := <-alertsWatcher.ResultChan()
-		result, err = alertReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: event.Object.(*coralogixv1alpha1.Alert).Name}})
+		_, err = alertReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: event.Object.(*coralogixv1alpha1.Alert).Name}})
 		if event.Type == watch.Deleted {
 			deletedAlertsEvents++
 		}

go.mod

@@ -4,7 +4,7 @@ go 1.22
 
 require (
 	github.com/go-logr/logr v1.2.4
-	github.com/golang/protobuf v1.5.3
+	github.com/golang/protobuf v1.5.4
 	github.com/grafana/grafana-api-golang-client v0.18.3
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/onsi/ginkgo/v2 v2.9.5
@@ -13,8 +13,8 @@ require (
 	github.com/stretchr/testify v1.8.2
 	go.uber.org/mock v0.3.0
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
-	google.golang.org/grpc v1.53.0
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/grpc v1.65.0
+	google.golang.org/protobuf v1.34.1
 	k8s.io/apimachinery v0.27.2
 	k8s.io/client-go v0.27.2
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
@@ -23,7 +23,7 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -36,10 +36,10 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -59,15 +59,14 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/net v0.16.0 // indirect
-	golang.org/x/oauth2 v0.6.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/oauth2 v0.20.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect