Address PR comments

Signed-off-by: Emmanuel Lodovice <[email protected]>

alertobserver/alertobserver.go

@@ -18,14 +18,15 @@ import (
 )
 
 const (
-	EventAlertReceived          string = "received"
-	EventAlertRejected          string = "rejected"
-	EventAlertAddedToAggrGroup  string = "addedAggrGroup"
-	EventAlertPipelineStart     string = "pipelineStart"
-	EventAlertPipelinePassStage string = "pipelinePassStage"
-	EventAlertMuted             string = "muted"
-	EventAlertSent              string = "sent"
-	EventAlertSendFailed        string = "sendFailed"
+	EventAlertReceived             string = "received"
+	EventAlertRejected             string = "rejected"
+	EventAlertAddedToAggrGroup     string = "addedAggrGroup"
+	EventAlertFailedAddToAggrGroup string = "failedAddAggrGroup"
+	EventAlertPipelineStart        string = "pipelineStart"
+	EventAlertPipelinePassStage    string = "pipelinePassStage"
+	EventAlertMuted                string = "muted"
+	EventAlertSent                 string = "sent"
+	EventAlertSendFailed           string = "sendFailed"
 )
 
 type AlertEventMeta map[string]interface{}

dispatch/dispatch.go

@@ -332,7 +332,11 @@ func (d *Dispatcher) processAlert(alert *types.Alert, route *Route) {
 	// If the group does not exist, create it. But check the limit first.
 	if limit := d.limits.MaxNumberOfAggregationGroups(); limit > 0 && d.aggrGroupsNum >= limit {
 		d.metrics.aggrGroupLimitReached.Inc()
-		level.Error(d.logger).Log("msg", "Too many aggregation groups, cannot create new group for alert", "groups", d.aggrGroupsNum, "limit", limit, "alert", alert.Name())
+		errMsg := "Too many aggregation groups, cannot create new group for alert"
+		level.Error(d.logger).Log("msg", errMsg, "groups", d.aggrGroupsNum, "limit", limit, "alert", alert.Name())
+		if d.alertLCObserver != nil {
+			d.alertLCObserver.Observe(alertobserver.EventAlertFailedAddToAggrGroup, []*types.Alert{alert}, alertobserver.AlertEventMeta{"msg": errMsg})
+		}
 		return
 	}
 

dispatch/dispatch_test.go

@@ -596,7 +596,8 @@ route:
 	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*types.Alert)}
 	m := NewDispatcherMetrics(true, prometheus.NewRegistry())
 	observer := alertobserver.NewFakeLifeCycleObserver()
-	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, nil, logger, m, observer)
+	lim := limits{groups: 1}
+	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, lim, logger, m, observer)
 	go dispatcher.Run()
 	defer dispatcher.Stop()
 
@@ -617,6 +618,17 @@ route:
 	groupFp := getGroupLabels(inputAlerts[0], route).Fingerprint()
 	groupKey := dispatcher.aggrGroupsPerRoute[route][groupFp].GroupKey()
 	require.Equal(t, groupKey, observer.MetaPerEvent[alertobserver.EventAlertAddedToAggrGroup][0]["groupKey"].(string))
+
+	inputAlerts = []*types.Alert{
+		newAlert(model.LabelSet{"alertname": "YetAnotherAlert", "cluster": "cc", "service": "db"}),
+	}
+	err = alerts.Put(inputAlerts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(400 * time.Millisecond)
+	require.Equal(t, 1, len(observer.AlertsPerEvent[alertobserver.EventAlertFailedAddToAggrGroup]))
+	require.Equal(t, inputAlerts[0].Fingerprint(), observer.AlertsPerEvent[alertobserver.EventAlertFailedAddToAggrGroup][0].Fingerprint())
 }
 
 type recordStage struct {