Ignore first NODATA - OK event (#113)

* What a beautiful code I have here. It would be shame if it doesn't work... * WTF... * I think, this hack should work * I win this round... No NODATA - OK anymore. * Fix tests * I ain't the sharpest tool in the shed * Fix EventTimeStamp in empty lastCheck * Add new CheckData field, remove duck tape * Trying to speed up tests * Refactoring NODATA - OK logic * Revert "Trying to speed up tests" This reverts commit cac064b. * fix bug * refactor GetOrCreateMetricState method, remove unnecessary assignments * Add tests
moira-alert · Sep 3, 2018 · 0d5babf · 0d5babf
1 parent ce0c988
commit 0d5babf
Show file tree

Hide file tree

Showing 10 changed files with 330 additions and 170 deletions.
diff --git a/api/dto/triggers.go b/api/dto/triggers.go
@@ -60,45 +60,49 @@ type TriggerModel struct {
 	Patterns []string `json:"patterns"`
 	// Shows if trigger is remote (graphite-backend) based or stored inside Moira-Redis DB
 	IsRemote bool `json:"is_remote"`
+	// If false, first event NODATA → OK will be omitted
+	NotifyAboutNewMetrics bool `json:"notify_about_new_metrics"`
 }
 
 // ToMoiraTrigger transforms TriggerModel to moira.Trigger
 func (model *TriggerModel) ToMoiraTrigger() *moira.Trigger {
 	return &moira.Trigger{
-		ID:          model.ID,
-		Name:        model.Name,
-		Desc:        model.Desc,
-		Targets:     model.Targets,
-		WarnValue:   model.WarnValue,
-		ErrorValue:  model.ErrorValue,
-		TriggerType: model.TriggerType,
-		Tags:        model.Tags,
-		TTLState:    model.TTLState,
-		TTL:         model.TTL,
-		Schedule:    model.Schedule,
-		Expression:  &model.Expression,
-		Patterns:    model.Patterns,
-		IsRemote:    model.IsRemote,
+		ID:                    model.ID,
+		Name:                  model.Name,
+		Desc:                  model.Desc,
+		Targets:               model.Targets,
+		WarnValue:             model.WarnValue,
+		ErrorValue:            model.ErrorValue,
+		TriggerType:           model.TriggerType,
+		Tags:                  model.Tags,
+		TTLState:              model.TTLState,
+		TTL:                   model.TTL,
+		Schedule:              model.Schedule,
+		Expression:            &model.Expression,
+		Patterns:              model.Patterns,
+		IsRemote:              model.IsRemote,
+		NotifyAboutNewMetrics: model.NotifyAboutNewMetrics,
 	}
 }
 
 // CreateTriggerModel transforms moira.Trigger to TriggerModel
 func CreateTriggerModel(trigger *moira.Trigger) TriggerModel {
 	return TriggerModel{
-		ID:          trigger.ID,
-		Name:        trigger.Name,
-		Desc:        trigger.Desc,
-		Targets:     trigger.Targets,
-		WarnValue:   trigger.WarnValue,
-		ErrorValue:  trigger.ErrorValue,
-		TriggerType: trigger.TriggerType,
-		Tags:        trigger.Tags,
-		TTLState:    trigger.TTLState,
-		TTL:         trigger.TTL,
-		Schedule:    trigger.Schedule,
-		Expression:  moira.UseString(trigger.Expression),
-		Patterns:    trigger.Patterns,
-		IsRemote:    trigger.IsRemote,
+		ID:                    trigger.ID,
+		Name:                  trigger.Name,
+		Desc:                  trigger.Desc,
+		Targets:               trigger.Targets,
+		WarnValue:             trigger.WarnValue,
+		ErrorValue:            trigger.ErrorValue,
+		TriggerType:           trigger.TriggerType,
+		Tags:                  trigger.Tags,
+		TTLState:              trigger.TTLState,
+		TTL:                   trigger.TTL,
+		Schedule:              trigger.Schedule,
+		Expression:            moira.UseString(trigger.Expression),
+		Patterns:              trigger.Patterns,
+		IsRemote:              trigger.IsRemote,
+		NotifyAboutNewMetrics: trigger.NotifyAboutNewMetrics,
 	}
 }
 

diff --git a/checker/check.go b/checker/check.go
@@ -65,6 +65,9 @@ func (triggerChecker *TriggerChecker) handleMetricsCheck() (moira.CheckData, err
 		Timestamp:      triggerChecker.Until,
 		EventTimestamp: triggerChecker.lastCheck.EventTimestamp,
 		Score:          triggerChecker.lastCheck.Score,
+		TriggerAlreadyProcessed: triggerChecker.lastCheck.TriggerAlreadyProcessed,
+		Suppressed:              triggerChecker.lastCheck.Suppressed,
+		SuppressedState:         triggerChecker.lastCheck.SuppressedState,
 	}
 
 	var triggerTimeSeries *triggerTimeSeries
@@ -128,7 +131,7 @@ func (triggerChecker *TriggerChecker) handleMetricsCheck() (moira.CheckData, err
 }
 
 func (triggerChecker *TriggerChecker) checkTimeSeries(timeSeries *target.TimeSeries, triggerTimeSeries *triggerTimeSeries) (lastState moira.MetricState, needToDeleteMetric bool, err error) {
-	lastState = triggerChecker.lastCheck.GetOrCreateMetricState(timeSeries.Name, timeSeries.StartTime-3600)
+	lastState = triggerChecker.lastCheck.GetOrCreateMetricState(timeSeries.Name, timeSeries.StartTime-3600, triggerChecker.trigger.NotifyAboutNewMetrics)
 	metricStates, err := triggerChecker.getTimeSeriesStepsStates(triggerTimeSeries, timeSeries, lastState)
 	if err != nil {
 		return
@@ -152,6 +155,10 @@ func (triggerChecker *TriggerChecker) checkTimeSeries(timeSeries *target.TimeSer
 func (triggerChecker *TriggerChecker) handleTriggerCheck(checkData moira.CheckData, checkingError error) (moira.CheckData, error) {
 	if checkingError == nil {
 		checkData.State = OK
+		if !checkData.TriggerAlreadyProcessed {
+			checkData.TriggerAlreadyProcessed = true
+			return checkData, nil
+		}
 		return triggerChecker.compareTriggerStates(checkData)
 	}