From d13cc7342c232c41d80090f7a647dd09920139d2 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Mon, 8 Jun 2020 16:03:19 +0500 Subject: [PATCH] feature: update templates * Create new package for templating * Added functions and methods for templating * Added recover to notifier * Fixed the display of templates * Added error display in messages if they could not parse templates --- api/dto/triggers.go | 20 +++++ api/handler/trigger.go | 43 ++++++----- api/handler/triggers.go | 9 +++ datatypes.go | 66 ++++++---------- datatypes_test.go | 42 ---------- go.mod | 2 +- go.sum | 2 +- notifier/notifier.go | 10 ++- templating/templating.go | 78 +++++++++++++++++++ templating/templating_test.go | 140 ++++++++++++++++++++++++++++++++++ 10 files changed, 305 insertions(+), 107 deletions(-) create mode 100644 templating/templating.go create mode 100644 templating/templating_test.go diff --git a/api/dto/triggers.go b/api/dto/triggers.go index 00b205d78..571429c2a 100644 --- a/api/dto/triggers.go +++ b/api/dto/triggers.go @@ -8,6 +8,8 @@ import ( "strconv" "time" + "github.com/moira-alert/moira/templating" + "github.com/moira-alert/moira" "github.com/moira-alert/moira/api" "github.com/moira-alert/moira/api/middleware" @@ -174,10 +176,12 @@ func (trigger *Trigger) Bind(request *http.Request) error { return api.ErrInvalidRequestContent{ValidationError: fmt.Errorf("pattern \"*\" is not allowed to use")} } } + middleware.SetTimeSeriesNames(request, metricsDataNames) if _, err := triggerExpression.Evaluate(); err != nil { return err } + return nil } @@ -312,6 +316,22 @@ func (*Trigger) Render(w http.ResponseWriter, r *http.Request) error { return nil } +func (trigger *Trigger) PopulatedDescription(events moira.NotificationEvents) error { + if trigger.Desc == nil { + return nil + } + + templatingEvents := moira.NotificationEventsToTemplatingEvents(events) + description, err := templating.Populate(trigger.Name, *trigger.Desc, templatingEvents) + if err != nil { + return fmt.Errorf("you have an error in your Go template: %v", err) + } + + *trigger.Desc = description + + return nil +} + type TriggerCheck struct { *moira.CheckData TriggerID string `json:"trigger_id"` diff --git a/api/handler/trigger.go b/api/handler/trigger.go index eae09ed34..8f9f8e68e 100644 --- a/api/handler/trigger.go +++ b/api/handler/trigger.go @@ -7,7 +7,6 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/render" - "github.com/moira-alert/moira" "github.com/moira-alert/moira/metric_source/local" "github.com/moira-alert/moira/metric_source/remote" @@ -54,16 +53,11 @@ func updateTrigger(writer http.ResponseWriter, request *http.Request) { render.Render(writer, request, api.ErrorInternalServer(err)) //nolint } - return - } - - if trigger.Desc != nil { - triggerData := moira.TriggerData{Desc: *trigger.Desc, Name: trigger.Name} - if _, err := triggerData.GetPopulatedDescription(moira.NotificationEvents{}); err != nil { - render.Render(writer, request, api.ErrorRender( //nolint - fmt.Errorf("You have an error in your Go template: %v", err))) - return + if err := checkingTemplateFilling(request, *trigger); err != nil { + render.Render(writer, request, err) //nolint } + + return } timeSeriesNames := middleware.GetTimeSeriesNames(request) @@ -92,21 +86,15 @@ func getTrigger(writer http.ResponseWriter, request *http.Request) { if triggerID == "testlog" { panic("Test for multi line logs") } + trigger, err := controller.GetTrigger(database, triggerID) if err != nil { render.Render(writer, request, err) //nolint return } - if needToPopulate := middleware.GetPopulated(request); needToPopulate && trigger.Desc != nil { - triggerData := moira.TriggerData{Desc: *trigger.Desc, Name: trigger.Name} - - eventsList, err := controller.GetTriggerEvents(database, triggerID, 0, 3) - if err != nil { - render.Render(writer, request, err) //nolint - } - - *trigger.Desc, _ = triggerData.GetPopulatedDescription(eventsList.List) + if err := checkingTemplateFilling(request, *trigger); err != nil { + middleware.GetLoggerEntry(request).Warning(err) } if err := render.Render(writer, request, trigger); err != nil { @@ -114,6 +102,23 @@ func getTrigger(writer http.ResponseWriter, request *http.Request) { } } +func checkingTemplateFilling(request *http.Request, trigger dto.Trigger) *api.ErrorResponse { + if !middleware.GetPopulated(request) { + return nil + } + + eventsList, err := controller.GetTriggerEvents(database, trigger.ID, 0, 3) + if err != nil { + return err + } + + if err := trigger.PopulatedDescription(eventsList.List); err != nil { + return api.ErrorRender(err) + } + + return nil +} + func getTriggerState(writer http.ResponseWriter, request *http.Request) { triggerID := middleware.GetTriggerID(request) triggerState, err := controller.GetTriggerLastCheck(database, triggerID) diff --git a/api/handler/triggers.go b/api/handler/triggers.go index 4477cc5b1..27acaba2b 100644 --- a/api/handler/triggers.go +++ b/api/handler/triggers.go @@ -64,6 +64,15 @@ func createTrigger(writer http.ResponseWriter, request *http.Request) { } return } + + if trigger.Desc != nil { + err := trigger.PopulatedDescription(moira.NotificationEvents{{}}) + if err != nil { + render.Render(writer, request, api.ErrorRender(err)) //nolint + return + } + } + timeSeriesNames := middleware.GetTimeSeriesNames(request) response, err := controller.CreateTrigger(database, &trigger.TriggerModel, timeSeriesNames) if err != nil { diff --git a/datatypes.go b/datatypes.go index c966b2646..694136bf6 100644 --- a/datatypes.go +++ b/datatypes.go @@ -3,12 +3,13 @@ package moira import ( "bytes" "fmt" - "html/template" "math" "sort" "strconv" "strings" "time" + + "github.com/moira-alert/moira/templating" ) const ( @@ -106,21 +107,31 @@ func (event *NotificationEvent) CreateMessage(location *time.Location) string { // NotificationEvents represents slice of NotificationEvent type NotificationEvents []NotificationEvent -type templateData struct { - Trigger templateTrigger - Events []templateEvent -} +func (trigger *TriggerData) PopulatedDescription(events NotificationEvents) error { + description, err := templating.Populate(trigger.Name, trigger.Desc, NotificationEventsToTemplatingEvents(events)) + if err != nil { + description = "Your description is using the wrong template. Since we were unable to populate your template with " + + "data, we return it so you can parse it.\n\n" + trigger.Desc + } -type templateEvent struct { - Metric string - MetricElements []string - Timestamp int64 - Value *float64 - State State + trigger.Desc = description + + return err } -type templateTrigger struct { - Name string `json:"name"` +func NotificationEventsToTemplatingEvents(events NotificationEvents) []templating.Event { + templatingEvents := make([]templating.Event, 0, len(events)) + for _, event := range events { + templatingEvents = append(templatingEvents, templating.Event{ + Metric: event.Metric, + MetricElements: strings.Split(event.Metric, "."), + Timestamp: event.Timestamp, + State: string(event.State), + Value: event.Value, + }) + } + + return templatingEvents } // TriggerData represents trigger object @@ -135,35 +146,6 @@ type TriggerData struct { Tags []string `json:"__notifier_trigger_tags"` } -func (trigger TriggerData) GetPopulatedDescription(events NotificationEvents) (string, error) { - buffer := new(bytes.Buffer) - templateEvents := make([]templateEvent, 0, len(events)) - - for _, data := range events { - event := templateEvent{ - Metric: data.Metric, - MetricElements: strings.Split(data.Metric, "."), - Timestamp: data.Timestamp, - State: data.State, - Value: data.Value, - } - - templateEvents = append(templateEvents, event) - } - - dataToExecute := templateData{ - Trigger: templateTrigger{Name: trigger.Name}, - Events: templateEvents, - } - - triggerTemplate := template.Must(template.New("populate-description").Parse(trigger.Desc)) - if err := triggerTemplate.Execute(buffer, dataToExecute); err != nil { - return "", err - } - - return buffer.String(), nil -} - // GetTriggerURI gets frontUri and returns triggerUrl, returns empty string on selfcheck and test notifications func (trigger TriggerData) GetTriggerURI(frontURI string) string { if trigger.ID != "" { diff --git a/datatypes_test.go b/datatypes_test.go index 7119e18c8..fb4c25d10 100644 --- a/datatypes_test.go +++ b/datatypes_test.go @@ -246,48 +246,6 @@ func TestTriggerData_GetTags(t *testing.T) { }) } -func TestTriggerData_TemplateDescription(t *testing.T) { - Convey("Test templates", t, func() { - var trigger = TriggerData{Name: "TestName"} - trigger.Desc = "\n" + - "Trigger name: {{.Trigger.Name}}\n" + - "{{range $v := .Events }}\n" + - "Metric: {{$v.Metric}}\n" + - "MetricElements: {{$v.MetricElements}}\n" + - "Timestamp: {{$v.Timestamp}}\n" + - "Value: {{$v.Value}}\n" + - "State: {{$v.State}}\n" + - "{{end}}\n" + - "https://grafana.yourhost.com/some-dashboard{{ range $i, $v := .Events }}{{ if ne $i 0 }}&{{ else }}?{{ end }}var-host={{ $v.Metric }}{{ end }}\n" - - var data = NotificationEvents{{Metric: "1"}, {Metric: "2"}} - - Convey("Test nil data", func() { - expected, err := trigger.GetPopulatedDescription(nil) - So(err, ShouldBeNil) - So(` -Trigger name: TestName - -https://grafana.yourhost.com/some-dashboard -`, ShouldResemble, expected) - }) - - Convey("Test data", func() { - expected, err := trigger.GetPopulatedDescription(data) - So(err, ShouldBeNil) - So("\nTrigger name: TestName\n\nMetric: 1\nMetricElements: [1]\nTimestamp: 0\nValue: <nil>\nState: \n\nMetric: 2\nMetricElements: [2]\nTimestamp: 0\nValue: <nil>\nState: \n\nhttps://grafana.yourhost.com/some-dashboard?var-host=1&var-host=2\n", ShouldResemble, expected) - }) - - Convey("Test description without templates", func() { - anotherText := "Another text" - trigger.Desc = anotherText - expected, err := trigger.GetPopulatedDescription(data) - So(err, ShouldBeNil) - So(anotherText, ShouldEqual, expected) - }) - }) -} - func TestScheduledNotification_GetKey(t *testing.T) { Convey("Get key", t, func() { notification := ScheduledNotification{ diff --git a/go.mod b/go.mod index 8880e05f2..fb3b78209 100644 --- a/go.mod +++ b/go.mod @@ -111,4 +111,4 @@ require ( gopkg.in/yaml.v2 v2.3.0 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect honnef.co/go/tools v0.0.1-2020.1.5 // indirect -) +) \ No newline at end of file diff --git a/go.sum b/go.sum index 9852da2f1..a38b6b774 100644 --- a/go.sum +++ b/go.sum @@ -956,4 +956,4 @@ modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= \ No newline at end of file diff --git a/notifier/notifier.go b/notifier/notifier.go index 42436b939..f33a73e2e 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -154,7 +154,13 @@ func (notifier *StandardNotifier) resend(pkg *NotificationPackage, reason string } func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan NotificationPackage) { + defer func() { + if err := recover(); err != nil { + notifier.logger.Warningf("Panic notifier: %v, ", err) + } + }() defer notifier.waitGroup.Done() + for pkg := range ch { plots, err := notifier.buildNotificationPackagePlots(pkg) if err != nil { @@ -167,9 +173,9 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific } } - pkg.Trigger.Desc, err = pkg.Trigger.GetPopulatedDescription(pkg.Events) + err = pkg.Trigger.PopulatedDescription(pkg.Events) if err != nil { - notifier.logger.Errorf("Error populate description: %v", err) + notifier.logger.Warningf("Error populate description:\n%v", err) } err = sender.SendEvents(pkg.Events, pkg.Contact, pkg.Trigger, plots, pkg.Throttled) diff --git a/templating/templating.go b/templating/templating.go new file mode 100644 index 000000000..73ed233a1 --- /dev/null +++ b/templating/templating.go @@ -0,0 +1,78 @@ +package templating + +import ( + "bytes" + "fmt" + "html/template" + "strings" + "time" +) + +const eventTimeFormat = "2006-01-02 15:04:05" + +type notification struct { + Trigger trigger + Events []Event +} + +type Event struct { + Metric string + MetricElements []string + Timestamp int64 + Value *float64 + State string +} + +func date(unixTime int64) string { + return time.Unix(unixTime, 0).Format(eventTimeFormat) +} + +func formatDate(unixTime int64, format string) string { + return time.Unix(unixTime, 0).Format(format) +} + +func (event Event) TimestampDecrease(second int64) int64 { + return event.Timestamp - second +} + +func (event Event) TimestampIncrease(second int64) int64 { + return event.Timestamp + second +} + +type trigger struct { + Name string `json:"name"` +} + +func Populate(name, description string, events []Event) (desc string, err error) { + defer func() { + if errRecover := recover(); errRecover != nil { + desc = description + err = fmt.Errorf("PANIC in populate: %v, Trigger name: %s, desc: %s, events:%#v", + err, name, description, events) + } + }() + + buffer := bytes.Buffer{} + funcMap := template.FuncMap{ + "date": date, + "formatDate": formatDate, + } + + dataToExecute := notification{ + Trigger: trigger{Name: name}, + Events: events, + } + + triggerTemplate := template.New("populate-description").Funcs(funcMap) + triggerTemplate, err = triggerTemplate.Parse(description) + if err != nil { + return description, err + } + + err = triggerTemplate.Execute(&buffer, dataToExecute) + if err != nil { + return description, err + } + + return strings.TrimSpace(buffer.String()), nil +} diff --git a/templating/templating_test.go b/templating/templating_test.go new file mode 100644 index 000000000..d5e72e08a --- /dev/null +++ b/templating/templating_test.go @@ -0,0 +1,140 @@ +package templating + +import ( + "fmt" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func Test_TemplateDescription(t *testing.T) { + Convey("Test templates", t, func() { + var Name = "TestName" + var Desc = "" + + "Trigger name: {{.Trigger.Name}}\n" + + "{{range $v := .Events }}\n" + + "Metric: {{$v.Metric}}\n" + + "MetricElements: {{$v.MetricElements}}\n" + + "Timestamp: {{$v.Timestamp}}\n" + + "Value: {{$v.Value}}\n" + + "State: {{$v.State}}\n" + + "{{end}}\n" + + "https://grafana.yourhost.com/some-dashboard" + + "{{ range $i, $v := .Events }}{{ if ne $i 0 }}&{{ else }}?" + + "{{ end }}var-host={{ $v.Metric }}{{ end }}" + + var testUnixTime = time.Now().Unix() + var events = []Event{{Metric: "1", Timestamp: testUnixTime}, {Metric: "2", Timestamp: testUnixTime}} + + Convey("Test nil data", func() { + expected, err := Populate(Name, Desc, nil) + if err != nil { + println("Error:", err.Error()) + } + So(err, ShouldBeNil) + So(`Trigger name: TestName + +https://grafana.yourhost.com/some-dashboard`, + ShouldResemble, expected) + }) + + Convey("Test data", func() { + expected, err := Populate(Name, Desc, events) + So(err, ShouldBeNil) + So(fmt.Sprintf("Trigger name: TestName\n\nMetric: 1\nMetricElements: []\nTimestamp: %d\nValue: <nil>"+ + "\nState: \n\nMetric: 2\nMetricElements: []\nTimestamp: %d\nValue: <nil>"+ + "\nState: \n\nhttps://grafana.yourhost.com/some-dashboard?var-host=1&var-host=2", testUnixTime, testUnixTime), + ShouldResemble, expected) + }) + + Convey("Test description without templates", func() { + anotherText := "Another text" + Desc = anotherText + + expected, err := Populate(Name, Desc, events) + So(err, ShouldBeNil) + So(anotherText, ShouldEqual, expected) + }) + + Convey("Test method Date", func() { + formatDate := time.Unix(testUnixTime, 0).Format(eventTimeFormat) + actual := fmt.Sprintf("%s | %s |", formatDate, formatDate) + Desc = "{{ range .Events }}{{ date .Timestamp }} | {{ end }}" + + expected, err := Populate(Name, Desc, events) + So(err, ShouldBeNil) + So(actual, ShouldEqual, expected) + }) + + Convey("Test method formatted Date", func() { + formatedDate := time.Unix(testUnixTime, 0).Format("2006-01-02 15:04:05") + actual := fmt.Sprintf("%s | %s |", formatedDate, formatedDate) + Desc = "{{ range .Events }}{{ formatDate .Timestamp \"2006-01-02 15:04:05\" }} | {{ end }}" + + expected, err := Populate(Name, Desc, events) + So(err, ShouldBeNil) + So(actual, ShouldEqual, expected) + }) + + Convey("Test method decrease and increase Date", func() { + var timeOffset int64 = 300 + + Convey("Date increase", func() { + increase := testUnixTime + timeOffset + actual := fmt.Sprintf("%d | %d |", increase, increase) + Desc = fmt.Sprintf("{{ range .Events }}{{ .TimestampIncrease %d }} | {{ end }}", timeOffset) + + expected, err := Populate(Name, Desc, events) + So(err, ShouldBeNil) + So(actual, ShouldEqual, expected) + }) + + Convey("Date decrease", func() { + increase := testUnixTime - timeOffset + actual := fmt.Sprintf("%d | %d |", increase, increase) + Desc = fmt.Sprintf("{{ range .Events }}{{ .TimestampDecrease %d }} | {{ end }}", timeOffset) + + expected, err := Populate(Name, Desc, events) + So(err, ShouldBeNil) + So(actual, ShouldEqual, expected) + }) + }) + + Convey("Bad functions", func() { + var timeOffset int64 = 300 + + Convey("Non-existent function", func() { + Desc = fmt.Sprintf("{{ range .Events }}{{ decrease %d }} | {{ end }}", timeOffset) + + expected, err := Populate(Name, Desc, events) + So(err, ShouldNotBeNil) + So(Desc, ShouldEqual, expected) + }) + + Convey("Non-existent method", func() { + Desc = fmt.Sprintf("{{ range .Events }}{{ .Decrease %d }} | {{ end }}", timeOffset) + + expected, err := Populate(Name, Desc, events) + So(err, ShouldNotBeNil) + So(Desc, ShouldEqual, expected) + }) + + Convey("Bad parameters", func() { + Desc = "{{ date \"bad\" }} " + + expected, err := Populate(Name, Desc, events) + So(err, ShouldNotBeNil) + So(Desc, ShouldEqual, expected) + }) + + Convey("No parameters", func() { + Desc = "{{ date }} " + + expected, err := Populate(Name, Desc, events) + So(err, ShouldNotBeNil) + So(Desc, ShouldEqual, expected) + }) + }) + }) +}