From 5cc9ddda3ac884d971029be61224efa42417bbb4 Mon Sep 17 00:00:00 2001 From: Dylan Arbour Date: Thu, 4 Apr 2024 18:40:49 -0400 Subject: [PATCH] Add exponential backoff for Slack requests Adds a 30 second retry to POST requests made to Slack's API, in the event that Slack experiences an intermittent failure. --- check/main.go | 3 ++- go.mod | 1 + go.sum | 2 ++ in/main.go | 3 ++- out/main.go | 5 ++++- slack/slack.go | 27 ++++++++++++++++++++------- slack/slack_test.go | 39 +++++++++++++++++++++++++-------------- 7 files changed, 56 insertions(+), 24 deletions(-) diff --git a/check/main.go b/check/main.go index bf93356..ef61776 100644 --- a/check/main.go +++ b/check/main.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "log" "os" @@ -11,6 +12,6 @@ import ( func main() { err := json.NewEncoder(os.Stdout).Encode(concourse.CheckResponse{}) if err != nil { - log.Fatalln(err) + log.Fatalln(fmt.Errorf("error: %s", err)) } } diff --git a/go.mod b/go.mod index a355e7b..a2dca59 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22 require ( github.com/Masterminds/semver/v3 v3.2.1 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/google/go-cmp v0.6.0 golang.org/x/oauth2 v0.21.0 ) diff --git a/go.sum b/go.sum index 399c6a3..297f37e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= diff --git a/in/main.go b/in/main.go index de97752..6163f23 100644 --- a/in/main.go +++ b/in/main.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "log" "os" @@ -11,6 +12,6 @@ import ( func main() { err := json.NewEncoder(os.Stdout).Encode(concourse.InResponse{Version: concourse.Version{"ver": "static"}}) if err != nil { - log.Fatalln(err) + log.Fatalln(fmt.Errorf("error: %s", err)) } } diff --git a/out/main.go b/out/main.go index 52c507d..172e42a 100644 --- a/out/main.go +++ b/out/main.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/arbourd/concourse-slack-alert-resource/concourse" "github.com/arbourd/concourse-slack-alert-resource/slack" @@ -130,6 +131,8 @@ func previousBuildName(s string) (string, error) { return strings.Trim(s, ".0"), nil } +var maxElapsedTime = 30 * time.Second + func out(input *concourse.OutRequest, path string) (*concourse.OutResponse, error) { if input.Source.URL == "" { return nil, errors.New("slack webhook url cannot be blank") @@ -153,7 +156,7 @@ func out(input *concourse.OutRequest, path string) (*concourse.OutResponse, erro } message := buildMessage(alert, metadata, path) - err := slack.Send(input.Source.URL, message) + err := slack.Send(input.Source.URL, message, maxElapsedTime) if err != nil { return nil, fmt.Errorf("error sending slack message: %w", err) } diff --git a/slack/slack.go b/slack/slack.go index 7562e0a..a588cc2 100644 --- a/slack/slack.go +++ b/slack/slack.go @@ -5,6 +5,9 @@ import ( "encoding/json" "fmt" "net/http" + "time" + + "github.com/cenkalti/backoff/v4" ) // Message represents a Slack API message @@ -35,20 +38,30 @@ type Field struct { } // Send sends the message to the webhook URL. -func Send(url string, m *Message) error { +func Send(url string, m *Message, maxRetryTime time.Duration) error { buf, err := json.Marshal(m) if err != nil { return err } - res, err := http.Post(url, "application/json", bytes.NewReader(buf)) + err = backoff.Retry( + func() error { + r, err := http.Post(url, "application/json", bytes.NewReader(buf)) + if err != nil { + return err + } + defer r.Body.Close() + + if r.StatusCode > 399 { + return fmt.Errorf("unexpected response status code: %d", r.StatusCode) + } + return nil + }, + backoff.NewExponentialBackOff(backoff.WithMaxElapsedTime(maxRetryTime)), + ) + if err != nil { return err } - defer res.Body.Close() - - if res.StatusCode != 200 { - return fmt.Errorf("unexpected status code: %d", res.StatusCode) - } return nil } diff --git a/slack/slack_test.go b/slack/slack_test.go index 9d7fc0a..a4e1511 100644 --- a/slack/slack_test.go +++ b/slack/slack_test.go @@ -4,37 +4,48 @@ import ( "net/http" "net/http/httptest" "testing" + "time" ) func TestSend(t *testing.T) { cases := map[string]struct { message *Message - err bool + backoff uint8 + wantErr bool }{ "ok": { message: &Message{Channel: "concourse"}, + backoff: 0, }, - "unauthorized": { - message: &Message{}, - err: true, + "retry ok": { + message: &Message{Channel: "concourse"}, + backoff: 1, + }, + "retry fail": { + message: &Message{Channel: "concourse"}, + backoff: 255, + wantErr: true, }, } for name, c := range cases { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if c.err { - http.Error(w, "", http.StatusUnauthorized) - } - })) - t.Run(name, func(t *testing.T) { - err := Send(s.URL, c.message) - if err != nil && !c.err { + tries := c.backoff + + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tries > 0 { + tries-- + http.Error(w, "", http.StatusUnauthorized) + } + })) + defer s.Close() + + err := Send(s.URL, c.message, 2*time.Second) + if err != nil && !c.wantErr { t.Fatalf("unexpected error from Send:\n\t(ERR): %s", err) - } else if err == nil && c.err { + } else if err == nil && c.wantErr { t.Fatalf("expected an error from Send:\n\t(GOT): nil") } }) - s.Close() } }