alerting: added slack as a notifier

go.mod

@@ -42,6 +42,7 @@ require (
 	github.com/prometheus/client_golang v1.12.2
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sethvargo/go-retry v0.1.0
+	github.com/slack-go/slack v0.10.3 // indirect
 	github.com/spf13/viper v1.11.0
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.7.1

go.sum

@@ -633,6 +633,7 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
@@ -832,6 +833,7 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -1417,6 +1419,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/slack-go/slack v0.10.3 h1:kKYwlKY73AfSrtAk9UHWCXXfitudkDztNI9GYBviLxw=
+github.com/slack-go/slack v0.10.3/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=

internal/alerting/alerter.go

@@ -1,14 +1,6 @@
 package alerting
 
 import (
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"os"
-	"runtime"
-	"time"
-
-	"github.com/PagerDuty/go-pagerduty"
 	"github.com/moov-io/achgateway/internal/service"
 )
 
@@ -22,85 +14,27 @@ func (mn *MockAlerter) AlertError(e error) error {
 	return nil
 }
 
-func NewAlerter(cfg service.ErrorAlerting) (Alerter, error) {
+func NewAlerters(cfg service.ErrorAlerting) ([]Alerter, error) {
+	var alerters []Alerter
 	switch {
+	case cfg.Slack != nil:
+		alerter, err := NewSlackAlerter(cfg.Slack)
+		if err != nil {
+			return nil, err
+		}
+		alerters = append(alerters, alerter)
+		fallthrough
 	case cfg.PagerDuty != nil:
-		return NewPagerDutyAlerter(cfg.PagerDuty)
-	}
-	return &MockAlerter{}, nil
-}
-
-type PagerDuty struct {
-	client     *pagerduty.Client
-	routingKey string
-}
-
-func NewPagerDutyAlerter(cfg *service.PagerDutyAlerting) (*PagerDuty, error) {
-	notifier := &PagerDuty{
-		client:     pagerduty.NewClient(cfg.ApiKey),
-		routingKey: cfg.RoutingKey,
-	}
-	if err := notifier.ping(); err != nil {
-		return nil, err
+		alerter, err := NewPagerDutyAlerter(cfg.PagerDuty)
+		if err != nil {
+			return nil, err
+		}
+		alerters = append(alerters, alerter)
 	}
-	return notifier, nil
-}
-
-func (pd *PagerDuty) AlertError(e error) error {
-	if e == nil {
-		return nil
-	}
-
-	details := make(map[string]string)
 
-	hostName, err := os.Hostname()
-	if err != nil {
-		return fmt.Errorf("getting host name: %v", err)
+	if len(alerters) == 0 {
+		return []Alerter{&MockAlerter{}}, nil
 	}
 
-	dedupKey := e.Error()
-	if _, file, line, ok := runtime.Caller(1); ok {
-		location := fmt.Sprintf("%s:%d", file, line)
-		details["location"] = location
-		dedupKey += location
-	}
-
-	errorHash := fmt.Sprintf("%x", sha256.Sum256([]byte(dedupKey)))
-
-	event := &pagerduty.V2Event{
-		RoutingKey: pd.routingKey,
-		Action:     "trigger",
-		DedupKey:   errorHash,
-		Payload: &pagerduty.V2Payload{
-			Summary:   e.Error(),
-			Source:    hostName,
-			Severity:  "critical",
-			Timestamp: time.Now().Format(time.RFC3339),
-			Details:   details,
-		},
-	}
-
-	_, err = pd.client.ManageEvent(event)
-	if err != nil {
-		return fmt.Errorf("creating event in PagerDuty: %v", err)
-	}
-
-	return nil
-}
-
-func (pd *PagerDuty) ping() error {
-	if pd == nil || pd.client == nil {
-		return errors.New("pagerduty: nil")
-	}
-
-	// make a call and verify we don't error
-	resp, err := pd.client.ListAbilities()
-	if err != nil {
-		return fmt.Errorf("pagerduty list abilities: %v", err)
-	}
-	if len(resp.Abilities) <= 0 {
-		return fmt.Errorf("pagerduty: missing abilities")
-	}
-
-	return nil
+	return alerters, nil
 }

internal/alerting/alerter_test.go

@@ -9,20 +9,59 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestPagerDutyErrorAlert(t *testing.T) {
-	if os.Getenv("PD_API_KEY") == "" {
-		t.Skip("Skip PagerDuty notification as PD_API_KEY and PD_ROUTING_KEY are not set")
+func TestNewAlerters(t *testing.T) {
+	if os.Getenv("PD_API_KEY") == "" && os.Getenv("SLACK_ACCESS_TOKEN") == "" {
+		t.Skip("Skip TestNewAlerters as PD_API_KEY and SLACK_ACCESS_TOKEN are not set")
 	}
+	var cfg service.ErrorAlerting
+	var alerters []Alerter
+	var err error
 
-	cfg := &service.PagerDutyAlerting{
-		ApiKey:     os.Getenv("PD_API_KEY"),
-		RoutingKey: os.Getenv("PD_ROUTING_KEY"),
+	if os.Getenv("PD_API_KEY") != "" {
+		cfg = service.ErrorAlerting{
+			PagerDuty: &service.PagerDutyAlerting{
+				ApiKey:     os.Getenv("PD_API_KEY"),
+				RoutingKey: os.Getenv("PD_ROUTING_KEY"),
+			},
+		}
+
+		alerters, err = NewAlerters(cfg)
+		require.NoError(t, err)
+		require.Len(t, alerters, 1)
+	}
+
+	if os.Getenv("SLACK_ACCESS_TOKEN") != "" {
+		cfg = service.ErrorAlerting{
+			Slack: &service.SlackAlerting{
+				AccessToken: os.Getenv("SLACK_ACCESS_TOKEN"),
+				ChannelID:   os.Getenv("SLACK_CHANNEL_ID"),
+			},
+		}
+
+		alerters, err = NewAlerters(cfg)
+		require.NoError(t, err)
+		require.Len(t, alerters, 1)
 	}
 
-	notifier, err := NewPagerDutyAlerter(cfg)
-	require.NoError(t, err)
-	require.NotNil(t, notifier)
+	if os.Getenv("PD_API_KEY") != "" && os.Getenv("SLACK_ACCESS_TOKEN") != "" {
+		cfg = service.ErrorAlerting{
+			PagerDuty: &service.PagerDutyAlerting{
+				ApiKey:     os.Getenv("PD_API_KEY"),
+				RoutingKey: os.Getenv("PD_ROUTING_KEY"),
+			},
+			Slack: &service.SlackAlerting{
+				AccessToken: os.Getenv("SLACK_ACCESS_TOKEN"),
+				ChannelID:   os.Getenv("SLACK_CHANNEL_ID"),
+			},
+		}
+
+		alerters, err = NewAlerters(cfg)
+		require.NoError(t, err)
+		require.Len(t, alerters, 2)
 
-	err = notifier.AlertError(errors.New("error message"))
-	require.NoError(t, err)
+		for _, alerter := range alerters {
+			err = alerter.AlertError(errors.New("error message"))
+			require.NoError(t, err)
+		}
+	}
 }

internal/alerting/pagerduty.go

@@ -0,0 +1,88 @@
+package alerting
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/PagerDuty/go-pagerduty"
+	"github.com/moov-io/achgateway/internal/service"
+)
+
+type PagerDuty struct {
+	client     *pagerduty.Client
+	routingKey string
+}
+
+func NewPagerDutyAlerter(cfg *service.PagerDutyAlerting) (*PagerDuty, error) {
+	notifier := &PagerDuty{
+		client:     pagerduty.NewClient(cfg.ApiKey),
+		routingKey: cfg.RoutingKey,
+	}
+	if err := notifier.ping(); err != nil {
+		return nil, err
+	}
+	return notifier, nil
+}
+
+func (pd *PagerDuty) AlertError(e error) error {
+	if e == nil {
+		return nil
+	}
+
+	details := make(map[string]string)
+
+	hostName, err := os.Hostname()
+	if err != nil {
+		return fmt.Errorf("getting host name: %v", err)
+	}
+
+	dedupKey := e.Error()
+	if _, file, line, ok := runtime.Caller(1); ok {
+		location := fmt.Sprintf("%s:%d", file, line)
+		details["location"] = location
+		dedupKey += location
+	}
+
+	errorHash := fmt.Sprintf("%x", sha256.Sum256([]byte(dedupKey)))
+
+	event := &pagerduty.V2Event{
+		RoutingKey: pd.routingKey,
+		Action:     "trigger",
+		DedupKey:   errorHash,
+		Payload: &pagerduty.V2Payload{
+			Summary:   e.Error(),
+			Source:    hostName,
+			Severity:  "critical",
+			Timestamp: time.Now().Format(time.RFC3339),
+			Details:   details,
+		},
+	}
+
+	_, err = pd.client.ManageEvent(event)
+	if err != nil {
+		return fmt.Errorf("creating event in PagerDuty: %v", err)
+	}
+
+	return nil
+}
+
+func (pd *PagerDuty) ping() error {
+	if pd == nil || pd.client == nil {
+		return errors.New("pagerduty: nil")
+	}
+
+	// make a call and verify we don't error
+	resp, err := pd.client.ListAbilities()
+	if err != nil {
+		return fmt.Errorf("pagerduty list abilities: %v", err)
+	}
+	if len(resp.Abilities) <= 0 {
+		return fmt.Errorf("pagerduty: missing abilities")
+	}
+
+	return nil
+}

internal/alerting/pagerduty_test.go

@@ -0,0 +1,28 @@
+package alerting
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/moov-io/achgateway/internal/service"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPagerDutyErrorAlert(t *testing.T) {
+	if os.Getenv("PD_API_KEY") == "" {
+		t.Skip("Skip PagerDuty notification as PD_API_KEY and PD_ROUTING_KEY are not set")
+	}
+
+	cfg := &service.PagerDutyAlerting{
+		ApiKey:     os.Getenv("PD_API_KEY"),
+		RoutingKey: os.Getenv("PD_ROUTING_KEY"),
+	}
+
+	notifier, err := NewPagerDutyAlerter(cfg)
+	require.NoError(t, err)
+	require.NotNil(t, notifier)
+
+	err = notifier.AlertError(errors.New("error message"))
+	require.NoError(t, err)
+}