feat: support multiple PagerDuty API keys

go.mod

@@ -4,6 +4,6 @@ go 1.21
 
 toolchain go1.21.3
 
-require github.com/PagerDuty/go-pagerduty v1.7.0
+require github.com/PagerDuty/go-pagerduty v1.8.0
 
 require github.com/google/go-querystring v1.1.0 // indirect

go.sum

@@ -1,8 +1,8 @@
-github.com/PagerDuty/go-pagerduty v1.7.0 h1:S1NcMKECxT5hJwV4VT+QzeSsSiv4oWl1s2821dUqG/8=
-github.com/PagerDuty/go-pagerduty v1.7.0/go.mod h1:PuFyJKRz1liIAH4h5KVXVD18Obpp1ZXRdxHvmGXooro=
+github.com/PagerDuty/go-pagerduty v1.8.0 h1:MTFqTffIcAervB83U7Bx6HERzLbyaSPL/+oxH3zyluI=
+github.com/PagerDuty/go-pagerduty v1.8.0/go.mod h1:nzIeAqyFSJAFkjWKvMzug0JtwDg+V+UoCWjFrfFH5mI=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"strings"
 	"text/template"
 	"time"
 
@@ -59,6 +60,13 @@ func main() {
 		summaryTemplate = DEFAULT_SUMMARY_TEMPLATE
 	}
 
+	routingKeysEnv := os.Getenv("PAGERDUTY_ROUTING_KEY")
+	routingKeys := strings.Split(routingKeysEnv, ",")
+	if len(routingKeys) == 0 || routingKeys[0] == "" {
+		log.Printf("PAGERDUTY_ROUTING_KEY is empty")
+		return
+	}
+
 	hostName, err := os.Hostname()
 	if err != nil {
 		log.Printf("failed to get host name: %+v", err)
@@ -73,9 +81,8 @@ func main() {
 		log.Printf("failed to get systemd status: %+v", err)
 	}
 	event := pagerduty.V2Event{
-		Action:     "trigger",
-		Client:     "systemd-failure-notification",
-		RoutingKey: os.Getenv("PAGERDUTY_ROUTING_KEY"),
+		Action: "trigger",
+		Client: "systemd-failure-notification",
 	}
 	details := make(map[string]string)
 	details["status"] = out.String()
@@ -90,31 +97,35 @@ func main() {
 	event.Payload = payload
 	event.DedupKey = fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%s-%s", hostName, unitName))))
 
-	var retryErr error
-	retryWaitDuration := time.Second
-	for retryCount := 0; retryCount <= RETRY_COUNT; retryCount += 1 {
-		if retryCount > 0 {
-			time.Sleep(retryWaitDuration)
-			retryWaitDuration *= 2
-		}
+	for i, routingKey := range routingKeys {
+		event.RoutingKey = routingKey
 
-		if _, err := pagerduty.ManageEventWithContext(context.Background(), event); err != nil {
-			log.Printf("failed to send to pagerduty: %+v", err)
-			retryErr = err
-		} else {
-			retryErr = nil
-			break
-		}
-	}
+		var retryErr error
+		retryWaitDuration := time.Second
+		for retryCount := 0; retryCount <= RETRY_COUNT; retryCount += 1 {
+			if retryCount > 0 {
+				time.Sleep(retryWaitDuration)
+				retryWaitDuration *= 2
+			}
 
-	if retryErr != nil {
-		f, err := os.Create(errorTouchPath)
-		if err != nil {
-			log.Printf("failed to touch error file: %+v", err)
-			return
+			if _, err := pagerduty.ManageEventWithContext(context.Background(), event); err != nil {
+				log.Printf("failed to send to pagerduty for routing key #%d: %+v", i+1, err)
+				retryErr = err
+			} else {
+				retryErr = nil
+				break
+			}
 		}
-		if err := f.Close(); err != nil {
-			log.Printf("failed to touch error file: %+v", err)
+
+		if retryErr != nil {
+			f, err := os.Create(errorTouchPath)
+			if err != nil {
+				log.Printf("failed to touch error file: %+v", err)
+				return
+			}
+			if err := f.Close(); err != nil {
+				log.Printf("failed to touch error file: %+v", err)
+			}
 		}
 	}
 }