From 25aff890e805e8568c3a44fa8249941be209fa78 Mon Sep 17 00:00:00 2001
From: Andre Ziviani <andrepziviani@gmail.com>
Date: Thu, 4 Apr 2024 15:11:47 -0300
Subject: [PATCH] feat: Expose events as metrics

---
 exporter/health.go  |  9 +++---
 exporter/metrics.go | 53 ++++++++++++++++++--------------
 go.mod              | 10 ++++++-
 go.sum              | 25 +++++++++++++---
 main.go             | 73 +++++++++++++++++++++++++++++++++------------
 5 files changed, 120 insertions(+), 50 deletions(-)

diff --git a/exporter/health.go b/exporter/health.go
index 3309b7a..191d4c2 100644
--- a/exporter/health.go
+++ b/exporter/health.go
@@ -21,15 +21,15 @@ func (m *Metrics) HealthOrganizationEnabled(ctx context.Context) bool {
 }
 
 func (m *Metrics) GetHealthEvents() []HealthEvent {
-	var events []HealthEvent
+	var tmp, events []HealthEvent
 
 	if m.organizationEnabled {
-		events = m.GetOrgEvents()
+		tmp = m.GetOrgEvents()
 	} else {
-		events = m.GetAccountEvents()
+		tmp = m.GetAccountEvents()
 	}
 
-	for _, e := range events {
+	for _, e := range tmp {
 		if ignoreEvents(m.ignoreEvents, *e.Event.EventTypeCode) {
 			continue
 		}
@@ -43,6 +43,7 @@ func (m *Metrics) GetHealthEvents() []HealthEvent {
 			continue
 		}
 
+		events = append(events, e)
 		m.SendSlackNotification(e)
 	}
 
diff --git a/exporter/metrics.go b/exporter/metrics.go
index 5ba90ba..71b83c4 100644
--- a/exporter/metrics.go
+++ b/exporter/metrics.go
@@ -11,24 +11,45 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/slack-go/slack"
 	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 )
 
-const (
-	namespace = "aws_health"
-)
-
-func NewMetrics(ctx context.Context, registry *prometheus.Registry, c *cli.Context) (*Metrics, error) {
+func NewMetrics(ctx context.Context, meter metric.Meter, c *cli.Context) (*Metrics, error) {
 	m := Metrics{}
 
 	m.init(ctx, c)
 
-	registry.MustRegister(&m)
-	registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
-	registry.MustRegister(collectors.NewGoCollector())
+	g, _ := meter.Int64ObservableGauge("event", metric.WithDescription("Status of AWS Health events"))
+	meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
+		events := m.GetHealthEvents()
+		for _, e := range events {
+			attributes := metric.WithAttributes(
+				attribute.Key("region").String(aws.ToString(e.Event.Region)),
+				attribute.Key("service").String(aws.ToString(e.Event.Service)),
+				attribute.Key("scope").String(string(e.Event.EventScopeCode)),
+				attribute.Key("category").String(string(e.Event.EventTypeCategory)),
+				attribute.Key("code").String(aws.ToString(e.Event.EventTypeCode)),
+			)
+
+			status := int64(1) // open
+			if e.Event.StatusCode != "open" {
+				status = int64(0) // closed
+			}
+
+			if len(e.AffectedAccounts) > 0 {
+				for _, account := range e.AffectedAccounts {
+					o.ObserveInt64(g, status, attributes, metric.WithAttributes(attribute.Key("account").String(account)))
+				}
+			} else {
+				o.ObserveInt64(g, status, attributes)
+			}
+		}
+
+		return nil
+	}, g)
 
 	return &m, nil
 }
@@ -89,15 +110,3 @@ func (m *Metrics) init(ctx context.Context, c *cli.Context) {
 	}
 
 }
-
-func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
-	prometheus.DescribeByCollect(m, ch)
-}
-
-func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
-	m.GetHealthEvents()
-}
-
-func sanitizeLabel(label string) string {
-	return strings.Replace(strings.Replace(label, ".", "_", -1), "/", "_", -1)
-}
diff --git a/go.mod b/go.mod
index d69bd62..2865fc7 100644
--- a/go.mod
+++ b/go.mod
@@ -15,6 +15,8 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/slack-go/slack v0.12.5
 	github.com/urfave/cli/v2 v2.27.1
+	go.opentelemetry.io/otel/exporters/prometheus v0.46.0
+	go.opentelemetry.io/otel/sdk/metric v1.24.0
 	k8s.io/api v0.26.0
 	k8s.io/apimachinery v0.26.0
 	k8s.io/client-go v0.26.0
@@ -34,7 +36,8 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
@@ -49,6 +52,11 @@ require (
 	github.com/prometheus/procfs v0.13.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
diff --git a/go.sum b/go.sum
index 8eaa0d1..8bb0d96 100644
--- a/go.sum
+++ b/go.sum
@@ -49,8 +49,11 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
 github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
@@ -127,14 +130,28 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
 github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
 github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0 h1:dg9y+7ArpumB6zwImJv47RHfdgOGQ1EMkzP5vLkEnTU=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0/go.mod h1:Ul4MtXqu/hJBM+v7a6dCF0nHwckPMLpIpLeCi4+zfdw=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ=
+go.opentelemetry.io/otel/exporters/prometheus v0.46.0/go.mod h1:ztwVUHe5DTR/1v7PeuGRnU5Bbd4QKYwApWmuutKsJSs=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
+go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
+go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
+go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8=
+go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
diff --git a/main.go b/main.go
index d51dabf..e912301 100644
--- a/main.go
+++ b/main.go
@@ -8,10 +8,15 @@ import (
 	"time"
 
 	"github.com/AndreZiviani/aws-health-exporter/exporter"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/contrib/instrumentation/runtime"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
 )
 
 func main() {
@@ -44,30 +49,20 @@ func main() {
 
 			log.Infof("Starting AWS Health Exporter. [log-level=%s]", c.String("log-level"))
 
-			ctx := context.TODO()
+			ctx := context.Background()
 
-			registry := prometheus.NewRegistry()
+			provider, err := newMeter()
+			if err != nil {
+				log.Fatal(err)
+			}
+			defer provider.Shutdown(ctx)
 
-			_, err = exporter.NewMetrics(ctx, registry, c)
+			_, err = exporter.NewMetrics(ctx, otel.Meter("aws-health-exporter"), c)
 			if err != nil {
 				log.Fatal(err)
 			}
 
-			log.Infof("Starting metric http endpoint [address=%s, path=%s, regions=%s]", c.String("listen-address"), c.String("metrics-path"), c.String("regions"))
-			http.Handle(c.String("metrics-path"), promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
-			http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-				w.WriteHeader(http.StatusOK)
-				w.Write([]byte(`<html>
-					<head><title>AWS Health Exporter</title></head>
-					<body>
-					<h1>AWS Health Exporter</h1>
-					<p><a href="` + c.String("metrics-path") + `">Metrics</a></p>
-					</body>
-					</html>
-				`))
-
-			})
-			log.Fatal(http.ListenAndServe(c.String("listen-address"), nil))
+			serveMetrics(c)
 
 			return nil
 		},
@@ -81,3 +76,43 @@ func main() {
 
 	return
 }
+
+func newMeter() (*metric.MeterProvider, error) {
+	promExporter, err := prometheus.New(prometheus.WithNamespace("aws_health"))
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := resource.Merge(resource.Default(),
+		resource.NewWithAttributes(semconv.SchemaURL,
+			semconv.ServiceName("aws-health-exporter"),
+			//semconv.ServiceVersion("0.1.0"),
+		))
+	if err != nil {
+		return nil, err
+	}
+
+	provider := metric.NewMeterProvider(metric.WithResource(res), metric.WithReader(promExporter))
+	otel.SetMeterProvider(provider)
+	runtime.Start()
+
+	return provider, nil
+}
+
+func serveMetrics(c *cli.Context) {
+	log.Infof("Starting metric http endpoint [address=%s, path=%s, regions=%s]", c.String("listen-address"), c.String("metrics-path"), c.String("regions"))
+	http.Handle(c.String("metrics-path"), promhttp.Handler())
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`<html>
+				<head><title>AWS Health Exporter</title></head>
+				<body>
+				<h1>AWS Health Exporter</h1>
+				<p><a href="` + c.String("metrics-path") + `">Metrics</a></p>
+				</body>
+				</html>
+			`))
+
+	})
+	log.Fatal(http.ListenAndServe(c.String("listen-address"), nil))
+}