From 88ae354128a5bf6d03a280a1717d11a60ac0fa39 Mon Sep 17 00:00:00 2001
From: corverroos <corver@obol.tech>
Date: Fri, 20 May 2022 07:15:21 +0200
Subject: [PATCH] app: add cluster identifiers to all metrics (#565)

Wraps all prometheus metrics in cluster identifiers.

category: feature
ticket: #548
---
 app/app.go         | 26 ++++++++++++++++++++++++--
 app/app_test.go    |  7 ++++---
 app/simnet_test.go |  1 +
 3 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/app/app.go b/app/app.go
index 898547730..2b6874a56 100644
--- a/app/app.go
+++ b/app/app.go
@@ -20,6 +20,7 @@ package app
 import (
 	"context"
 	"crypto/ecdsa"
+	"encoding/hex"
 	"net/http"
 	"net/http/pprof"
 	"time"
@@ -32,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/automaxprocs/maxprocs"
 
@@ -100,6 +102,8 @@ type TestConfig struct {
 	SimnetBMockOpts []beaconmock.Option
 	// BroadcastCallback is called when a duty is completed and sent to the broadcast component.
 	BroadcastCallback func(context.Context, core.Duty, core.PubKey, core.AggSignedData) error
+	// DisablePromWrap disables wrapping prometheus metrics with cluster identifiers.
+	DisablePromWrap bool
 }
 
 // Run is the entrypoint for running a charon DVC instance.
@@ -114,7 +118,6 @@ func Run(ctx context.Context, conf Config) (err error) {
 	}()
 
 	_, _ = maxprocs.Set()
-	initStartupMetrics()
 	if err := log.InitLogger(conf.Log); err != nil {
 		return err
 	}
@@ -142,6 +145,12 @@ func Run(ctx context.Context, conf Config) (err error) {
 		return err
 	}
 
+	lockHash, err := lock.HashTreeRoot()
+	if err != nil {
+		return err
+	}
+	lockHashHex := hex.EncodeToString(lockHash[:])[:7]
+
 	tcpNode, localEnode, err := wireP2P(ctx, life, conf, lock)
 	if err != nil {
 		return err
@@ -152,12 +161,25 @@ func Run(ctx context.Context, conf Config) (err error) {
 		return err
 	}
 
-	log.Info(ctx, "Lock loaded",
+	log.Info(ctx, "Lock file loaded",
+		z.Str("cluster_hash", lockHashHex),
+		z.Str("cluster_name", lock.Name),
 		z.Int("peers", len(lock.Operators)),
 		z.Str("peer_id", p2p.ShortID(tcpNode.ID())),
 		z.Int("peer_index", nodeIdx.PeerIdx),
 		z.Str("enr", localEnode.Node().String()))
 
+	if !conf.TestConfig.DisablePromWrap {
+		// Wrap prometheus metrics with cluster and node identifiers.
+		prometheus.DefaultRegisterer = prometheus.WrapRegistererWith(prometheus.Labels{
+			"cluster_hash":    lockHashHex,
+			"cluster_name":    lock.Name,
+			"cluster_enr":     lock.Operators[nodeIdx.PeerIdx].ENR,
+			"cluster_peer_id": p2p.ShortID(tcpNode.ID()),
+		}, prometheus.DefaultRegisterer)
+	}
+	initStartupMetrics()
+
 	wireMonitoringAPI(life, conf.MonitoringAddr, localEnode)
 
 	if err := wireCoreWorkflow(ctx, life, conf, lock, nodeIdx, tcpNode); err != nil {
diff --git a/app/app_test.go b/app/app_test.go
index 43136749b..e819c7fa6 100644
--- a/app/app_test.go
+++ b/app/app_test.go
@@ -174,9 +174,10 @@ func pingCluster(t *testing.T, test pingTest) {
 			MonitoringAddr:   testutil.AvailableAddr(t).String(), // Random monitoring address
 			ValidatorAPIAddr: testutil.AvailableAddr(t).String(), // Random validatorapi address
 			TestConfig: app.TestConfig{
-				Lock:         &lock,
-				P2PKey:       p2pKeys[i],
-				PingCallback: asserter.Callback(t, i),
+				Lock:            &lock,
+				P2PKey:          p2pKeys[i],
+				PingCallback:    asserter.Callback(t, i),
+				DisablePromWrap: true,
 			},
 			P2P: p2p.Config{
 				UDPBootnodes:    bootnodes,
diff --git a/app/simnet_test.go b/app/simnet_test.go
index e5b57f344..695cadd0c 100644
--- a/app/simnet_test.go
+++ b/app/simnet_test.go
@@ -153,6 +153,7 @@ func testSimnet(t *testing.T, args simnetArgs, propose bool) {
 				Lock:               &args.Lock,
 				P2PKey:             args.P2PKeys[i],
 				DisablePing:        true,
+				DisablePromWrap:    true,
 				SimnetKeys:         []*bls_sig.SecretKey{args.SimnetKeys[i]},
 				ParSigExFunc:       parSigExFunc,
 				LcastTransportFunc: lcastTransportFunc,