feat: add OpenTelemetry to node

AztecProtocol · Jun 21, 2024 · 2149d25 · 2149d25
1 parent 7746363
commit 2149d25
Show file tree

Hide file tree

Showing 61 changed files with 2,301 additions and 83 deletions.
diff --git a/cspell.json b/cspell.json
@@ -169,6 +169,9 @@
     "nullifer",
     "offchain",
     "onchain",
+    "opentelemetry",
+    "otel",
+    "OTLP",
     "otterscan",
     "outdir",
     "overlayfs",
@@ -253,6 +256,7 @@
     "typegen",
     "typeparam",
     "undeployed",
+    "undici",
     "unexclude",
     "unexcluded",
     "unprefixed",
@@ -270,6 +274,7 @@
     "viem",
     "wasms",
     "webassembly",
+    "WITGEN",
     "workdir",
     "yamux",
     "yarnrc",
@@ -301,5 +306,7 @@
     "lib",
     "*.cmake"
   ],
-  "flagWords": ["anonymous"]
-}
+  "flagWords": [
+    "anonymous"
+  ]
+}
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -28,6 +28,8 @@ services:
       - aztec:/var/lib/aztec
     ports:
       - 8080:8080/tcp
+    profiles:
+      - pxe
 
   node:
     image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70}
@@ -59,18 +61,34 @@ services:
       P2P_ENABLED: true
       PEER_ID_PRIVATE_KEY:
       AZTEC_PORT: 8999
+      OTEL_COLLECTOR_BASE_URL: ${OTEL_COLLECTOR_BASE_URL:-http://otel-collector:4318}
     secrets:
       - ethereum-host
       - p2p-boot-node
-    entrypoint: [
-        "/bin/sh",
-        "-c",
-        "export ETHEREUM_HOST=$$(cat /var/run/secrets/ethereum-host);\
-        export BOOTSTRAP_NODES=$$(cat /var/run/secrets/p2p-boot-node);\
-        test -z \"$$PEER_ID_PRIVATE_KEY\" -a ! -f /var/lib/aztec/p2p-private-key && node /usr/src/yarn-project/cli/dest/bin/index.js generate-p2p-private-key | head -1 | cut -d' ' -f 3 | tee /var/lib/aztec/p2p-private-key || echo 'Re-using existing P2P private key';\
-        test -z \"$$PEER_ID_PRIVATE_KEY\" && export PEER_ID_PRIVATE_KEY=$$(cat /var/lib/aztec/p2p-private-key);\
-        node /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver",
-      ]
+    entrypoint: |
+      /bin/sh -c '
+        export ETHEREUM_HOST=$$(cat /var/run/secrets/ethereum-host)
+        export BOOTSTRAP_NODES=$$(cat /var/run/secrets/p2p-boot-node)
+
+        test -z "$$PEER_ID_PRIVATE_KEY" -a ! -f /var/lib/aztec/p2p-private-key && node /usr/src/yarn-project/cli/dest/bin/index.js generate-p2p-private-key | head -1 | cut -d" " -f 3 | tee /var/lib/aztec/p2p-private-key || echo "Re-using existing P2P private key"
+        test -z "$$PEER_ID_PRIVATE_KEY" && export PEER_ID_PRIVATE_KEY=$$(cat /var/lib/aztec/p2p-private-key)
+
+        # if the stack is started with --profile metrics --profile node, give the collector a chance to start before the node
+        i=0
+        max=3
+        while ! curl --head --silent $$OTEL_COLLECTOR_BASE_URL > /dev/null; do
+          echo "OpenTelemetry collector not up. Retrying after 1s";
+          sleep 1;
+          i=$$((i+1));
+          if [ $$i -eq $$max ]; then
+            echo "OpenTelemetry collector at $$OTEL_COLLECTOR_BASE_URL not up after $${max}s. Running without metrics";
+            unset OTEL_COLLECTOR_BASE_URL;
+            break
+          fi;
+        done;
+
+        node /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver
+      '
     volumes:
       - aztec:/var/lib/aztec
     profiles:
@@ -94,8 +112,86 @@ services:
     profiles:
       - cli
 
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib
+    configs:
+      - source: otel-collector-config
+        target: /etc/otelcol-contrib/config.yaml
+    profiles:
+      - metrics
+
+  prometheus:
+    image: prom/prometheus
+    profiles:
+      - metrics
+    configs:
+      - source: prometheus-config
+        target: /etc/prometheus/prometheus.yml
+
+  grafana:
+    image: grafana/grafana
+    ports:
+      - 3000:3000
+    profiles:
+      - metrics
+    volumes:
+      - ./grafana_dashboards:/etc/grafana/provisioning/dashboards
+      - grafana:/var/lib/grafana
+    configs:
+      - source: grafana-sources
+        target: /etc/grafana/provisioning/datasources/default.yml
+
 volumes:
   aztec:
+  grafana:
+
+configs:
+  grafana-sources:
+    content: |
+      apiVersion: 1
+      datasources:
+        - name: Prometheus
+          uid: aztec-node-metrics
+          type: prometheus
+          url: http://prometheus:9090
+          editable: false
+          isDefault: true
+          jsonData:
+            timeInterval: 10s
+
+  prometheus-config:
+    content: |
+      global:
+        evaluation_interval: 30s
+        scrape_interval: 10s
+      scrape_configs:
+        - job_name: otel-collector
+          static_configs:
+          - targets: ['otel-collector:8888']
+        - job_name: aztec
+          static_configs:
+          - targets: ['otel-collector:8889']
+  otel-collector-config:
+    content: |
+      receivers:
+        otlp:
+          protocols:
+            http:
+
+      processors:
+        batch:
+
+      exporters:
+        prometheus:
+          endpoint: 0.0.0.0:8889
+          metric_expiration: 5m
+
+      service:
+        pipelines:
+          metrics:
+            receivers: [otlp]
+            processors: [batch]
+            exporters: [prometheus]
 
 secrets:
   aztec-node-url: