chore: simplify otel config, 1val setup, fix pod dns, retries

spartan/aztec-network/files/config/deploy-l1-contracts.sh

@@ -9,12 +9,18 @@ export PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4
 
 # Run the deploy-l1-contracts command and capture the output
 output=""
-# if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command
-if [ "$INIT_VALIDATORS" = "true" ]; then
-  output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --validators $2 --l1-chain-id $CHAIN_ID)
-else
-  output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --l1-chain-id $CHAIN_ID)
-fi
+MAX_RETRIES=5
+RETRY_DELAY=60
+for attempt in $(seq 1 $MAX_RETRIES); do
+  # if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command
+  if [ "${INIT_VALIDATORS:-false}" = "true" ]; then
+    output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --validators "$2" --l1-chain-id "$CHAIN_ID") && break
+  else
+    output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --l1-chain-id "$CHAIN_ID") && break
+  fi
+  echo "Attempt $attempt failed. Retrying in $RETRY_DELAY seconds..."
+  sleep "$RETRY_DELAY"
+done || { echo "All l1 contract deploy attempts failed."; exit 1; }
 
 echo "$output"
 

spartan/aztec-network/files/config/setup-service-addresses.sh

@@ -81,17 +81,25 @@ fi
 
 if [ "${PROVER_BROKER_EXTERNAL_HOST}" != "" ]; then
     PROVER_BROKER_ADDR="${PROVER_BROKER_EXTERNAL_HOST}"
-elif [ "${NETWORK_PUBLIC}" = "true" ]; then
-    PROVER_BROKER_ADDR=$(get_service_address "prover-broker" "${PROVER_BROKER_PORT}")
 else
     PROVER_BROKER_ADDR="http://${SERVICE_NAME}-prover-broker.${NAMESPACE}:${PROVER_BROKER_PORT}"
 fi
 
+# Configure OTEL_COLLECTOR_ENDPOINT if not set in values file
+if [ "${TELEMETRY:-false}" = "true" ] && [ "${OTEL_COLLECTOR_ENDPOINT}" = "" ]; then
+    OTEL_COLLECTOR_PORT=${OTEL_COLLECTOR_PORT:-4318}
+    OTEL_COLLECTOR_ENDPOINT="http://metrics-opentelemetry-collector.metrics:$OTEL_COLLECTOR_PORT"
+fi
 
 # Write addresses to file for sourcing
 echo "export ETHEREUM_HOST=${ETHEREUM_ADDR}" >> /shared/config/service-addresses
 echo "export BOOT_NODE_HOST=${BOOT_NODE_ADDR}" >> /shared/config/service-addresses
 echo "export PROVER_NODE_HOST=${PROVER_NODE_ADDR}" >> /shared/config/service-addresses
 echo "export PROVER_BROKER_HOST=${PROVER_BROKER_ADDR}" >> /shared/config/service-addresses
+echo "export OTEL_COLLECTOR_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT" >> /shared/config/service-addresses
+echo "export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/logs" >> /shared/config/service-addresses
+echo "export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/metrics" >> /shared/config/service-addresses
+echo "export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/traces" >> /shared/config/service-addresses
+
 echo "Addresses configured:"
 cat /shared/config/service-addresses

spartan/aztec-network/templates/_helpers.tpl

@@ -68,30 +68,6 @@ http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace
 http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }}
 {{- end -}}
 
-{{- define "aztec-network.otelCollectorMetricsEndpoint" -}}
-{{- if .Values.telemetry.enabled -}}
-{{- if .Values.telemetry.otelCollectorEndpoint -}}
-{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/metrics
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "aztec-network.otelCollectorTracesEndpoint" -}}
-{{- if .Values.telemetry.enabled -}}
-{{- if .Values.telemetry.otelCollectorEndpoint -}}
-{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/traces
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "aztec-network.otelCollectorLogsEndpoint" -}}
-{{- if .Values.telemetry.enabled -}}
-{{- if .Values.telemetry.otelCollectorEndpoint -}}
-{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/logs
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
 {{- define "helpers.flag" -}}
 {{- $name := index . 0 -}}
 {{- $value := index . 1 -}}
@@ -153,6 +129,10 @@ Service Address Setup Container
       value: "{{ .Values.network.public }}"
     - name: NAMESPACE
       value: {{ .Release.Namespace }}
+    - name: TELEMETRY
+      value: "{{ .Values.telemetry.enabled }}"
+    - name: OTEL_COLLECTOR_ENDPOINT
+      value: "{{ .Values.telemetry.otelCollectorEndpoint }}"
     - name: EXTERNAL_ETHEREUM_HOST
       value: "{{ .Values.ethereum.externalHost }}"
     - name: ETHEREUM_PORT

spartan/aztec-network/templates/boot-node.yaml

@@ -17,6 +17,7 @@ spec:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: boot-node
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       {{- if .Values.network.public }}
       hostNetwork: true
       {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }}
@@ -32,17 +33,18 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               echo "Awaiting ethereum node at ${ETHEREUM_HOST}"
               until curl -s -X POST -H 'Content-Type: application/json' \
                 -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
                 ${ETHEREUM_HOST} | grep -q reth; do
-                echo "Waiting for Ethereum node..."
+                echo "Waiting for Ethereum node ${ETHEREUM_HOST}..."
                 sleep 5
               done
               echo "Ethereum node is ready!"
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -150,12 +152,6 @@ spec:
               value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
             - name: PROVER_REAL_PROOFS
               value: "{{ .Values.aztec.realProofs }}"
             - name: PXE_PROVER_ENABLED

spartan/aztec-network/templates/deploy-l1-verifier.yaml

@@ -44,20 +44,29 @@ spec:
               chmod +x /tmp/setup-service-addresses.sh
               /tmp/setup-service-addresses.sh
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
 
               until curl -s -X GET "$BOOT_NODE_HOST/status"; do
                 echo "Waiting for Aztec node $BOOT_NODE_HOST..."
                 sleep 5
               done
               echo "Boot node is ready!"
 
-              export ROLLUP_CONTRACT_ADDRESS=$(curl -X POST -H 'Content-Type: application/json' \
+              l1_contracts=$(curl -X POST -H 'Content-Type: application/json' \
                   -d '{"jsonrpc":"2.0","method":"node_getL1ContractAddresses","params":[],"id":1}' \
-                  "$BOOT_NODE_HOST" \
-                  | jq -r '.result.rollupAddress.value')
-
+                  "$BOOT_NODE_HOST")
+              echo "L1 Contracts"
+              echo $l1_contracts
+              export ROLLUP_CONTRACT_ADDRESS=$(echo $l1_contracts | jq -r '.result.rollupAddress')
+              [ -z "$ROLLUP_CONTRACT_ADDRESS" ] && echo "Could not retrieve rollup address!" && exit 1
               echo "Rollup contract address: $ROLLUP_CONTRACT_ADDRESS"
-              node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real
+              MAX_RETRIES=5
+              RETRY_DELAY=60
+              for attempt in $(seq 1 $MAX_RETRIES); do
+                node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real
+                echo "Attempt $attempt failed. Retrying in $RETRY_DELAY seconds..."
+                sleep "$RETRY_DELAY"
+              done || { echo "All L1 verifier deploy attempts failed." >&2; exit 1; }
               echo "L1 verifier deployed"
           env:
             - name: NODE_NO_WARNINGS

spartan/aztec-network/templates/prover-agent.yaml

@@ -17,6 +17,7 @@ spec:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: prover-agent
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       {{- if .Values.proverAgent.gke.spotEnabled }}
       affinity:
         nodeAffinity:
@@ -50,14 +51,15 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               until curl -s -X POST ${PROVER_BROKER_HOST}/status; do
                 echo "Waiting for broker ${PROVER_BROKER_HOST} ..."
                 sleep 5
               done
               echo "Broker is ready!"
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -97,12 +99,6 @@ spec:
               value: {{ join "," .Values.proverAgent.proofTypes | quote }}
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
           resources:
             {{- toYaml .Values.proverAgent.resources | nindent 12 }}
 {{- end }}

spartan/aztec-network/templates/prover-broker.yaml

@@ -17,6 +17,7 @@ spec:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: prover-broker
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       serviceAccountName: {{ include "aztec-network.fullname" . }}-node
       {{- if .Values.network.public }}
       hostNetwork: true
@@ -36,9 +37,10 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -78,15 +80,11 @@ spec:
               value: "{{ .Values.proverBroker.dataDirectory }}"
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
           resources:
             {{- toYaml .Values.proverBroker.resources | nindent 12 }}
+{{- end }}
 ---
+# Headless service for StatefulSet DNS entries
 apiVersion: v1
 kind: Service
 metadata:
@@ -101,4 +99,3 @@ spec:
   ports:
     - port: {{ .Values.proverBroker.service.nodePort }}
       name: node
-{{ end }}

spartan/aztec-network/templates/prover-node.yaml

@@ -17,6 +17,7 @@ spec:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: prover-node
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       {{- if .Values.network.public }}
       hostNetwork: true
       {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }}
@@ -35,7 +36,7 @@ spec:
               until curl -s -X POST -H 'Content-Type: application/json' \
                 -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
                 ${ETHEREUM_HOST} | grep -q reth; do
-                echo "Waiting for Ethereum node..."
+                echo "Waiting for Ethereum node ${ETHEREUM_HOST}..."
                 sleep 5
               done
               echo "Ethereum node is ready!"
@@ -51,8 +52,8 @@ spec:
               fi
 
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -142,12 +143,6 @@ spec:
               value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
             - name: L1_CHAIN_ID
               value: "{{ .Values.ethereum.chainId }}"
             - name: P2P_ENABLED

spartan/aztec-network/templates/pxe.yaml

@@ -16,6 +16,7 @@ spec:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: pxe
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       serviceAccountName: {{ include "aztec-network.fullname" . }}-node
       {{- if .Values.network.public }}
       hostNetwork: true
@@ -37,6 +38,7 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               until curl --head --silent ${BOOT_NODE_HOST}/status; do
                 echo "Waiting for boot node..."
                 sleep 5
@@ -68,6 +70,7 @@ spec:
             - "-c"
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.network.public }}
               # If the network is public, we need to use the boot node URL
               export AZTEC_NODE_URL=${BOOT_NODE_HOST}

spartan/aztec-network/templates/rbac.yaml

@@ -55,4 +55,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: {{ include "aztec-network.fullname" . }}-node
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ .Release.Namespace }}

spartan/aztec-network/templates/setup-l2-contracts.yaml

@@ -48,6 +48,7 @@ spec:
               chmod +x /tmp/setup-service-addresses.sh
               /tmp/setup-service-addresses.sh
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               export AZTEC_NODE_URL=$BOOT_NODE_HOST
               export PXE_URL=$BOOT_NODE_HOST
               until curl -s -X POST -H 'content-type: application/json' \
@@ -61,6 +62,8 @@ spec:
               node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js setup-protocol-contracts --skipProofWait --l1-chain-id {{ .Values.ethereum.chainId }}
               echo "L2 contracts initialized"
           env:
+            - name: TELEMETRY
+              value: "{{ .Values.telemetry.enabled }}"
             - name: DEBUG
               value: "aztec:*"
             - name: LOG_LEVEL

spartan/aztec-network/templates/transaction-bot.yaml

@@ -38,6 +38,7 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.bot.nodeUrl }}
               export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }}
               {{- else if .Values.network.public }}
@@ -63,6 +64,7 @@ spec:
             - "-c"
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.bot.nodeUrl }}
               export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }}
               {{- else if .Values.network.public }}