chore: simplify otel config, 1val setup, fix pod dns, retries (#10344)

There were a few things in the way of getting a devnet up - Added retries as we had 'intrinsic gas too low' before reth got a chance to sync - use cluster dns explicitly before host networkinge - work around an issue with broker external host --------- Co-authored-by: Alex Gherghisan <[email protected]>
AztecProtocol · Dec 6, 2024 · be91d80 · be91d80
1 parent 2c2169b
commit be91d80
Show file tree

Hide file tree

Showing 25 changed files with 188 additions and 102 deletions.
diff --git a/spartan/aztec-network/files/config/deploy-l1-contracts.sh b/spartan/aztec-network/files/config/deploy-l1-contracts.sh
@@ -6,12 +6,19 @@ CHAIN_ID=$1
 
 # Run the deploy-l1-contracts command and capture the output
 output=""
-# if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command
-if [ "$INIT_VALIDATORS" = "true" ]; then
-  output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --validators $2 --l1-chain-id $CHAIN_ID)
-else
-  output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --l1-chain-id $CHAIN_ID)
-fi
+MAX_RETRIES=5
+RETRY_DELAY=60
+for attempt in $(seq 1 $MAX_RETRIES); do
+  # if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command
+  if [ "${INIT_VALIDATORS:-false}" = "true" ]; then
+    output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --validators $2 --l1-chain-id $CHAIN_ID) && break
+  else
+    output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --l1-chain-id $CHAIN_ID) && break
+  fi
+  echo "Attempt $attempt failed. Retrying in $RETRY_DELAY seconds..."
+  sleep "$RETRY_DELAY"
+done || { echo "All l1 contract deploy attempts failed."; exit 1; }
+
 
 echo "$output"
 

diff --git a/spartan/aztec-network/files/config/setup-service-addresses.sh b/spartan/aztec-network/files/config/setup-service-addresses.sh
@@ -81,17 +81,29 @@ fi
 
 if [ "${PROVER_BROKER_EXTERNAL_HOST}" != "" ]; then
     PROVER_BROKER_ADDR="${PROVER_BROKER_EXTERNAL_HOST}"
-elif [ "${NETWORK_PUBLIC}" = "true" ]; then
-    PROVER_BROKER_ADDR=$(get_service_address "prover-broker" "${PROVER_BROKER_PORT}")
 else
     PROVER_BROKER_ADDR="http://${SERVICE_NAME}-prover-broker.${NAMESPACE}:${PROVER_BROKER_PORT}"
 fi
 
+# Configure OTEL_COLLECTOR_ENDPOINT if not set in values file
+if [ "${TELEMETRY:-false}" = "true" ] && [ "${OTEL_COLLECTOR_ENDPOINT}" = "" ]; then
+    OTEL_COLLECTOR_PORT=${OTEL_COLLECTOR_PORT:-4318}
+    OTEL_COLLECTOR_ENDPOINT="http://metrics-opentelemetry-collector.metrics:$OTEL_COLLECTOR_PORT"
+fi
 
 # Write addresses to file for sourcing
 echo "export ETHEREUM_HOST=${ETHEREUM_ADDR}" >> /shared/config/service-addresses
 echo "export BOOT_NODE_HOST=${BOOT_NODE_ADDR}" >> /shared/config/service-addresses
 echo "export PROVER_NODE_HOST=${PROVER_NODE_ADDR}" >> /shared/config/service-addresses
 echo "export PROVER_BROKER_HOST=${PROVER_BROKER_ADDR}" >> /shared/config/service-addresses
+
+if [ "${OTEL_COLLECTOR_ENDPOINT}" != "" ]; then
+  echo "export OTEL_COLLECTOR_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT" >> /shared/config/service-addresses
+  echo "export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/logs" >> /shared/config/service-addresses
+  echo "export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/metrics" >> /shared/config/service-addresses
+  echo "export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/traces" >> /shared/config/service-addresses
+fi
+
+
 echo "Addresses configured:"
 cat /shared/config/service-addresses
diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl
@@ -68,30 +68,6 @@ http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace
 http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }}
 {{- end -}}
 
-{{- define "aztec-network.otelCollectorMetricsEndpoint" -}}
-{{- if .Values.telemetry.enabled -}}
-{{- if .Values.telemetry.otelCollectorEndpoint -}}
-{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/metrics
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "aztec-network.otelCollectorTracesEndpoint" -}}
-{{- if .Values.telemetry.enabled -}}
-{{- if .Values.telemetry.otelCollectorEndpoint -}}
-{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/traces
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "aztec-network.otelCollectorLogsEndpoint" -}}
-{{- if .Values.telemetry.enabled -}}
-{{- if .Values.telemetry.otelCollectorEndpoint -}}
-{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/logs
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
 {{- define "helpers.flag" -}}
 {{- $name := index . 0 -}}
 {{- $value := index . 1 -}}
@@ -153,6 +129,10 @@ Service Address Setup Container
       value: "{{ .Values.network.public }}"
     - name: NAMESPACE
       value: {{ .Release.Namespace }}
+    - name: TELEMETRY
+      value: "{{ .Values.telemetry.enabled }}"
+    - name: OTEL_COLLECTOR_ENDPOINT
+      value: "{{ .Values.telemetry.otelCollectorEndpoint }}"
     - name: EXTERNAL_ETHEREUM_HOST
       value: "{{ .Values.ethereum.externalHost }}"
     - name: ETHEREUM_PORT

diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
@@ -17,6 +17,7 @@ spec:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: boot-node
     spec:
+      dnsPolicy: ClusterFirstWithHostNet
       {{- if .Values.network.public }}
       hostNetwork: true
       {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }}
@@ -32,17 +33,18 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               echo "Awaiting ethereum node at ${ETHEREUM_HOST}"
               until curl -s -X POST -H 'Content-Type: application/json' \
                 -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
                 ${ETHEREUM_HOST} | grep -q reth; do
-                echo "Waiting for Ethereum node..."
+                echo "Waiting for Ethereum node ${ETHEREUM_HOST}..."
                 sleep 5
               done
               echo "Ethereum node is ready!"
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -70,6 +72,8 @@ spec:
             - name: scripts
               mountPath: /scripts
           env:
+            - name: TELEMETRY
+              value: "{{ .Values.telemetry.enabled }}"
             - name: INIT_VALIDATORS
               value: "true"
             - name: MNEMONIC
@@ -152,12 +156,6 @@ spec:
               value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
             - name: PROVER_REAL_PROOFS
               value: "{{ .Values.aztec.realProofs }}"
             - name: PXE_PROVER_ENABLED

diff --git a/spartan/aztec-network/templates/deploy-l1-verifier.yaml b/spartan/aztec-network/templates/deploy-l1-verifier.yaml
@@ -44,20 +44,29 @@ spec:
               chmod +x /tmp/setup-service-addresses.sh
               /tmp/setup-service-addresses.sh
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
 
               until curl -s -X GET "$BOOT_NODE_HOST/status"; do
                 echo "Waiting for Aztec node $BOOT_NODE_HOST..."
                 sleep 5
               done
               echo "Boot node is ready!"
 
-              export ROLLUP_CONTRACT_ADDRESS=$(curl -X POST -H 'Content-Type: application/json' \
+              l1_contracts=$(curl -X POST -H 'Content-Type: application/json' \
                   -d '{"jsonrpc":"2.0","method":"node_getL1ContractAddresses","params":[],"id":1}' \
-                  "$BOOT_NODE_HOST" \
-                  | jq -r '.result.rollupAddress.value')
-
+                  "$BOOT_NODE_HOST")
+              echo "L1 Contracts"
+              echo $l1_contracts
+              export ROLLUP_CONTRACT_ADDRESS=$(echo $l1_contracts | jq -r '.result.rollupAddress')
+              [ -z "$ROLLUP_CONTRACT_ADDRESS" ] && echo "Could not retrieve rollup address!" && exit 1
               echo "Rollup contract address: $ROLLUP_CONTRACT_ADDRESS"
-              node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real
+              MAX_RETRIES=5
+              RETRY_DELAY=60
+              for attempt in $(seq 1 $MAX_RETRIES); do
+                node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real
+                echo "Attempt $attempt failed. Retrying in $RETRY_DELAY seconds..."
+                sleep "$RETRY_DELAY"
+              done || { echo "All L1 verifier deploy attempts failed." >&2; exit 1; }
               echo "L1 verifier deployed"
           env:
             - name: NODE_NO_WARNINGS

diff --git a/spartan/aztec-network/templates/prover-agent.yaml b/spartan/aztec-network/templates/prover-agent.yaml
@@ -34,6 +34,7 @@ spec:
       serviceAccountName: {{ include "aztec-network.fullname" . }}-node
       {{- if .Values.network.public }}
       hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
       {{- end }}
       volumes:
         - name: config
@@ -50,14 +51,15 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               until curl -s -X POST ${PROVER_BROKER_HOST}/status; do
                 echo "Waiting for broker ${PROVER_BROKER_HOST} ..."
                 sleep 5
               done
               echo "Broker is ready!"
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -95,12 +97,6 @@ spec:
               value: {{ join "," .Values.proverAgent.proofTypes | quote }}
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
           resources:
             {{- toYaml .Values.proverAgent.resources | nindent 12 }}
 {{- end }}
diff --git a/spartan/aztec-network/templates/prover-broker.yaml b/spartan/aztec-network/templates/prover-broker.yaml
@@ -20,6 +20,7 @@ spec:
       serviceAccountName: {{ include "aztec-network.fullname" . }}-node
       {{- if .Values.network.public }}
       hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
       {{- end }}
       volumes:
         - name: config
@@ -36,9 +37,10 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -76,15 +78,11 @@ spec:
               value: "{{ .Values.proverBroker.dataDirectory }}"
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
           resources:
             {{- toYaml .Values.proverBroker.resources | nindent 12 }}
+{{- end }}
 ---
+# Headless service for StatefulSet DNS entries
 apiVersion: v1
 kind: Service
 metadata:
@@ -99,4 +97,3 @@ spec:
   ports:
     - port: {{ .Values.proverBroker.service.nodePort }}
       name: node
-{{ end }}
diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml
@@ -19,6 +19,7 @@ spec:
     spec:
       {{- if .Values.network.public }}
       hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
       {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }}
       {{- end }}
       serviceAccountName: {{ include "aztec-network.fullname" . }}-node
@@ -35,7 +36,7 @@ spec:
               until curl -s -X POST -H 'Content-Type: application/json' \
                 -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
                 ${ETHEREUM_HOST} | grep -q reth; do
-                echo "Waiting for Ethereum node..."
+                echo "Waiting for Ethereum node ${ETHEREUM_HOST}..."
                 sleep 5
               done
               echo "Ethereum node is ready!"
@@ -51,8 +52,8 @@ spec:
               fi
 
               {{- if .Values.telemetry.enabled }}
-              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
-                echo "Waiting for OpenTelemetry collector..."
+              until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
+                echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
@@ -142,12 +143,6 @@ spec:
               value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
-            - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
-            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
             - name: L1_CHAIN_ID
               value: "{{ .Values.ethereum.chainId }}"
             - name: P2P_ENABLED

diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml
@@ -19,6 +19,7 @@ spec:
       serviceAccountName: {{ include "aztec-network.fullname" . }}-node
       {{- if .Values.network.public }}
       hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
       {{- end }}
       volumes:
         - name: config
@@ -37,6 +38,7 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               until curl --head --silent ${BOOT_NODE_HOST}/status; do
                 echo "Waiting for boot node..."
                 sleep 5
@@ -68,6 +70,7 @@ spec:
             - "-c"
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.network.public }}
               # If the network is public, we need to use the boot node URL
               export AZTEC_NODE_URL=${BOOT_NODE_HOST}

diff --git a/spartan/aztec-network/templates/rbac.yaml b/spartan/aztec-network/templates/rbac.yaml
@@ -55,4 +55,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: {{ include "aztec-network.fullname" . }}-node
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ .Release.Namespace }}
diff --git a/spartan/aztec-network/templates/setup-l2-contracts.yaml b/spartan/aztec-network/templates/setup-l2-contracts.yaml
@@ -48,19 +48,22 @@ spec:
               chmod +x /tmp/setup-service-addresses.sh
               /tmp/setup-service-addresses.sh
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               export AZTEC_NODE_URL=$BOOT_NODE_HOST
               export PXE_URL=$BOOT_NODE_HOST
               until curl -s -X POST -H 'content-type: application/json' \
                 -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \
                 $PXE_URL | grep -q '"enr:-'; do
-                echo "Waiting for PXE service..."
+                echo "Waiting for PXE service at ${PXE_URL}..."
                 sleep 5
               done
               echo "PXE service is ready!"
               set -e
               node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js setup-protocol-contracts --skipProofWait --l1-chain-id {{ .Values.ethereum.chainId }}
               echo "L2 contracts initialized"
           env:
+            - name: TELEMETRY
+              value: "{{ .Values.telemetry.enabled }}"
             - name: LOG_LEVEL
               value: "debug"
             - name: NETWORK_PUBLIC

diff --git a/spartan/aztec-network/templates/transaction-bot.yaml b/spartan/aztec-network/templates/transaction-bot.yaml
@@ -38,6 +38,7 @@ spec:
             - -c
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.bot.nodeUrl }}
               export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }}
               {{- else if .Values.network.public }}
@@ -63,6 +64,7 @@ spec:
             - "-c"
             - |
               source /shared/config/service-addresses
+              cat /shared/config/service-addresses
               {{- if .Values.bot.nodeUrl }}
               export AZTEC_NODE_URL={{ .Values.bot.nodeUrl }}
               {{- else if .Values.network.public }}