diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 54b780691ba..4e021a67ff0 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -15,16 +15,19 @@ source: COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts . build-contracts: + ARG RAYON_NUM_THREADS FROM +source # Install transpiler COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler RUN yarn - RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler ./bootstrap.sh + ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS + RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler ./bootstrap.sh SAVE ARTIFACT noir-contracts build-protocol-circuits: + ARG RAYON_NUM_THREADS LOCALLY LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin) FROM +source @@ -34,10 +37,13 @@ build-protocol-circuits: # Install bb COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb RUN yarn + + ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS RUN cd noir-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo ./bootstrap.sh SAVE ARTIFACT noir-protocol-circuits build-mock-protocol-circuits: + ARG RAYON_NUM_THREADS LOCALLY LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin) FROM +source @@ -47,10 +53,14 @@ build-mock-protocol-circuits: # Install bb COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb RUN yarn + + ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS + RUN echo "building with num threads $RAYON_NUM_THREADS" RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo ./bootstrap.sh SAVE ARTIFACT mock-protocol-circuits build: + ARG RAYON_NUM_THREADS FROM +source BUILD +build-contracts BUILD +build-protocol-circuits diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index 2f03276bd62..e715196dd22 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -33,9 +33,8 @@ case "$(uname)" in echo "Parallel builds not supported on this operating system" ;; esac -# This value may be too low. # If builds fail with an amount of free memory greater than this value then it should be increased. -MIN_PARALLEL_BUILD_MEMORY=32854492 +MIN_PARALLEL_BUILD_MEMORY=134217728 yarn diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 6435a258b43..777c04d8e98 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -20,7 +20,7 @@ node ./scripts/generate_variants.js echo "Compiling protocol circuits..." NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} -$NARGO compile --silence-warnings +RAYON_NUM_THREADS=1 $NARGO compile --silence-warnings BB_HASH=${BB_HASH:-$(cd ../../ && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)} echo Using BB hash $BB_HASH diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl index 632b4614882..cae1bdbf528 100644 --- a/spartan/aztec-network/templates/_helpers.tpl +++ b/spartan/aztec-network/templates/_helpers.tpl @@ -67,11 +67,11 @@ http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} {{- end -}} {{- define "aztec-network.otelCollectorMetricsEndpoint" -}} -{{ include "aztec-network.metricsHost" . }}:{{ .Values.metrics.ports.otlp }}/v1/metrics +http://metrics-opentelemetry-collector.metrics:4318/v1/metrics {{- end -}} {{- define "aztec-network.otelCollectorTracesEndpoint" -}} -{{ include "aztec-network.metricsHost" . }}:{{ .Values.metrics.ports.otlp }}/v1/traces +http://metrics-opentelemetry-collector.metrics:4318/v1/traces {{- end -}} diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml index ec2b2f762ae..4f9e4454f46 100644 --- a/spartan/aztec-network/templates/boot-node.yaml +++ b/spartan/aztec-network/templates/boot-node.yaml @@ -53,7 +53,7 @@ spec: - name: ETHEREUM_HOST value: {{ include "aztec-network.ethereumHost" . | quote }} containers: - - name: aztec + - name: boot-node image: {{ .Values.images.aztec.image }} command: [ diff --git a/spartan/aztec-network/templates/metrics.yaml b/spartan/aztec-network/templates/metrics.yaml deleted file mode 100644 index fc3d6d3ee15..00000000000 --- a/spartan/aztec-network/templates/metrics.yaml +++ /dev/null @@ -1,191 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "aztec-network.fullname" . }}-metrics - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - {{- include "aztec-network.selectorLabels" . | nindent 6 }} - app: metrics - template: - metadata: - labels: - {{- include "aztec-network.selectorLabels" . | nindent 8 }} - app: metrics - spec: - securityContext: - fsGroup: {{ .Values.metrics.fileSystemUser }} # required for Grafana write access to storage volume - containers: - - name: otel-collector - image: "{{ .Values.images.otelCollector.image }}" - imagePullPolicy: {{ .Values.images.otelCollector.pullPolicy }} - ports: - - containerPort: {{ .Values.metrics.ports.otlp }} - name: otlp - - containerPort: {{ .Values.metrics.ports.metrics }} - name: metrics - args: - - "--config=/etc/otel-collector-config.yaml" - volumeMounts: - - name: metrics-config - mountPath: /etc/otel-collector-config.yaml - subPath: otel-collector-config.yaml - - - name: prometheus - image: "{{ .Values.images.prometheus.image }}" - ports: - - containerPort: {{ .Values.metrics.ports.prometheus }} - volumeMounts: - - name: metrics-config - mountPath: /etc/prometheus/prometheus.yml - subPath: prometheus.yaml - - - name: grafana - image: "{{ .Values.images.grafana.image }}" - ports: - - containerPort: {{ .Values.metrics.ports.grafana }} - volumeMounts: - - name: metrics-config - mountPath: /etc/grafana/provisioning/dashboards/default.yml - subPath: grafana-default-dashboard.yml - - name: metrics-config - mountPath: /etc/grafana/provisioning/dashboards/aztec/aztec-dashboard.json - subPath: grafana-aztec-dashboard.json - - name: metrics-config - mountPath: /etc/grafana/provisioning/datasources/default.yaml - subPath: grafana-sources.yaml - - name: grafana-storage - mountPath: /var/lib/grafana - securityContext: - runAsUser: {{ .Values.metrics.fileSystemUser }} # required for Grafana write access to storage volume - - - name: jaeger - image: "{{ .Values.images.jaeger.image }}" - ports: - - containerPort: {{ .Values.metrics.ports.jaeger }} - volumes: - - name: metrics-config - configMap: - name: {{ include "aztec-network.fullname" . }}-metrics - - name: grafana-storage - persistentVolumeClaim: - claimName: {{ include "aztec-network.fullname" . }}-grafana-pvc ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "aztec-network.fullname" . }}-grafana-pvc - labels: - app: {{ include "aztec-network.name" . }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.metrics.storage }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "aztec-network.fullname" . }}-metrics - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -spec: - type: {{if .Values.network.public }}"LoadBalancer"{{ else }}"ClusterIP"{{ end }} - ports: - - port: {{ .Values.metrics.ports.otlp }} - targetPort: otlp - protocol: TCP - name: otlp - - port: {{ .Values.metrics.ports.metrics }} - targetPort: metrics - protocol: TCP - name: metrics - - port: {{ .Values.metrics.ports.prometheus }} - targetPort: {{ .Values.metrics.ports.prometheus }} - protocol: TCP - name: prometheus - - port: {{ .Values.metrics.ports.grafana }} - targetPort: {{ .Values.metrics.ports.grafana }} - protocol: TCP - name: grafana - - port: {{ .Values.metrics.ports.jaeger }} - targetPort: {{ .Values.metrics.ports.jaeger }} - protocol: TCP - name: jaeger-ui - - port: 14250 - targetPort: 14250 - protocol: TCP - name: jaeger-collector - selector: - {{- include "aztec-network.selectorLabels" . | nindent 4 }} - app: metrics ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "aztec-network.fullname" . }}-metrics - labels: - {{- include "aztec-network.labels" . | nindent 4 }} -data: - otel-collector-config.yaml: | - receivers: - otlp: - protocols: - http: - endpoint: "0.0.0.0:{{ .Values.metrics.ports.otlp }}" - - processors: - batch: - - exporters: - prometheus: - endpoint: "0.0.0.0:8889" - metric_expiration: 5m - otlp/jaeger: - endpoint: "0.0.0.0:4317" - tls: - insecure: true - service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [otlp/jaeger] - metrics: - receivers: [otlp] - processors: [batch] - exporters: [prometheus] - - prometheus.yaml: | - global: - evaluation_interval: 15s - scrape_interval: 15s - scrape_configs: - - job_name: otel-collector - static_configs: - - targets: ['localhost:8888'] - - job_name: aztec - static_configs: - - targets: ['localhost:8889'] - - grafana-sources.yaml: | - apiVersion: 1 - datasources: - - name: Prometheus - uid: aztec-node-metrics - type: prometheus - url: http://localhost:{{ .Values.metrics.ports.prometheus }} - editable: false - isDefault: true - jsonData: - timeInterval: 10s - - grafana-aztec-dashboard.json: | - {{ .Files.Get "files/grafana_dashboards/aztec/aztec-dashboard-all-in-one.json" | nindent 4 }} - - grafana-default-dashboard.yml: | - {{ .Files.Get "files/grafana_dashboards/default.yml" | nindent 4 }} diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml index dfcb8b4bbb5..109360cd583 100644 --- a/spartan/aztec-network/templates/prover-node.yaml +++ b/spartan/aztec-network/templates/prover-node.yaml @@ -34,7 +34,7 @@ spec: - name: ETHEREUM_HOST value: {{ include "aztec-network.ethereumHost" . | quote }} containers: - - name: aztec + - name: prover-node image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} command: diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml index 3725960a0b4..4066b796e2d 100644 --- a/spartan/aztec-network/templates/pxe.yaml +++ b/spartan/aztec-network/templates/pxe.yaml @@ -18,7 +18,7 @@ spec: app: pxe spec: containers: - - name: aztec + - name: pxe image: "{{ .Values.images.aztec.image }}" command: - "/bin/bash" diff --git a/spartan/aztec-network/templates/transaction-bot.yaml b/spartan/aztec-network/templates/transaction-bot.yaml index 798a28de571..cd0438eaa19 100644 --- a/spartan/aztec-network/templates/transaction-bot.yaml +++ b/spartan/aztec-network/templates/transaction-bot.yaml @@ -26,7 +26,7 @@ spec: - | until curl -s {{ include "aztec-network.bootNodeUrl" . }}/status; do echo waiting for aztec-node; sleep 2; done containers: - - name: aztec + - name: transaction-bot image: "{{ .Values.images.aztec.image }}" command: - "/bin/bash" diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml index 205a32a74cd..ee8c3c99e34 100644 --- a/spartan/aztec-network/templates/validator.yaml +++ b/spartan/aztec-network/templates/validator.yaml @@ -34,7 +34,7 @@ spec: - name: ETHEREUM_HOST value: {{ include "aztec-network.ethereumHost" . | quote }} containers: - - name: aztec + - name: validator image: "{{ .Values.images.aztec.image }}" imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} command: diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index 588eac5c34a..594f4084d9c 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -12,18 +12,6 @@ images: foundry: image: ghcr.io/foundry-rs/foundry@sha256:ce4b236f6760fdeb08e82267c9fa17647d29a374760bfe7ee01998fb8c0aaad7 pullPolicy: IfNotPresent - otelCollector: - image: otel/opentelemetry-collector-contrib - pullPolicy: IfNotPresent - prometheus: - image: prom/prometheus - pullPolicy: IfNotPresent - grafana: - image: grafana/grafana - pullPolicy: IfNotPresent - jaeger: - image: jaegertracing/all-in-one - pullPolicy: IfNotPresent bootNode: replicas: 1 @@ -31,7 +19,7 @@ bootNode: p2pPort: 40400 nodePort: 8080 logLevel: "debug" - debug: "aztec:*,-aztec:avm_simulator:*" + debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*" sequencer: maxSecondsBetweenBlocks: 0 minTxsPerBlock: 1 @@ -49,12 +37,12 @@ bootNode: storage: "8Gi" validator: - replicas: 1 + replicas: 0 service: p2pPort: 40400 nodePort: 8080 logLevel: "debug" - debug: "aztec:*,-aztec:avm_simulator:*" + debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*" sequencer: maxSecondsBetweenBlocks: 0 minTxsPerBlock: 1 @@ -76,7 +64,7 @@ proverNode: service: nodePort: 8080 logLevel: "debug" - debug: "aztec:*" + debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*" realProofs: false proverAgentEnabled: true resources: @@ -90,7 +78,7 @@ proverNode: pxe: logLevel: "debug" - debug: "aztec:*" + debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*" replicas: 1 service: port: 8080 @@ -111,7 +99,7 @@ pxe: bot: logLevel: "debug" - debug: "aztec:*" + debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*" replicas: 1 botPrivateKey: "0xcafe" txIntervalSeconds: 5 @@ -166,19 +154,3 @@ ethereum: limits: memory: "4Gi" cpu: "1" - -metrics: - ports: - otlp: 4316 - metrics: 4315 - prometheus: 9090 - grafana: 3000 - jaeger: 16686 - readinessProbe: - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - storage: "8Gi" - fileSystemUser: 472 \ No newline at end of file diff --git a/spartan/aztec-network/values/16-validators.yaml b/spartan/aztec-network/values/16-validators.yaml new file mode 100644 index 00000000000..0fd3faed552 --- /dev/null +++ b/spartan/aztec-network/values/16-validators.yaml @@ -0,0 +1,12 @@ +validator: + debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service" + replicas: 16 + resources: + requests: + memory: "512Mi" + validator: + disabled: false + +bootNode: + validator: + disabled: true diff --git a/spartan/metrics/Chart.lock b/spartan/metrics/Chart.lock index 85b5ec4e110..f47e003fee5 100644 --- a/spartan/metrics/Chart.lock +++ b/spartan/metrics/Chart.lock @@ -2,8 +2,17 @@ dependencies: - name: opentelemetry-collector repository: https://open-telemetry.github.io/opentelemetry-helm-charts version: 0.104.0 -- name: eck-stack - repository: https://helm.elastic.co - version: 0.12.1 -digest: sha256:e95083de14387953eb4093eb2c6b98cf889e532cd097b21b15b1896cfa117e7c -generated: "2024-09-06T19:31:29.086654335-04:00" +- name: loki + repository: https://grafana.github.io/helm-charts + version: 6.16.0 +- name: grafana + repository: https://grafana.github.io/helm-charts + version: 8.5.2 +- name: tempo + repository: https://grafana.github.io/helm-charts + version: 1.10.3 +- name: prometheus + repository: https://prometheus-community.github.io/helm-charts + version: 25.27.0 +digest: sha256:a8d35ff18d4850d37501946edc716a642c3743f50050cd99b724c7a95f7ee453 +generated: "2024-10-06T06:26:54.299530386-04:00" diff --git a/spartan/metrics/Chart.yaml b/spartan/metrics/Chart.yaml index 04a896c0fce..60355c2a92e 100644 --- a/spartan/metrics/Chart.yaml +++ b/spartan/metrics/Chart.yaml @@ -27,6 +27,18 @@ dependencies: - name: opentelemetry-collector version: 0.104.0 repository: https://open-telemetry.github.io/opentelemetry-helm-charts - - name: eck-stack - version: 0.12.1 - repository: https://helm.elastic.co + - name: loki + version: "6.16.0" + repository: "https://grafana.github.io/helm-charts" + condition: loki.enabled + - name: grafana + version: "8.5.2" + repository: "https://grafana.github.io/helm-charts" + condition: grafana.enabled + - name: tempo + version: "1.10.3" + repository: "https://grafana.github.io/helm-charts" + condition: tempo.enabled + - name: prometheus + version: "25.27.0" + repository: "https://prometheus-community.github.io/helm-charts" diff --git a/spartan/metrics/README.md b/spartan/metrics/README.md new file mode 100644 index 00000000000..5b9759300c7 --- /dev/null +++ b/spartan/metrics/README.md @@ -0,0 +1,31 @@ +This is a chart intended to be installed once per k8s cluster, and provides monitoring for all aztec network deployments within the cluster. + +## Installation + +```bash +# From the spartan/metrics directory +./install.sh +``` + +## Access +```bash +# From the spartan/metrics directory +./forward.sh +``` + +Follow the instructions printed to access the dashboard. + +## How does it work + +This chart installs: + - a daemonset of otel-collectors + - These automatically gather all logs from all pods in the cluster + - Aztec nodes also push metrics and traces to these collectors + - a prometheus instance + - This scrapes metrics from the otel-collectors + - a loki instance + - This stores logs from the otel-collectors + - a tempo instance + - This stores traces from the otel-collectors + - a grafana instance + - This provides a dashboard for the metrics, logs, and traces diff --git a/spartan/metrics/forward.sh b/spartan/metrics/forward.sh new file mode 100755 index 00000000000..8e1f80e707c --- /dev/null +++ b/spartan/metrics/forward.sh @@ -0,0 +1,9 @@ +#!/bin/bash +echo "Run this script after the metrics pods all come up. After running this script, port forward 3000 locally" +echo "Then navigate to https://localhost:3000 and login with user admin, and password as follows:" +echo +kubectl get secrets -n metrics metrics-grafana -o jsonpath='{.data.admin-password}' | base64 --decode +# skip two empty lines +echo +echo +kubectl port-forward -n metrics service/metrics-grafana 3000:80 diff --git a/spartan/metrics/install.sh b/spartan/metrics/install.sh new file mode 100755 index 00000000000..fda4203601f --- /dev/null +++ b/spartan/metrics/install.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eu + +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts +helm repo add grafana https://grafana.github.io/helm-charts +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm dependency update +helm upgrade metrics . -n metrics --install --create-namespace --atomic diff --git a/spartan/metrics/values.yaml b/spartan/metrics/values.yaml index d792b7d9fee..3d169270f68 100644 --- a/spartan/metrics/values.yaml +++ b/spartan/metrics/values.yaml @@ -1,16 +1,32 @@ opentelemetry-collector: - extraEnvs: - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - name: elasticsearch-es-elastic-user - key: elastic - mode: daemonset + service: + enabled: true + image: repository: "otel/opentelemetry-collector-contrib" + ports: + otlp-http: + enabled: true + containerPort: 4318 + servicePort: 4318 + hostPort: 4318 + protocol: TCP + otel-metrics: + enabled: true + containerPort: 8888 + servicePort: 8888 + hostPort: 8888 + protocol: TCP + aztec-metrics: + enabled: true + containerPort: 8889 + servicePort: 8889 + hostPort: 8889 + protocol: TCP + presets: logsCollection: enabled: true @@ -19,58 +35,324 @@ opentelemetry-collector: enabled: true config: exporters: - debug: {} - elasticsearch: - endpoint: "https://elasticsearch-es-http.metrics.svc:9200/" + # debug: + # verbosity: detailed + otlphttp/logs: + endpoint: http://metrics-loki.metrics:3100/otlp + otlp/tempo: + endpoint: http://metrics-tempo.metrics:4317 tls: - insecure_skip_verify: true - auth: - authenticator: basicauth + insecure: true + prometheus: + endpoint: ${env:MY_POD_IP}:8889 + metric_expiration: 5m extensions: - basicauth: - client_auth: - username: elastic - password: ${ELASTICSEARCH_PASSWORD} - # The health_check extension is mandatory for this chart. - # Without the health_check extension the collector will fail the readiness and liveliness probes. - # The health_check extension can be modified, but should never be removed. health_check: endpoint: ${env:MY_POD_IP}:13133 processors: batch: {} - transform: - error_mode: ignore - log_statements: - - context: log - statements: - - merge_maps(attributes, ParseJSON(body), "upsert") where IsMatch(body, "^\\{") - # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl/contexts/ottllog - - set(severity_number, 1) where attributes["level"] == "trace" - - set(severity_number, 5) where attributes["level"] == "debug" - - set(severity_number, 9) where attributes["level"] == "info" - - set(severity_number, 13) where attributes["level"] == "warn" - - set(severity_number, 17) where attributes["level"] == "error" - - set(severity_number, 21) where attributes["level"] == "fatal" - - set(body, attributes["message"]) - - set(attributes["level"], nil) - - set(attributes["message"], nil) receivers: otlp: protocols: http: endpoint: ${env:MY_POD_IP}:4318 + grpc: + endpoint: ${env:MY_POD_IP}:4317 service: - extensions: [basicauth, health_check] + extensions: [health_check] telemetry: metrics: address: ${env:MY_POD_IP}:8888 pipelines: logs: + receivers: + - otlp + processors: + - batch exporters: - - elasticsearch - - debug + - otlphttp/logs + traces: + receivers: + - otlp processors: - - transform - batch + exporters: + - otlp/tempo + metrics: receivers: - otlp + processors: + - batch + exporters: + - prometheus + # - debug + +# Enable and configure the Loki subchart +loki: + deploymentMode: SingleBinary + loki: + auth_enabled: false + commonConfig: + replication_factor: 1 + storage: + type: "filesystem" + schemaConfig: + configs: + - from: "2024-01-01" + store: tsdb + index: + prefix: loki_index_ + period: 24h + object_store: filesystem # we're storing on filesystem so there's no real persistence here. + schema: v13 + singleBinary: + replicas: 1 + read: + replicas: 0 + backend: + replicas: 0 + write: + replicas: 0 + +# Enable and configure the Tempo subchart +tempo: + minio: + enabled: true + mode: standalone + rootUser: grafana-tempo + rootPassword: supersecret + buckets: + # Default Tempo storage bucket + - name: tempo-traces + policy: none + purge: false + traces: + otlp: + grpc: + enabled: true + http: + enabled: true + zipkin: + enabled: false + jaeger: + thriftHttp: + enabled: false + opencensus: + enabled: false + +prometheus: + server: + global: + evaluation_interval: 15s + scrape_interval: 15s + serverFiles: + prometheus.yml: + scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["metrics-opentelemetry-collector.metrics:8888"] + - job_name: aztec + static_configs: + - targets: ["metrics-opentelemetry-collector.metrics:8889"] + +# Enable and configure Grafana +grafana: + datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Loki + type: loki + url: http://metrics-loki.metrics:3100 + - name: Tempo + type: tempo + url: http://metrics-tempo.metrics:3100 + - name: Prometheus + type: prometheus + uid: spartan-metrics-prometheus + isDefault: true + url: http://metrics-prometheus-server.metrics:80 + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + dashboards: + default: + # unfortunately, we can't use the `file` helper here, so we have to inline the dashboard + # json. This is a limitation of Helm. + # See https://github.com/helm/helm/issues/1892 + spartan-dashboard: + json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "aztec_archiver_block_height", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "L2 Block Height", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "smoke", + "value": "smoke" + }, + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "definition": "label_values(k8s_namespace_name)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Deployment", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(k8s_namespace_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Spartan Deployments", + "uid": "ae01y5sn1bls0a", + "version": 1, + "weekStart": "" + } diff --git a/spartan/scripts/forward_metrics_dashboard.sh b/spartan/scripts/forward_metrics_dashboard.sh deleted file mode 100755 index 8619cbe5cc8..00000000000 --- a/spartan/scripts/forward_metrics_dashboard.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -echo "Run this script after the metrics pods all come up. After running this script, port forward 5601 locally" -echo "Then navigate to https://localhost:5601 and login with user elastic password admin" -kubectl port-forward -n metrics service/metrics-eck-kibana-kb-http 5601:5601 \ No newline at end of file diff --git a/spartan/scripts/install_eck_operator.sh b/spartan/scripts/install_eck_operator.sh deleted file mode 100755 index 5923893ea61..00000000000 --- a/spartan/scripts/install_eck_operator.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -ex - -version=2.14.0 - -kubectl create -f https://download.elastic.co/downloads/eck/$version/crds.yaml -kubectl apply -f https://download.elastic.co/downloads/eck/$version/operator.yaml diff --git a/spartan/scripts/setup_metrics_dashboard.sh b/spartan/scripts/setup_metrics_dashboard.sh deleted file mode 100755 index 3b618116883..00000000000 --- a/spartan/scripts/setup_metrics_dashboard.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts -helm repo add elastic https://helm.elastic.co -helm repo update -helm dependency build "$(git rev-parse --show-toplevel)/spartan/metrics/" -kubectl create -f https://download.elastic.co/downloads/eck/2.14.0/crds.yaml 2>/dev/null -kubectl create secret generic elasticsearch-es-elastic-user --from-literal=elastic=admin -n metrics -kubectl apply -f https://download.elastic.co/downloads/eck/2.14.0/operator.yaml -helm upgrade --install metrics "$(git rev-parse --show-toplevel)/spartan/metrics/" --wait --namespace metrics --create-namespace --wait-for-jobs=true --timeout=30m \ No newline at end of file diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 2011b633a3b..ab6bf4c2ad6 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -31,6 +31,7 @@ deps: RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules build: + ARG RAYON_NUM_THREADS # Prefetch targets to not wait for +deps. BUILD ../barretenberg/cpp/+build BUILD ../barretenberg/ts/+build @@ -57,6 +58,7 @@ build-dev: SAVE ARTIFACT /usr/src /usr/src cli-base: + ARG RAYON_NUM_THREADS FROM +build # Remove a bunch of stuff that we don't need that takes up space. RUN rm -rf \ @@ -145,12 +147,14 @@ export-cli-wallet: SAVE IMAGE --push aztecprotocol/cli-wallet:${DIST_TAG}${ARCH:+-$ARCH} aztec-prod: + ARG RAYON_NUM_THREADS FROM +cli-base RUN yarn workspaces focus @aztec/aztec --production && yarn cache clean COPY --dir +rollup-verifier-contract/usr/src/bb /usr/src SAVE ARTIFACT /usr/src /usr/src aztec: + ARG RAYON_NUM_THREADS FROM ubuntu:noble RUN apt update && apt install nodejs curl jq -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY +aztec-prod/usr/src /usr/src @@ -191,6 +195,7 @@ export-aztec-faucet: # We care about creating a slimmed down e2e image because we have to serialize it from earthly to docker for running. end-to-end-prod: + ARG RAYON_NUM_THREADS FROM +cli-base RUN yarn workspaces focus @aztec/end-to-end @aztec/cli-wallet --production && yarn cache clean COPY --dir +rollup-verifier-contract/usr/src/bb /usr/src @@ -224,6 +229,7 @@ end-to-end-base: RUN ln -s /usr/src/yarn-project/.yarn/releases/yarn-3.6.3.cjs /usr/local/bin/yarn end-to-end: + ARG RAYON_NUM_THREADS FROM +end-to-end-base COPY +anvil/anvil /opt/foundry/bin/anvil @@ -238,10 +244,12 @@ scripts-prod: SAVE ARTIFACT /usr/src /usr/src all: + ARG RAYON_NUM_THREADS BUILD +aztec BUILD +end-to-end export-aztec: + ARG RAYON_NUM_THREADS ARG EARTHLY_GIT_HASH FROM +aztec SAVE IMAGE aztecprotocol/aztec:$EARTHLY_GIT_HASH @@ -258,6 +266,7 @@ export-end-to-end: SAVE IMAGE aztecprotocol/end-to-end:$EARTHLY_GIT_HASH export-e2e-test-images: + ARG RAYON_NUM_THREADS BUILD +export-aztec BUILD +export-end-to-end diff --git a/yarn-project/aztec/src/logging.ts b/yarn-project/aztec/src/logging.ts index e407b1c3f39..cd5fef656b0 100644 --- a/yarn-project/aztec/src/logging.ts +++ b/yarn-project/aztec/src/logging.ts @@ -30,11 +30,37 @@ function createWinstonLocalFileLogger() { }); } +function extractNegativePatterns(debugString: string): string[] { + return ( + debugString + .split(',') + .filter(p => p.startsWith('-')) + // Remove the leading '-' from the pattern + .map(p => p.slice(1)) + // Remove any '*' from the pattern + .map(p => p.replace('*', '')) + ); +} + /** Creates a winston logger that logs everything to stdout in json format */ -function createWinstonJsonStdoutLogger() { +function createWinstonJsonStdoutLogger( + debugString: string = process.env.DEBUG ?? + 'aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*', +) { + const ignorePatterns = extractNegativePatterns(debugString); + const ignoreAztecPattern = format(info => { + if (ignorePatterns.some(pattern => info.module.startsWith(pattern))) { + return false; // Skip logging this message + } + return info; + }); return winston.createLogger({ level: process.env.LOG_LEVEL ?? 'info', - transports: [new winston.transports.Console({ format: format.combine(format.timestamp(), format.json()) })], + transports: [ + new winston.transports.Console({ + format: format.combine(format.timestamp(), ignoreAztecPattern(), format.json()), + }), + ], }); }