Update peerdas

ethpandaops · Jul 10, 2024 · e19e395 · e19e395
2 parents a507bd5 + cf83d6e
commit e19e395
Show file tree

Hide file tree

Showing 103 changed files with 7,953 additions and 4,325 deletions.
diff --git a/.github/workflows/goreleaser.yaml b/.github/workflows/goreleaser.yaml
@@ -9,7 +9,9 @@ jobs:
   goreleaser:
     permissions:
       contents: write
-    runs-on: ubuntu-latest
+    runs-on:
+      - self-hosted-ghr
+      - size-l-x64
     steps:
       - name: Checkout
         uses: actions/checkout@v3

diff --git a/.github/workflows/sentry-smoke-test.yaml b/.github/workflows/sentry-smoke-test.yaml
@@ -18,18 +18,12 @@ jobs:
       run: |
         docker build -t ethpandaops/xatu:local .
         echo "Xatu image is built."
-    - name: Install Kurtosis
-      run: |
-        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
-        sudo apt update
-        sudo apt install kurtosis-cli
     - name: Create Kurtosis config file
       run: |
-        cat <<EOF > network_params.yaml
+        cat <<EOF > /tmp/network_params.yaml
           participants:
             - el_type: geth
               cl_type: teku
-              cl_image: ethpandaops/teku:master-16c4354
             - el_type: nethermind
               cl_type: prysm
             - el_type: erigon
@@ -56,38 +50,23 @@ jobs:
             - voluntary_exit
             - contribution_and_proof
         <<EOF
-
-    - name: Start Ethereum network with Kurtosis and Run Xatu stack in parallel
-      timeout-minutes: 20
+    - name: Run Xatu stack
+      timeout-minutes: 10
       shell: bash
       run: |
-        echo "Starting Kurtosis..."
-        kurtosis run --enclave xatu github.com/kurtosis-tech/ethereum-package --args-file network_params.yaml & 
-        KURTOSIS_PID=$!
-
-        echo "Starting Xatu stack..."
         docker compose up --detach --quiet-pull &
-        XATU_STACK_PID=$!
-
-        echo "Waiting for Kurtosis to start..."
-        wait $KURTOSIS_PID
-        KURTOSIS_EXIT_CODE=$?
-
-        if [ $KURTOSIS_EXIT_CODE -ne 0 ]; then
-          echo "Kurtosis failed to start."
-          exit $KURTOSIS_EXIT_CODE
-        fi
-
-        echo "Waiting for Xatu stack to start..."
-        wait $XATU_STACK_PID
-        XATU_STACK_EXIT_CODE=$?
-
-        if [ $XATU_STACK_EXIT_CODE -ne 0 ]; then
-          echo "Xatu stack failed to start."
-          exit $XATU_STACK_EXIT_CODE
-        fi
-
-        echo "Kurtosis and Xatu stack have started successfully."
+    - name: Setup kurtosis testnet and run assertoor tests
+      id: kurtosis-setup
+      uses: ethpandaops/kurtosis-assertoor-github-action@v1
+      with:
+        ethereum_package_args: /tmp/network_params.yaml
+        await_assertoor_tests: false
+        enclave_name: xatu
+    - name: Show all kurtosis containers
+      env:
+        SERVICES: ${{ steps.kurtosis-setup.outputs.services }}
+      run: |
+        echo $SERVICES
     - name: Add all xatu-sentry containers to the xatu network
       run: |
         for container in $(docker ps --filter name=xatu-sentry --format "{{.Names}}"); do docker network connect xatu_xatu-net $container; echo $container; docker restart $container; done
@@ -139,7 +118,7 @@ jobs:
         # Check for any data in the tables before digging in to the individual sentries
         for table in "${tables[@]}"; do
             pretty_print "Checking $table table..." "none"
-            data_count=$(docker exec clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.$table" || true)
+            data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.$table" || true)
             if [[ $data_count -gt 0 ]]; then
                 pretty_print "$table table has $data_count entries" "green"
             else
@@ -155,7 +134,7 @@ jobs:
             for sentry in "${all_sentries[@]}"; do
                 pretty_print "Checking $table table for $sentry..." "none"
                 while true; do
-                    data_count=$(docker exec clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.$table WHERE meta_client_name = '$sentry'" || true)
+                    data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.$table WHERE meta_client_name = '$sentry'" || true)
                     if [[ $data_count -gt 0 ]]; then
                         pretty_print "$table has $data_count entries from $sentry" "green"
                         break

diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,5 @@ GeoLite2-City.mmdb
 __debug_bin*
 .vscode/launch.json
 sage.yaml
-.goreleaser.yaml.new
+.goreleaser.yaml.new
+.aider*
diff --git a/README.md b/README.md
@@ -2,6 +2,10 @@
 
 Ethereum network monitoring with collection clients and a centralized server for data pipelining.
 
+
+> [!IMPORTANT]  
+> EthPandaOps publishes all the data we collect with Xatu. Check it out [here](https://github.com/ethpandaops/xatu-data)
+
 ## Overview
 
 Xatu can run in multiple modes. Each mode can be run independently. The following diagram shows the different modes and how they interact with each other and other services.

diff --git a/deploy/local/docker-compose/grafana/datasources/datasources.yaml b/deploy/local/docker-compose/grafana/datasources/datasources.yaml
@@ -30,4 +30,11 @@ datasources:
       sslmode: disable
       tlsSkipVerify: true
       postgresVersion: 1500
-      database: xatu
+      database: xatu
+  - name: Tempo
+    type: tempo
+    access: proxy
+    uid: EbPG8fYoz
+    url: http://tempo:3200
+    jsonData:
+      httpMethod: GET
diff --git a/deploy/local/docker-compose/tempo.yaml b/deploy/local/docker-compose/tempo.yaml
@@ -0,0 +1,59 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 15m                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /var/tempo/wal             # where to store the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
diff --git a/deploy/local/docker-compose/vector-http-kafka.yaml b/deploy/local/docker-compose/vector-http-kafka.yaml
@@ -63,6 +63,10 @@ transforms:
       libp2p_trace_join: .event.name == "LIBP2P_TRACE_JOIN"
       libp2p_trace_handle_metadata: .event.name == "LIBP2P_TRACE_HANDLE_METADATA"
       libp2p_trace_handle_status: .event.name == "LIBP2P_TRACE_HANDLE_STATUS"
+      libp2p_trace_gossipsub_beacon_block: .event.name == "LIBP2P_TRACE_GOSSIPSUB_BEACON_BLOCK"
+      libp2p_trace_gossipsub_beacon_attestation: .event.name == "LIBP2P_TRACE_GOSSIPSUB_BEACON_ATTESTATION"
+      libp2p_trace_gossipsub_blob_sidecar: .event.name == "LIBP2P_TRACE_GOSSIPSUB_BLOB_SIDECAR"
+      beacon_api_eth_v1_beacon_validators: .event.name == "BEACON_API_ETH_V1_BEACON_VALIDATORS"
       libp2p_trace_gossipsub_beacon_block : .event.name == "LIBP2P_TRACE_GOSSIPSUB_BEACON_BLOCK"
       libp2p_trace_gossipsub_beacon_attestation : .event.name == "LIBP2P_TRACE_GOSSIPSUB_BEACON_ATTESTATION"
       libp2p_trace_gossipsub_beacon_data_column_sidecar : .event.name == "LIBP2P_TRACE_GOSSIPSUB_BEACON_DATA_COLUMN_SIDECAR"
@@ -857,6 +861,38 @@ sinks:
       enabled: true
     encoding:
       codec: json
+  libp2p_trace_gossipsub_blob_sidecar_kafka:
+    type: kafka
+    buffer:
+      max_events: 500000
+    batch:
+      timeout_secs: 0.5
+    inputs:
+      - xatu_server_events_router.libp2p_trace_gossipsub_blob_sidecar
+    bootstrap_servers: "${KAFKA_BROKERS}"
+    key_field: "event.id"
+    topic: libp2p-trace-gossipsub-blob-sidecar
+    compression: snappy
+    healthcheck:
+      enabled: true
+    encoding:
+      codec: json
+  beacon_api_eth_v1_beacon_validators_kafka:
+    type: kafka
+    buffer:
+      max_events: 500000
+    batch:
+      timeout_secs: 0.5
+    inputs:
+      - xatu_server_events_router.beacon_api_eth_v1_beacon_validators
+    bootstrap_servers: "${KAFKA_BROKERS}"
+    key_field: "event.id"
+    topic: beacon-api-eth-v1-beacon-validators
+    compression: snappy
+    healthcheck:
+      enabled: true
+    encoding:
+      codec: json
   libp2p_trace_gossipsub_beacon_data_column_sidecar_kafka:
     type: kafka
     buffer: