Fix the e2e test

EspressoSystems · Apr 17, 2024 · 4053075 · 4053075
1 parent d8d444f
commit 4053075
Show file tree

Hide file tree

Showing 3 changed files with 147 additions and 59 deletions.
diff --git a/system_tests/espresso-e2e/.env b/system_tests/espresso-e2e/.env
@@ -12,32 +12,46 @@ L1_BLOCK_TIME_SEC=3
 ASYNC_STD_THREAD_COUNT=1
 
 # Internal port inside container
-ESPRESSO_WEB_SERVER_PORT=40000
+ESPRESSO_CDN_SERVER_PORT=40000
+ESPRESSO_CDN_SERVER_METRICS_PORT=9090
 ESPRESSO_ORCHESTRATOR_PORT=40001
-ESPRESSO_CONSENSUS_SERVER_PORT=40002
-ESPRESSO_DA_SERVER_PORT=40003
-ESPRESSO_SEQUENCER_DA_SERVER_URL=http://da-server:$ESPRESSO_WEB_SERVER_PORT
-ESPRESSO_SEQUENCER_CONSENSUS_SERVER_URL=http://consensus-server:$ESPRESSO_WEB_SERVER_PORT
-ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://orchestrator:$ESPRESSO_ORCHESTRATOR_PORT
+ESPRESSO_ORCHESTRATOR_NUM_NODES=2
+ESPRESSO_ORCHESTRATOR_START_DELAY=5s
+ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT=30s
+ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS=50
+ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME=1s
+ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=2s
+ESPRESSO_SEQUENCER_CDN_ENDPOINT=marshal-0:${ESPRESSO_CDN_SERVER_PORT}
+ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://orchestrator:${ESPRESSO_ORCHESTRATOR_PORT}
 ESPRESSO_SEQUENCER_API_PORT=50000
+ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT=42000
 ESPRESSO_SEQUENCER1_API_PORT=50001
-ESPRESSO_SEQUENCER_URL=http://espresso-sequencer0:$ESPRESSO_SEQUENCER_API_PORT
+ESPRESSO_SEQUENCER_URL=http://espresso-sequencer0:${ESPRESSO_SEQUENCER_API_PORT}
 ESPRESSO_SEQUENCER_STORAGE_PATH=/store/sequencer
 ESPRESSO_SEQUENCER_L1_PORT=8545
 ESPRESSO_SEQUENCER_L1_WS_PORT=8546
-ESPRESSO_SEQUENCER_L1_PROVIDER=http://host.docker.internal:$ESPRESSO_SEQUENCER_L1_PORT
-ESPRESSO_SEQUENCER_L1_WS_PROVIDER=ws://host.docker.internal:$ESPRESSO_SEQUENCER_L1_WS_PORT
-ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG=true
+ESPRESSO_SEQUENCER_L1_PROVIDER=http://host.docker.internal:${ESPRESSO_SEQUENCER_L1_PORT}
+ESPRESSO_SEQUENCER_L1_WS_PROVIDER=ws://host.docker.internal:${ESPRESSO_SEQUENCER_L1_WS_PORT}
 ESPRESSO_COMMITMENT_ETH_MNEMONIC="indoor dish desk flag debris potato excuse depart ticket judge file exit"
 ESPRESSO_SEQUENCER_ETH_MNEMONIC="test test test test test test test test test test test junk"
-ESPRESSO_DEPLOYER_ACCOUNT_INDEX=5
+ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS=0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f
 ESPRESSO_COMMITMENT_TASK_PORT=60000
+ESPRESSO_STATE_RELAY_SERVER_PORT=40004
+ESPRESSO_SEQUENCER_DB_PORT=5432
+ESPRESSO_STATE_RELAY_SERVER_URL=http://state-relay-server:${ESPRESSO_STATE_RELAY_SERVER_PORT}
+
+ESPRESSO_STATE_SIGNATURE_WEIGHT_THRESHOLD=3
 
 # Ethereum accounts (note 11-15 are used by the sequencer nodes)
 ESPRESSO_SEQUENCER_HOTSHOT_ACCOUNT_INDEX=5
+ESPRESSO_BUILDER_ETH_ACCOUNT_INDEX=5
+ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX=5
+ESPRESSO_DEPLOYER_ACCOUNT_INDEX=5
 
 # Conctracts
 ESPRESSO_SEQUENCER_HOTSHOT_ADDRESS=0x217788c286797d56cd59af5e493f3699c39cbbe8
+ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS=0xb075b82c7a23e0994df4793422a1f03dbcf9136f
+ESPRESSO_SEQUENCER_LIGHTCLIENT_ADDRESS=$ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS
 
 # Example sequencer demo private keys
 ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_0=BLS_SIGNING_KEY~lNDh4Pn-pTAyzyprOAFdXHwhrKhEwqwtMtkD3CZF4x3o
@@ -46,9 +60,14 @@ ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_1=BLS_SIGNING_KEY~-DO72m_SFl6NQMYknm
 ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_0=SCHNORR_SIGNING_KEY~XxPSER8Vh3nFj_m7cUQ--96JfKrycrSKyRQximkQigCo
 ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_1=SCHNORR_SIGNING_KEY~2NpKtvY5F0u1LWgYws-JeX1vDdp5CfECuaMMYxyz4gDM
 
-ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX=8
-ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS=0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f
+# Builder service
+ESPRESSO_BUILDER_L1_PROVIDER=${ESPRESSO_SEQUENCER_L1_PROVIDER}
+ESPRESSO_BUILDER_ETH_MNEMONIC=${ESPRESSO_SEQUENCER_ETH_MNEMONIC}
+ESPRESSO_BUILDER_SERVER_PORT=41003
+ESPRESSO_BUILDER_PRIVATE_STAKING_KEY=BLS_SIGNING_KEY~tI9He_sCnEbfEajycUXz9Scfy6ocLr0yL9ceD53s8QPa
+ESPRESSO_BUILDER_PRIVATE_STATE_KEY=SCHNORR_SIGNING_KEY~IftHINvgzqcd9agX13HHY3Uhz8vsH46i8soKgV7ZUQV-
+ESPRESSO_BUILDER_CHANNEL_CAPACITY=1024
+ESPRESSO_BUILDER_BOOTSTRAPPED_VIEW=0
 
-ESPRESSO_STATE_RELAY_SERVER_PORT=40004
-ESPRESSO_STATE_SIGNATURE_WEIGHT_THRESHOLD=3
-ESPRESSO_STATE_RELAY_SERVER_URL=http://state-relay-server:$ESPRESSO_STATE_RELAY_SERVER_PORT
+ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0=7000
+ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1=7001
diff --git a/system_tests/espresso-e2e/docker-compose.yaml b/system_tests/espresso-e2e/docker-compose.yaml
@@ -22,37 +22,86 @@ services:
       - "$ESPRESSO_ORCHESTRATOR_PORT:$ESPRESSO_ORCHESTRATOR_PORT"
     environment:
       - ESPRESSO_ORCHESTRATOR_PORT
-      - ESPRESSO_ORCHESTRATOR_NUM_NODES=2
-      - ESPRESSO_ORCHESTRATOR_START_DELAY=5s
-      - ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT=30s
-      - ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS=1
-      - ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME=0s
-      - ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=1s
+      - ESPRESSO_ORCHESTRATOR_NUM_NODES
+      - ESPRESSO_ORCHESTRATOR_START_DELAY
+      - ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT
+      - ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS
+      - ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME
+      - ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME
       - RUST_LOG
       - RUST_LOG_FORMAT
 
-  da-server:
-    image: ghcr.io/espressosystems/espresso-sequencer/web-server:arbitrum-integrationmusl
-    ports:
-      - "$ESPRESSO_DA_SERVER_PORT:$ESPRESSO_WEB_SERVER_PORT"
+  # We use KeyDB (a Redis variant) to maintain consistency between
+  # different parts of the CDN
+  keydb:
+    image: docker.io/eqalpha/keydb:latest
+    command: ["--requirepass", "changemeplease!!"]
+    healthcheck:
+      # Attempt to PING the database
+      test: keydb-cli --pass changemeplease!! --raw incr PING
+      interval: 5s
+      timeout: 4s
+      retries: 20
+
+  # The CDN system's main entry point; where users contact first.
+  marshal-0:
     environment:
-      - ESPRESSO_WEB_SERVER_PORT
-      - RUST_LOG=error
-      - RUST_LOG_FORMAT
+      - RUST_LOG
+      - ESPRESSO_CDN_SERVER_METRICS_PORT
+    image: ghcr.io/espressosystems/espresso-sequencer/cdn-marshal:main
+    command:
+      - cdn-marshal
+      - -d
+      - redis://:changemeplease!!@keydb:6379
+      - --bind-port
+      - $ESPRESSO_CDN_SERVER_PORT
+      - --metrics-enabled
+      - --metrics-port
+      - $ESPRESSO_CDN_SERVER_METRICS_PORT
     depends_on:
-      orchestrator:
+      keydb:
         condition: service_healthy
 
-  consensus-server:
-    image: ghcr.io/espressosystems/espresso-sequencer/web-server:arbitrum-integrationmusl
-    ports:
-      - "$ESPRESSO_CONSENSUS_SERVER_PORT:$ESPRESSO_WEB_SERVER_PORT"
+  # A broker is the main message-routing unit of the CDN
+  broker-0:
     environment:
-      - ESPRESSO_WEB_SERVER_PORT
-      - RUST_LOG=error
-      - RUST_LOG_FORMAT
+      - RUST_LOG
+      - ESPRESSO_CDN_SERVER_METRICS_PORT
+    image: ghcr.io/espressosystems/espresso-sequencer/cdn-broker:main
+    command:
+    - cdn-broker
+    - -d
+    - redis://:changemeplease!!@keydb:6379
+    - --public-advertise-address
+    - broker-0:1738
+    - --private-advertise-address
+    - broker-0:1739
+    - --metrics-enabled
+    - --metrics-port
+    - $ESPRESSO_CDN_SERVER_METRICS_PORT
     depends_on:
-      orchestrator:
+      keydb:
+        condition: service_healthy
+
+  # A broker is the main message-routing unit of the CDN
+  broker-1:
+    environment:
+      - RUST_LOG
+      - ESPRESSO_CDN_SERVER_METRICS_PORT
+    image: ghcr.io/espressosystems/espresso-sequencer/cdn-broker:main
+    command:
+    - cdn-broker
+    - -d
+    - redis://:changemeplease!!@keydb:6379
+    - --public-advertise-address
+    - broker-1:1738
+    - --private-advertise-address
+    - broker-1:1739
+    - --metrics-enabled
+    - --metrics-port
+    - $ESPRESSO_CDN_SERVER_METRICS_PORT
+    depends_on:
+      keydb:
         condition: service_healthy
 
   espresso-sequencer0:
@@ -63,15 +112,16 @@ services:
     command: sequencer -- http -- query -- submit -- status
     environment:
       - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL
-      - ESPRESSO_SEQUENCER_DA_SERVER_URL
-      - ESPRESSO_SEQUENCER_CONSENSUS_SERVER_URL
+      - ESPRESSO_SEQUENCER_CDN_ENDPOINT
       - ESPRESSO_SEQUENCER_API_PORT
+      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
       - ESPRESSO_SEQUENCER_STORAGE_PATH
       - ESPRESSO_SEQUENCER_L1_PROVIDER
-      - ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG
       - ESPRESSO_STATE_RELAY_SERVER_URL
       - ESPRESSO_SEQUENCER_PRIVATE_STAKING_KEY=$ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_0
-      - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_1
+      - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_0
+      - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0
+      - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=espresso-sequencer0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0
       - ESPRESSO_SEQUENCER_ETH_MNEMONIC
       - ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX
       - ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS
@@ -81,12 +131,16 @@ services:
     depends_on:
       orchestrator:
         condition: service_healthy
-      consensus-server:
-        condition: service_healthy
-      da-server:
+      sequencer-db:
         condition: service_healthy
       state-relay-server:
         condition: service_healthy
+      broker-0:
+        condition: service_healthy
+      broker-1:
+        condition: service_healthy
+      marshal-0:
+        condition: service_healthy
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -98,30 +152,35 @@ services:
     command: sequencer -- http
     environment:
       - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL
-      - ESPRESSO_SEQUENCER_DA_SERVER_URL
-      - ESPRESSO_SEQUENCER_CONSENSUS_SERVER_URL
+      - ESPRESSO_SEQUENCER_CDN_ENDPOINT
       - ESPRESSO_SEQUENCER_API_PORT
+      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
       - ESPRESSO_SEQUENCER_STORAGE_PATH
       - ESPRESSO_SEQUENCER_L1_PROVIDER
-      - ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG
+      - ESPRESSO_STATE_RELAY_SERVER_URL
       - ESPRESSO_SEQUENCER_PRIVATE_STAKING_KEY=$ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_1
       - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_1
+      - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1
+      - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=espresso-sequencer1:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1
       - ESPRESSO_SEQUENCER_ETH_MNEMONIC
       - ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX
       - ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS
-      - ESPRESSO_STATE_RELAY_SERVER_URL
+      - ESPRESSO_SEQUENCER_STATE_PEERS=http://espresso-sequencer0:$ESPRESSO_SEQUENCER_API_PORT
       - RUST_LOG
       - RUST_LOG_FORMAT
-      - ESPRESSO_SEQUENCER_STATE_PEERS=http://espresso-sequencer0:$ESPRESSO_SEQUENCER_API_PORT
     depends_on:
       orchestrator:
         condition: service_healthy
-      consensus-server:
-        condition: service_healthy
-      da-server:
+      sequencer-db:
         condition: service_healthy
       state-relay-server:
         condition: service_healthy
+      broker-0:
+        condition: service_healthy
+      broker-1:
+        condition: service_healthy
+      marshal-0:
+        condition: service_healthy
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -141,10 +200,6 @@ services:
     depends_on:
       espresso-sequencer0:
         condition: service_healthy
-      consensus-server:
-        condition: service_healthy
-      da-server:
-        condition: service_healthy
       deploy-contracts:
         condition: service_completed_successfully
 
@@ -160,3 +215,19 @@ services:
       - ESPRESSO_STATE_SIGNATURE_WEIGHT_THRESHOLD
       - RUST_LOG
       - RUST_LOG_FORMAT
+
+  sequencer-db:
+    image: postgres
+    user: postgres
+    ports:
+      - "$ESPRESSO_SEQUENCER_DB_PORT:5432"
+    environment:
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_USER=root
+    healthcheck:
+      # Postgres can be falsely "ready" once before running init scripts.
+      # See https://github.com/docker-library/postgres/issues/146 for discussion.
+      test: "pg_isready && sleep 1 && pg_isready"
+      interval: 5s
+      timeout: 4s
+      retries: 20
diff --git a/system_tests/espresso_e2e_test.go b/system_tests/espresso_e2e_test.go
@@ -56,8 +56,6 @@ func runEspresso(t *testing.T, ctx context.Context) func() {
 	invocation := []string{"compose", "up", "-d", "--build"}
 	nodes := []string{
 		"orchestrator",
-		"da-server",
-		"consensus-server",
 		"espresso-sequencer0",
 		"espresso-sequencer1",
 		"commitment-task",