From 4053075c9105421e73d020cbe68409e196122d6d Mon Sep 17 00:00:00 2001 From: ImJeremyHe Date: Wed, 17 Apr 2024 14:49:02 +0800 Subject: [PATCH] Fix the e2e test --- system_tests/espresso-e2e/.env | 51 ++++-- system_tests/espresso-e2e/docker-compose.yaml | 153 +++++++++++++----- system_tests/espresso_e2e_test.go | 2 - 3 files changed, 147 insertions(+), 59 deletions(-) diff --git a/system_tests/espresso-e2e/.env b/system_tests/espresso-e2e/.env index da69425997..5af96d5898 100644 --- a/system_tests/espresso-e2e/.env +++ b/system_tests/espresso-e2e/.env @@ -12,32 +12,46 @@ L1_BLOCK_TIME_SEC=3 ASYNC_STD_THREAD_COUNT=1 # Internal port inside container -ESPRESSO_WEB_SERVER_PORT=40000 +ESPRESSO_CDN_SERVER_PORT=40000 +ESPRESSO_CDN_SERVER_METRICS_PORT=9090 ESPRESSO_ORCHESTRATOR_PORT=40001 -ESPRESSO_CONSENSUS_SERVER_PORT=40002 -ESPRESSO_DA_SERVER_PORT=40003 -ESPRESSO_SEQUENCER_DA_SERVER_URL=http://da-server:$ESPRESSO_WEB_SERVER_PORT -ESPRESSO_SEQUENCER_CONSENSUS_SERVER_URL=http://consensus-server:$ESPRESSO_WEB_SERVER_PORT -ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://orchestrator:$ESPRESSO_ORCHESTRATOR_PORT +ESPRESSO_ORCHESTRATOR_NUM_NODES=2 +ESPRESSO_ORCHESTRATOR_START_DELAY=5s +ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT=30s +ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS=50 +ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME=1s +ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=2s +ESPRESSO_SEQUENCER_CDN_ENDPOINT=marshal-0:${ESPRESSO_CDN_SERVER_PORT} +ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://orchestrator:${ESPRESSO_ORCHESTRATOR_PORT} ESPRESSO_SEQUENCER_API_PORT=50000 +ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT=42000 ESPRESSO_SEQUENCER1_API_PORT=50001 -ESPRESSO_SEQUENCER_URL=http://espresso-sequencer0:$ESPRESSO_SEQUENCER_API_PORT +ESPRESSO_SEQUENCER_URL=http://espresso-sequencer0:${ESPRESSO_SEQUENCER_API_PORT} ESPRESSO_SEQUENCER_STORAGE_PATH=/store/sequencer ESPRESSO_SEQUENCER_L1_PORT=8545 ESPRESSO_SEQUENCER_L1_WS_PORT=8546 -ESPRESSO_SEQUENCER_L1_PROVIDER=http://host.docker.internal:$ESPRESSO_SEQUENCER_L1_PORT -ESPRESSO_SEQUENCER_L1_WS_PROVIDER=ws://host.docker.internal:$ESPRESSO_SEQUENCER_L1_WS_PORT -ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG=true +ESPRESSO_SEQUENCER_L1_PROVIDER=http://host.docker.internal:${ESPRESSO_SEQUENCER_L1_PORT} +ESPRESSO_SEQUENCER_L1_WS_PROVIDER=ws://host.docker.internal:${ESPRESSO_SEQUENCER_L1_WS_PORT} ESPRESSO_COMMITMENT_ETH_MNEMONIC="indoor dish desk flag debris potato excuse depart ticket judge file exit" ESPRESSO_SEQUENCER_ETH_MNEMONIC="test test test test test test test test test test test junk" -ESPRESSO_DEPLOYER_ACCOUNT_INDEX=5 +ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS=0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f ESPRESSO_COMMITMENT_TASK_PORT=60000 +ESPRESSO_STATE_RELAY_SERVER_PORT=40004 +ESPRESSO_SEQUENCER_DB_PORT=5432 +ESPRESSO_STATE_RELAY_SERVER_URL=http://state-relay-server:${ESPRESSO_STATE_RELAY_SERVER_PORT} + +ESPRESSO_STATE_SIGNATURE_WEIGHT_THRESHOLD=3 # Ethereum accounts (note 11-15 are used by the sequencer nodes) ESPRESSO_SEQUENCER_HOTSHOT_ACCOUNT_INDEX=5 +ESPRESSO_BUILDER_ETH_ACCOUNT_INDEX=5 +ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX=5 +ESPRESSO_DEPLOYER_ACCOUNT_INDEX=5 # Conctracts ESPRESSO_SEQUENCER_HOTSHOT_ADDRESS=0x217788c286797d56cd59af5e493f3699c39cbbe8 +ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS=0xb075b82c7a23e0994df4793422a1f03dbcf9136f +ESPRESSO_SEQUENCER_LIGHTCLIENT_ADDRESS=$ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS # Example sequencer demo private keys ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_0=BLS_SIGNING_KEY~lNDh4Pn-pTAyzyprOAFdXHwhrKhEwqwtMtkD3CZF4x3o @@ -46,9 +60,14 @@ ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_1=BLS_SIGNING_KEY~-DO72m_SFl6NQMYknm ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_0=SCHNORR_SIGNING_KEY~XxPSER8Vh3nFj_m7cUQ--96JfKrycrSKyRQximkQigCo ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_1=SCHNORR_SIGNING_KEY~2NpKtvY5F0u1LWgYws-JeX1vDdp5CfECuaMMYxyz4gDM -ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX=8 -ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS=0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f +# Builder service +ESPRESSO_BUILDER_L1_PROVIDER=${ESPRESSO_SEQUENCER_L1_PROVIDER} +ESPRESSO_BUILDER_ETH_MNEMONIC=${ESPRESSO_SEQUENCER_ETH_MNEMONIC} +ESPRESSO_BUILDER_SERVER_PORT=41003 +ESPRESSO_BUILDER_PRIVATE_STAKING_KEY=BLS_SIGNING_KEY~tI9He_sCnEbfEajycUXz9Scfy6ocLr0yL9ceD53s8QPa +ESPRESSO_BUILDER_PRIVATE_STATE_KEY=SCHNORR_SIGNING_KEY~IftHINvgzqcd9agX13HHY3Uhz8vsH46i8soKgV7ZUQV- +ESPRESSO_BUILDER_CHANNEL_CAPACITY=1024 +ESPRESSO_BUILDER_BOOTSTRAPPED_VIEW=0 -ESPRESSO_STATE_RELAY_SERVER_PORT=40004 -ESPRESSO_STATE_SIGNATURE_WEIGHT_THRESHOLD=3 -ESPRESSO_STATE_RELAY_SERVER_URL=http://state-relay-server:$ESPRESSO_STATE_RELAY_SERVER_PORT \ No newline at end of file +ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0=7000 +ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1=7001 diff --git a/system_tests/espresso-e2e/docker-compose.yaml b/system_tests/espresso-e2e/docker-compose.yaml index d78a04046b..df56aa87c5 100644 --- a/system_tests/espresso-e2e/docker-compose.yaml +++ b/system_tests/espresso-e2e/docker-compose.yaml @@ -22,37 +22,86 @@ services: - "$ESPRESSO_ORCHESTRATOR_PORT:$ESPRESSO_ORCHESTRATOR_PORT" environment: - ESPRESSO_ORCHESTRATOR_PORT - - ESPRESSO_ORCHESTRATOR_NUM_NODES=2 - - ESPRESSO_ORCHESTRATOR_START_DELAY=5s - - ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT=30s - - ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS=1 - - ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME=0s - - ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=1s + - ESPRESSO_ORCHESTRATOR_NUM_NODES + - ESPRESSO_ORCHESTRATOR_START_DELAY + - ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT + - ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS + - ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME + - ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME - RUST_LOG - RUST_LOG_FORMAT - da-server: - image: ghcr.io/espressosystems/espresso-sequencer/web-server:arbitrum-integrationmusl - ports: - - "$ESPRESSO_DA_SERVER_PORT:$ESPRESSO_WEB_SERVER_PORT" + # We use KeyDB (a Redis variant) to maintain consistency between + # different parts of the CDN + keydb: + image: docker.io/eqalpha/keydb:latest + command: ["--requirepass", "changemeplease!!"] + healthcheck: + # Attempt to PING the database + test: keydb-cli --pass changemeplease!! --raw incr PING + interval: 5s + timeout: 4s + retries: 20 + + # The CDN system's main entry point; where users contact first. + marshal-0: environment: - - ESPRESSO_WEB_SERVER_PORT - - RUST_LOG=error - - RUST_LOG_FORMAT + - RUST_LOG + - ESPRESSO_CDN_SERVER_METRICS_PORT + image: ghcr.io/espressosystems/espresso-sequencer/cdn-marshal:main + command: + - cdn-marshal + - -d + - redis://:changemeplease!!@keydb:6379 + - --bind-port + - $ESPRESSO_CDN_SERVER_PORT + - --metrics-enabled + - --metrics-port + - $ESPRESSO_CDN_SERVER_METRICS_PORT depends_on: - orchestrator: + keydb: condition: service_healthy - consensus-server: - image: ghcr.io/espressosystems/espresso-sequencer/web-server:arbitrum-integrationmusl - ports: - - "$ESPRESSO_CONSENSUS_SERVER_PORT:$ESPRESSO_WEB_SERVER_PORT" + # A broker is the main message-routing unit of the CDN + broker-0: environment: - - ESPRESSO_WEB_SERVER_PORT - - RUST_LOG=error - - RUST_LOG_FORMAT + - RUST_LOG + - ESPRESSO_CDN_SERVER_METRICS_PORT + image: ghcr.io/espressosystems/espresso-sequencer/cdn-broker:main + command: + - cdn-broker + - -d + - redis://:changemeplease!!@keydb:6379 + - --public-advertise-address + - broker-0:1738 + - --private-advertise-address + - broker-0:1739 + - --metrics-enabled + - --metrics-port + - $ESPRESSO_CDN_SERVER_METRICS_PORT depends_on: - orchestrator: + keydb: + condition: service_healthy + + # A broker is the main message-routing unit of the CDN + broker-1: + environment: + - RUST_LOG + - ESPRESSO_CDN_SERVER_METRICS_PORT + image: ghcr.io/espressosystems/espresso-sequencer/cdn-broker:main + command: + - cdn-broker + - -d + - redis://:changemeplease!!@keydb:6379 + - --public-advertise-address + - broker-1:1738 + - --private-advertise-address + - broker-1:1739 + - --metrics-enabled + - --metrics-port + - $ESPRESSO_CDN_SERVER_METRICS_PORT + depends_on: + keydb: condition: service_healthy espresso-sequencer0: @@ -63,15 +112,16 @@ services: command: sequencer -- http -- query -- submit -- status environment: - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL - - ESPRESSO_SEQUENCER_DA_SERVER_URL - - ESPRESSO_SEQUENCER_CONSENSUS_SERVER_URL + - ESPRESSO_SEQUENCER_CDN_ENDPOINT - ESPRESSO_SEQUENCER_API_PORT + - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT - ESPRESSO_SEQUENCER_STORAGE_PATH - ESPRESSO_SEQUENCER_L1_PROVIDER - - ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG - ESPRESSO_STATE_RELAY_SERVER_URL - ESPRESSO_SEQUENCER_PRIVATE_STAKING_KEY=$ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_0 - - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_1 + - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_0 + - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0 + - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=espresso-sequencer0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0 - ESPRESSO_SEQUENCER_ETH_MNEMONIC - ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX - ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS @@ -81,12 +131,16 @@ services: depends_on: orchestrator: condition: service_healthy - consensus-server: - condition: service_healthy - da-server: + sequencer-db: condition: service_healthy state-relay-server: condition: service_healthy + broker-0: + condition: service_healthy + broker-1: + condition: service_healthy + marshal-0: + condition: service_healthy extra_hosts: - "host.docker.internal:host-gateway" @@ -98,30 +152,35 @@ services: command: sequencer -- http environment: - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL - - ESPRESSO_SEQUENCER_DA_SERVER_URL - - ESPRESSO_SEQUENCER_CONSENSUS_SERVER_URL + - ESPRESSO_SEQUENCER_CDN_ENDPOINT - ESPRESSO_SEQUENCER_API_PORT + - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT - ESPRESSO_SEQUENCER_STORAGE_PATH - ESPRESSO_SEQUENCER_L1_PROVIDER - - ESPRESSO_SEQUENCER_L1_USE_LATEST_BLOCK_TAG + - ESPRESSO_STATE_RELAY_SERVER_URL - ESPRESSO_SEQUENCER_PRIVATE_STAKING_KEY=$ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_1 - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_1 + - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1 + - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=espresso-sequencer1:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1 - ESPRESSO_SEQUENCER_ETH_MNEMONIC - ESPRESSO_SEQUENCER_ETH_ACCOUNT_INDEX - ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS - - ESPRESSO_STATE_RELAY_SERVER_URL + - ESPRESSO_SEQUENCER_STATE_PEERS=http://espresso-sequencer0:$ESPRESSO_SEQUENCER_API_PORT - RUST_LOG - RUST_LOG_FORMAT - - ESPRESSO_SEQUENCER_STATE_PEERS=http://espresso-sequencer0:$ESPRESSO_SEQUENCER_API_PORT depends_on: orchestrator: condition: service_healthy - consensus-server: - condition: service_healthy - da-server: + sequencer-db: condition: service_healthy state-relay-server: condition: service_healthy + broker-0: + condition: service_healthy + broker-1: + condition: service_healthy + marshal-0: + condition: service_healthy extra_hosts: - "host.docker.internal:host-gateway" @@ -141,10 +200,6 @@ services: depends_on: espresso-sequencer0: condition: service_healthy - consensus-server: - condition: service_healthy - da-server: - condition: service_healthy deploy-contracts: condition: service_completed_successfully @@ -160,3 +215,19 @@ services: - ESPRESSO_STATE_SIGNATURE_WEIGHT_THRESHOLD - RUST_LOG - RUST_LOG_FORMAT + + sequencer-db: + image: postgres + user: postgres + ports: + - "$ESPRESSO_SEQUENCER_DB_PORT:5432" + environment: + - POSTGRES_PASSWORD=password + - POSTGRES_USER=root + healthcheck: + # Postgres can be falsely "ready" once before running init scripts. + # See https://github.com/docker-library/postgres/issues/146 for discussion. + test: "pg_isready && sleep 1 && pg_isready" + interval: 5s + timeout: 4s + retries: 20 diff --git a/system_tests/espresso_e2e_test.go b/system_tests/espresso_e2e_test.go index 47ffdc19a7..bb9dc6e562 100644 --- a/system_tests/espresso_e2e_test.go +++ b/system_tests/espresso_e2e_test.go @@ -56,8 +56,6 @@ func runEspresso(t *testing.T, ctx context.Context) func() { invocation := []string{"compose", "up", "-d", "--build"} nodes := []string{ "orchestrator", - "da-server", - "consensus-server", "espresso-sequencer0", "espresso-sequencer1", "commitment-task",