Merge branch 'main' into main

.config/nextest.toml

@@ -1,10 +1,33 @@
 [profile.default]
-slow-timeout = "2m"
-default-filter = 'not test(slow_)'
+# Kill tests after 3 periods of 2m, because they are probably hanging
+slow-timeout = { period = "2m", terminate-after = 3 }
+default-filter = 'not (test(slow_) | package(tests))'
 retries = 2
 
+[[profile.default.overrides]]
+# This test is fast if it works (usually about 150ms) but sometimes hangs,
+# especially when running concurrently with other tests. Kill it after a while
+# and retry up to 10 times.
+filter = 'test(test_process_client_handling_stream_subscribe_node_identity)'
+slow-timeout = { period = "2s", terminate-after = 1 }
+retries  = 10
+
+# The restart tests run an entire sequencing network, and so are quite resource intensive.
+[[profile.default.overrides]]
+filter = 'test(slow_test_restart)'
+threads-required = 'num-cpus'
+
 [profile.slow]
+slow-timeout = "2m"
 default-filter = 'test(slow_)'
+retries = 2
 
 [profile.all]
+slow-timeout = "2m"
 default-filter = 'all()'
+retries = 2
+
+[profile.integration]
+slow-timeout = "2m"
+default-filter = 'package(tests)'
+retries = 2

.env

@@ -37,12 +37,12 @@ ESPRESSO_SEQUENCER_GENESIS_FILE=/genesis/demo.toml
 ESPRESSO_SEQUENCER_L1_PORT=8545
 ESPRESSO_SEQUENCER_L1_WS_PORT=8546
 ESPRESSO_SEQUENCER_L1_PROVIDER=http://demo-l1-network:${ESPRESSO_SEQUENCER_L1_PORT}
+ESPRESSO_SEQUENCER_L1_WS_PROVIDER=ws://demo-l1-network:${ESPRESSO_SEQUENCER_L1_WS_PORT}
 ESPRESSO_NODE_VALIDATOR_PORT=9000
 
 # Only allow 1 block to be processed for events at a time, simulating a very bad L1 provider.
 ESPRESSO_SEQUENCER_L1_EVENTS_MAX_BLOCK_RANGE=1
 ESPRESSO_SEQUENCER_ETH_MNEMONIC="test test test test test test test test test test test junk"
-ESPRESSO_COMMITMENT_TASK_PORT=30010
 ESPRESSO_SEQUENCER0_DB_PORT=5432
 ESPRESSO_SEQUENCER1_DB_PORT=5433
 MARKETPLACE_SOLVER_POSTGRES_PORT=5434
@@ -52,15 +52,14 @@ ESPRESSO_STATE_RELAY_SERVER_URL=http://state-relay-server:${ESPRESSO_STATE_RELAY
 ESPRESSO_BLOCK_EXPLORER_PORT=3000
 
 # Ethereum accounts (note 11-15 are used by the sequencer nodes)
-ESPRESSO_SEQUENCER_HOTSHOT_ACCOUNT_INDEX=5
 ESPRESSO_SEQUENCER_STATE_PROVER_ACCOUNT_INDEX=7
 ESPRESSO_BUILDER_ETH_ACCOUNT_INDEX=8
 ESPRESSO_DEPLOYER_ACCOUNT_INDEX=9
 
 # Contracts
-ESPRESSO_SEQUENCER_HOTSHOT_ADDRESS=0xb19b36b1456e65e3a6d514d3f715f204bd59f431
 ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS=0x0c8e79f3534b00d9a3d4a856b665bf4ebc22f2ba
 ESPRESSO_SEQUENCER_LIGHTCLIENT_ADDRESS=$ESPRESSO_SEQUENCER_LIGHT_CLIENT_PROXY_ADDRESS
+ESPRESSO_SEQUENCER_PERMISSIONED_PROVER=0x14dc79964da2c08b23698b3d3cc7ca32193d9955
 
 # Example sequencer demo private keys
 ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_0=BLS_SIGNING_KEY~lNDh4Pn-pTAyzyprOAFdXHwhrKhEwqwtMtkD3CZF4x3o
@@ -88,6 +87,8 @@ ESPRESSO_SEQUENCER_STAKE_TABLE_CAPACITY=10
 # Foundry
 # The mnemonic used by foundry to deploy contracts.
 MNEMONIC="test test test test test test test test test test test junk"
+# The mnemonic offset, starts at zero
+MNEMONIC_OFFSET=0
 # The RPC URL for deploying to the sepolia network.
 SEPOLIA_RPC_URL=
 
@@ -130,3 +131,23 @@ ESPRESSO_SEQUENCER_FETCH_RATE_LIMIT=25
 
 # Query service stress test
 ESPRESSO_NASTY_CLIENT_PORT=24011
+
+# Test setting upgradable contract address after deployment.
+#
+# Setting this address to a random address effectively relinuquishes ownership of the contracts,
+# which ensures that all services work correctly in the demo without admin access. In a real
+# deployment, we would set this to the address of a multisig wallet.
+ESPRESSO_SEQUENCER_ETH_MULTISIG_ADDRESS=8626f6940e2eb28930efb4cef49b2d1f2c9c1199
+
+# Set this to the number of blocks you would like to confirm the sequencer can reach
+INTEGRATION_TEST_EXPECTED_BLOCK_HEIGHT=10
+
+INTEGRATION_TEST_HOST=localhost
+INTEGRATION_TEST_PROTO=http
+
+# Version of sequencer protocol we want to test. If this is set to
+# `03`, marketplace upgrade will be tested.
+INTEGRATION_TEST_SEQUENCER_VERSION=02
+
+# max database connections 
+ESPRESSO_SEQUENCER_DATABASE_MAX_CONNECTIONS=25

.env.contracts.example

@@ -1,45 +1,40 @@
-# Since foundry and openzeppelin defender expect secrets to be in the .env file, remember to source this file before your forge script command
+# Since foundry expects secrets to be in the .env file, remember to source this file before your forge script command
 ## e.g. `source .env.contracts && force script $YOUR_SCRIPT_NAME`
 
-# Openzeppelin Defender Deployment Profile
-export DEFENDER_KEY=
-export DEFENDER_SECRET=
-export FEE_CONTRACT_SALT=
-export LIGHT_CLIENT_SALT=
-export FEE_CONTRACT_UPGRADE_NAME= #e.g "FeeContract.sol"
-export LIGHT_CLIENT_UPGRADE_NAME=
-export FOUNDRY_OUT=contracts/out
-
 # The Ethereum address of the safe multisig wallet used to deploy and operate the contracts.
 export SAFE_MULTISIG_ADDRESS=
 # The Ethereum private key of the wallet used for the proposing multisig transactions.
 export SAFE_ORCHESTRATOR_PRIVATE_KEY=
 
-# Light Client
-export LIGHT_CLIENT_PROXY_CONTRACT_ADDRESS=
-export APPROVED_PROVER_ADDRESS=
-
-# Plonk Verification Library Deployment with Defender
-export PLONK_VERIFIER_SALT=# Openzeppelin Defender Deployment Profile
-export DEFENDER_KEY=
-export DEFENDER_SECRET=
-export FEE_CONTRACT_SALT=
-export LIGHT_CLIENT_SALT=
-export FEE_CONTRACT_UPGRADE_NAME= #e.g "FeeContract.sol"
-export LIGHT_CLIENT_UPGRADE_NAME=
-export FOUNDRY_OUT=contracts/out
+export RPC_URL=
 
-# The Ethereum address of the safe multisig wallet used to deploy and operate the contracts.
-export SAFE_MULTISIG_ADDRESS=
-# The Ethereum private key of the wallet used for the proposing multisig transactions.
-export SAFE_ORCHESTRATOR_PRIVATE_KEY=
+# Plonk Verifier
+export PLONK_VERIFIER_SALT=
+export PLONK_VERIFIER_ADDRESS=
 
 # Light Client
-export LIGHT_CLIENT_PROXY_CONTRACT_ADDRESS=
-export APPROVED_PROVER_ADDRESS=
-
-# Plonk Verification Library Deployment with Defender
-export PLONK_VERIFIER_SALT=
+export LIGHT_CLIENT_CONTRACT_PROXY_ADDRESS=
+export PERMISSIONED_PROVER_ADDRESS=
+export STATE_HISTORY_RETENTION_PERIOD=
+export LIGHT_CLIENT_CONTRACT_UPGRADE_NAME=
+export LIGHT_CLIENT_CONTRACT_ORIGINAL_NAME = # e.g. "LightClient.sol"
+export STATE_HISTORY_RETENTION_PERIOD= #in seconds
+export NUM_INIT_VALIDATORS=
+export LIGHT_CLIENT_CONTRACT_ADDRESS=
+
+# Fee Contract
+export FEE_CONTRACT_PROXY_ADDRESS=
+export FEE_CONTRACT_UPGRADE_NAME= 
+export FEE_CONTRACT_ORIGINAL_NAME= #e.g "FeeContract.sol"
+export FEE_CONTRACT_ADDRESS=
 
 # The etherscan API key is needed to verify contracts on etherscan.
-export ETHERSCAN_API_KEY=
+export ETHERSCAN_API_KEY=
+export SOLC_VERSION=
+
+# foundry scripts
+export DEPLOYER_PRIVATE_KEY= #include the 0x prefix
+export DEPLOYER_MNEMONIC=
+export DEPLOYER_MNEMONIC_OFFSET=
+export DEPLOYER_HARDWARE_WALLET_ADDRESS=
+export USE_HARDWARE_WALLET=false # values {true, false}

.github/dependabot.yml

@@ -2,10 +2,20 @@ version: 2
 updates:
   - package-ecosystem: "github-actions"
     directory: "/"
+    # Group all updates together
+    groups:
+      all:
+        patterns:
+        - "*"
     schedule:
       interval: "daily"
 
   - package-ecosystem: "cargo"
     directory: "/"
+    # Group all updates together
+    groups:
+      all:
+        patterns:
+        - "*"
     schedule:
       interval: "daily"

.github/workflows/audit.yml

@@ -21,15 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      # See https://github.com/rustsec/audit-check for docs
-      # TODO: re-enable if https://github.com/rustsec/audit-check/pull/20 is merged
-      # - uses: rustsec/audit-check@v1
-      #   with:
-      #     token: ${{ secrets.GITHUB_TOKEN }}
 
-      # Currently the rustsec/audit-check action regenerates the Cargo.lock
-      # file. Our binaries are built using the committed lock file.
-      # Re-generating the lock file can hide vulnerabilities. We therefore run
-      # cargo audit directly which respects our lock file.
-      - run: cargo audit
+      - uses: rustsec/audit-check@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/backport.yml

@@ -0,0 +1,18 @@
+# Create a backport PR to branch "foo" from a merged PR by adding a PR label "backport foo"
+name: Backport merged pull request
+on:
+  pull_request_target:
+    types: [closed]
+permissions:
+  contents: write # so it can comment
+  pull-requests: write # so it can create pull requests
+jobs:
+  backport:
+    name: Backport pull request
+    runs-on: ubuntu-latest
+    # Don't run on closed unmerged pull requests
+    if: github.event.pull_request.merged
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create backport pull requests
+        uses: korthout/backport-action@v3