Merge pull request #254 from EspressoSystems/ma/sync-nitro-v3.2.1-ng

Sync nitro v3.2.1
EspressoSystems · Nov 7, 2024 · fb8e74e · fb8e74e
2 parents d908e3d + 83ac01d
commit fb8e74e
Show file tree

Hide file tree

Showing 349 changed files with 15,570 additions and 3,846 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -9,6 +9,7 @@ go-ethereum/tests
 **/*.yml
 contracts/build
 contracts/cache/
+safe-smart-account/build/
 solgen/go
 **/node_modules
 

diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml
@@ -71,17 +71,20 @@ jobs:
       - name: Install rust stable
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: "1.76"
+          toolchain: 'stable'
           components: 'llvm-tools-preview, rustfmt, clippy'
 
       - name: Install rust nightly
         uses: dtolnay/rust-toolchain@nightly
         id: install-rust-nightly
         with:
-          toolchain: "nightly-2024-02-04"
+          toolchain: 'nightly-2024-08-06'
           targets: 'wasm32-wasi, wasm32-unknown-unknown'
           components: 'rust-src, rustfmt, clippy'
 
+      - name: Set STYLUS_NIGHTLY_VER environment variable
+        run: echo "STYLUS_NIGHTLY_VER=+$(rustup toolchain list | grep '^nightly' | head -n1 | cut -d' ' -f1)" >> "$GITHUB_ENV"
+
       - name: Cache Rust intermediate build products
         uses: actions/cache@v3
         with:
@@ -158,28 +161,19 @@ jobs:
         run: echo "$HOME/wabt-prefix/bin" >> "$GITHUB_PATH"
 
       - name: Make arbitrator libraries
-        run: make -j wasm-ci-build STYLUS_NIGHTLY_VER="+nightly-2024-02-04"
+        run: make -j wasm-ci-build
 
       - name: Clippy check
         run: cargo clippy --all --manifest-path arbitrator/Cargo.toml -- -D warnings
 
       - name: Run rust tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: -p arbutil -p prover -p jit -p stylus --release --manifest-path arbitrator/prover/Cargo.toml
+        run: cargo test -p arbutil -p prover -p jit -p stylus --release --manifest-path arbitrator/prover/Cargo.toml
 
       - name: Rustfmt
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: -p arbutil -p prover -p jit -p stylus --manifest-path arbitrator/Cargo.toml -- --check
+        run: cargo fmt -p arbutil -p prover -p jit -p stylus --manifest-path arbitrator/Cargo.toml -- --check
 
       - name: Rustfmt - langs/rust
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all --manifest-path arbitrator/langs/rust/Cargo.toml -- --check
+        run: cargo fmt --all --manifest-path arbitrator/langs/rust/Cargo.toml -- --check
 
       - name: Make proofs from test cases
         run: make -j test-gen-proofs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,7 +30,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test-mode: [defaults, race, challenge, stylus, long, redis]
+        # Run the defaults and race tests with `path` and `hash` state scheme.
+        # the others we run with the default test-state-scheme which at the time
+        # of writing this is `path`.
+        include:
+          - test-mode: defaults
+            test-state-scheme: path
+
+          - test-mode: defaults
+            test-state-scheme: hash
+
+          - test-mode: race
+            test-state-scheme: path
+
+          - test-mode: race
+            test-state-scheme: hash
+
+          - test-mode: challenge
+          - test-mode: stylus
+          - test-mode: long
+          - test-mode: redis
 
     steps:
       - name: Checkout
@@ -65,9 +84,14 @@ jobs:
 
       - name: Install dependencies
         run: >
-          sudo apt update && sudo apt install -y wabt gotestsum
+          sudo apt update && sudo apt install -y wabt
           cmake build-essential bison golang clang make wabt
 
+      - name: Install latest gotestsum
+        uses: autero1/[email protected]
+        with:
+          gotestsum_version: v1.12.0
+
       - name: Setup nodejs
         if: false # fails to find yarn.lock when running in symlinked directory, doesn't seem to be needed
         uses: actions/setup-node@v3
@@ -89,7 +113,21 @@ jobs:
       - name: Install rust stable
         uses: dtolnay/rust-toolchain@stable
         with:
-          targets: 'wasm32-unknown-unknown, wasm32-wasi'
+          # TODO: remove pinning: https://github.com/EspressoSystems/nitro-espresso-integration/issues/257
+          toolchain: '1.81.0'
+          targets: 'wasm32-wasi, wasm32-unknown-unknown'
+          components: 'llvm-tools-preview, rustfmt, clippy'
+
+      - name: Install rust nightly
+        uses: dtolnay/rust-toolchain@nightly
+        id: install-rust-nightly
+        with:
+          toolchain: 'nightly-2024-08-06'
+          targets: 'wasm32-wasi, wasm32-unknown-unknown'
+          components: 'rust-src, rustfmt, clippy'
+
+      - name: Set STYLUS_NIGHTLY_VER environment variable
+        run: echo "STYLUS_NIGHTLY_VER=+$(rustup toolchain list | grep '^nightly' | head -n1 | cut -d' ' -f1)" >> "$GITHUB_ENV"
 
       - name: Install cbindgen
         run: cargo install cbindgen
@@ -99,21 +137,6 @@ jobs:
         with:
           version: nightly
 
-      - name: Install rust nightly
-        uses: actions-rs/toolchain@v1
-        id: install-rust-nightly
-        with:
-          profile: minimal
-          toolchain: "nightly"
-
-      - name: Install rust wasm targets
-        run: rustup target add wasm32-wasi wasm32-unknown-unknown
-
-      - name: Install nightly wasm targets
-        run: |
-          rustup component add rust-src --toolchain nightly
-          rustup target add wasm32-unknown-unknown --toolchain nightly
-
       - name: Cache Build Products
         uses: actions/cache@v3
         with:
@@ -182,50 +205,46 @@ jobs:
         run: |
           echo "GOMEMLIMIT=6GiB" >> "$GITHUB_ENV"
           echo "GOGC=80" >> "$GITHUB_ENV"
+          echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_ENV"
 
-      - name: run tests without race detection
+      - name: run tests without race detection (path and hash state scheme)
         if: matrix.test-mode == 'defaults'
         shell: bash
+        env:
+          TEST_STATE_SCHEME: ${{ matrix.test-state-scheme }}
         run: |
-          skip_tests=`grep -vE '^\s*#|^\s*$' ci_skip_tests | sed 's/.*/^&$/g' | tr '\n' '|' | sed 's/|$//'`
-          packages=`go list ./...`
-          stdbuf -oL gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 --no-color=false -- ./... -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 35m -skip "$skip_tests"  > >(stdbuf -oL tee full.log | grep -vE "INFO|seal")
+          scripts/espresso-run-test-ci
 
-      - name: run tests with race detection
+      - name: run tests with race detection (path and hash state scheme)
         if: matrix.test-mode == 'race'
         shell: bash
+        env:
+          TEST_STATE_SCHEME: ${{ matrix.test-state-scheme }}
         run: |
-          skip_tests=`grep -vE '^\s*#|^\s*$' ci_skip_tests | sed 's/.*/^&$/g' | tr '\n' '|' | sed 's/|$//'`
-          packages=`go list ./...`
-          stdbuf -oL gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 --no-color=false -- ./... -race -skip "$skip_tests" -timeout=40m > >(stdbuf -oL tee full.log | grep -vE "INFO|seal")
+          scripts/espresso-run-test-ci -race
 
       - name: run redis tests
         if: matrix.test-mode == 'redis'
-        run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -timeout 20m -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./...
+        run: |
+          TEST_REDIS=redis://localhost:6379/0 scripts/espresso-run-test-ci -p 1 -run TestRedis ./arbnode/... ./system_tests/...
 
       - name: run challenge tests
         if: matrix.test-mode == 'challenge'
         shell: bash
         run: |
-          skip_tests=`grep -vE '^\s*#|^\s*$' ci_skip_tests | sed 's/.*/^&$/g' | tr '\n' '|' | sed 's/|$//'`
-          packages=`go list ./...`
-          stdbuf -oL gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 --no-color=false -- ./... -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/...   -tags=challengetest -run=TestChallenge -skip "$skip_tests"> >(stdbuf -oL tee full.log | grep -vE "INFO|seal")
+          scripts/espresso-run-test-ci -run=TestChallenge -tags=challengetest
 
       - name: run stylus tests
         if: matrix.test-mode == 'stylus'
         shell: bash
         run: |
-          skip_tests=`grep -vE '^\s*#|^\s*$' ci_skip_tests | sed 's/.*/^&$/g' | tr '\n' '|' | sed 's/|$//'`
-          packages=`go list ./...`
-          stdbuf -oL gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 --no-color=false -- ./... -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -parallel=8 -tags=stylustest -run="TestProgramArbitrator" -skip "$skip_tests"> >(stdbuf -oL tee full.log | grep -vE "INFO|seal")
+          scripts/espresso-run-test-ci -tags=stylustest -run=TestProgramArbitrator
 
       - name: run long stylus tests
         if: matrix.test-mode == 'long'
         shell: bash
         run: |
-          skip_tests=`grep -vE '^\s*#|^\s*$' ci_skip_tests | sed 's/.*/^&$/g' | tr '\n' '|' | sed 's/|$//'`
-          packages=`go list ./...`
-          stdbuf -oL gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 --no-color=false -- ./... -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -parallel=8 -tags=stylustest -run="TestProgramLong" -skip "$skip_tests"> >(stdbuf -oL tee full.log | grep -vE "INFO|seal")
+          scripts/espresso-run-test-ci -tags=stylustest -run=TestProgramLong
 
       - name: Archive detailed run log
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/espresso-e2e.yml b/.github/workflows/espresso-e2e.yml
@@ -25,9 +25,14 @@ jobs:
 
       - name: Install dependencies
         run: >
-          sudo apt update && sudo apt install -y wabt gotestsum
+          sudo apt update && sudo apt install -y wabt
           cmake build-essential bison golang clang make wabt
 
+      - name: Install latest gotestsum
+        uses: autero1/[email protected]
+        with:
+          gotestsum_version: 1.12.0
+
       - name: free disk space
         uses: jlumbroso/free-disk-space@main
         with:
@@ -58,6 +63,8 @@ jobs:
       - name: Install rust stable
         uses: dtolnay/rust-toolchain@stable
         with:
+          # TODO: remove pinning: https://github.com/EspressoSystems/nitro-espresso-integration/issues/257
+          toolchain: '1.81.0'
           targets: 'wasm32-unknown-unknown, wasm32-wasi'
 
       - name: Install Foundry
@@ -66,19 +73,12 @@ jobs:
           version: nightly
 
       - name: Install rust nightly
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@nightly
         id: install-rust-nightly
         with:
-          profile: minimal
-          toolchain: "nightly"
-
-      - name: Install rust wasm targets
-        run: rustup target add wasm32-wasi wasm32-unknown-unknown
-
-      - name: Install nightly wasm targets
-        run: |
-          rustup component add rust-src --toolchain nightly
-          rustup target add wasm32-unknown-unknown --toolchain nightly
+          toolchain: 'nightly-2024-08-06'
+          targets: 'wasm32-wasi, wasm32-unknown-unknown'
+          components: 'rust-src, rustfmt, clippy'
 
       - name: Cache Build Products
         uses: actions/cache@v3
@@ -132,12 +132,6 @@ jobs:
         with:
           version: '2.22.0'
 
-      - name: Run E2E test
-        run: |
-          packages=`go list ./... | grep system_tests`
-          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -v -timeout 35m -p 1 ./... -run 'TestEspressoE2E'
-
-      - name: Run sovereign sequencer test
+      - name: Run Espresso tests
         run: |
-          packages=`go list ./... | grep system_tests`
-          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -v -timeout 15m -p 1 ./... -run 'TestSovereignSequencer'
+          scripts/run-espresso-ci-job
diff --git a/.github/workflows/merge-checks.yml b/.github/workflows/merge-checks.yml
@@ -17,7 +17,7 @@ jobs:
         run: |
           set -x pipefail
           status_state="pending"
-          if ${{ contains(github.event.*.labels.*.name, 'design-approved') }}; then
+          if ${{ contains(github.event.pull_request.labels.*.name, 'design-approved') && !contains(github.event.pull_request.labels.*.name, 'after-next-version') }}; then
             status_state="success"
           else
             resp="$(curl -sSL --fail-with-body \

diff --git a/.github/workflows/shellcheck-ci.yml b/.github/workflows/shellcheck-ci.yml
@@ -0,0 +1,30 @@
+name: ShellCheck CI
+run-name: ShellCheck CI triggered from @${{ github.actor }} of ${{ github.head_ref }}
+
+on:
+  workflow_dispatch:
+  merge_group:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  shellcheck:
+    name: Run ShellCheck
+    runs-on: ubuntu-8
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+        with:
+          ignore_paths: >-
+            ./fastcache/**
+            ./contracts/**
+            ./safe-smart-account/**
+            ./go-ethereum/**
+            ./nitro-testnode/**
+            ./brotli/**
+            ./arbitrator/**
diff --git a/.github/workflows/submodule-pin-check.sh b/.github/workflows/submodule-pin-check.sh