Altda devnet monitoring

.circleci/config.yml

@@ -222,7 +222,7 @@ jobs:
           condition: <<parameters.notify>>
           steps:
             - notify-failures-on-develop:
-                mentions: "@proofs-squad"
+                mentions: "@proofs-team"
 
   cannon-build-test-vectors:
     docker:
@@ -721,6 +721,8 @@ jobs:
           command: interfaces-check-no-build
       - run-contracts-check:
           command: size-check
+      - run-contracts-check:
+          command: unused-imports-check-no-build
 
   contracts-bedrock-validate-spacers:
     docker:
@@ -1142,7 +1144,7 @@ jobs:
               exit 1
             fi
       - notify-failures-on-develop:
-          mentions: "@proofs-squad"
+          mentions: "@proofs-team"
 
   cannon-stf-verify:
     docker:
@@ -1269,7 +1271,7 @@ jobs:
             make verify-sepolia
           working_directory: op-program
       - notify-failures-on-develop:
-          mentions: "@proofs-squad"
+          mentions: "@proofs-team"
 
   op-program-compat:
     docker:
@@ -1367,7 +1369,7 @@ jobs:
           name: Build contracts
           environment:
             FOUNDRY_PROFILE: ci
-          command: just build
+          command: just forge-build
           working_directory: packages/contracts-bedrock
       - run:
           name: Publish artifacts
@@ -1420,7 +1422,7 @@ workflows:
       - semgrep-scan
       - semgrep-scan:
           name: semgrep-scan-local
-          scan_command: semgrep scan --timeout=100 --config=./.semgrep --strict --error .
+          scan_command: semgrep scan --timeout=100 --config=./.semgrep --error .
       - go-lint:
           requires:
             - go-mod-download
@@ -1726,7 +1728,7 @@ workflows:
           target: test-cannon
           parallelism: 8
           notify: true
-          mentions: "@proofs-squad"
+          mentions: "@proofs-team"
           requires:
             - contracts-bedrock-build
             - cannon-prestate

.semgrep/sol-rules.yaml

@@ -2,14 +2,14 @@ rules:
   - id: sol-safety-deployutils-args
     languages: [solidity]
     severity: ERROR
-    pattern-regex: DeployUtils\.(create1|create2|create1AndSave|create2AndSave)\s*\(\s*\{[^}]*?_args\s*:\s*(?!\s*DeployUtils\.encodeConstructor\()\s*[^}]*?\}\s*\)
     message: _args parameter should be wrapped with DeployUtils.encodeConstructor
+    pattern-regex: DeployUtils\.(create1|create2|create1AndSave|create2AndSave)\s*\(\s*\{[^}]*?_args\s*:\s*(?!\s*DeployUtils\.encodeConstructor\()\s*[^}]*?\}\s*\)
 
   - id: sol-style-input-arg-fmt
     languages: [solidity]
     severity: ERROR
-    pattern-regex: function\s+\w+\s*\(\s*([^)]*?\b\w+\s+(?!_)(?!memory\b)(?!calldata\b)(?!storage\b)(?!payable\b)\w+\s*(?=,|\)))
     message: Named inputs to functions must be prepended with an underscore
+    pattern-regex: function\s+\w+\s*\(\s*([^)]*?\b\w+\s+(?!_)(?!memory\b)(?!calldata\b)(?!storage\b)(?!payable\b)\w+\s*(?=,|\)))
     paths:
       exclude:
         - op-chain-ops/script/testdata/scripts/ScriptExample.s.sol
@@ -25,8 +25,8 @@ rules:
   - id: sol-style-return-arg-fmt
     languages: [solidity]
     severity: ERROR
-    pattern-regex: returns\s*(\w+\s*)?\(\s*([^)]*?\b\w+\s+(?!memory\b)(?!calldata\b)(?!storage\b)(?!payable\b)\w+(?<!_)\s*(?=,|\)))
     message: Named return arguments to functions must be appended with an underscore
+    pattern-regex: returns\s*(\w+\s*)?\(\s*([^)]*?\b\w+\s+(?!memory\b)(?!calldata\b)(?!storage\b)(?!payable\b)\w+(?<!_)\s*(?=,|\)))
     paths:
       exclude:
         - op-chain-ops/script/testdata/scripts/ScriptExample.s.sol
@@ -47,3 +47,37 @@ rules:
     message: vm.expectRevert() must specify the revert reason
     patterns:
       - pattern: vm.expectRevert()
+
+  - id: sol-style-malformed-require
+    languages: [solidity]
+    severity: ERROR
+    message: Require statement style is malformed
+    patterns:
+      - pattern: require(..., $ERR);
+      - pattern-not: require($ERR);
+      - focus-metavariable: $ERR
+      - pattern-not-regex: \"(\w+:\s[^"]+)\"
+      - pattern-not-regex: string\.concat\(\"(\w+:\s[^"]+)\"\,[^"]+\)
+      - pattern-not-regex: \"([a-zA-Z0-9\s]+-[a-zA-Z0-9\s]+)\"
+      - pattern-not-regex: \"([a-zA-Z0-9\s]+-[a-zA-Z0-9\s]+-[a-zA-Z0-9\s]+)\"
+    paths:
+      exclude:
+        - packages/contracts-bedrock/src/libraries/Bytes.sol
+        - packages/contracts-bedrock/src/legacy/LegacyMintableERC20.sol
+        - packages/contracts-bedrock/src/cannon/MIPS.sol
+        - packages/contracts-bedrock/src/cannon/MIPS2.sol
+        - packages/contracts-bedrock/src/cannon/libraries/MIPSMemory.sol
+        - packages/contracts-bedrock/src/cannon/libraries/MIPSInstructions.sol
+
+  - id: sol-style-malformed-revert
+    languages: [solidity]
+    severity: ERROR
+    message: Revert statement style is malformed
+    patterns:
+      - pattern: revert($MSG);
+      - pattern-not: revert $ERR(...);
+      - focus-metavariable: $MSG
+      - pattern-not-regex: \"(\w+:\s[^"]+)\"
+    paths:
+      exclude:
+        - packages/contracts-bedrock/src/cannon/libraries/MIPSInstructions.sol

Makefile

@@ -186,6 +186,12 @@ devnet-up: pre-devnet ## Starts the local devnet
 	PYTHONPATH=./bedrock-devnet $(PYTHON) ./bedrock-devnet/main.py --monorepo-dir=.
 .PHONY: devnet-up
 
+altda-devnet-up:
+	@( \
+		set -a; . ./.env; set +a; \
+		DEVNET_ALTDA=true GENERIC_ALTDA=true TRAFFIC_GEN=true make devnet-up; \
+	)
+
 devnet-test: pre-devnet ## Runs tests on the local devnet
 	make -C op-e2e test-devnet
 .PHONY: devnet-test

README.md

@@ -11,6 +11,7 @@
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 **Table of Contents**
 
+- [devnet with eigenda alt-da](#devnet-with-eigenda-alt-da)
 - [What is Optimism?](#what-is-optimism)
 - [Documentation](#documentation)
 - [Specification](#specification)
@@ -26,6 +27,32 @@
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
+## devnet with eigenda alt-da
+
+First you will need to build the dev docker image for eigenda-proxy, until we make a new release.
+
+```bash
+git clone https://github.com/Layr-Labs/eigenda-proxy.git
+make docker-build
+```
+
+Now create a `.env` file in the root of this repository with the following content (you will need to add an ecdsa private key that is authorized on the eigenda disperser, as well as an ethereum rpc endpoint):
+
+```bash
+MEMSTORE_ENABLED=false
+EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0x54A03db2784E3D0aCC08344D05385d0b62d4F432 #holesky preprod
+EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX=
+EIGENDA_PROXY_EIGENDA_ETH_RPC=
+```
+
+Finally run `make altda-devnet-up` to start the devnet with the eigenda proxy as alt-da server, which sends batches to `disperser-preprod-holesky.eigenda.xyz:443`.
+
+This will also start a grafana instance, which you can access at `http://localhost:3000` with user:pw `admin:admin`.
+
+![](./ops-bedrock/monitoring/grafana/grafana-dashboard-overview-high-throughput.png)
+
+The devnet starts an instance of https://github.com/Layr-Labs/evm-tx-load-gen which is submitting transactions with large amounts of calldata to reach 1.5MB/s of throughput.
+
 ## What is Optimism?
 
 [Optimism](https://www.optimism.io/) is a project dedicated to scaling Ethereum's technology and expanding its ability to coordinate people from across the world to build effective decentralized economies and governance systems. The [Optimism Collective](https://www.optimism.io/vision) builds open-source software that powers scalable blockchains and aims to address key governance and economic challenges in the wider Ethereum ecosystem. Optimism operates on the principle of **impact=profit**, the idea that individuals who positively impact the Collective should be proportionally rewarded with profit. **Change the incentives and you change the world.**

bedrock-devnet/devnet/__init__.py

@@ -31,6 +31,7 @@
 DEVNET_L2OO = os.getenv('DEVNET_L2OO') == "true"
 DEVNET_ALTDA = os.getenv('DEVNET_ALTDA') == "true"
 GENERIC_ALTDA = os.getenv('GENERIC_ALTDA') == "true"
+TRAFFIC_GEN = os.getenv('TRAFFIC_GEN') == "true"
 
 class Bunch:
     def __init__(self, **kwds):
@@ -173,6 +174,12 @@ def devnet_l2_allocs(paths):
 
 # Bring up the devnet where the contracts are deployed to L1
 def devnet_deploy(paths):
+
+    log.info('Starting peripheral telemetry services.')
+    run_command(['docker', 'compose', 'up', '-d', 'prometheus', 'grafana', 'loki', 'promtail', 'ethereum-metrics-exporter-l2'], cwd=paths.ops_bedrock_dir, env={
+        'PWD': paths.ops_bedrock_dir
+    })
+
     if os.path.exists(paths.genesis_l1_path):
         log.info('L1 genesis already generated.')
     else:
@@ -247,11 +254,7 @@ def devnet_deploy(paths):
     wait_for_rpc_server('127.0.0.1:9545')
 
     # Print out the addresses being used for easier debugging.
-    l2_output_oracle = addresses['L2OutputOracleProxy']
-    dispute_game_factory = addresses['DisputeGameFactoryProxy']
     batch_inbox_address = rollup_config['batch_inbox_address']
-    log.info(f'Using L2OutputOracle {l2_output_oracle}')
-    log.info(f'Using DisputeGameFactory {dispute_game_factory}')
     log.info(f'Using batch inbox {batch_inbox_address}')
 
     # Set up the base docker environment.
@@ -264,7 +267,11 @@ def devnet_deploy(paths):
     # Must be done selectively because op-proposer throws if both are set.
     if DEVNET_L2OO:
         docker_env['L2OO_ADDRESS'] = l2_output_oracle
+        l2_output_oracle = addresses['L2OutputOracleProxy']
+        log.info(f'Using L2OutputOracle {l2_output_oracle}')
     else:
+        dispute_game_factory = addresses['DisputeGameFactoryProxy']
+        log.info(f'Using DisputeGameFactory {dispute_game_factory}')
         docker_env['DGF_ADDRESS'] = dispute_game_factory
         docker_env['DG_TYPE'] = '254'
         docker_env['PROPOSAL_INTERVAL'] = '12s'
@@ -287,15 +294,20 @@ def devnet_deploy(paths):
     log.info('Bringing up `op-node`, `op-proposer` and `op-batcher`.')
     run_command(['docker', 'compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher', 'artifact-server'], cwd=paths.ops_bedrock_dir, env=docker_env)
 
+    # Optionally bring up traffic-gen.
+    if TRAFFIC_GEN:
+      log.info('Bringing up `evm-tx-load-gen`.')
+      run_command(['docker', 'compose', 'up', '-d', 'evm-tx-load-gen'], cwd=paths.ops_bedrock_dir, env=docker_env)
+
     # Optionally bring up op-challenger.
     if not DEVNET_L2OO:
         log.info('Bringing up `op-challenger`.')
         run_command(['docker', 'compose', 'up', '-d', 'op-challenger'], cwd=paths.ops_bedrock_dir, env=docker_env)
 
     # Optionally bring up Alt-DA Mode components.
     if DEVNET_ALTDA:
-        log.info('Bringing up `da-server`, `sentinel`.') # TODO(10141): We don't have public sentinel images yet
-        run_command(['docker', 'compose', 'up', '-d', 'da-server'], cwd=paths.ops_bedrock_dir, env=docker_env)
+        log.info('Bringing up `da-server`, `redis-cache`.') # TODO(10141): We don't have public sentinel images yet
+        run_command(['docker', 'compose', 'up', '-d', 'da-server', 'redis-cache'], cwd=paths.ops_bedrock_dir, env=docker_env)
 
     # Fin.
     log.info('Devnet ready.')

cannon/README.md

@@ -29,8 +29,8 @@ cd ../cannon
 make cannon
 
 # Transform MIPS op-program client binary into first VM state.
-# This outputs state.json (VM state) and meta.json (for debug symbols).
-./bin/cannon load-elf --type singlethreaded --path=../op-program/bin/op-program-client.elf
+# This outputs state.bin.gz (VM state) and meta.json (for debug symbols).
+./bin/cannon load-elf --type singlethreaded-2 --path=../op-program/bin/op-program-client.elf
 
 # Run cannon emulator (with example inputs)
 # Note that the server-mode op-program command is passed into cannon (after the --),
@@ -45,7 +45,7 @@ make cannon
     --proof-at '=<TRACE_INDEX>' \
     --stop-at '=<STOP_INDEX>' \
     --snapshot-at '%1000000000' \
-    --input ./state.json \
+    --input ./state.bin.gz \
     -- \
     ../op-program/bin/op-program \
     --network op-mainnet \

cannon/cmd/load_elf.go

@@ -32,7 +32,7 @@ var (
 	LoadELFOutFlag = &cli.PathFlag{
 		Name:     "out",
 		Usage:    "Output path to write state to. State is dumped to stdout if set to '-'. Not written if empty. Use file extension '.bin', '.bin.gz', or '.json' for binary, compressed binary, or JSON formats.",
-		Value:    "state.json",
+		Value:    "state.bin.gz",
 		Required: false,
 	}
 	LoadELFMetaFlag = &cli.PathFlag{

cannon/mipsevm/multithreaded/state.go

@@ -234,7 +234,6 @@ func (s *State) EncodeThreadProof() []byte {
 	out := make([]byte, 0, THREAD_WITNESS_SIZE)
 	out = append(out, threadBytes[:]...)
 	out = append(out, otherThreadsWitness[:]...)
-
 	return out
 }