feat: add Reth node bindings (#1092)

* generalize geth into node

* add basic reth setup

* basic spawning and capturing of ports works, shutdown does not

* add auth_port, p2p_port

* clean up

* fix geth tests

* simplify abstraction

* clean up

* add explicit `dev` mode command, reth remains open

* --chain takes path or string, not id

* add support for UDP endpoint extraction from keys

* clean up, add dev flag

* tag as dev

* add option to set block time

* add block time

* disable peer persistence if discovery is disabled

* update binaries

* add reth install in github workflow

* fix build

* fix doctest

* windows geth build is zipped

* fix name

* fix path to reth binary

* tar is flat

* fix url

* fix output path

* reth is already extracted to the root

* reth renders via stdout

* disable dev by default, enabling it with .dev()

* make sure to exit when running into fatal error

* add tests to cover the feature set

* try debugging windows

* run binary install in parallel

* disable discovery on blocktime dev

* attempt to fix on windows by not using tempdir

* re-enable tempdir, attempt to get full error log on windows

* ignore tests on windows, CI of Windows is not compatible:

[crates\node-bindings\src\nodes\reth.rs:369:13] &line = "2024-09-04T12:39:10.637528Z  INFO reth::cli: Opening database path=\"C:\\\\Users\\\\RUNNER~1\\\\AppData\\\\Local\\\\Temp\\\\.tmp4XoSee\\\\db\"\n"
[crates\node-bindings\src\nodes\reth.rs:369:13] &line = "2024-09-04T12:39:10.731699Z ERROR reth::cli: shutting down due to error\n"
Error: failed to open the database: environment or database is not compatible with the requested operation or flags (-30784)

Location:
    /project/crates/storage/db/src/mdbx.rs:28:8

* only install on linux

* temporarily re-enable windows to test error catching

* revert

.github/scripts/install_test_binaries.sh

@@ -3,7 +3,8 @@
 # Note: intended for use only with CI (x86_64 Ubuntu, MacOS or Windows)
 set -e
 
-GETH_BUILD=${GETH_BUILD:-"1.14.0-87246f3c"}
+GETH_BUILD=${GETH_BUILD:-"1.14.8-a9523b64"}
+RETH_BUILD=${RETH_BUILD:-"1.0.6"}
 
 BIN_DIR=${BIN_DIR:-"$HOME/bin"}
 
@@ -17,32 +18,49 @@ main() {
         echo "$BIN_DIR" >> "$GITHUB_PATH"
     fi
 
-    install_geth
+    install_geth &
+    install_reth &
 
-    echo ""
-    echo "Installed Geth:"
-    geth version
+    wait
 }
 
 # Installs geth from https://geth.ethereum.org/downloads
 install_geth() {
     case "$PLATFORM" in
-        linux|darwin)
-            name="geth-$PLATFORM-amd64-$GETH_BUILD"
-            curl -s "https://gethstore.blob.core.windows.net/builds/$name.tar.gz" | tar -xzf -
-            mv -f "$name/geth" ./
-            rm -rf "$name"
+        linux)
+            NAME="geth-$PLATFORM-amd64-$GETH_BUILD"
+            curl -sL "https://gethstore.blob.core.windows.net/builds/$NAME.tar.gz" | tar -xzf -
+            mv -f "$NAME/geth" ./
+            rm -rf "$NAME"
             chmod +x geth
             ;;
         *)
-            name="geth-windows-amd64-$GETH_BUILD"
-            zip="$name.zip"
-            curl -so "$zip" "https://gethstore.blob.core.windows.net/builds/$zip"
-            unzip "$zip"
-            mv -f "$name/geth.exe" ./
-            rm -rf "$name" "$zip"
+            NAME="geth-windows-amd64-$GETH_BUILD"
+            curl -so $NAME.zip "https://gethstore.blob.core.windows.net/builds/$NAME.zip"
+            unzip $NAME.zip
+            mv -f "$NAME/geth.exe" ./
+            rm -rf "$NAME" "$NAME.zip"
+            ;;
+    esac
+
+    echo ""
+    echo "Installed Geth:"
+    geth version
+}
+
+# Install reth from https://github.com/paradigmxyz/reth/releases
+install_reth() {
+    case "$PLATFORM" in
+        linux)
+            NAME="reth-v$RETH_BUILD-x86_64-unknown-linux-gnu"
+            curl -sL "https://github.com/paradigmxyz/reth/releases/download/v$RETH_BUILD/$NAME.tar.gz" | tar -xzf -
+            chmod +x reth
+
+            echo ""
+            echo "Installed Reth:"
+            reth --version
             ;;
     esac
 }
 
-main
+main

crates/node-bindings/src/lib.rs

@@ -11,37 +11,35 @@ extern crate tracing;
 
 use alloy_primitives::U256;
 
-pub mod anvil;
-pub use anvil::{Anvil, AnvilInstance};
+pub mod nodes;
+pub use nodes::{
+    anvil::{self, Anvil, AnvilInstance},
+    geth::{self, Geth, GethInstance},
+    reth::{self, Reth, RethInstance},
+};
 
-pub mod geth;
-pub use geth::{Geth, GethInstance};
+mod node;
+pub use node::*;
+
+mod utils;
+use utils::*;
 
 /// 1 Ether = 1e18 Wei == 0x0de0b6b3a7640000 Wei
 pub const WEI_IN_ETHER: U256 = U256::from_limbs([0x0de0b6b3a7640000, 0x0, 0x0, 0x0]);
 
 /// The number of blocks from the past for which the fee rewards are fetched for fee estimation.
 pub const EIP1559_FEE_ESTIMATION_PAST_BLOCKS: u64 = 10;
+
 /// The default percentile of gas premiums that are fetched for fee estimation.
 pub const EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE: f64 = 5.0;
+
 /// The default max priority fee per gas, used in case the base fee is within a threshold.
 pub const EIP1559_FEE_ESTIMATION_DEFAULT_PRIORITY_FEE: u64 = 3_000_000_000;
+
 /// The threshold for base fee below which we use the default priority fee, and beyond which we
 /// estimate an appropriate value for priority fee.
 pub const EIP1559_FEE_ESTIMATION_PRIORITY_FEE_TRIGGER: u64 = 100_000_000_000;
+
 /// The threshold max change/difference (in %) at which we will ignore the fee history values
 /// under it.
 pub const EIP1559_FEE_ESTIMATION_THRESHOLD_MAX_CHANGE: i64 = 200;
-
-/// A bit of hack to find an unused TCP port.
-///
-/// Does not guarantee that the given port is unused after the function exists, just that it was
-/// unused before the function started (i.e., it does not reserve a port).
-fn unused_port() -> u16 {
-    let listener = std::net::TcpListener::bind("127.0.0.1:0")
-        .expect("Failed to create TCP listener to find unused port");
-
-    let local_addr =
-        listener.local_addr().expect("Failed to read TCP listener local_addr to find unused port");
-    local_addr.port()
-}

crates/node-bindings/src/node.rs

@@ -0,0 +1,68 @@
+//! Node-related types and constants.
+
+use std::time::Duration;
+use thiserror::Error;
+
+/// How long we will wait for the node to indicate that it is ready.
+pub const NODE_STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
+
+/// Timeout for waiting for the node to add a peer.
+pub const NODE_DIAL_LOOP_TIMEOUT: Duration = Duration::from_secs(20);
+
+/// Errors that can occur when working with a node instance.
+#[derive(Debug)]
+pub enum NodeInstanceError {
+    /// Timed out waiting for a message from node's stderr.
+    Timeout(String),
+
+    /// A line could not be read from the node's stderr.
+    ReadLineError(std::io::Error),
+
+    /// The child node process's stderr was not captured.
+    NoStderr,
+
+    /// The child node process's stdout was not captured.
+    NoStdout,
+}
+
+/// Errors that can occur when working with the node.
+#[derive(Debug, Error)]
+pub enum NodeError {
+    /// The chain id was not set.
+    #[error("the chain ID was not set")]
+    ChainIdNotSet,
+    /// Could not create the data directory.
+    #[error("could not create directory: {0}")]
+    CreateDirError(std::io::Error),
+    /// No stderr was captured from the child process.
+    #[error("no stderr was captured from the process")]
+    NoStderr,
+    /// No stdout was captured from the child process.
+    #[error("no stdout was captured from the process")]
+    NoStdout,
+    /// Timed out waiting for the node to start.
+    #[error("timed out waiting for node to spawn; is the node binary installed?")]
+    Timeout,
+    /// Encountered a fatal error.
+    #[error("fatal error: {0}")]
+    Fatal(String),
+    /// A line could not be read from the node stderr.
+    #[error("could not read line from node stderr: {0}")]
+    ReadLineError(std::io::Error),
+    /// Genesis error
+    #[error("genesis error occurred: {0}")]
+    GenesisError(String),
+    /// Node init error
+    #[error("node init error occurred")]
+    InitError,
+    /// Spawn node error
+    #[error("could not spawn node: {0}")]
+    SpawnError(std::io::Error),
+    /// Wait error
+    #[error("could not wait for node to exit: {0}")]
+    WaitError(std::io::Error),
+
+    /// Clique private key error
+    #[error("clique address error: {0}")]
+    CliqueAddressError(String),
+}