feat(mine): Add an internal Zcash miner to Zebra (#8136)

* Patch equihash to use the solver branch

* Add an internal-miner feature and set up its dependencies

* Remove 'Experimental' from mining RPC docs

* Fix a nightly clippy::question_mark lint

* Move a byte array utility function to zebra-chain

* fixup! Add an internal-miner feature and set up its dependencies

* Add an equihash::Solution::solve() method with difficulty checks

* Check solution is valid before returning it

* Add a TODO to check for peers before mining

* Move config validation into GetBlockTemplateRpcImpl::new()

* fixup! fixup! Add an internal-miner feature and set up its dependencies

* Use the same generic constraints for GetBlockTemplateRpcImpl struct and impls

* Start adding an internal miner component

* Add the miner task to the start command

* Add basic miner code

* Split out a method to mine one block

* Spawn to a blocking thread

* Wait until a valid template is available

* Handle shutdown

* Run mining on low priority threads

* Ignore some invalid solutions

* Use a difference nonce for each solver thread

* Update TODOs

* Change the patch into a renamed dependency to simplify crate releases

* Clean up instrumentation and TODOs

* Make RPC instances cloneable and clean up generics

* Make LongPollId Copy so it's easier to use

* Add API to restart mining if there's a new block template

* Actually restart mining if there's a new block template

* Tidy instrumentation

* fixup! Move config validation into GetBlockTemplateRpcImpl::new()

* fixup! Make RPC instances cloneable and clean up generics

* Run the template generator and one miner concurrently

* Reduce logging

* Fix a bug in getblocktemplate RPC tip change detection

* Work around some watch channel change bugs

* Rate-limit template changes in the receiver

* Run one mining solver per available core

* Use updated C code with double-free protection

* Update to the latest solver branch

* Return and submit all valid solutions

* Document what INPUT_LENGTH means

* Fix watch channel change detection

* Don't return early when a mining task fails

* Spawn async miner tasks to avoid cooperative blocking, deadlocks, and improve shutdown responsiveness

* Make existing parallelism docs and configs consistent

* Add a mining parallelism config

* Use the minimum of the configured or available threads for mining

* Ignore optional feature fields in tests

* Downgrade some frequent logs to debug

* Document new zebrad features and tasks

* Describe the internal-miner feature in the CHANGELOG

* Update dependency to de-duplicate equihash solutions

* Use futures::StreamExt instead of TryStreamExt

* Fix a panic message typo

CHANGELOG.md

@@ -5,10 +5,20 @@ All notable changes to Zebra are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org).
 
+## [Zebra 1.6.0](https://github.com/ZcashFoundation/zebra/releases/tag/v1.6.0) - TODO: 2024-01-??
+
+This release:
+- TODO: summary of other important changes
+- adds an experimental `internal-miner` feature, which mines blocks within `zebrad`. This feature
+  is only supported on testnet. Use a more efficient GPU or ASIC for mainnet mining.
+
+TODO: the rest of the changelog
+
+
 ## [Zebra 1.5.0](https://github.com/ZcashFoundation/zebra/releases/tag/v1.5.0) - 2023-11-28
 
 This release:
-- fixes a panic that was introduced in Zebra v1.4.0, which happens in rare circumstances when reading cached sprout or history trees. 
+- fixes a panic that was introduced in Zebra v1.4.0, which happens in rare circumstances when reading cached sprout or history trees.
 - further improves how Zebra recovers from network interruptions and prevents potential network hangs.
 - limits the ability of synthetic nodes to spread throughout the network through Zebra to address some of the Ziggurat red team report.
 

Cargo.lock

@@ -1346,6 +1346,16 @@ dependencies = [
  "byteorder",
 ]
 
+[[package]]
+name = "equihash"
+version = "0.2.0"
+source = "git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp#251098313920466958fcd05b25e151d4edd3a1b1"
+dependencies = [
+ "blake2b_simd",
+ "byteorder",
+ "cc",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.1"
@@ -4353,6 +4363,20 @@ dependencies = [
  "syn 2.0.40",
 ]
 
+[[package]]
+name = "thread-priority"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b72cb4958060ee2d9540cef68bb3871fd1e547037772c7fe7650d5d1cbec53b3"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if 1.0.0",
+ "libc",
+ "log",
+ "rustversion",
+ "winapi",
+]
+
 [[package]]
 name = "thread_local"
 version = "1.1.7"
@@ -5619,7 +5643,7 @@ dependencies = [
  "blake2s_simd",
  "bls12_381",
  "byteorder",
- "equihash",
+ "equihash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ff",
  "fpe",
  "group",
@@ -5717,7 +5741,8 @@ dependencies = [
  "criterion",
  "displaydoc",
  "ed25519-zebra",
- "equihash",
+ "equihash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "equihash 0.2.0 (git+https://github.com/ZcashFoundation/librustzcash.git?branch=equihash-solver-tromp)",
  "futures",
  "group",
  "halo2_proofs",
@@ -5873,7 +5898,6 @@ dependencies = [
  "jsonrpc-core",
  "jsonrpc-derive",
  "jsonrpc-http-server",
- "num_cpus",
  "proptest",
  "rand 0.8.5",
  "serde",
@@ -6072,6 +6096,7 @@ dependencies = [
  "serde_json",
  "tempfile",
  "thiserror",
+ "thread-priority",
  "tinyvec",
  "tokio",
  "tokio-stream",

Cargo.toml

@@ -55,6 +55,12 @@ opt-level = 3
 [profile.dev.package.bls12_381]
 opt-level = 3
 
+[profile.dev.package.byteorder]
+opt-level = 3
+
+[profile.dev.package.equihash]
+opt-level = 3
+
 [profile.dev.package.zcash_proofs]
 opt-level = 3
 

deny.toml

@@ -86,6 +86,11 @@ skip-tree = [
     # wait for hdwallet to upgrade
     { name = "ring", version = "=0.16.20" },
 
+    # wait for the equihash/solver feature to merge
+    # https://github.com/zcash/librustzcash/pull/1083
+    # https://github.com/zcash/librustzcash/pull/1088
+    { name = "equihash", version = "=0.2.0" },
+
     # zebra-utils dependencies
 
     # wait for structopt upgrade (or upgrade to clap 4)
@@ -137,6 +142,10 @@ unknown-git = "deny"
 allow-registry = ["https://github.com/rust-lang/crates.io-index"]
 # List of URLs for allowed Git repositories
 allow-git = [
+    # TODO: remove this after the equihash solver branch is merged and released.
+    #
+    # "cargo deny" will log a warning in builds without the internal-miner feature. That's ok.
+    "https://github.com/ZcashFoundation/librustzcash.git"
 ]
 
 [sources.allow-org]

zebra-chain/Cargo.toml

@@ -29,11 +29,19 @@ async-error = [
     "tokio",
 ]
 
-# Experimental mining RPC support
+# Mining RPC support
 getblocktemplate-rpcs = [
     "zcash_address",
 ]
 
+# Experimental internal miner support
+internal-miner = [
+    # TODO: replace with "equihash/solver" when that feature is merged and released:
+    # https://github.com/zcash/librustzcash/pull/1083
+    # https://github.com/zcash/librustzcash/pull/1088
+    "equihash-solver",
+]
+
 # Experimental elasticsearch support
 elasticsearch = []
 
@@ -61,7 +69,21 @@ blake2s_simd = "1.0.2"
 bridgetree = "0.4.0"
 bs58 = { version = "0.5.0", features = ["check"] }
 byteorder = "1.5.0"
+
 equihash = "0.2.0"
+# Experimental internal miner support
+#
+# TODO: remove "equihash-solver" when the "equihash/solver" feature is merged and released:
+# https://github.com/zcash/librustzcash/pull/1083
+# https://github.com/zcash/librustzcash/pull/1088
+#
+# Use the solver PR:
+# - latest: branch = "equihash-solver-tromp",
+# - crashing with double-frees: rev = "da26c34772f4922eb13b4a1e7d88a969bbcf6a91",
+equihash-solver = { version = "0.2.0", git = "https://github.com/ZcashFoundation/librustzcash.git", branch = "equihash-solver-tromp", features = ["solver"], package = "equihash", optional = true }
+# or during development, use the locally checked out and modified version of equihash:
+#equihash-solver = { version = "0.2.0", path = "../../librustzcash/components/equihash", features = ["solver"], package = "equihash", optional = true }
+
 group = "0.13.0"
 incrementalmerkletree = "0.5.0"
 jubjub = "0.10.0"

zebra-chain/src/block/header.rs

@@ -123,6 +123,11 @@ impl Header {
             ))?
         }
     }
+
+    /// Compute the hash of this header.
+    pub fn hash(&self) -> Hash {
+        Hash::from(self)
+    }
 }
 
 /// A header with a count of the number of transactions in its block.

zebra-chain/src/primitives.rs

@@ -12,6 +12,8 @@ mod address;
 #[cfg(feature = "getblocktemplate-rpcs")]
 pub use address::Address;
 
+pub mod byte_array;
+
 pub use ed25519_zebra as ed25519;
 pub use reddsa;
 pub use redjubjub;

zebra-chain/src/primitives/byte_array.rs

@@ -0,0 +1,14 @@
+//! Functions for modifying byte arrays.
+
+/// Increments `byte_array` by 1, interpreting it as a big-endian integer.
+/// If the big-endian integer overflowed, sets all the bytes to zero, and returns `true`.
+pub fn increment_big_endian(byte_array: &mut [u8]) -> bool {
+    // Increment the last byte in the array that is less than u8::MAX, and clear any bytes after it
+    // to increment the next value in big-endian (lexicographic) order.
+    let is_wrapped_overflow = byte_array.iter_mut().rev().all(|v| {
+        *v = v.wrapping_add(1);
+        v == &0
+    });
+
+    is_wrapped_overflow
+}

zebra-chain/src/work/equihash.rs

@@ -12,16 +12,24 @@ use crate::{
     },
 };
 
-/// The error type for Equihash
+#[cfg(feature = "internal-miner")]
+use crate::serialization::AtLeastOne;
+
+/// The error type for Equihash validation.
 #[non_exhaustive]
 #[derive(Debug, thiserror::Error)]
 #[error("invalid equihash solution for BlockHeader")]
 pub struct Error(#[from] equihash::Error);
 
+/// The error type for Equihash solving.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, thiserror::Error)]
+#[error("solver was cancelled")]
+pub struct SolverCancelled;
+
 /// The size of an Equihash solution in bytes (always 1344).
 pub(crate) const SOLUTION_SIZE: usize = 1344;
 
-/// Equihash Solution.
+/// Equihash Solution in compressed format.
 ///
 /// A wrapper around [u8; 1344] because Rust doesn't implement common
 /// traits like `Debug`, `Clone`, etc for collections like array
@@ -53,18 +61,138 @@ impl Solution {
             .zcash_serialize(&mut input)
             .expect("serialization into a vec can't fail");
 
+        // The part of the header before the nonce and solution.
+        // This data is kept constant during solver runs, so the verifier API takes it separately.
         let input = &input[0..Solution::INPUT_LENGTH];
 
         equihash::is_valid_solution(n, k, input, nonce.as_ref(), solution)?;
 
         Ok(())
     }
 
-    #[cfg(feature = "getblocktemplate-rpcs")]
+    /// Returns a [`Solution`] containing the bytes from `solution`.
+    /// Returns an error if `solution` is the wrong length.
+    pub fn from_bytes(solution: &[u8]) -> Result<Self, SerializationError> {
+        if solution.len() != SOLUTION_SIZE {
+            return Err(SerializationError::Parse(
+                "incorrect equihash solution size",
+            ));
+        }
+
+        let mut bytes = [0; SOLUTION_SIZE];
+        // Won't panic, because we just checked the length.
+        bytes.copy_from_slice(solution);
+
+        Ok(Self(bytes))
+    }
+
     /// Returns a [`Solution`] of `[0; SOLUTION_SIZE]` to be used in block proposals.
+    #[cfg(feature = "getblocktemplate-rpcs")]
     pub fn for_proposal() -> Self {
         Self([0; SOLUTION_SIZE])
     }
+
+    /// Mines and returns one or more [`Solution`]s based on a template `header`.
+    /// The returned header contains a valid `nonce` and `solution`.
+    ///
+    /// If `cancel_fn()` returns an error, returns early with `Err(SolverCancelled)`.
+    ///
+    /// The `nonce` in the header template is taken as the starting nonce. If you are running multiple
+    /// solvers at the same time, start them with different nonces.
+    /// The `solution` in the header template is ignored.
+    ///
+    /// This method is CPU and memory-intensive. It uses 144 MB of RAM and one CPU core while running.
+    /// It can run for minutes or hours if the network difficulty is high.
+    #[cfg(feature = "internal-miner")]
+    #[allow(clippy::unwrap_in_result)]
+    pub fn solve<F>(
+        mut header: Header,
+        mut cancel_fn: F,
+    ) -> Result<AtLeastOne<Header>, SolverCancelled>
+    where
+        F: FnMut() -> Result<(), SolverCancelled>,
+    {
+        use crate::shutdown::is_shutting_down;
+
+        let mut input = Vec::new();
+        header
+            .zcash_serialize(&mut input)
+            .expect("serialization into a vec can't fail");
+        // Take the part of the header before the nonce and solution.
+        // This data is kept constant for this solver run.
+        let input = &input[0..Solution::INPUT_LENGTH];
+
+        while !is_shutting_down() {
+            // Don't run the solver if we'd just cancel it anyway.
+            cancel_fn()?;
+
+            let solutions = equihash_solver::tromp::solve_200_9_compressed(input, || {
+                // Cancel the solver if we have a new template.
+                if cancel_fn().is_err() {
+                    return None;
+                }
+
+                // This skips the first nonce, which doesn't matter in practice.
+                Self::next_nonce(&mut header.nonce);
+                Some(*header.nonce)
+            });
+
+            let mut valid_solutions = Vec::new();
+
+            // If we got any solutions, try submitting them, because the new template might just
+            // contain some extra transactions. Mining extra transactions is optional.
+            for solution in &solutions {
+                header.solution = Self::from_bytes(solution)
+                    .expect("unexpected invalid solution: incorrect length");
+
+                // TODO: work out why we sometimes get invalid solutions here
+                if let Err(error) = header.solution.check(&header) {
+                    info!(?error, "found invalid solution for header");
+                    continue;
+                }
+
+                if Self::difficulty_is_valid(&header) {
+                    valid_solutions.push(header);
+                }
+            }
+
+            match valid_solutions.try_into() {
+                Ok(at_least_one_solution) => return Ok(at_least_one_solution),
+                Err(_is_empty_error) => debug!(
+                    solutions = ?solutions.len(),
+                    "found valid solutions which did not pass the validity or difficulty checks"
+                ),
+            }
+        }
+
+        Err(SolverCancelled)
+    }
+
+    /// Modifies `nonce` to be the next integer in big-endian order.
+    /// Wraps to zero if the next nonce would overflow.
+    #[cfg(feature = "internal-miner")]
+    fn next_nonce(nonce: &mut [u8; 32]) {
+        let _ignore_overflow = crate::primitives::byte_array::increment_big_endian(&mut nonce[..]);
+    }
+
+    /// Returns `true` if the `nonce` and `solution` in `header` meet the difficulty threshold.
+    ///
+    /// Assumes that the difficulty threshold in the header is valid.
+    #[cfg(feature = "internal-miner")]
+    fn difficulty_is_valid(header: &Header) -> bool {
+        // Simplified from zebra_consensus::block::check::difficulty_is_valid().
+        let difficulty_threshold = header
+            .difficulty_threshold
+            .to_expanded()
+            .expect("unexpected invalid header template: invalid difficulty threshold");
+
+        // TODO: avoid calculating this hash multiple times
+        let hash = header.hash();
+
+        // Note: this comparison is a u256 integer comparison, like zcashd and bitcoin. Greater
+        // values represent *less* work.
+        hash <= difficulty_threshold
+    }
 }
 
 impl PartialEq<Solution> for Solution {
@@ -109,17 +237,6 @@ impl ZcashSerialize for Solution {
 impl ZcashDeserialize for Solution {
     fn zcash_deserialize<R: io::Read>(mut reader: R) -> Result<Self, SerializationError> {
         let solution: Vec<u8> = (&mut reader).zcash_deserialize_into()?;
-
-        if solution.len() != SOLUTION_SIZE {
-            return Err(SerializationError::Parse(
-                "incorrect equihash solution size",
-            ));
-        }
-
-        let mut bytes = [0; SOLUTION_SIZE];
-        // Won't panic, because we just checked the length.
-        bytes.copy_from_slice(&solution);
-
-        Ok(Self(bytes))
+        Self::from_bytes(&solution)
     }
 }