From 16a8331d52fcfa416e764f713e674fe9968a7e7e Mon Sep 17 00:00:00 2001 From: Ivan Date: Fri, 15 Apr 2022 14:33:18 +0300 Subject: [PATCH] Merge pull request #335 from subspace/farmer-docs Add docs for farmer --- .github/workflows/rust.yaml | 27 +++++++++++++++ crates/subspace-farmer/README.md | 21 ++++++------ crates/subspace-farmer/src/archiving.rs | 1 + crates/subspace-farmer/src/commitments.rs | 7 ++++ crates/subspace-farmer/src/farming.rs | 13 ++++--- crates/subspace-farmer/src/lib.rs | 34 +++++++++++-------- crates/subspace-farmer/src/multi_farming.rs | 5 ++- crates/subspace-farmer/src/object_mappings.rs | 1 + crates/subspace-farmer/src/plot.rs | 16 +++++---- crates/subspace-farmer/src/plotting.rs | 5 +++ 10 files changed, 93 insertions(+), 37 deletions(-) diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 99831ecf18bcd..c692d3358d378 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -109,6 +109,33 @@ jobs: args: -- -D warnings if: runner.os == 'macOS' + cargo-docs: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Rust toolchain + uses: actions-rs/toolchain@v1 + # TODO: Below can be removed when https://github.com/actions-rs/toolchain/issues/126 is resolved + with: + toolchain: nightly-2022-02-15 + target: wasm32-unknown-unknown + override: true + + - name: Configure cache + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Check Documentation + run: cargo doc --all --no-deps --lib + env: + RUSTDOCFLAGS: "-D rustdoc::broken-intra-doc-links -D rustdoc::private_intra_doc_links" + cargo-test: strategy: matrix: diff --git a/crates/subspace-farmer/README.md b/crates/subspace-farmer/README.md index a573bc302ca29..d2c1f869a7801 100644 --- a/crates/subspace-farmer/README.md +++ b/crates/subspace-farmer/README.md @@ -73,26 +73,27 @@ Windows {FOLDERID_LocalAppData} C:\Users\Alice\AppData\Local ``` -## Design +## Architecture The farmer typically runs two processes in parallel: plotting and farming. ### Plotting -0. A new Schnorr key pair is generated, and the farmer ID is derived from the public key. -1. For every archived piece of the history encoding is created by applying the time-asymmetric SLOTH permutation as `encode(genesis_piece, farmer_public_key_hash, plot_index)` -2. Each encoding is written directly to disk. -3. A commitment, or tag, to each encoding is created as `hmac(encoding, salt)` and stored within a binary search tree (BST). + +Think of it as the following pipeline: + +1. [Farmer receives new blocks from the blockchain](src/archiving.rs) +2. [Archives each of them](src/archiving.rs) +3. [Encodes each archived piece by applying the time-asymmetric SLOTH permutation as `encode(genesis_piece, farmer_public_key_hash, plot_index)`](src/plotting.rs) +4. [Each encoding is written to the disk](src/plotting.rs) +3. [A commitment, or tag, to each encoding is created as `hmac(encoding, salt)` and stored within a binary search tree (BST)](src/plotting.rs). This process currently takes ~ 36 hours per TiB on a quad-core machine, but for 1 GiB plotting should take between a few seconds and a few minutes. -### Solving +### [Farming](src/farming.rs) + 1. Connect to a client and subscribe to `slot_notifications` via JSON-RPC. 2. Given a global challenge as `hash(randomness || slot_index)` and `SOLUTION_RANGE`. 3. Derive local challenge as `hash(global_challenge || farmer_public_key_hash)`. 4. Query the BST for the nearest tag to the local challenge. 5. If it within `SOLUTION_RANGE` return a `SOLUTION` else return `None` 6. All the above can and will happen in parallel to plotting process, so it is possible to participate right away - - - - diff --git a/crates/subspace-farmer/src/archiving.rs b/crates/subspace-farmer/src/archiving.rs index d3c1ba7d0b2c2..fb07b44d62276 100644 --- a/crates/subspace-farmer/src/archiving.rs +++ b/crates/subspace-farmer/src/archiving.rs @@ -42,6 +42,7 @@ pub struct PiecesToPlot { pub pieces: FlatPieces, } +/// Abstraction around archiving blocks and updating global object map pub struct Archiving { stop_sender: Option>, archiving_handle: Option>, diff --git a/crates/subspace-farmer/src/commitments.rs b/crates/subspace-farmer/src/commitments.rs index b336e5a3c479c..024fa3d42faaa 100644 --- a/crates/subspace-farmer/src/commitments.rs +++ b/crates/subspace-farmer/src/commitments.rs @@ -55,6 +55,13 @@ struct Inner { commitment_databases: Mutex, } +/// `Commitments` is a database for commitments. +/// +/// You can think of it as 2 mappings from *piece tags* to *plot offsets*. +/// +/// Overall it is just wrapper around 2 databases (as we know just 2 salts - +/// current and the next one). Second one is filled in the background in the +/// `Plotting` process. #[derive(Debug, Clone)] pub struct Commitments { inner: Arc, diff --git a/crates/subspace-farmer/src/farming.rs b/crates/subspace-farmer/src/farming.rs index 57ebea943627b..434010437c8c4 100644 --- a/crates/subspace-farmer/src/farming.rs +++ b/crates/subspace-farmer/src/farming.rs @@ -25,10 +25,13 @@ pub enum FarmingError { #[error("Plot read error: {0}")] PlotRead(std::io::Error), } -/// `Farming` struct is the abstraction of the farming process + +/// `Farming` structure is an abstraction of the farming process for a single replica plot farming. +/// +/// Farming instance can be stopped by dropping or it is possible to wait for it to exit on its own. /// -/// Farming Instance that stores a channel to stop/pause the background farming task -/// and a handle to make it possible to wait on this background task +/// At high level it receives a new challenge from the consensus and tries to find solution for it +/// in its `Commitments` database. pub struct Farming { stop_sender: async_oneshot::Sender<()>, handle: Option>>, @@ -42,7 +45,7 @@ impl Farming { commitments: Commitments, client: T, identity: Identity, - reward_adress: PublicKey, + reward_address: PublicKey, ) -> Self { // Oneshot channels, that will be used for interrupt/stop the process let (stop_sender, stop_receiver) = async_oneshot::oneshot(); @@ -55,7 +58,7 @@ impl Farming { &plot, &commitments, &identity, - reward_adress, + reward_address, )), stop_receiver, ) diff --git a/crates/subspace-farmer/src/lib.rs b/crates/subspace-farmer/src/lib.rs index 62fe610338b72..e31d72ca01c6d 100644 --- a/crates/subspace-farmer/src/lib.rs +++ b/crates/subspace-farmer/src/lib.rs @@ -1,21 +1,25 @@ -//! subspace-farmer library implementation overview +#![feature(try_blocks, hash_drain_filter, int_log, io_error_other)] + +//! # `subspace-farmer` library implementation overview //! -//! This library provides droppable/interruptable instances of two processes that can be -//! run in parallel: `plotting` and `farming`. +//! This library provides droppable/interruptable instances of two processes that can be run in +//! parallel: `plotting` and `farming`. //! -//! During plotting we create a binary plot file, which contains subspace-encoded pieces one -//! after another as well as RocksDB key-value database with tags, where key is tag (first 8 bytes -//! of `hmac(encoding, salt)`) and value is an offset of corresponding encoded piece in the plot (we -//! can do this because all pieces have the same size). So for every 4096 bytes we also store a -//! record with 8-bytes tag and 8-bytes index (+some overhead of RocksDB itself). +//! During plotting we create: +//! * a binary plot file, which contains subspace-encoded pieces one after another +//! * a RocksDB commitments database, where key is a tag (first 8 bytes of `hmac(encoding, salt)`) +//! and value is an offset of corresponding encoded piece in the plot (we can do this because all +//! pieces have the same size). //! -//! During farming we receive a global challenge and need to find a solution, given target and -//! solution range. In order to find solution we derive local challenge as our target and do range -//! query in RocksDB. For that we interpret target as 64-bit unsigned integer, and find all of the -//! keys in tags database that are `target ± solution range` (while also handing overflow/underflow) -//! converted back to bytes. - -#![feature(try_blocks, hash_drain_filter, int_log, io_error_other)] +//! In short, for every 4096 bytes we also store a record with 8-bytes tag and 8-bytes index (+some +//! overhead of RocksDB itself). +//! +//! During farming we receive a global challenge and need to find a solution based on *target* and +//! *solution range*. In order to find solution, we derive *local challenge* and use first 8 bytes +//! (the same as tag size) as our *target* and do range query in RocksDB. For that we interpret +//! *target* as 64-bit big-endian unsigned integer and find all of the keys in tags database that +//! are `target ± ½ * solution range` (while also handing overflow/underflow) when interpreted as +//! 64-bit unsigned integers. pub(crate) mod archiving; pub(crate) mod commitments; diff --git a/crates/subspace-farmer/src/multi_farming.rs b/crates/subspace-farmer/src/multi_farming.rs index 2fb2fc366ad86..7d541ee22106d 100644 --- a/crates/subspace-farmer/src/multi_farming.rs +++ b/crates/subspace-farmer/src/multi_farming.rs @@ -9,7 +9,10 @@ use std::{path::Path, sync::Arc, time::Duration}; use subspace_core_primitives::PublicKey; use subspace_solving::SubspaceCodec; -/// Abstraction around having multiple plots, farmings and archiving +/// Abstraction around having multiple `Plot`s, `Farming`s and `Plotting`s. +/// +/// It is needed because of the limit of a single plot size from the consensus +/// (`pallet_subspace::MaxPlotSize`) in order to support any amount of disk space from user. pub struct MultiFarming { pub plots: Arc>, farmings: Vec, diff --git a/crates/subspace-farmer/src/object_mappings.rs b/crates/subspace-farmer/src/object_mappings.rs index 23824e519550f..9e6d3fba7449d 100644 --- a/crates/subspace-farmer/src/object_mappings.rs +++ b/crates/subspace-farmer/src/object_mappings.rs @@ -15,6 +15,7 @@ pub enum ObjectMappingError { Db(rocksdb::Error), } +/// `ObjectMappings` is a mapping from arbitrary object hash to its location in archived history. #[derive(Debug, Clone)] pub struct ObjectMappings { db: Arc, diff --git a/crates/subspace-farmer/src/plot.rs b/crates/subspace-farmer/src/plot.rs index 267eee9c452c0..465ff8b9b23be 100644 --- a/crates/subspace-farmer/src/plot.rs +++ b/crates/subspace-farmer/src/plot.rs @@ -161,13 +161,17 @@ pub fn retrieve_piece_from_plots( .transpose() } -/// `Plot` struct is an abstraction on top of both plot and tags database. +/// `Plot` is an abstraction for plotted pieces and some mappings. /// -/// It converts requests to internal reads/writes to the plot and tags database. It -/// prioritizes reads over writes by having separate queues for reads and writes requests, read -/// requests are executed until exhausted after which at most 1 write request is handled and the -/// cycle repeats. This allows finding solution with as little delay as possible while introducing -/// changes to the plot at the same time (re-plotting on salt changes or extending plot size). +/// Pieces plotted for single identity, that's why it is required to supply both address of single +/// replica farmer and maximum amount of pieces to be stored. It offloads disk writing to separate +/// worker, which runs in the background. +/// +/// The worker converts requests to internal reads/writes to the plot database to direct disk +/// reads/writes. It prioritizes reads over writes by having separate queues for high and low +/// priority requests, read requests are executed until exhausted after which at most 1 write +/// request is handled and the cycle repeats. This allows finding solution with as little delay as +/// possible while introducing changes to the plot at the same time. #[derive(Clone)] pub struct Plot { inner: Arc, diff --git a/crates/subspace-farmer/src/plotting.rs b/crates/subspace-farmer/src/plotting.rs index de7b5cfbf9dab..b0a8a0338610b 100644 --- a/crates/subspace-farmer/src/plotting.rs +++ b/crates/subspace-farmer/src/plotting.rs @@ -1,3 +1,8 @@ +//! Module with callbacks related to plotting pieces. It will: +//! * encode pieces +//! * write them to the plot +//! * update commitments accordingly to change in piece set + #[cfg(test)] mod tests;