Merge pull request paritytech#335 from subspace/farmer-docs

Add docs for farmer

.github/workflows/rust.yaml

@@ -109,6 +109,33 @@ jobs:
           args: -- -D warnings
         if: runner.os == 'macOS'
 
+  cargo-docs:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Rust toolchain
+        uses: actions-rs/toolchain@v1
+        # TODO: Below can be removed when https://github.com/actions-rs/toolchain/issues/126 is resolved
+        with:
+          toolchain: nightly-2022-02-15
+          target: wasm32-unknown-unknown
+          override: true
+
+      - name: Configure cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: ${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
+
+      - name: Check Documentation
+        run: cargo doc --all --no-deps --lib
+        env:
+          RUSTDOCFLAGS: "-D rustdoc::broken-intra-doc-links -D rustdoc::private_intra_doc_links"
+
   cargo-test:
     strategy:
       matrix:

crates/subspace-farmer/README.md

@@ -73,26 +73,27 @@ Windows
 {FOLDERID_LocalAppData}             C:\Users\Alice\AppData\Local
 ```
 
-## Design
+## Architecture
 
 The farmer typically runs two processes in parallel: plotting and farming.
 
 ### Plotting
-0. A new Schnorr key pair is generated, and the farmer ID is derived from the public key.
-1. For every archived piece of the history encoding is created by applying the time-asymmetric SLOTH permutation as `encode(genesis_piece, farmer_public_key_hash, plot_index)`
-2. Each encoding is written directly to disk.
-3. A commitment, or tag, to each encoding is created as `hmac(encoding, salt)` and stored within a binary search tree (BST).
+
+Think of it as the following pipeline:
+
+1. [Farmer receives new blocks from the blockchain](src/archiving.rs)
+2. [Archives each of them](src/archiving.rs)
+3. [Encodes each archived piece by applying the time-asymmetric SLOTH permutation as `encode(genesis_piece, farmer_public_key_hash, plot_index)`](src/plotting.rs)
+4. [Each encoding is written to the disk](src/plotting.rs)
+3. [A commitment, or tag, to each encoding is created as `hmac(encoding, salt)` and stored within a binary search tree (BST)](src/plotting.rs).
 
 This process currently takes ~ 36 hours per TiB on a quad-core machine, but for 1 GiB plotting should take between a few seconds and a few minutes.
 
-### Solving
+### [Farming](src/farming.rs)
+
 1. Connect to a client and subscribe to `slot_notifications` via JSON-RPC.
 2. Given a global challenge as `hash(randomness || slot_index)` and `SOLUTION_RANGE`.
 3. Derive local challenge as `hash(global_challenge || farmer_public_key_hash)`.
 4. Query the BST for the nearest tag to the local challenge.
 5. If it within `SOLUTION_RANGE` return a `SOLUTION` else return `None`
 6. All the above can and will happen in parallel to plotting process, so it is possible to participate right away
-
-
-
-

crates/subspace-farmer/src/archiving.rs

@@ -42,6 +42,7 @@ pub struct PiecesToPlot {
     pub pieces: FlatPieces,
 }
 
+/// Abstraction around archiving blocks and updating global object map
 pub struct Archiving {
     stop_sender: Option<oneshot::Sender<()>>,
     archiving_handle: Option<JoinHandle<()>>,

crates/subspace-farmer/src/commitments.rs

@@ -55,6 +55,13 @@ struct Inner {
     commitment_databases: Mutex<CommitmentDatabases>,
 }
 
+/// `Commitments` is a database for commitments.
+///
+/// You can think of it as 2 mappings from *piece tags* to *plot offsets*.
+///
+/// Overall it is just wrapper around 2 databases (as we know just 2 salts -
+/// current and the next one). Second one is filled in the background in the
+/// `Plotting` process.
 #[derive(Debug, Clone)]
 pub struct Commitments {
     inner: Arc<Inner>,

crates/subspace-farmer/src/farming.rs

@@ -25,10 +25,13 @@ pub enum FarmingError {
     #[error("Plot read error: {0}")]
     PlotRead(std::io::Error),
 }
-/// `Farming` struct is the abstraction of the farming process
+
+/// `Farming` structure is an abstraction of the farming process for a single replica plot farming.
+///
+/// Farming instance can be stopped by dropping or it is possible to wait for it to exit on its own.
 ///
-/// Farming Instance that stores a channel to stop/pause the background farming task
-/// and a handle to make it possible to wait on this background task
+/// At high level it receives a new challenge from the consensus and tries to find solution for it
+/// in its `Commitments` database.
 pub struct Farming {
     stop_sender: async_oneshot::Sender<()>,
     handle: Option<JoinHandle<Result<(), FarmingError>>>,
@@ -42,7 +45,7 @@ impl Farming {
         commitments: Commitments,
         client: T,
         identity: Identity,
-        reward_adress: PublicKey,
+        reward_address: PublicKey,
     ) -> Self {
         // Oneshot channels, that will be used for interrupt/stop the process
         let (stop_sender, stop_receiver) = async_oneshot::oneshot();
@@ -55,7 +58,7 @@ impl Farming {
                     &plot,
                     &commitments,
                     &identity,
-                    reward_adress,
+                    reward_address,
                 )),
                 stop_receiver,
             )

crates/subspace-farmer/src/lib.rs

@@ -1,21 +1,25 @@
-//! subspace-farmer library implementation overview
+#![feature(try_blocks, hash_drain_filter, int_log, io_error_other)]
+
+//! # `subspace-farmer` library implementation overview
 //!
-//! This library provides droppable/interruptable instances of two processes that can be
-//! run in parallel: `plotting` and `farming`.
+//! This library provides droppable/interruptable instances of two processes that can be run in
+//! parallel: `plotting` and `farming`.
 //!
-//! During plotting we create a binary plot file, which contains subspace-encoded pieces one
-//! after another as well as RocksDB key-value database with tags, where key is tag (first 8 bytes
-//! of `hmac(encoding, salt)`) and value is an offset of corresponding encoded piece in the plot (we
-//! can do this because all pieces have the same size). So for every 4096 bytes we also store a
-//! record with 8-bytes tag and 8-bytes index (+some overhead of RocksDB itself).
+//! During plotting we create:
+//! * a binary plot file, which contains subspace-encoded pieces one after another
+//! * a RocksDB commitments database, where key is a tag (first 8 bytes of `hmac(encoding, salt)`)
+//!   and value is an offset of corresponding encoded piece in the plot (we can do this because all
+//!   pieces have the same size).
 //!
-//! During farming we receive a global challenge and need to find a solution, given target and
-//! solution range. In order to find solution we derive local challenge as our target and do range
-//! query in RocksDB. For that we interpret target as 64-bit unsigned integer, and find all of the
-//! keys in tags database that are `target ± solution range` (while also handing overflow/underflow)
-//! converted back to bytes.
-
-#![feature(try_blocks, hash_drain_filter, int_log, io_error_other)]
+//! In short, for every 4096 bytes we also store a record with 8-bytes tag and 8-bytes index (+some
+//! overhead of RocksDB itself).
+//!
+//! During farming we receive a global challenge and need to find a solution based on *target* and
+//! *solution range*. In order to find solution, we derive *local challenge* and use first 8 bytes
+//! (the same as tag size) as our *target* and do range query in RocksDB. For that we interpret
+//! *target* as 64-bit big-endian unsigned integer and find all of the keys in tags database that
+//! are `target ± ½ * solution range` (while also handing overflow/underflow) when interpreted as
+//! 64-bit unsigned integers.
 
 pub(crate) mod archiving;
 pub(crate) mod commitments;

crates/subspace-farmer/src/multi_farming.rs

@@ -9,7 +9,10 @@ use std::{path::Path, sync::Arc, time::Duration};
 use subspace_core_primitives::PublicKey;
 use subspace_solving::SubspaceCodec;
 
-/// Abstraction around having multiple plots, farmings and archiving
+/// Abstraction around having multiple `Plot`s, `Farming`s and `Plotting`s.
+///
+/// It is needed because of the limit of a single plot size from the consensus
+/// (`pallet_subspace::MaxPlotSize`) in order to support any amount of disk space from user.
 pub struct MultiFarming {
     pub plots: Arc<Vec<Plot>>,
     farmings: Vec<Farming>,

crates/subspace-farmer/src/object_mappings.rs

@@ -15,6 +15,7 @@ pub enum ObjectMappingError {
     Db(rocksdb::Error),
 }
 
+/// `ObjectMappings` is a mapping from arbitrary object hash to its location in archived history.
 #[derive(Debug, Clone)]
 pub struct ObjectMappings {
     db: Arc<DB>,

crates/subspace-farmer/src/plot.rs

@@ -161,13 +161,17 @@ pub fn retrieve_piece_from_plots(
         .transpose()
 }
 
-/// `Plot` struct is an abstraction on top of both plot and tags database.
+/// `Plot` is an abstraction for plotted pieces and some mappings.
 ///
-/// It converts requests to internal reads/writes to the plot and tags database. It
-/// prioritizes reads over writes by having separate queues for reads and writes requests, read
-/// requests are executed until exhausted after which at most 1 write request is handled and the
-/// cycle repeats. This allows finding solution with as little delay as possible while introducing
-/// changes to the plot at the same time (re-plotting on salt changes or extending plot size).
+/// Pieces plotted for single identity, that's why it is required to supply both address of single
+/// replica farmer and maximum amount of pieces to be stored. It offloads disk writing to separate
+/// worker, which runs in the background.
+///
+/// The worker converts requests to internal reads/writes to the plot database to direct disk
+/// reads/writes. It prioritizes reads over writes by having separate queues for high and low
+/// priority requests, read requests are executed until exhausted after which at most 1 write
+/// request is handled and the cycle repeats. This allows finding solution with as little delay as
+/// possible while introducing changes to the plot at the same time.
 #[derive(Clone)]
 pub struct Plot {
     inner: Arc<Inner>,

crates/subspace-farmer/src/plotting.rs

@@ -1,3 +1,8 @@
+//! Module with callbacks related to plotting pieces. It will:
+//! * encode pieces
+//! * write them to the plot
+//! * update commitments accordingly to change in piece set
+
 #[cfg(test)]
 mod tests;