refactor(tree_backfiller): adjust thread management. add documentatio…

…n. general refactor to improve readability of the crate.
rpcpool · Jan 22, 2024 · e6bd076 · e6bd076
1 parent e9e0afd
commit e6bd076
Show file tree

Hide file tree

Showing 10 changed files with 114 additions and 172 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/tree_backfiller/Cargo.toml b/tree_backfiller/Cargo.toml
@@ -32,7 +32,6 @@ thiserror = "1.0.31"
 serde_json = "1.0.81"
 cadence = "0.29.0"
 cadence-macros = "0.29.0"
-hyper = "0.14.23"
 anchor-client = "0.28.0"
 tokio = { version = "1.26.0", features = ["full", "tracing"] }
 sqlx = { version = "0.6.2", features = [
@@ -69,29 +68,15 @@ mpl-bubblegum = "1.0.1-beta.3"
 spl-account-compression = { version = "0.2.0", features = ["no-entrypoint"] }
 spl-concurrent-merkle-tree = "0.2.0"
 uuid = "1.0.0"
-async-trait = "0.1.53"
-num-traits = "0.2.15"
-blockbuster = "0.9.0-beta.1"
 figment = { version = "0.10.6", features = ["env", "toml", "yaml"] }
 solana-sdk = "~1.16.16"
 solana-client = "~1.16.16"
-spl-token = { version = ">= 3.5.0, < 5.0", features = ["no-entrypoint"] }
 solana-transaction-status = "~1.16.16"
 solana-account-decoder = "~1.16.16"
-solana-geyser-plugin-interface = "~1.16.16"
-solana-sdk-macro = "~1.16.16"
-rand = "0.8.5"
 rust-crypto = "0.2.36"
 url = "2.3.1"
 anchor-lang = "0.28.0"
 borsh = "~0.10.3"
-stretto = { version = "0.7", features = ["async"] }
-tokio-stream = "0.1.12"
-tracing-subscriber = { version = "0.3.16", features = [
-  "json",
-  "env-filter",
-  "ansi",
-] }
 clap = { version = "4.2.2", features = ["derive", "cargo", "env"] }
 
 [lints]

diff --git a/tree_backfiller/README.md b/tree_backfiller/README.md
@@ -1,6 +1,6 @@
 # Tree Backfiller
 
-The Tree Backfiller crawls all trees on-chain and backfills any transactions related to a tree that have not already been observed.
+The tree backfiller CLI assists in detecting and indexing missing updates within a compression tree that are managed by the MPL bubblegum program.
 
 ## Commands
 
@@ -10,7 +10,21 @@ Command line arguments can also be set through environment variables.
 
 The `run` command initiates the crawling and backfilling process. It requires the Solana RPC URL, the database URL, and the messenger Redis URL.
 
+```mermaid
+flowchart TB;
+    start((Start)) -->|Initialize components| init[Initialize RPC, DB]
+    init -->|Fetch trees from DB| fetchTreesDB[Fetch Trees from DB]
+    fetchTreesDB -->|Find gaps in trees| findGapsDB[Find Gaps in DB]
+    findGapsDB -->|Enqueue gap fill tasks| enqueueGapFills[Enqueue Gap Fills]
+    enqueueGapFills -->|Gap Worker Manager| gapWorkerManager[Gap Worker Manager]
+    gapWorkerManager -->|Crawl Solana RPC for signatures| crawlSignatures[Crawl Solana RPC]
+    crawlSignatures -->|Enqueue signatures| enqueueSignatures[Enqueue Signatures]
+    enqueueSignatures -->|Transaction Worker Manager| transactionWorkerManager[Transaction Worker Manager]
+    transactionWorkerManager -->|Fetch transactions from Solana RPC| fetchTransactionsRPC[Fetch Transactions RPC]
+    fetchTransactionsRPC -->|Process transactions| processTransactions[Process Transactions]
+    processTransactions --> end((End))
 ```
+
 Usage: das-tree-backfiller run [OPTIONS] --solana-rpc-url <SOLANA_RPC_URL> --database-url <DATABASE_URL> --messenger-redis-url <MESSENGER_REDIS_URL>
 
 Options: