feat: integrating Metaplex DAS' backfiller

Cargo.toml

@@ -7,12 +7,15 @@ edition = "2021"
 members = [
     "migration",
     "digital_asset_types",
-    "program_transformers"
+    "program_transformers",
+    "blockbuster",
+    "core",
+    "bubblegum-backfill"
 ]
 
 [dependencies]
+anchor-lang = {workspace=true}
 anyhow = {workspace=true}
-blockbuster = {workspace=true}
 digital_asset_types = { workspace = true, features = ["json_types", "sql_types"] }
 dotenv = {workspace=true}
 futures = {workspace=true}
@@ -45,9 +48,34 @@ serde_json = {workspace=true}
 spl-concurrent-merkle-tree = {workspace=true}
 thiserror = {workspace=true}
 url = {workspace=true}
-base64 = "0.22.0"
-reqwest = "0.11.24"
-
+das-core = {workspace=true}
+heck = {workspace=true}
+backon = {workspace=true}
+clap = {workspace=true}
+derive_more = {workspace=true}
+figment = {workspace=true}
+indicatif = {workspace=true}
+plerkle_messenger = {workspace=true}
+reqwest = {workspace=true}
+solana-account-decoder = {workspace=true}
+bytemuck = {workspace=true}
+mpl-core = {workspace=true}
+mpl-token-metadata = {workspace=true}
+solana-zk-token-sdk = {workspace=true}
+spl-noop = {workspace=true}
+spl-pod = {workspace=true}
+anchor-client = {workspace=true}
+spl-token-2022  = {workspace=true}
+spl-token-group-interface  = {workspace=true}
+das-bubblegum-backfill = {workspace=true}
+spl-token-metadata-interface = {workspace=true}
+flatbuffers = {workspace=true}
+rand = {workspace=true}
+solana-geyser-plugin-interface = {workspace=true}
+solana-program = {workspace=true}
+blockbuster = {workspace=true}
+signal-hook = {workspace=true}
+base64 = {workspace=true}
 
 
 [[bin]]
@@ -61,45 +89,76 @@ repository = "https://github.com/WilfredAlmeida/LightDAS"
 version = "0.1.0"
 
 [workspace.dependencies]
-anyhow = "1.0.79"
-blockbuster = {git="https://github.com/rpcpool/blockbuster", branch="rm-plerkle-101"}
-cadence = "0.29.0"
-cadence-macros = "0.29.0"
+anyhow = "1.0.86"
+base64 = "0.22.1"
+blockbuster = { path = "blockbuster" }
+cadence = "0.29.1"
+cadence-macros = "0.29.1"
 dotenv = "0.15.0"
 digital_asset_types = { path = "digital_asset_types" }
-futures = "0.3.28"
-lazy_static = "1.4.0"
-mpl-bubblegum = "1.2.0"
-plerkle_serialization = "1.6.0"
-sea-orm = { version = "0.10.6", features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] }
-solana-client = "1.16.2"
-solana-rpc-client-api = "1.16.2"
-solana-sdk = "1.16.27"
-solana-transaction-status = "1.16.2"
-sqlx = "0.6.2"
-tokio = "1.30.0"
-log = "0.4.17"
-async-trait = "0.1.60"
+futures = "0.3.30"
+lazy_static = "1.5.0"
+mpl-bubblegum = "1.4.0"
+plerkle_serialization = "1.8.0"
+sea-orm = { version = "0.10.7", features = ["macros", "runtime-tokio-rustls", "sqlx-postgres", "with-chrono", "mock"] }
+solana-client = "~1.17"
+solana-rpc-client-api = "~1.17"
+solana-sdk = "~1.17"
+solana-transaction-status = "~1.17"
+sqlx = "0.6.3"
+tokio = "1.38.0"
+log = "0.4.22"
+async-trait = "0.1.80"
 borsh = "~0.10.3"
 borsh-derive = "~0.10.3"
 bs58 = "0.4.0"
 indexmap = "1.9.3"
 jsonpath_lib = "0.3.0"
 mime_guess = "2.0.4"
 num-derive = "0.3.3"
-num-traits = "0.2.15"
+num-traits = "0.2.19"
 program_transformers = { path = "program_transformers" }
-schemars = "0.8.6"
-schemars_derive = "0.8.6"
-sea-query = "0.28.1"
-serde = "1.0.137"
-serde_json = "1.0.81"
+das-bubblegum-backfill = { path = "bubblegum-backfill" }
+das-core = { path = "core" }
+schemars = "0.8.21"
+schemars_derive = "0.8.21"
+sea-query = "0.28.5"
+serde = "1.0.203"
+serde_json = "1.0.118"
 spl-concurrent-merkle-tree = "0.2.0"
-spl-account-compression = "0.2.0"
+spl-account-compression = "0.3.0"
 spl-token = ">= 3.5.0, < 5.0"
-thiserror = "1.0.31"
-tracing = "0.1.35"
-url = "2.3.1"
+thiserror = "1.0.61"
+tracing = "0.1.40"
+url = "2.5.2"
+heck = "0.5.0"
+backon = "0.4.4"
+clap = "4.5.7"
+derive_more = { version = "0.99.18" }
+figment = "0.10.19"
+indicatif = "0.17.8"
+plerkle_messenger = "1.8.0"
+reqwest = "0.11.27"
+signal-hook = "0.3.17"
+solana-account-decoder = "~1.17"
+bytemuck = { version = "1.16.1", features = ["derive"] }
+mpl-core = { version = "0.7.1", features = ["serde"] }
+mpl-token-metadata = "4.1.2"
+solana-zk-token-sdk = "~1.17"
+spl-noop = "0.2.0"
+spl-pod = { version = "0.1.0", features = ["serde-traits"] }
+
+
+spl-token-metadata-interface = "0.2.0"
+flatbuffers = "23.5.26"
+rand = "0.8.5"
+solana-geyser-plugin-interface = "~1.17"
+solana-program = "~1.17"
+anchor-client = "0.29.0"
+anchor-lang = "0.29.0"
+spl-token-2022 = { version = "1.0", features = ["no-entrypoint"] }
+spl-token-group-interface = "0.1.0"
+
 
 [workspace.lints.clippy]
 clone_on_ref_ptr = "deny"

README.md

@@ -10,25 +10,14 @@ It allows you to index specific Merkle Trees that you care about. This repositor
 - Upsert the Metaplex's DAS database
 ![LightDAS drawio](https://github.com/WilfredAlmeida/LightDAS/assets/60785452/323da5a6-de11-45a0-bdd2-e5b28d547e71)
 
-
-### Backfilling and Live Transactions Indexing Process
-![lightdas-queue](https://github.com/WilfredAlmeida/LightDAS/assets/60785452/e24fcc69-e5fa-406e-99a3-1cce22815740)
-1. LightDAS is started and calls `getSignaturesForAddress` on the Merkle Tree
-2. It returns transaction signatures from the latest to the first
-3. These are put into the queue one by one via push front. So when step 1 is completed, the queue will have all transaction signatures in an order of first to latest for backfilling
-4. LightDAS in parallel to step 1, also calls `logsSubscribe` on the Merkle Trees and listens to live transaction happening on them
-5. These are put into the queue one by one via push-back
-6. After step 1 is completed and all transaction signatures are present in the queue, a processes task starts in parallel and, processes transactions
-7. These processed transactions are inserted into the Metaplex DAS Database
-8. The Metaplex DAS API serves DAS requests by interacting with the DAS database
-
 ### Reasons we are building LigthDAS
 - Running a standard DAS API is expensive and complicated
 - It gives you data off all of the NFTs on chain, but do you really need all of it?
 - There are select DAS offerings thus creating a monopolistic environment
 
 With LightDAS, you can have your own DAS API without the nft ingester or any other heavy lifting. The components you need to get your DAS running are:
 - LightDAS ingester (us)
+- DAS Backfiller (Metaplex)
 - DAS API Handler (Metaplex)
 - DAS Database (Metaplex)
 - Graphite Monitoring (Metaplex)
@@ -51,6 +40,7 @@ Follow the steps mentioned below
   - `DATABASE_URL`: Default is `postgres://solana:solana@localhost:5432/solana`, use this unless you changed anything
 - Execute `cargo run`
 - This will download and compile the code with all needed dependencies. Grab a coffee this takes a while
+- The first run will fail and complain about no tree addresses being found to index, you need to add tree addresses to index in the database. See the `#trees config` section below
 - Once running, you'll see the logs of the tasks being performed
 - Under heavy loads, we have faced RPC rate limits
 - RPC Costs per NFT Mint:
@@ -59,14 +49,33 @@ Follow the steps mentioned below
     - `getTransaction`: 50 credits
 - Overall, each NFT mint will cost you 100 RPC credits
 
+### Trees Config
+1. The address of the trees to be indexed needs to be provided via the database
+2. LightDAS creates a table with the following schema 
+   ```
+   CREATE TABLE IF NOT EXISTS LD_MERKLE_TREES (
+      ADDRESS VARCHAR(255),
+      TAG VARCHAR(255) NULL,
+      CAPACITY INT NULL,
+      MAX_DEPTH INT NULL,
+      CANOPY_DEPTH INT NULL,
+      MAX_BUFFER_SIZE INT NULL,
+      SHOULD_INDEX BOOLEAN DEFAULT TRUE,
+      CREATED_AT TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+      UPDATED_AT TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    );
+   ```
+3. You need to add your addresses in the table `ld_merkle_trees`
+4. To update tree addresses dynamically:
+  1. Update the above table
+  2. Send a SIGHUP signal to the LightDAS process
+  3. LightDAS handles the signal and update it's indexing without disrupting existing tasks
+
 **Currently LightDAS supports only Compressed NFTs**:
 
 ### Testing
 If the program is running without any errors then the database is populated with information on new NFT mints. You can query the RPC API locally. It runs on the default URL `http://localhost:9090/`
 
-### Note
-1. Asynchronous backfilling is broken and the logic needs to be redone
-2. Backfilling is disabled by default, enable it by uncommenting the block at `src/main.rs:148`
 
 ### Support
 If you need any help, have any thoughts, or need to get in touch, DM [Wilfred](https://twitter.com/WilfredAlmeida_) on Twitter/X or open an issue.
@@ -77,7 +86,6 @@ We have some open [RFCs](https://github.com/WilfredAlmeida/LightDAS/labels/rfc)
 The following is our roadmap in decreasing order of priority:  
 - Test API responses correctness against standard DAS API responses
 - Publish benchmarking results of testing with different RPC providers under various deployment environments
-- Test out if LightDAS can work as a full fledged DAS. Since we're watching Merkle trees, we can also watch the Bubblegum program and index all NFT operations.
 
 ### The Future of LightDAS
 Our vision for LightDAS is to keep it an open-source public good for everyone. We aim to be a compliment to DAS, not to compete against it. Eventually, we would like to streamline the setup process to only need a single binary with minimal dependencies so it's easy for a project to setup a light DAS client to watch a tree and start serving requests. The future decisions for LightDAS will be based on community feedback and discussions.

blockbuster/.gitignore

@@ -0,0 +1,11 @@
+# Generated by Cargo
+# will have compiled files and executables
+/target/
+
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+*.iml
+blockbuster/target
+target/

blockbuster/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+authors = ["Metaplex Developers <[email protected]>"]
+description = "Metaplex canonical program parsers, for indexing, analytics etc...."
+edition = "2021"
+license = "AGPL-3.0"
+name = "blockbuster"
+readme = "../README.md"
+
+version = "2.3.0"
+
+[dependencies]
+anchor-lang = {workspace = true}
+async-trait = {workspace = true}
+borsh = {workspace = true}
+bs58 = {workspace = true}
+bytemuck = {workspace = true}
+lazy_static = {workspace = true}
+log = {workspace = true}
+mpl-bubblegum = {workspace = true}
+mpl-core = {workspace = true, features = ["serde"]}
+mpl-token-metadata = {workspace = true, features = ["serde"]}
+serde = {workspace = true}
+solana-sdk = {workspace = true}
+solana-transaction-status = {workspace = true}
+solana-zk-token-sdk = {workspace = true}
+spl-account-compression = {workspace = true, features = ["no-entrypoint"]}
+spl-noop = {workspace = true, features = ["no-entrypoint"]}
+spl-pod = {workspace = true, features = ["serde-traits"]}
+spl-token = {workspace = true, features = ["no-entrypoint"]}
+spl-token-2022 = {workspace = true, features = ["no-entrypoint"]}
+spl-token-group-interface = {workspace = true}
+spl-token-metadata-interface = {workspace = true}
+thiserror = {workspace = true}
+
+[dev-dependencies]
+flatbuffers = {workspace = true}
+plerkle_serialization = {workspace = true}
+rand = {workspace = true}
+serde_json = {workspace = true}
+solana-client = {workspace = true}
+solana-geyser-plugin-interface = {workspace = true}
+spl-concurrent-merkle-tree = {workspace = true}

blockbuster/README.md

@@ -0,0 +1,22 @@
+# BlockBuster
+
+BlockBuster -> "Busting Solana blocks into little pieces to index and operate on the programs therein" - Noone - 1995
+
+This repository is the home for Metaplex Program Parsers. Program parsers are canonical libraries that take a transaction or account update from a geyser plugin and parse them correctly according to Metaplex smart contracts. This sort of parsing is hard to automate as it must contain some knowledge of the API structure of the contract which is not fully describable yet via IDLs. Things like remaining accounts, optional accounts and complex instruction data are not always 100% clear what they mean without knowledge of the contract. 
+
+## Mode of Operation
+This library works best as a consumer of messages sent via a geyser plugin using the [Plerkle Serialization](https://github.com/metaplex-foundation/digital-asset-validator-plugin) library by metaplex. The types from that library are FlatBuffer based currently, and are the wire format of messages coming out of Plerkle into the rest of the infrastructure.
+For more information about Plerkle and the [Digital Asset RPC infrastructure](https://github.com/metaplex-foundation/digital-asset-validator-plugin) It can however be used in any general programs provided you can create the data in the FlatBuffer types.
+
+## Scope
+
+This library contains parsers for the following programs and the parsers are specific to how these contracts relate to metaplex assets.
+
+* Gummyroll (Solana)
+* Bubblegum (Metaplex)
+* Spl Token (Solana)
+* Token Metadata (Metaplex)
+* Auction House (Metaplex)
+* Candy Machine (Metaplex)
+* Hydra (Metaplex)
+

blockbuster/src/error.rs

@@ -0,0 +1,34 @@
+use std::io::Error;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum BlockbusterError {
+    #[error("Instruction Data Parsing Error")]
+    InstructionParsingError,
+    #[error("IO Error {0}")]
+    IOError(String),
+    #[error("Could not deserialize data")]
+    DeserializationError,
+    #[error("Missing Bubblegum event data")]
+    MissingBubblegumEventData,
+    #[error("Data length is invalid.")]
+    InvalidDataLength,
+    #[error("Unknown anchor account discriminator.")]
+    UnknownAccountDiscriminator,
+    #[error("Account type is not valid")]
+    InvalidAccountType,
+    #[error("Master edition version is invalid")]
+    FailedToDeserializeToMasterEdition,
+    #[error("Uninitialized account type")]
+    UninitializedAccount,
+    #[error("Account Type Not implemented")]
+    AccountTypeNotImplemented,
+    #[error("Could not deserialize data: {0}")]
+    CustomDeserializationError(String),
+}
+
+impl From<std::io::Error> for BlockbusterError {
+    fn from(err: Error) -> Self {
+        BlockbusterError::IOError(err.to_string())
+    }
+}