Enable indexing Merkle trees on other SVMs (#211)

* Update to Solana 1.18

* Add Bubblegum parsing for mpl-account-compression trees

* Refactor and update docker prepare script

* Suppress warning on use of Borsh 0.10 usage

* Move common functionality to helpers

* Download Bubblegum from devnet

* Add integration test for transferring an mpl-account-compression asset

Cargo.toml

@@ -68,7 +68,9 @@ metrics = "0.20.1"
 migration = {path = "migration"}
 mime_guess = "2.0.4"
 mpl-bubblegum = "1.2.0"
+mpl-account-compression = "0.4.2"
 mpl-core = {version = "0.8.0-beta.1", features = ["serde"]}
+mpl-noop = "0.2.1"
 mpl-token-metadata = "4.1.1"
 nft_ingester = {path = "nft_ingester"}
 num-derive = "0.3.3"
@@ -93,16 +95,16 @@ sea-query = "0.28.1"
 serde = "1.0.137"
 serde_json = "1.0.81"
 serial_test = "2.0.0"
-solana-account-decoder = "~1.17"
-solana-client = "~1.17"
-solana-geyser-plugin-interface = "~1.17"
-solana-program = "~1.17"
-solana-sdk = "~1.17"
-solana-transaction-status = "~1.17"
-solana-zk-token-sdk = "1.17.16"
-spl-account-compression = "0.3.0"
+solana-account-decoder = "~1.18.15"
+solana-client = "~1.18.15"
+solana-geyser-plugin-interface = "~1.18.15"
+solana-program = "~1.18.15"
+solana-sdk = "~1.18.15"
+solana-transaction-status = "~1.18.15"
+solana-zk-token-sdk = "~1.18.15"
+spl-account-compression = "0.4.2"
 spl-associated-token-account = ">= 1.1.3, < 3.0"
-spl-concurrent-merkle-tree = "0.2.0"
+spl-concurrent-merkle-tree = "0.4.1"
 spl-noop = "0.2.0"
 spl-pod = {version = "0.1.0", features = ["serde-traits"]}
 spl-token = ">= 3.5.0, < 5.0"

blockbuster/Cargo.toml

@@ -16,8 +16,10 @@ bs58 = {workspace = true}
 bytemuck = {workspace = true}
 lazy_static = {workspace = true}
 log = {workspace = true}
+mpl-account-compression = {workspace = true, features = ["no-entrypoint"]}
 mpl-bubblegum = {workspace = true}
 mpl-core = {workspace = true, features = ["serde"]}
+mpl-noop = {workspace = true, features = ["no-entrypoint"]}
 mpl-token-metadata = {workspace = true, features = ["serde"]}
 serde = {workspace = true}
 solana-sdk = {workspace = true}

blockbuster/src/programs/bubblegum/mod.rs

@@ -18,12 +18,6 @@ pub use mpl_bubblegum::{
     InstructionName, LeafSchemaEvent, ID,
 };
 use solana_sdk::pubkey::Pubkey;
-pub use spl_account_compression::events::{
-    AccountCompressionEvent::{self, ApplicationData, ChangeLog},
-    ApplicationDataEvent, ChangeLogEvent, ChangeLogEventV1,
-};
-
-use spl_noop;
 
 #[derive(Eq, PartialEq)]
 pub enum Payload {
@@ -57,7 +51,7 @@ pub enum Payload {
 //TODO add more of the parsing here to minimize program transformer code
 pub struct BubblegumInstruction {
     pub instruction: InstructionName,
-    pub tree_update: Option<ChangeLogEventV1>,
+    pub tree_update: Option<spl_account_compression::events::ChangeLogEventV1>,
     pub leaf_update: Option<LeafSchemaEvent>,
     pub payload: Option<Payload>,
 }
@@ -129,6 +123,11 @@ impl ProgramParser for BubblegumParser {
         if let Some(ixs) = inner_ix {
             for (pid, cix) in ixs.iter() {
                 if pid == &spl_noop::id() && !cix.data.is_empty() {
+                    use spl_account_compression::events::{
+                        AccountCompressionEvent::{self, ApplicationData, ChangeLog},
+                        ApplicationDataEvent, ChangeLogEvent,
+                    };
+
                     match AccountCompressionEvent::try_from_slice(&cix.data) {
                         Ok(result) => match result {
                             ChangeLog(changelog_event) => {
@@ -138,27 +137,41 @@ impl ProgramParser for BubblegumParser {
                             ApplicationData(app_data) => {
                                 let ApplicationDataEvent::V1(app_data) = app_data;
                                 let app_data = app_data.application_data;
+                                b_inst.leaf_update =
+                                    Some(get_bubblegum_leaf_schema_event(app_data)?);
+                            }
+                        },
+                        Err(e) => {
+                            warn!(
+                                "Error while deserializing txn {:?} with spl-noop data: {:?}",
+                                txn_id, e
+                            );
+                        }
+                    }
+                } else if pid == &mpl_noop::id() && !cix.data.is_empty() {
+                    use mpl_account_compression::events::{
+                        AccountCompressionEvent::{self, ApplicationData, ChangeLog},
+                        ApplicationDataEvent, ChangeLogEvent,
+                    };
 
-                                let event_type_byte = if !app_data.is_empty() {
-                                    &app_data[0..1]
-                                } else {
-                                    return Err(BlockbusterError::DeserializationError);
-                                };
-
-                                match BubblegumEventType::try_from_slice(event_type_byte)? {
-                                    BubblegumEventType::Uninitialized => {
-                                        return Err(BlockbusterError::MissingBubblegumEventData);
-                                    }
-                                    BubblegumEventType::LeafSchemaEvent => {
-                                        b_inst.leaf_update =
-                                            Some(LeafSchemaEvent::try_from_slice(&app_data)?);
-                                    }
-                                }
+                    match AccountCompressionEvent::try_from_slice(&cix.data) {
+                        Ok(result) => match result {
+                            ChangeLog(mpl_changelog_event) => {
+                                let ChangeLogEvent::V1(mpl_changelog_event) = mpl_changelog_event;
+                                let spl_change_log_event =
+                                    convert_mpl_to_spl_change_log_event(mpl_changelog_event);
+                                b_inst.tree_update = Some(spl_change_log_event);
+                            }
+                            ApplicationData(app_data) => {
+                                let ApplicationDataEvent::V1(app_data) = app_data;
+                                let app_data = app_data.application_data;
+                                b_inst.leaf_update =
+                                    Some(get_bubblegum_leaf_schema_event(app_data)?);
                             }
                         },
                         Err(e) => {
                             warn!(
-                                "Error while deserializing txn {:?} with noop data: {:?}",
+                                "Error while deserializing txn {:?} with mpl-noop data: {:?}",
                                 txn_id, e
                             );
                         }
@@ -215,6 +228,39 @@ impl ProgramParser for BubblegumParser {
     }
 }
 
+fn get_bubblegum_leaf_schema_event(app_data: Vec<u8>) -> Result<LeafSchemaEvent, BlockbusterError> {
+    let event_type_byte = if !app_data.is_empty() {
+        &app_data[0..1]
+    } else {
+        return Err(BlockbusterError::DeserializationError);
+    };
+
+    match BubblegumEventType::try_from_slice(event_type_byte)? {
+        BubblegumEventType::Uninitialized => Err(BlockbusterError::MissingBubblegumEventData),
+        BubblegumEventType::LeafSchemaEvent => Ok(LeafSchemaEvent::try_from_slice(&app_data)?),
+    }
+}
+
+// Convert from mpl-account-compression `ChangeLogEventV1` to
+// spl-account-compression `ChangeLogEventV1`.
+fn convert_mpl_to_spl_change_log_event(
+    mpl_changelog_event: mpl_account_compression::events::ChangeLogEventV1,
+) -> spl_account_compression::events::ChangeLogEventV1 {
+    spl_account_compression::events::ChangeLogEventV1 {
+        id: mpl_changelog_event.id,
+        path: mpl_changelog_event
+            .path
+            .iter()
+            .map(|path_node| spl_account_compression::state::PathNode {
+                node: path_node.node,
+                index: path_node.index,
+            })
+            .collect(),
+        seq: mpl_changelog_event.seq,
+        index: mpl_changelog_event.index,
+    }
+}
+
 // See Bubblegum documentation for offsets and positions:
 // https://github.com/metaplex-foundation/mpl-bubblegum/blob/main/programs/bubblegum/README.md#-verify_creator-and-unverify_creator
 fn build_creator_verification_payload(

blockbuster/src/programs/token_metadata/mod.rs

@@ -4,15 +4,16 @@ use crate::{
     programs::ProgramParseResult,
 };
 use borsh::BorshDeserialize;
-use solana_sdk::{borsh0_10::try_from_slice_unchecked, pubkey::Pubkey, pubkeys};
-
 use mpl_token_metadata::{
     accounts::{
         CollectionAuthorityRecord, DeprecatedMasterEditionV1, Edition, EditionMarker,
         MasterEdition, Metadata, UseAuthorityRecord,
     },
     types::Key,
 };
+#[allow(deprecated)]
+use solana_sdk::borsh0_10::try_from_slice_unchecked;
+use solana_sdk::{pubkey::Pubkey, pubkeys};
 
 pubkeys!(
     token_metadata_id,
@@ -79,6 +80,7 @@ impl ProgramParser for TokenMetadataParser {
         let key = Key::try_from_slice(&account_data[0..1])?;
         let token_metadata_account_state = match key {
             Key::EditionV1 => {
+                #[allow(deprecated)]
                 let account: Edition = try_from_slice_unchecked(account_data)?;
 
                 TokenMetadataAccountState {
@@ -87,6 +89,7 @@ impl ProgramParser for TokenMetadataParser {
                 }
             }
             Key::MasterEditionV1 => {
+                #[allow(deprecated)]
                 let account: DeprecatedMasterEditionV1 = try_from_slice_unchecked(account_data)?;
 
                 TokenMetadataAccountState {
@@ -95,6 +98,7 @@ impl ProgramParser for TokenMetadataParser {
                 }
             }
             Key::MasterEditionV2 => {
+                #[allow(deprecated)]
                 let account: MasterEdition = try_from_slice_unchecked(account_data)?;
 
                 TokenMetadataAccountState {
@@ -103,6 +107,7 @@ impl ProgramParser for TokenMetadataParser {
                 }
             }
             Key::UseAuthorityRecord => {
+                #[allow(deprecated)]
                 let account: UseAuthorityRecord = try_from_slice_unchecked(account_data)?;
 
                 TokenMetadataAccountState {
@@ -111,6 +116,7 @@ impl ProgramParser for TokenMetadataParser {
                 }
             }
             Key::EditionMarker => {
+                #[allow(deprecated)]
                 let account: EditionMarker = try_from_slice_unchecked(account_data)?;
 
                 TokenMetadataAccountState {
@@ -119,6 +125,7 @@ impl ProgramParser for TokenMetadataParser {
                 }
             }
             Key::CollectionAuthorityRecord => {
+                #[allow(deprecated)]
                 let account: CollectionAuthorityRecord = try_from_slice_unchecked(account_data)?;
 
                 TokenMetadataAccountState {

docker-compose.yaml

@@ -98,7 +98,7 @@ services:
     volumes:
       - ./db-data/:/var/lib/postgresql/data/:rw
   solana:
-    image: ghcr.io/metaplex-foundation/plerkle-test-validator:v1.6.0-1.69.0-v1.16.6
+    image: ghcr.io/metaplex-foundation/plerkle-test-validator:v1.9.0-1.75.0-v1.18.11
     volumes:
       - ./programs:/so/:ro
       - ./ledger:/config:rw

integration_tests/tests/integration_tests/cnft_tests.rs

@@ -348,3 +348,27 @@ async fn test_mint_verify_collection() {
 
     run_get_asset_scenario_test(&setup, asset_id, seeds, Order::AllPermutations).await;
 }
+
+#[tokio::test]
+#[serial]
+#[named]
+async fn test_mint_transfer_mpl_programs() {
+    let name = trim_test_name(function_name!());
+    let setup = TestSetup::new_with_options(
+        name.clone(),
+        TestSetupOptions {
+            network: Some(Network::Devnet),
+        },
+    )
+    .await;
+
+    let asset_id = "ZzTjJVwo66cRyBB5zNWNhUWDdPB6TqzyXDcwjUnpSJC";
+
+    let seeds: Vec<SeedEvent> = seed_txns([
+        "3iJ6XzhUXxGQYEEUnfkbZGdrkgS2o9vXUpsXALet3Co6sFQ2h7J21J4dTgSka8qoKiUFUzrXZFHfkqss1VFivnAG",
+        "4gV14HQBm8GCXjSTHEXjrhUNGmsBiyNdWY9hhCapH9cshmqbPKxn2kUU1XbajZ9j1Pxng95onzR6dx5bYqxQRh2a",
+        "T571TWE76frw6mWxYoHDrTdxYq7hJSyCtVEG4qmemPPtsc1CCKdknn9rTMAVcdeukLfwB1G97LZLH8eHLvuByoA",
+    ]);
+
+    run_get_asset_scenario_test(&setup, asset_id, seeds, Order::AllPermutations).await;
+}

integration_tests/tests/integration_tests/snapshots/integration_tests__cnft_tests__mint_transfer_mpl_programs.snap

@@ -0,0 +1,62 @@
+---
+source: integration_tests/tests/integration_tests/cnft_tests.rs
+assertion_line: 36
+expression: response
+---
+{
+  "interface": "V1_NFT",
+  "id": "ZzTjJVwo66cRyBB5zNWNhUWDdPB6TqzyXDcwjUnpSJC",
+  "content": {
+    "$schema": "https://schema.metaplex.com/nft1.0.json",
+    "json_uri": "https://example.com/my-nft.json",
+    "files": [],
+    "metadata": {
+      "name": "My NFT",
+      "symbol": "",
+      "token_standard": "NonFungible"
+    },
+    "links": {}
+  },
+  "authorities": [
+    {
+      "address": "6aGkAY47sgJopPFYYsQaTrsQvV2gAqZzZgyVVT12VzwF",
+      "scopes": [
+        "full"
+      ]
+    }
+  ],
+  "compression": {
+    "eligible": false,
+    "compressed": true,
+    "data_hash": "HB6sKWxroCdwkChjxckW3CF3fWupZHhPEua62GF46Ljs",
+    "creator_hash": "EKDHSGbrGztomDfuiV4iqiZ6LschDJPsFiXjZ83f92Md",
+    "asset_hash": "GrLckNqQqJK6qKuNP6vhhhnfHCTXpoBXVjDg7JhJ4wF1",
+    "tree": "DBMSBx9wYU5WTbYYq9shqK7pffR1EhbzjiHAGd7vAp7Z",
+    "seq": 2,
+    "leaf_id": 0
+  },
+  "grouping": [],
+  "royalty": {
+    "royalty_model": "creators",
+    "target": null,
+    "percent": 0.05,
+    "basis_points": 500,
+    "primary_sale_happened": false,
+    "locked": false
+  },
+  "creators": [],
+  "ownership": {
+    "frozen": false,
+    "delegated": false,
+    "delegate": null,
+    "ownership_model": "single",
+    "owner": "CrPXWqKpoHUDKSPyr6S4hy2wjFC1Nm8NmKVow4zYRDXf"
+  },
+  "supply": {
+    "print_max_supply": 0,
+    "print_current_supply": 0,
+    "edition_nonce": null
+  },
+  "mutable": true,
+  "burnt": false
+}