Add code to index Bubblegum Update Metadata

das_api/Cargo.toml

@@ -33,9 +33,9 @@ schemars = "0.8.6"
 schemars_derive = "0.8.6"
 open-rpc-derive = { version = "0.0.4"}
 open-rpc-schema = { version = "0.0.4"}
-blockbuster = { git = "https://github.com/metaplex-foundation/blockbuster.git", rev = "e5d96d62969af42ce1c7f10cca88dc1c4f643598" }
+blockbuster = { git = "https://github.com/metaplex-foundation/blockbuster.git", rev = "f5f7a3dc2c95021e61211401dbbb447bef3514a9" }
 anchor-lang = "0.28.0"
 mpl-token-metadata = { version = "=2.0.0-beta.1",  features = ["serde-feature"] }
 mpl-candy-machine-core = { version = "2.0.1", features = ["no-entrypoint"] }
-mpl-bubblegum = "1.0.1-beta.2"
+mpl-bubblegum = "1.0.1-beta.3"
 mpl-candy-guard = { version = "2.0.0", features = ["no-entrypoint"] }

digital_asset_types/Cargo.toml

@@ -18,7 +18,7 @@ solana-sdk = "~1.16.16"
 num-traits = "0.2.15"
 num-derive = "0.3.3"
 thiserror = "1.0.31"
-blockbuster = { git = "https://github.com/metaplex-foundation/blockbuster.git", rev = "e5d96d62969af42ce1c7f10cca88dc1c4f643598" }
+blockbuster = { git = "https://github.com/metaplex-foundation/blockbuster.git", rev = "f5f7a3dc2c95021e61211401dbbb447bef3514a9" }
 jsonpath_lib = "0.3.0"
 mime_guess = "2.0.4"
 url = "2.3.1"

docker-compose.yaml

@@ -89,7 +89,7 @@ services:
     volumes:
       - ./db-data/:/var/lib/postgresql/data/:rw
   solana:
-    image: ghcr.io/metaplex-foundation/plerkle-test-validator:v1.5.1-1.64.0-v1.14.15
+    image: ghcr.io/metaplex-foundation/plerkle-test-validator:v1.6.0-1.69.0-v1.16.6
     volumes:
     - ./programs:/so/:ro
     - ./ledger:/config:rw

nft_ingester/Cargo.toml

@@ -29,13 +29,13 @@ flatbuffers = "23.1.21"
 lazy_static = "1.4.0"
 regex = "1.5.5"
 digital_asset_types = { path = "../digital_asset_types", features = ["json_types", "sql_types"] }
-mpl-bubblegum = "1.0.1-beta.2"
+mpl-bubblegum = "1.0.1-beta.3"
 spl-account-compression = { version = "0.2.0", features = ["no-entrypoint"] }
 spl-concurrent-merkle-tree = "0.2.0"
 uuid = "1.0.0"
 async-trait = "0.1.53"
 num-traits = "0.2.15"
-blockbuster = { git = "https://github.com/metaplex-foundation/blockbuster.git", rev = "e5d96d62969af42ce1c7f10cca88dc1c4f643598" }
+blockbuster = { git = "https://github.com/metaplex-foundation/blockbuster.git", rev = "f5f7a3dc2c95021e61211401dbbb447bef3514a9" }
 figment = { version = "0.10.6", features = ["env", "toml", "yaml"] }
 cadence = "0.29.0"
 cadence-macros = "0.29.0"

nft_ingester/src/program_transformers/bubblegum/collection_verification.rs

@@ -26,7 +26,7 @@ where
         let (collection, verify) = match payload {
             Payload::CollectionVerification {
                 collection, verify, ..
-            } => (collection.clone(), verify.clone()),
+            } => (collection, verify),
             _ => {
                 return Err(IngesterError::ParsingError(
                     "Ix not parsed correctly".to_string(),
@@ -64,8 +64,8 @@ where
             txn,
             id_bytes.to_vec(),
             Some(Collection {
-                key: collection.clone(),
-                verified: verify,
+                key: *collection,
+                verified: *verify,
             }),
             bundle.slot as i64,
             seq as i64,

nft_ingester/src/program_transformers/bubblegum/db.rs

@@ -1,6 +1,7 @@
 use crate::error::IngesterError;
 use digital_asset_types::dao::{
-    asset, asset_creators, asset_grouping, backfill_items, cl_audits, cl_items,
+    asset, asset_creators, asset_data, asset_grouping, backfill_items, cl_audits, cl_items,
+    sea_orm_active_enums::{ChainMutability, Mutability},
 };
 use log::{debug, info};
 use mpl_bubblegum::types::Collection;
@@ -9,8 +10,6 @@ use sea_orm::{
 };
 use spl_account_compression::events::ChangeLogEventV1;
 
-use std::convert::From;
-
 pub async fn save_changelog_event<'c, T>(
     change_log_event: &ChangeLogEventV1,
     slot: u64,
@@ -68,7 +67,7 @@ where
             ..Default::default()
         };
 
-        let mut audit_item: Option<cl_audits::ActiveModel> = if (cl_audits) {
+        let audit_item: Option<cl_audits::ActiveModel> = if cl_audits {
             let mut ai: cl_audits::ActiveModel = item.clone().into();
             ai.tx = Set(txn_id.to_string());
             Some(ai)
@@ -135,6 +134,7 @@ where
     //TODO -> set maximum size of path and break into multiple statements
 }
 
+#[allow(clippy::too_many_arguments)]
 pub async fn upsert_asset_with_leaf_info<T>(
     txn: &T,
     id: Vec<u8>,
@@ -441,3 +441,105 @@ where
 
     Ok(())
 }
+
+pub async fn upsert_asset_data<T>(
+    txn: &T,
+    id: Vec<u8>,
+    chain_data_mutability: ChainMutability,
+    chain_data: JsonValue,
+    metadata_url: String,
+    metadata_mutability: Mutability,
+    metadata: JsonValue,
+    slot_updated: i64,
+    reindex: Option<bool>,
+    raw_name: Vec<u8>,
+    raw_symbol: Vec<u8>,
+    seq: i64,
+) -> Result<(), IngesterError>
+where
+    T: ConnectionTrait + TransactionTrait,
+{
+    let model = asset_data::ActiveModel {
+        id: Set(id),
+        chain_data_mutability: Set(chain_data_mutability),
+        chain_data: Set(chain_data),
+        metadata_url: Set(metadata_url),
+        metadata_mutability: Set(metadata_mutability),
+        metadata: Set(metadata),
+        slot_updated: Set(slot_updated),
+        reindex: Set(reindex),
+        raw_name: Set(raw_name),
+        raw_symbol: Set(raw_symbol),
+        //seq: Set(seq),
+    };
+
+    let mut query = asset_data::Entity::insert(model)
+        .on_conflict(
+            OnConflict::columns([asset_data::Column::Id])
+                .update_columns([
+                    asset_data::Column::ChainDataMutability,
+                    asset_data::Column::ChainData,
+                    asset_data::Column::MetadataUrl,
+                    asset_data::Column::MetadataMutability,
+                    //TODO DEAL WITH THIS
+                    //asset_data::Column::Metadata,
+                    asset_data::Column::SlotUpdated,
+                    asset_data::Column::Reindex,
+                    asset_data::Column::RawName,
+                    asset_data::Column::RawSymbol,
+                    //asset_data::Column::Seq,
+                ])
+                .to_owned(),
+        )
+        .build(DbBackend::Postgres);
+    query.sql = format!(
+    // TODO DEAL WITH THIS
+    "{} WHERE (excluded.slot_updated > asset_data.slot_updated) AND (excluded.seq >= asset_data.seq OR asset_data.seq IS NULL)",
+    query.sql
+);
+    txn.execute(query)
+        .await
+        .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?;
+
+    Ok(())
+}
+
+pub async fn upsert_asset_with_royalty_amount<T>(
+    txn: &T,
+    id: Vec<u8>,
+    royalty_amount: i32,
+    seq: i64,
+) -> Result<(), IngesterError>
+where
+    T: ConnectionTrait + TransactionTrait,
+{
+    let model = asset::ActiveModel {
+        id: Set(id),
+        royalty_amount: Set(royalty_amount),
+        //royalty_amount_seq: Set(Some(seq)),
+        ..Default::default()
+    };
+
+    let mut query = asset::Entity::insert(model)
+        .on_conflict(
+            OnConflict::column(asset::Column::Id)
+                .update_columns([
+                    asset::Column::RoyaltyAmount,
+                    //asset::Column::RoyaltyAmountSeq,
+                ])
+                .to_owned(),
+        )
+        .build(DbBackend::Postgres);
+
+    query.sql = format!(
+        // TODO DEAL WITH THIS
+        "{} WHERE (NOT asset.was_decompressed) AND (excluded.royalty_amount_seq >= asset.royalty_amount_seq OR royalty_amount_seq.seq IS NULL)",
+        query.sql
+    );
+
+    txn.execute(query)
+        .await
+        .map_err(|db_err| IngesterError::StorageWriteError(db_err.to_string()))?;
+
+    Ok(())
+}

nft_ingester/src/program_transformers/bubblegum/mint_v1.rs

@@ -1,8 +1,9 @@
 use crate::{
     error::IngesterError,
     program_transformers::bubblegum::{
-        save_changelog_event, upsert_asset_with_compression_info, upsert_asset_with_leaf_info,
-        upsert_asset_with_owner_and_delegate_info, upsert_asset_with_seq, upsert_collection_info,
+        save_changelog_event, upsert_asset_data, upsert_asset_with_compression_info,
+        upsert_asset_with_leaf_info, upsert_asset_with_owner_and_delegate_info,
+        upsert_asset_with_royalty_amount, upsert_asset_with_seq, upsert_collection_info,
     },
     tasks::{DownloadMetadata, IntoTaskData, TaskData},
 };
@@ -17,7 +18,7 @@ use blockbuster::{
 use chrono::Utc;
 use digital_asset_types::{
     dao::{
-        asset, asset_authority, asset_creators, asset_data, asset_v1_account_attachments,
+        asset, asset_authority, asset_creators, asset_v1_account_attachments,
         sea_orm_active_enums::{ChainMutability, Mutability, OwnerType, RoyaltyTargetType},
     },
     json::ChainDataV1,
@@ -62,7 +63,14 @@ where
                 let (edition_attachment_address, _) = find_master_edition_account(&id);
                 let id_bytes = id.to_bytes();
                 let slot_i = bundle.slot as i64;
+
                 let uri = metadata.uri.replace('\0', "");
+                if uri.is_empty() {
+                    return Err(IngesterError::DeserializationError(
+                        "URI is empty".to_string(),
+                    ));
+                }
+
                 let name = metadata.name.clone().into_bytes();
                 let symbol = metadata.symbol.clone().into_bytes();
                 let mut chain_data = ChainDataV1 {
@@ -84,46 +92,23 @@ where
                     true => ChainMutability::Mutable,
                     false => ChainMutability::Immutable,
                 };
-                if uri.is_empty() {
-                    return Err(IngesterError::DeserializationError(
-                        "URI is empty".to_string(),
-                    ));
-                }
-                let data = asset_data::ActiveModel {
-                    id: Set(id_bytes.to_vec()),
-                    chain_data_mutability: Set(chain_mutability),
-                    chain_data: Set(chain_data_json),
-                    metadata_url: Set(uri),
-                    metadata: Set(JsonValue::String("processing".to_string())),
-                    metadata_mutability: Set(Mutability::Mutable),
-                    slot_updated: Set(slot_i),
-                    reindex: Set(Some(true)),
-                    raw_name: Set(name.to_vec()),
-                    raw_symbol: Set(symbol.to_vec()),
-                    ..Default::default()
-                };
 
-                let mut query = asset_data::Entity::insert(data)
-                    .on_conflict(
-                        OnConflict::columns([asset_data::Column::Id])
-                            .update_columns([
-                                asset_data::Column::ChainDataMutability,
-                                asset_data::Column::ChainData,
-                                asset_data::Column::MetadataUrl,
-                                asset_data::Column::MetadataMutability,
-                                asset_data::Column::SlotUpdated,
-                                asset_data::Column::Reindex,
-                            ])
-                            .to_owned(),
-                    )
-                    .build(DbBackend::Postgres);
-                query.sql = format!(
-                    "{} WHERE excluded.slot_updated > asset_data.slot_updated",
-                    query.sql
-                );
-                txn.execute(query)
-                    .await
-                    .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?;
+                upsert_asset_data(
+                    txn,
+                    id_bytes.to_vec(),
+                    chain_mutability,
+                    chain_data_json,
+                    uri,
+                    Mutability::Mutable,
+                    JsonValue::String("processing".to_string()),
+                    slot_i,
+                    Some(true),
+                    name.to_vec(),
+                    symbol.to_vec(),
+                    seq as i64,
+                )
+                .await?;
+
                 // Insert into `asset` table.
                 let delegate = if owner == delegate || delegate.to_bytes() == [0; 32] {
                     None
@@ -143,7 +128,6 @@ where
                     nonce: Set(Some(nonce as i64)),
                     royalty_target_type: Set(RoyaltyTargetType::Creators),
                     royalty_target: Set(None),
-                    royalty_amount: Set(metadata.seller_fee_basis_points as i32), //basis points
                     asset_data: Set(Some(id_bytes.to_vec())),
                     slot_updated: Set(Some(slot_i)),
                     ..Default::default()
@@ -160,7 +144,6 @@ where
                                 asset::Column::SpecificationAssetClass,
                                 asset::Column::RoyaltyTargetType,
                                 asset::Column::RoyaltyTarget,
-                                asset::Column::RoyaltyAmount,
                                 asset::Column::AssetData,
                             ])
                             .to_owned(),
@@ -176,6 +159,14 @@ where
                     .await
                     .map_err(|db_err| IngesterError::AssetIndexError(db_err.to_string()))?;
 
+                upsert_asset_with_royalty_amount(
+                    txn,
+                    id_bytes.to_vec(),
+                    metadata.seller_fee_basis_points as i32,
+                    seq as i64,
+                )
+                .await?;
+
                 // Partial update of asset table with just compression info elements.
                 upsert_asset_with_compression_info(
                     txn,
@@ -248,6 +239,7 @@ where
                         if creators_set.contains(&c.address) {
                             continue;
                         }
+
                         db_creator_infos.push(asset_creators::ActiveModel {
                             asset_id: Set(id_bytes.to_vec()),
                             creator: Set(c.address.to_bytes().to_vec()),

nft_ingester/src/program_transformers/bubblegum/mod.rs

@@ -17,6 +17,7 @@ mod delegate;
 mod mint_v1;
 mod redeem;
 mod transfer;
+mod update_metadata;
 
 pub use db::*;
 
@@ -53,7 +54,8 @@ where
         InstructionName::VerifyCollection => "VerifyCollection",
         InstructionName::UnverifyCollection => "UnverifyCollection",
         InstructionName::SetAndVerifyCollection => "SetAndVerifyCollection",
-        InstructionName::SetDecompressibleState | InstructionName::UpdateMetadata => todo!(),
+        InstructionName::SetDecompressibleState => "SetDecompressibleState",
+        InstructionName::UpdateMetadata => "UpdateMetadata",
     };
     info!("BGUM instruction txn={:?}: {:?}", ix_str, bundle.txn_id);
 
@@ -92,6 +94,13 @@ where
         | InstructionName::SetAndVerifyCollection => {
             collection_verification::process(parsing_result, bundle, txn, cl_audits).await?;
         }
+        InstructionName::SetDecompressibleState => (), // Nothing to index.
+        InstructionName::UpdateMetadata => {
+            let task =
+                update_metadata::update_metadata(parsing_result, bundle, txn, cl_audits).await?;
+
+            task_manager.send(task)?;
+        }
         _ => debug!("Bubblegum: Not Implemented Instruction"),
     }
     Ok(())