From 213dac51bfbd8c005799cbf1bbfe204bac7b5aa2 Mon Sep 17 00:00:00 2001 From: Michael Danenberg <56533526+danenbm@users.noreply.github.com> Date: Fri, 20 Oct 2023 15:21:11 -0700 Subject: [PATCH] Use new sequence numbers for Bubblegum Update Metadata --- digital_asset_types/tests/common.rs | 12 +++++--- digital_asset_types/tests/json_parsing.rs | 6 ++-- .../src/program_transformers/bubblegum/db.rs | 30 +++++++++---------- .../program_transformers/bubblegum/mint_v1.rs | 8 ++--- .../bubblegum/update_metadata.rs | 18 ++++++----- .../token_metadata/v1_asset.rs | 17 +++++------ nft_ingester/src/tasks/common/mod.rs | 18 +++++------ 7 files changed, 58 insertions(+), 51 deletions(-) diff --git a/digital_asset_types/tests/common.rs b/digital_asset_types/tests/common.rs index bbe3cf509..ef551f251 100644 --- a/digital_asset_types/tests/common.rs +++ b/digital_asset_types/tests/common.rs @@ -83,8 +83,10 @@ pub fn create_asset_data( metadata: JsonValue::String("processing".to_string()), slot_updated: 0, reindex: None, - raw_name: metadata.name.into_bytes().to_vec().clone(), - raw_symbol: metadata.symbol.into_bytes().to_vec().clone(), + raw_name: Some(metadata.name.into_bytes().to_vec().clone()), + raw_symbol: Some(metadata.symbol.into_bytes().to_vec().clone()), + base_info_seq: Some(0), + download_metadata_seq: Some(0), }, ) } @@ -157,6 +159,7 @@ pub fn create_asset( owner_delegate_seq: Some(0), was_decompressed: false, leaf_seq: Some(0), + royalty_amount_seq: Some(0), }, ) } @@ -182,9 +185,10 @@ pub fn create_asset_creator( creator, share, verified, - seq: Some(0), + verified_seq: Some(0), slot_updated: Some(0), position: 0, + base_info_seq: Some(0), }, ) } @@ -231,7 +235,7 @@ pub fn create_asset_grouping( id: row_num, group_key: "collection".to_string(), slot_updated: Some(0), - verified: Some(false), + verified: false, group_info_seq: Some(0), }, ) diff --git a/digital_asset_types/tests/json_parsing.rs b/digital_asset_types/tests/json_parsing.rs index 765f14bc6..c10ca12e3 100644 --- a/digital_asset_types/tests/json_parsing.rs +++ b/digital_asset_types/tests/json_parsing.rs @@ -34,8 +34,10 @@ pub async fn parse_onchain_json(json: serde_json::Value) -> Content { metadata: json, slot_updated: 0, reindex: None, - raw_name: String::from("Handalf").into_bytes().to_vec(), - raw_symbol: String::from("").into_bytes().to_vec(), + raw_name: Some(String::from("Handalf").into_bytes().to_vec()), + raw_symbol: Some(String::from("").into_bytes().to_vec()), + base_info_seq: Some(0), + download_metadata_seq: Some(0), }; v1_content_from_json(&asset_data).unwrap() diff --git a/nft_ingester/src/program_transformers/bubblegum/db.rs b/nft_ingester/src/program_transformers/bubblegum/db.rs index 4e1391cf7..44e41f240 100644 --- a/nft_ingester/src/program_transformers/bubblegum/db.rs +++ b/nft_ingester/src/program_transformers/bubblegum/db.rs @@ -354,7 +354,7 @@ where asset_id: Set(asset_id), creator: Set(creator), verified: Set(verified), - seq: Set(Some(seq)), + verified_seq: Set(Some(seq)), ..Default::default() }; @@ -366,7 +366,7 @@ where ]) .update_columns([ asset_creators::Column::Verified, - asset_creators::Column::Seq, + asset_creators::Column::VerifiedSeq, ]) .to_owned(), ) @@ -403,7 +403,7 @@ where asset_id: Set(asset_id), group_key: Set("collection".to_string()), group_value: Set(group_value), - verified: Set(Some(verified)), + verified: Set(verified), slot_updated: Set(Some(slot_updated)), group_info_seq: Set(Some(seq)), ..Default::default() @@ -450,7 +450,7 @@ pub async fn upsert_asset_data( reindex: Option, raw_name: Vec, raw_symbol: Vec, - _seq: i64, + seq: i64, ) -> Result<(), IngesterError> where T: ConnectionTrait + TransactionTrait, @@ -464,9 +464,10 @@ where metadata: Set(metadata), slot_updated: Set(slot_updated), reindex: Set(reindex), - raw_name: Set(raw_name), - raw_symbol: Set(raw_symbol), - //base_info_seq: Set(seq), + raw_name: Set(Some(raw_name)), + raw_symbol: Set(Some(raw_symbol)), + base_info_seq: Set(Some(seq)), + ..Default::default() }; let mut query = asset_data::Entity::insert(model) @@ -477,15 +478,14 @@ where asset_data::Column::ChainData, asset_data::Column::MetadataUrl, asset_data::Column::MetadataMutability, - // Don't update Metadata if it already exists. Even if we are doing - // and update_metadata and there's a new URI, the new background task - // will overwrite it. - //asset_data::Column::Metadata, + // Don't update asset_data::Column::Metadata if it already exists. Even if we + // are indexing `update_metadata`` and there's a new URI, the new background + // task will overwrite it. asset_data::Column::SlotUpdated, asset_data::Column::Reindex, asset_data::Column::RawName, asset_data::Column::RawSymbol, - //asset_data::Column::BaseInfoSeq, + asset_data::Column::BaseInfoSeq, ]) .to_owned(), ) @@ -505,7 +505,7 @@ pub async fn upsert_asset_with_royalty_amount( txn: &T, id: Vec, royalty_amount: i32, - _seq: i64, + seq: i64, ) -> Result<(), IngesterError> where T: ConnectionTrait + TransactionTrait, @@ -513,7 +513,7 @@ where let model = asset::ActiveModel { id: Set(id.clone()), royalty_amount: Set(royalty_amount), - //royalty_amount_seq: Set(Some(seq)), + royalty_amount_seq: Set(Some(seq)), ..Default::default() }; @@ -522,7 +522,7 @@ where OnConflict::column(asset::Column::Id) .update_columns([ asset::Column::RoyaltyAmount, - //asset::Column::RoyaltyAmountSeq, + asset::Column::RoyaltyAmountSeq, ]) .to_owned(), ) diff --git a/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs b/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs index 3b3603891..7a5be0033 100644 --- a/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs +++ b/nft_ingester/src/program_transformers/bubblegum/mint_v1.rs @@ -241,7 +241,7 @@ where position: Set(i as i16), share: Set(c.share as i32), slot_updated: Set(Some(slot_i)), - //base_info_seq: Set(Some(seq as i64)) + base_info_seq: Set(Some(seq as i64)), ..Default::default() }); @@ -249,7 +249,7 @@ where asset_id: Set(id_bytes.to_vec()), creator: Set(c.address.to_bytes().to_vec()), verified: Set(c.verified), - //verified_seq: Set(Some(seq as i64)), + verified_seq: Set(Some(seq as i64)), ..Default::default() }); @@ -267,7 +267,7 @@ where asset_creators::Column::Position, asset_creators::Column::Share, asset_creators::Column::SlotUpdated, - //asset_creators::Column::BaseInfoSeq, + asset_creators::Column::BaseInfoSeq, ]) .to_owned(), ) @@ -289,7 +289,7 @@ where ]) .update_columns([ asset_creators::Column::Verified, - //asset_creators::Column::VerifiedSeq, + asset_creators::Column::VerifiedSeq, ]) .to_owned(), ) diff --git a/nft_ingester/src/program_transformers/bubblegum/update_metadata.rs b/nft_ingester/src/program_transformers/bubblegum/update_metadata.rs index 204a4154c..6c13eb097 100644 --- a/nft_ingester/src/program_transformers/bubblegum/update_metadata.rs +++ b/nft_ingester/src/program_transformers/bubblegum/update_metadata.rs @@ -189,6 +189,7 @@ where position: Set(i as i16), share: Set(c.share as i32), slot_updated: Set(Some(slot_i)), + base_info_seq: Set(Some(seq as i64)), ..Default::default() }); @@ -196,7 +197,7 @@ where asset_id: Set(id_bytes.to_vec()), creator: Set(c.address.to_bytes().to_vec()), verified: Set(c.verified), - seq: Set(Some(seq as i64)), + verified_seq: Set(Some(seq as i64)), ..Default::default() }); @@ -215,11 +216,12 @@ where .filter( Condition::all() .add(asset_creators::Column::AssetId.eq(id_bytes.to_vec())) - .add(asset_creators::Column::Creator.is_in(db_creators_to_remove)), // .add( - // Condition::any() - // .add(asset_creators::Column::VerifiedSeq.lt(seq as i64)) - // .add(asset_creators::Column::VerifiedSeq.is_null()), - // ), + .add(asset_creators::Column::Creator.is_in(db_creators_to_remove)) + .add( + Condition::any() + .add(asset_creators::Column::VerifiedSeq.lt(seq as i64)) + .add(asset_creators::Column::VerifiedSeq.is_null()), + ), ) .exec(txn) .await?; @@ -237,7 +239,7 @@ where asset_creators::Column::Position, asset_creators::Column::Share, asset_creators::Column::SlotUpdated, - //asset_creators::Column::BaseInfoSeq, + asset_creators::Column::BaseInfoSeq, ]) .to_owned(), ) @@ -260,7 +262,7 @@ where ]) .update_columns([ asset_creators::Column::Verified, - //asset_creators::Column::VerifiedSeq, + asset_creators::Column::VerifiedSeq, ]) .to_owned(), ) diff --git a/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs b/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs index 1bfedac84..7950d8e45 100644 --- a/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs +++ b/nft_ingester/src/program_transformers/token_metadata/v1_asset.rs @@ -147,8 +147,10 @@ pub async fn save_v1_asset( slot_updated: Set(slot_i), reindex: Set(Some(true)), id: Set(id.to_vec()), - raw_name: Set(name.to_vec()), - raw_symbol: Set(symbol.to_vec()), + raw_name: Set(Some(name.to_vec())), + raw_symbol: Set(Some(symbol.to_vec())), + download_metadata_seq: Set(Some(0)), + ..Default::default() }; let txn = conn.begin().await?; let mut query = asset_data::Entity::insert(asset_data_model) @@ -161,7 +163,8 @@ pub async fn save_v1_asset( asset_data::Column::MetadataMutability, asset_data::Column::SlotUpdated, asset_data::Column::Reindex, - //TODO RAW NAME + asset_data::Column::RawName, + asset_data::Column::RawSymbol, ]) .to_owned(), ) @@ -195,9 +198,6 @@ pub async fn save_v1_asset( asset_data: Set(Some(id.to_vec())), slot_updated: Set(Some(slot_i)), burnt: Set(false), - //data_hash, - //creator_hash, - //leaf_seq, ..Default::default() }; let mut query = asset::Entity::insert(model) @@ -282,7 +282,7 @@ pub async fn save_v1_asset( asset_id: Set(id.to_vec()), group_key: Set("collection".to_string()), group_value: Set(Some(c.key.to_string())), - verified: Set(Some(c.verified)), + verified: Set(c.verified), seq: Set(None), slot_updated: Set(Some(slot_i)), ..Default::default() @@ -333,7 +333,6 @@ pub async fn save_v1_asset( creator: Set(c.address.to_bytes().to_vec()), share: Set(c.share as i32), verified: Set(c.verified), - seq: Set(Some(0)), slot_updated: Set(Some(slot_i)), position: Set(i as i16), ..Default::default() @@ -360,7 +359,7 @@ pub async fn save_v1_asset( asset_creators::Column::Creator, asset_creators::Column::Share, asset_creators::Column::Verified, - asset_creators::Column::Seq, + asset_creators::Column::VerifiedSeq, asset_creators::Column::SlotUpdated, ]) .to_owned(), diff --git a/nft_ingester/src/tasks/common/mod.rs b/nft_ingester/src/tasks/common/mod.rs index 2cf1cca18..19f1a12e9 100644 --- a/nft_ingester/src/tasks/common/mod.rs +++ b/nft_ingester/src/tasks/common/mod.rs @@ -111,22 +111,22 @@ impl BgTask for DownloadMetadataTask { id: Unchanged(download_metadata.asset_data_id.clone()), metadata: Set(body), reindex: Set(Some(false)), - //download_metadata_seq: Set(Some(download_metadata.seq)), + download_metadata_seq: Set(Some(download_metadata.seq)), ..Default::default() }; debug!( "download metadata for {:?}", bs58::encode(download_metadata.asset_data_id.clone()).into_string() ); - let query = asset_data::Entity::update(model) + let mut query = asset_data::Entity::update(model) .filter(asset_data::Column::Id.eq(download_metadata.asset_data_id.clone())); - // if download_metadata.seq != 0 { - // query.filter( - // Condition::any() - // .add(asset_data::Column::DownloadMetadataSeq.lt(download_metadata.seq)) - // .add(asset_data::Column::DownloadMetadataSeq.is_null()), - // ); - // } + if download_metadata.seq != 0 { + query = query.filter( + Condition::any() + .add(asset_data::Column::DownloadMetadataSeq.lt(download_metadata.seq)) + .add(asset_data::Column::DownloadMetadataSeq.is_null()), + ); + } query.exec(db).await.map(|_| ()).map_err(|db| { IngesterError::TaskManagerError(format!( "Database error with {}, error: {}",