From 700cd9faa0d1fd83e0a6f59c197b426c0732bf18 Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Mon, 9 Dec 2024 15:39:22 -0500 Subject: [PATCH] Remove Selects From Standard NFT Indexing (#183) * Use Example Configs (#180) Co-authored-by: Ahzam Akhtar * Remove select insert from indexing mint * Drop master edition lookup save v1 asset sets the specification and mint the supply * Remove lookup records while ingesting asset * Only set owner_type based on the mint for standard nfts --------- Co-authored-by: Ahzam Akhtar --- grpc-ingest/.gitignore | 3 + grpc-ingest/README.md | 10 +- ...edis.yml => config-grpc2redis.example.yml} | 0 ...gester.yml => config-ingester.example.yml} | 0 ...monitor.yml => config-monitor.example.yml} | 0 program_transformers/src/asset_upserts.rs | 13 +- .../src/mpl_core_program/v1_asset.rs | 117 ++++++++-------- program_transformers/src/token/mod.rs | 86 +++++------- .../src/token_metadata/master_edition.rs | 24 +--- .../src/token_metadata/v1_asset.rs | 127 +----------------- 10 files changed, 130 insertions(+), 250 deletions(-) create mode 100644 grpc-ingest/.gitignore rename grpc-ingest/{config-grpc2redis.yml => config-grpc2redis.example.yml} (100%) rename grpc-ingest/{config-ingester.yml => config-ingester.example.yml} (100%) rename grpc-ingest/{config-monitor.yml => config-monitor.example.yml} (100%) diff --git a/grpc-ingest/.gitignore b/grpc-ingest/.gitignore new file mode 100644 index 000000000..191f3badd --- /dev/null +++ b/grpc-ingest/.gitignore @@ -0,0 +1,3 @@ +config-grpc2redis.yaml +config-ingester.yaml +config-monitor.yaml diff --git a/grpc-ingest/README.md b/grpc-ingest/README.md index 011b2a062..1f3a46393 100644 --- a/grpc-ingest/README.md +++ b/grpc-ingest/README.md @@ -14,7 +14,15 @@ docker compose up db redis prometheus INIT_FILE_PATH=./init.sql sea migrate up --database-url=postgres://solana:solana@localhost:5432/solana ``` -### Config for grpc2redis [./config-grpc2redis.yml](./config-grpc2redis.yml) +### Configs + +Example config files are available at +- [./config-grpc2redis.example.yml](./config-grpc2redis.example.yml) +- [./config-ingester.example.yml](./config-ingester.example.yml) +- [./config-monitor.example.yml](./config-monitor.example.yml) + +Copy these files and modify them as needed to setup the project. + ### Run grpc2redis service diff --git a/grpc-ingest/config-grpc2redis.yml b/grpc-ingest/config-grpc2redis.example.yml similarity index 100% rename from grpc-ingest/config-grpc2redis.yml rename to grpc-ingest/config-grpc2redis.example.yml diff --git a/grpc-ingest/config-ingester.yml b/grpc-ingest/config-ingester.example.yml similarity index 100% rename from grpc-ingest/config-ingester.yml rename to grpc-ingest/config-ingester.example.yml diff --git a/grpc-ingest/config-monitor.yml b/grpc-ingest/config-monitor.example.yml similarity index 100% rename from grpc-ingest/config-monitor.yml rename to grpc-ingest/config-monitor.example.yml diff --git a/program_transformers/src/asset_upserts.rs b/program_transformers/src/asset_upserts.rs index 5561ff5fd..ee0b62713 100644 --- a/program_transformers/src/asset_upserts.rs +++ b/program_transformers/src/asset_upserts.rs @@ -47,7 +47,7 @@ pub async fn upsert_assets_token_account_columns= asset.slot_updated_token_account OR asset.slot_updated_token_account IS NULL", + "{} WHERE (excluded.slot_updated_token_account >= asset.slot_updated_token_account OR asset.slot_updated_token_account IS NULL) AND asset.owner_type = 'single'", query.sql); txn_or_conn.execute(query).await?; Ok(()) @@ -64,11 +64,18 @@ pub async fn upsert_assets_mint_account_columns Result<(), DbErr> { + let owner_type = if columns.supply == Decimal::from(1) { + OwnerType::Single + } else { + OwnerType::Token + }; + let active_model = asset::ActiveModel { id: Set(columns.mint), supply: Set(columns.supply), supply_mint: Set(columns.supply_mint), slot_updated_mint_account: Set(Some(columns.slot_updated_mint_account as i64)), + owner_type: Set(owner_type), ..Default::default() }; let mut query = asset::Entity::insert(active_model) @@ -78,6 +85,7 @@ pub async fn upsert_assets_mint_account_columns, - pub owner_type: OwnerType, pub specification_asset_class: Option, pub royalty_amount: i32, pub asset_data: Option>, @@ -112,7 +119,6 @@ pub async fn upsert_assets_metadata_account_columns Result<(), DbErr> { let active_model = asset::ActiveModel { id: Set(columns.mint), - owner_type: Set(columns.owner_type), specification_version: Set(Some(SpecificationVersions::V1)), specification_asset_class: Set(columns.specification_asset_class), tree_id: Set(None), @@ -142,7 +148,6 @@ pub async fn upsert_assets_metadata_account_columns( None }; - upsert_assets_metadata_account_columns( - AssetMetadataAccountColumns { - mint: id_vec.clone(), - owner_type: ownership_type, - specification_asset_class: Some(class), - royalty_amount: royalty_amount as i32, - asset_data: Some(id_vec.clone()), - slot_updated_metadata_account: slot, - mpl_core_plugins: Some(plugins_json), - mpl_core_unknown_plugins: unknown_plugins_json, - mpl_core_collection_num_minted: asset.num_minted.map(|val| val as i32), - mpl_core_collection_current_size: asset.current_size.map(|val| val as i32), - mpl_core_plugins_json_version: Some(1), - mpl_core_external_plugins: Some(external_plugins_json), - mpl_core_unknown_external_plugins: unknown_external_plugins_json, - }, - &txn, - ) - .await?; - - let supply = Decimal::from(1); - - // Note: these need to be separate for Token Metadata but here could be one upsert. - upsert_assets_mint_account_columns( - AssetMintAccountColumns { - mint: id_vec.clone(), - supply_mint: None, - supply, - slot_updated_mint_account: slot, - }, - &txn, - ) - .await?; - - // Get transfer delegate from `TransferDelegate` plugin if available. let transfer_delegate = asset .plugins @@ -346,7 +307,6 @@ pub async fn save_v1_asset( PluginAuthority::None => None, }); - // Get frozen status from `FreezeDelegate` plugin if available. let frozen = asset .plugins .get(&PluginType::FreezeDelegate) @@ -359,19 +319,68 @@ pub async fn save_v1_asset( }) .unwrap_or(false); - // TODO: these upserts needed to be separate for Token Metadata but here could be one upsert. - upsert_assets_token_account_columns( - AssetTokenAccountColumns { - mint: id_vec.clone(), - owner, - frozen, - // Note use transfer delegate for the existing delegate field. - delegate: transfer_delegate.clone(), - slot_updated_token_account: Some(slot_i), - }, - &txn, - ) - .await?; + let asset_model = asset::ActiveModel { + id: ActiveValue::Set(id_vec.clone()), + owner_type: ActiveValue::Set(ownership_type), + supply: ActiveValue::Set(Decimal::from(1)), + supply_mint: ActiveValue::Set(None), + slot_updated_mint_account: ActiveValue::Set(Some(slot as i64)), + specification_version: ActiveValue::Set(Some(SpecificationVersions::V1)), + specification_asset_class: ActiveValue::Set(Some(class)), + royalty_amount: ActiveValue::Set(royalty_amount as i32), + asset_data: ActiveValue::Set(Some(id_vec.clone())), + slot_updated_metadata_account: ActiveValue::Set(Some(slot as i64)), + mpl_core_plugins: ActiveValue::Set(Some(plugins_json)), + mpl_core_unknown_plugins: ActiveValue::Set(unknown_plugins_json), + mpl_core_collection_num_minted: ActiveValue::Set(asset.num_minted.map(|val| val as i32)), + mpl_core_collection_current_size: ActiveValue::Set( + asset.current_size.map(|val| val as i32), + ), + mpl_core_plugins_json_version: ActiveValue::Set(Some(1)), + mpl_core_external_plugins: ActiveValue::Set(Some(external_plugins_json)), + mpl_core_unknown_external_plugins: ActiveValue::Set(unknown_external_plugins_json), + owner: ActiveValue::Set(owner), + frozen: ActiveValue::Set(frozen), + delegate: ActiveValue::Set(transfer_delegate.clone()), + slot_updated_token_account: ActiveValue::Set(Some(slot_i)), + ..Default::default() + }; + + let mut query = asset::Entity::insert(asset_model) + .on_conflict( + OnConflict::columns([asset::Column::Id]) + .update_columns([ + asset::Column::OwnerType, + asset::Column::Supply, + asset::Column::SupplyMint, + asset::Column::SlotUpdatedMintAccount, + asset::Column::SpecificationVersion, + asset::Column::SpecificationAssetClass, + asset::Column::RoyaltyAmount, + asset::Column::AssetData, + asset::Column::SlotUpdatedMetadataAccount, + asset::Column::MplCorePlugins, + asset::Column::MplCoreUnknownPlugins, + asset::Column::MplCoreCollectionNumMinted, + asset::Column::MplCoreCollectionCurrentSize, + asset::Column::MplCorePluginsJsonVersion, + asset::Column::MplCoreExternalPlugins, + asset::Column::MplCoreUnknownExternalPlugins, + asset::Column::Owner, + asset::Column::Frozen, + asset::Column::Delegate, + asset::Column::SlotUpdatedTokenAccount, + ]) + .to_owned(), + ) + .build(DbBackend::Postgres); + + query.sql = format!( + "{} WHERE excluded.slot_updated_metadata_account >= asset.slot_updated_metadata_account OR asset.slot_updated_metadata_account IS NULL", + query.sql + ); + + txn.execute(query).await?; //----------------------- // asset_grouping table diff --git a/program_transformers/src/token/mod.rs b/program_transformers/src/token/mod.rs index d46360317..64a83d300 100644 --- a/program_transformers/src/token/mod.rs +++ b/program_transformers/src/token/mod.rs @@ -8,12 +8,10 @@ use { AccountInfo, DownloadMetadataNotifier, }, blockbuster::programs::token_account::TokenProgramAccount, - digital_asset_types::dao::{asset, sea_orm_active_enums::OwnerType, token_accounts, tokens}, + digital_asset_types::dao::{token_accounts, tokens}, sea_orm::{ - entity::{ActiveValue, ColumnTrait}, - query::{QueryFilter, QueryTrait}, - sea_query::query::OnConflict, - ConnectionTrait, DatabaseConnection, DbBackend, EntityTrait, TransactionTrait, + entity::ActiveValue, query::QueryTrait, sea_query::query::OnConflict, ConnectionTrait, + DatabaseConnection, DbBackend, EntityTrait, TransactionTrait, }, solana_sdk::program_option::COption, spl_token::state::AccountState, @@ -49,6 +47,8 @@ pub async fn handle_token_program_account<'a, 'b>( close_authority: ActiveValue::Set(None), }; + let txn = db.begin().await?; + let mut query = token_accounts::Entity::insert(model) .on_conflict( OnConflict::columns([token_accounts::Column::Pubkey]) @@ -70,30 +70,22 @@ pub async fn handle_token_program_account<'a, 'b>( "{} WHERE excluded.slot_updated > token_accounts.slot_updated", query.sql ); - db.execute(query).await?; - let txn = db.begin().await?; - let asset_update: Option = asset::Entity::find_by_id(mint.clone()) - .filter(asset::Column::OwnerType.eq("single")) - .one(&txn) + txn.execute(query).await?; + + if ta.amount == 1 { + upsert_assets_token_account_columns( + AssetTokenAccountColumns { + mint: mint.clone(), + owner: Some(owner.clone()), + frozen, + delegate, + slot_updated_token_account: Some(account_info.slot as i64), + }, + &txn, + ) .await?; - if let Some(_asset) = asset_update { - // will only update owner if token account balance is non-zero - // since the asset is marked as single then the token account balance can only be 1. Greater implies a fungible token in which case no si - // TODO: this does not guarantee in case when wallet receives an amount of 1 for a token but its supply is more. is unlikely since mints often have a decimal - if ta.amount == 1 { - upsert_assets_token_account_columns( - AssetTokenAccountColumns { - mint: mint.clone(), - owner: Some(owner.clone()), - frozen, - delegate, - slot_updated_token_account: Some(account_info.slot as i64), - }, - &txn, - ) - .await?; - } } + txn.commit().await?; Ok(()) } @@ -118,6 +110,8 @@ pub async fn handle_token_program_account<'a, 'b>( freeze_authority: ActiveValue::Set(freeze_auth), }; + let txn = db.begin().await?; + let mut query = tokens::Entity::insert(model) .on_conflict( OnConflict::columns([tokens::Column::Mint]) @@ -134,34 +128,26 @@ pub async fn handle_token_program_account<'a, 'b>( .to_owned(), ) .build(DbBackend::Postgres); + query.sql = format!( "{} WHERE excluded.slot_updated >= tokens.slot_updated", query.sql ); - db.execute(query).await?; - let asset_update: Option = asset::Entity::find_by_id(account_key.clone()) - .filter( - asset::Column::OwnerType - .eq(OwnerType::Single) - .or(asset::Column::OwnerType - .eq(OwnerType::Unknown) - .and(asset::Column::Supply.eq(1))), - ) - .one(db) - .await?; - if let Some(_asset) = asset_update { - upsert_assets_mint_account_columns( - AssetMintAccountColumns { - mint: account_key.clone(), - supply_mint: Some(account_key), - supply: m.supply.into(), - slot_updated_mint_account: account_info.slot, - }, - db, - ) - .await?; - } + txn.execute(query).await?; + + upsert_assets_mint_account_columns( + AssetMintAccountColumns { + mint: account_key.clone(), + supply_mint: Some(account_key), + supply: m.supply.into(), + slot_updated_mint_account: account_info.slot, + }, + &txn, + ) + .await?; + + txn.commit().await?; Ok(()) } diff --git a/program_transformers/src/token_metadata/master_edition.rs b/program_transformers/src/token_metadata/master_edition.rs index 791368af6..a95304e5f 100644 --- a/program_transformers/src/token_metadata/master_edition.rs +++ b/program_transformers/src/token_metadata/master_edition.rs @@ -5,13 +5,11 @@ use { types::Key, }, digital_asset_types::dao::{ - asset, asset_v1_account_attachments, extensions, - sea_orm_active_enums::{SpecificationAssetClass, V1AccountAttachments}, + asset_v1_account_attachments, sea_orm_active_enums::V1AccountAttachments, }, sea_orm::{ - entity::{ActiveModelTrait, ActiveValue, EntityTrait, RelationTrait}, - prelude::*, - query::{JoinType, QuerySelect, QueryTrait}, + entity::{ActiveValue, EntityTrait}, + query::QueryTrait, sea_query::query::OnConflict, ConnectionTrait, DatabaseTransaction, DbBackend, }, @@ -65,15 +63,6 @@ pub async fn save_master_edition( txn: &DatabaseTransaction, ) -> ProgramTransformerResult<()> { let id_bytes = id.to_bytes().to_vec(); - let master_edition: Option<(asset_v1_account_attachments::Model, Option)> = - asset_v1_account_attachments::Entity::find_by_id(id.to_bytes().to_vec()) - .find_also_related(asset::Entity) - .join( - JoinType::InnerJoin, - extensions::asset::Relation::AssetData.def(), - ) - .one(txn) - .await?; let ser = serde_json::to_value(me_data) .map_err(|e| ProgramTransformerError::SerializatonError(e.to_string()))?; @@ -85,13 +74,6 @@ pub async fn save_master_edition( ..Default::default() }; - if let Some((_me, Some(asset))) = master_edition { - let mut updatable: asset::ActiveModel = asset.into(); - updatable.supply = ActiveValue::Set(Decimal::from(1)); - updatable.specification_asset_class = ActiveValue::Set(Some(SpecificationAssetClass::Nft)); - updatable.update(txn).await?; - } - let query = asset_v1_account_attachments::Entity::insert(model) .on_conflict( OnConflict::columns([asset_v1_account_attachments::Column::Id]) diff --git a/program_transformers/src/token_metadata/v1_asset.rs b/program_transformers/src/token_metadata/v1_asset.rs index 2c3a38149..a75575d7f 100644 --- a/program_transformers/src/token_metadata/v1_asset.rs +++ b/program_transformers/src/token_metadata/v1_asset.rs @@ -1,12 +1,8 @@ use { crate::{ - asset_upserts::{ - upsert_assets_metadata_account_columns, upsert_assets_mint_account_columns, - upsert_assets_token_account_columns, AssetMetadataAccountColumns, - AssetMintAccountColumns, AssetTokenAccountColumns, - }, + asset_upserts::{upsert_assets_metadata_account_columns, AssetMetadataAccountColumns}, error::{ProgramTransformerError, ProgramTransformerResult}, - find_model_with_retry, DownloadMetadataInfo, + DownloadMetadataInfo, }, blockbuster::token_metadata::{ accounts::{MasterEdition, Metadata}, @@ -17,21 +13,19 @@ use { asset, asset_authority, asset_creators, asset_data, asset_grouping, asset_v1_account_attachments, sea_orm_active_enums::{ - ChainMutability, Mutability, OwnerType, SpecificationAssetClass, - SpecificationVersions, V1AccountAttachments, + ChainMutability, Mutability, SpecificationAssetClass, SpecificationVersions, + V1AccountAttachments, }, - token_accounts, tokens, }, json::ChainDataV1, }, sea_orm::{ - entity::{ActiveValue, ColumnTrait, EntityTrait}, - query::{JsonValue, Order, QueryFilter, QueryOrder, QueryTrait}, + entity::{ActiveValue, EntityTrait}, + query::{JsonValue, QueryTrait}, sea_query::query::OnConflict, - ConnectionTrait, DbBackend, DbErr, Statement, TransactionTrait, + ConnectionTrait, DbBackend, Statement, TransactionTrait, }, solana_sdk::pubkey, - sqlx::types::Decimal, tracing::warn, }; @@ -62,87 +56,8 @@ pub async fn burn_v1_asset( Ok(()) } -const RETRY_INTERVALS: &[u64] = &[0, 5, 10]; static WSOL_PUBKEY: pubkey::Pubkey = pubkey!("So11111111111111111111111111111111111111112"); -pub async fn index_and_fetch_mint_data( - conn: &T, - mint_pubkey_vec: Vec, -) -> ProgramTransformerResult> { - // Gets the token and token account for the mint to populate the asset. - // This is required when the token and token account are indexed, but not the metadata account. - // If the metadata account is indexed, then the token and ta ingester will update the asset with the correct data. - let token: Option = find_model_with_retry( - conn, - "token", - &tokens::Entity::find_by_id(mint_pubkey_vec.clone()), - RETRY_INTERVALS, - ) - .await?; - - if let Some(token) = token { - upsert_assets_mint_account_columns( - AssetMintAccountColumns { - mint: mint_pubkey_vec.clone(), - supply_mint: Some(token.mint.clone()), - supply: token.supply, - slot_updated_mint_account: token.slot_updated as u64, - }, - conn, - ) - .await - .map_err(|db_err| ProgramTransformerError::AssetIndexError(db_err.to_string()))?; - Ok(Some(token)) - } else { - // warn!( - // target: "Mint not found", - // "Mint not found in 'tokens' table for mint {}", - // bs58::encode(&mint_pubkey_vec).into_string() - // ); - Ok(None) - } -} - -async fn index_token_account_data( - conn: &T, - mint_pubkey_vec: Vec, -) -> ProgramTransformerResult<()> { - let token_account: Option = find_model_with_retry( - conn, - "owners", - &token_accounts::Entity::find() - .filter(token_accounts::Column::Mint.eq(mint_pubkey_vec.clone())) - .filter(token_accounts::Column::Amount.gt(0)) - .order_by(token_accounts::Column::SlotUpdated, Order::Desc), - RETRY_INTERVALS, - ) - .await - .map_err(|e: DbErr| ProgramTransformerError::DatabaseError(e.to_string()))?; - - if let Some(token_account) = token_account { - upsert_assets_token_account_columns( - AssetTokenAccountColumns { - mint: mint_pubkey_vec.clone(), - owner: Some(token_account.owner), - delegate: token_account.delegate, - frozen: token_account.frozen, - slot_updated_token_account: Some(token_account.slot_updated), - }, - conn, - ) - .await - .map_err(|db_err| ProgramTransformerError::AssetIndexError(db_err.to_string()))?; - } else { - // warn!( - // target: "Account not found", - // "Token acc not found in 'owners' table for mint {}", - // bs58::encode(&mint_pubkey_vec).into_string() - // ); - } - - Ok(()) -} - pub async fn save_v1_asset( conn: &T, metadata: &Metadata, @@ -170,40 +85,13 @@ pub async fn save_v1_asset( } _ => SpecificationAssetClass::Unknown, }; - let mut ownership_type = match class { - SpecificationAssetClass::FungibleAsset => OwnerType::Token, - SpecificationAssetClass::FungibleToken => OwnerType::Token, - SpecificationAssetClass::Nft | SpecificationAssetClass::ProgrammableNft => { - OwnerType::Single - } - _ => OwnerType::Unknown, - }; // Wrapped Solana is a special token that has supply 0 (infinite). // It's a fungible token with a metadata account, but without any token standard, meaning the code above will misabel it as an NFT. if mint_pubkey == WSOL_PUBKEY { - ownership_type = OwnerType::Token; class = SpecificationAssetClass::FungibleToken; } - let token: Option = - index_and_fetch_mint_data(conn, mint_pubkey_vec.clone()).await?; - - // get supply of token, default to 1 since most cases will be NFTs. Token mint ingester will properly set supply if token_result is None - let supply = token.map(|t| t.supply).unwrap_or_else(|| Decimal::from(1)); - // Map unknown ownership types based on the supply. - if ownership_type == OwnerType::Unknown { - ownership_type = match supply { - s if s == Decimal::from(1) => OwnerType::Single, - s if s > Decimal::from(1) => OwnerType::Token, - _ => OwnerType::Unknown, - }; - }; - - if (ownership_type == OwnerType::Single) | (ownership_type == OwnerType::Unknown) { - index_token_account_data(conn, mint_pubkey_vec.clone()).await?; - } - let name = metadata.name.clone().into_bytes(); let symbol = metadata.symbol.clone().into_bytes(); let mut chain_data = ChainDataV1 { @@ -274,7 +162,6 @@ pub async fn save_v1_asset( upsert_assets_metadata_account_columns( AssetMetadataAccountColumns { mint: mint_pubkey_vec.clone(), - owner_type: ownership_type, specification_asset_class: Some(class), royalty_amount: metadata.seller_fee_basis_points as i32, asset_data: Some(mint_pubkey_vec.clone()),