Remove Selects From Standard NFT Indexing (#183)

* Use Example Configs (#180) Co-authored-by: Ahzam Akhtar <[email protected]> * Remove select insert from indexing mint * Drop master edition lookup save v1 asset sets the specification and mint the supply * Remove lookup records while ingesting asset * Only set owner_type based on the mint for standard nfts --------- Co-authored-by: Ahzam Akhtar <[email protected]>
rpcpool · Dec 9, 2024 · 700cd9f · 700cd9f
1 parent ea897eb
commit 700cd9f
Show file tree

Hide file tree

Showing 10 changed files with 130 additions and 250 deletions.
diff --git a/grpc-ingest/.gitignore b/grpc-ingest/.gitignore
@@ -0,0 +1,3 @@
+config-grpc2redis.yaml
+config-ingester.yaml
+config-monitor.yaml
diff --git a/grpc-ingest/README.md b/grpc-ingest/README.md
@@ -14,7 +14,15 @@ docker compose up db redis prometheus
 INIT_FILE_PATH=./init.sql sea migrate up --database-url=postgres://solana:solana@localhost:5432/solana
 ```
 
-### Config for grpc2redis [./config-grpc2redis.yml](./config-grpc2redis.yml)
+### Configs
+
+Example config files are available at
+- [./config-grpc2redis.example.yml](./config-grpc2redis.example.yml)
+- [./config-ingester.example.yml](./config-ingester.example.yml)
+- [./config-monitor.example.yml](./config-monitor.example.yml) 
+
+Copy these files and modify them as needed to setup the project. 
+
 
 ### Run grpc2redis service
 

diff --git a/grpc-ingest/config-grpc2redis.yml → grpc-ingest/config-grpc2redis.example.yml b/grpc-ingest/config-grpc2redis.yml → grpc-ingest/config-grpc2redis.example.yml
diff --git a/grpc-ingest/config-ingester.yml → grpc-ingest/config-ingester.example.yml b/grpc-ingest/config-ingester.yml → grpc-ingest/config-ingester.example.yml
diff --git a/grpc-ingest/config-monitor.yml → grpc-ingest/config-monitor.example.yml b/grpc-ingest/config-monitor.yml → grpc-ingest/config-monitor.example.yml
diff --git a/program_transformers/src/asset_upserts.rs b/program_transformers/src/asset_upserts.rs
@@ -47,7 +47,7 @@ pub async fn upsert_assets_token_account_columns<T: ConnectionTrait + Transactio
         .build(DbBackend::Postgres);
 
     query.sql = format!(
-    "{} WHERE excluded.slot_updated_token_account >= asset.slot_updated_token_account OR asset.slot_updated_token_account IS NULL",
+    "{} WHERE (excluded.slot_updated_token_account >= asset.slot_updated_token_account OR asset.slot_updated_token_account IS NULL) AND asset.owner_type = 'single'",
     query.sql);
     txn_or_conn.execute(query).await?;
     Ok(())
@@ -64,11 +64,18 @@ pub async fn upsert_assets_mint_account_columns<T: ConnectionTrait + Transaction
     columns: AssetMintAccountColumns,
     txn_or_conn: &T,
 ) -> Result<(), DbErr> {
+    let owner_type = if columns.supply == Decimal::from(1) {
+        OwnerType::Single
+    } else {
+        OwnerType::Token
+    };
+
     let active_model = asset::ActiveModel {
         id: Set(columns.mint),
         supply: Set(columns.supply),
         supply_mint: Set(columns.supply_mint),
         slot_updated_mint_account: Set(Some(columns.slot_updated_mint_account as i64)),
+        owner_type: Set(owner_type),
         ..Default::default()
     };
     let mut query = asset::Entity::insert(active_model)
@@ -78,6 +85,7 @@ pub async fn upsert_assets_mint_account_columns<T: ConnectionTrait + Transaction
                     asset::Column::Supply,
                     asset::Column::SupplyMint,
                     asset::Column::SlotUpdatedMintAccount,
+                    asset::Column::OwnerType,
                 ])
                 .to_owned(),
         )
@@ -92,7 +100,6 @@ pub async fn upsert_assets_mint_account_columns<T: ConnectionTrait + Transaction
 
 pub struct AssetMetadataAccountColumns {
     pub mint: Vec<u8>,
-    pub owner_type: OwnerType,
     pub specification_asset_class: Option<SpecificationAssetClass>,
     pub royalty_amount: i32,
     pub asset_data: Option<Vec<u8>>,
@@ -112,7 +119,6 @@ pub async fn upsert_assets_metadata_account_columns<T: ConnectionTrait + Transac
 ) -> Result<(), DbErr> {
     let active_model = asset::ActiveModel {
         id: Set(columns.mint),
-        owner_type: Set(columns.owner_type),
         specification_version: Set(Some(SpecificationVersions::V1)),
         specification_asset_class: Set(columns.specification_asset_class),
         tree_id: Set(None),
@@ -142,7 +148,6 @@ pub async fn upsert_assets_metadata_account_columns<T: ConnectionTrait + Transac
         .on_conflict(
             OnConflict::columns([asset::Column::Id])
                 .update_columns([
-                    asset::Column::OwnerType,
                     asset::Column::SpecificationVersion,
                     asset::Column::SpecificationAssetClass,
                     asset::Column::TreeId,

diff --git a/program_transformers/src/mpl_core_program/v1_asset.rs b/program_transformers/src/mpl_core_program/v1_asset.rs
@@ -1,10 +1,5 @@
 use {
     crate::{
-        asset_upserts::{
-            upsert_assets_metadata_account_columns, upsert_assets_mint_account_columns,
-            upsert_assets_token_account_columns, AssetMetadataAccountColumns,
-            AssetMintAccountColumns, AssetTokenAccountColumns,
-        },
         error::{ProgramTransformerError, ProgramTransformerResult},
         find_model_with_retry, DownloadMetadataInfo,
     },
@@ -17,6 +12,7 @@ use {
             asset, asset_authority, asset_creators, asset_data, asset_grouping,
             sea_orm_active_enums::{
                 ChainMutability, Mutability, OwnerType, SpecificationAssetClass,
+                SpecificationVersions,
             },
         },
         json::ChainDataV1,
@@ -300,41 +296,6 @@ pub async fn save_v1_asset<T: ConnectionTrait + TransactionTrait>(
         None
     };
 
-    upsert_assets_metadata_account_columns(
-        AssetMetadataAccountColumns {
-            mint: id_vec.clone(),
-            owner_type: ownership_type,
-            specification_asset_class: Some(class),
-            royalty_amount: royalty_amount as i32,
-            asset_data: Some(id_vec.clone()),
-            slot_updated_metadata_account: slot,
-            mpl_core_plugins: Some(plugins_json),
-            mpl_core_unknown_plugins: unknown_plugins_json,
-            mpl_core_collection_num_minted: asset.num_minted.map(|val| val as i32),
-            mpl_core_collection_current_size: asset.current_size.map(|val| val as i32),
-            mpl_core_plugins_json_version: Some(1),
-            mpl_core_external_plugins: Some(external_plugins_json),
-            mpl_core_unknown_external_plugins: unknown_external_plugins_json,
-        },
-        &txn,
-    )
-    .await?;
-
-    let supply = Decimal::from(1);
-
-    // Note: these need to be separate for Token Metadata but here could be one upsert.
-    upsert_assets_mint_account_columns(
-        AssetMintAccountColumns {
-            mint: id_vec.clone(),
-            supply_mint: None,
-            supply,
-            slot_updated_mint_account: slot,
-        },
-        &txn,
-    )
-    .await?;
-
-    // Get transfer delegate from `TransferDelegate` plugin if available.
     let transfer_delegate =
         asset
             .plugins
@@ -346,7 +307,6 @@ pub async fn save_v1_asset<T: ConnectionTrait + TransactionTrait>(
                 PluginAuthority::None => None,
             });
 
-    // Get frozen status from `FreezeDelegate` plugin if available.
     let frozen = asset
         .plugins
         .get(&PluginType::FreezeDelegate)
@@ -359,19 +319,68 @@ pub async fn save_v1_asset<T: ConnectionTrait + TransactionTrait>(
         })
         .unwrap_or(false);
 
-    // TODO: these upserts needed to be separate for Token Metadata but here could be one upsert.
-    upsert_assets_token_account_columns(
-        AssetTokenAccountColumns {
-            mint: id_vec.clone(),
-            owner,
-            frozen,
-            // Note use transfer delegate for the existing delegate field.
-            delegate: transfer_delegate.clone(),
-            slot_updated_token_account: Some(slot_i),
-        },
-        &txn,
-    )
-    .await?;
+    let asset_model = asset::ActiveModel {
+        id: ActiveValue::Set(id_vec.clone()),
+        owner_type: ActiveValue::Set(ownership_type),
+        supply: ActiveValue::Set(Decimal::from(1)),
+        supply_mint: ActiveValue::Set(None),
+        slot_updated_mint_account: ActiveValue::Set(Some(slot as i64)),
+        specification_version: ActiveValue::Set(Some(SpecificationVersions::V1)),
+        specification_asset_class: ActiveValue::Set(Some(class)),
+        royalty_amount: ActiveValue::Set(royalty_amount as i32),
+        asset_data: ActiveValue::Set(Some(id_vec.clone())),
+        slot_updated_metadata_account: ActiveValue::Set(Some(slot as i64)),
+        mpl_core_plugins: ActiveValue::Set(Some(plugins_json)),
+        mpl_core_unknown_plugins: ActiveValue::Set(unknown_plugins_json),
+        mpl_core_collection_num_minted: ActiveValue::Set(asset.num_minted.map(|val| val as i32)),
+        mpl_core_collection_current_size: ActiveValue::Set(
+            asset.current_size.map(|val| val as i32),
+        ),
+        mpl_core_plugins_json_version: ActiveValue::Set(Some(1)),
+        mpl_core_external_plugins: ActiveValue::Set(Some(external_plugins_json)),
+        mpl_core_unknown_external_plugins: ActiveValue::Set(unknown_external_plugins_json),
+        owner: ActiveValue::Set(owner),
+        frozen: ActiveValue::Set(frozen),
+        delegate: ActiveValue::Set(transfer_delegate.clone()),
+        slot_updated_token_account: ActiveValue::Set(Some(slot_i)),
+        ..Default::default()
+    };
+
+    let mut query = asset::Entity::insert(asset_model)
+        .on_conflict(
+            OnConflict::columns([asset::Column::Id])
+                .update_columns([
+                    asset::Column::OwnerType,
+                    asset::Column::Supply,
+                    asset::Column::SupplyMint,
+                    asset::Column::SlotUpdatedMintAccount,
+                    asset::Column::SpecificationVersion,
+                    asset::Column::SpecificationAssetClass,
+                    asset::Column::RoyaltyAmount,
+                    asset::Column::AssetData,
+                    asset::Column::SlotUpdatedMetadataAccount,
+                    asset::Column::MplCorePlugins,
+                    asset::Column::MplCoreUnknownPlugins,
+                    asset::Column::MplCoreCollectionNumMinted,
+                    asset::Column::MplCoreCollectionCurrentSize,
+                    asset::Column::MplCorePluginsJsonVersion,
+                    asset::Column::MplCoreExternalPlugins,
+                    asset::Column::MplCoreUnknownExternalPlugins,
+                    asset::Column::Owner,
+                    asset::Column::Frozen,
+                    asset::Column::Delegate,
+                    asset::Column::SlotUpdatedTokenAccount,
+                ])
+                .to_owned(),
+        )
+        .build(DbBackend::Postgres);
+
+    query.sql = format!(
+        "{} WHERE excluded.slot_updated_metadata_account >= asset.slot_updated_metadata_account OR asset.slot_updated_metadata_account IS NULL",
+        query.sql
+    );
+
+    txn.execute(query).await?;
 
     //-----------------------
     // asset_grouping table

diff --git a/program_transformers/src/token/mod.rs b/program_transformers/src/token/mod.rs
@@ -8,12 +8,10 @@ use {
         AccountInfo, DownloadMetadataNotifier,
     },
     blockbuster::programs::token_account::TokenProgramAccount,
-    digital_asset_types::dao::{asset, sea_orm_active_enums::OwnerType, token_accounts, tokens},
+    digital_asset_types::dao::{token_accounts, tokens},
     sea_orm::{
-        entity::{ActiveValue, ColumnTrait},
-        query::{QueryFilter, QueryTrait},
-        sea_query::query::OnConflict,
-        ConnectionTrait, DatabaseConnection, DbBackend, EntityTrait, TransactionTrait,
+        entity::ActiveValue, query::QueryTrait, sea_query::query::OnConflict, ConnectionTrait,
+        DatabaseConnection, DbBackend, EntityTrait, TransactionTrait,
     },
     solana_sdk::program_option::COption,
     spl_token::state::AccountState,
@@ -49,6 +47,8 @@ pub async fn handle_token_program_account<'a, 'b>(
                 close_authority: ActiveValue::Set(None),
             };
 
+            let txn = db.begin().await?;
+
             let mut query = token_accounts::Entity::insert(model)
                 .on_conflict(
                     OnConflict::columns([token_accounts::Column::Pubkey])
@@ -70,30 +70,22 @@ pub async fn handle_token_program_account<'a, 'b>(
                 "{} WHERE excluded.slot_updated > token_accounts.slot_updated",
                 query.sql
             );
-            db.execute(query).await?;
-            let txn = db.begin().await?;
-            let asset_update: Option<asset::Model> = asset::Entity::find_by_id(mint.clone())
-                .filter(asset::Column::OwnerType.eq("single"))
-                .one(&txn)
+            txn.execute(query).await?;
+
+            if ta.amount == 1 {
+                upsert_assets_token_account_columns(
+                    AssetTokenAccountColumns {
+                        mint: mint.clone(),
+                        owner: Some(owner.clone()),
+                        frozen,
+                        delegate,
+                        slot_updated_token_account: Some(account_info.slot as i64),
+                    },
+                    &txn,
+                )
                 .await?;
-            if let Some(_asset) = asset_update {
-                // will only update owner if token account balance is non-zero
-                // since the asset is marked as single then the token account balance can only be 1. Greater implies a fungible token in which case no si
-                // TODO: this does not guarantee in case when wallet receives an amount of 1 for a token but its supply is more. is unlikely since mints often have a decimal
-                if ta.amount == 1 {
-                    upsert_assets_token_account_columns(
-                        AssetTokenAccountColumns {
-                            mint: mint.clone(),
-                            owner: Some(owner.clone()),
-                            frozen,
-                            delegate,
-                            slot_updated_token_account: Some(account_info.slot as i64),
-                        },
-                        &txn,
-                    )
-                    .await?;
-                }
             }
+
             txn.commit().await?;
             Ok(())
         }
@@ -118,6 +110,8 @@ pub async fn handle_token_program_account<'a, 'b>(
                 freeze_authority: ActiveValue::Set(freeze_auth),
             };
 
+            let txn = db.begin().await?;
+
             let mut query = tokens::Entity::insert(model)
                 .on_conflict(
                     OnConflict::columns([tokens::Column::Mint])
@@ -134,34 +128,26 @@ pub async fn handle_token_program_account<'a, 'b>(
                         .to_owned(),
                 )
                 .build(DbBackend::Postgres);
+
             query.sql = format!(
                 "{} WHERE excluded.slot_updated >= tokens.slot_updated",
                 query.sql
             );
-            db.execute(query).await?;
 
-            let asset_update: Option<asset::Model> = asset::Entity::find_by_id(account_key.clone())
-                .filter(
-                    asset::Column::OwnerType
-                        .eq(OwnerType::Single)
-                        .or(asset::Column::OwnerType
-                            .eq(OwnerType::Unknown)
-                            .and(asset::Column::Supply.eq(1))),
-                )
-                .one(db)
-                .await?;
-            if let Some(_asset) = asset_update {
-                upsert_assets_mint_account_columns(
-                    AssetMintAccountColumns {
-                        mint: account_key.clone(),
-                        supply_mint: Some(account_key),
-                        supply: m.supply.into(),
-                        slot_updated_mint_account: account_info.slot,
-                    },
-                    db,
-                )
-                .await?;
-            }
+            txn.execute(query).await?;
+
+            upsert_assets_mint_account_columns(
+                AssetMintAccountColumns {
+                    mint: account_key.clone(),
+                    supply_mint: Some(account_key),
+                    supply: m.supply.into(),
+                    slot_updated_mint_account: account_info.slot,
+                },
+                &txn,
+            )
+            .await?;
+
+            txn.commit().await?;
 
             Ok(())
         }