Danenbm/update metadata parsing

README.md

@@ -34,7 +34,7 @@ Because this is a multi component system the easiest way to develop or locally t
 #### Regenerating DB Types
 Edit the init.sql, then run `docker compose up db`
 Then with a local `DATABASE_URL` var exported like this `export DATABASE_URL=postgres://solana:solana@localhost/solana` you can run
-` sea-orm-cli generate entity -o ./digital_asset_types/src/dao/generated/ --database-url $DATABASE_URL --with-serde both --expanded-format`
+`sea-orm-cli generate entity -o ./digital_asset_types/src/dao/generated/ --database-url $DATABASE_URL --with-serde both --expanded-format`
 
 If you need to install `sea-orm-cli` run `cargo install sea-orm-cli`.
 

das_api/Cargo.toml

@@ -33,7 +33,7 @@ schemars = "0.8.6"
 schemars_derive = "0.8.6"
 open-rpc-derive = { version = "0.0.4"}
 open-rpc-schema = { version = "0.0.4"}
-blockbuster = "0.9.0-beta.1"
+blockbuster = "=0.9.0-beta.3"
 anchor-lang = "0.28.0"
 mpl-token-metadata = { version = "=2.0.0-beta.1",  features = ["serde-feature"] }
 mpl-candy-machine-core = { version = "2.0.1", features = ["no-entrypoint"] }

digital_asset_types/Cargo.toml

@@ -18,7 +18,7 @@ solana-sdk = "~1.16.16"
 num-traits = "0.2.15"
 num-derive = "0.3.3"
 thiserror = "1.0.31"
-blockbuster = "0.9.0-beta.1"
+blockbuster = "=0.9.0-beta.3"
 jsonpath_lib = "0.3.0"
 mime_guess = "2.0.4"
 url = "2.3.1"

digital_asset_types/src/dao/generated/asset.rs

@@ -46,6 +46,8 @@ pub struct Model {
     pub owner_delegate_seq: Option<i64>,
     pub was_decompressed: bool,
     pub leaf_seq: Option<i64>,
+    pub royalty_amount_seq: Option<i64>,
+    pub creators_added_seq: Option<i64>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
@@ -78,6 +80,8 @@ pub enum Column {
     OwnerDelegateSeq,
     WasDecompressed,
     LeafSeq,
+    RoyaltyAmountSeq,
+    CreatorsAddedSeq,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
@@ -133,6 +137,8 @@ impl ColumnTrait for Column {
             Self::OwnerDelegateSeq => ColumnType::BigInteger.def().null(),
             Self::WasDecompressed => ColumnType::Boolean.def(),
             Self::LeafSeq => ColumnType::BigInteger.def().null(),
+            Self::RoyaltyAmountSeq => ColumnType::BigInteger.def().null(),
+            Self::CreatorsAddedSeq => ColumnType::BigInteger.def().null(),
         }
     }
 }

digital_asset_types/src/dao/generated/asset_creators.rs

@@ -19,7 +19,7 @@ pub struct Model {
     pub creator: Vec<u8>,
     pub share: i32,
     pub verified: bool,
-    pub seq: Option<i64>,
+    pub verified_seq: Option<i64>,
     pub slot_updated: Option<i64>,
     pub position: i16,
 }
@@ -31,7 +31,7 @@ pub enum Column {
     Creator,
     Share,
     Verified,
-    Seq,
+    VerifiedSeq,
     SlotUpdated,
     Position,
 }
@@ -62,7 +62,7 @@ impl ColumnTrait for Column {
             Self::Creator => ColumnType::Binary.def(),
             Self::Share => ColumnType::Integer.def(),
             Self::Verified => ColumnType::Boolean.def(),
-            Self::Seq => ColumnType::BigInteger.def().null(),
+            Self::VerifiedSeq => ColumnType::BigInteger.def().null(),
             Self::SlotUpdated => ColumnType::BigInteger.def().null(),
             Self::Position => ColumnType::SmallInteger.def(),
         }

digital_asset_types/src/dao/generated/asset_data.rs

@@ -24,8 +24,10 @@ pub struct Model {
     pub metadata: Json,
     pub slot_updated: i64,
     pub reindex: Option<bool>,
-    pub raw_name: Vec<u8>,
-    pub raw_symbol: Vec<u8>,
+    pub raw_name: Option<Vec<u8>>,
+    pub raw_symbol: Option<Vec<u8>>,
+    pub base_info_seq: Option<i64>,
+    pub download_metadata_seq: Option<i64>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
@@ -40,6 +42,8 @@ pub enum Column {
     Reindex,
     RawName,
     RawSymbol,
+    BaseInfoSeq,
+    DownloadMetadataSeq,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
@@ -70,9 +74,11 @@ impl ColumnTrait for Column {
             Self::MetadataMutability => Mutability::db_type(),
             Self::Metadata => ColumnType::JsonBinary.def(),
             Self::SlotUpdated => ColumnType::BigInteger.def(),
-            Self::Reindex => ColumnType::Boolean.def(),
-            Self::RawName => ColumnType::Binary.def(),
-            Self::RawSymbol => ColumnType::Binary.def(),
+            Self::Reindex => ColumnType::Boolean.def().null(),
+            Self::RawName => ColumnType::Binary.def().null(),
+            Self::RawSymbol => ColumnType::Binary.def().null(),
+            Self::BaseInfoSeq => ColumnType::BigInteger.def().null(),
+            Self::DownloadMetadataSeq => ColumnType::BigInteger.def().null(),
         }
     }
 }

digital_asset_types/src/dao/generated/asset_grouping.rs

@@ -1,4 +1,4 @@
-//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
+//! SeaORM Entity. Generated by sea-orm-codegen 0.9.3
 
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
@@ -12,15 +12,15 @@ impl EntityName for Entity {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Serialize, Deserialize)]
 pub struct Model {
     pub id: i64,
     pub asset_id: Vec<u8>,
     pub group_key: String,
     pub group_value: Option<String>,
     pub seq: Option<i64>,
     pub slot_updated: Option<i64>,
-    pub verified: Option<bool>,
+    pub verified: bool,
     pub group_info_seq: Option<i64>,
 }
 

digital_asset_types/src/dao/generated/cl_audits.rs

@@ -22,7 +22,7 @@ pub struct Model {
     pub seq: i64,
     pub level: i64,
     pub hash: Vec<u8>,
-    pub created_at: Option<DateTime>,
+    pub created_at: DateTime,
     pub tx: String,
 }
 

digital_asset_types/src/dao/generated/sea_orm_active_enums.rs

@@ -4,58 +4,12 @@ use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "mutability")]
-pub enum Mutability {
-    #[sea_orm(string_value = "immutable")]
-    Immutable,
-    #[sea_orm(string_value = "mutable")]
-    Mutable,
-    #[sea_orm(string_value = "unknown")]
-    Unknown,
-}
-#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(
-    rs_type = "String",
-    db_type = "Enum",
-    enum_name = "v1_account_attachments"
-)]
-pub enum V1AccountAttachments {
-    #[sea_orm(string_value = "edition")]
-    Edition,
-    #[sea_orm(string_value = "edition_marker")]
-    EditionMarker,
-    #[sea_orm(string_value = "master_edition_v1")]
-    MasterEditionV1,
-    #[sea_orm(string_value = "master_edition_v2")]
-    MasterEditionV2,
-    #[sea_orm(string_value = "unknown")]
-    Unknown,
-}
-#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "task_status")]
-pub enum TaskStatus {
-    #[sea_orm(string_value = "failed")]
-    Failed,
-    #[sea_orm(string_value = "pending")]
-    Pending,
-    #[sea_orm(string_value = "running")]
-    Running,
-    #[sea_orm(string_value = "success")]
-    Success,
-}
-#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(
-    rs_type = "String",
-    db_type = "Enum",
-    enum_name = "royalty_target_type"
-)]
-pub enum RoyaltyTargetType {
-    #[sea_orm(string_value = "creators")]
-    Creators,
-    #[sea_orm(string_value = "fanout")]
-    Fanout,
+#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "owner_type")]
+pub enum OwnerType {
     #[sea_orm(string_value = "single")]
     Single,
+    #[sea_orm(string_value = "token")]
+    Token,
     #[sea_orm(string_value = "unknown")]
     Unknown,
 }
@@ -88,12 +42,20 @@ pub enum SpecificationAssetClass {
     Unknown,
 }
 #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "chain_mutability")]
-pub enum ChainMutability {
-    #[sea_orm(string_value = "immutable")]
-    Immutable,
-    #[sea_orm(string_value = "mutable")]
-    Mutable,
+#[sea_orm(
+    rs_type = "String",
+    db_type = "Enum",
+    enum_name = "v1_account_attachments"
+)]
+pub enum V1AccountAttachments {
+    #[sea_orm(string_value = "edition")]
+    Edition,
+    #[sea_orm(string_value = "edition_marker")]
+    EditionMarker,
+    #[sea_orm(string_value = "master_edition_v1")]
+    MasterEditionV1,
+    #[sea_orm(string_value = "master_edition_v2")]
+    MasterEditionV2,
     #[sea_orm(string_value = "unknown")]
     Unknown,
 }
@@ -114,12 +76,50 @@ pub enum SpecificationVersions {
     V2,
 }
 #[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "owner_type")]
-pub enum OwnerType {
+#[sea_orm(
+    rs_type = "String",
+    db_type = "Enum",
+    enum_name = "royalty_target_type"
+)]
+pub enum RoyaltyTargetType {
+    #[sea_orm(string_value = "creators")]
+    Creators,
+    #[sea_orm(string_value = "fanout")]
+    Fanout,
     #[sea_orm(string_value = "single")]
     Single,
-    #[sea_orm(string_value = "token")]
-    Token,
+    #[sea_orm(string_value = "unknown")]
+    Unknown,
+}
+#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
+#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "task_status")]
+pub enum TaskStatus {
+    #[sea_orm(string_value = "failed")]
+    Failed,
+    #[sea_orm(string_value = "pending")]
+    Pending,
+    #[sea_orm(string_value = "running")]
+    Running,
+    #[sea_orm(string_value = "success")]
+    Success,
+}
+#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
+#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "chain_mutability")]
+pub enum ChainMutability {
+    #[sea_orm(string_value = "immutable")]
+    Immutable,
+    #[sea_orm(string_value = "mutable")]
+    Mutable,
+    #[sea_orm(string_value = "unknown")]
+    Unknown,
+}
+#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
+#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "mutability")]
+pub enum Mutability {
+    #[sea_orm(string_value = "immutable")]
+    Immutable,
+    #[sea_orm(string_value = "mutable")]
+    Mutable,
     #[sea_orm(string_value = "unknown")]
     Unknown,
 }

digital_asset_types/tests/common.rs

@@ -83,8 +83,10 @@ pub fn create_asset_data(
             metadata: JsonValue::String("processing".to_string()),
             slot_updated: 0,
             reindex: None,
-            raw_name: metadata.name.into_bytes().to_vec().clone(),
-            raw_symbol: metadata.symbol.into_bytes().to_vec().clone(),
+            raw_name: Some(metadata.name.into_bytes().to_vec().clone()),
+            raw_symbol: Some(metadata.symbol.into_bytes().to_vec().clone()),
+            base_info_seq: Some(0),
+            download_metadata_seq: Some(0),
         },
     )
 }
@@ -157,6 +159,8 @@ pub fn create_asset(
             owner_delegate_seq: Some(0),
             was_decompressed: false,
             leaf_seq: Some(0),
+            royalty_amount_seq: Some(0),
+            creators_added_seq: Some(0),
         },
     )
 }
@@ -182,7 +186,7 @@ pub fn create_asset_creator(
             creator,
             share,
             verified,
-            seq: Some(0),
+            verified_seq: Some(0),
             slot_updated: Some(0),
             position: 0,
         },
@@ -231,7 +235,7 @@ pub fn create_asset_grouping(
             id: row_num,
             group_key: "collection".to_string(),
             slot_updated: Some(0),
-            verified: Some(false),
+            verified: false,
             group_info_seq: Some(0),
         },
     )

digital_asset_types/tests/json_parsing.rs

@@ -34,8 +34,10 @@ pub async fn parse_onchain_json(json: serde_json::Value) -> Content {
         metadata: json,
         slot_updated: 0,
         reindex: None,
-        raw_name: String::from("Handalf").into_bytes().to_vec(),
-        raw_symbol: String::from("").into_bytes().to_vec(),
+        raw_name: Some(String::from("Handalf").into_bytes().to_vec()),
+        raw_symbol: Some(String::from("").into_bytes().to_vec()),
+        base_info_seq: Some(0),
+        download_metadata_seq: Some(0),
     };
 
     v1_content_from_json(&asset_data).unwrap()

migration/src/lib.rs

@@ -30,6 +30,7 @@ mod m20230724_120101_add_group_info_seq;
 mod m20230726_013107_remove_not_null_constraint_from_group_value;
 mod m20230918_182123_add_raw_name_symbol;
 mod m20230919_072154_cl_audits;
+mod m20231019_120101_add_seq_numbers_bgum_update_metadata;
 
 pub struct Migrator;
 
@@ -67,6 +68,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20230726_013107_remove_not_null_constraint_from_group_value::Migration),
             Box::new(m20230918_182123_add_raw_name_symbol::Migration),
             Box::new(m20230919_072154_cl_audits::Migration),
+            Box::new(m20231019_120101_add_seq_numbers_bgum_update_metadata::Migration),
         ]
     }
 }