Introduce AccountsDataStorage

ledger-tool/src/main.rs

@@ -47,16 +47,19 @@ use {
         transaction_status_service::TransactionStatusService,
     },
     solana_runtime::{
+        account_storage::meta::{StoredAccountMeta, StoredMeta},
         accounts::Accounts,
         accounts_background_service::{
             AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService,
             PrunedBanksRequestHandler, SnapshotRequestHandler,
         },
+        accounts_data_storage::{meta_entries::AccountMetaFlags, AccountsDataStorage},
         accounts_db::{
             AccountsDb, AccountsDbConfig, CalcAccountsHashDataSource, FillerAccountsConfig,
         },
         accounts_index::{AccountsIndexConfig, IndexLimitMb, ScanConfig},
         accounts_update_notifier_interface::AccountsUpdateNotifier,
+        append_vec::AppendVec,
         bank::{Bank, RewardCalculationEvent, TotalAccountsStats},
         bank_forks::BankForks,
         cost_model::CostModel,
@@ -2369,6 +2372,24 @@ fn main() {
                            If no file name is specified, it will print the metadata of all ledger files.")
             )
         )
+        .subcommand(
+            SubCommand::with_name("new_ads_file")
+            .about("Create a new accounts-data-storage file from an existing append_vec file.")
+            .arg(
+                Arg::with_name("append_vec")
+                    .long("append-vec")
+                    .takes_value(true)
+                    .value_name("APPEND_VEC_FILE_NAME")
+                    .help("The name of the append vec file.")
+            )
+            .arg(
+                Arg::with_name("ads_file_name")
+                    .long("ads-file-name")
+                    .takes_value(true)
+                    .value_name("ADS_FILE_NAME")
+                    .help("The name of the output ads file.")
+            )
+        )
         .get_matches();
 
     info!("{} {}", crate_name!(), solana_version::version!());
@@ -4330,6 +4351,90 @@ fn main() {
                     eprintln!("{err}");
                 }
             }
+            ("new_ads_file", Some(arg_matches)) => {
+                let append_vec_path = value_t_or_exit!(arg_matches, "append_vec", String);
+                let ads_file_path =
+                    PathBuf::from(value_t_or_exit!(arg_matches, "ads_file_name", String));
+                let append_vec_len = std::fs::metadata(&append_vec_path).unwrap().len() as usize;
+                let mut append_vec =
+                    AppendVec::new_from_file_unchecked(append_vec_path, append_vec_len)
+                        .expect("should succeed");
+                append_vec.set_no_remove_on_drop();
+                let ads = AccountsDataStorage::new(&ads_file_path, true /* create */);
+                ads.write_from_append_vec(&append_vec).unwrap();
+
+                let footer = ads.read_footer_block().unwrap();
+                info!("footer = {:?}", footer);
+
+                // read append-vec
+                let mut num_accounts = 0;
+                let mut offset = 0;
+                let mut account_map: HashMap<Pubkey, StoredAccountMeta> = HashMap::new();
+                while let Some((account, next_offset)) = append_vec.get_account(offset) {
+                    offset = next_offset;
+                    num_accounts += 1;
+                    account_map.insert(*account.pubkey(), account);
+                }
+                assert_eq!(num_accounts, footer.account_meta_count as u64);
+                info!("# accounts from append_vec = {:?}", num_accounts);
+
+                let mut metas = ads
+                    .read_account_metas_block(
+                        footer.account_metas_offset,
+                        footer.account_meta_count,
+                    )
+                    .unwrap();
+
+                for i in 0..footer.account_meta_count as usize {
+                    let account_pubkey = ads
+                        .read_account_pubkey(&footer, i.try_into().unwrap())
+                        .unwrap();
+                    let account_owner = ads
+                        .read_owner(footer.owners_offset, metas[i].owner_local_id)
+                        .unwrap();
+                    if account_pubkey == Pubkey::default() {
+                        continue;
+                    }
+                    if account_owner == Pubkey::default() {
+                        continue;
+                    }
+                    let av_account = &account_map[&account_pubkey];
+                    info!("verifing account {:?}", account_pubkey);
+                    info!(
+                        "    lamport: append_vec({:?}) vs ads_file ({:?})",
+                        av_account.lamports(),
+                        metas[i].lamports
+                    );
+                    assert_eq!(av_account.lamports(), metas[i].lamports);
+
+                    let data_block = ads.read_account_data_block(&footer, &mut metas, i).unwrap();
+                    let account_data_from_storage = metas[i].get_account_data(&data_block);
+
+                    let stored_meta_from_storage = StoredMeta {
+                        // we no longer store write_version anymore
+                        write_version_obsolete: av_account.write_version(),
+                        // load the first pubkey in the storage
+                        pubkey: account_pubkey,
+                        data_len: (account_data_from_storage.len()) as u64,
+                    };
+                    assert_eq!(&stored_meta_from_storage, av_account.meta());
+                    info!(" StoredMeta: append_vec({:?})", av_account.meta());
+                    info!("             ads_file  ({:?})", stored_meta_from_storage);
+
+                    let account_from_storage = AccountSharedData {
+                        lamports: metas[i].lamports,
+                        data: Arc::new(account_data_from_storage.to_vec()),
+                        owner: account_owner,
+                        executable: metas[i].flags_get(AccountMetaFlags::EXECUTABLE),
+                        rent_epoch: metas[i].rent_epoch(&data_block).unwrap_or(0),
+                    };
+                    let av_shared_meta = av_account.clone_account();
+
+                    info!(" AccountSharedData: append_vec({:?})", av_shared_meta);
+                    info!("                    ads_file  ({:?})", account_from_storage);
+                    assert_eq!(account_from_storage, av_shared_meta);
+                }
+            }
             ("", _) => {
                 eprintln!("{}", matches.usage());
                 exit(1);

runtime/Cargo.toml

@@ -31,6 +31,7 @@ lru = { workspace = true }
 lz4 = { workspace = true }
 memmap2 = { workspace = true }
 modular-bitfield = { workspace = true }
+num_enum = { workspace = true }
 num-derive = { workspace = true }
 num-traits = { workspace = true }
 num_cpus = { workspace = true }

runtime/src/account_storage/meta.rs

@@ -1,5 +1,8 @@
 use {
-    crate::{append_vec::AppendVecAccountMeta, storable_accounts::StorableAccounts},
+    crate::{
+        accounts_data_storage::reader::TieredAccountMeta, append_vec::AppendVecAccountMeta,
+        storable_accounts::StorableAccounts,
+    },
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount},
         hash::Hash,
@@ -99,80 +102,93 @@ impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V:
 #[derive(PartialEq, Eq, Debug)]
 pub enum StoredAccountMeta<'a> {
     AppendVec(AppendVecAccountMeta<'a>),
+    Tiered(TieredAccountMeta<'a>),
 }
 
 impl<'a> StoredAccountMeta<'a> {
     /// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
     pub fn clone_account(&self) -> AccountSharedData {
         match self {
             Self::AppendVec(av) => av.clone_account(),
+            Self::Tiered(ts) => ts.clone_account(),
         }
     }
 
     pub fn pubkey(&self) -> &Pubkey {
         match self {
             Self::AppendVec(av) => av.pubkey(),
+            Self::Tiered(ts) => ts.pubkey(),
         }
     }
 
     pub fn hash(&self) -> &Hash {
         match self {
             Self::AppendVec(av) => av.hash(),
+            Self::Tiered(ts) => ts.hash(),
         }
     }
 
     pub fn stored_size(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.stored_size(),
+            Self::Tiered(ts) => ts.stored_size(),
         }
     }
 
     pub fn offset(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.offset(),
+            Self::Tiered(ts) => ts.offset(),
         }
     }
 
     pub fn data(&self) -> &[u8] {
         match self {
             Self::AppendVec(av) => av.data(),
+            Self::Tiered(ts) => ts.data(),
         }
     }
 
     pub fn data_len(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.data_len(),
+            Self::Tiered(ts) => ts.data_len(),
         }
     }
 
     pub fn write_version(&self) -> StoredMetaWriteVersion {
         match self {
             Self::AppendVec(av) => av.write_version(),
+            Self::Tiered(ts) => ts.write_version(),
         }
     }
 
     pub fn meta(&self) -> &StoredMeta {
         match self {
             Self::AppendVec(av) => av.meta(),
+            Self::Tiered(_) => unreachable!(),
         }
     }
 
     pub fn set_meta(&mut self, meta: &'a StoredMeta) {
         match self {
             Self::AppendVec(av) => av.set_meta(meta),
+            Self::Tiered(_) => unreachable!(),
         }
     }
 
     pub(crate) fn sanitize(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.sanitize(),
+            Self::Tiered(_) => unimplemented!(),
         }
     }
 
     #[allow(dead_code)]
     pub(crate) fn sanitize_executable(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.sanitize_executable(),
+            Self::Tiered(_) => unimplemented!(),
         }
         // Sanitize executable to ensure higher 7-bits are cleared correctly.
     }
@@ -181,13 +197,15 @@ impl<'a> StoredAccountMeta<'a> {
     pub(crate) fn sanitize_lamports(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.sanitize_lamports(),
+            Self::Tiered(_) => unimplemented!(),
         }
     }
 
     #[allow(dead_code)]
     pub(crate) fn ref_executable_byte(&self) -> &u8 {
         match self {
             Self::AppendVec(av) => av.ref_executable_byte(),
+            Self::Tiered(_) => unimplemented!(),
         }
     }
 }
@@ -196,26 +214,31 @@ impl<'a> ReadableAccount for StoredAccountMeta<'a> {
     fn lamports(&self) -> u64 {
         match self {
             Self::AppendVec(av) => av.lamports(),
+            Self::Tiered(ts) => ts.lamports(),
         }
     }
     fn data(&self) -> &[u8] {
         match self {
             Self::AppendVec(av) => av.data(),
+            Self::Tiered(ts) => ts.data(),
         }
     }
     fn owner(&self) -> &Pubkey {
         match self {
             Self::AppendVec(av) => av.owner(),
+            Self::Tiered(ts) => ts.owner(),
         }
     }
     fn executable(&self) -> bool {
         match self {
             Self::AppendVec(av) => av.executable(),
+            Self::Tiered(ts) => ts.executable(),
         }
     }
     fn rent_epoch(&self) -> Epoch {
         match self {
             Self::AppendVec(av) => av.rent_epoch(),
+            Self::Tiered(ts) => ts.rent_epoch(),
         }
     }
 }