Merge branch 'master' of github.com:lijunwangs/solana

Cargo.toml

@@ -226,7 +226,7 @@ itertools = "0.10.5"
 jemallocator = { package = "tikv-jemallocator", version = "0.4.1", features = [
     "unprefixed_malloc_on_supported_platforms",
 ] }
-js-sys = "0.3.66"
+js-sys = "0.3.67"
 json5 = "0.4.1"
 jsonrpc-core = "18.0.0"
 jsonrpc-core-client = "18.0.0"
@@ -280,7 +280,7 @@ quote = "1.0"
 rand = "0.8.5"
 rand_chacha = "0.3.1"
 raptorq = "1.8.0"
-rayon = "1.8.0"
+rayon = "1.8.1"
 rcgen = "0.10.0"
 reed-solomon-erasure = "6.0.0"
 regex = "1.10.2"

accounts-db/src/accounts_db.rs

@@ -98,6 +98,7 @@ use {
         borrow::{Borrow, Cow},
         boxed::Box,
         collections::{hash_map, BTreeSet, HashMap, HashSet},
+        fs,
         hash::{Hash as StdHash, Hasher as StdHasher},
         io::Result as IoResult,
         ops::{Range, RangeBounds},
@@ -1214,11 +1215,11 @@ pub fn create_accounts_run_and_snapshot_dirs(
         // to this new version using run and snapshot directories.
         // The run/ content cleanup will be done at a later point.  The snapshot/ content persists
         // across the process boot, and will be purged by the account_background_service.
-        if fs_err::remove_dir_all(&account_dir).is_err() {
+        if fs::remove_dir_all(&account_dir).is_err() {
             delete_contents_of_path(&account_dir);
         }
-        fs_err::create_dir_all(&run_path)?;
-        fs_err::create_dir_all(&snapshot_path)?;
+        fs::create_dir_all(&run_path)?;
+        fs::create_dir_all(&snapshot_path)?;
     }
 
     Ok((run_path, snapshot_path))
@@ -1246,20 +1247,26 @@ pub fn create_all_accounts_run_and_snapshot_dirs(
 /// to delete the top level directory it might be able to
 /// delete the contents of that directory.
 pub fn delete_contents_of_path(path: impl AsRef<Path>) {
-    match fs_err::read_dir(path.as_ref()) {
+    match fs::read_dir(&path) {
         Err(err) => {
-            warn!("Failed to delete contents: {err}")
+            warn!(
+                "Failed to delete contents of '{}': could not read dir: {err}",
+                path.as_ref().display(),
+            )
         }
         Ok(dir_entries) => {
             for entry in dir_entries.flatten() {
                 let sub_path = entry.path();
                 let result = if sub_path.is_dir() {
-                    fs_err::remove_dir_all(&sub_path)
+                    fs::remove_dir_all(&sub_path)
                 } else {
-                    fs_err::remove_file(&sub_path)
+                    fs::remove_file(&sub_path)
                 };
                 if let Err(err) = result {
-                    warn!("Failed to delete contents: {err}");
+                    warn!(
+                        "Failed to delete contents of '{}': {err}",
+                        sub_path.display(),
+                    );
                 }
             }
         }
@@ -2463,8 +2470,7 @@ impl AccountsDb {
             let accounts_hash_cache_path =
                 base_working_path.join(Self::DEFAULT_ACCOUNTS_HASH_CACHE_DIR);
             if !accounts_hash_cache_path.exists() {
-                fs_err::create_dir(&accounts_hash_cache_path)
-                    .expect("create accounts hash cache dir");
+                fs::create_dir(&accounts_hash_cache_path).expect("create accounts hash cache dir");
             }
             accounts_hash_cache_path
         });

accounts-db/src/hardened_unpack.rs

@@ -446,28 +446,35 @@ fn is_valid_snapshot_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool
     }
 }
 
+#[derive(Error, Debug)]
+pub enum OpenGenesisConfigError {
+    #[error("unpack error: {0}")]
+    Unpack(#[from] UnpackError),
+    #[error("Genesis load error: {0}")]
+    Load(#[from] std::io::Error),
+}
+
 pub fn open_genesis_config(
     ledger_path: &Path,
     max_genesis_archive_unpacked_size: u64,
-) -> GenesisConfig {
-    GenesisConfig::load(ledger_path).unwrap_or_else(|load_err| {
-        let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
-        unpack_genesis_archive(
-            &genesis_package,
-            ledger_path,
-            max_genesis_archive_unpacked_size,
-        )
-        .unwrap_or_else(|unpack_err| {
+) -> std::result::Result<GenesisConfig, OpenGenesisConfigError> {
+    match GenesisConfig::load(ledger_path) {
+        Ok(genesis_config) => Ok(genesis_config),
+        Err(load_err) => {
             warn!(
-                "Failed to open ledger genesis_config at {:?}: {}, {}",
-                ledger_path, load_err, unpack_err,
+                "Failed to load genesis_config at {ledger_path:?}: {load_err}. \
+                Will attempt to unpack genesis archive and then retry loading."
             );
-            std::process::exit(1);
-        });
 
-        // loading must succeed at this moment
-        GenesisConfig::load(ledger_path).unwrap()
-    })
+            let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
+            unpack_genesis_archive(
+                &genesis_package,
+                ledger_path,
+                max_genesis_archive_unpacked_size,
+            )?;
+            GenesisConfig::load(ledger_path).map_err(OpenGenesisConfigError::Load)
+        }
+    }
 }
 
 pub fn unpack_genesis_archive(

accounts-db/src/tiered_storage.rs

@@ -19,13 +19,14 @@ use {
         storable_accounts::StorableAccounts,
     },
     error::TieredStorageError,
-    footer::{AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat},
+    footer::{AccountBlockFormat, AccountMetaFormat},
     index::IndexBlockFormat,
+    owners::OwnersBlockFormat,
     readable::TieredStorageReader,
     solana_sdk::account::ReadableAccount,
     std::{
         borrow::Borrow,
-        fs::OpenOptions,
+        fs::{self, OpenOptions},
         path::{Path, PathBuf},
         sync::OnceLock,
     },
@@ -53,8 +54,11 @@ pub struct TieredStorage {
 
 impl Drop for TieredStorage {
     fn drop(&mut self) {
-        if let Err(err) = fs_err::remove_file(&self.path) {
-            panic!("TieredStorage failed to remove backing storage file: {err}");
+        if let Err(err) = fs::remove_file(&self.path) {
+            panic!(
+                "TieredStorage failed to remove backing storage file '{}': {err}",
+                self.path.display(),
+            );
         }
     }
 }

accounts-db/src/tiered_storage/footer.rs

@@ -4,6 +4,7 @@ use {
         file::TieredStorageFile,
         index::IndexBlockFormat,
         mmap_utils::{get_pod, get_type},
+        owners::OwnersBlockFormat,
         TieredStorageResult,
     },
     bytemuck::{Pod, Zeroable},
@@ -78,23 +79,6 @@ pub enum AccountBlockFormat {
     Lz4 = 1,
 }
 
-#[repr(u16)]
-#[derive(
-    Clone,
-    Copy,
-    Debug,
-    Default,
-    Eq,
-    Hash,
-    PartialEq,
-    num_enum::IntoPrimitive,
-    num_enum::TryFromPrimitive,
-)]
-pub enum OwnersBlockFormat {
-    #[default]
-    LocalIndex = 0,
-}
-
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 #[repr(C)]
 pub struct TieredStorageFooter {
@@ -353,7 +337,7 @@ mod tests {
         let path = get_append_vec_path("test_file_footer");
         let expected_footer = TieredStorageFooter {
             account_meta_format: AccountMetaFormat::Hot,
-            owners_block_format: OwnersBlockFormat::LocalIndex,
+            owners_block_format: OwnersBlockFormat::AddressesOnly,
             index_block_format: IndexBlockFormat::AddressesThenOffsets,
             account_block_format: AccountBlockFormat::AlignedRaw,
             account_entry_count: 300,

accounts-db/src/tiered_storage/hot.rs

@@ -8,13 +8,11 @@ use {
         tiered_storage::{
             byte_block,
             file::TieredStorageFile,
-            footer::{
-                AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter,
-            },
+            footer::{AccountBlockFormat, AccountMetaFormat, TieredStorageFooter},
             index::{AccountOffset, IndexBlockFormat, IndexOffset},
             meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
             mmap_utils::{get_pod, get_slice},
-            owners::{OwnerOffset, OwnersBlock},
+            owners::{OwnerOffset, OwnersBlockFormat},
             readable::TieredReadableAccount,
             TieredStorageError, TieredStorageFormat, TieredStorageResult,
         },
@@ -29,7 +27,7 @@ use {
 pub const HOT_FORMAT: TieredStorageFormat = TieredStorageFormat {
     meta_entry_size: std::mem::size_of::<HotAccountMeta>(),
     account_meta_format: AccountMetaFormat::Hot,
-    owners_block_format: OwnersBlockFormat::LocalIndex,
+    owners_block_format: OwnersBlockFormat::AddressesOnly,
     index_block_format: IndexBlockFormat::AddressesThenOffsets,
     account_block_format: AccountBlockFormat::AlignedRaw,
 };
@@ -331,7 +329,9 @@ impl HotStorageReader {
     /// Returns the address of the account owner given the specified
     /// owner_offset.
     fn get_owner_address(&self, owner_offset: OwnerOffset) -> TieredStorageResult<&Pubkey> {
-        OwnersBlock::get_owner_address(&self.mmap, &self.footer, owner_offset)
+        self.footer
+            .owners_block_format
+            .get_owner_address(&self.mmap, &self.footer, owner_offset)
     }
 
     /// Returns Ok(index_of_matching_owner) if the account owner at
@@ -466,13 +466,11 @@ pub mod tests {
         crate::tiered_storage::{
             byte_block::ByteBlockWriter,
             file::TieredStorageFile,
-            footer::{
-                AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter,
-                FOOTER_SIZE,
-            },
+            footer::{AccountBlockFormat, AccountMetaFormat, TieredStorageFooter, FOOTER_SIZE},
             hot::{HotAccountMeta, HotStorageReader},
             index::{AccountIndexWriterEntry, IndexBlockFormat, IndexOffset},
             meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
+            owners::OwnersBlockFormat,
         },
         assert_matches::assert_matches,
         memoffset::offset_of,
@@ -638,7 +636,7 @@ pub mod tests {
         let path = temp_dir.path().join("test_hot_storage_footer");
         let expected_footer = TieredStorageFooter {
             account_meta_format: AccountMetaFormat::Hot,
-            owners_block_format: OwnersBlockFormat::LocalIndex,
+            owners_block_format: OwnersBlockFormat::AddressesOnly,
             index_block_format: IndexBlockFormat::AddressesThenOffsets,
             account_block_format: AccountBlockFormat::AlignedRaw,
             account_entry_count: 300,
@@ -825,7 +823,10 @@ pub mod tests {
         {
             let file = TieredStorageFile::new_writable(&path).unwrap();
 
-            OwnersBlock::write_owners_block(&file, &addresses).unwrap();
+            footer
+                .owners_block_format
+                .write_owners_block(&file, &addresses)
+                .unwrap();
 
             // while the test only focuses on account metas, writing a footer
             // here is necessary to make it a valid tiered-storage file.
@@ -892,7 +893,10 @@ pub mod tests {
             // the owners_block_offset set to the end of the accounts blocks.
             footer.owners_block_offset = footer.index_block_offset;
 
-            OwnersBlock::write_owners_block(&file, &owner_addresses).unwrap();
+            footer
+                .owners_block_format
+                .write_owners_block(&file, &owner_addresses)
+                .unwrap();
 
             // while the test only focuses on account metas, writing a footer
             // here is necessary to make it a valid tiered-storage file.
@@ -1025,7 +1029,10 @@ pub mod tests {
 
             // write owners block
             footer.owners_block_offset = current_offset as u64;
-            OwnersBlock::write_owners_block(&file, &owners).unwrap();
+            footer
+                .owners_block_format
+                .write_owners_block(&file, &owners)
+                .unwrap();
 
             footer.write_footer_block(&file).unwrap();
         }