feat(provider): Add StorageProvider impl, table changes (paradigmxyz#172

)

* feat(provider): Add StorageProvider impl, table changes

* unwind header numbers by walker (paradigmxyz#174)

* readme, fmt

* fix tests

* Update crates/interfaces/src/provider/db_provider/storage.rs

Co-authored-by: Roman Krasiuk <[email protected]>

* Update crates/interfaces/src/provider/db_provider/storage.rs

Co-authored-by: Roman Krasiuk <[email protected]>

Co-authored-by: Roman Krasiuk <[email protected]>

README.md

@@ -8,7 +8,7 @@
 
 ## Build
 
-To build this project we are currently using Rust nightly for GAT support, that is planed to release in rust 1.65 (4th Nov 2022). GAT's are used for the `Database` trait in `reth-interface`. 
+Rust minimum required version to build this project is 1.65.0 published 02.11.2022
 
 ## Docs
 

crates/common/rlp/src/decode.rs

@@ -42,7 +42,7 @@ mod alloc_impl {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum DecodeError {
     Overflow,
     LeadingZero,

crates/db/src/kv/cursor.rs

@@ -45,10 +45,6 @@ impl<'tx, K: TransactionKind, T: Table> DbCursorRO<'tx, T> for Cursor<'tx, K, T>
         decode!(self.inner.first())
     }
 
-    fn seek(&mut self, key: <T as Table>::SeekKey) -> reth_interfaces::db::PairResult<T> {
-        decode!(self.inner.set_range(key.encode().as_ref()))
-    }
-
     fn seek_exact(&mut self, key: <T as Table>::Key) -> reth_interfaces::db::PairResult<T> {
         decode!(self.inner.set_key(key.encode().as_ref()))
     }
@@ -87,6 +83,10 @@ impl<'tx, K: TransactionKind, T: Table> DbCursorRO<'tx, T> for Cursor<'tx, K, T>
 }
 
 impl<'tx, K: TransactionKind, T: DupSort> DbDupCursorRO<'tx, T> for Cursor<'tx, K, T> {
+    fn seek(&mut self, key: <T as DupSort>::SubKey) -> reth_interfaces::db::PairResult<T> {
+        decode!(self.inner.set_range(key.encode().as_ref()))
+    }
+
     /// Returns the next `(key, value)` pair of a DUPSORT table.
     fn next_dup(&mut self) -> PairResult<T> {
         decode!(self.inner.next_dup())

crates/db/src/kv/mod.rs

@@ -138,10 +138,13 @@ pub mod test_utils {
 #[cfg(test)]
 mod tests {
     use super::{test_utils, Env, EnvKind};
-    use reth_interfaces::db::{
-        models::ShardedKey,
-        tables::{AccountHistory, Headers, PlainAccountState, PlainStorageState},
-        Database, DbCursorRO, DbDupCursorRO, DbTx, DbTxMut,
+    use reth_interfaces::{
+        db::{
+            models::ShardedKey,
+            tables::{AccountHistory, Headers, PlainAccountState, PlainStorageState},
+            Database, DbCursorRO, DbDupCursorRO, DbTx, DbTxMut,
+        },
+        provider::{ProviderImpl, StateProviderFactory},
     };
     use reth_libmdbx::{NoWriteMap, WriteMap};
     use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, H256, U256};
@@ -315,6 +318,13 @@ mod tests {
             assert_eq!(list200, list);
         }
     }
+
+    #[test]
+    fn common_history_provider() {
+        let db = test_utils::create_test_db::<WriteMap>(EnvKind::RW);
+        let provider = ProviderImpl::new(db);
+        let _ = provider.latest();
+    }
 }
 
 #[cfg(test)]

crates/db/src/kv/tx.rs

@@ -55,12 +55,12 @@ impl<'a, K: TransactionKind, E: EnvironmentKind> DbTxMutGAT<'a> for Tx<'_, K, E>
 
 impl<'tx, K: TransactionKind, E: EnvironmentKind> DbTx<'tx> for Tx<'tx, K, E> {
     // Iterate over read only values in database.
-    fn cursor<T: Table>(&'tx self) -> Result<<Self as DbTxGAT<'tx>>::Cursor<T>, Error> {
+    fn cursor<T: Table>(&self) -> Result<<Self as DbTxGAT<'_>>::Cursor<T>, Error> {
         self.new_cursor()
     }
 
     /// Iterate over read only values in database.
-    fn cursor_dup<T: DupSort>(&'tx self) -> Result<<Self as DbTxGAT<'tx>>::DupCursor<T>, Error> {
+    fn cursor_dup<T: DupSort>(&self) -> Result<<Self as DbTxGAT<'_>>::DupCursor<T>, Error> {
         self.new_cursor()
     }
 

crates/executor/src/executor.rs

@@ -2,7 +2,10 @@ use crate::{
     revm_wrap::{self, State, SubState},
     Config,
 };
-use reth_interfaces::executor::{BlockExecutor, Error, ExecutorDb};
+use reth_interfaces::{
+    executor::{BlockExecutor, Error},
+    provider::StateProvider,
+};
 use reth_primitives::BlockLocked;
 use revm::{AnalysisKind, SpecId, EVM};
 
@@ -19,7 +22,7 @@ impl Executor {
     }
 
     /// Verify block. Execute all transaction and compare results.
-    pub fn verify<DB: ExecutorDb>(&self, block: &BlockLocked, db: DB) -> Result<(), Error> {
+    pub fn verify<DB: StateProvider>(&self, block: &BlockLocked, db: DB) -> Result<(), Error> {
         let db = SubState::new(State::new(db));
         let mut evm = EVM::new();
         evm.database(db);

crates/executor/src/revm_wrap.rs

@@ -1,18 +1,17 @@
-use reth_interfaces::executor::ExecutorDb;
+use reth_interfaces::{provider::StateProvider, Error};
 use reth_primitives::{BlockLocked, Transaction, TransactionKind, H160, H256, U256};
 use revm::{
     db::{CacheDB, DatabaseRef},
     BlockEnv, TransactTo, TxEnv,
 };
-use std::convert::Infallible;
 
 /// SubState of database. Uses revm internal cache with binding to reth DbExecutor trait.
 pub type SubState<DB> = CacheDB<State<DB>>;
 
 /// Wrapper around ExeuctorDb that implements revm database trait
-pub struct State<DB: ExecutorDb>(DB);
+pub struct State<DB: StateProvider>(DB);
 
-impl<DB: ExecutorDb> State<DB> {
+impl<DB: StateProvider> State<DB> {
     /// Create new State with generic ExecutorDb.
     pub fn new(db: DB) -> Self {
         Self(db)
@@ -34,11 +33,11 @@ impl<DB: ExecutorDb> State<DB> {
     }
 }
 
-impl<DB: ExecutorDb> DatabaseRef for State<DB> {
-    type Error = Infallible;
+impl<DB: StateProvider> DatabaseRef for State<DB> {
+    type Error = Error;
 
     fn basic(&self, address: H160) -> Result<Option<revm::AccountInfo>, Self::Error> {
-        Ok(self.0.basic_account(address).map(|account| revm::AccountInfo {
+        Ok(self.0.basic_account(address)?.map(|account| revm::AccountInfo {
             balance: account.balance,
             nonce: account.nonce,
             code_hash: account.bytecode_hash,
@@ -47,19 +46,19 @@ impl<DB: ExecutorDb> DatabaseRef for State<DB> {
     }
 
     fn code_by_hash(&self, code_hash: H256) -> Result<revm::Bytecode, Self::Error> {
-        let (bytecode, size) = self.0.bytecode_by_hash(code_hash).unwrap_or_default();
-        Ok(unsafe { revm::Bytecode::new_checked(bytecode.0, size, Some(code_hash)) })
+        let bytecode = self.0.bytecode_by_hash(code_hash)?.unwrap_or_default();
+        Ok(revm::Bytecode::new_raw(bytecode.0))
     }
 
     fn storage(&self, address: H160, index: U256) -> Result<U256, Self::Error> {
         let mut h_index = H256::zero();
         index.to_big_endian(h_index.as_bytes_mut());
 
-        Ok(U256::from_big_endian(self.0.storage(address, h_index).unwrap_or_default().as_ref()))
+        Ok(self.0.storage(address, h_index)?.unwrap_or_default())
     }
 
     fn block_hash(&self, number: U256) -> Result<H256, Self::Error> {
-        Ok(self.0.block_hash(number).unwrap_or_default())
+        Ok(self.0.block_hash(number)?.unwrap_or_default())
     }
 }
 

crates/interfaces/src/db/mock.rs

@@ -107,10 +107,6 @@ impl<'tx, T: Table> DbCursorRO<'tx, T> for CursorMock {
         todo!()
     }
 
-    fn seek(&mut self, _key: T::SeekKey) -> super::PairResult<T> {
-        todo!()
-    }
-
     fn seek_exact(&mut self, _key: T::Key) -> super::PairResult<T> {
         todo!()
     }
@@ -147,6 +143,10 @@ impl<'tx, T: DupSort> DbDupCursorRO<'tx, T> for CursorMock {
         todo!()
     }
 
+    fn seek(&mut self, _key: T::SubKey) -> super::PairResult<T> {
+        todo!()
+    }
+
     fn next_no_dup(&mut self) -> super::PairResult<T> {
         todo!()
     }

crates/interfaces/src/db/mod.rs

@@ -101,9 +101,9 @@ pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> {
     /// freeing of memory pages
     fn commit(self) -> Result<bool, Error>;
     /// Iterate over read only values in table.
-    fn cursor<T: Table>(&'tx self) -> Result<<Self as DbTxGAT<'tx>>::Cursor<T>, Error>;
+    fn cursor<T: Table>(&self) -> Result<<Self as DbTxGAT<'_>>::Cursor<T>, Error>;
     /// Iterate over read only values in dup sorted table.
-    fn cursor_dup<T: DupSort>(&'tx self) -> Result<<Self as DbTxGAT<'tx>>::DupCursor<T>, Error>;
+    fn cursor_dup<T: DupSort>(&self) -> Result<<Self as DbTxGAT<'_>>::DupCursor<T>, Error>;
 }
 
 /// Read write transaction that allows writing to database
@@ -115,11 +115,11 @@ pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> {
     /// Clears database.
     fn clear<T: Table>(&self) -> Result<(), Error>;
     /// Cursor mut
-    fn cursor_mut<T: Table>(&self) -> Result<<Self as DbTxMutGAT<'tx>>::CursorMut<T>, Error>;
+    fn cursor_mut<T: Table>(&self) -> Result<<Self as DbTxMutGAT<'_>>::CursorMut<T>, Error>;
     /// DupCursor mut.
     fn cursor_dup_mut<T: DupSort>(
         &self,
-    ) -> Result<<Self as DbTxMutGAT<'tx>>::DupCursorMut<T>, Error>;
+    ) -> Result<<Self as DbTxMutGAT<'_>>::DupCursorMut<T>, Error>;
 }
 
 /// Alias type for a `(key, value)` result coming from a cursor.
@@ -134,9 +134,6 @@ pub trait DbCursorRO<'tx, T: Table> {
     /// First item in table
     fn first(&mut self) -> PairResult<T>;
 
-    /// Seeks for a `(key, value)` pair greater or equal than `key`.
-    fn seek(&mut self, key: T::SeekKey) -> PairResult<T>;
-
     /// Seeks for the exact `(key, value)` pair with `key`.
     fn seek_exact(&mut self, key: T::Key) -> PairResult<T>;
 
@@ -164,6 +161,9 @@ pub trait DbCursorRO<'tx, T: Table> {
 
 /// Read only curor over DupSort table.
 pub trait DbDupCursorRO<'tx, T: DupSort> {
+    /// Seeks for a `(key, value)` pair greater or equal than `key`.
+    fn seek(&mut self, key: T::SubKey) -> PairResult<T>;
+
     /// Returns the next `(key, value)` pair of a DUPSORT table.
     fn next_dup(&mut self) -> PairResult<T>;
 

crates/interfaces/src/db/models/sharded_key.rs

@@ -12,7 +12,7 @@ use reth_primitives::TxNumber;
 /// `Address | 200` -> data is from transaction 0 to 200.
 ///
 /// `Address | 300` -> data is from transaction 201 to 300.
-#[derive(Debug, Default, Clone, PartialEq)]
+#[derive(Debug, Default, Clone, Eq, PartialEq)]
 pub struct ShardedKey<T> {
     /// The key for this type.
     pub key: T,

crates/interfaces/src/db/table.rs

@@ -62,8 +62,6 @@ pub trait Table: Send + Sync + Debug + 'static {
     type Key: Key;
     /// Value element of `Table`.
     type Value: Value;
-    /// Seek Key element of `Table`.
-    type SeekKey: Key;
 }
 
 /// DupSort allows for keys not to be repeated in the database,

crates/interfaces/src/db/tables.rs

@@ -9,7 +9,8 @@ use crate::db::{
     DupSort,
 };
 use reth_primitives::{
-    Account, Address, BlockNumber, Header, IntegerList, Receipt, StorageEntry, TxNumber, H256,
+    Account, Address, BlockHash, BlockNumber, Header, IntegerList, Receipt, StorageEntry, TxNumber,
+    H256,
 };
 
 /// Enum for the type of table present in libmdbx.
@@ -22,7 +23,7 @@ pub enum TableType {
 }
 
 /// Default tables that should be present inside database.
-pub const TABLES: [(TableType, &str); 19] = [
+pub const TABLES: [(TableType, &str); 20] = [
     (TableType::Table, CanonicalHeaders::const_name()),
     (TableType::Table, HeaderTD::const_name()),
     (TableType::Table, HeaderNumbers::const_name()),
@@ -35,6 +36,7 @@ pub const TABLES: [(TableType, &str); 19] = [
     (TableType::Table, Logs::const_name()),
     (TableType::Table, PlainAccountState::const_name()),
     (TableType::DupSort, PlainStorageState::const_name()),
+    (TableType::Table, Bytecodes::const_name()),
     (TableType::Table, AccountHistory::const_name()),
     (TableType::Table, StorageHistory::const_name()),
     (TableType::DupSort, AccountChangeSet::const_name()),
@@ -59,7 +61,6 @@ macro_rules! table {
             const NAME: &'static str = $name::const_name();
             type Key = $key;
             type Value = $value;
-            type SeekKey = $seek;
         }
 
         impl $name {
@@ -89,7 +90,7 @@ macro_rules! dupsort {
             $(#[$docs])+
             ///
             #[doc = concat!("`DUPSORT` table with subkey being: [`", stringify!($subkey), "`].")]
-            $name => $key => $value
+            $name => $key => $value => $subkey
         );
         impl DupSort for $name {
             type SubKey = $subkey;
@@ -111,7 +112,7 @@ table!(
 
 table!(
     /// Stores the block number corresponding to an header.
-    HeaderNumbers => BlockNumHash => BlockNumber);
+    HeaderNumbers => BlockHash => BlockNumber);
 
 table!(
     /// Stores header bodies.
@@ -145,6 +146,10 @@ table!(
     /// Stores the current state of an Account.
     PlainAccountState => Address => Account);
 
+table!(
+    /// Stores all smart contract bytecodes.
+    Bytecodes => H256 => Bytecode);
+
 dupsort!(
     /// Stores the current value of a storage key.
     PlainStorageState => Address => [H256] StorageEntry);
@@ -156,10 +161,10 @@ table!(
     /// use reth_primitives::{Address, IntegerList};
     /// use reth_interfaces::db::{DbTx, DbTxMut, DbCursorRO, Database, models::ShardedKey, tables::AccountHistory};
     /// use reth_db::{kv::{EnvKind, Env, test_utils}, mdbx::WriteMap};
-    /// use std::str::FromStr;
+    /// use std::{str::FromStr,sync::Arc};
     /// 
     /// fn main() {
-    ///     let db: Env<WriteMap> = test_utils::create_test_db(EnvKind::RW);
+    ///     let db: Arc<Env<WriteMap>> = test_utils::create_test_db(EnvKind::RW);
     ///     let account = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047").unwrap();
     /// 
     ///     // Setup if each shard can only take 1 transaction.
@@ -242,3 +247,5 @@ pub type RlpTotalDifficulty = Vec<u8>;
 pub type RlpTxBody = Vec<u8>;
 /// Temporary placeholder type for DB.
 pub type AddressStorageKey = Vec<u8>;
+/// Temporary placeholder type for DB.
+pub type Bytecode = Vec<u8>;

crates/interfaces/src/error.rs

@@ -13,4 +13,7 @@ pub enum Error {
 
     #[error(transparent)]
     Database(#[from] crate::db::Error),
+
+    #[error(transparent)]
+    Provider(#[from] crate::provider::Error),
 }

crates/interfaces/src/executor.rs

@@ -1,7 +1,5 @@
 use async_trait::async_trait;
-use reth_primitives::{
-    Account, Address, Block, BlockNumber, Bytes, StorageKey, StorageValue, H256, U256,
-};
+use reth_primitives::Block;
 use thiserror::Error;
 
 /// Takes block and executes it, returns error
@@ -20,21 +18,3 @@ pub enum Error {
     #[error("Example of error.")]
     VerificationFailed,
 }
-
-/// Function needed for executor
-pub trait ExecutorDb {
-    /// Get Block by BlockNumber.
-    fn block(&self, _height: BlockNumber) -> Option<Block>;
-
-    /// Get storage.
-    fn storage(&self, account: Address, storage_key: StorageKey) -> Option<StorageValue>;
-
-    /// Get basic account information.
-    fn basic_account(&self, adderss: Address) -> Option<Account>;
-
-    /// Get account code by its hash
-    fn bytecode_by_hash(&self, code_hash: H256) -> Option<(Bytes, usize)>;
-
-    /// Get block hash by number.
-    fn block_hash(&self, number: U256) -> Option<H256>;
-}