Skip to content

Commit

Permalink
Fix and optimize IndexedDbBlockHeaderStorage
Browse files Browse the repository at this point in the history 
* Make `height: BeBigUint` instead of `u64`
* Fix `BlockHeaderStorageTable::TICKER_HEIGHT_INDEX` index
  • Loading branch information
@sergeyboyko0791
sergeyboyko0791 committed Feb 24, 2023
1 parent b4acf2c commit 4deacf5
Show file tree
Hide file tree
Showing 6 changed files with 178 additions and 65 deletions.
7 changes: 3 additions & 4 deletions mm2src/coins/utxo/utxo_block_header_storage/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ impl BlockHeaderStorage {
})
}

#[cfg(test)]
pub(crate) fn into_inner(self) -> Box<dyn BlockHeaderStorageOps> { self.inner }
}

Expand Down Expand Up @@ -108,7 +107,7 @@ impl BlockHeaderStorageOps for BlockHeaderStorage {
async fn is_table_empty(&self) -> Result<(), BlockHeaderStorageError> { self.inner.is_table_empty().await }
}

#[cfg(test)]
#[cfg(any(test, target_arch = "wasm32"))]
mod block_headers_storage_tests {
use super::*;
use chain::BlockHeaderBits;
Expand Down Expand Up @@ -313,7 +312,7 @@ mod native_tests {
fn test_remove_headers_up_to_height() { block_on(test_remove_headers_up_to_height_impl(FOR_COIN_GET)) }
}

#[cfg(all(test, target_arch = "wasm32"))]
#[cfg(target_arch = "wasm32")]
mod wasm_test {
use super::*;
use crate::utxo::utxo_block_header_storage::block_headers_storage_tests::*;
Expand All @@ -323,7 +322,7 @@ mod wasm_test {

wasm_bindgen_test_configure!(run_in_browser);

const FOR_COIN: &str = "RICK";
const FOR_COIN: &str = "tBTC";

#[wasm_bindgen_test]
async fn test_storage_init() {
Expand Down
8 changes: 4 additions & 4 deletions mm2src/coins/utxo/utxo_block_header_storage/wasm/block_header_table.rs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
use mm2_db::indexed_db::{DbUpgrader, OnUpgradeResult, TableSignature};
use mm2_db::indexed_db::{BeBigUint, DbUpgrader, OnUpgradeResult, TableSignature};

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct BlockHeaderStorageTable {
pub height: u64,
pub height: BeBigUint,
pub bits: u32,
pub hash: String,
pub raw_header: String,
pub ticker: String,
}

impl BlockHeaderStorageTable {
pub const HEIGHT_TICKER_INDEX: &str = "block_height_ticker_index";
pub const TICKER_HEIGHT_INDEX: &str = "block_height_ticker_index";
pub const HASH_TICKER_INDEX: &str = "block_hash_ticker_index";
}

Expand All @@ -21,7 +21,7 @@ impl TableSignature for BlockHeaderStorageTable {
match (old_version, new_version) {
(0, 1) => {
let table = upgrader.create_table(Self::table_name())?;
table.create_multi_index(Self::HEIGHT_TICKER_INDEX, &["height", "ticker"], true)?;
table.create_multi_index(Self::TICKER_HEIGHT_INDEX, &["ticker", "height"], true)?;
table.create_multi_index(Self::HASH_TICKER_INDEX, &["hash", "ticker"], true)?;
table.create_index("ticker", false)?;
},
Expand Down
95 changes: 58 additions & 37 deletions mm2src/coins/utxo/utxo_block_header_storage/wasm/indexeddb_block_header_storage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ use super::BlockHeaderStorageTable;
use async_trait::async_trait;
use chain::BlockHeader;
use mm2_core::mm_ctx::MmArc;
use mm2_db::indexed_db::{ConstructibleDb, DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder,
use mm2_db::indexed_db::{BeBigUint, ConstructibleDb, DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder,
InitDbResult, MultiIndex, SharedDb};
use mm2_err_handle::prelude::*;
use num_traits::ToPrimitive;
use primitives::hash::H256;
use serialization::Reader;
use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps};
Expand Down Expand Up @@ -92,15 +93,15 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
let bits: u32 = header.bits.into();
let headers_to_store = BlockHeaderStorageTable {
ticker: ticker.clone(),
height,
height: BeBigUint::from(height),
bits,
hash,
raw_header,
};
let index_keys = MultiIndex::new(BlockHeaderStorageTable::HEIGHT_TICKER_INDEX)
.with_value(&height)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
let index_keys = MultiIndex::new(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
.with_value(&BeBigUint::from(height))
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

block_headers_db
Expand Down Expand Up @@ -148,10 +149,10 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.table::<BlockHeaderStorageTable>()
.await
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;
let index_keys = MultiIndex::new(BlockHeaderStorageTable::HEIGHT_TICKER_INDEX)
.with_value(&height)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
let index_keys = MultiIndex::new(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
.with_value(&BeBigUint::from(height))
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

Ok(block_headers_db
Expand All @@ -177,22 +178,31 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.await
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

// Todo: use open_cursor with direction to optimze this process.
let res = block_headers_db
let maybe_item = block_headers_db
.cursor_builder()
.only("ticker", ticker.clone())
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.open_cursor("ticker")
// We need to provide any constraint on the `height` property
// since `ticker_height` consists of both `ticker` and `height` properties.
.bound("height", BeBigUint::from(0u64), BeBigUint::from(u64::MAX))
// Cursor returns values from the lowest to highest key indexes.
// But we need to get the most highest height, so reverse the cursor direction.
.reverse()
.open_cursor(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.collect()
.next()
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.into_iter()
.map(|(_item_id, item)| item.height)
.collect::<Vec<_>>();
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?;

Ok(res.into_iter().max())
if let Some((_item_id, item)) = maybe_item {
let height = item
.height
.to_u64()
.ok_or_else(|| BlockHeaderStorageError::get_err(&ticker, "height is too large".to_string()))?;
return Ok(Some(height));
}
Ok(None)
}

async fn get_last_block_header_with_non_max_bits(
Expand All @@ -214,26 +224,29 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.await
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

// Todo: use open_cursor with direction to optimze this process.
let res = block_headers_db
let mut cursor = block_headers_db
.cursor_builder()
.only("ticker", ticker.clone())
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.open_cursor("ticker")
// We need to provide any constraint on the `height` property
// since `ticker_height` consists of both `ticker` and `height` properties.
.bound("height", BeBigUint::from(0u64), BeBigUint::from(u64::MAX))
// Cursor returns values from the lowest to highest key indexes.
// But we need to get the most highest height, so reverse the cursor direction.
.reverse()
.open_cursor(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.collect()
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?;

while let Some((_item_id, header)) = cursor
.next()
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.into_iter()
.map(|(_item_id, item)| item)
.collect::<Vec<_>>();
let res = res
.into_iter()
.filter_map(|e| if e.bits != max_bits { Some(e) } else { None })
.collect::<Vec<_>>();

for header in res {
{
if header.bits == max_bits {
continue;
}

let serialized = &hex::decode(header.raw_header).map_err(|e| BlockHeaderStorageError::DecodeError {
coin: ticker.clone(),
reason: e.to_string(),
Expand Down Expand Up @@ -274,11 +287,19 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

Ok(block_headers_db
let maybe_item = block_headers_db
.get_item_by_unique_multi_index(index_keys)
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.map(|raw| raw.1.height as i64))
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?;

if let Some((_item_id, header)) = maybe_item {
let height = header
.height
.to_i64()
.ok_or_else(|| BlockHeaderStorageError::get_err(&ticker, "height is too large".to_string()))?;
return Ok(Some(height));
}
Ok(None)
}

async fn remove_headers_up_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError> {
Expand All @@ -298,10 +319,10 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

for height in 0..=to_height {
let index_keys = MultiIndex::new(BlockHeaderStorageTable::HEIGHT_TICKER_INDEX)
.with_value(&height)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
let index_keys = MultiIndex::new(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
.with_value(&BeBigUint::from(height))
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

block_headers_db
Expand Down
2 changes: 1 addition & 1 deletion mm2src/mm2_db/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,4 @@ serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] }
wasm-bindgen = { version = "0.2.50", features = ["nightly"] }
wasm-bindgen-futures = { version = "0.4.1" }
wasm-bindgen-test = { version = "0.3.2" }
web-sys = { version = "0.3.55", features = ["console", "CloseEvent", "DomException", "ErrorEvent", "IdbDatabase", "IdbCursor", "IdbCursorWithValue", "IdbFactory", "IdbIndex", "IdbIndexParameters", "IdbObjectStore", "IdbObjectStoreParameters", "IdbOpenDbRequest", "IdbKeyRange", "IdbTransaction", "IdbTransactionMode", "IdbVersionChangeEvent", "MessageEvent", "WebSocket"] }
web-sys = { version = "0.3.55", features = ["console", "CloseEvent", "DomException", "ErrorEvent", "IdbDatabase", "IdbCursor", "IdbCursorWithValue", "IdbCursorDirection", "IdbFactory", "IdbIndex", "IdbIndexParameters", "IdbObjectStore", "IdbObjectStoreParameters", "IdbOpenDbRequest", "IdbKeyRange", "IdbTransaction", "IdbTransactionMode", "IdbVersionChangeEvent", "MessageEvent", "WebSocket"] }
14 changes: 13 additions & 1 deletion mm2src/mm2_db/src/indexed_db/drivers/cursor/cursor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use serde_json::{self as json, Value as Json};
use std::convert::TryInto;
use wasm_bindgen::prelude::*;
use wasm_bindgen::JsCast;
use web_sys::{IdbCursorWithValue, IdbIndex, IdbKeyRange, IdbRequest};
use web_sys::{IdbCursorDirection, IdbCursorWithValue, IdbIndex, IdbKeyRange, IdbRequest};

mod empty_cursor;
mod multi_key_bound_cursor;
Expand Down Expand Up @@ -87,6 +87,7 @@ pub enum CursorBoundValue {
pub struct CursorFilters {
pub(crate) only_keys: Vec<(String, Json)>,
pub(crate) bound_keys: Vec<(String, CursorBoundValue, CursorBoundValue)>,
pub(crate) reverse: bool,
}

impl From<u32> for CursorBoundValue {
Expand Down Expand Up @@ -202,10 +203,21 @@ pub(crate) struct CursorDriver {

impl CursorDriver {
pub(crate) fn init_cursor(db_index: IdbIndex, filters: CursorFilters) -> CursorResult<CursorDriver> {
let reverse = filters.reverse;
let inner = IdbCursorEnum::new(filters)?;

let cursor_request_result = match inner.key_range()? {
Some(key_range) if reverse => {
db_index.open_cursor_with_range_and_direction(&key_range, IdbCursorDirection::Prev)
},
Some(key_range) => db_index.open_cursor_with_range(&key_range),
// Please note that `IndexedDb` doesn't allow to open a cursor with a direction
// but without a key range.
None if reverse => {
return MmError::err(CursorError::ErrorOpeningCursor {
description: format!("Direction cannot be specified without a range"),
})
},
None => db_index.open_cursor(),
};
let cursor_request = cursor_request_result.map_err(|e| CursorError::ErrorOpeningCursor {
Expand Down
Loading

0 comments on commit 4deacf5

Please sign in to comment.