From 275530159961788c31e36078fb11e586e7ccc9a9 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Mon, 10 Oct 2022 17:01:32 +0400 Subject: [PATCH] added migration fix --- .../core/src/chain_storage/lmdb_db/lmdb.rs | 48 +++++++ .../core/src/chain_storage/lmdb_db/lmdb_db.rs | 128 ++++++++++++++++-- infrastructure/tari_script/src/lib.rs | 2 +- 3 files changed, 169 insertions(+), 9 deletions(-) diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs index 75e238b088..a1bc2dcd11 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs @@ -445,3 +445,51 @@ pub fn lmdb_clear(txn: &WriteTransaction<'_>, db: &Database) -> Result( + txn: &WriteTransaction<'_>, + db: &Database, + f: F, +) -> Result<(), ChainStorageError> +where + F: Fn(V) -> Option, + V: DeserializeOwned, + R: Serialize, +{ + let mut access = txn.access(); + let mut cursor = txn.cursor(db).map_err(|e| { + error!(target: LOG_TARGET, "Could not get read cursor from lmdb: {:?}", e); + ChainStorageError::AccessError(e.to_string()) + })?; + let iter = CursorIter::new( + MaybeOwned::Borrowed(&mut cursor), + &access, + |c, a| c.first(a), + Cursor::next::<[u8], [u8]>, + )?; + let items = iter + .map(|r| r.map(|(k, v)| (k.to_vec(), v.to_vec()))) + .collect::, _>>()?; + + for (key, val) in items { + // let (key, val) = row?; + let val = deserialize::(&val)?; + if let Some(ret) = f(val) { + let ret_bytes = serialize(&ret)?; + access.put(db, &key, &ret_bytes, put::Flags::empty()).map_err(|e| { + if let lmdb_zero::Error::Code(code) = &e { + if *code == lmdb_zero::error::MAP_FULL { + return ChainStorageError::DbResizeRequired; + } + } + error!( + target: LOG_TARGET, + "Could not replace value in lmdb transaction: {:?}", e + ); + ChainStorageError::AccessError(e.to_string()) + })?; + } + } + Ok(()) +} diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 93539a2583..6527949d39 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -264,6 +264,8 @@ impl LMDBDatabase { _file_lock: Arc::new(file_lock), }; + run_migrations(&db)?; + Ok(db) } @@ -2436,6 +2438,7 @@ enum MetadataKey { HorizonData, DeletedBitmap, BestBlockTimestamp, + MigrationVersion, } impl MetadataKey { @@ -2448,14 +2451,15 @@ impl MetadataKey { impl fmt::Display for MetadataKey { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - MetadataKey::ChainHeight => f.write_str("Current chain height"), - MetadataKey::AccumulatedWork => f.write_str("Total accumulated work"), - MetadataKey::PruningHorizon => f.write_str("Pruning horizon"), - MetadataKey::PrunedHeight => f.write_str("Effective pruned height"), - MetadataKey::BestBlock => f.write_str("Chain tip block hash"), - MetadataKey::HorizonData => f.write_str("Database info"), - MetadataKey::DeletedBitmap => f.write_str("Deleted bitmap"), - MetadataKey::BestBlockTimestamp => f.write_str("Chain tip block timestamp"), + MetadataKey::ChainHeight => write!(f, "Current chain height"), + MetadataKey::AccumulatedWork => write!(f, "Total accumulated work"), + MetadataKey::PruningHorizon => write!(f, "Pruning horizon"), + MetadataKey::PrunedHeight => write!(f, "Effective pruned height"), + MetadataKey::BestBlock => write!(f, "Chain tip block hash"), + MetadataKey::HorizonData => write!(f, "Database info"), + MetadataKey::DeletedBitmap => write!(f, "Deleted bitmap"), + MetadataKey::BestBlockTimestamp => write!(f, "Chain tip block timestamp"), + MetadataKey::MigrationVersion => write!(f, "Migration version"), } } } @@ -2471,6 +2475,7 @@ enum MetadataValue { HorizonData(HorizonData), DeletedBitmap(DeletedBitmap), BestBlockTimestamp(u64), + MigrationVersion(u64), } impl fmt::Display for MetadataValue { @@ -2486,6 +2491,7 @@ impl fmt::Display for MetadataValue { write!(f, "Deleted Bitmap ({} indexes)", deleted.bitmap().cardinality()) }, MetadataValue::BestBlockTimestamp(timestamp) => write!(f, "Chain tip block timestamp is {}", timestamp), + MetadataValue::MigrationVersion(n) => write!(f, "Migration version {}", n), } } } @@ -2579,3 +2585,109 @@ impl CompositeKey { type InputKey = CompositeKey; type KernelKey = CompositeKey; type OutputKey = CompositeKey; + +fn run_migrations(db: &LMDBDatabase) -> Result<(), ChainStorageError> { + const MIGRATION_VERSION: u64 = 1; + let txn = db.read_transaction()?; + + let k = MetadataKey::MigrationVersion; + let val = lmdb_get::<_, MetadataValue>(&*txn, &db.metadata_db, &k.as_u32())?; + let n = match val { + Some(MetadataValue::MigrationVersion(n)) => n, + Some(_) | None => 0, + }; + info!( + target: LOG_TARGET, + "Blockchain database is at v{} (required version: {})", n, MIGRATION_VERSION + ); + drop(txn); + + if n < MIGRATION_VERSION { + tari_script_execution_stack_bug_migration::migrate(db)?; + info!(target: LOG_TARGET, "Migrated database to version {}", MIGRATION_VERSION); + let txn = db.write_transaction()?; + lmdb_replace( + &txn, + &db.metadata_db, + &k.as_u32(), + &MetadataValue::MigrationVersion(MIGRATION_VERSION), + )?; + txn.commit()?; + } + + Ok(()) +} + +// TODO: remove +mod tari_script_execution_stack_bug_migration { + use serde::{Deserialize, Serialize}; + use tari_common_types::types::{ComSignature, PublicKey}; + use tari_crypto::ristretto::{pedersen::PedersenCommitment, RistrettoPublicKey, RistrettoSchnorr}; + use tari_script::{ExecutionStack, HashValue, ScalarValue, StackItem}; + + use super::*; + use crate::{ + chain_storage::lmdb_db::lmdb::lmdb_map_inplace, + transactions::transaction_components::{SpentOutput, TransactionInputVersion}, + }; + + pub fn migrate(db: &LMDBDatabase) -> Result<(), ChainStorageError> { + let txn = db.write_transaction()?; + error!(target: LOG_TARGET, "IN MIGTATE"); + lmdb_map_inplace(&txn, &db.inputs_db, |mut v: TransactionInputRowDataV0| { + error!(target: LOG_TARGET, "HERE {:?}", v); + let mut items = Vec::with_capacity(v.input.input_data.items.len()); + while let Some(item) = v.input.input_data.items.pop() { + error!(target: LOG_TARGET, "in while {:?}", item); + if let StackItemV0::Commitment(ref commitment) = item { + debug!(target: LOG_TARGET, "Migrating item: {:?}", item); + let pk = PublicKey::from_bytes(commitment.as_bytes()).unwrap(); + items.push(StackItem::PublicKey(pk)); + } else { + items.push(unsafe { mem::transmute(item) }); + } + + error!(target: LOG_TARGET, "end of while {}", items.len()); + } + error!(target: LOG_TARGET, "done while ",); + let mut v = unsafe { mem::transmute::<_, TransactionInputRowData>(v) }; + error!(target: LOG_TARGET, "done input transmute ",); + v.input.input_data = ExecutionStack::new(items); + Some(v) + })?; + txn.commit()?; + error!(target: LOG_TARGET, "done map inplace ",); + Ok(()) + } + + #[derive(Debug, Serialize, Deserialize)] + pub(crate) struct TransactionInputRowDataV0 { + pub input: TransactionInputV0, + pub header_hash: HashOutput, + pub mmr_position: u32, + pub hash: HashOutput, + } + + #[derive(Debug, Serialize, Deserialize)] + pub struct TransactionInputV0 { + version: TransactionInputVersion, + spent_output: SpentOutput, + input_data: ExecutionStackV0, + script_signature: ComSignature, + } + + #[derive(Debug, Serialize, Deserialize)] + struct ExecutionStackV0 { + items: Vec, + } + + #[derive(Debug, Serialize, Deserialize)] + enum StackItemV0 { + Number(i64), + Hash(HashValue), + Scalar(ScalarValue), + Commitment(PedersenCommitment), + PublicKey(RistrettoPublicKey), + Signature(RistrettoSchnorr), + } +} diff --git a/infrastructure/tari_script/src/lib.rs b/infrastructure/tari_script/src/lib.rs index e796c55a4d..81ef3d5e7f 100644 --- a/infrastructure/tari_script/src/lib.rs +++ b/infrastructure/tari_script/src/lib.rs @@ -24,7 +24,7 @@ mod serde; mod stack; pub use error::ScriptError; -pub use op_codes::{slice_to_boxed_hash, slice_to_hash, HashValue, Opcode}; +pub use op_codes::{slice_to_boxed_hash, slice_to_hash, HashValue, Message, Opcode, ScalarValue}; pub use script::TariScript; pub use script_commitment::{ScriptCommitment, ScriptCommitmentError, ScriptCommitmentFactory}; pub use script_context::ScriptContext;