fix: revisit 7+ blocks reorg handling (#553)

Co-authored-by: Ludo Galabru <[email protected]>

components/chainhook-cli/src/cli/mod.rs

@@ -8,8 +8,10 @@ use crate::scan::stacks::{
 use crate::service::http_api::document_predicate_api_server;
 use crate::service::Service;
 use crate::storage::{
-    get_last_block_height_inserted, get_stacks_block_at_block_height, is_stacks_block_present,
-    open_readonly_stacks_db_conn,
+    delete_confirmed_entry_from_stacks_blocks, get_last_block_height_inserted,
+    get_last_unconfirmed_block_height_inserted, get_stacks_block_at_block_height,
+    insert_unconfirmed_entry_in_stacks_blocks, is_stacks_block_present,
+    open_readonly_stacks_db_conn, open_readwrite_stacks_db_conn,
 };
 
 use chainhook_sdk::chainhooks::types::{
@@ -217,6 +219,32 @@ enum StacksDbCommand {
     /// Retrieve a block from the Stacks db
     #[clap(name = "get", bin_name = "get")]
     GetBlock(GetBlockDbCommand),
+    /// Deletes a block from the confirmed block db and moves it to the unconfirmed block db.
+    #[clap(name = "unconfirm", bin_name = "unconfirm")]
+    UnconfirmBlock(UnconfirmBlockDbCommand),
+    /// Get latest blocks from the unconfirmed and confirmed block db.
+    #[clap(name = "get-latest", bin_name = "get-latest")]
+    GetLatest(GetLatestBlocksDbCommand),
+}
+
+#[derive(Parser, PartialEq, Clone, Debug)]
+struct GetLatestBlocksDbCommand {
+    /// Load config file path
+    #[clap(long = "config-path")]
+    pub config_path: Option<String>,
+    /// The number of blocks from the chain tip to fetch.
+    #[clap(long = "count")]
+    pub count: u64,
+}
+
+#[derive(Parser, PartialEq, Clone, Debug)]
+struct UnconfirmBlockDbCommand {
+    /// Load config file path
+    #[clap(long = "config-path")]
+    pub config_path: Option<String>,
+    /// The block height to unconfirm
+    #[clap(long = "block-height")]
+    pub block_height: u64,
 }
 
 #[derive(Parser, PartialEq, Clone, Debug)]
@@ -557,6 +585,134 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
             }
         },
         Command::Stacks(subcmd) => match subcmd {
+            StacksCommand::Db(StacksDbCommand::UnconfirmBlock(cmd)) => {
+                let config = Config::default(false, false, false, &cmd.config_path)?;
+                let stacks_db_rw =
+                    open_readwrite_stacks_db_conn(&config.expected_cache_path(), &ctx)
+                        .expect("unable to read stacks_db");
+
+                match get_stacks_block_at_block_height(cmd.block_height, true, 3, &stacks_db_rw) {
+                    Ok(Some(block)) => {
+                        let mut delete_confirmed = false;
+                        let mut insert_unconfirmed = false;
+                        match get_stacks_block_at_block_height(
+                            cmd.block_height,
+                            false,
+                            3,
+                            &stacks_db_rw,
+                        ) {
+                            Ok(Some(_)) => {
+                                warn!(ctx.expect_logger(), "Block {} was found in both the confirmed and unconfirmed database. Deleting from confirmed database.", cmd.block_height);
+                                delete_confirmed = true;
+                            }
+                            Ok(None) => {
+                                info!(ctx.expect_logger(), "Block {} found in confirmed database. Deleting from confirmed database and inserting into unconfirmed.", cmd.block_height);
+                                delete_confirmed = true;
+                                insert_unconfirmed = true;
+                            }
+                            Err(e) => {
+                                error!(
+                                    ctx.expect_logger(),
+                                    "Error making request to database: {e}",
+                                );
+                            }
+                        }
+                        if delete_confirmed {
+                            delete_confirmed_entry_from_stacks_blocks(
+                                &block.block_identifier,
+                                &stacks_db_rw,
+                                &ctx,
+                            )?;
+                        }
+                        if insert_unconfirmed {
+                            insert_unconfirmed_entry_in_stacks_blocks(&block, &stacks_db_rw, &ctx)?;
+                        }
+                    }
+                    Ok(None) => {
+                        warn!(ctx.expect_logger(), "Block {} not present in the confirmed database. No database changes were made by this command.", cmd.block_height);
+                    }
+                    Err(e) => {
+                        error!(ctx.expect_logger(), "Error making request to database: {e}",);
+                    }
+                }
+            }
+            StacksCommand::Db(StacksDbCommand::GetLatest(cmd)) => {
+                let config = Config::default(false, false, false, &cmd.config_path)?;
+                let stacks_db = open_readonly_stacks_db_conn(&config.expected_cache_path(), &ctx)
+                    .expect("unable to read stacks_db");
+
+                match get_last_block_height_inserted(&stacks_db, &ctx) {
+                    Some(confirmed_tip) => {
+                        let min_block = confirmed_tip - cmd.count;
+                        info!(
+                            ctx.expect_logger(),
+                            "Getting confirmed blocks {} through {}", min_block, confirmed_tip
+                        );
+                        let mut confirmed_blocks = vec![];
+                        let mut cursor = confirmed_tip;
+                        while cursor > min_block {
+                            match get_stacks_block_at_block_height(cursor, true, 3, &stacks_db) {
+                                Ok(Some(block)) => {
+                                    confirmed_blocks.push(block.block_identifier.index);
+                                    cursor -= 1;
+                                }
+                                Ok(None) => {
+                                    warn!(
+                                        ctx.expect_logger(),
+                                        "Block {} not present in confirmed database", cursor
+                                    );
+                                }
+                                Err(e) => {
+                                    error!(ctx.expect_logger(), "{e}",);
+                                }
+                            }
+                        }
+                        info!(
+                            ctx.expect_logger(),
+                            "Found confirmed blocks: {:?}", confirmed_blocks
+                        );
+                    }
+                    None => {
+                        warn!(ctx.expect_logger(), "No confirmed blocks found in db");
+                    }
+                };
+
+                match get_last_unconfirmed_block_height_inserted(&stacks_db, &ctx) {
+                    Some(unconfirmed_tip) => {
+                        let min_block = unconfirmed_tip - cmd.count;
+                        info!(
+                            ctx.expect_logger(),
+                            "Getting unconfirmed blocks {} through {}", min_block, unconfirmed_tip
+                        );
+                        let mut confirmed_blocks = vec![];
+                        let mut cursor = unconfirmed_tip;
+                        while cursor > min_block {
+                            match get_stacks_block_at_block_height(cursor, false, 3, &stacks_db) {
+                                Ok(Some(block)) => {
+                                    confirmed_blocks.push(block.block_identifier.index);
+                                    cursor -= 1;
+                                }
+                                Ok(None) => {
+                                    warn!(
+                                        ctx.expect_logger(),
+                                        "Block {} not present in unconfirmed database", cursor
+                                    );
+                                }
+                                Err(e) => {
+                                    error!(ctx.expect_logger(), "{e}",);
+                                }
+                            }
+                        }
+                        info!(
+                            ctx.expect_logger(),
+                            "Found unconfirmed blocks: {:?}", confirmed_blocks
+                        );
+                    }
+                    None => {
+                        warn!(ctx.expect_logger(), "No unconfirmed blocks found in db");
+                    }
+                };
+            }
             StacksCommand::Db(StacksDbCommand::GetBlock(cmd)) => {
                 let config = Config::default(false, false, false, &cmd.config_path)?;
                 let stacks_db = open_readonly_stacks_db_conn(&config.expected_cache_path(), &ctx)

components/chainhook-cli/src/scan/stacks.rs

@@ -523,7 +523,6 @@ pub async fn consolidate_local_stacks_chainstate_using_csv(
         "Building local chainstate from Stacks archive file"
     );
 
-
     let downloaded_new_dataset = download_stacks_dataset_if_required(config, ctx).await?;
 
     if downloaded_new_dataset {
@@ -558,7 +557,6 @@ pub async fn consolidate_local_stacks_chainstate_using_csv(
             ) {
                 Ok(block) => block,
                 Err(e) => {
-
                     error!(
                         &ctx.expect_logger(),
                         "Failed to standardize stacks block: {e}"

components/chainhook-cli/src/storage/mod.rs

@@ -148,12 +148,16 @@ pub fn insert_unconfirmed_entry_in_stacks_blocks(
     stacks_db_rw
         .put(&key, &block_bytes.to_string().as_bytes())
         .map_err(|e| format!("unable to insert blocks: {}", e))?;
-    stacks_db_rw
-        .put(
-            get_last_unconfirmed_insert_key(),
-            block.block_identifier.index.to_be_bytes(),
-        )
-        .map_err(|e| format!("unable to insert metadata: {}", e))?;
+    let previous_last_inserted =
+        get_last_unconfirmed_block_height_inserted(stacks_db_rw, _ctx).unwrap_or(0);
+    if block.block_identifier.index > previous_last_inserted {
+        stacks_db_rw
+            .put(
+                get_last_unconfirmed_insert_key(),
+                block.block_identifier.index.to_be_bytes(),
+            )
+            .map_err(|e| format!("unable to insert metadata: {}", e))?;
+    }
     Ok(())
 }
 
@@ -168,6 +172,17 @@ pub fn delete_unconfirmed_entry_from_stacks_blocks(
         .map_err(|e| format!("unable to delete blocks: {}", e))
 }
 
+pub fn delete_confirmed_entry_from_stacks_blocks(
+    block_identifier: &BlockIdentifier,
+    stacks_db_rw: &DB,
+    _ctx: &Context,
+) -> Result<(), String> {
+    let key = get_block_key(&block_identifier);
+    stacks_db_rw
+        .delete(&key)
+        .map_err(|e| format!("unable to delete blocks: {}", e))
+}
+
 pub fn get_last_unconfirmed_block_height_inserted(stacks_db: &DB, _ctx: &Context) -> Option<u64> {
     stacks_db
         .get(get_last_unconfirmed_insert_key())

components/chainhook-sdk/src/indexer/fork_scratch_pad.rs

@@ -232,7 +232,7 @@ impl ForkScratchPad {
             return;
         }
         // Any block beyond 6th ancestor is considered as confirmed and can be pruned
-        let cut_off = &canonical_segment[5];
+        let cut_off = &canonical_segment[(CONFIRMED_SEGMENT_MINIMUM_LENGTH - 2) as usize];
 
         // Prune forks using the confirmed block
         let mut blocks_to_prune = vec![];

components/chainhook-sdk/src/indexer/mod.rs

@@ -194,10 +194,7 @@ impl Indexer {
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ChainSegment {
-    pub amount_of_btc_spent: u64,
-    pub most_recent_confirmed_block_height: u64,
     pub block_ids: VecDeque<BlockIdentifier>,
-    confirmed_blocks_inbox: Vec<BlockIdentifier>,
 }
 
 #[derive(Clone, Debug)]
@@ -220,22 +217,13 @@ pub struct ChainSegmentDivergence {
 impl ChainSegment {
     pub fn new() -> ChainSegment {
         let block_ids = VecDeque::new();
-        ChainSegment {
-            block_ids,
-            most_recent_confirmed_block_height: 0,
-            confirmed_blocks_inbox: vec![],
-            amount_of_btc_spent: 0,
-        }
+        ChainSegment { block_ids }
     }
 
     fn is_empty(&self) -> bool {
         self.block_ids.is_empty()
     }
 
-    fn is_block_id_older_than_segment(&self, block_identifier: &BlockIdentifier) -> bool {
-        block_identifier.index < self.most_recent_confirmed_block_height
-    }
-
     fn is_block_id_newer_than_segment(&self, block_identifier: &BlockIdentifier) -> bool {
         if let Some(tip) = self.block_ids.front() {
             return block_identifier.index > (tip.index + 1);
@@ -256,10 +244,6 @@ impl ChainSegment {
         block: &dyn AbstractBlock,
         ctx: &Context,
     ) -> Result<(), ChainSegmentIncompatibility> {
-        if self.is_block_id_older_than_segment(&block.get_identifier()) {
-            // Could be facing a deep fork...
-            return Err(ChainSegmentIncompatibility::OutdatedBlock);
-        }
         if self.is_block_id_newer_than_segment(&block.get_identifier()) {
             // Chain segment looks outdated, we should just prune it?
             return Err(ChainSegmentIncompatibility::OutdatedSegment);
@@ -319,8 +303,7 @@ impl ChainSegment {
     }
 
     pub fn get_length(&self) -> u64 {
-        let len: u64 = self.block_ids.len().try_into().unwrap();
-        self.most_recent_confirmed_block_height + len
+        self.block_ids.len().try_into().unwrap()
     }
 
     pub fn keep_blocks_from_oldest_to_block_identifier(