merge bitcoin#21526: UpdateTip/CheckBlockIndex assumeutxo support

src/Makefile.test_util.include

@@ -9,6 +9,7 @@ EXTRA_LIBRARIES += \
 
 TEST_UTIL_H = \
     test/util/blockfilter.h \
+    test/util/chainstate.h \
     test/util/index.h \
     test/util/logging.h \
     test/util/mining.h \

src/chain.h

@@ -124,6 +124,14 @@ enum BlockStatus: uint32_t {
     BLOCK_FAILED_MASK        =   BLOCK_FAILED_VALID | BLOCK_FAILED_CHILD,
 
     BLOCK_CONFLICT_CHAINLOCK =   128, //!< conflicts with chainlock system
+
+    /**
+     * If set, this indicates that the block index entry is assumed-valid.
+     * Certain diagnostics will be skipped in e.g. CheckBlockIndex().
+     * It almost certainly means that the block's full validation is pending
+     * on a background chainstate. See `doc/assumeutxo.md`.
+     */
+    BLOCK_ASSUMED_VALID      =   256,
 };
 
 /** The block chain is a tree shaped structure starting with the
@@ -291,14 +299,24 @@ class CBlockIndex
         return ((nStatus & BLOCK_VALID_MASK) >= nUpTo);
     }
 
+    //! @returns true if the block is assumed-valid; this means it is queued to be
+    //!   validated by a background chainstate.
+    bool IsAssumedValid() const { return nStatus & BLOCK_ASSUMED_VALID; }
+
     //! Raise the validity level of this block index entry.
     //! Returns true if the validity was changed.
     bool RaiseValidity(enum BlockStatus nUpTo)
     {
         assert(!(nUpTo & ~BLOCK_VALID_MASK)); // Only validity flags allowed.
-        if (nStatus & BLOCK_FAILED_MASK)
-            return false;
+        if (nStatus & BLOCK_FAILED_MASK) return false;
+
         if ((nStatus & BLOCK_VALID_MASK) < nUpTo) {
+            // If this block had been marked assumed-valid and we're raising
+            // its validity to a certain point, there is no longer an assumption.
+            if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) {
+                nStatus &= ~BLOCK_ASSUMED_VALID;
+            }
+
             nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo;
             return true;
         }

src/test/evo_deterministicmns_tests.cpp

@@ -251,7 +251,7 @@ void FuncDIP3Activation(TestChainSetup& setup)
     int nHeight = chainman.ActiveChain().Height();
 
     // We start one block before DIP3 activation, so mining a block with a DIP3 transaction should fail
-    auto block = std::make_shared<CBlock>(setup.CreateBlock(txns, setup.coinbaseKey));
+    auto block = std::make_shared<CBlock>(setup.CreateBlock(txns, setup.coinbaseKey, chainman.ActiveChainstate()));
     chainman.ProcessNewBlock(Params(), block, true, nullptr);
     BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight);
     BOOST_ASSERT(block->GetHash() != chainman.ActiveChain().Tip()->GetBlockHash());
@@ -261,7 +261,7 @@ void FuncDIP3Activation(TestChainSetup& setup)
     setup.CreateAndProcessBlock({}, setup.coinbaseKey);
     BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1);
     // Mining a block with a DIP3 transaction should succeed now
-    block = std::make_shared<CBlock>(setup.CreateBlock(txns, setup.coinbaseKey));
+    block = std::make_shared<CBlock>(setup.CreateBlock(txns, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip());
     BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 2);
@@ -288,7 +288,7 @@ void FuncV19Activation(TestChainSetup& setup)
 
     int nHeight = chainman.ActiveChain().Height();
 
-    auto block = std::make_shared<CBlock>(setup.CreateBlock({tx_reg}, setup.coinbaseKey));
+    auto block = std::make_shared<CBlock>(setup.CreateBlock({tx_reg}, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19));
     ++nHeight;
@@ -306,7 +306,7 @@ void FuncV19Activation(TestChainSetup& setup)
     operator_key_new.MakeNewKey();
     auto tx_upreg = CreateProUpRegTx(chainman.ActiveChain(), *(setup.m_node.mempool), utxos, tx_reg_hash, owner_key, operator_key_new.GetPublicKey(), owner_key.GetPubKey().GetID(), collateralScript, setup.coinbaseKey);
 
-    block = std::make_shared<CBlock>(setup.CreateBlock({tx_upreg}, setup.coinbaseKey));
+    block = std::make_shared<CBlock>(setup.CreateBlock({tx_upreg}, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19));
     ++nHeight;
@@ -326,7 +326,7 @@ void FuncV19Activation(TestChainSetup& setup)
     FillableSigningProvider signing_provider;
     signing_provider.AddKeyPubKey(collateral_key, collateral_key.GetPubKey());
     BOOST_ASSERT(SignSignature(signing_provider, CTransaction(tx_reg), tx_spend, 0, SIGHASH_ALL));
-    block = std::make_shared<CBlock>(setup.CreateBlock({tx_spend}, setup.coinbaseKey));
+    block = std::make_shared<CBlock>(setup.CreateBlock({tx_spend}, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     BOOST_ASSERT(!DeploymentActiveAfter(chainman.ActiveChain().Tip(), Params().GetConsensus(), Consensus::DEPLOYMENT_V19));
     ++nHeight;
@@ -614,7 +614,7 @@ void FuncTestMempoolReorg(TestChainSetup& setup)
     FundTransaction(chainman.ActiveChain(), tx_collateral, utxos, scriptCollateral, dmn_types::Regular.collat_amount, setup.coinbaseKey);
     SignTransaction(*(setup.m_node.mempool), tx_collateral, setup.coinbaseKey);
 
-    auto block = std::make_shared<CBlock>(setup.CreateBlock({tx_collateral}, setup.coinbaseKey));
+    auto block = std::make_shared<CBlock>(setup.CreateBlock({tx_collateral}, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     setup.m_node.dmnman->UpdatedBlockTip(chainman.ActiveChain().Tip());
     BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1);
@@ -756,7 +756,7 @@ void FuncVerifyDB(TestChainSetup& setup)
     FundTransaction(chainman.ActiveChain(), tx_collateral, utxos, scriptCollateral, dmn_types::Regular.collat_amount, setup.coinbaseKey);
     SignTransaction(*(setup.m_node.mempool), tx_collateral, setup.coinbaseKey);
 
-    auto block = std::make_shared<CBlock>(setup.CreateBlock({tx_collateral}, setup.coinbaseKey));
+    auto block = std::make_shared<CBlock>(setup.CreateBlock({tx_collateral}, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip());
     BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 1);
@@ -788,7 +788,7 @@ void FuncVerifyDB(TestChainSetup& setup)
 
     auto tx_reg_hash = tx_reg.GetHash();
 
-    block = std::make_shared<CBlock>(setup.CreateBlock({tx_reg}, setup.coinbaseKey));
+    block = std::make_shared<CBlock>(setup.CreateBlock({tx_reg}, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip());
     BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 2);
@@ -800,7 +800,7 @@ void FuncVerifyDB(TestChainSetup& setup)
     collateral_utxos.emplace(payload.collateralOutpoint, std::make_pair(1, 1000));
     auto proUpRevTx = CreateProUpRevTx(chainman.ActiveChain(), *(setup.m_node.mempool), collateral_utxos, tx_reg_hash, operatorKey, collateralKey);
 
-    block = std::make_shared<CBlock>(setup.CreateBlock({proUpRevTx}, setup.coinbaseKey));
+    block = std::make_shared<CBlock>(setup.CreateBlock({proUpRevTx}, setup.coinbaseKey, chainman.ActiveChainstate()));
     BOOST_ASSERT(chainman.ProcessNewBlock(Params(), block, true, nullptr));
     dmnman.UpdatedBlockTip(chainman.ActiveChain().Tip());
     BOOST_CHECK_EQUAL(chainman.ActiveChain().Height(), nHeight + 3);

src/test/util/chainstate.h

@@ -0,0 +1,54 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+//
+#ifndef BITCOIN_TEST_UTIL_CHAINSTATE_H
+#define BITCOIN_TEST_UTIL_CHAINSTATE_H
+
+#include <clientversion.h>
+#include <fs.h>
+#include <node/context.h>
+#include <node/utxo_snapshot.h>
+#include <rpc/blockchain.h>
+#include <validation.h>
+
+#include <univalue.h>
+
+#include <boost/test/unit_test.hpp>
+
+const auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){};
+
+/**
+ * Create and activate a UTXO snapshot, optionally providing a function to
+ * malleate the snapshot.
+ */
+template<typename F = decltype(NoMalleation)>
+static bool
+CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation)
+{
+    // Write out a snapshot to the test's tempdir.
+    //
+    int height;
+    WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight());
+    fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height);
+    FILE* outfile{fsbridge::fopen(snapshot_path, "wb")};
+    CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION};
+
+    UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile);
+    BOOST_TEST_MESSAGE(
+        "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write());
+
+    // Read the written snapshot in and then activate it.
+    //
+    FILE* infile{fsbridge::fopen(snapshot_path, "rb")};
+    CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION};
+    SnapshotMetadata metadata;
+    auto_infile >> metadata;
+
+    malleation(auto_infile, metadata);
+
+    return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true);
+}
+
+
+#endif // BITCOIN_TEST_UTIL_CHAINSTATE_H

src/test/util/setup_common.cpp

@@ -373,28 +373,41 @@ void TestChainSetup::mineBlocks(int num_blocks)
     }
 }
 
-CBlock TestChainSetup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey)
+CBlock TestChainSetup::CreateAndProcessBlock(
+    const std::vector<CMutableTransaction>& txns,
+    const CScript& scriptPubKey,
+    CChainState* chainstate)
 {
+    if (!chainstate) {
+        chainstate = &Assert(m_node.chainman)->ActiveChainstate();
+    }
+
     const CChainParams& chainparams = Params();
-    auto block = CreateBlock(txns, scriptPubKey);
+    auto block = this->CreateBlock(txns, scriptPubKey, *chainstate);
 
     std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block);
     Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr);
 
     return block;
 }
 
-CBlock TestChainSetup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CKey& scriptKey)
+CBlock TestChainSetup::CreateAndProcessBlock(
+    const std::vector<CMutableTransaction>& txns,
+    const CKey& scriptKey,
+    CChainState* chainstate)
 {
     CScript scriptPubKey = CScript() <<  ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG;
-    return CreateAndProcessBlock(txns, scriptPubKey);
+    return CreateAndProcessBlock(txns, scriptPubKey, chainstate);
 }
 
-CBlock TestChainSetup::CreateBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey)
+CBlock TestChainSetup::CreateBlock(
+    const std::vector<CMutableTransaction>& txns,
+    const CScript& scriptPubKey,
+    CChainState& chainstate)
 {
     const CChainParams& chainparams = Params();
     CTxMemPool empty_pool;
-    CBlock block = BlockAssembler(m_node.chainman->ActiveChainstate(), m_node, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block;
+    CBlock block = BlockAssembler(chainstate, m_node, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block;
 
     std::vector<CTransactionRef> llmqCommitments;
     for (const auto& tx : block.vtx) {
@@ -443,10 +456,13 @@ CBlock TestChainSetup::CreateBlock(const std::vector<CMutableTransaction>& txns,
     return result;
 }
 
-CBlock TestChainSetup::CreateBlock(const std::vector<CMutableTransaction>& txns, const CKey& scriptKey)
+CBlock TestChainSetup::CreateBlock(
+    const std::vector<CMutableTransaction>& txns,
+    const CKey& scriptKey,
+    CChainState& chainstate)
 {
     CScript scriptPubKey = CScript() <<  ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG;
-    return CreateBlock(txns, scriptPubKey);
+    return CreateBlock(txns, scriptPubKey, chainstate);
 }
 
 

src/test/util/setup_common.h

@@ -127,15 +127,25 @@ struct TestChainSetup : public RegTestingSetup
     /**
      * Create a new block with just given transactions, coinbase paying to
      * scriptPubKey, and try to add it to the current chain.
+     * If no chainstate is specified, default to the active.
      */
     CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns,
-                                 const CScript& scriptPubKey);
+                                 const CScript& scriptPubKey,
+                                 CChainState* chainstate = nullptr);
     CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns,
-                                 const CKey& scriptKey);
+                                 const CKey& scriptKey,
+                                 CChainState* chainstate = nullptr);
+
+    /**
+     * Create a new block with just given transactions, coinbase paying to
+     * scriptPubKey.
+     */
     CBlock CreateBlock(const std::vector<CMutableTransaction>& txns,
-                       const CScript& scriptPubKey);
+                       const CScript& scriptPubKey,
+                       CChainState& chainstate);
     CBlock CreateBlock(const std::vector<CMutableTransaction>& txns,
-                       const CKey& scriptKey);
+                       const CKey& scriptKey,
+                       CChainState& chainstate);
 
     //! Mine a series of new blocks on the active chain.
     void mineBlocks(int num_blocks);

src/test/validation_chainstate_tests.cpp

@@ -2,6 +2,7 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 //
+#include <chainparams.h>
 #include <consensus/validation.h>
 #include <evo/evodb.h>
 #include <index/txindex.h>
@@ -10,6 +11,8 @@
 #include <llmq/instantsend.h>
 #include <random.h>
 #include <sync.h>
+#include <rpc/blockchain.h>
+#include <test/util/chainstate.h>
 #include <test/util/setup_common.h>
 #include <uint256.h>
 #include <validation.h>
@@ -78,4 +81,77 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches)
     WITH_LOCK(::cs_main, manager.Unload());
 }
 
+//! Test UpdateTip behavior for both active and background chainstates.
+//!
+//! When run on the background chainstate, UpdateTip should do a subset
+//! of what it does for the active chainstate.
+BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup)
+{
+    ChainstateManager& chainman = *Assert(m_node.chainman);
+    uint256 curr_tip = ::g_best_block;
+
+    // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can
+    // be found.
+    mineBlocks(10);
+
+    // After adding some blocks to the tip, best block should have changed.
+    BOOST_CHECK(::g_best_block != curr_tip);
+
+    BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root));
+
+    // Ensure our active chain is the snapshot chainstate.
+    BOOST_CHECK(chainman.IsSnapshotActive());
+
+    curr_tip = ::g_best_block;
+
+    // Mine a new block on top of the activated snapshot chainstate.
+    mineBlocks(1);  // Defined in TestChain100Setup.
+
+    // After adding some blocks to the snapshot tip, best block should have changed.
+    BOOST_CHECK(::g_best_block != curr_tip);
+
+    curr_tip = ::g_best_block;
+
+    CChainState* background_cs;
+
+    BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2);
+    for (CChainState* cs : chainman.GetAll()) {
+        if (cs != &chainman.ActiveChainstate()) {
+            background_cs = cs;
+        }
+    }
+    BOOST_CHECK(background_cs);
+
+    // Create a block to append to the validation chain.
+    std::vector<CMutableTransaction> noTxns;
+    CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG;
+    CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, *background_cs);
+    auto pblock = std::make_shared<const CBlock>(validation_block);
+    BlockValidationState state;
+    CBlockIndex* pindex = nullptr;
+    const CChainParams& chainparams = Params();
+    bool newblock = false;
+
+    // TODO: much of this is inlined from ProcessNewBlock(); just reuse PNB()
+    // once it is changed to support multiple chainstates.
+    {
+        LOCK(::cs_main);
+        bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus());
+        BOOST_CHECK(checked);
+        bool accepted = background_cs->AcceptBlock(
+            pblock, state, &pindex, true, nullptr, &newblock);
+        BOOST_CHECK(accepted);
+    }
+    // UpdateTip is called here
+    bool block_added = background_cs->ActivateBestChain(state, pblock);
+
+    // Ensure tip is as expected
+    BOOST_CHECK_EQUAL(background_cs->m_chain.Tip()->GetBlockHash(), validation_block.GetHash());
+
+    // g_best_block should be unchanged after adding a block to the background
+    // validation chain.
+    BOOST_CHECK(block_added);
+    BOOST_CHECK_EQUAL(curr_tip, ::g_best_block);
+}
+
 BOOST_AUTO_TEST_SUITE_END()