merge bitcoin#19806: UTXO snapshot activation

src/chain.h

@@ -161,14 +161,27 @@ class CBlockIndex
 
     //! Number of transactions in this block.
     //! Note: in a potential headers-first mode, this number cannot be relied upon
+    //! Note: this value is faked during UTXO snapshot load to ensure that
+    //! LoadBlockIndex() will load index entries for blocks that we lack data for.
+    //! @sa ActivateSnapshot
     unsigned int nTx{0};
 
     //! (memory only) Number of transactions in the chain up to and including this block.
     //! This value will be non-zero only if and only if transactions for this block and all its parents are available.
     //! Change to 64-bit type when necessary; won't happen before 2030
+    //!
+    //! Note: this value is faked during use of a UTXO snapshot because we don't
+    //! have the underlying block data available during snapshot load.
+    //! @sa AssumeutxoData
+    //! @sa ActivateSnapshot
     unsigned int nChainTx{0};
 
     //! Verification status of this block. See enum BlockStatus
+    //!
+    //! Note: this value is modified to show BLOCK_OPT_WITNESS during UTXO snapshot
+    //! load to avoid the block index being spuriously rewound.
+    //! @sa RewindBlockIndex
+    //! @sa ActivateSnapshot
     uint32_t nStatus{0};
 
     //! block header

src/chainparams.cpp

@@ -9,7 +9,6 @@
 #include <chainparamsseeds.h>
 #include <consensus/merkle.h>
 #include <llmq/params.h>
-#include <tinyformat.h>
 #include <util/ranges.h>
 #include <util/system.h>
 #include <util/underlying.h>
@@ -311,6 +310,10 @@ class CMainParams : public CChainParams {
             }
         };
 
+        m_assumeutxo_data = MapAssumeutxo{
+         // TODO to be specified in a future patch.
+        };
+
         // getchaintxstats 17280 00000000000000261bdbe99c01fcba992e577efa6cc41aae564b8ca9f112b2a3
         chainTxData = ChainTxData{
                 1680866408, // * UNIX timestamp of last known number of transactions (Block 1718597)
@@ -477,6 +480,10 @@ class CTestNetParams : public CChainParams {
             }
         };
 
+        m_assumeutxo_data = MapAssumeutxo{
+            // TODO to be specified in a future patch.
+        };
+
         // getchaintxstats 17280 0000005c35514190ef3c38d322f69412553dc7e1107ed5f92adc2935b90acc51
         chainTxData = ChainTxData{
                 1680868209, // * UNIX timestamp of last known number of transactions (Block 771537)
@@ -846,6 +853,17 @@ class CRegTestParams : public CChainParams {
             }
         };
 
+        m_assumeutxo_data = MapAssumeutxo{
+            {
+                110,
+                {uint256S("0x533d91c2aee01848b86693c226da68b1ba3f47bf266d9082a32bb2df4dafd7d8"), 110},
+            },
+            {
+                210,
+                {uint256S("0x4282d2b2a90444a8e5d3f80250c5a0cacde5a7000f2b03b0982013be98c2ba53"), 210},
+            },
+        };
+
         chainTxData = ChainTxData{
             0,
             0,
@@ -1326,3 +1344,9 @@ void SelectParams(const std::string& network)
     SelectBaseParams(network);
     globalChainParams = CreateChainParams(network);
 }
+
+std::ostream& operator<<(std::ostream& o, const AssumeutxoData& aud)
+{
+    o << strprintf("AssumeutxoData(%s, %s)", aud.hash_serialized.ToString(), aud.nChainTx);
+    return o;
+}

src/chainparams.h

@@ -26,6 +26,26 @@ struct CCheckpointData {
     MapCheckpoints mapCheckpoints;
 };
 
+/**
+ * Holds configuration for use during UTXO snapshot load and validation. The contents
+ * here are security critical, since they dictate which UTXO snapshots are recognized
+ * as valid.
+ */
+struct AssumeutxoData {
+    //! The expected hash of the deserialized UTXO set.
+    const uint256 hash_serialized;
+
+    //! Used to populate the nChainTx value, which is used during BlockManager::LoadBlockIndex().
+    //!
+    //! We need to hardcode the value here because this is computed cumulatively using block data,
+    //! which we do not necessarily have at the time of snapshot load.
+    const unsigned int nChainTx;
+};
+
+std::ostream& operator<<(std::ostream& o, const AssumeutxoData& aud);
+
+using MapAssumeutxo = std::map<int, const AssumeutxoData>;
+
 /**
  * Holds various statistics on transactions within a chain. Used to estimate
  * verification progress during chain sync.
@@ -97,6 +117,11 @@ class CChainParams
     int ExtCoinType() const { return nExtCoinType; }
     const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; }
     const CCheckpointData& Checkpoints() const { return checkpointData; }
+
+    //! Get allowed assumeutxo configuration.
+    //! @see ChainstateManager
+    const MapAssumeutxo& Assumeutxo() const { return m_assumeutxo_data; }
+
     const ChainTxData& TxData() const { return chainTxData; }
     void UpdateDIP3Parameters(int nActivationHeight, int nEnforcementHeight);
     void UpdateDIP8Parameters(int nActivationHeight);
@@ -139,6 +164,7 @@ class CChainParams
     bool m_is_mockable_chain;
     int nLLMQConnectionRetryTimeout;
     CCheckpointData checkpointData;
+    MapAssumeutxo m_assumeutxo_data;
     ChainTxData chainTxData;
     int nPoolMinParticipants;
     int nPoolMaxParticipants;

src/coins.cpp

@@ -99,6 +99,14 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi
     cachedCoinsUsage += it->second.coin.DynamicMemoryUsage();
 }
 
+void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) {
+    cachedCoinsUsage += coin.DynamicMemoryUsage();
+    cacheCoins.emplace(
+        std::piecewise_construct,
+        std::forward_as_tuple(std::move(outpoint)),
+        std::forward_as_tuple(std::move(coin), CCoinsCacheEntry::DIRTY));
+}
+
 void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) {
     bool fCoinbase = tx.IsCoinBase();
     const uint256& txid = tx.GetHash();

src/coins.h

@@ -20,6 +20,8 @@
 #include <functional>
 #include <unordered_map>
 
+class ChainstateManager;
+
 /**
  * A UTXO entry.
  *
@@ -155,6 +157,7 @@ struct CCoinsCacheEntry
 
     CCoinsCacheEntry() : flags(0) {}
     explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {}
+    CCoinsCacheEntry(Coin&& coin_, unsigned char flag) : coin(std::move(coin_)), flags(flag) {}
 };
 
 typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CCoinsMap;
@@ -292,6 +295,15 @@ class CCoinsViewCache : public CCoinsViewBacked
      */
     void AddCoin(const COutPoint& outpoint, Coin&& coin, bool possible_overwrite);
 
+    /**
+     * Emplace a coin into cacheCoins without performing any checks, marking
+     * the emplaced coin as dirty.
+     *
+     * NOT FOR GENERAL USE. Used only when loading coins from a UTXO snapshot.
+     * @sa ChainstateManager::PopulateAndValidateSnapshot()
+     */
+    void EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin);
+
     /**
      * Spend a coin. Pass moveto in order to get the deleted data.
      * If no unspent output exists for the passed outpoint, this call

src/node/coinstats.cpp

@@ -56,6 +56,18 @@ static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& has
     muhash.Insert(MakeUCharSpan(ss));
 }
 
+//! Warning: be very careful when changing this! assumeutxo and UTXO snapshot
+//! validation commitments are reliant on the hash constructed by this
+//! function.
+//!
+//! If the construction of this hash is changed, it will invalidate
+//! existing UTXO snapshots. This will not result in any kind of consensus
+//! failure, but it will force clients that were expecting to make use of
+//! assumeutxo to do traditional IBD instead.
+//!
+//! It is also possible, though very unlikely, that a change in this
+//! construction could cause a previously invalid (and potentially malicious)
+//! UTXO snapshot to be considered valid.
 template <typename T>
 static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs)
 {

src/rpc/blockchain.cpp

@@ -2716,10 +2716,19 @@ UniValue dumptxoutset(const JSONRPCRequest& request)
 
     FILE* file{fsbridge::fopen(temppath, "wb")};
     CAutoFile afile{file, SER_DISK, CLIENT_VERSION};
+    NodeContext& node = EnsureNodeContext(request.context);
+    UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), afile);
+    fs::rename(temppath, path);
+
+    result.pushKV("path", path.string());
+    return result;
+}
+
+UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile)
+{
     std::unique_ptr<CCoinsViewCursor> pcursor;
     CCoinsStats stats;
     CBlockIndex* tip;
-    NodeContext& node = EnsureNodeContext(request.context);
 
     {
         // We need to lock cs_main to ensure that the coinsdb isn't written to
@@ -2736,13 +2745,13 @@ UniValue dumptxoutset(const JSONRPCRequest& request)
         //
         LOCK(::cs_main);
 
-        ::ChainstateActive().ForceFlushStateToDisk();
+        chainstate.ForceFlushStateToDisk();
 
-        if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) {
+        if (!GetUTXOStats(&chainstate.CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) {
             throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set");
         }
 
-        pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor());
+        pcursor = std::unique_ptr<CCoinsViewCursor>(chainstate.CoinsDB().Cursor());
         tip = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock);
         CHECK_NONFATAL(tip);
     }
@@ -2767,13 +2776,12 @@ UniValue dumptxoutset(const JSONRPCRequest& request)
     }
 
     afile.fclose();
-    fs::rename(temppath, path);
 
     UniValue result(UniValue::VOBJ);
     result.pushKV("coins_written", stats.coins_count);
     result.pushKV("base_hash", tip->GetBlockHash().ToString());
     result.pushKV("base_height", tip->nHeight);
-    result.pushKV("path", path.string());
+
     return result;
 }
 

src/rpc/blockchain.h

@@ -7,6 +7,7 @@
 
 #include <amount.h>
 #include <context.h>
+#include <streams.h>
 #include <sync.h>
 
 #include <stdint.h>
@@ -17,6 +18,7 @@ extern RecursiveMutex cs_main;
 class CBlock;
 class CBlockIndex;
 class CBlockPolicyEstimator;
+class CChainState;
 class CTxMemPool;
 class ChainstateManager;
 class UniValue;
@@ -61,4 +63,10 @@ ChainstateManager& EnsureChainman(const CoreContext& context);
 CBlockPolicyEstimator& EnsureFeeEstimator(const CoreContext& context);
 LLMQContext& EnsureLLMQContext(const CoreContext& context);
 
+/**
+ * Helper to create UTXO snapshots given a chainstate and a file handle.
+ * @return a UniValue map containing metadata about the snapshot.
+ */
+UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile);
+
 #endif

src/test/block_reward_reallocation_tests.cpp

@@ -38,7 +38,7 @@ using SimpleUTXOMap = std::map<COutPoint, std::pair<int, CAmount>>;
 struct TestChainBRRBeforeActivationSetup : public TestChainSetup
 {
     // Force fast DIP3 activation
-    TestChainBRRBeforeActivationSetup() : TestChainSetup(497, {"-dip3params=30:50"}) {}
+    TestChainBRRBeforeActivationSetup() : TestChainSetup(497, false, {"-dip3params=30:50"}) {}
 };
 
 static SimpleUTXOMap BuildSimpleUtxoMap(const std::vector<CTransactionRef>& txs)

src/test/dynamic_activation_thresholds_tests.cpp

@@ -34,7 +34,7 @@ static constexpr int threshold(int attempt)
 
 struct TestChainDATSetup : public TestChainSetup
 {
-    TestChainDATSetup() : TestChainSetup(window - 2, {"-vbparams=testdummy:0:999999999999:100:80:60:5"}) {}
+    TestChainDATSetup() : TestChainSetup(window - 2, false, {"-vbparams=testdummy:0:999999999999:100:80:60:5"}) {}
 
     void signal(int num_blocks, bool expected_lockin)
     {

src/test/util/setup_common.cpp

@@ -258,15 +258,43 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const
     }
 }
 
-TestChainSetup::TestChainSetup(int blockCount, const std::vector<const char*>& extra_args)
+TestChainSetup::TestChainSetup(int num_blocks, bool deterministic, const std::vector<const char*>& extra_args)
     : RegTestingSetup(extra_args)
 {
+    m_deterministic = deterministic;
+
+    if (m_deterministic) {
+        SetMockTime(1598887952);
+        constexpr std::array<unsigned char, 32> vchKey = {
+            {
+                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
+            }
+        };
+        coinbaseKey.Set(vchKey.begin(), vchKey.end(), false);
+    } else {
+        coinbaseKey.MakeNewKey(true);
+    }
+
     // Generate a 100-block chain:
-    coinbaseKey.MakeNewKey(true);
+    this->mineBlocks(num_blocks);
+
+    if (m_deterministic) {
+        LOCK(::cs_main);
+        assert(
+            m_node.chainman->ActiveChain().Tip()->GetBlockHash().ToString() ==
+            "5d52e2382788a10ac5bf8156f0de41a8f1ef13496b1895e4d7eab7989c23255b");
+    }
+}
+
+void TestChainSetup::mineBlocks(int num_blocks)
+{
     CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG;
-    for (int i = 0; i < blockCount; i++) {
+    for (int i = 0; i < num_blocks; i++) {
         std::vector<CMutableTransaction> noTxns;
         CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey);
+        if (m_deterministic) {
+            SetMockTime(GetTime() + 1);
+        }
         m_coinbase_txns.push_back(b.vtx[0]);
     }
 
@@ -369,6 +397,9 @@ TestChainSetup::~TestChainSetup()
     g_txindex->Interrupt();
     g_txindex->Stop();
     g_txindex.reset();
+    if (m_deterministic) {
+        SetMockTime(0);
+    }
 }
 
 CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx)

src/test/util/setup_common.h

@@ -79,7 +79,6 @@ struct BasicTestingSetup {
     explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {});
     ~BasicTestingSetup();
 
-private:
     std::unique_ptr<CConnman> connman;
     const fs::path m_path_root;
 };
@@ -112,7 +111,7 @@ class CScript;
 
 struct TestChainSetup : public RegTestingSetup
 {
-    TestChainSetup(int blockCount, const std::vector<const char*>& extra_args = {});
+    TestChainSetup(int num_blocks, bool deterministic = false, const std::vector<const char*>& extra_args = {});
     ~TestChainSetup();
 
     /**
@@ -128,6 +127,10 @@ struct TestChainSetup : public RegTestingSetup
     CBlock CreateBlock(const std::vector<CMutableTransaction>& txns,
                        const CKey& scriptKey);
 
+    //! Mine a series of new blocks on the active chain.
+    void mineBlocks(int num_blocks);
+
+    bool m_deterministic;
     std::vector<CTransactionRef> m_coinbase_txns; // For convenience, coinbase transactions
     CKey coinbaseKey; // private/public key needed to spend coinbase transactions
 };
@@ -136,7 +139,11 @@ struct TestChainSetup : public RegTestingSetup
  * Testing fixture that pre-creates a 100-block REGTEST-mode block chain
  */
 struct TestChain100Setup : public TestChainSetup {
-    TestChain100Setup() : TestChainSetup(100) {}
+    TestChain100Setup() : TestChainSetup(100, false) {}
+};
+
+struct TestChain100DeterministicSetup : public TestChainSetup {
+    TestChain100DeterministicSetup() : TestChainSetup(100, true) { }
 };
 
 struct TestChainDIP3Setup : public TestChainSetup