From 8d009e42984bc5eea7d6f50ee8afc97cc773b8b5 Mon Sep 17 00:00:00 2001 From: Peter Bushnell Date: Tue, 12 Jul 2022 08:33:30 +0100 Subject: [PATCH] Migrate undos to dedicated DB --- src/flushablestorage.h | 3 + src/init.cpp | 55 ++++++- src/masternodes/masternodes.cpp | 38 ++--- src/masternodes/masternodes.h | 14 +- src/masternodes/mn_checks.cpp | 8 - src/masternodes/undo.h | 17 ++ src/masternodes/undos.cpp | 48 +++++- src/masternodes/undos.h | 27 ++- src/miner.cpp | 3 +- src/test/setup_common.cpp | 6 +- src/test/storage_tests.cpp | 29 ++-- src/validation.cpp | 155 ++++++++++-------- src/validation.h | 12 +- .../test_framework/test_framework.py | 1 + 14 files changed, 277 insertions(+), 139 deletions(-) diff --git a/src/flushablestorage.h b/src/flushablestorage.h index fd41d96a0d..3d50ea0c85 100644 --- a/src/flushablestorage.h +++ b/src/flushablestorage.h @@ -503,6 +503,9 @@ class CStorageView { } } } + CFlushableStorageKV& GetStorage() { + return static_cast(DB()); + } virtual bool Flush() { return DB().Flush(); } void Discard() { DB().Discard(); } diff --git a/src/init.cpp b/src/init.cpp index ca0518ee6e..2d29462362 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1340,6 +1340,48 @@ void SetupAnchorSPVDatabases(bool resync) { } } + +void MigrateDBs() +{ + auto it = pundosView->LowerBound(UndoSourceKey{}); + if (it.Valid()) { + return; + } + + auto time = GetTimeMillis(); + + // Migrate Undos + auto migrationStarted{false}; + auto mnview(*pcustomcsview); + pcustomcsview->ForEachUndo([&](const UndoKey& key, const CUndo& undo){ + if (!migrationStarted) { + migrationStarted = true; + LogPrintf("Migrating undo entries, might take a while...\n"); + } + + pundosView->SetUndo({{key.height, key.txid}, UndoSource::CustomView}, undo); + mnview.DelUndo(key); + + return true; + }); + + if (migrationStarted) { + pundosView->Flush(); + pundosDB->Flush(); + + auto& map = mnview.GetStorage().GetRaw(); + const auto compactBegin = map.begin()->first; + const auto compactEnd = map.rbegin()->first; + mnview.Flush(); + pcustomcsview->Flush(); + pcustomcsDB->Flush(); + pcustomcsDB->Compact(compactBegin, compactEnd); + + LogPrintf("Migrating undo data finished.\n"); + LogPrint(BCLog::BENCH, " - Migrating undo data takes: %dms\n", GetTimeMillis() - time); + } +} + bool SetupInterruptArg(const std::string &argName, std::string &hashStore, int &heightStore) { // Experimental: Block height or hash to invalidate on and stop sync auto val = gArgs.GetArg(argName, ""); @@ -1717,7 +1759,7 @@ bool AppInitMain(InitInterfaces& interfaces) pcustomcsDB.reset(); pcustomcsDB = std::make_unique(GetDataDir() / "enhancedcs", nCustomCacheSize, false, fReset || fReindexChainState); pcustomcsview.reset(); - pcustomcsview = std::make_unique(*pcustomcsDB.get()); + pcustomcsview = std::make_unique(*pcustomcsDB); if (!fReset && !fReindexChainState) { if (!pcustomcsDB->IsEmpty() && pcustomcsview->GetDbVersion() != CCustomCSView::DbVersion) { @@ -1749,6 +1791,15 @@ bool AppInitMain(InitInterfaces& interfaces) pvaultHistoryDB = std::make_unique(GetDataDir() / "vault", nCustomCacheSize, false, fReset || fReindexChainState); } + // Create Undo DB + pundosDB.reset(); + pundosDB = std::make_unique(GetDataDir() / "undos", nCustomCacheSize, false, fReset || fReindexChainState); + pundosView.reset(); + pundosView = std::make_unique(*pundosDB); + + // Migrate FutureSwaps and Undos to their own new DBs. + MigrateDBs(); + // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!::ChainstateActive().CoinsDB().Upgrade()) { @@ -1757,7 +1808,7 @@ bool AppInitMain(InitInterfaces& interfaces) } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get())) { + if (!ReplayBlocks(chainparams, &::ChainstateActive().CoinsDB(), pcustomcsview.get(), pundosView.get())) { strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; break; } diff --git a/src/masternodes/masternodes.cpp b/src/masternodes/masternodes.cpp index 1ca14e3a58..915a2df623 100644 --- a/src/masternodes/masternodes.cpp +++ b/src/masternodes/masternodes.cpp @@ -709,7 +709,10 @@ void CSettingsView::SetDexStatsLastHeight(const int32_t height) std::optional CSettingsView::GetDexStatsLastHeight() { - return ReadBy(DEX_STATS_LAST_HEIGHT); + if (auto res = ReadBy(DEX_STATS_LAST_HEIGHT)) { + return res; + } + return {}; } void CSettingsView::SetDexStatsEnabled(const bool enabled) @@ -878,16 +881,6 @@ void CCustomCSView::CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::Ancho } } -void CCustomCSView::OnUndoTx(uint256 const & txid, uint32_t height) -{ - const auto undo = GetUndo(UndoKey{height, txid}); - if (!undo) { - return; // not custom tx, or no changes done - } - CUndo::Revert(GetStorage(), *undo); // revert the changes of this tx - DelUndo(UndoKey{height, txid}); // erase undo data, it served its purpose -} - bool CCustomCSView::CanSpend(const uint256 & txId, int height) const { auto node = GetMasternode(txId); @@ -1085,28 +1078,31 @@ Res CCustomCSView::PopulateCollateralData(CCollateralLoans& result, CVaultId con return Res::Ok(); } -uint256 CCustomCSView::MerkleRoot() { +uint256 CCustomCSView::MerkleRoot(CUndosView& undo) { auto rawMap = GetStorage().GetRaw(); if (rawMap.empty()) { return {}; } auto isAttributes = [](const TBytes& key) { - MapKV map = {std::make_pair(key, TBytes{})}; // Attributes should not be part of merkle root static const std::string attributes("ATTRIBUTES"); + MapKV map = {std::make_pair(key, TBytes{})}; auto it = NewKVIterator(attributes, map); return it.Valid() && it.Key() == attributes; }; - - auto it = NewKVIterator(UndoKey{}, rawMap); + auto& undoStorage = undo.GetStorage(); + auto& undoMap = undoStorage.GetRaw(); + auto it = NewKVIterator(UndoSourceKey{}, undoMap); for (; it.Valid(); it.Next()) { - CUndo value = it.Value(); - auto& map = value.before; - for (auto it = map.begin(); it != map.end();) { - isAttributes(it->first) ? map.erase(it++) : ++it; + if (it.Key().key == UndoSource::CustomView) { + CUndo value = it.Value(); + auto& map = value.before; + for (auto it = map.begin(); it != map.end();) { + isAttributes(it->first) ? map.erase(it++) : ++it; + } + auto key = std::make_pair(CUndosBaseView::ByUndoKey::prefix(), static_cast(it.Key())); + rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value); } - auto key = std::make_pair(CUndosView::ByUndoKey::prefix(), static_cast(it.Key())); - rawMap[DbTypeToBytes(key)] = DbTypeToBytes(value); } std::vector hashes; diff --git a/src/masternodes/masternodes.h b/src/masternodes/masternodes.h index a9db52b6e0..6f6ea335e2 100644 --- a/src/masternodes/masternodes.h +++ b/src/masternodes/masternodes.h @@ -390,7 +390,7 @@ class CCustomCSView , public CTokensView , public CAccountsView , public CCommunityBalancesView - , public CUndosView + , public CUndosBaseView , public CPoolPairView , public CGovView , public CAnchorConfirmsView @@ -411,7 +411,7 @@ class CCustomCSView CTokensView :: ID, Symbol, CreationTx, LastDctId, CAccountsView :: ByBalanceKey, ByHeightKey, ByFuturesSwapKey, CCommunityBalancesView :: ById, - CUndosView :: ByUndoKey, + CUndosBaseView :: ByUndoKey, CPoolPairView :: ByID, ByPair, ByShare, ByIDPair, ByPoolSwap, ByReserves, ByRewardPct, ByRewardLoanPct, ByPoolReward, ByDailyReward, ByCustomReward, ByTotalLiquidity, ByDailyLoanReward, ByPoolLoanReward, ByTokenDexFeePct, @@ -461,9 +461,6 @@ class CCustomCSView /// @todo newbase move to networking? void CreateAndRelayConfirmMessageIfNeed(const CAnchorIndex::AnchorRec* anchor, const uint256 & btcTxHash, const CKey &masternodeKey); - // simplified version of undo, without any unnecessary undo data - void OnUndoTx(uint256 const & txid, uint32_t height); - bool CanSpend(const uint256 & txId, int height) const; bool CalculateOwnerRewards(CScript const & owner, uint32_t height); @@ -487,12 +484,7 @@ class CCustomCSView void SetGlobalCustomTxExpiration(const uint32_t height); uint32_t GetGlobalCustomTxExpiration() const; - uint256 MerkleRoot(); - - // we construct it as it - CFlushableStorageKV& GetStorage() { - return static_cast(DB()); - } + uint256 MerkleRoot(CUndosView& undo); virtual CAccountHistoryStorage* GetAccountHistoryStore(); CVaultHistoryStorage* GetVaultHistoryStore(); diff --git a/src/masternodes/mn_checks.cpp b/src/masternodes/mn_checks.cpp index 9589dd6a9d..d0b447db2d 100644 --- a/src/masternodes/mn_checks.cpp +++ b/src/masternodes/mn_checks.cpp @@ -3774,15 +3774,7 @@ Res ApplyCustomTx(CCustomCSView& mnview, const CCoinsViewCache& coins, const CTr return res; } - // construct undo - auto& flushable = view.GetStorage(); - auto undo = CUndo::Construct(mnview.GetStorage(), flushable.GetRaw()); - // flush changes view.Flush(); - // write undo - if (!undo.before.empty()) { - mnview.SetUndo(UndoKey{height, tx.GetHash()}, undo); - } return res; } diff --git a/src/masternodes/undo.h b/src/masternodes/undo.h index b9f6e1e967..27c5d59676 100644 --- a/src/masternodes/undo.h +++ b/src/masternodes/undo.h @@ -12,6 +12,11 @@ #include #include +// Enum for future support of multiple sources of undo data, not just CustomView. +enum UndoSource : uint8_t { + CustomView = 0, +}; + struct UndoKey { uint32_t height; // height is there to be able to prune older undos using lexicographic iteration uint256 txid; @@ -25,6 +30,18 @@ struct UndoKey { } }; +struct UndoSourceKey : UndoKey { + uint8_t key; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITEAS(UndoKey, *this); + READWRITE(key); + } +}; + struct CUndo { MapKV before; diff --git a/src/masternodes/undos.cpp b/src/masternodes/undos.cpp index 47bd1d9bfd..40a12efe81 100644 --- a/src/masternodes/undos.cpp +++ b/src/masternodes/undos.cpp @@ -4,29 +4,61 @@ #include -void CUndosView::ForEachUndo(std::function)> callback, UndoKey const & start) +void CUndosBaseView::ForEachUndo(std::function)> callback, UndoKey const & start) { ForEach(callback, start); } -Res CUndosView::SetUndo(UndoKey const & key, CUndo const & undo) +Res CUndosBaseView::DelUndo(UndoKey const & key) { - WriteBy(key, undo); + EraseBy(key); return Res::Ok(); } -Res CUndosView::DelUndo(UndoKey const & key) +void CUndosView::ForEachUndo(std::function)> callback, const UndoSourceKey& start) { - EraseBy(key); + ForEach(callback, start); +} + +void CUndosView::AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height) +{ + auto& flushable = cache.GetStorage(); + auto& rawMap = flushable.GetRaw(); + if (!rawMap.empty()) { + SetUndo({{height, txid}, key}, CUndo::Construct(source.GetStorage(), rawMap)); + } +} + +Res CUndosView::SetUndo(UndoSourceKey const & key, CUndo const & undo) +{ + WriteBy(key, undo); return Res::Ok(); } -std::optional CUndosView::GetUndo(UndoKey const & key) const +void CUndosView::OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height) +{ + const auto undo = GetUndo({{height, txid}, key}); + if (!undo) { + return; // not custom tx, or no changes done + } + CUndo::Revert(source.GetStorage(), *undo); // revert the changes of this tx + DelUndo({{height, txid}, key}); // erase undo data, it served its purpose +} + +std::optional CUndosView::GetUndo(UndoSourceKey const & key) const { CUndo val; - bool ok = ReadBy(key, val); - if (ok) { + if (ReadBy(key, val)) { return val; } return {}; } + +Res CUndosView::DelUndo(const UndoSourceKey & key) +{ + EraseBy(key); + return Res::Ok(); +} + +std::unique_ptr pundosDB; +std::unique_ptr pundosView; diff --git a/src/masternodes/undos.h b/src/masternodes/undos.h index bf9ebe1f1c..dbb3b19ba4 100644 --- a/src/masternodes/undos.h +++ b/src/masternodes/undos.h @@ -9,17 +9,36 @@ #include #include -class CUndosView : public virtual CStorageView { +class CUndosBaseView : public virtual CStorageView { public: void ForEachUndo(std::function)> callback, UndoKey const & start = {}); - std::optional GetUndo(UndoKey const & key) const; - Res SetUndo(UndoKey const & key, CUndo const & undo); - Res DelUndo(UndoKey const & key); + Res DelUndo(const UndoKey & key); // tags struct ByUndoKey { static constexpr uint8_t prefix() { return 'u'; } }; }; +class CUndosView : public CStorageView +{ +public: + CUndosView(CUndosView& other) : CStorageView(new CFlushableStorageKV(other.DB())) {} + explicit CUndosView(CStorageKV& st) : CStorageView(new CFlushableStorageKV(st)) {} + + void ForEachUndo(std::function)> callback, const UndoSourceKey& start = {}); + + [[nodiscard]] std::optional GetUndo(UndoSourceKey const & key) const; + Res SetUndo(const UndoSourceKey& key, const CUndo& undo); + Res DelUndo(const UndoSourceKey & key); + + void AddUndo(const UndoSource key, CStorageView & source, CStorageView & cache, uint256 const & txid, uint32_t height); + void OnUndoTx(const UndoSource key, CStorageView & source, uint256 const & txid, uint32_t height); + + // tags + struct ByMultiUndoKey { static constexpr uint8_t prefix() { return 'n'; } }; +}; + +extern std::unique_ptr pundosDB; +extern std::unique_ptr pundosView; #endif //DEFI_MASTERNODES_UNDOS_H diff --git a/src/miner.cpp b/src/miner.cpp index 68d846a408..4293b5159d 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -213,6 +213,7 @@ std::unique_ptr BlockAssembler::CreateNewBlock(const CScript& sc int nPackagesSelected = 0; int nDescendantsUpdated = 0; CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); if (!blockTime) { UpdateTime(pblock, consensus, pindexPrev); // update time before tx packaging } @@ -354,7 +355,7 @@ std::unique_ptr BlockAssembler::CreateNewBlock(const CScript& sc && nHeight < chainparams.GetConsensus().EunosKampungHeight) { // includes coinbase account changes ApplyGeneralCoinbaseTx(mnview, *(pblock->vtx[0]), nHeight, nFees, chainparams.GetConsensus()); - pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot()); + pblock->hashMerkleRoot = Hash2(pblock->hashMerkleRoot, mnview.MerkleRoot(undosView)); } LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", 0.001 * (nTime1 - nTimeStart), nPackagesSelected, nDescendantsUpdated, 0.001 * (nTime2 - nTime1), 0.001 * (nTime2 - nTimeStart)); diff --git a/src/test/setup_common.cpp b/src/test/setup_common.cpp index a29ad05681..82b148106a 100644 --- a/src/test/setup_common.cpp +++ b/src/test/setup_common.cpp @@ -113,7 +113,11 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha pcustomcsDB.reset(); pcustomcsDB = std::make_unique(GetDataDir() / "enhancedcs", nMinDbCache << 20, true, true); - pcustomcsview = std::make_unique(*pcustomcsDB.get()); + pcustomcsview = std::make_unique(*pcustomcsDB); + + pundosDB.reset(); + pundosDB = std::make_unique(GetDataDir() / "undos", nMinDbCache << 20, true, true); + pundosView = std::make_unique(*pundosDB); panchorauths.reset(); panchorauths = std::make_unique(); diff --git a/src/test/storage_tests.cpp b/src/test/storage_tests.cpp index a54d8ee213..746004badb 100644 --- a/src/test/storage_tests.cpp +++ b/src/test/storage_tests.cpp @@ -81,13 +81,18 @@ BOOST_AUTO_TEST_CASE(flushableType) BOOST_AUTO_TEST_CASE(undo) { - CStorageKV & base_raw = pcustomcsview->GetStorage(); + CCustomCSView view(*pcustomcsview); + CUndosView undoView(*pundosView); + auto& base_raw = view.GetStorage(); + auto& undo_raw = undoView.GetStorage(); + auto undoStart = TakeSnapshot(undo_raw); + // place some "old" record - pcustomcsview->Write("testkey1", "value0"); + view.Write("testkey1", "value0"); auto snapStart = TakeSnapshot(base_raw); - CCustomCSView mnview(*pcustomcsview); + auto mnview(view); BOOST_CHECK(mnview.Write("testkey1", "value1")); // modify BOOST_CHECK(mnview.Write("testkey2", "value2")); // insert @@ -107,17 +112,21 @@ BOOST_AUTO_TEST_CASE(undo) BOOST_CHECK(snap1.at(ToBytes("testkey2")) == ToBytes("value2")); // write undo - pcustomcsview->SetUndo(UndoKey{1, uint256S("0x1")}, undo); + auto snap_undo1 = TakeSnapshot(base_raw); + undoView.SetUndo({{1, uint256S("0x1")}, UndoSource::CustomView}, undo); - auto snap2 = TakeSnapshot(base_raw); - BOOST_CHECK(snap2.size() - snap1.size() == 1); // undo - BOOST_CHECK(snap2.size() - snapStart.size() == 2); // onew new record + undo + auto snap_undo = TakeSnapshot(undo_raw); + BOOST_CHECK_EQUAL(snap_undo.size() - undoStart.size(), 1); // undo - pcustomcsview->OnUndoTx(uint256S("0x1"), 2); // fail + auto snap2 = TakeSnapshot(base_raw); + undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x1"), 2); // fail + mnview.Flush(); BOOST_CHECK(snap2 == TakeSnapshot(base_raw)); - pcustomcsview->OnUndoTx(uint256S("0x2"), 1); // fail + undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x2"), 1); // fail + mnview.Flush(); BOOST_CHECK(snap2 == TakeSnapshot(base_raw)); - pcustomcsview->OnUndoTx(uint256S("0x1"), 1); // success + undoView.OnUndoTx(UndoSource::CustomView, mnview, uint256S("0x1"), 1); // success + mnview.Flush(); BOOST_CHECK(snapStart == TakeSnapshot(base_raw)); } diff --git a/src/validation.cpp b/src/validation.cpp index 352c73f22d..ea1df71e97 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1691,7 +1691,7 @@ static bool GetCreationTransactions(const CBlock& block, const uint32_t id, cons /** Undo the effects of this block (with given index) on the UTXO set represented by coins. * When FAILED is returned, view is left in an indeterminate state. */ -DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, std::vector & disconnectedAnchorConfirms) +DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, CUndosView& undosView, std::vector & disconnectedAnchorConfirms) { bool fClean = true; @@ -1712,7 +1712,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI } // special case: possible undo (first) of custom 'complex changes' for the whole block (expired orders and/or prices) - mnview.OnUndoTx(uint256(), (uint32_t) pindex->nHeight); // undo for "zero hash" + undosView.OnUndoTx(UndoSource::CustomView, mnview, {}, static_cast(pindex->nHeight)); pburnHistoryDB->EraseAccountHistoryHeight(pindex->nHeight); if (paccountHistoryDB) { @@ -1834,7 +1834,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI } // process transactions revert for masternodes - mnview.OnUndoTx(tx.GetHash(), (uint32_t) pindex->nHeight); + undosView.OnUndoTx(UndoSource::CustomView, mnview, tx.GetHash(), pindex->nHeight); } // one time downgrade to revert CInterestRateV2 structure @@ -2448,6 +2448,53 @@ bool ApplyGovVars(CCustomCSView& cache, const CBlockIndex& pindex, const std::ma return false; } +static void PruneUndos(CUndosView& view, const int height) { + bool pruneStarted = false; + auto time = GetTimeMillis(); + CUndosView pruned(view); + view.ForEachUndo([&](const UndoSourceKey& key, const CLazySerialize&) { + if (static_cast(key.height) >= height) { // don't erase checkpoint height + return false; + } + if (!pruneStarted) { + pruneStarted = true; + LogPrintf("Pruning undo data prior %d, it can take a while...\n", height); + } + return pruned.DelUndo(key).ok; + }); + if (pruneStarted) { + auto& map = pruned.GetStorage().GetRaw(); + compactBegin = map.begin()->first; + compactEnd = map.rbegin()->first; + pruned.Flush(); + LogPrintf("Pruning undo data finished.\n"); + LogPrint(BCLog::BENCH, " - Pruning undo data takes: %dms\n", GetTimeMillis() - time); + } +} + +static void PruneInterest(CCustomCSView& mnview, const CChainParams& chainparams, int height) { + if (height <= chainparams.GetConsensus().FortCanningHillHeight) { + return; + } + CCustomCSView view(mnview); + mnview.ForEachVaultInterest([&](const CVaultId& vaultId, DCT_ID tokenId, CInterestRate) { + view.EraseBy(std::make_pair(vaultId, tokenId)); + return true; + }); + view.Flush(); +} + +static void PruneData(CCustomCSView& mnview, CUndosView& undosView, const CChainParams& chainparams, int height) { + auto &checkpoints = chainparams.Checkpoints().mapCheckpoints; + auto it = checkpoints.lower_bound(height); + if (it != checkpoints.begin()) { + --it; + PruneUndos(undosView, it->first); + // we can safely delete old interest keys + PruneInterest(mnview, chainparams, height); + } +} + bool StopOrInterruptConnect(const CBlockIndex *pIndex, CValidationState& state) { if (!fStopOrInterrupt) return false; @@ -2476,8 +2523,8 @@ bool StopOrInterruptConnect(const CBlockIndex *pIndex, CValidationState& state) /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock () * can fail if those validity checks fail (among other reasons). */ -bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, CCustomCSView& mnview, const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck) +bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& mnview, CUndosView& undosView, + const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck) { AssertLockHeld(cs_main); assert(pindex); @@ -2544,13 +2591,17 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl pcustomcsview->CreateDFIToken(); // init view|db with genesis here for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = *block.vtx[i]; + CCustomCSView mnviewCopy(mnview); CHistoryWriters writers{paccountHistoryDB.get(), nullptr, nullptr}; - const auto res = ApplyCustomTx(mnview, view, *block.vtx[i], chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); + const auto res = ApplyCustomTx(mnviewCopy, view, *block.vtx[i], chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); if (!res.ok) { return error("%s: Genesis block ApplyCustomTx failed. TX: %s Error: %s", - __func__, block.vtx[i]->GetHash().ToString(), res.msg); + __func__, tx.GetHash().ToString(), res.msg); } - AddCoins(view, *block.vtx[i], 0); + undosView.AddUndo(UndoSource::CustomView, mnview, mnviewCopy, tx.GetHash(), pindex->nHeight); + mnviewCopy.Flush(); + AddCoins(view, tx, 0); } } return true; @@ -2813,8 +2864,9 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl __func__, tx.GetHash().ToString(), FormatStateMessage(state)); } + CCustomCSView mnviewCopy(accountsView); CHistoryWriters writers{paccountHistoryDB.get(), pburnHistoryDB.get(), pvaultHistoryDB.get()}; - const auto res = ApplyCustomTx(mnview, view, tx, chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); + const auto res = ApplyCustomTx(mnviewCopy, view, tx, chainparams.GetConsensus(), pindex->nHeight, pindex->GetBlockTime(), nullptr, nullptr, i, &writers); if (!res.ok && (res.code & CustomTxErrCodes::Fatal)) { if (pindex->nHeight >= chainparams.GetConsensus().EunosHeight) { return state.Invalid(ValidationInvalidReason::CONSENSUS, @@ -2826,6 +2878,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl __func__, tx.GetHash().ToString(), res.msg); } } + undosView.AddUndo(UndoSource::CustomView, accountsView, mnviewCopy, tx.GetHash(), pindex->nHeight); + mnviewCopy.Flush(); // log if (!fJustCheck && !res.msg.empty()) { if (res.ok) { @@ -2967,7 +3021,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl && pindex->nHeight < chainparams.GetConsensus().EunosKampungHeight) { bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); - if (block.hashMerkleRoot != Hash2(hashMerkleRoot2, accountsView.MerkleRoot())) { + if (block.hashMerkleRoot != Hash2(hashMerkleRoot2, accountsView.MerkleRoot(undosView))) { return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-txnmrklroot", "hashMerkleRoot mismatch"); } @@ -3039,15 +3093,9 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // Masternode updates ProcessMasternodeUpdates(pindex, cache, view, chainparams); - // construct undo - auto& flushable = cache.GetStorage(); - auto undo = CUndo::Construct(mnview.GetStorage(), flushable.GetRaw()); - // flush changes to underlying view + undosView.AddUndo(UndoSource::CustomView, mnview, cache, {}, pindex->nHeight); + cache.Flush(); - // write undo - if (!undo.before.empty()) { - mnview.SetUndo(UndoKey{static_cast(pindex->nHeight), uint256() }, undo); // "zero hash" - } } // Write any UTXO burns @@ -3068,41 +3116,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl } mnview.SetLastHeight(pindex->nHeight); - auto &checkpoints = chainparams.Checkpoints().mapCheckpoints; - auto it = checkpoints.lower_bound(pindex->nHeight); - if (it != checkpoints.begin()) { - --it; - bool pruneStarted = false; - auto time = GetTimeMillis(); - CCustomCSView pruned(mnview); - mnview.ForEachUndo([&](UndoKey const & key, CLazySerialize) { - if (key.height >= static_cast(it->first)) { // don't erase checkpoint height - return false; - } - if (!pruneStarted) { - pruneStarted = true; - LogPrintf("Pruning undo data prior %d, it can take a while...\n", it->first); - } - return pruned.DelUndo(key).ok; - }); - if (pruneStarted) { - auto& map = pruned.GetStorage().GetRaw(); - compactBegin = map.begin()->first; - compactEnd = map.rbegin()->first; - pruned.Flush(); - LogPrintf("Pruning undo data finished.\n"); - LogPrint(BCLog::BENCH, " - Pruning undo data takes: %dms\n", GetTimeMillis() - time); - } - // we can safety delete old interest keys - if (it->first > chainparams.GetConsensus().FortCanningHillHeight) { - CCustomCSView view(mnview); - mnview.ForEachVaultInterest([&](const CVaultId& vaultId, DCT_ID tokenId, CInterestRate) { - view.EraseBy(std::make_pair(vaultId, tokenId)); - return true; - }); - view.Flush(); - } - } + PruneData(mnview, undosView, chainparams, pindex->nHeight); if (pindex->nHeight >= chainparams.GetConsensus().GreatWorldHeight) { // Remove any TXs from mempool that are now expired @@ -5050,10 +5064,11 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(&CoinsTip()); - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); std::vector disconnectedConfirms; - if (DisconnectBlock(block, pindexDelete, view, mnview, disconnectedConfirms) != DISCONNECT_OK) { + if (DisconnectBlock(block, pindexDelete, view, mnview, undosView, disconnectedConfirms) != DISCONNECT_OK) { // no usable history if (paccountHistoryDB) { paccountHistoryDB->Discard(); @@ -5067,7 +5082,7 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha m_disconnectTip = false; return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); } - bool flushed = view.Flush() && mnview.Flush(); + bool flushed = view.Flush() && mnview.Flush() && undosView.Flush(); assert(flushed); // flush history @@ -5216,9 +5231,10 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); { CCoinsViewCache view(&CoinsTip()); - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); std::vector rewardedAnchors; - bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, mnview, chainparams, rewardedAnchors); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, mnview, undosView, chainparams, rewardedAnchors); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) { @@ -5238,7 +5254,7 @@ bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainp } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime3 - nTime2) * MILLI, nTimeConnectTotal * MICRO, nTimeConnectTotal * MILLI / nBlocksTotal); - bool flushed = view.Flush() && mnview.Flush(); + bool flushed = view.Flush() && mnview.Flush() && undosView.Flush(); assert(flushed); // flush history @@ -6760,7 +6776,8 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); CCoinsViewCache viewNew(&::ChainstateActive().CoinsTip()); std::vector dummyRewardedAnchors; - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); uint256 block_hash(block.GetHash()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; @@ -6775,7 +6792,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, mnview, chainparams, dummyRewardedAnchors, true)) + if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, mnview, undosView, chainparams, dummyRewardedAnchors, true)) return false; assert(state.IsValid()); @@ -7158,7 +7175,8 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); CCoinsViewCache coins(coinsview); - CCustomCSView mnview(*pcustomcsview.get()); + CCustomCSView mnview(*pcustomcsview); + CUndosView undosView(*pundosView); CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; @@ -7203,7 +7221,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); std::vector disconnectedConfirms; // dummy - DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins, mnview, disconnectedConfirms); + DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins, mnview, undosView, disconnectedConfirms); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -7238,7 +7256,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); std::vector dummyRewardedAnchors; - if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, mnview, chainparams, dummyRewardedAnchors)) + if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, mnview, undosView, chainparams, dummyRewardedAnchors)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); if (ShutdownRequested()) return true; } @@ -7274,12 +7292,13 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i return true; } -bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview) +bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview, CUndosView* undosView) { LOCK(cs_main); CCoinsViewCache cache(view); CCustomCSView mncache(*mnview); + CUndosView undosCache(*undosView); std::vector hashHeads = view->GetHeadBlocks(); if (hashHeads.empty()) return true; // We're already in a consistent state. @@ -7318,7 +7337,7 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCu } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); std::vector disconnectedConfirms; // dummy - DisconnectResult res = DisconnectBlock(block, pindexOld, cache, mncache, disconnectedConfirms); + DisconnectResult res = DisconnectBlock(block, pindexOld, cache, mncache, undosCache, disconnectedConfirms); if (res == DISCONNECT_FAILED) { return error("RollbackBlock(): DisconnectBlock failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } @@ -7346,8 +7365,8 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view, CCu return true; } -bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview) { - return ::ChainstateActive().ReplayBlocks(params, view, mnview); +bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* mnview, CUndosView* undosView) { + return ::ChainstateActive().ReplayBlocks(params, view, mnview, undosView); } //! Helper for CChainState::RewindBlockIndex diff --git a/src/validation.h b/src/validation.h index 35fc0b33df..b3aeffdf28 100644 --- a/src/validation.h +++ b/src/validation.h @@ -48,6 +48,7 @@ class CConnman; class CScriptCheck; class CBlockPolicyEstimator; class CTxMemPool; +class CUndosView; class CValidationState; struct ChainTxData; @@ -437,7 +438,7 @@ class CVerifyDB { }; /** Replay blocks that aren't fully applied to the database. */ -bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache); +bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache, CUndosView* undosView); CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -725,9 +726,10 @@ class CChainState { bool AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: - DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, std::vector & disconnectedAnchorConfirms); - bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, CCustomCSView& cache, const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, CUndosView& undosView, + std::vector & disconnectedAnchorConfirms); + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, CCustomCSView& cache, CUndosView& undosView, + const CChainParams& chainparams, std::vector & rewardedAnchors, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. bool DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); @@ -737,7 +739,7 @@ class CChainState { bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache); + bool ReplayBlocks(const CChainParams& params, CCoinsView* view, CCustomCSView* cache, CUndosView* undosView); bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); bool LoadGenesisBlock(const CChainParams& chainparams); diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index ed19bc35dc..b0ffd07835 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -621,6 +621,7 @@ def cache_path(*paths): # Remove custom dirs shutil.rmtree(cache_path('burn')) + shutil.rmtree(cache_path('undos')) for entry in os.listdir(cache_path()): if entry not in ['chainstate', 'blocks', 'enhancedcs', 'anchors', 'history']: # Only keep chainstate and blocks folder