Skip to content

Commit

Permalink
Merge #19927: validation: Reduce direct g_chainman usage
Browse files Browse the repository at this point in the history 
72a1d5c validation: Remove review-only comments + assertions (Carl Dong)
3756853 docs: Move FindFilesToPrune{,Manual} doxygen comment (Carl Dong)
485899a style: Make FindFilesToPrune{,Manual} match style guide (Carl Dong)
3f5b5f3 validation: Move FindFilesToPrune{,Manual} to BlockManager (Carl Dong)
f8d4975 validation: Move PruneOneBlockFile to BlockManager (Carl Dong)
74f73c7 validation: Pass in chainman to UnloadBlockIndex (Carl Dong)
4668ded validation: Move ~CMainCleanup logic to ~BlockManager (Carl Dong)

Pull request description:

  This PR paves the way for de-globalizing `g_chainman` entirely by removing the usage of `g_chainman` in the following functions/methods:
  - `~CMainCleanup`
  - `CChainState::FlushStateToDisk`
  - `UnloadBlockIndex`

  The remaining direct uses of `g_chainman` are as follows:
  1. In initialization codepaths:
  	- `AppTests`
  	- `AppInitMain`
  	- `TestingSetup::TestingSetup`
  2. `::ChainstateActive`
  3. `LookupBlockIndex`
  	- Note: `LookupBlockIndex` is used extensively throughout the codebase and require a much larger set of changes, therefore I've left it out of this initial PR

ACKs for top commit:
  MarcoFalke:
    re-ACK 72a1d5c 👚
  jnewbery:
    utACK 72a1d5c

Tree-SHA512: 944a4fa8405eecf39706ff944375d6824373aaeea849d11473f08181eff26b12f70043a8348a5b08e6e9021b243b481842fbdfbc7c3140ca795fce3688b7f5c3
  • Loading branch information
MarcoFalke committed Sep 23, 2020
2 parents 43305e9 + 72a1d5c commit 1b313ca
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 69 deletions.
2 changes: 1 addition & 1 deletion src/init.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1563,7 +1563,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA
chainman.m_total_coinstip_cache = nCoinCacheUsage;
chainman.m_total_coinsdb_cache = nCoinDBCache;

UnloadBlockIndex(node.mempool.get());
UnloadBlockIndex(node.mempool.get(), chainman);

// new CBlockTreeDB tries to delete the existing file, which
// fails if it's still open from the previous loop. Close it first:
Expand Down
7 changes: 5 additions & 2 deletions src/qt/test/apptests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,11 @@ void AppTests::appTests()
// Reset global state to avoid interfering with later tests.
LogInstance().DisconnectTestLogger();
AbortShutdown();
UnloadBlockIndex(/* mempool */ nullptr);
WITH_LOCK(::cs_main, g_chainman.Reset());
{
LOCK(cs_main);
UnloadBlockIndex(/* mempool */ nullptr, g_chainman);
g_chainman.Reset();
}
}

//! Entry point for BitcoinGUI tests.
Expand Down
2 changes: 1 addition & 1 deletion src/test/util/setup_common.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ TestingSetup::~TestingSetup()
m_node.connman.reset();
m_node.banman.reset();
m_node.args = nullptr;
UnloadBlockIndex(m_node.mempool.get());
UnloadBlockIndex(m_node.mempool.get(), *m_node.chainman);
m_node.mempool.reset();
m_node.scheduler.reset();
m_node.chainman->Reset();
Expand Down
86 changes: 29 additions & 57 deletions src/validation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,6 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc

std::unique_ptr<CBlockTreeDB> pblocktree;

// See definition for documentation
static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight);
static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight);
bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr);
static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false);
static FlatFileSeq BlockFileSeq();
Expand Down Expand Up @@ -2299,11 +2296,11 @@ bool CChainState::FlushStateToDisk(
if (nManualPruneHeight > 0) {
LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH);

FindFilesToPruneManual(g_chainman, setFilesToPrune, nManualPruneHeight);
m_blockman.FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight, m_chain.Height());
} else {
LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH);

FindFilesToPrune(g_chainman, setFilesToPrune, chainparams.PruneAfterHeight());
m_blockman.FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight(), m_chain.Height(), IsInitialBlockDownload());
fCheckForPruning = false;
}
if (!setFilesToPrune.empty()) {
Expand Down Expand Up @@ -3909,12 +3906,12 @@ uint64_t CalculateCurrentUsage()
return retval;
}

void ChainstateManager::PruneOneBlockFile(const int fileNumber)
void BlockManager::PruneOneBlockFile(const int fileNumber)
{
AssertLockHeld(cs_main);
LOCK(cs_LastBlockFile);

for (const auto& entry : m_blockman.m_block_index) {
for (const auto& entry : m_block_index) {
CBlockIndex* pindex = entry.second;
if (pindex->nFile == fileNumber) {
pindex->nStatus &= ~BLOCK_HAVE_DATA;
Expand All @@ -3928,12 +3925,12 @@ void ChainstateManager::PruneOneBlockFile(const int fileNumber)
// to be downloaded again in order to consider its chain, at which
// point it would be considered as a candidate for
// m_blocks_unlinked or setBlockIndexCandidates.
auto range = m_blockman.m_blocks_unlinked.equal_range(pindex->pprev);
auto range = m_blocks_unlinked.equal_range(pindex->pprev);
while (range.first != range.second) {
std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first;
range.first++;
if (_it->second == pindex) {
m_blockman.m_blocks_unlinked.erase(_it);
m_blocks_unlinked.erase(_it);
}
}
}
Expand All @@ -3954,22 +3951,23 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
}
}

/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight)
void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height)
{
assert(fPruneMode && nManualPruneHeight > 0);

LOCK2(cs_main, cs_LastBlockFile);
if (::ChainActive().Tip() == nullptr)
if (chain_tip_height < 0) {
return;
}

// last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip)
unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, ::ChainActive().Tip()->nHeight - MIN_BLOCKS_TO_KEEP);
int count=0;
unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP);
int count = 0;
for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune)
if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) {
continue;
chainman.PruneOneBlockFile(fileNumber);
}
PruneOneBlockFile(fileNumber);
setFilesToPrune.insert(fileNumber);
count++;
}
Expand All @@ -3987,64 +3985,52 @@ void PruneBlockFilesManual(int nManualPruneHeight)
}
}

/**
* Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target.
* The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
* space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
* (which in this case means the blockchain must be re-downloaded.)
*
* Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set.
* Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
* Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 1000 on regtest).
* Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
* The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
* A db flag records the fact that at least some block files have been pruned.
*
* @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
*/
static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight)
void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, bool is_ibd)
{
LOCK2(cs_main, cs_LastBlockFile);
if (::ChainActive().Tip() == nullptr || nPruneTarget == 0) {
if (chain_tip_height < 0 || nPruneTarget == 0) {
return;
}
if ((uint64_t)::ChainActive().Tip()->nHeight <= nPruneAfterHeight) {
if ((uint64_t)chain_tip_height <= nPruneAfterHeight) {
return;
}

unsigned int nLastBlockWeCanPrune = ::ChainActive().Tip()->nHeight - MIN_BLOCKS_TO_KEEP;
unsigned int nLastBlockWeCanPrune = chain_tip_height - MIN_BLOCKS_TO_KEEP;
uint64_t nCurrentUsage = CalculateCurrentUsage();
// We don't check to prune until after we've allocated new space for files
// So we should leave a buffer under our target to account for another allocation
// before the next pruning.
uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
uint64_t nBytesToPrune;
int count=0;
int count = 0;

if (nCurrentUsage + nBuffer >= nPruneTarget) {
// On a prune event, the chainstate DB is flushed.
// To avoid excessive prune events negating the benefit of high dbcache
// values, we should not prune too rapidly.
// So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon.
if (::ChainstateActive().IsInitialBlockDownload()) {
if (is_ibd) {
// Since this is only relevant during IBD, we use a fixed 10%
nBuffer += nPruneTarget / 10;
}

for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize;

if (vinfoBlockFile[fileNumber].nSize == 0)
if (vinfoBlockFile[fileNumber].nSize == 0) {
continue;
}

if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target?
if (nCurrentUsage + nBuffer < nPruneTarget) { // are we below our target?
break;
}

// don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning
if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune)
if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) {
continue;
}

chainman.PruneOneBlockFile(fileNumber);
PruneOneBlockFile(fileNumber);
// Queue up the files for removal
setFilesToPrune.insert(fileNumber);
nCurrentUsage -= nBytesToPrune;
Expand Down Expand Up @@ -4602,10 +4588,10 @@ void CChainState::UnloadBlockIndex() {
// May NOT be used after any connections are up as much
// of the peer-processing logic assumes a consistent
// block index state
void UnloadBlockIndex(CTxMemPool* mempool)
void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman)
{
LOCK(cs_main);
g_chainman.Unload();
chainman.Unload();
pindexBestInvalid = nullptr;
pindexBestHeader = nullptr;
if (mempool) mempool->clear();
Expand Down Expand Up @@ -5220,20 +5206,6 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin
return std::min<double>(pindex->nChainTx / fTxTotal, 1.0);
}

class CMainCleanup
{
public:
CMainCleanup() {}
~CMainCleanup() {
// block headers
BlockMap::iterator it1 = g_chainman.BlockIndex().begin();
for (; it1 != g_chainman.BlockIndex().end(); it1++)
delete (*it1).second;
g_chainman.BlockIndex().clear();
}
};
static CMainCleanup instance_of_cmaincleanup;

Optional<uint256> ChainstateManager::SnapshotBlockhash() const {
if (m_active_chainstate != nullptr) {
// If a snapshot chainstate exists, it will always be our active.
Expand Down
38 changes: 33 additions & 5 deletions src/validation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi
/** Ensures we have a genesis block in the block tree, possibly writing one to disk. */
bool LoadGenesisBlock(const CChainParams& chainparams);
/** Unload database information */
void UnloadBlockIndex(CTxMemPool* mempool);
void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman);
/** Run an instance of the script checking thread */
void ThreadScriptCheck(int worker_num);
/**
Expand Down Expand Up @@ -352,7 +352,31 @@ struct CBlockIndexWorkComparator
* This data is used mostly in `CChainState` - information about, e.g.,
* candidate tips is not maintained here.
*/
class BlockManager {
class BlockManager
{
friend CChainState;

private:
/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height);

/**
* Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target.
* The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
* space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
* (which in this case means the blockchain must be re-downloaded.)
*
* Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set.
* Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
* Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 1000 on regtest).
* Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
* The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
* A db flag records the fact that at least some block files have been pruned.
*
* @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
*/
void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, bool is_ibd);

public:
BlockMap m_block_index GUARDED_BY(cs_main);

Expand Down Expand Up @@ -403,6 +427,9 @@ class BlockManager {
/** Create a new block index entry for a given block hash */
CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);

//! Mark one block file as pruned (modify associated database entries)
void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main);

/**
* If a block header hasn't already been seen, call CheckBlockHeader on it, ensure
* that it doesn't descend from an invalid block, and then add it to m_block_index.
Expand All @@ -412,6 +439,10 @@ class BlockManager {
BlockValidationState& state,
const CChainParams& chainparams,
CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main);

~BlockManager() {
Unload();
}
};

/**
Expand Down Expand Up @@ -895,9 +926,6 @@ class ChainstateManager
*/
bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main);

//! Mark one block file as pruned (modify associated database entries)
void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main);

//! Load the block tree and coins database from disk, initializing state if we're running with -reindex
bool LoadBlockIndex(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main);

Expand Down
6 changes: 3 additions & 3 deletions src/wallet/test/wallet_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup)
// Prune the older block file.
{
LOCK(cs_main);
Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile);
Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(oldTip->GetBlockPos().nFile);
}
UnlinkPrunedFiles({oldTip->GetBlockPos().nFile});

Expand All @@ -152,7 +152,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup)
// Prune the remaining block file.
{
LOCK(cs_main);
Assert(m_node.chainman)->PruneOneBlockFile(newTip->GetBlockPos().nFile);
Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(newTip->GetBlockPos().nFile);
}
UnlinkPrunedFiles({newTip->GetBlockPos().nFile});

Expand Down Expand Up @@ -189,7 +189,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup)
// Prune the older block file.
{
LOCK(cs_main);
Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile);
Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(oldTip->GetBlockPos().nFile);
}
UnlinkPrunedFiles({oldTip->GetBlockPos().nFile});

Expand Down

0 comments on commit 1b313ca

Please sign in to comment.