Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Captive core fast startup #2994

Merged
merged 5 commits into from
May 13, 2021
Merged

Captive core fast startup #2994

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4e92ccf
Move configuration validation up - check validity right after app cre…
marta-lokhova Apr 1, 2021
27e8d33
Application initialization: re-arrange bucket and database initilization
marta-lokhova Apr 2, 2021
ab2a445
Introduce new mode to store ledger headers only
marta-lokhova May 5, 2021
8257fae
Add additional peer loading and storing functionality
marta-lokhova May 10, 2021
7cde809
Allow rebuilding in-memory state from buckets
marta-lokhova May 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/bucket/BucketManagerImpl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,10 @@ BucketManagerImpl::BucketManagerImpl(Application& app)
, mBucketSnapMerge(app.getMetrics().NewTimer({"bucket", "snap", "merge"}))
, mSharedBucketsSize(
app.getMetrics().NewCounter({"bucket", "memory", "shared"}))
, mDeleteEntireBucketDirInDtor(app.getConfig().isInMemoryMode())
// Minimal DB is stored in the buckets dir, so delete it only when
// mode does not use minimal DB
, mDeleteEntireBucketDirInDtor(
app.getConfig().isInMemoryModeWithoutMinimalDB())
{
}

Expand Down
2 changes: 1 addition & 1 deletion src/herder/HerderImpl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ HerderImpl::processExternalized(uint64 slotIndex, StellarValue const& value)
TxSetFramePtr externalizedSet = mPendingEnvelopes.getTxSet(value.txSetHash);

// save the SCP messages in the database
if (mApp.getConfig().MODE_STORES_HISTORY)
if (mApp.getConfig().MODE_STORES_HISTORY_MISC)
{
mApp.getHerderPersistence().saveSCPHistory(
static_cast<uint32>(slotIndex),
Expand Down
3 changes: 2 additions & 1 deletion src/herder/test/HerderTests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2586,7 +2586,7 @@ TEST_CASE("slot herder policy", "[herder]")
Config cfg(getTestConfig());

// start in sync
cfg.FORCE_SCP = true;
cfg.FORCE_SCP = false;
MonsieurNicolas marked this conversation as resolved.
Show resolved Hide resolved
cfg.MANUAL_CLOSE = false;
cfg.NODE_SEED = v0SecretKey;
cfg.MAX_SLOTS_TO_REMEMBER = 5;
Expand All @@ -2599,6 +2599,7 @@ TEST_CASE("slot herder policy", "[herder]")

VirtualClock clock;
Application::pointer app = createTestApplication(clock, cfg);
app->start();
MonsieurNicolas marked this conversation as resolved.
Show resolved Hide resolved

auto& herder = static_cast<HerderImpl&>(app->getHerder());

Expand Down
10 changes: 5 additions & 5 deletions src/ledger/LedgerManagerImpl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ LedgerManagerImpl::loadLastKnownLedger(
CLOG_INFO(Ledger, "Last closed ledger (LCL) hash is {}", lastLedger);
Hash lastLedgerHash = hexToBin256(lastLedger);

if (mApp.getConfig().MODE_STORES_HISTORY)
if (mApp.getConfig().MODE_STORES_HISTORY_LEDGERHEADERS)
{
auto currentLedger =
LedgerHeaderUtils::loadByHash(getDatabase(), lastLedgerHash);
Expand Down Expand Up @@ -663,7 +663,7 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData)
}
// Note: Index from 1 rather than 0 to match the behavior of
// storeTransaction and storeTransactionFee.
if (mApp.getConfig().MODE_STORES_HISTORY)
if (mApp.getConfig().MODE_STORES_HISTORY_MISC)
{
Upgrades::storeUpgradeHistory(getDatabase(), ledgerSeq,
lupgrade, changes,
Expand Down Expand Up @@ -896,7 +896,7 @@ LedgerManagerImpl::processFeesSeqNums(
// txs counting from 1, not 0. We preserve this for the time being
// in case anyone depends on it.
++index;
if (mApp.getConfig().MODE_STORES_HISTORY)
if (mApp.getConfig().MODE_STORES_HISTORY_MISC)
{
storeTransactionFee(mApp.getDatabase(), ledgerSeq, tx, changes,
index);
Expand Down Expand Up @@ -1014,7 +1014,7 @@ LedgerManagerImpl::applyTransactions(
// txs counting from 1, not 0. We preserve this for the time being
// in case anyone depends on it.
++index;
if (mApp.getConfig().MODE_STORES_HISTORY)
if (mApp.getConfig().MODE_STORES_HISTORY_MISC)
{
auto ledgerSeq = ltx.loadHeader().current().ledgerSeq;
storeTransaction(mApp.getDatabase(), ledgerSeq, tx, tm,
Expand Down Expand Up @@ -1044,7 +1044,7 @@ void
LedgerManagerImpl::storeCurrentLedger(LedgerHeader const& header)
{
ZoneScoped;
if (mApp.getConfig().MODE_STORES_HISTORY)
if (mApp.getConfig().MODE_STORES_HISTORY_LEDGERHEADERS)
{
LedgerHeaderUtils::storeInDatabase(mApp.getDatabase(), header);
}
Expand Down
8 changes: 8 additions & 0 deletions src/main/Application.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ class Application

virtual AbstractLedgerTxnParent& getLedgerTxnRoot() = 0;

virtual void validateAndLogConfig() = 0;

// Factory: create a new Application object bound to `clock`, with a local
// copy made of `cfg`
static pointer create(VirtualClock& clock, Config const& cfg,
Expand All @@ -297,10 +299,16 @@ class Application
auto ret = std::make_shared<T>(clock, cfg, std::forward<Args>(args)...);
ret->initialize(newDB);
validateNetworkPassphrase(ret);
ret->validateAndLogConfig();

return ret;
}

// This method is used in in-memory mode: when rebuilding state from buckets
// is not possible, this method resets the database state back to genesis
// (while preserving the overlay data).
virtual void resetDBForInMemoryMode() = 0;

protected:
Application()
{
Expand Down
76 changes: 49 additions & 27 deletions src/main/ApplicationImpl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
#include "medida/timer.h"
#include "overlay/BanManager.h"
#include "overlay/OverlayManager.h"
#include "overlay/OverlayManagerImpl.h"
#include "process/ProcessManager.h"
#include "scp/LocalNode.h"
#include "scp/QuorumSetUtils.h"
Expand Down Expand Up @@ -135,13 +136,24 @@ ApplicationImpl::ApplicationImpl(VirtualClock& clock, Config const& cfg)
void
ApplicationImpl::initialize(bool createNewDB)
{
// Subtle: initialize the bucket manager first before initializing the
// database. This is needed as some modes in core (such as in-memory) use a
// small database inside the bucket directory.
mBucketManager = BucketManager::create(*this);
MonsieurNicolas marked this conversation as resolved.
Show resolved Hide resolved

bool initNewDB =
createNewDB || mConfig.DATABASE.value == "sqlite3://:memory:";
if (initNewDB)
{
mBucketManager->dropAll();
}

mDatabase = createDatabase();
mPersistentState = std::make_unique<PersistentState>(*this);
mOverlayManager = createOverlayManager();
mLedgerManager = createLedgerManager();
mHerder = createHerder();
mHerderPersistence = HerderPersistence::create(*this);
mBucketManager = BucketManager::create(*this);
mCatchupManager = CatchupManager::create(*this);
mHistoryArchiveManager = std::make_unique<HistoryArchiveManager>(*this);
mHistoryManager = HistoryManager::create(*this);
Expand Down Expand Up @@ -192,13 +204,13 @@ ApplicationImpl::initialize(bool createNewDB)
SponsorshipCountIsValid::registerInvariant(*this);
enableInvariantsFromConfig();

if (createNewDB || mConfig.DATABASE.value == "sqlite3://:memory:")
if (initNewDB)
{
newDB();
}
else
{
upgradeDB();
mDatabase->upgradeToCurrentSchema();
}

// Subtle: process manager should come to existence _after_ BucketManager
Expand All @@ -213,16 +225,9 @@ ApplicationImpl::newDB()
{
mDatabase->initialize();
mDatabase->upgradeToCurrentSchema();
mBucketManager->dropAll();
mLedgerManager->startNewLedger();
}

void
ApplicationImpl::upgradeDB()
{
mDatabase->upgradeToCurrentSchema();
}

void
ApplicationImpl::reportCfgMetrics()
{
Expand Down Expand Up @@ -394,28 +399,27 @@ ApplicationImpl::~ApplicationImpl()
LOG_INFO(DEFAULT_LOG, "Application destroyed");
}

void
ApplicationImpl::resetDBForInMemoryMode()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This worries me / reminds me a bit of a "schema migration", or just .. an alternative path for creating a DB (albeit a minimal one) that does not necessarily arrive at the same place we'd arrive if we reinitialized. I gather you're trying to preserve the overlay state here (but nothing else); would it be possible to do that explicitly? That is:

  1. load all the overlay state you want to preserve into a temporary data structure in memory.
  2. reinitialize the database using the normal code path.
  3. reinsert just the overlay state you wanted to preserve (data which we're a bit lax about anyways)

{
// Load the peer information and reinitialize the DB
auto& pm = getOverlayManager().getPeerManager();
auto peerData = pm.loadAllPeers();
newDB();
pm.storePeers(peerData);

LOG_INFO(DEFAULT_LOG, "In-memory state is reset back to genesis");
}

uint64_t
ApplicationImpl::timeNow()
{
return VirtualClock::to_time_t(getClock().system_now());
}

void
ApplicationImpl::start()
ApplicationImpl::validateAndLogConfig()
{
if (mStarted)
{
CLOG_INFO(Ledger, "Skipping application start up");
return;
}
CLOG_INFO(Ledger, "Starting up application");
mStarted = true;

if (mConfig.TESTING_UPGRADE_DATETIME.time_since_epoch().count() != 0)
{
mHerder->setUpgrades(mConfig);
}

if (mConfig.FORCE_SCP && !mConfig.NODE_IS_VALIDATOR)
{
throw std::invalid_argument(
Expand Down Expand Up @@ -454,10 +458,11 @@ ApplicationImpl::start()

if (getHistoryArchiveManager().hasAnyWritableHistoryArchive())
{
if (!mConfig.MODE_STORES_HISTORY)
if (!mConfig.modeStoresAllHistory())
{
throw std::invalid_argument("MODE_STORES_HISTORY is not set, but "
"some history archives are writable");
throw std::invalid_argument(
"Core is not configured to store history, but "
"some history archives are writable");
}
}

Expand All @@ -467,6 +472,23 @@ ApplicationImpl::start()
}

mConfig.logBasicInfo();
}

void
ApplicationImpl::start()
{
if (mStarted)
{
CLOG_INFO(Ledger, "Skipping application start up");
return;
}
CLOG_INFO(Ledger, "Starting up application");
mStarted = true;

if (mConfig.TESTING_UPGRADE_DATETIME.time_since_epoch().count() != 0)
{
mHerder->setUpgrades(mConfig);
}

bool done = false;
mLedgerManager->loadLastKnownLedger([this,
Expand Down
7 changes: 5 additions & 2 deletions src/main/ApplicationImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ class ApplicationImpl : public Application
// returns.
virtual void joinAllThreads() override;

virtual void validateAndLogConfig() override;

virtual std::string
manualClose(std::optional<uint32_t> const& manualLedgerSeq,
std::optional<TimePoint> const& manualCloseTime) override;
Expand All @@ -120,6 +122,8 @@ class ApplicationImpl : public Application

virtual AbstractLedgerTxnParent& getLedgerTxnRoot() override;

virtual void resetDBForInMemoryMode() override;

protected:
std::unique_ptr<LedgerManager>
mLedgerManager; // allow to change that for tests
Expand All @@ -143,9 +147,9 @@ class ApplicationImpl : public Application
asio::io_context mWorkerIOContext;
std::unique_ptr<asio::io_context::work> mWork;

std::unique_ptr<BucketManager> mBucketManager;
std::unique_ptr<Database> mDatabase;
std::unique_ptr<OverlayManager> mOverlayManager;
std::unique_ptr<BucketManager> mBucketManager;
std::unique_ptr<CatchupManager> mCatchupManager;
std::unique_ptr<HerderPersistence> mHerderPersistence;
std::unique_ptr<HistoryArchiveManager> mHistoryArchiveManager;
Expand Down Expand Up @@ -195,7 +199,6 @@ class ApplicationImpl : public Application
Hash mNetworkID;

void newDB();
void upgradeDB();

void shutdownMainIOContext();
void shutdownWorkScheduler();
Expand Down
Loading