Track token and pool splits in account history

src/flushablestorage.h

@@ -506,7 +506,7 @@ class CStorageView {
         }
     }
 
-    bool Flush() { return DB().Flush(); }
+    virtual bool Flush() { return DB().Flush(); }
     void Discard() { DB().Discard(); }
     size_t SizeEstimate() const { return DB().SizeEstimate(); }
 

src/init.cpp

@@ -1720,10 +1720,12 @@ bool AppInitMain(InitInterfaces& interfaces)
                 paccountHistoryDB.reset();
                 if (gArgs.GetBoolArg("-acindex", DEFAULT_ACINDEX)) {
                     paccountHistoryDB = std::make_unique<CAccountHistoryStorage>(GetDataDir() / "history", nCustomCacheSize, false, fReset || fReindexChainState);
+                    paccountHistoryDB->CreateMultiIndexIfNeeded();
                 }
 
                 pburnHistoryDB.reset();
                 pburnHistoryDB = std::make_unique<CBurnHistoryStorage>(GetDataDir() / "burn", nCustomCacheSize, false, fReset || fReindexChainState);
+                pburnHistoryDB->CreateMultiIndexIfNeeded();
 
                 // Create vault history DB
                 pvaultHistoryDB.reset();

src/masternodes/accountshistory.cpp

@@ -8,25 +8,112 @@
 #include <masternodes/vaulthistory.h>
 #include <key_io.h>
 
-void CAccountsHistoryView::ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, CLazySerialize<AccountHistoryValue>)> callback, AccountHistoryKey const & start)
+struct AccountHistoryKeyNew {
+    uint32_t blockHeight;
+    CScript owner;
+    uint32_t txn; // for order in block
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        if (ser_action.ForRead()) {
+            READWRITE(WrapBigEndian(blockHeight));
+            blockHeight = ~blockHeight;
+        } else {
+            uint32_t blockHeight_ = ~blockHeight;
+            READWRITE(WrapBigEndian(blockHeight_));
+        }
+
+        READWRITE(owner);
+
+        if (ser_action.ForRead()) {
+            READWRITE(WrapBigEndian(txn));
+            txn = ~txn;
+        } else {
+            uint32_t txn_ = ~txn;
+            READWRITE(WrapBigEndian(txn_));
+        }
+    }
+};
+
+static AccountHistoryKeyNew Convert(AccountHistoryKey const & key) {
+    return {key.blockHeight, key.owner, key.txn};
+}
+
+static AccountHistoryKey Convert(AccountHistoryKeyNew const & key) {
+    return {key.owner, key.blockHeight, key.txn};
+}
+
+void CAccountsHistoryView::CreateMultiIndexIfNeeded()
 {
-    ForEach<ByAccountHistoryKey, AccountHistoryKey, AccountHistoryValue>(callback, start);
+    AccountHistoryKeyNew anyNewKey{~0u, {}, ~0u};
+    if (auto it = LowerBound<ByAccountHistoryKeyNew>(anyNewKey); it.Valid()) {
+        return;
+    }
+
+    LogPrintf("Adding multi index in progress...\n");
+
+    auto startTime = GetTimeMillis();
+
+    AccountHistoryKey startKey{{}, ~0u, ~0u};
+    auto it = LowerBound<ByAccountHistoryKey>(startKey);
+    for (; it.Valid(); it.Next()) {
+        WriteBy<ByAccountHistoryKeyNew>(Convert(it.Key()), '\0');
+    }
+
+    Flush();
+
+    LogPrint(BCLog::BENCH, "    - Multi index took: %dms\n", GetTimeMillis() - startTime);
 }
 
-Res CAccountsHistoryView::WriteAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value)
+void CAccountsHistoryView::ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, AccountHistoryValue)> callback,
+                                                 const CScript & owner, uint32_t height, uint32_t txn)
 {
-    WriteBy<ByAccountHistoryKey>(key, value);
-    return Res::Ok();
+    if (!owner.empty()) {
+        ForEach<ByAccountHistoryKey, AccountHistoryKey, AccountHistoryValue>(callback, {owner, height, txn});
+        return;
+    }
+
+    ForEach<ByAccountHistoryKeyNew, AccountHistoryKeyNew, char>([&](AccountHistoryKeyNew const & newKey, char) {
+        auto key = Convert(newKey);
+        auto value = ReadAccountHistory(key);
+        assert(value);
+        return callback(key, *value);
+    }, {height, owner, txn});
 }
 
 std::optional<AccountHistoryValue> CAccountsHistoryView::ReadAccountHistory(AccountHistoryKey const & key) const
 {
     return ReadBy<ByAccountHistoryKey, AccountHistoryValue>(key);
 }
 
+Res CAccountsHistoryView::WriteAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value)
+{
+    WriteBy<ByAccountHistoryKey>(key, value);
+    WriteBy<ByAccountHistoryKeyNew>(Convert(key), '\0');
+    return Res::Ok();
+}
+
 Res CAccountsHistoryView::EraseAccountHistory(const AccountHistoryKey& key)
 {
     EraseBy<ByAccountHistoryKey>(key);
+    EraseBy<ByAccountHistoryKeyNew>(Convert(key));
+    return Res::Ok();
+}
+
+Res CAccountsHistoryView::EraseAccountHistoryHeight(uint32_t height)
+{
+    std::vector<AccountHistoryKey> keysToDelete;
+
+    auto it = LowerBound<ByAccountHistoryKeyNew>(AccountHistoryKeyNew{height, {}, ~0u});
+    for (; it.Valid() && it.Key().blockHeight == height; it.Next()) {
+        keysToDelete.push_back(Convert(it.Key()));
+    }
+
+    for (const auto& key : keysToDelete) {
+        EraseAccountHistory(key);
+    }
     return Res::Ok();
 }
 
@@ -75,78 +162,9 @@ bool CAccountsHistoryWriter::Flush()
     return CCustomCSView::Flush();
 }
 
-CAccountsHistoryEraser::CAccountsHistoryEraser(CCustomCSView & storage, uint32_t height, uint32_t txn, CHistoryErasers& erasers)
-    : CStorageView(new CFlushableStorageKV(static_cast<CStorageKV&>(storage.GetStorage()))), height(height), txn(txn), erasers(erasers)
-{
-}
-
-Res CAccountsHistoryEraser::AddBalance(CScript const & owner, CTokenAmount)
-{
-    erasers.AddBalance(owner, vaultID);
-    return Res::Ok();
-}
-
-Res CAccountsHistoryEraser::SubBalance(CScript const & owner, CTokenAmount)
-{
-    erasers.SubBalance(owner, vaultID);
-    return Res::Ok();
-}
-
-bool CAccountsHistoryEraser::Flush()
-{
-    erasers.Flush(height, txn, vaultID);
-    return Res::Ok(); // makes sure no changes are applyed to underlaying view
-}
-
-CHistoryErasers::CHistoryErasers(CAccountHistoryStorage* historyView, CBurnHistoryStorage* burnView, CVaultHistoryStorage* vaultView)
-    : historyView(historyView), burnView(burnView), vaultView(vaultView) {}
-
-void CHistoryErasers::AddBalance(const CScript& owner, const uint256& vaultID)
-{
-    if (historyView) {
-        accounts.insert(owner);
-    }
-    if (burnView && owner == Params().GetConsensus().burnAddress) {
-        burnAccounts.insert(owner);
-    }
-    if (vaultView && !vaultID.IsNull()) {
-        vaults.insert(vaultID);
-    }
-}
-
-void CHistoryErasers::SubFeeBurn(const CScript& owner)
-{
-    if (burnView) {
-        burnAccounts.insert(owner);
-    }
-}
-
-void CHistoryErasers::SubBalance(const CScript& owner, const uint256& vaultID)
-{
-    if (historyView) {
-        accounts.insert(owner);
-    }
-    if (burnView && owner == Params().GetConsensus().burnAddress) {
-        burnAccounts.insert(owner);
-    }
-    if (vaultView && !vaultID.IsNull()) {
-        vaults.insert(vaultID);
-    }
-}
-
-void CHistoryErasers::Flush(const uint32_t height, const uint32_t txn, const uint256& vaultID)
-{
-    if (historyView) {
-        for (const auto& account : accounts) {
-            historyView->EraseAccountHistory({account, height, txn});
-        }
-    }
-    if (burnView) {
-        for (const auto& account : burnAccounts) {
-            burnView->EraseAccountHistory({account, height, txn});
-        }
-    }
-}
+CAccountHistoryStorage* CAccountsHistoryWriter::GetAccountHistoryStore() {
+    return writers ? writers->GetAccountHistoryStore() : nullptr;
+};
 
 CHistoryWriters::CHistoryWriters(CAccountHistoryStorage* historyView, CBurnHistoryStorage* burnView, CVaultHistoryStorage* vaultView)
     : historyView(historyView), burnView(burnView), vaultView(vaultView) {}

src/masternodes/accountshistory.h

@@ -59,19 +59,24 @@ struct AccountHistoryValue {
 class CAccountsHistoryView : public virtual CStorageView
 {
 public:
+    void CreateMultiIndexIfNeeded();
+    Res EraseAccountHistoryHeight(uint32_t height);
+    [[nodiscard]] std::optional<AccountHistoryValue> ReadAccountHistory(AccountHistoryKey const & key) const;
     Res WriteAccountHistory(AccountHistoryKey const & key, AccountHistoryValue const & value);
-    std::optional<AccountHistoryValue> ReadAccountHistory(AccountHistoryKey const & key) const;
     Res EraseAccountHistory(AccountHistoryKey const & key);
-    void ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, CLazySerialize<AccountHistoryValue>)> callback, AccountHistoryKey const & start = {});
+    void ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, AccountHistoryValue)> callback,
+                               const CScript& owner = {}, uint32_t height = std::numeric_limits<uint32_t>::max(), uint32_t txn = std::numeric_limits<uint32_t>::max());
 
     // tags
-    struct ByAccountHistoryKey { static constexpr uint8_t prefix() { return 'h'; } };
+    struct ByAccountHistoryKey    { static constexpr uint8_t prefix() { return 'h'; } };
+    struct ByAccountHistoryKeyNew { static constexpr uint8_t prefix() { return 'H'; } };
 };
 
 class CAccountHistoryStorage : public CAccountsHistoryView
                              , public CAuctionHistoryView
 {
 public:
+    CAccountHistoryStorage(CAccountHistoryStorage& accountHistory) : CStorageView(new CFlushableStorageKV(accountHistory.DB())) {}
     CAccountHistoryStorage(const fs::path& dbName, std::size_t cacheSize, bool fMemory = false, bool fWipe = false);
 };
 
@@ -82,75 +87,44 @@ class CBurnHistoryStorage : public CAccountsHistoryView
 };
 
 class CHistoryWriters {
-    CAccountHistoryStorage* historyView;
-    CBurnHistoryStorage* burnView;
+    CAccountHistoryStorage* historyView{};
+    CBurnHistoryStorage* burnView{};
     std::map<CScript, TAmounts> diffs;
     std::map<CScript, TAmounts> burnDiffs;
     std::map<uint256, std::map<CScript,TAmounts>> vaultDiffs;
 
 public:
-    CVaultHistoryStorage* vaultView;
+    CVaultHistoryStorage* vaultView{};
     CLoanSchemeCreation globalLoanScheme;
     std::string schemeID;
 
     CHistoryWriters(CAccountHistoryStorage* historyView, CBurnHistoryStorage* burnView, CVaultHistoryStorage* vaultView);
 
+    CAccountHistoryStorage* GetAccountHistoryStore() { return historyView; };
+
     void AddBalance(const CScript& owner, const CTokenAmount amount, const uint256& vaultID);
     void AddFeeBurn(const CScript& owner, const CAmount amount);
     void SubBalance(const CScript& owner, const CTokenAmount amount, const uint256& vaultID);
     void Flush(const uint32_t height, const uint256& txid, const uint32_t txn, const uint8_t type, const uint256& vaultID);
 };
 
-class CHistoryErasers {
-    CAccountHistoryStorage* historyView;
-    CBurnHistoryStorage* burnView;
-    std::set<CScript> accounts;
-    std::set<CScript> burnAccounts;
-    std::set<uint256> vaults;
-
-public:
-    CVaultHistoryStorage* vaultView;
-    bool removeLoanScheme{false};
-    uint256 schemeCreationTxid;
-
-    CHistoryErasers(CAccountHistoryStorage* historyView, CBurnHistoryStorage* burnView, CVaultHistoryStorage* vaultView);
-
-    void AddBalance(const CScript& owner, const uint256& vaultID);
-    void SubFeeBurn(const CScript& owner);
-    void SubBalance(const CScript& owner, const uint256& vaultID);
-    void Flush(const uint32_t height, const uint32_t txn, const uint256& vaultID);
-};
-
 class CAccountsHistoryWriter : public CCustomCSView
 {
     const uint32_t height;
     const uint32_t txn;
     const uint256 txid;
     const uint8_t type;
-    CHistoryWriters* writers;
+    CHistoryWriters* writers{};
 
 public:
     uint256 vaultID;
 
     CAccountsHistoryWriter(CCustomCSView & storage, uint32_t height, uint32_t txn, const uint256& txid, uint8_t type, CHistoryWriters* writers);
     Res AddBalance(CScript const & owner, CTokenAmount amount) override;
     Res SubBalance(CScript const & owner, CTokenAmount amount) override;
-    bool Flush();
-};
+    bool Flush() override;
 
-class CAccountsHistoryEraser : public CCustomCSView
-{
-    const uint32_t height;
-    const uint32_t txn;
-    CHistoryErasers& erasers;
-
-public:
-    uint256 vaultID;
-
-    CAccountsHistoryEraser(CCustomCSView & storage, uint32_t height, uint32_t txn, CHistoryErasers& erasers);
-    Res AddBalance(CScript const & owner, CTokenAmount amount) override;
-    Res SubBalance(CScript const & owner, CTokenAmount amount) override;
-    bool Flush();
+    CAccountHistoryStorage* GetAccountHistoryStore() override;
 };
 
 extern std::unique_ptr<CAccountHistoryStorage> paccountHistoryDB;

src/masternodes/govvariables/attributes.cpp

@@ -6,8 +6,8 @@
 
 #include <masternodes/accountshistory.h> /// CAccountsHistoryWriter
 #include <masternodes/masternodes.h> /// CCustomCSView
-#include <masternodes/mn_checks.h> /// GetAggregatePrice
-#include <masternodes/mn_checks.h> /// CustomTxType
+#include <masternodes/mn_checks.h> /// GetAggregatePrice / CustomTxType
+#include <validation.h> /// GetNextAccPosition
 
 #include <amount.h> /// GetDecimaleString
 #include <core_io.h> /// ValueFromAmount
@@ -548,22 +548,25 @@ Res ATTRIBUTES::RefundFuturesContracts(CCustomCSView &mnview, const uint32_t hei
     CDataStructureV0 liveKey{AttributeTypes::Live, ParamIDs::Economy, EconomyKeys::DFIP2203Current};
     auto balances = GetValue(liveKey, CBalances{});
 
-    auto txn = std::numeric_limits<uint32_t>::max();
+    CAccountHistoryStorage* historyStore{mnview.GetAccountHistoryStore()};
+    const auto currentHeight = mnview.GetLastHeight() + 1;
 
     for (const auto& [key, value] : userFuturesValues) {
 
         mnview.EraseFuturesUserValues(key);
 
-        CHistoryWriters subWriters{paccountHistoryDB.get(), nullptr, nullptr};
-        CAccountsHistoryWriter subView(mnview, height, txn--, {}, uint8_t(CustomTxType::FutureSwapRefund), &subWriters);
+        CHistoryWriters subWriters{historyStore, nullptr, nullptr};
+        CAccountsHistoryWriter subView(mnview, currentHeight, GetNextAccPosition(), {}, uint8_t(CustomTxType::FutureSwapRefund), &subWriters);
+
         auto res = subView.SubBalance(*contractAddressValue, value.source);
         if (!res) {
             return res;
         }
         subView.Flush();
 
-        CHistoryWriters addWriters{paccountHistoryDB.get(), nullptr, nullptr};
-        CAccountsHistoryWriter addView(mnview, height, txn--, {}, uint8_t(CustomTxType::FutureSwapRefund), &addWriters);
+        CHistoryWriters addWriters{historyStore, nullptr, nullptr};
+        CAccountsHistoryWriter addView(mnview, currentHeight, GetNextAccPosition(), {}, uint8_t(CustomTxType::FutureSwapRefund), &addWriters);
+
         res = addView.AddBalance(key.owner, value.source);
         if (!res) {
             return res;