Better handling of account history

src/flushablestorage.h

@@ -313,10 +313,29 @@ class CFlushableStorageKV : public CStorageKV {
     MapKV changed;
 };
 
+template<typename T>
+struct CLazySerialize
+{
+    CStorageKVIterator& it;
+
+    operator T()
+    {
+        return get();
+    }
+
+    T get()
+    {
+        T value;
+        BytesToDbType(it.Value(), value);
+        return value;
+    }
+};
+
 class CStorageView {
 public:
+    CStorageView() = default;
     CStorageView(CStorageKV * st) : storage(st) {}
-    CStorageView() {}
+    virtual ~CStorageView() = default;
 
     template<typename KeyType>
     bool Exists(const KeyType& key) const {
@@ -374,18 +393,15 @@ class CStorageView {
     }
 
     template<typename By, typename KeyType, typename ValueType>
-    bool ForEach(std::function<bool(KeyType const &, ValueType &)> callback, KeyType const & start = KeyType()) const {
+    bool ForEach(std::function<bool(KeyType const &, CLazySerialize<ValueType>)> callback, KeyType const & start = KeyType()) const {
         auto& self = const_cast<CStorageView&>(*this);
         auto key = std::make_pair(By::prefix, start);
 
         auto it = self.DB().NewIterator();
         for(it->Seek(DbTypeToBytes(key)); it->Valid() && (BytesToDbType(it->Key(), key), key.first == By::prefix); it->Next()) {
             boost::this_thread::interruption_point();
 
-            ValueType value;
-            BytesToDbType(it->Value(), value);
-
-            if (!callback(key.second, value))
+            if (!callback(key.second, CLazySerialize<ValueType>{*it}))
                 break;
         }
         return true;

src/init.cpp

@@ -1585,6 +1585,21 @@ bool AppInitMain(InitInterfaces& interfaces)
                 pcustomcsDB = MakeUnique<CStorageLevelDB>(GetDataDir() / "enhancedcs", nMinDbCache << 20, false, fReset || fReindexChainState);
                 pcustomcsview.reset();
                 pcustomcsview = MakeUnique<CCustomCSView>(*pcustomcsDB.get());
+                if (!fReset && gArgs.GetBoolArg("-acindex", false)) {
+                    bool hasAccountHistory = false, hasRewardHistory = false;
+                    pcustomcsview->ForEachAccountHistory([&](AccountHistoryKey const &, CLazySerialize<AccountHistoryValue>) {
+                        hasAccountHistory = true;
+                        return false;
+                    });
+                    pcustomcsview->ForEachRewardHistory([&](RewardHistoryKey const &, CLazySerialize<RewardHistoryValue>) {
+                        hasRewardHistory = true;
+                        return false;
+                    });
+                    if (hasAccountHistory && !hasRewardHistory) {
+                        strLoadError = _("Account history needs rebuild").translated;
+                        break;
+                    }
+                }
 
                 panchorauths.reset();
                 panchorauths = MakeUnique<CAnchorAuthIndex>();

src/masternodes/accounts.cpp

@@ -7,9 +7,9 @@
 /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!!
 const unsigned char CAccountsView::ByBalanceKey::prefix = 'a';
 
-void CAccountsView::ForEachBalance(std::function<bool(CScript const & owner, CTokenAmount const & amount)> callback, BalanceKey start) const
+void CAccountsView::ForEachBalance(std::function<bool(CScript const &, CTokenAmount const &)> callback, BalanceKey const & start) const
 {
-    ForEach<ByBalanceKey, BalanceKey, CAmount>([&callback] (BalanceKey const & key, CAmount const & val) {
+    ForEach<ByBalanceKey, BalanceKey, CAmount>([&callback] (BalanceKey const & key, CAmount val) {
         return callback(key.owner, CTokenAmount{key.tokenID, val});
     }, start);
 }

src/masternodes/accounts.h

@@ -14,12 +14,13 @@
 class CAccountsView : public virtual CStorageView
 {
 public:
-    void ForEachBalance(std::function<bool(CScript const & owner, CTokenAmount const & amount)> callback, BalanceKey start = {}) const;
+    void ForEachBalance(std::function<bool(CScript const &, CTokenAmount const &)> callback, BalanceKey const & start = {}) const;
     CTokenAmount GetBalance(CScript const & owner, DCT_ID tokenID) const;
 
-    Res AddBalance(CScript const & owner, CTokenAmount amount);
+    virtual Res AddBalance(CScript const & owner, CTokenAmount amount);
+    virtual Res SubBalance(CScript const & owner, CTokenAmount amount);
+
     Res AddBalances(CScript const & owner, CBalances const & balances);
-    Res SubBalance(CScript const & owner, CTokenAmount amount);
     Res SubBalances(CScript const & owner, CBalances const & balances);
 
     // tags

src/masternodes/accountshistory.cpp

@@ -10,53 +10,26 @@
 
 /// @attention make sure that it does not overlap with those in masternodes.cpp/tokens.cpp/undos.cpp/accounts.cpp !!!
 const unsigned char CAccountsHistoryView::ByAccountHistoryKey::prefix = 'h'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB
+const unsigned char CRewardsHistoryView::ByRewardHistoryKey::prefix = 'R'; // don't intersects with CMintedHeadersView::MintedHeaders::prefix due to different DB
 
-void CAccountsHistoryView::ForEachAccountHistory(std::function<bool(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diffs)> callback, AccountHistoryKey start) const
+void CAccountsHistoryView::ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, CLazySerialize<AccountHistoryValue>)> callback, AccountHistoryKey const & start) const
 {
-    ForEach<ByAccountHistoryKey, AccountHistoryKey, AccountHistoryValue>([&callback] (AccountHistoryKey const & key, AccountHistoryValue const & val) {
-        return callback(key.owner,key.blockHeight, key.txn, val.txid, val.category, val.diff);
-    }, start);
+    ForEach<ByAccountHistoryKey, AccountHistoryKey, AccountHistoryValue>(callback, start);
 }
 
-Res CAccountsHistoryView::SetAccountHistory(const CScript & owner, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category, TAmounts const & diff)
+Res CAccountsHistoryView::SetAccountHistory(const AccountHistoryKey& key, const AccountHistoryValue& value)
 {
-////  left for debug:
-//    std::string ownerStr;
-//    CTxDestination dest;
-//    if (!ExtractDestination(owner, dest)) {
-//        ownerStr = owner.GetHex();
-//    } else
-//        ownerStr = EncodeDestination(dest);
-//    LogPrintf("DEBUG: SetAccountHistory: owner: %s, ownerStr: %s, block: %i, txn: %d, txid: %s, diffs: %ld\n", owner.GetHex().c_str(), ownerStr.c_str(), height, txn, txid.ToString().c_str(), diff.size());
-
-    WriteBy<ByAccountHistoryKey>(AccountHistoryKey{owner, height, txn}, AccountHistoryValue{txid, category, diff});
+    WriteBy<ByAccountHistoryKey>(key, value);
     return Res::Ok();
 }
 
-bool CAccountsHistoryView::TrackAffectedAccounts(CStorageKV const & before, MapKV const & diff, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category) {
-    // txn set to max if called from CreateNewBlock to check account balances, do not track here.
-    if (!gArgs.GetBoolArg("-acindex", false) || txn == std::numeric_limits<uint32_t>::max())
-        return false;
-
-    std::map<CScript, TAmounts> balancesDiff;
-    using TKey = std::pair<unsigned char, BalanceKey>;
-
-    for (auto it = diff.lower_bound({CAccountsView::ByBalanceKey::prefix}); it != diff.end() && it->first.at(0) == CAccountsView::ByBalanceKey::prefix; ++it) {
-        CAmount oldAmount = 0, newAmount = 0;
+void CRewardsHistoryView::ForEachRewardHistory(std::function<bool(RewardHistoryKey const &, CLazySerialize<RewardHistoryValue>)> callback, RewardHistoryKey const & start) const
+{
+    ForEach<ByRewardHistoryKey, RewardHistoryKey, RewardHistoryValue>(callback, start);
+}
 
-        if (it->second) {
-            BytesToDbType(*it->second, newAmount);
-        }
-        TBytes beforeVal;
-        if (before.Read(it->first, beforeVal)) {
-            BytesToDbType(beforeVal, oldAmount);
-        }
-        TKey balanceKey;
-        BytesToDbType(it->first, balanceKey);
-        balancesDiff[balanceKey.second.owner][balanceKey.second.tokenID] = newAmount - oldAmount;
-    }
-    for (auto const & kv : balancesDiff) {
-        SetAccountHistory(kv.first, height, txn, txid, category, kv.second);
-    }
-    return true;
+Res CRewardsHistoryView::SetRewardHistory(const RewardHistoryKey& key, const RewardHistoryValue& value)
+{
+    WriteBy<ByRewardHistoryKey>(key, value);
+    return Res::Ok();
 }

src/masternodes/accountshistory.h

@@ -53,15 +53,61 @@ struct AccountHistoryValue {
     }
 };
 
+struct RewardHistoryKey {
+    CScript owner;
+    uint32_t blockHeight;
+    DCT_ID poolID; // for order in block
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        READWRITE(owner);
+
+        if (ser_action.ForRead()) {
+            READWRITE(WrapBigEndian(blockHeight));
+            blockHeight = ~blockHeight;
+        }
+        else {
+            uint32_t blockHeight_ = ~blockHeight;
+            READWRITE(WrapBigEndian(blockHeight_));
+        }
+
+        READWRITE(VARINT(poolID.v));
+    }
+};
+
+struct RewardHistoryValue {
+    unsigned char category;
+    TAmounts diff;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action) {
+        READWRITE(category);
+        READWRITE(diff);
+    }
+};
+
 class CAccountsHistoryView : public virtual CStorageView
 {
 public:
-    Res SetAccountHistory(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diff);
-    void ForEachAccountHistory(std::function<bool(CScript const & owner, uint32_t height, uint32_t txn, uint256 const & txid, unsigned char category, TAmounts const & diff)> callback, AccountHistoryKey start) const;
-    bool TrackAffectedAccounts(CStorageKV const & before, MapKV const & diff, uint32_t height, uint32_t txn, const uint256 & txid, unsigned char category);
+    Res SetAccountHistory(AccountHistoryKey const & key, AccountHistoryValue const & value);
+    void ForEachAccountHistory(std::function<bool(AccountHistoryKey const &, CLazySerialize<AccountHistoryValue>)> callback, AccountHistoryKey const & start = {}) const;
 
     // tags
     struct ByAccountHistoryKey { static const unsigned char prefix; };
 };
 
+class CRewardsHistoryView : public virtual CStorageView
+{
+public:
+    Res SetRewardHistory(RewardHistoryKey const & key, RewardHistoryValue const & value);
+    void ForEachRewardHistory(std::function<bool(RewardHistoryKey const &, CLazySerialize<RewardHistoryValue>)> callback, RewardHistoryKey const & start = {}) const;
+
+    // tags
+    struct ByRewardHistoryKey { static const unsigned char prefix; };
+};
+
 #endif //DEFI_MASTERNODES_ACCOUNTSHISTORY_H

src/masternodes/criminals.cpp

@@ -43,11 +43,11 @@ bool CMintedHeadersView::FetchMintedHeaders(const uint256 & txid, const uint64_t
     }
 
     blockHeaders.clear();
-    ForEach<MintedHeaders,DBMNBlockHeadersKey,CBlockHeader>([&txid, &mintedBlocks, &blockHeaders] (DBMNBlockHeadersKey const & key, CBlockHeader & blockHeader) {
+    ForEach<MintedHeaders,DBMNBlockHeadersKey,CBlockHeader>([&txid, &mintedBlocks, &blockHeaders] (DBMNBlockHeadersKey const & key, CLazySerialize<CBlockHeader> blockHeader) {
         if (key.masternodeID == txid &&
             key.mintedBlocks == mintedBlocks) {
 
-            blockHeaders.emplace(key.blockHash, std::move(blockHeader));
+            blockHeaders.emplace(key.blockHash, blockHeader.get());
             return true; // continue
         }
         return false; // break!
@@ -76,12 +76,12 @@ void CCriminalProofsView::RemoveCriminalProofs(const uint256 & mnId) {
 CCriminalProofsView::CMnCriminals CCriminalProofsView::GetUnpunishedCriminals() {
 
     CMnCriminals result;
-    ForEach<Proofs, uint256, CDoubleSignFact>([&result] (uint256 const & id, CDoubleSignFact & proof) {
+    ForEach<Proofs, uint256, CDoubleSignFact>([&result] (uint256 const & id, CLazySerialize<CDoubleSignFact> proof) {
 
         // matching with already punished. and this is the ONLY measure!
         auto node = pcustomcsview->GetMasternode(id); // assert?
         if (node && node->banTx.IsNull()) {
-            result.emplace(id, std::move(proof));
+            result.emplace(id, proof.get());
         }
         return true; // continue
     });

src/masternodes/govvariables/lp_splits.cpp

@@ -50,12 +50,12 @@ Res LP_SPLITS::Validate(const CCustomCSView & mnview) const {
 }
 
 Res LP_SPLITS::Apply(CCustomCSView & mnview) {
-    mnview.ForEachPoolPair([&] (const DCT_ID poolId, const CPoolPair & pool) {
+    mnview.ForEachPoolPair([&] (const DCT_ID poolId, CPoolPair pool) {
         // we ought to reset previous value:
-        const_cast<CPoolPair &>(pool).rewardPct = 0;
+        pool.rewardPct = 0;
         auto it = splits.find(poolId);
         if (it != splits.end()) {
-            const_cast<CPoolPair &>(pool).rewardPct = it->second;
+            pool.rewardPct = it->second;
         }
 
         mnview.SetPoolPair(poolId, pool);

src/masternodes/incentivefunding.cpp

@@ -28,9 +28,9 @@ Res CCommunityBalancesView::SetCommunityBalance(CommunityAccountType account, CA
     return Res::Ok();
 }
 
-void CCommunityBalancesView::ForEachCommunityBalance(std::function<bool (CommunityAccountType, CAmount)> callback) const
+void CCommunityBalancesView::ForEachCommunityBalance(std::function<bool (CommunityAccountType, CLazySerialize<CAmount>)> callback) const
 {
-    ForEach<ById, unsigned char, CAmount>([&callback] (unsigned char const & key, CAmount const & val) {
+    ForEach<ById, unsigned char, CAmount>([&callback] (unsigned char const & key, CLazySerialize<CAmount> val) {
         return callback(CommunityAccountCodeToType(key), val);
     }, '\0');
 

src/masternodes/incentivefunding.h

@@ -35,7 +35,7 @@ class CCommunityBalancesView : public virtual CStorageView
     CAmount GetCommunityBalance(CommunityAccountType account) const;
     Res SetCommunityBalance(CommunityAccountType account, CAmount amount);
 
-    void ForEachCommunityBalance(std::function<bool(CommunityAccountType account, CAmount amount)> callback) const;
+    void ForEachCommunityBalance(std::function<bool(CommunityAccountType, CLazySerialize<CAmount>)> callback) const;
 
     Res AddCommunityBalance(CommunityAccountType account, CAmount amount);
     Res SubCommunityBalance(CommunityAccountType account, CAmount amount);

src/masternodes/masternodes.cpp

@@ -209,11 +209,9 @@ boost::optional<uint256> CMasternodesView::GetMasternodeIdByOwner(const CKeyID &
     return ReadBy<Owner, uint256>(id);
 }
 
-void CMasternodesView::ForEachMasternode(std::function<bool (const uint256 &, CMasternode &)> callback, uint256 const & start)
+void CMasternodesView::ForEachMasternode(std::function<bool (const uint256 &, CLazySerialize<CMasternode>)> callback, uint256 const & start)
 {
-    ForEach<ID, uint256, CMasternode>([&callback] (uint256 const & txid, CMasternode & node) {
-        return callback(txid, node);
-    }, start);
+    ForEach<ID, uint256, CMasternode>(callback, start);
 }
 
 void CMasternodesView::IncrementMintedBy(const CKeyID & minter)
@@ -426,7 +424,7 @@ void CAnchorRewardsView::RemoveRewardForAnchor(const CAnchorRewardsView::AnchorT
     EraseBy<BtcTx>(btcTxHash);
 }
 
-void CAnchorRewardsView::ForEachAnchorReward(std::function<bool (const CAnchorRewardsView::AnchorTxHash &, CAnchorRewardsView::RewardTxHash &)> callback)
+void CAnchorRewardsView::ForEachAnchorReward(std::function<bool (const CAnchorRewardsView::AnchorTxHash &, CLazySerialize<CAnchorRewardsView::RewardTxHash>)> callback)
 {
     ForEach<BtcTx, AnchorTxHash, RewardTxHash>(callback);
 }
@@ -442,7 +440,7 @@ CTeamView::CTeam CCustomCSView::CalcNextTeam(const uint256 & stakeModifier)
     int anchoringTeamSize = Params().GetConsensus().mn.anchoringTeamSize;
 
     std::map<arith_uint256, CKeyID, std::less<arith_uint256>> priorityMN;
-    ForEachMasternode([&stakeModifier, &priorityMN] (uint256 const & id, CMasternode & node) {
+    ForEachMasternode([&stakeModifier, &priorityMN] (uint256 const & id, CMasternode node) {
         if(!node.IsActive())
             return true;
 
@@ -525,3 +523,65 @@ bool CCustomCSView::CanSpend(const uint256 & txId, int height) const
     return !pair || pair->second.destructionTx != uint256{} || pair->second.IsPoolShare();
 }
 
+CAccountsHistoryStorage::CAccountsHistoryStorage(CCustomCSView & storage, uint32_t height, uint32_t txn, const uint256& txid, uint8_t type)
+    : CStorageView(new CFlushableStorageKV(storage.GetRaw())), height(height), txn(txn), txid(txid), type(type)
+{
+    acindex = gArgs.GetBoolArg("-acindex", false);
+}
+
+Res CAccountsHistoryStorage::AddBalance(CScript const & owner, CTokenAmount amount)
+{
+    auto res = CCustomCSView::AddBalance(owner, amount);
+    if (acindex && res.ok) {
+        diffs[owner][amount.nTokenId] += amount.nValue;
+    }
+    return res;
+}
+
+Res CAccountsHistoryStorage::SubBalance(CScript const & owner, CTokenAmount amount)
+{
+    auto res = CCustomCSView::SubBalance(owner, amount);
+    if (acindex && res.ok) {
+        diffs[owner][amount.nTokenId] -= amount.nValue;
+    }
+    return res;
+}
+
+bool CAccountsHistoryStorage::Flush()
+{
+    if (acindex) {
+        for (const auto& diff : diffs) {
+            SetAccountHistory({diff.first, height, txn}, {txid, type, diff.second});
+        }
+    }
+    return CCustomCSView::Flush();
+}
+
+CRewardsHistoryStorage::CRewardsHistoryStorage(CCustomCSView & storage, uint32_t height)
+    : CStorageView(new CFlushableStorageKV(storage.GetRaw())), height(height)
+{
+    acindex = gArgs.GetBoolArg("-acindex", false);
+}
+
+Res CRewardsHistoryStorage::AddBalance(CScript const & owner, DCT_ID poolID, uint8_t type, CTokenAmount amount)
+{
+    auto res = CCustomCSView::AddBalance(owner, amount);
+    if (acindex && res.ok) {
+        auto& tuple = diffs[owner];
+        std::get<0>(tuple) = poolID;
+        std::get<1>(tuple) = type;
+        std::get<2>(tuple)[amount.nTokenId] += amount.nValue;
+    }
+    return res;
+}
+
+bool CRewardsHistoryStorage::Flush()
+{
+    if (acindex) {
+        for (const auto& diff : diffs) {
+            const auto& tuple = diff.second;
+            SetRewardHistory({diff.first, height, std::get<0>(tuple)}, {std::get<1>(tuple), std::get<2>(tuple)});
+        }
+    }
+    return CCustomCSView::Flush();
+}