Skip to content

Commit

Permalink
Implement "paging" for blockchain.*.get_history
Browse files Browse the repository at this point in the history 
Closes issue #180.  Optional 2nd and 3rd args to blockchain.*.get_history
can be used to limit the size of the returned history by looking only
within a certain block height range.

Implementation follows the suggested API for Electrum protocol version 2.0
proposed protocol spec, see:

https://github.com/spesmilo/electrumx/pull/90/files#diff-ffa7e37c618564f8c58d9072b5ef234e6e9b2cdf1d71d16be7dc356146cb475fR327
  • Loading branch information
@cculianu
cculianu committed Jun 28, 2023
1 parent 7c3d644 commit f6543b9
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 15 deletions.
44 changes: 35 additions & 9 deletions src/Servers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1497,10 +1497,13 @@ void Server::impl_get_balance(Client *c, const RPC::BatchId batchId, const RPC::

/// called from get_mempool and get_history to retrieve the mempool for a hashx synchronously. Returns the
/// QVariantMap suitable for placing into the resulting response.
QVariantList ServerBase::getHistoryCommon(const HashX &sh, bool mempoolOnly)
QVariantList ServerBase::getHistoryCommon(const HashX &sh, bool mempoolOnly, const GetHistory_FromToBH &fromTo)
{
QVariantList resp;
const auto items = storage->getHistory(sh, !mempoolOnly, true); // these are already sorted
const bool includeConfirmed = !mempoolOnly;
const bool includeMempool = mempoolOnly || !fromTo.second.has_value();
// the `items` result is already sorted
const auto items = storage->getHistory(sh, includeConfirmed, includeMempool, fromTo.first, fromTo.second);
for (const auto & item : items) {
QVariantMap m{
{ "tx_hash" , Util::ToHexFast(item.hash) },
Expand All @@ -1513,20 +1516,43 @@ QVariantList ServerBase::getHistoryCommon(const HashX &sh, bool mempoolOnly)
return resp;
}

auto Server::parseFromToBlockHeightCommon(const RPC::Message &m) const -> GetHistory_FromToBH
{
GetHistory_FromToBH ret{0u, std::nullopt};
const QVariantList l(m.paramsList());
if (l.size() > 1) {
bool ok;
const int tmp = l[1].toInt(&ok);
if (tmp >= 0) ret.first = static_cast<BlockHeight>(tmp);
if (!ok || tmp < 0) throw RPCError("Bad from_height argument at position 2", RPC::ErrorCodes::Code_InvalidParams);
}
if (l.size() > 2) {
bool ok;
const int tmp = l[2].toInt(&ok);
if (ok && tmp > -1) ret.second = static_cast<BlockHeight>(tmp);
if (!ok || (ret.second && ret.first > *ret.second))
throw RPCError("Bad to_height argument at position 3", RPC::ErrorCodes::Code_InvalidParams);
}
return ret;
}

void Server::rpc_blockchain_scripthash_get_history(Client *c, const RPC::BatchId batchId, const RPC::Message &m)
{
const auto sh = parseFirstHashParamCommon(m);
impl_get_history(c, batchId, m, sh);
const auto fromTo = parseFromToBlockHeightCommon(m);
impl_get_history(c, batchId, m, sh, fromTo);
}
void Server::rpc_blockchain_address_get_history(Client *c, const RPC::BatchId batchId, const RPC::Message &m)
{
const auto sh = parseFirstAddrParamToShCommon(m);
impl_get_history(c, batchId, m, sh);
const auto fromTo = parseFromToBlockHeightCommon(m);
impl_get_history(c, batchId, m, sh, fromTo);
}
void Server::impl_get_history(Client *c, const RPC::BatchId batchId, const RPC::Message &m, const HashX &sh)
void Server::impl_get_history(Client *c, const RPC::BatchId batchId, const RPC::Message &m, const HashX &sh,
const GetHistory_FromToBH &fromTo)
{
generic_do_async(c, batchId, m.id, [sh, this] {
return getHistoryCommon(sh, false);
generic_do_async(c, batchId, m.id, [sh, fromTo, this] {
return getHistoryCommon(sh, false, fromTo);
});
}

Expand Down Expand Up @@ -2155,7 +2181,7 @@ HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::registry){
{ {"server.version", true, false, PR{0,2}, }, MP(rpc_server_version) },

{ {"blockchain.address.get_balance", true, false, PR{1,2}, }, MP(rpc_blockchain_address_get_balance) },
{ {"blockchain.address.get_history", true, false, PR{1,1}, }, MP(rpc_blockchain_address_get_history) },
{ {"blockchain.address.get_history", true, false, PR{1,3}, }, MP(rpc_blockchain_address_get_history) },
{ {"blockchain.address.get_mempool", true, false, PR{1,1}, }, MP(rpc_blockchain_address_get_mempool) },
{ {"blockchain.address.get_scripthash", true, false, PR{1,1}, }, MP(rpc_blockchain_address_get_scripthash) },
{ {"blockchain.address.listunspent", true, false, PR{1,2}, }, MP(rpc_blockchain_address_listunspent) },
Expand All @@ -2171,7 +2197,7 @@ HEY_COMPILER_PUT_STATIC_HERE(Server::StaticData::registry){
{ {"blockchain.relayfee", true, false, PR{0,0}, }, MP(rpc_blockchain_relayfee) },

{ {"blockchain.scripthash.get_balance", true, false, PR{1,2}, }, MP(rpc_blockchain_scripthash_get_balance) },
{ {"blockchain.scripthash.get_history", true, false, PR{1,1}, }, MP(rpc_blockchain_scripthash_get_history) },
{ {"blockchain.scripthash.get_history", true, false, PR{1,3}, }, MP(rpc_blockchain_scripthash_get_history) },
{ {"blockchain.scripthash.get_mempool", true, false, PR{1,1}, }, MP(rpc_blockchain_scripthash_get_mempool) },
{ {"blockchain.scripthash.listunspent", true, false, PR{1,2}, }, MP(rpc_blockchain_scripthash_listunspent) },
{ {"blockchain.scripthash.subscribe", true, false, PR{1,1}, }, MP(rpc_blockchain_scripthash_subscribe) },
Expand Down
10 changes: 8 additions & 2 deletions src/Servers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
#include <optional>
#include <shared_mutex>
#include <type_traits>
#include <utility>
#include <variant>

struct TcpServerError : public Exception
Expand Down Expand Up @@ -315,11 +316,13 @@ protected slots:


// --- Helpers used by Server* and AdminServer subclasses ---
using GetHistory_FromToBH = std::pair<BlockHeight, std::optional<BlockHeight>>;
static constexpr GetHistory_FromToBH default_GetHistory_FromToBH{0, std::nullopt};

/// Called from get_mempool and get_history to retrieve the mempool and/or history for a hashx synchronously.
/// Also called by Admin server's 'query_address'
/// Returns the QVariantMap suitable for placing into the resulting response.
QVariantList getHistoryCommon(const HashX & scriptHash, bool mempoolOnly);
QVariantList getHistoryCommon(const HashX & scriptHash, bool mempoolOnly, const GetHistory_FromToBH & = default_GetHistory_FromToBH);
/// Called for get_balance and also Admin server's query_address
QVariantMap getBalanceCommon(const HashX & scriptHash, Storage::TokenFilterOption tokenFilter);
/// Called for listunspent and also Admin server's query_address
Expand Down Expand Up @@ -420,7 +423,7 @@ class Server : public ServerBase
// Impl. for blockchain.scripthash.* & blockchain.address.* methods (both sets call into these).
// Note: Validation should have already been done by caller.
void impl_get_balance(Client *, RPC::BatchId, const RPC::Message &, const HashX &scriptHash, Storage::TokenFilterOption tokenFilter);
void impl_get_history(Client *, RPC::BatchId, const RPC::Message &, const HashX &scriptHash);
void impl_get_history(Client *, RPC::BatchId, const RPC::Message &, const HashX &scriptHash, const GetHistory_FromToBH &);
void impl_get_mempool(Client *, RPC::BatchId, const RPC::Message &, const HashX &scriptHash);
void impl_listunspent(Client *, RPC::BatchId, const RPC::Message &, const HashX &scriptHash, Storage::TokenFilterOption tokenFilter);
void impl_generic_subscribe(SubsMgr *, Client *, RPC::BatchId, const RPC::Message &, const HashX &key,
Expand All @@ -443,6 +446,9 @@ class Server : public ServerBase
/// Helper used by blockchain.*.listunspent *.get_balance to parse optional 2nd arg
Storage::TokenFilterOption parseTokenFilterOptionCommon(Client *c, const RPC::Message &m, size_t argPos) const;

/// Helper used by blockchain.*.get_history to get the from_height and to_height optional params, if any
GetHistory_FromToBH parseFromToBlockHeightCommon(const RPC::Message &m) const;

/// Basically a namespace for our rpc dispatch tables, etc
struct StaticData {
struct MethodMember : public RPC::Method { Member_t member = nullptr; }; ///< used to associate the method spec with a pointer to member
Expand Down
12 changes: 9 additions & 3 deletions src/Storage.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3615,7 +3615,8 @@ static auto GetMaxHistoryCtrFunc(const QString &name, const HashX &hashX, size_t
};
}

auto Storage::getHistory(const HashX & hashX, bool conf, bool unconf) const -> History
auto Storage::getHistory(const HashX & hashX, bool conf, bool unconf, BlockHeight fromHeight,
std::optional<BlockHeight> optToHeight) const -> History
{
History ret;
if (hashX.length() != HashLen)
Expand All @@ -3637,8 +3638,13 @@ auto Storage::getHistory(const HashX & hashX, bool conf, bool unconf) const -> H
// low hanging fruit for optimization -- thus I am leaving this comment here so I can remember to come
// back and optmize the below. /TODO
for (auto num : nums) {
auto hash = hashForTxNum(num).value(); // may throw, but that indicates some database inconsistency. we catch below
auto height = heightForTxNum(num).value(); // may throw, same deal
const BlockHeight height = heightForTxNum(num).value(); // may throw, same deal

// Assumption for this loop: the nums are in order!
if (optToHeight && height >= *optToHeight) break; // threshold of "to height" reached
else if (height < fromHeight) continue; // keep looping until we hit a height that at least "from height"

const auto hash = hashForTxNum(num).value(); // may throw, but that indicates some database inconsistency. we catch below
ret.emplace_back(/* HistoryItem: */ hash, int(height));
}
}
Expand Down
3 changes: 2 additions & 1 deletion src/Storage.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,8 @@ class Storage final : public Mgr, public ThreadObjectMixin

/// Thread-safe. Will return an empty vector if the confirmed history size exceeds MaxHistory, or a truncated
/// vector if the confirmed + unconfirmed history exceeds MaxHistory.
History getHistory(const HashX &, bool includeConfirmed, bool includeMempool) const;
History getHistory(const HashX &, bool includeConfirmed, bool includeMempool, BlockHeight fromHeight = 0,
std::optional<BlockHeight> optToHeight = std::nullopt) const;

struct UnspentItem : HistoryItem {
IONum tx_pos = 0;
Expand Down

0 comments on commit f6543b9

Please sign in to comment.