Skip to content

Commit

Permalink
Enforce restricted # pool txs served via RPC + optimize chunked reqs …
Browse files Browse the repository at this point in the history
…[release-v0.18]

- `/getblocks.bin` respects the `RESTRICTED_TX_COUNT` (=100) when
returning pool txs via a restricted RPC daemon.
- A restricted RPC daemon includes a max of `RESTRICTED_TX_COUNT` txs
in the `added_pool_txs` field, and returns any remaining pool hashes
in the `remaining_added_pool_txids` field. The client then requests
the remaining txs via `/gettransactions` in chunks.
- `/gettransactions` no longer does expensive no-ops for ALL pool txs
if the client requests a subset of pool txs. Instead it searches for
the txs the client explicitly requests.
- Reset `m_pool_info_query_time` when a user:
  (1) rescans the chain (so the wallet re-requests the whole pool)
  (2) changes the daemon their wallets points to (a new daemon would
      have a different view of the pool)
- `/getblocks.bin` respects the `req.prune` field when returning
pool txs.
- Pool extension fields in response to `/getblocks.bin` are optional
with default 0'd values.
  • Loading branch information
j-berman authored and woodser committed Jul 24, 2023
1 parent d81309e commit 66e571d
Show file tree
Hide file tree
Showing 12 changed files with 245 additions and 168 deletions.
2 changes: 1 addition & 1 deletion src/cryptonote_core/blockchain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2065,7 +2065,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
cryptonote::blobdata blob;
if (m_tx_pool.have_tx(txid, relay_category::legacy))
{
if (m_tx_pool.get_transaction_info(txid, td))
if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/))
{
bei.block_cumulative_weight += td.weight;
}
Expand Down
9 changes: 7 additions & 2 deletions src/cryptonote_core/cryptonote_core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1727,6 +1727,11 @@ namespace cryptonote
return true;
}
//-----------------------------------------------------------------------------------------------
bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const
{
return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes);
}
//-----------------------------------------------------------------------------------------------
bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const
{
m_mempool.get_transactions(txs, include_sensitive_data);
Expand All @@ -1739,9 +1744,9 @@ namespace cryptonote
return true;
}
//-----------------------------------------------------------------------------------------------
bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, std::vector<tx_memory_pool::tx_details>& added_txs, std::vector<crypto::hash>& removed_txs, bool& incremental) const
bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
{
return m_mempool.get_pool_info(start_time, include_sensitive_txes, added_txs, removed_txs, incremental);
return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental);
}
//-----------------------------------------------------------------------------------------------
bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const
Expand Down
11 changes: 10 additions & 1 deletion src/cryptonote_core/cryptonote_core.h
Original file line number Diff line number Diff line change
Expand Up @@ -509,13 +509,22 @@ namespace cryptonote
*/
bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const;

/**
* @copydoc tx_memory_pool::get_pool_transactions_info
* @param include_sensitive_txes include private transactions
*
* @note see tx_memory_pool::get_pool_transactions_info
*/
bool get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes = false) const;

/**
* @copydoc tx_memory_pool::get_pool_info
* @param include_sensitive_txes include private transactions
* @param max_tx_count max allowed added_txs in response
*
* @note see tx_memory_pool::get_pool_info
*/
bool get_pool_info(time_t start_time, bool include_sensitive_txes, std::vector<tx_memory_pool::tx_details>& added_txs, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;

/**
* @copydoc tx_memory_pool::get_transactions
Expand Down
78 changes: 48 additions & 30 deletions src/cryptonote_core/tx_pool.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -614,7 +614,7 @@ namespace cryptonote
return true;
}
//---------------------------------------------------------------------------------
bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const
bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const
{
PERF_TIMER(get_transaction_info);
CRITICAL_REGION_LOCAL(m_transactions_lock);
Expand All @@ -626,7 +626,12 @@ namespace cryptonote
txpool_tx_meta_t meta;
if (!m_blockchain.get_txpool_tx_meta(txid, meta))
{
MERROR("Failed to find tx in txpool");
LOG_PRINT_L2("Failed to find tx in txpool: " << txid);
return false;
}
if (!include_sensitive_data && !meta.matches(relay_category::broadcasted))
{
// We don't want sensitive data && the tx is sensitive, so no need to return it
return false;
}
cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
Expand All @@ -652,12 +657,13 @@ namespace cryptonote
td.kept_by_block = meta.kept_by_block;
td.last_failed_height = meta.last_failed_height;
td.last_failed_id = meta.last_failed_id;
td.receive_time = meta.receive_time;
td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time;
td.receive_time = include_sensitive_data ? meta.receive_time : 0;
td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0;
td.relayed = meta.relayed;
td.do_not_relay = meta.do_not_relay;
td.double_spend_seen = meta.double_spend_seen;
td.sensitive = !meta.matches(relay_category::broadcasted);
if (include_blob)
td.tx_blob = std::move(txblob);
}
catch (const std::exception &e)
{
Expand All @@ -667,6 +673,25 @@ namespace cryptonote

return true;
}
//------------------------------------------------------------------
bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const
{
CRITICAL_REGION_LOCAL(m_transactions_lock);
CRITICAL_REGION_LOCAL1(m_blockchain);

txs.clear();

for (const auto &it: txids)
{
tx_details details;
bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/);
if (success)
{
txs.push_back(std::make_pair(it, std::move(details)));
}
}
return true;
}
//---------------------------------------------------------------------------------
bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const
{
Expand Down Expand Up @@ -940,7 +965,7 @@ namespace cryptonote
}, false, category);
}
//------------------------------------------------------------------
bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, std::vector<tx_details>& added_txs, std::vector<crypto::hash>& removed_txs, bool& incremental) const
bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
{
CRITICAL_REGION_LOCAL(m_transactions_lock);
CRITICAL_REGION_LOCAL1(m_blockchain);
Expand Down Expand Up @@ -968,46 +993,39 @@ namespace cryptonote
}

added_txs.clear();
remaining_added_txids.clear();
removed_txs.clear();

std::vector<crypto::hash> txids;
if (!incremental)
{
LOG_PRINT_L2("Giving back the whole pool");
// Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the
// anonymous method somehow results in an LMDB error with transactions we have to build a list of
// ids first and get the full info afterwards
std::vector<crypto::hash> txids;
const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted;
m_blockchain.for_all_txpool_txes([&txids](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){
txids.push_back(txid);
return true;
}, false, category);
tx_details details;
for (const auto &it: txids)
get_transaction_hashes(txids, include_sensitive);
if (txids.size() > max_tx_count)
{
bool success = get_transaction_info(it, details);
if (success)
{
added_txs.push_back(std::move(details));
}
remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end());
txids.erase(txids.begin() + max_tx_count, txids.end());
}
get_transactions_info(txids, added_txs, include_sensitive);
return true;
}

// Give back incrementally, based on time of entry into the map
tx_details details;
for (const auto &pit : m_added_txs_by_id)
{
if (pit.second >= start_time)
{
bool success = get_transaction_info(pit.first, details);
if (success)
{
if (include_sensitive || !details.sensitive)
{
added_txs.push_back(std::move(details));
}
}
}
txids.push_back(pit.first);
}
get_transactions_info(txids, added_txs, include_sensitive);
if (added_txs.size() > max_tx_count)
{
remaining_added_txids.reserve(added_txs.size() - max_tx_count);
for (size_t i = max_tx_count; i < added_txs.size(); ++i)
remaining_added_txids.push_back(added_txs[i].first);
added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end());
}

std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time);
Expand Down
11 changes: 8 additions & 3 deletions src/cryptonote_core/tx_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,7 @@ namespace cryptonote
struct tx_details
{
transaction tx; //!< the transaction
cryptonote::blobdata tx_blob; //!< the transaction's binary blob
size_t blob_size; //!< the transaction's size
size_t weight; //!< the transaction's weight
uint64_t fee; //!< the transaction's fee amount
Expand Down Expand Up @@ -461,13 +462,17 @@ namespace cryptonote
bool do_not_relay; //!< to avoid relay this transaction to the network

bool double_spend_seen; //!< true iff another tx was seen double spending this one
bool sensitive;
};

/**
* @brief get infornation about a single transaction
*/
bool get_transaction_info(const crypto::hash &txid, tx_details &td) const;
bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const;

/**
* @brief get information about multiple transactions
*/
bool get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive_data = false) const;

/**
* @brief get transactions not in the passed set
Expand All @@ -479,7 +484,7 @@ namespace cryptonote
*
* @return true on success, false on error
*/
bool get_pool_info(time_t start_time, bool include_sensitive, std::vector<tx_details>& added_txs, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;

private:

Expand Down
55 changes: 22 additions & 33 deletions src/rpc/core_rpc_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -630,31 +630,34 @@ namespace cryptonote
const bool restricted = m_restricted && ctx;
const bool request_has_rpc_origin = ctx != NULL;
const bool allow_sensitive = !request_has_rpc_origin || !restricted;
const size_t max_tx_count = restricted ? RESTRICTED_TRANSACTIONS_COUNT : std::numeric_limits<size_t>::max();

bool incremental;
std::vector<tx_memory_pool::tx_details> added_pool_txs;
bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, added_pool_txs, res.removed_pool_txids, incremental);
std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> added_pool_txs;
bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, max_tx_count, added_pool_txs, res.remaining_added_pool_txids, res.removed_pool_txids, incremental);
if (success)
{
res.added_pool_txs.clear();
if (m_rpc_payment)
{
CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + res.removed_pool_txids.size() * COST_PER_POOL_HASH);
CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH);
}
for (auto tx_detail: added_pool_txs)
for (const auto &added_pool_tx: added_pool_txs)
{
COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info;
info.tx_hash = cryptonote::get_transaction_hash(tx_detail.tx);
info.tx_hash = added_pool_tx.first;
std::stringstream oss;
binary_archive<true> ar(oss);
bool r = ::serialization::serialize(ar, tx_detail.tx);
bool r = req.prune
? const_cast<cryptonote::transaction&>(added_pool_tx.second.tx).serialize_base(ar)
: ::serialization::serialize(ar, const_cast<cryptonote::transaction&>(added_pool_tx.second.tx));
if (!r)
{
res.status = "Failed to serialize transaction";
return true;
}
info.tx_blob = oss.str();
info.double_spend_seen = tx_detail.double_spend_seen;
info.double_spend_seen = added_pool_tx.second.double_spend_seen;
res.added_pool_txs.push_back(std::move(info));
}
}
Expand Down Expand Up @@ -993,17 +996,16 @@ namespace cryptonote
// try the pool for any missing txes
size_t found_in_pool = 0;
std::unordered_set<crypto::hash> pool_tx_hashes;
std::unordered_map<crypto::hash, tx_info> per_tx_pool_tx_info;
std::unordered_map<crypto::hash, tx_memory_pool::tx_details> per_tx_pool_tx_details;
if (!missed_txs.empty())
{
std::vector<tx_info> pool_tx_info;
std::vector<spent_key_image_info> pool_key_image_info;
bool r = m_core.get_pool_transactions_and_spent_keys_info(pool_tx_info, pool_key_image_info, !request_has_rpc_origin || !restricted);
std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> pool_txs;
bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted);
if(r)
{
// sort to match original request
std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs;
std::vector<tx_info>::const_iterator i;
std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>::const_iterator i;
unsigned txs_processed = 0;
for (const crypto::hash &h: vh)
{
Expand All @@ -1023,36 +1025,23 @@ namespace cryptonote
sorted_txs.push_back(std::move(txs[txs_processed]));
++txs_processed;
}
else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end())
else if ((i = std::find_if(pool_txs.begin(), pool_txs.end(), [h](const std::pair<crypto::hash, tx_memory_pool::tx_details> &pt) { return h == pt.first; })) != pool_txs.end())
{
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_from_blob(i->tx_blob, tx))
{
res.status = "Failed to parse and validate tx from blob";
return true;
}
const tx_memory_pool::tx_details &td = i->second;
std::stringstream ss;
binary_archive<true> ba(ss);
bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
bool r = const_cast<cryptonote::transaction&>(td.tx).serialize_base(ba);
if (!r)
{
res.status = "Failed to serialize transaction base";
return true;
}
const cryptonote::blobdata pruned = ss.str();
const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx);
sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size())));
const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx);
sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size())));
missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h));
pool_tx_hashes.insert(h);
const std::string hash_string = epee::string_tools::pod_to_hex(h);
for (const auto &ti: pool_tx_info)
{
if (ti.id_hash == hash_string)
{
per_tx_pool_tx_info.insert(std::make_pair(h, ti));
break;
}
}
per_tx_pool_tx_details.insert(std::make_pair(h, td));
++found_in_pool;
}
}
Expand Down Expand Up @@ -1148,8 +1137,8 @@ namespace cryptonote
{
e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max();
e.confirmations = 0;
auto it = per_tx_pool_tx_info.find(tx_hash);
if (it != per_tx_pool_tx_info.end())
auto it = per_tx_pool_tx_details.find(tx_hash);
if (it != per_tx_pool_tx_details.end())
{
e.double_spend_seen = it->second.double_spend_seen;
e.relayed = it->second.relayed;
Expand Down
16 changes: 12 additions & 4 deletions src/rpc/core_rpc_server_commands_defs.h
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ namespace cryptonote
uint64_t daemon_time;
uint8_t pool_info_extent;
std::vector<pool_tx_info> added_pool_txs;
std::vector<crypto::hash> remaining_added_pool_txids;
std::vector<crypto::hash> removed_pool_txids;

BEGIN_KV_SERIALIZE_MAP()
Expand All @@ -244,10 +245,17 @@ namespace cryptonote
KV_SERIALIZE(start_height)
KV_SERIALIZE(current_height)
KV_SERIALIZE(output_indices)
KV_SERIALIZE(daemon_time)
KV_SERIALIZE(pool_info_extent)
KV_SERIALIZE(added_pool_txs)
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids)
KV_SERIALIZE_OPT(daemon_time, (uint64_t) 0)
KV_SERIALIZE_OPT(pool_info_extent, (uint8_t) 0)
if (pool_info_extent != POOL_INFO_EXTENT::NONE)
{
KV_SERIALIZE(added_pool_txs)
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(remaining_added_pool_txids)
}
if (pool_info_extent == POOL_INFO_EXTENT::INCREMENTAL)
{
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids)
}
END_KV_SERIALIZE_MAP()
};
typedef epee::misc_utils::struct_init<response_t> response;
Expand Down
Loading

0 comments on commit 66e571d

Please sign in to comment.