[WIP] decoy selection: spend like coinbase-ish outputs

Implements solution recommended here: monero-project/research-lab#109

WIP: right now a coinbase output ditribution cache on the daemon side makes it feasible to request this information from the dameon, but no wallet code is done yet.

src/rpc/core_rpc_server.cpp

@@ -174,6 +174,11 @@ namespace cryptonote
     , m_was_bootstrap_ever_used(false)
     , disable_rpc_ban(false)
     , m_rpc_payment_allow_free_loopback(false)
+    , m_cb_out_dist_cache
+    (
+      [&cr](uint64_t a, size_t b, std::vector<block>& c) -> bool { return cr.get_blocks(a, b, c); },
+      [&cr](uint64_t a) -> crypto::hash { return cr.get_block_id_by_height(a); }
+    )
   {}
   //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::set_bootstrap_daemon(
@@ -3340,6 +3345,21 @@ namespace cryptonote
 
         res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress});
       }
+
+      if (req.coinbase)
+      {
+        std::vector<uint64_t> num_cb_outs_per_block;
+        const uint64_t t1 = epee::misc_utils::get_tick_count();
+        if (!m_cb_out_dist_cache.get_coinbase_output_distribution(req.from_height, req_to_height, num_cb_outs_per_block))
+        {
+          res.status = "Failed to get coinbase output distribution";
+          return true;
+        }
+        const uint64_t t2 = epee::misc_utils::get_tick_count();
+        const uint64_t elap = t2 - t1;
+        std::cout << "get_coinbase_output_distribution took " << elap << " ms" << std::endl;
+        res.coinbase_distribution = {{std::move(num_cb_outs_per_block), req.from_height, 0}, 0, "", req.binary, req.compress};
+      }
     }
     catch (const std::exception &e)
     {
@@ -3394,10 +3414,25 @@ namespace cryptonote
 
         res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress});
       }
+
+      if (req.coinbase)
+      {
+        std::vector<uint64_t> num_cb_outs_per_block;
+        const uint64_t t1 = epee::misc_utils::get_tick_count();
+        if (!m_cb_out_dist_cache.get_coinbase_output_distribution(req.from_height, req_to_height, num_cb_outs_per_block))
+        {
+          res.status = "Failed to get coinbase output distribution";
+          return true;
+        }
+        const uint64_t t2 = epee::misc_utils::get_tick_count();
+        const uint64_t elap = t2 - t1;
+        std::cout << "get_coinbase_output_distribution took " << elap << " ms" << std::endl;
+        res.coinbase_distribution = {{std::move(num_cb_outs_per_block), req.from_height, 0}, 0, "", req.binary, req.compress};
+      }
     }
     catch (const std::exception &e)
     {
-      res.status = "Failed to get output distribution";
+      res.status = "Failed to get output distribution: error thrown";
       return true;
     }
 

src/rpc/core_rpc_server.h

@@ -42,6 +42,7 @@
 #include "cryptonote_core/cryptonote_core.h"
 #include "p2p/net_node.h"
 #include "cryptonote_protocol/cryptonote_protocol_handler.h"
+#include "rpc_handler.h"
 #include "rpc_payment.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -300,6 +301,7 @@ namespace cryptonote
     std::unique_ptr<rpc_payment> m_rpc_payment;
     bool disable_rpc_ban;
     bool m_rpc_payment_allow_free_loopback;
+    rpc::CoinbaseOutputDistributionCache m_cb_out_dist_cache;
   };
 }
 

src/rpc/core_rpc_server_commands_defs.h

@@ -2447,6 +2447,7 @@ namespace cryptonote
       bool cumulative;
       bool binary;
       bool compress;
+      bool coinbase;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_request_base)
@@ -2456,6 +2457,7 @@ namespace cryptonote
         KV_SERIALIZE_OPT(cumulative, false)
         KV_SERIALIZE_OPT(binary, true)
         KV_SERIALIZE_OPT(compress, false)
+        KV_SERIALIZE_OPT(coinbase, false)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -2468,6 +2470,12 @@ namespace cryptonote
       bool binary;
       bool compress;
 
+      bool operator==(const distribution& rhs) const
+      {
+        return data == rhs.data && amount == rhs.amount && compressed_data == rhs.compressed_data &&
+          binary == rhs.binary && compress == rhs.compress;
+      }
+
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(amount)
         KV_SERIALIZE_N(data.start_height, "start_height")
@@ -2505,10 +2513,12 @@ namespace cryptonote
     struct response_t: public rpc_access_response_base
     {
       std::vector<distribution> distributions;
+      distribution coinbase_distribution;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_response_base)
         KV_SERIALIZE(distributions)
+        KV_SERIALIZE_OPT(coinbase_distribution, distribution{})
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;

src/rpc/rpc_handler.cpp

@@ -109,5 +109,180 @@ namespace rpc
 
       return process_distribution(cumulative, start_height, std::move(distribution), base);
   }
+
+  bool CoinbaseOutputDistributionCache::get_coinbase_output_distribution
+  (
+    const uint64_t start_height, // inclusive
+    const uint64_t stop_height, // inclusive
+    std::vector<uint64_t>& num_cb_outs_per_block,
+    bool* only_used_cache/* = nullptr*/
+  )
+  {
+    if (only_used_cache)
+      *only_used_cache = true;
+
+    if (stop_height < start_height)
+    {
+      num_cb_outs_per_block.clear();
+      return true;
+    }
+    else if (stop_height > CRYPTONOTE_MAX_BLOCK_NUMBER)
+    {
+      MERROR("Request for ridiculous height: " << stop_height);
+      return false;
+    }
+
+    // Start accessing the cache
+
+    std::lock_guard<decltype(m_mutex)> ll(m_mutex);
+
+    rollback_for_reorgs();
+
+    if (m_num_cb_outs_per_block.empty())
+    {
+      // If we have nothing cached, we can adjust the cache start height to the first request.
+      // Since realistically, we will only use this cache for post-RCT coinbase blocks, we can save
+      // some space and DB fetching.
+      m_height_begin = start_height;
+    }
+
+    const uint64_t num_missing_front = (start_height < m_height_begin) ? (m_height_begin - start_height) : 0;
+    const uint64_t num_missing_back = (stop_height >= height_end()) ? (stop_height - height_end() + 1) : 0;
+
+    if (only_used_cache)
+      *only_used_cache = num_missing_front || num_missing_back;
+
+    if (!fetch_and_extend(start_height, num_missing_front, true))
+    {
+      return false;
+    }
+    else if (!fetch_and_extend(height_end(), num_missing_back, false))
+    {
+      return false;
+    }
+
+    CHECK_AND_ASSERT_MES(m_height_begin <= start_height, false, "internal bug: extend front");
+    CHECK_AND_ASSERT_MES(height_end() > stop_height, false, "internal bug: extend back");
+
+    const size_t num_queried_blocks = stop_height - start_height + 1; // +1 since inclusive range
+    const auto res_begin = m_num_cb_outs_per_block.begin() + (start_height - m_height_begin);
+    const auto res_end = res_begin + num_queried_blocks;
+    num_cb_outs_per_block = {res_begin, res_end};
+
+    return true;
+  }
+
+  void CoinbaseOutputDistributionCache::rollback_for_reorgs()
+  {
+    bool top_9_potent_invalid = false;
+    bool top_99_potent_invalid = false;
+    bool all_potent_invalid = false;
+
+    const uint64_t num_blocks_cached = m_num_cb_outs_per_block.size();
+    if (num_blocks_cached >= 1 && m_last_1_hash != m_get_block_id_by_height(height_end() - 1))
+    {
+      top_9_potent_invalid = true;
+      if (num_blocks_cached >= 10 && m_last_10_hash != m_get_block_id_by_height(height_end() - 10))
+      {
+        top_99_potent_invalid = true;
+        if (num_blocks_cached >= 100 && m_last_100_hash != m_get_block_id_by_height(height_end() - 100))
+        {
+          all_potent_invalid = true;
+        }
+      }
+    }
+
+    if (all_potent_invalid)
+    {
+      m_num_cb_outs_per_block.clear();
+    }
+    else if (top_99_potent_invalid)
+    {
+      for (size_t i = 0; i < 99; ++i) m_num_cb_outs_per_block.pop_back();
+    }
+    else if (top_9_potent_invalid)
+    {
+      for (size_t i = 0; i < 9; ++i) m_num_cb_outs_per_block.pop_back();
+    }
+    else
+    {
+      return; // Everything is valid! Nothing to do
+    }
+
+    save_current_checkpoints();
+  }
+
+  bool CoinbaseOutputDistributionCache::fetch_and_extend
+  (
+    const uint64_t block_start_offset,
+    const size_t count,
+    const bool cache_front
+  )
+  {
+    CHECK_AND_ASSERT_MES(!(cache_front && count > m_height_begin), false, "Trying to extend front past 0");
+
+    std::vector<uint64_t> extension;
+    extension.reserve(count);
+
+    // To save memory, and not hog blockchain locks, we only request MAX_GET_BLOCKS_CHUNK at a time.
+    // However, we don't want to fetch part of the blocks, modify our cache accordingly, run into an
+    // issue, then abort in an inconsistent state. So, the variable extension will contain a compact
+    // list of only the values that are needed. Then, post-fetch, we add all the values in extension
+    // to our cache in one go.
+    static constexpr size_t MAX_GET_BLOCKS_CHUNK = 1000;
+    for (size_t num_fetched = 0; num_fetched < count; num_fetched += MAX_GET_BLOCKS_CHUNK)
+    {
+      const size_t remaining = count - num_fetched;
+      const size_t chunk_size = std::min(remaining, MAX_GET_BLOCKS_CHUNK);
+      const size_t chunk_start = block_start_offset + num_fetched;
+
+      std::vector<block> blocks;
+      if (!m_get_blocks(chunk_start, chunk_size, blocks))
+      {
+        MERROR("Could not get blocks " << block_start_offset << "/+" << count);
+        return false;
+      }
+
+      for (const auto& b : blocks)
+      {
+        extension.push_back(b.miner_tx.vout.size());
+      }
+    }
+
+    CHECK_AND_ASSERT_MES(extension.size() == count, false, "internal bug: chunking wrong size");
+
+    if (cache_front)
+    {
+      for (auto it = extension.rbegin(); it < extension.rend(); ++it)
+      {
+        m_num_cb_outs_per_block.push_front(*it);
+      }
+
+      // Adding blocks to front of the deque changes our starting height
+      m_height_begin -= count;
+    }
+    else
+    {
+      for (auto it = extension.begin(); it < extension.end(); ++it)
+      {
+        m_num_cb_outs_per_block.push_back(*it);
+      }
+    }
+
+    if (count)
+    {
+      save_current_checkpoints();
+    }
+
+    return true;
+  }
+
+  void CoinbaseOutputDistributionCache::save_current_checkpoints()
+  {
+    const uint64_t num_cached = m_num_cb_outs_per_block.size();
+    if (num_cached >= 1) m_last_1_hash = m_get_block_id_by_height(height_end() - 1);
+    if (num_cached >= 10) m_last_10_hash = m_get_block_id_by_height(height_end() - 10);
+    if (num_cached >= 100) m_last_100_hash = m_get_block_id_by_height(height_end() - 100);
+  }
 } // rpc
 } // cryptonote

src/rpc/rpc_handler.h

@@ -30,13 +30,15 @@
 
 #include <boost/optional/optional.hpp>
 #include <cstdint>
+#include <deque>
 #include <string>
 #include <vector>
 #include "byte_slice.h"
 #include "crypto/hash.h"
 
 namespace cryptonote
 {
+class block;
 class core;
 
 namespace rpc
@@ -47,6 +49,11 @@ struct output_distribution_data
   std::vector<std::uint64_t> distribution;
   std::uint64_t start_height;
   std::uint64_t base;
+
+  bool operator==(const output_distribution_data& rhs) const
+  {
+    return distribution == rhs.distribution && start_height == rhs.start_height && base == rhs.base;
+  }
 };
 
 class RpcHandler
@@ -61,6 +68,59 @@ class RpcHandler
       get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height);
 };
 
+class CoinbaseOutputDistributionCache
+{
+public:
+  using get_blocks_f = std::function<bool(uint64_t, size_t, std::vector<block>&)>;
+  using get_block_id_by_height_f = std::function<crypto::hash(uint64_t)>;
+
+  template <class GetBlocksFunc, class GetBlockIdByHeightFunc>
+  CoinbaseOutputDistributionCache
+  (
+    GetBlocksFunc&& gbf,
+    GetBlockIdByHeightFunc&& gbibhf
+  )
+    : m_num_cb_outs_per_block()
+    , m_height_begin()
+    , m_last_1_hash()
+    , m_last_10_hash()
+    , m_last_100_hash()
+    , m_get_blocks(gbf)
+    , m_get_block_id_by_height(gbibhf)
+    , m_mutex()
+  {}
+
+  bool get_coinbase_output_distribution
+  (
+    uint64_t start_height, // inclusive
+    uint64_t stop_height, // inclusive
+    std::vector<uint64_t>& num_cb_outs_per_block,
+    bool* only_used_cache = nullptr
+  );
+
+private:
+  uint64_t height_end() const // exclusive
+  {
+    return m_height_begin + m_num_cb_outs_per_block.size();
+  };
+
+  void rollback_for_reorgs();
+
+  bool fetch_and_extend(uint64_t block_start_offset, size_t count, bool cache_front);
+
+  void save_current_checkpoints();
+
+  std::deque<uint64_t> m_num_cb_outs_per_block; // NOT cumulative like RCT offsets
+  uint64_t m_height_begin; // inclusive
+  crypto::hash m_last_1_hash;
+  crypto::hash m_last_10_hash;
+  crypto::hash m_last_100_hash;
+
+  get_blocks_f m_get_blocks;
+  get_block_id_by_height_f m_get_block_id_by_height;
+
+  std::mutex m_mutex;
+};
 
 }  // rpc