Upstream merge 5

CMakeLists.txt

@@ -851,10 +851,10 @@ if (${BOOST_IGNORE_SYSTEM_PATHS} STREQUAL "ON")
 endif()
 
 set(OLD_LIB_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+set(Boost_NO_BOOST_CMAKE ON)
 if(STATIC)
   if(MINGW)
     set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
-    set(Boost_NO_BOOST_CMAKE ON)
   endif()
 
   set(Boost_USE_STATIC_LIBS ON)

contrib/epee/include/serialization/keyvalue_serialization_overloads.h

@@ -35,6 +35,11 @@
 
 namespace epee
 {
+  namespace
+  {
+    template<class C> void hint_resize(C &container, size_t size) {}
+    template<class C> void hint_resize(std::vector<C> &container, size_t size) { container.reserve(size); }
+  }
   namespace serialization
   {
 
@@ -158,6 +163,7 @@ namespace epee
           false, 
           "size in blob " << loaded_size << " not have not zero modulo for sizeof(value_type) = " << sizeof(typename stl_container::value_type) << ", type " << typeid(typename stl_container::value_type).name());
         size_t count = (loaded_size/sizeof(typename stl_container::value_type));
+        hint_resize(container, count);
         for(size_t i = 0; i < count; i++)
           container.insert(container.end(), *(pelem++));
       }

contrib/epee/include/storages/parserse_base_utils.h

@@ -28,6 +28,8 @@
 
 #pragma once 
 
+#include <algorithm>
+
 namespace epee 
 {
 namespace misc_utils
@@ -36,8 +38,12 @@ namespace misc_utils
   {
     inline std::string transform_to_escape_sequence(const std::string& src)
     {
-      //std::stringstream res;
+      static const char escaped[] = "\b\f\n\r\t\v\"\\/";
+      if (std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)) == src.end())
+        return src;
+
       std::string res;
+      res.reserve(2 * src.size());
       for(std::string::const_iterator it = src.begin(); it!=src.end(); ++it)
       {
         switch(*it)
@@ -84,6 +90,7 @@ namespace misc_utils
       inline void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
       {
         val.clear();
+        val.reserve(std::distance(star_end_string, buf_end));
         bool escape_mode = false;
         std::string::const_iterator it = star_end_string;
         ++it;

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -573,8 +573,8 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
   LOG_PRINT_L1("Space used:      " << size_used);
   LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used);
   LOG_PRINT_L1("Size threshold:  " << threshold_size);
-  float resize_percent_old = RESIZE_PERCENT;
-  LOG_PRINT_L1(boost::format("Percent used: %.04f  Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old);
+  float resize_percent = RESIZE_PERCENT;
+  LOG_PRINT_L1(boost::format("Percent used: %.04f  Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent);
 
   if (threshold_size > 0)
   {
@@ -587,10 +587,6 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
       return false;
   }
 
-  std::mt19937 engine(std::random_device{}());
-  std::uniform_real_distribution<double> fdis(0.6, 0.9);
-  double resize_percent = fdis(engine);
-
   if ((double)size_used / mei.me_mapsize  > resize_percent)
   {
     LOG_PRINT_L1("Threshold met (percent-based)");
@@ -1230,8 +1226,12 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
     throw DB_ERROR("Database could not be opened");
   }
 
-  if (tools::is_hdd(filename.c_str()))
-    MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible");
+  boost::optional<bool> is_hdd_result = tools::is_hdd(filename.c_str());
+  if (is_hdd_result)
+  {
+    if (is_hdd_result.value())
+        MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible");
+  }
 
   m_folder = filename;
 
@@ -2001,14 +2001,25 @@ std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std
 
   MDB_val v;
 
+  uint64_t prev_height = heights[0];
   for (uint64_t height: heights)
   {
-    MDB_val_set(v, height);
-    result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+    if (height == prev_height + 1)
+    {
+      MDB_val k2;
+      result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT);
+    }
+    else
+    {
+      v.mv_size = sizeof(uint64_t);
+      v.mv_data = (void*)&height;
+      result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+    }
     if (result)
       throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str()));
     const mdb_block_info *bi = (const mdb_block_info *)v.mv_data;
     res.push_back(bi->bi_cum_rct);
+    prev_height = height;
   }
 
   TXN_POSTFIX_RDONLY();

src/blockchain_db/lmdb/db_lmdb.h

@@ -465,7 +465,7 @@ class BlockchainLMDB : public BlockchainDB
 #endif
 #endif
 
-  constexpr static float RESIZE_PERCENT = 0.8f;
+  constexpr static float RESIZE_PERCENT = 0.9f;
 };
 
 }  // namespace cryptonote

src/common/util.cpp

@@ -45,6 +45,13 @@
 #include <string>
 #endif
 
+//tools::is_hdd
+#ifdef __GLIBC__
+  #include <sstream>
+  #include <sys/sysmacros.h>
+  #include <fstream>
+#endif
+
 #include "unbound.h"
 
 #include "include_base_utils.h"
@@ -761,62 +768,41 @@ std::string get_nix_version_display_string()
 #endif
   }
 
-  bool is_hdd(const char *path)
+  boost::optional<bool> is_hdd(const char *file_path)
   {
 #ifdef __GLIBC__
-    std::string device = "";
-    struct stat st, dst;
-    if (stat(path, &st) < 0)
-      return 0;
-
-    DIR *dir = opendir("/dev/block");
-    if (!dir)
-      return 0;
-    struct dirent *de;
-    while ((de = readdir(dir)))
-    {
-      if (strcmp(de->d_name, ".") && strcmp(de->d_name, ".."))
+    struct stat st;
+    std::string prefix;
+    if(stat(file_path, &st) == 0)
+    {
+      std::ostringstream s;
+      s << "/sys/dev/block/" << major(st.st_dev) << ":" << minor(st.st_dev);
+      prefix = s.str();
+    }
+    else
+    {
+      return boost::none;
+    }
+    std::string attr_path = prefix + "/queue/rotational";
+    std::ifstream f(attr_path, std::ios_base::in);
+    if(not f.is_open())
+    {
+      attr_path = prefix + "/../queue/rotational";
+      f.open(attr_path, std::ios_base::in);
+      if(not f.is_open())
       {
-        std::string dev_path = std::string("/dev/block/") + de->d_name;
-        char resolved[PATH_MAX];
-        if (realpath(dev_path.c_str(), resolved) && !strncmp(resolved, "/dev/", 5))
-        {
-          if (stat(resolved, &dst) == 0)
-          {
-            if (dst.st_rdev == st.st_dev)
-            {
-              // take out trailing digits (eg, sda1 -> sda)
-              char *ptr = resolved;
-              while (*ptr)
-                ++ptr;
-              while (ptr > resolved && isdigit(*--ptr))
-                *ptr = 0;
-              device = resolved + 5;
-              break;
-            }
-          }
-        }
+          return boost::none;
       }
     }
-    closedir(dir);
-
-    if (device.empty())
-      return 0;
-
-    std::string sys_path = "/sys/block/" + device + "/queue/rotational";
-    FILE *f = fopen(sys_path.c_str(), "r");
-    if (!f)
-      return false;
-    char s[8];
-    char *ptr = fgets(s, sizeof(s), f);
-    fclose(f);
-    if (!ptr)
-      return 0;
-    s[sizeof(s) - 1] = 0;
-    int n = atoi(s); // returns 0 on parse error
-    return n == 1;
+    unsigned short val = 0xdead;
+    f >> val;
+    if(not f.fail())
+    {
+      return (val == 1);
+    }
+    return boost::none;
 #else
-    return 0;
+    return boost::none;
 #endif
   }
 

src/common/util.h

@@ -230,7 +230,7 @@ namespace tools
   bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash);
   bool sha256sum(const std::string &filename, crypto::hash &hash);
 
-  bool is_hdd(const char *path);
+  boost::optional<bool> is_hdd(const char *path);
 
   boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str);
 

src/crypto/keccak.c

@@ -132,3 +132,77 @@ void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md)
 {
     keccak(in, inlen, md, sizeof(state_t));
 }
+
+#define KECCAK_FINALIZED 0x80000000
+#define KECCAK_BLOCKLEN 136
+#define KECCAK_WORDS 17
+#define KECCAK_DIGESTSIZE 32
+#define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0)))
+#define KECCAK_PROCESS_BLOCK(st, block) { \
+    for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \
+        ((st))[i_] ^= ((block))[i_]; \
+    }; \
+    keccakf(st, KECCAK_ROUNDS); }
+
+
+void keccak_init(KECCAK_CTX * ctx){
+    memset(ctx, 0, sizeof(KECCAK_CTX));
+}
+
+void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen){
+    if (ctx->rest & KECCAK_FINALIZED) {
+        local_abort("Bad keccak use");
+    }
+
+    const size_t idx = ctx->rest;
+    ctx->rest = (ctx->rest + inlen) % KECCAK_BLOCKLEN;
+
+    // fill partial block
+    if (idx) {
+        size_t left = KECCAK_BLOCKLEN - idx;
+        memcpy((char*)ctx->message + idx, in, (inlen < left ? inlen : left));
+        if (inlen < left) return;
+
+        KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message);
+
+        in  += left;
+        inlen -= left;
+    }
+
+    const bool is_aligned = IS_ALIGNED_64(in);
+    while (inlen >= KECCAK_BLOCKLEN) {
+        const uint64_t* aligned_message_block;
+        if (is_aligned) {
+            aligned_message_block = (uint64_t*)in;
+        } else {
+            memcpy(ctx->message, in, KECCAK_BLOCKLEN);
+            aligned_message_block = ctx->message;
+        }
+
+        KECCAK_PROCESS_BLOCK(ctx->hash, aligned_message_block);
+        in  += KECCAK_BLOCKLEN;
+        inlen -= KECCAK_BLOCKLEN;
+    }
+    if (inlen) {
+        memcpy(ctx->message, in, inlen);
+    }
+}
+
+void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){
+    if (!(ctx->rest & KECCAK_FINALIZED))
+    {
+        // clear the rest of the data queue
+        memset((char*)ctx->message + ctx->rest, 0, KECCAK_BLOCKLEN - ctx->rest);
+        ((char*)ctx->message)[ctx->rest] |= 0x01;
+        ((char*)ctx->message)[KECCAK_BLOCKLEN - 1] |= 0x80;
+
+        // process final block
+        KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message);
+        ctx->rest = KECCAK_FINALIZED; // mark context as finalized
+    }
+
+    static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, "");
+    if (md) {
+        memcpy(md, ctx->hash, KECCAK_DIGESTSIZE);
+    }
+}

src/crypto/keccak.h

@@ -15,6 +15,17 @@
 #define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y))))
 #endif
 
+// SHA3 Algorithm context.
+typedef struct KECCAK_CTX
+{
+  // 1600 bits algorithm hashing state
+  uint64_t hash[25];
+  // 1088-bit buffer for leftovers, block size = 136 B for 256-bit keccak
+  uint64_t message[17];
+  // count of bytes in the message[] buffer
+  size_t rest;
+} KECCAK_CTX;
+
 // compute a keccak hash (md) of given byte length from "in"
 void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen);
 
@@ -23,4 +34,7 @@ void keccakf(uint64_t st[25], int norounds);
 
 void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md);
 
+void keccak_init(KECCAK_CTX * ctx);
+void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen);
+void keccak_finish(KECCAK_CTX * ctx, uint8_t *md);
 #endif

src/cryptonote_core/cryptonote_tx_utils.cpp

@@ -543,6 +543,7 @@ namespace cryptonote
 
       uint64_t amount_in = 0, amount_out = 0;
       rct::ctkeyV inSk;
+      inSk.reserve(sources.size());
       // mixRing indexing is done the other way round for simple
       rct::ctkeyM mixRing(use_simple_rct ? sources.size() : n_total_outs);
       rct::keyV destinations;
@@ -559,6 +560,7 @@ namespace cryptonote
         ctkey.dest = rct::sk2rct(in_contexts[i].in_ephemeral.sec);
         ctkey.mask = sources[i].mask;
         inSk.push_back(ctkey);
+        memwipe(&ctkey, sizeof(rct::ctkey));
         // inPk: (public key, commitment)
         // will be done when filling in mixRing
         if (msout)
@@ -617,6 +619,7 @@ namespace cryptonote
         tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, bulletproof, hwdev);
       else
         tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, bulletproof, hwdev); // same index assumption
+      memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey));
 
       CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout");