From c46c4141d961f0e3f1ec76730624dc2e97dd86d2 Mon Sep 17 00:00:00 2001 From: Rob Patro Date: Sat, 28 Jul 2018 12:07:47 -0400 Subject: [PATCH 1/4] back on develop after 0.11.1 --- current_version.txt | 2 +- doc/source/conf.py | 2 +- docker/Dockerfile | 2 +- include/SalmonConfig.hpp | 4 ++-- scripts/fetchRapMap.sh | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/current_version.txt b/current_version.txt index 1015d4eb5..6229b383f 100644 --- a/current_version.txt +++ b/current_version.txt @@ -1,3 +1,3 @@ VERSION_MAJOR 0 VERSION_MINOR 11 -VERSION_PATCH 1 +VERSION_PATCH 2 diff --git a/doc/source/conf.py b/doc/source/conf.py index 405dd9ff2..04f61aa6d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -57,7 +57,7 @@ # The short X.Y version. version = '0.11' # The full version, including alpha/beta/rc tags. -release = '0.11.1' +release = '0.11.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docker/Dockerfile b/docker/Dockerfile index c7d206966..abb86337d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ MAINTAINER salmon.maintainer@gmail.com ENV PACKAGES git gcc make g++ cmake libboost-all-dev liblzma-dev libbz2-dev \ ca-certificates zlib1g-dev curl unzip autoconf -ENV SALMON_VERSION 0.11.1 +ENV SALMON_VERSION 0.11.2 # salmon binary will be installed in /home/salmon/bin/salmon diff --git a/include/SalmonConfig.hpp b/include/SalmonConfig.hpp index 01a9d798d..e06f0b447 100644 --- a/include/SalmonConfig.hpp +++ b/include/SalmonConfig.hpp @@ -27,8 +27,8 @@ namespace salmon { constexpr char majorVersion[] = "0"; constexpr char minorVersion[] = "11"; -constexpr char patchVersion[] = "1"; -constexpr char version[] = "0.11.1"; +constexpr char patchVersion[] = "2"; +constexpr char version[] = "0.11.2"; constexpr uint32_t indexVersion = 2; constexpr char requiredQuasiIndexVersion[] = "q5"; } // namespace salmon diff --git a/scripts/fetchRapMap.sh b/scripts/fetchRapMap.sh index 2d39298cb..a6ea50205 100755 --- a/scripts/fetchRapMap.sh +++ b/scripts/fetchRapMap.sh @@ -22,8 +22,8 @@ if [ -d ${INSTALL_DIR}/src/rapmap ] ; then rm -fr ${INSTALL_DIR}/src/rapmap fi -SVER=salmon-v0.11.1 -#SVER=develop-salmon +#SVER=salmon-v0.11.1 +SVER=develop-salmon #SVER=pe-chaining EXPECTED_SHA256=489c365f859b647564374830325e2a9407a8d5d066b2c4e89f3f671c3a39221b From fb357db88fa88eb7a274b27c5d3600762926c349 Mon Sep 17 00:00:00 2001 From: Rob Patro Date: Tue, 31 Jul 2018 21:55:55 -0400 Subject: [PATCH 2/4] use faster hash for alignment cache --- CMakeLists.txt | 2 +- include/tsl/bhopscotch_map.h | 675 +++++++++ include/tsl/bhopscotch_set.h | 529 ++++++++ include/tsl/hopscotch_growth_policy.h | 295 ++++ include/tsl/hopscotch_hash.h | 1801 +++++++++++++++++++++++++ include/tsl/hopscotch_map.h | 679 ++++++++++ include/tsl/hopscotch_set.h | 525 +++++++ src/CMakeLists.txt | 11 + src/SalmonQuantify.cpp | 12 +- 9 files changed, 4523 insertions(+), 6 deletions(-) create mode 100644 include/tsl/bhopscotch_map.h create mode 100644 include/tsl/bhopscotch_set.h create mode 100644 include/tsl/hopscotch_growth_policy.h create mode 100644 include/tsl/hopscotch_hash.h create mode 100644 include/tsl/hopscotch_map.h create mode 100644 include/tsl/hopscotch_set.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 073707da0..a98093a41 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ endif() ## Set the standard required compile flags # Nov 18th --- removed -DHAVE_CONFIG_H set (REMOVE_WARNING_FLAGS "-Wno-unused-function;-Wno-unused-local-typedefs") -set (TGT_COMPILE_FLAGS "-ftree-vectorize;-funroll-loops;-fPIC;-fomit-frame-pointer;-O3") +set (TGT_COMPILE_FLAGS "-ftree-vectorize;-funroll-loops;-fPIC;-fomit-frame-pointer;-O3;-DNDEBUG") set (TGT_WARN_FLAGS "-Wall;-Wno-unknown-pragmas;-Wno-reorder;-Wno-unused-variable;-Wreturn-type;-Werror=return-type;${REMOVE_WARNING_FLAGS}") if (APPLE) diff --git a/include/tsl/bhopscotch_map.h b/include/tsl/bhopscotch_map.h new file mode 100644 index 000000000..e5bcbf32a --- /dev/null +++ b/include/tsl/bhopscotch_map.h @@ -0,0 +1,675 @@ +/** + * MIT License + * + * Copyright (c) 2017 Tessil + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_BHOPSCOTCH_MAP_H +#define TSL_BHOPSCOTCH_MAP_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hopscotch_hash.h" + + +namespace tsl { + + +/** + * Similar to tsl::hopscotch_map but instead of using a list for overflowing elements it uses + * a binary search tree. It thus needs an additional template parameter Compare. Compare should + * be arithmetically coherent with KeyEqual. + * + * The binary search tree allows the map to have a worst-case scenario of O(log n) for search + * and delete, even if the hash function maps all the elements to the same bucket. + * For insert, the amortized worst case is O(log n), but the worst case is O(n) in case of rehash. + * + * This makes the map resistant to DoS attacks (but doesn't preclude you to have a good hash function, + * as an element in the bucket array is faster to retrieve than in the tree). + * + * @copydoc hopscotch_map + */ +template, + class KeyEqual = std::equal_to, + class Compare = std::less, + class Allocator = std::allocator>, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false, + class GrowthPolicy = tsl::hh::power_of_two_growth_policy<2>> +class bhopscotch_map { +private: + template + using has_is_transparent = tsl::detail_hopscotch_hash::has_is_transparent; + + class KeySelect { + public: + using key_type = Key; + + const key_type& operator()(const std::pair& key_value) const { + return key_value.first; + } + + const key_type& operator()(std::pair& key_value) { + return key_value.first; + } + }; + + class ValueSelect { + public: + using value_type = T; + + const value_type& operator()(const std::pair& key_value) const { + return key_value.second; + } + + value_type& operator()(std::pair& key_value) { + return key_value.second; + } + }; + + + // TODO Not optimal as we have to use std::pair as ValueType which forbid + // us to move the key in the bucket array, we have to use copy. Optimize. + using overflow_container_type = std::map; + using ht = detail_hopscotch_hash::hopscotch_hash, KeySelect, ValueSelect, + Hash, KeyEqual, + Allocator, NeighborhoodSize, + StoreHash, GrowthPolicy, + overflow_container_type>; + +public: + using key_type = typename ht::key_type; + using mapped_type = T; + using value_type = typename ht::value_type; + using size_type = typename ht::size_type; + using difference_type = typename ht::difference_type; + using hasher = typename ht::hasher; + using key_equal = typename ht::key_equal; + using key_compare = Compare; + using allocator_type = typename ht::allocator_type; + using reference = typename ht::reference; + using const_reference = typename ht::const_reference; + using pointer = typename ht::pointer; + using const_pointer = typename ht::const_pointer; + using iterator = typename ht::iterator; + using const_iterator = typename ht::const_iterator; + + + /* + * Constructors + */ + bhopscotch_map() : bhopscotch_map(ht::DEFAULT_INIT_BUCKETS_SIZE) { + } + + explicit bhopscotch_map(size_type bucket_count, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator(), + const Compare& comp = Compare()) : + m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR, comp) + { + } + + bhopscotch_map(size_type bucket_count, + const Allocator& alloc) : bhopscotch_map(bucket_count, Hash(), KeyEqual(), alloc) + { + } + + bhopscotch_map(size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : bhopscotch_map(bucket_count, hash, KeyEqual(), alloc) + { + } + + explicit bhopscotch_map(const Allocator& alloc) : bhopscotch_map(ht::DEFAULT_INIT_BUCKETS_SIZE, alloc) { + } + + template + bhopscotch_map(InputIt first, InputIt last, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : bhopscotch_map(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + + template + bhopscotch_map(InputIt first, InputIt last, + size_type bucket_count, + const Allocator& alloc) : bhopscotch_map(first, last, bucket_count, Hash(), KeyEqual(), alloc) + { + } + + template + bhopscotch_map(InputIt first, InputIt last, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : bhopscotch_map(first, last, bucket_count, hash, KeyEqual(), alloc) + { + } + + bhopscotch_map(std::initializer_list init, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : + bhopscotch_map(init.begin(), init.end(), bucket_count, hash, equal, alloc) + { + } + + bhopscotch_map(std::initializer_list init, + size_type bucket_count, + const Allocator& alloc) : + bhopscotch_map(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), alloc) + { + } + + bhopscotch_map(std::initializer_list init, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : + bhopscotch_map(init.begin(), init.end(), bucket_count, hash, KeyEqual(), alloc) + { + } + + + bhopscotch_map& operator=(std::initializer_list ilist) { + m_ht.clear(); + + m_ht.reserve(ilist.size()); + m_ht.insert(ilist.begin(), ilist.end()); + + return *this; + } + + allocator_type get_allocator() const { return m_ht.get_allocator(); } + + + /* + * Iterators + */ + iterator begin() noexcept { return m_ht.begin(); } + const_iterator begin() const noexcept { return m_ht.begin(); } + const_iterator cbegin() const noexcept { return m_ht.cbegin(); } + + iterator end() noexcept { return m_ht.end(); } + const_iterator end() const noexcept { return m_ht.end(); } + const_iterator cend() const noexcept { return m_ht.cend(); } + + + /* + * Capacity + */ + bool empty() const noexcept { return m_ht.empty(); } + size_type size() const noexcept { return m_ht.size(); } + size_type max_size() const noexcept { return m_ht.max_size(); } + + /* + * Modifiers + */ + void clear() noexcept { m_ht.clear(); } + + + + + std::pair insert(const value_type& value) { + return m_ht.insert(value); + } + + template::value>::type* = nullptr> + std::pair insert(P&& value) { + return m_ht.insert(std::forward

(value)); + } + + std::pair insert(value_type&& value) { + return m_ht.insert(std::move(value)); + } + + + iterator insert(const_iterator hint, const value_type& value) { + return m_ht.insert(hint, value); + } + + template::value>::type* = nullptr> + iterator insert(const_iterator hint, P&& value) { + return m_ht.insert(hint, std::forward

(value)); + } + + iterator insert(const_iterator hint, value_type&& value) { + return m_ht.insert(hint, std::move(value)); + } + + + template + void insert(InputIt first, InputIt last) { + m_ht.insert(first, last); + } + + void insert(std::initializer_list ilist) { + m_ht.insert(ilist.begin(), ilist.end()); + } + + + + + template + std::pair insert_or_assign(const key_type& k, M&& obj) { + return m_ht.insert_or_assign(k, std::forward(obj)); + } + + template + std::pair insert_or_assign(key_type&& k, M&& obj) { + return m_ht.insert_or_assign(std::move(k), std::forward(obj)); + } + + template + iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj) { + return m_ht.insert_or_assign(hint, k, std::forward(obj)); + } + + template + iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj) { + return m_ht.insert_or_assign(hint, std::move(k), std::forward(obj)); + } + + + + /** + * Due to the way elements are stored, emplace will need to move or copy the key-value once. + * The method is equivalent to insert(value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + std::pair emplace(Args&&... args) { + return m_ht.emplace(std::forward(args)...); + } + + + + + /** + * Due to the way elements are stored, emplace_hint will need to move or copy the key-value once. + * The method is equivalent to insert(hint, value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + iterator emplace_hint(const_iterator hint, Args&&... args) { + return m_ht.emplace_hint(hint, std::forward(args)...); + } + + + + + template + std::pair try_emplace(const key_type& k, Args&&... args) { + return m_ht.try_emplace(k, std::forward(args)...); + } + + template + std::pair try_emplace(key_type&& k, Args&&... args) { + return m_ht.try_emplace(std::move(k), std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args) { + return m_ht.try_emplace(hint, k, std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args) { + return m_ht.try_emplace(hint, std::move(k), std::forward(args)...); + } + + + + + iterator erase(iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator first, const_iterator last) { return m_ht.erase(first, last); } + size_type erase(const key_type& key) { return m_ht.erase(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + size_type erase(const key_type& key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type erase(const K& key) { return m_ht.erase(key); } + + /** + * @copydoc erase(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type erase(const K& key, std::size_t precalculated_hash) { return m_ht.erase(key, precalculated_hash); } + + + + + void swap(bhopscotch_map& other) { other.m_ht.swap(m_ht); } + + /* + * Lookup + */ + T& at(const Key& key) { return m_ht.at(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + T& at(const Key& key, std::size_t precalculated_hash) { return m_ht.at(key, precalculated_hash); } + + const T& at(const Key& key) const { return m_ht.at(key); } + + /** + * @copydoc at(const Key& key, std::size_t precalculated_hash) + */ + const T& at(const Key& key, std::size_t precalculated_hash) const { return m_ht.at(key, precalculated_hash); } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + T& at(const K& key) { return m_ht.at(key); } + + /** + * @copydoc at(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + T& at(const K& key, std::size_t precalculated_hash) { return m_ht.at(key, precalculated_hash); } + + /** + * @copydoc at(const K& key) + */ + template::value && has_is_transparent::value>::type* = nullptr> + const T& at(const K& key) const { return m_ht.at(key); } + + /** + * @copydoc at(const K& key, std::size_t precalculated_hash) + */ + template::value && has_is_transparent::value>::type* = nullptr> + const T& at(const K& key, std::size_t precalculated_hash) const { return m_ht.at(key, precalculated_hash); } + + + + + T& operator[](const Key& key) { return m_ht[key]; } + T& operator[](Key&& key) { return m_ht[std::move(key)]; } + + + + + size_type count(const Key& key) const { return m_ht.count(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + size_type count(const Key& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type count(const K& key) const { return m_ht.count(key); } + + /** + * @copydoc count(const K& key) const + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type count(const K& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } + + + + + iterator find(const Key& key) { return m_ht.find(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + iterator find(const Key& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + const_iterator find(const Key& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const Key& key, std::size_t precalculated_hash) + */ + const_iterator find(const Key& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + iterator find(const K& key) { return m_ht.find(key); } + + /** + * @copydoc find(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + iterator find(const K& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + /** + * @copydoc find(const K& key) + */ + template::value && has_is_transparent::value>::type* = nullptr> + const_iterator find(const K& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const K& key, std::size_t precalculated_hash) + */ + template::value && has_is_transparent::value>::type* = nullptr> + const_iterator find(const K& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } + + + + + std::pair equal_range(const Key& key) { return m_ht.equal_range(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + std::pair equal_range(const Key& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key) { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * @copydoc equal_range(const K& key) + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const K& key, std::size_t precalculated_hash) + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + + + + /* + * Bucket interface + */ + size_type bucket_count() const { return m_ht.bucket_count(); } + size_type max_bucket_count() const { return m_ht.max_bucket_count(); } + + + /* + * Hash policy + */ + float load_factor() const { return m_ht.load_factor(); } + float max_load_factor() const { return m_ht.max_load_factor(); } + void max_load_factor(float ml) { m_ht.max_load_factor(ml); } + + void rehash(size_type count_) { m_ht.rehash(count_); } + void reserve(size_type count_) { m_ht.reserve(count_); } + + + /* + * Observers + */ + hasher hash_function() const { return m_ht.hash_function(); } + key_equal key_eq() const { return m_ht.key_eq(); } + key_compare key_comp() const { return m_ht.key_comp(); } + + /* + * Other + */ + + /** + * Convert a const_iterator to an iterator. + */ + iterator mutable_iterator(const_iterator pos) { + return m_ht.mutable_iterator(pos); + } + + size_type overflow_size() const noexcept { return m_ht.overflow_size(); } + + friend bool operator==(const bhopscotch_map& lhs, const bhopscotch_map& rhs) { + if(lhs.size() != rhs.size()) { + return false; + } + + for(const auto& element_lhs : lhs) { + const auto it_element_rhs = rhs.find(element_lhs.first); + if(it_element_rhs == rhs.cend() || element_lhs.second != it_element_rhs->second) { + return false; + } + } + + return true; + } + + friend bool operator!=(const bhopscotch_map& lhs, const bhopscotch_map& rhs) { + return !operator==(lhs, rhs); + } + + friend void swap(bhopscotch_map& lhs, bhopscotch_map& rhs) { + lhs.swap(rhs); + } + + + +private: + ht m_ht; +}; + + +/** + * Same as `tsl::bhopscotch_map`. + */ +template, + class KeyEqual = std::equal_to, + class Compare = std::less, + class Allocator = std::allocator>, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false> +using bhopscotch_pg_map = bhopscotch_map; + +} // end namespace tsl + +#endif diff --git a/include/tsl/bhopscotch_set.h b/include/tsl/bhopscotch_set.h new file mode 100644 index 000000000..86d563ced --- /dev/null +++ b/include/tsl/bhopscotch_set.h @@ -0,0 +1,529 @@ +/** + * MIT License + * + * Copyright (c) 2017 Tessil + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_BHOPSCOTCH_SET_H +#define TSL_BHOPSCOTCH_SET_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hopscotch_hash.h" + + +namespace tsl { + + +/** + * Similar to tsl::hopscotch_set but instead of using a list for overflowing elements it uses + * a binary search tree. It thus needs an additional template parameter Compare. Compare should + * be arithmetically coherent with KeyEqual. + * + * The binary search tree allows the set to have a worst-case scenario of O(log n) for search + * and delete, even if the hash function maps all the elements to the same bucket. + * For insert, the amortized worst case is O(log n), but the worst case is O(n) in case of rehash. + * + * This makes the set resistant to DoS attacks (but doesn't preclude you to have a good hash function, + * as an element in the bucket array is faster to retrieve than in the tree). + * + * @copydoc hopscotch_set + */ +template, + class KeyEqual = std::equal_to, + class Compare = std::less, + class Allocator = std::allocator, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false, + class GrowthPolicy = tsl::hh::power_of_two_growth_policy<2>> +class bhopscotch_set { +private: + template + using has_is_transparent = tsl::detail_hopscotch_hash::has_is_transparent; + + class KeySelect { + public: + using key_type = Key; + + const key_type& operator()(const Key& key) const { + return key; + } + + key_type& operator()(Key& key) { + return key; + } + }; + + + using overflow_container_type = std::set; + using ht = tsl::detail_hopscotch_hash::hopscotch_hash; + +public: + using key_type = typename ht::key_type; + using value_type = typename ht::value_type; + using size_type = typename ht::size_type; + using difference_type = typename ht::difference_type; + using hasher = typename ht::hasher; + using key_equal = typename ht::key_equal; + using key_compare = Compare; + using allocator_type = typename ht::allocator_type; + using reference = typename ht::reference; + using const_reference = typename ht::const_reference; + using pointer = typename ht::pointer; + using const_pointer = typename ht::const_pointer; + using iterator = typename ht::iterator; + using const_iterator = typename ht::const_iterator; + + + /* + * Constructors + */ + bhopscotch_set() : bhopscotch_set(ht::DEFAULT_INIT_BUCKETS_SIZE) { + } + + explicit bhopscotch_set(size_type bucket_count, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator(), + const Compare& comp = Compare()) : + m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR, comp) + { + } + + bhopscotch_set(size_type bucket_count, + const Allocator& alloc) : bhopscotch_set(bucket_count, Hash(), KeyEqual(), alloc) + { + } + + bhopscotch_set(size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : bhopscotch_set(bucket_count, hash, KeyEqual(), alloc) + { + } + + explicit bhopscotch_set(const Allocator& alloc) : bhopscotch_set(ht::DEFAULT_INIT_BUCKETS_SIZE, alloc) { + } + + template + bhopscotch_set(InputIt first, InputIt last, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : bhopscotch_set(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + + template + bhopscotch_set(InputIt first, InputIt last, + size_type bucket_count, + const Allocator& alloc) : bhopscotch_set(first, last, bucket_count, Hash(), KeyEqual(), alloc) + { + } + + template + bhopscotch_set(InputIt first, InputIt last, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : bhopscotch_set(first, last, bucket_count, hash, KeyEqual(), alloc) + { + } + + bhopscotch_set(std::initializer_list init, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : + bhopscotch_set(init.begin(), init.end(), bucket_count, hash, equal, alloc) + { + } + + bhopscotch_set(std::initializer_list init, + size_type bucket_count, + const Allocator& alloc) : + bhopscotch_set(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), alloc) + { + } + + bhopscotch_set(std::initializer_list init, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : + bhopscotch_set(init.begin(), init.end(), bucket_count, hash, KeyEqual(), alloc) + { + } + + + bhopscotch_set& operator=(std::initializer_list ilist) { + m_ht.clear(); + + m_ht.reserve(ilist.size()); + m_ht.insert(ilist.begin(), ilist.end()); + + return *this; + } + + allocator_type get_allocator() const { return m_ht.get_allocator(); } + + + /* + * Iterators + */ + iterator begin() noexcept { return m_ht.begin(); } + const_iterator begin() const noexcept { return m_ht.begin(); } + const_iterator cbegin() const noexcept { return m_ht.cbegin(); } + + iterator end() noexcept { return m_ht.end(); } + const_iterator end() const noexcept { return m_ht.end(); } + const_iterator cend() const noexcept { return m_ht.cend(); } + + + /* + * Capacity + */ + bool empty() const noexcept { return m_ht.empty(); } + size_type size() const noexcept { return m_ht.size(); } + size_type max_size() const noexcept { return m_ht.max_size(); } + + /* + * Modifiers + */ + void clear() noexcept { m_ht.clear(); } + + + + + std::pair insert(const value_type& value) { return m_ht.insert(value); } + std::pair insert(value_type&& value) { return m_ht.insert(std::move(value)); } + + iterator insert(const_iterator hint, const value_type& value) { return m_ht.insert(hint, value); } + iterator insert(const_iterator hint, value_type&& value) { return m_ht.insert(hint, std::move(value)); } + + template + void insert(InputIt first, InputIt last) { m_ht.insert(first, last); } + void insert(std::initializer_list ilist) { m_ht.insert(ilist.begin(), ilist.end()); } + + + + + /** + * Due to the way elements are stored, emplace will need to move or copy the key-value once. + * The method is equivalent to insert(value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + std::pair emplace(Args&&... args) { return m_ht.emplace(std::forward(args)...); } + + + + + /** + * Due to the way elements are stored, emplace_hint will need to move or copy the key-value once. + * The method is equivalent to insert(hint, value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + iterator emplace_hint(const_iterator hint, Args&&... args) { + return m_ht.emplace_hint(hint, std::forward(args)...); + } + + + + + iterator erase(iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator first, const_iterator last) { return m_ht.erase(first, last); } + size_type erase(const key_type& key) { return m_ht.erase(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + size_type erase(const key_type& key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type erase(const K& key) { return m_ht.erase(key); } + + /** + * @copydoc erase(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type erase(const K& key, std::size_t precalculated_hash) { return m_ht.erase(key, precalculated_hash); } + + + + + void swap(bhopscotch_set& other) { other.m_ht.swap(m_ht); } + + + /* + * Lookup + */ + size_type count(const Key& key) const { return m_ht.count(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + size_type count(const Key& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type count(const K& key) const { return m_ht.count(key); } + + /** + * @copydoc count(const K& key) const + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + size_type count(const K& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } + + + + + iterator find(const Key& key) { return m_ht.find(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + iterator find(const Key& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + const_iterator find(const Key& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const Key& key, std::size_t precalculated_hash) + */ + const_iterator find(const Key& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + iterator find(const K& key) { return m_ht.find(key); } + + /** + * @copydoc find(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + iterator find(const K& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + /** + * @copydoc find(const K& key) + */ + template::value && has_is_transparent::value>::type* = nullptr> + const_iterator find(const K& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + const_iterator find(const K& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } + + + + + std::pair equal_range(const Key& key) { return m_ht.equal_range(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + std::pair equal_range(const Key& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent + * and Compare::is_transparent exist. + * If so, K must be hashable and comparable to Key. + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key) { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * @copydoc equal_range(const K& key) + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const K& key, std::size_t precalculated_hash) + */ + template::value && has_is_transparent::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + + + + /* + * Bucket interface + */ + size_type bucket_count() const { return m_ht.bucket_count(); } + size_type max_bucket_count() const { return m_ht.max_bucket_count(); } + + + /* + * Hash policy + */ + float load_factor() const { return m_ht.load_factor(); } + float max_load_factor() const { return m_ht.max_load_factor(); } + void max_load_factor(float ml) { m_ht.max_load_factor(ml); } + + void rehash(size_type count_) { m_ht.rehash(count_); } + void reserve(size_type count_) { m_ht.reserve(count_); } + + + /* + * Observers + */ + hasher hash_function() const { return m_ht.hash_function(); } + key_equal key_eq() const { return m_ht.key_eq(); } + key_compare key_comp() const { return m_ht.key_comp(); } + + + /* + * Other + */ + + /** + * Convert a const_iterator to an iterator. + */ + iterator mutable_iterator(const_iterator pos) { + return m_ht.mutable_iterator(pos); + } + + size_type overflow_size() const noexcept { return m_ht.overflow_size(); } + + friend bool operator==(const bhopscotch_set& lhs, const bhopscotch_set& rhs) { + if(lhs.size() != rhs.size()) { + return false; + } + + for(const auto& element_lhs : lhs) { + const auto it_element_rhs = rhs.find(element_lhs); + if(it_element_rhs == rhs.cend()) { + return false; + } + } + + return true; + } + + friend bool operator!=(const bhopscotch_set& lhs, const bhopscotch_set& rhs) { + return !operator==(lhs, rhs); + } + + friend void swap(bhopscotch_set& lhs, bhopscotch_set& rhs) { + lhs.swap(rhs); + } + +private: + ht m_ht; +}; + + +/** + * Same as `tsl::bhopscotch_set`. + */ +template, + class KeyEqual = std::equal_to, + class Compare = std::less, + class Allocator = std::allocator, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false> +using bhopscotch_pg_set = bhopscotch_set; + +} // end namespace tsl + +#endif diff --git a/include/tsl/hopscotch_growth_policy.h b/include/tsl/hopscotch_growth_policy.h new file mode 100644 index 000000000..65b1845de --- /dev/null +++ b/include/tsl/hopscotch_growth_policy.h @@ -0,0 +1,295 @@ +/** + * MIT License + * + * Copyright (c) 2018 Tessil + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_HOPSCOTCH_GROWTH_POLICY_H +#define TSL_HOPSCOTCH_GROWTH_POLICY_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace tsl { +namespace hh { + +/** + * Grow the hash table by a factor of GrowthFactor keeping the bucket count to a power of two. It allows + * the table to use a mask operation instead of a modulo operation to map a hash to a bucket. + * + * GrowthFactor must be a power of two >= 2. + */ +template +class power_of_two_growth_policy { +public: + /** + * Called on the hash table creation and on rehash. The number of buckets for the table is passed in parameter. + * This number is a minimum, the policy may update this value with a higher value if needed (but not lower). + * + * If 0 is given, min_bucket_count_in_out must still be 0 after the policy creation and + * bucket_for_hash must always return 0 in this case. + */ + explicit power_of_two_growth_policy(std::size_t& min_bucket_count_in_out) { + if(min_bucket_count_in_out > max_bucket_count()) { + throw std::length_error("The hash table exceeds its maxmimum size."); + } + + if(min_bucket_count_in_out > 0) { + min_bucket_count_in_out = round_up_to_power_of_two(min_bucket_count_in_out); + m_mask = min_bucket_count_in_out - 1; + } + else { + m_mask = 0; + } + } + + /** + * Return the bucket [0, bucket_count()) to which the hash belongs. + * If bucket_count() is 0, it must always return 0. + */ + std::size_t bucket_for_hash(std::size_t hash) const noexcept { + return hash & m_mask; + } + + /** + * Return the bucket count to use when the bucket array grows on rehash. + */ + std::size_t next_bucket_count() const { + if((m_mask + 1) > max_bucket_count() / GrowthFactor) { + throw std::length_error("The hash table exceeds its maxmimum size."); + } + + return (m_mask + 1) * GrowthFactor; + } + + /** + * Return the maximum number of buckets supported by the policy. + */ + std::size_t max_bucket_count() const { + // Largest power of two. + return (std::numeric_limits::max() / 2) + 1; + } + + /** + * Reset the growth policy as if it was created with a bucket count of 0. + * After a clear, the policy must always return 0 when bucket_for_hash is called. + */ + void clear() noexcept { + m_mask = 0; + } + +private: + static std::size_t round_up_to_power_of_two(std::size_t value) { + if(is_power_of_two(value)) { + return value; + } + + if(value == 0) { + return 1; + } + + --value; + for(std::size_t i = 1; i < sizeof(std::size_t) * CHAR_BIT; i *= 2) { + value |= value >> i; + } + + return value + 1; + } + + static constexpr bool is_power_of_two(std::size_t value) { + return value != 0 && (value & (value - 1)) == 0; + } + +private: + static_assert(is_power_of_two(GrowthFactor) && GrowthFactor >= 2, "GrowthFactor must be a power of two >= 2."); + + std::size_t m_mask; +}; + + +/** + * Grow the hash table by GrowthFactor::num / GrowthFactor::den and use a modulo to map a hash + * to a bucket. Slower but it can be useful if you want a slower growth. + */ +template> +class mod_growth_policy { +public: + explicit mod_growth_policy(std::size_t& min_bucket_count_in_out) { + if(min_bucket_count_in_out > max_bucket_count()) { + throw std::length_error("The hash table exceeds its maxmimum size."); + } + + if(min_bucket_count_in_out > 0) { + m_mod = min_bucket_count_in_out; + } + else { + m_mod = 1; + } + } + + std::size_t bucket_for_hash(std::size_t hash) const noexcept { + return hash % m_mod; + } + + std::size_t next_bucket_count() const { + if(m_mod == max_bucket_count()) { + throw std::length_error("The hash table exceeds its maxmimum size."); + } + + const double next_bucket_count = std::ceil(double(m_mod) * REHASH_SIZE_MULTIPLICATION_FACTOR); + if(!std::isnormal(next_bucket_count)) { + throw std::length_error("The hash table exceeds its maxmimum size."); + } + + if(next_bucket_count > double(max_bucket_count())) { + return max_bucket_count(); + } + else { + return std::size_t(next_bucket_count); + } + } + + std::size_t max_bucket_count() const { + return MAX_BUCKET_COUNT; + } + + void clear() noexcept { + m_mod = 1; + } + +private: + static constexpr double REHASH_SIZE_MULTIPLICATION_FACTOR = 1.0 * GrowthFactor::num / GrowthFactor::den; + static const std::size_t MAX_BUCKET_COUNT = + std::size_t(double( + std::numeric_limits::max() / REHASH_SIZE_MULTIPLICATION_FACTOR + )); + + static_assert(REHASH_SIZE_MULTIPLICATION_FACTOR >= 1.1, "Growth factor should be >= 1.1."); + + std::size_t m_mod; +}; + + + +namespace detail { + +static constexpr const std::array PRIMES = {{ + 1ul, 5ul, 17ul, 29ul, 37ul, 53ul, 67ul, 79ul, 97ul, 131ul, 193ul, 257ul, 389ul, 521ul, 769ul, 1031ul, + 1543ul, 2053ul, 3079ul, 6151ul, 12289ul, 24593ul, 49157ul, 98317ul, 196613ul, 393241ul, 786433ul, + 1572869ul, 3145739ul, 6291469ul, 12582917ul, 25165843ul, 50331653ul, 100663319ul, 201326611ul, + 402653189ul, 805306457ul, 1610612741ul, 3221225473ul, 4294967291ul +}}; + +template +static constexpr std::size_t mod(std::size_t hash) { return hash % PRIMES[IPrime]; } + +// MOD_PRIME[iprime](hash) returns hash % PRIMES[iprime]. This table allows for faster modulo as the +// compiler can optimize the modulo code better with a constant known at the compilation. +static constexpr const std::array MOD_PRIME = {{ + &mod<0>, &mod<1>, &mod<2>, &mod<3>, &mod<4>, &mod<5>, &mod<6>, &mod<7>, &mod<8>, &mod<9>, &mod<10>, + &mod<11>, &mod<12>, &mod<13>, &mod<14>, &mod<15>, &mod<16>, &mod<17>, &mod<18>, &mod<19>, &mod<20>, + &mod<21>, &mod<22>, &mod<23>, &mod<24>, &mod<25>, &mod<26>, &mod<27>, &mod<28>, &mod<29>, &mod<30>, + &mod<31>, &mod<32>, &mod<33>, &mod<34>, &mod<35>, &mod<36>, &mod<37> , &mod<38>, &mod<39> +}}; + +} + +/** + * Grow the hash table by using prime numbers as bucket count. Slower than tsl::hh::power_of_two_growth_policy in + * general but will probably distribute the values around better in the buckets with a poor hash function. + * + * To allow the compiler to optimize the modulo operation, a lookup table is used with constant primes numbers. + * + * With a switch the code would look like: + * \code + * switch(iprime) { // iprime is the current prime of the hash table + * case 0: hash % 5ul; + * break; + * case 1: hash % 17ul; + * break; + * case 2: hash % 29ul; + * break; + * ... + * } + * \endcode + * + * Due to the constant variable in the modulo the compiler is able to optimize the operation + * by a series of multiplications, substractions and shifts. + * + * The 'hash % 5' could become something like 'hash - (hash * 0xCCCCCCCD) >> 34) * 5' in a 64 bits environement. + */ +class prime_growth_policy { +public: + explicit prime_growth_policy(std::size_t& min_bucket_count_in_out) { + auto it_prime = std::lower_bound(detail::PRIMES.begin(), + detail::PRIMES.end(), min_bucket_count_in_out); + if(it_prime == detail::PRIMES.end()) { + throw std::length_error("The hash table exceeds its maxmimum size."); + } + + m_iprime = static_cast(std::distance(detail::PRIMES.begin(), it_prime)); + if(min_bucket_count_in_out > 0) { + min_bucket_count_in_out = *it_prime; + } + else { + min_bucket_count_in_out = 0; + } + } + + std::size_t bucket_for_hash(std::size_t hash) const noexcept { + return detail::MOD_PRIME[m_iprime](hash); + } + + std::size_t next_bucket_count() const { + if(m_iprime + 1 >= detail::PRIMES.size()) { + throw std::length_error("The hash table exceeds its maxmimum size."); + } + + return detail::PRIMES[m_iprime + 1]; + } + + std::size_t max_bucket_count() const { + return detail::PRIMES.back(); + } + + void clear() noexcept { + m_iprime = 0; + } + +private: + unsigned int m_iprime; + + static_assert(std::numeric_limits::max() >= detail::PRIMES.size(), + "The type of m_iprime is not big enough."); +}; + +} +} + +#endif diff --git a/include/tsl/hopscotch_hash.h b/include/tsl/hopscotch_hash.h new file mode 100644 index 000000000..1031c0c9a --- /dev/null +++ b/include/tsl/hopscotch_hash.h @@ -0,0 +1,1801 @@ +/** + * MIT License + * + * Copyright (c) 2017 Tessil + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_HOPSCOTCH_HASH_H +#define TSL_HOPSCOTCH_HASH_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "hopscotch_growth_policy.h" + + + +#if (defined(__GNUC__) && (__GNUC__ == 4) && (__GNUC_MINOR__ < 9)) +#define TSL_NO_RANGE_ERASE_WITH_CONST_ITERATOR +#endif + + + +/* + * Only activate tsl_assert if TSL_DEBUG is defined. + * This way we avoid the performance hit when NDEBUG is not defined with assert as tsl_assert is used a lot + * (people usually compile with "-O3" and not "-O3 -DNDEBUG"). + */ +#ifndef tsl_assert + #ifdef TSL_DEBUG + #define tsl_assert(expr) assert(expr) + #else + #define tsl_assert(expr) (static_cast(0)) + #endif +#endif + +namespace tsl { + +namespace detail_hopscotch_hash { + + +template +struct make_void { + using type = void; +}; + + +template +struct has_is_transparent : std::false_type { +}; + +template +struct has_is_transparent::type> : std::true_type { +}; + + +template +struct has_key_compare : std::false_type { +}; + +template +struct has_key_compare::type> : std::true_type { +}; + + +template +struct is_power_of_two_policy: std::false_type { +}; + +template +struct is_power_of_two_policy>: std::true_type { +}; + + + + + +/* + * smallest_type_for_min_bits::type returns the smallest type that can fit MinBits. + */ +static const std::size_t SMALLEST_TYPE_MAX_BITS_SUPPORTED = 64; +template +class smallest_type_for_min_bits { +}; + +template +class smallest_type_for_min_bits 0) && (MinBits <= 8)>::type> { +public: + using type = std::uint_least8_t; +}; + +template +class smallest_type_for_min_bits 8) && (MinBits <= 16)>::type> { +public: + using type = std::uint_least16_t; +}; + +template +class smallest_type_for_min_bits 16) && (MinBits <= 32)>::type> { +public: + using type = std::uint_least32_t; +}; + +template +class smallest_type_for_min_bits 32) && (MinBits <= 64)>::type> { +public: + using type = std::uint_least64_t; +}; + + + +/* + * Each bucket may store up to three elements: + * - An aligned storage to store a value_type object with placement-new. + * - An (optional) hash of the value in the bucket. + * - An unsigned integer of type neighborhood_bitmap used to tell us which buckets in the neighborhood of the + * current bucket contain a value with a hash belonging to the current bucket. + * + * For a bucket 'bct', a bit 'i' (counting from 0 and from the least significant bit to the most significant) + * set to 1 means that the bucket 'bct + i' contains a value with a hash belonging to bucket 'bct'. + * The bits used for that, start from the third least significant bit. + * The two least significant bits are reserved: + * - The least significant bit is set to 1 if there is a value in the bucket storage. + * - The second least significant bit is set to 1 if there is an overflow. More than NeighborhoodSize values + * give the same hash, all overflow values are stored in the m_overflow_elements list of the map. + * + * Details regarding hopscotch hashing an its implementation can be found here: + * https://tessil.github.io/2016/08/29/hopscotch-hashing.html + */ +static const std::size_t NB_RESERVED_BITS_IN_NEIGHBORHOOD = 2; + + +using truncated_hash_type = std::uint_least32_t; + +/** + * Helper class that store a truncated hash if StoreHash is true and nothing otherwise. + */ +template +class hopscotch_bucket_hash { +public: + bool bucket_hash_equal(std::size_t /*hash*/) const noexcept { + return true; + } + + truncated_hash_type truncated_bucket_hash() const noexcept { + return 0; + } + +protected: + void copy_hash(const hopscotch_bucket_hash& ) noexcept { + } + + void set_hash(truncated_hash_type /*hash*/) noexcept { + } +}; + +template<> +class hopscotch_bucket_hash { +public: + bool bucket_hash_equal(std::size_t hash) const noexcept { + return m_hash == truncated_hash_type(hash); + } + + truncated_hash_type truncated_bucket_hash() const noexcept { + return m_hash; + } + +protected: + void copy_hash(const hopscotch_bucket_hash& bucket) noexcept { + m_hash = bucket.m_hash; + } + + void set_hash(truncated_hash_type hash) noexcept { + m_hash = hash; + } + +private: + truncated_hash_type m_hash; +}; + + +template +class hopscotch_bucket: public hopscotch_bucket_hash { +private: + static const std::size_t MIN_NEIGHBORHOOD_SIZE = 4; + static const std::size_t MAX_NEIGHBORHOOD_SIZE = SMALLEST_TYPE_MAX_BITS_SUPPORTED - NB_RESERVED_BITS_IN_NEIGHBORHOOD; + + + static_assert(NeighborhoodSize >= 4, "NeighborhoodSize should be >= 4."); + // We can't put a variable in the message, ensure coherence + static_assert(MIN_NEIGHBORHOOD_SIZE == 4, ""); + + static_assert(NeighborhoodSize <= 62, "NeighborhoodSize should be <= 62."); + // We can't put a variable in the message, ensure coherence + static_assert(MAX_NEIGHBORHOOD_SIZE == 62, ""); + + + static_assert(!StoreHash || NeighborhoodSize <= 30, + "NeighborhoodSize should be <= 30 if StoreHash is true."); + // We can't put a variable in the message, ensure coherence + static_assert(MAX_NEIGHBORHOOD_SIZE - 32 == 30, ""); + + using bucket_hash = hopscotch_bucket_hash; + +public: + using value_type = ValueType; + using neighborhood_bitmap = + typename smallest_type_for_min_bits::type; + + + hopscotch_bucket() noexcept: bucket_hash(), m_neighborhood_infos(0) { + tsl_assert(empty()); + } + + + hopscotch_bucket(const hopscotch_bucket& bucket) + noexcept(std::is_nothrow_copy_constructible::value): bucket_hash(bucket), + m_neighborhood_infos(0) + { + if(!bucket.empty()) { + ::new (static_cast(std::addressof(m_value))) value_type(bucket.value()); + } + + m_neighborhood_infos = bucket.m_neighborhood_infos; + } + + hopscotch_bucket(hopscotch_bucket&& bucket) + noexcept(std::is_nothrow_move_constructible::value) : bucket_hash(std::move(bucket)), + m_neighborhood_infos(0) + { + if(!bucket.empty()) { + ::new (static_cast(std::addressof(m_value))) value_type(std::move(bucket.value())); + } + + m_neighborhood_infos = bucket.m_neighborhood_infos; + } + + hopscotch_bucket& operator=(const hopscotch_bucket& bucket) + noexcept(std::is_nothrow_copy_constructible::value) + { + if(this != &bucket) { + remove_value(); + + bucket_hash::operator=(bucket); + if(!bucket.empty()) { + ::new (static_cast(std::addressof(m_value))) value_type(bucket.value()); + } + + m_neighborhood_infos = bucket.m_neighborhood_infos; + } + + return *this; + } + + hopscotch_bucket& operator=(hopscotch_bucket&& ) = delete; + + ~hopscotch_bucket() noexcept { + if(!empty()) { + destroy_value(); + } + } + + neighborhood_bitmap neighborhood_infos() const noexcept { + return neighborhood_bitmap(m_neighborhood_infos >> NB_RESERVED_BITS_IN_NEIGHBORHOOD); + } + + void set_overflow(bool has_overflow) noexcept { + if(has_overflow) { + m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos | 2); + } + else { + m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos & ~2); + } + } + + bool has_overflow() const noexcept { + return (m_neighborhood_infos & 2) != 0; + } + + bool empty() const noexcept { + return (m_neighborhood_infos & 1) == 0; + } + + void toggle_neighbor_presence(std::size_t ineighbor) noexcept { + tsl_assert(ineighbor <= NeighborhoodSize); + m_neighborhood_infos = neighborhood_bitmap( + m_neighborhood_infos ^ (1ull << (ineighbor + NB_RESERVED_BITS_IN_NEIGHBORHOOD))); + } + + bool check_neighbor_presence(std::size_t ineighbor) const noexcept { + tsl_assert(ineighbor <= NeighborhoodSize); + if(((m_neighborhood_infos >> (ineighbor + NB_RESERVED_BITS_IN_NEIGHBORHOOD)) & 1) == 1) { + return true; + } + + return false; + } + + value_type& value() noexcept { + tsl_assert(!empty()); + return *reinterpret_cast(std::addressof(m_value)); + } + + const value_type& value() const noexcept { + tsl_assert(!empty()); + return *reinterpret_cast(std::addressof(m_value)); + } + + template + void set_value_of_empty_bucket(truncated_hash_type hash, Args&&... value_type_args) { + tsl_assert(empty()); + + ::new (static_cast(std::addressof(m_value))) value_type(std::forward(value_type_args)...); + set_empty(false); + this->set_hash(hash); + } + + void swap_value_into_empty_bucket(hopscotch_bucket& empty_bucket) { + tsl_assert(empty_bucket.empty()); + if(!empty()) { + ::new (static_cast(std::addressof(empty_bucket.m_value))) value_type(std::move(value())); + empty_bucket.copy_hash(*this); + empty_bucket.set_empty(false); + + destroy_value(); + set_empty(true); + } + } + + void remove_value() noexcept { + if(!empty()) { + destroy_value(); + set_empty(true); + } + } + + void clear() noexcept { + if(!empty()) { + destroy_value(); + } + + m_neighborhood_infos = 0; + tsl_assert(empty()); + } + + static std::size_t max_size() noexcept { + if(StoreHash) { + return std::numeric_limits::max(); + } + else { + return std::numeric_limits::max(); + } + } + + static truncated_hash_type truncate_hash(std::size_t hash) noexcept { + return truncated_hash_type(hash); + } + +private: + void set_empty(bool is_empty) noexcept { + if(is_empty) { + m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos & ~1); + } + else { + m_neighborhood_infos = neighborhood_bitmap(m_neighborhood_infos | 1); + } + } + + void destroy_value() noexcept { + tsl_assert(!empty()); + value().~value_type(); + } + +private: + using storage = typename std::aligned_storage::type; + + neighborhood_bitmap m_neighborhood_infos; + storage m_value; +}; + + +/** + * Internal common class used by (b)hopscotch_map and (b)hopscotch_set. + * + * ValueType is what will be stored by hopscotch_hash (usually std::pair for a map and Key for a set). + * + * KeySelect should be a FunctionObject which takes a ValueType in parameter and returns a reference to the key. + * + * ValueSelect should be a FunctionObject which takes a ValueType in parameter and returns a reference to the value. + * ValueSelect should be void if there is no value (in a set for example). + * + * OverflowContainer will be used as containers for overflown elements. Usually it should be a list + * or a set/map. + */ +template +class hopscotch_hash: private Hash, private KeyEqual, private GrowthPolicy { +private: + template + using has_mapped_type = typename std::integral_constant::value>; + + static_assert(noexcept(std::declval().bucket_for_hash(std::size_t(0))), "GrowthPolicy::bucket_for_hash must be noexcept."); + static_assert(noexcept(std::declval().clear()), "GrowthPolicy::clear must be noexcept."); + +public: + template + class hopscotch_iterator; + + using key_type = typename KeySelect::key_type; + using value_type = ValueType; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using hasher = Hash; + using key_equal = KeyEqual; + using allocator_type = Allocator; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator = hopscotch_iterator; + using const_iterator = hopscotch_iterator; + +private: + using hopscotch_bucket = tsl::detail_hopscotch_hash::hopscotch_bucket; + using neighborhood_bitmap = typename hopscotch_bucket::neighborhood_bitmap; + + using buckets_allocator = typename std::allocator_traits::template rebind_alloc; + using buckets_container_type = std::vector; + + using overflow_container_type = OverflowContainer; + + static_assert(std::is_same::value, + "OverflowContainer should have ValueType as type."); + + static_assert(std::is_same::value, + "Invalid allocator, not the same type as the value_type."); + + + using iterator_buckets = typename buckets_container_type::iterator; + using const_iterator_buckets = typename buckets_container_type::const_iterator; + + using iterator_overflow = typename overflow_container_type::iterator; + using const_iterator_overflow = typename overflow_container_type::const_iterator; + +public: + /** + * The `operator*()` and `operator->()` methods return a const reference and const pointer respectively to the + * stored value type. + * + * In case of a map, to get a modifiable reference to the value associated to a key (the `.second` in the + * stored pair), you have to call `value()`. + */ + template + class hopscotch_iterator { + friend class hopscotch_hash; + private: + using iterator_bucket = typename std::conditional::type; + using iterator_overflow = typename std::conditional::type; + + + hopscotch_iterator(iterator_bucket buckets_iterator, iterator_bucket buckets_end_iterator, + iterator_overflow overflow_iterator) noexcept : + m_buckets_iterator(buckets_iterator), m_buckets_end_iterator(buckets_end_iterator), + m_overflow_iterator(overflow_iterator) + { + } + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = const typename hopscotch_hash::value_type; + using difference_type = std::ptrdiff_t; + using reference = value_type&; + using pointer = value_type*; + + + hopscotch_iterator() noexcept { + } + + hopscotch_iterator(const hopscotch_iterator& other) noexcept : + m_buckets_iterator(other.m_buckets_iterator), m_buckets_end_iterator(other.m_buckets_end_iterator), + m_overflow_iterator(other.m_overflow_iterator) + { + } + + const typename hopscotch_hash::key_type& key() const { + if(m_buckets_iterator != m_buckets_end_iterator) { + return KeySelect()(m_buckets_iterator->value()); + } + + return KeySelect()(*m_overflow_iterator); + } + + template::value>::type* = nullptr> + typename std::conditional< + IsConst, + const typename U::value_type&, + typename U::value_type&>::type value() const + { + if(m_buckets_iterator != m_buckets_end_iterator) { + return U()(m_buckets_iterator->value()); + } + + return U()(*m_overflow_iterator); + } + + reference operator*() const { + if(m_buckets_iterator != m_buckets_end_iterator) { + return m_buckets_iterator->value(); + } + + return *m_overflow_iterator; + } + + pointer operator->() const { + if(m_buckets_iterator != m_buckets_end_iterator) { + return std::addressof(m_buckets_iterator->value()); + } + + return std::addressof(*m_overflow_iterator); + } + + hopscotch_iterator& operator++() { + if(m_buckets_iterator == m_buckets_end_iterator) { + ++m_overflow_iterator; + return *this; + } + + do { + ++m_buckets_iterator; + } while(m_buckets_iterator != m_buckets_end_iterator && m_buckets_iterator->empty()); + + return *this; + } + + hopscotch_iterator operator++(int) { + hopscotch_iterator tmp(*this); + ++*this; + + return tmp; + } + + friend bool operator==(const hopscotch_iterator& lhs, const hopscotch_iterator& rhs) { + return lhs.m_buckets_iterator == rhs.m_buckets_iterator && + lhs.m_overflow_iterator == rhs.m_overflow_iterator; + } + + friend bool operator!=(const hopscotch_iterator& lhs, const hopscotch_iterator& rhs) { + return !(lhs == rhs); + } + + private: + iterator_bucket m_buckets_iterator; + iterator_bucket m_buckets_end_iterator; + iterator_overflow m_overflow_iterator; + }; + +public: + template::value>::type* = nullptr> + hopscotch_hash(size_type bucket_count, + const Hash& hash, + const KeyEqual& equal, + const Allocator& alloc, + float max_load_factor) : Hash(hash), + KeyEqual(equal), + GrowthPolicy(bucket_count), + m_buckets(alloc), + m_overflow_elements(alloc), + m_first_or_empty_bucket(static_empty_bucket_ptr()), + m_nb_elements(0) + { + if(bucket_count > max_bucket_count()) { + throw std::length_error("The map exceeds its maxmimum size."); + } + + if(bucket_count > 0) { + static_assert(NeighborhoodSize - 1 > 0, ""); + + // Can't directly construct with the appropriate size in the initializer + // as m_buckets(bucket_count, alloc) is not supported by GCC 4.8 + m_buckets.resize(bucket_count + NeighborhoodSize - 1); + m_first_or_empty_bucket = m_buckets.data(); + } + + + this->max_load_factor(max_load_factor); + + + // Check in the constructor instead of outside of a function to avoi compilation issues + // when value_type is not complete. + static_assert(std::is_nothrow_move_constructible::value || + std::is_copy_constructible::value, + "value_type must be either copy constructible or nothrow move constructible."); + } + + template::value>::type* = nullptr> + hopscotch_hash(size_type bucket_count, + const Hash& hash, + const KeyEqual& equal, + const Allocator& alloc, + float max_load_factor, + const typename OC::key_compare& comp) : Hash(hash), + KeyEqual(equal), + GrowthPolicy(bucket_count), + m_buckets(alloc), + m_overflow_elements(comp, alloc), + m_first_or_empty_bucket(static_empty_bucket_ptr()), + m_nb_elements(0) + { + + if(bucket_count > max_bucket_count()) { + throw std::length_error("The map exceeds its maxmimum size."); + } + + if(bucket_count > 0) { + static_assert(NeighborhoodSize - 1 > 0, ""); + + // Can't directly construct with the appropriate size in the initializer + // as m_buckets(bucket_count, alloc) is not supported by GCC 4.8 + m_buckets.resize(bucket_count + NeighborhoodSize - 1); + m_first_or_empty_bucket = m_buckets.data(); + } + + + this->max_load_factor(max_load_factor); + + + // Check in the constructor instead of outside of a function to avoi compilation issues + // when value_type is not complete. + static_assert(std::is_nothrow_move_constructible::value || + std::is_copy_constructible::value, + "value_type must be either copy constructible or nothrow move constructible."); + } + + hopscotch_hash(const hopscotch_hash& other): + Hash(other), + KeyEqual(other), + GrowthPolicy(other), + m_buckets(other.m_buckets), + m_overflow_elements(other.m_overflow_elements), + m_first_or_empty_bucket(m_buckets.empty()?static_empty_bucket_ptr(): + m_buckets.data()), + m_nb_elements(other.m_nb_elements), + m_max_load_factor(other.m_max_load_factor), + m_max_load_threshold_rehash(other.m_max_load_threshold_rehash), + m_min_load_threshold_rehash(other.m_min_load_threshold_rehash) + { + } + + hopscotch_hash(hopscotch_hash&& other) + noexcept( + std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value + ): + Hash(std::move(static_cast(other))), + KeyEqual(std::move(static_cast(other))), + GrowthPolicy(std::move(static_cast(other))), + m_buckets(std::move(other.m_buckets)), + m_overflow_elements(std::move(other.m_overflow_elements)), + m_first_or_empty_bucket(m_buckets.empty()?static_empty_bucket_ptr(): + m_buckets.data()), + m_nb_elements(other.m_nb_elements), + m_max_load_factor(other.m_max_load_factor), + m_max_load_threshold_rehash(other.m_max_load_threshold_rehash), + m_min_load_threshold_rehash(other.m_min_load_threshold_rehash) + { + other.GrowthPolicy::clear(); + other.m_buckets.clear(); + other.m_overflow_elements.clear(); + other.m_first_or_empty_bucket = static_empty_bucket_ptr(); + other.m_nb_elements = 0; + other.m_max_load_threshold_rehash = 0; + other.m_min_load_threshold_rehash = 0; + } + + hopscotch_hash& operator=(const hopscotch_hash& other) { + if(&other != this) { + Hash::operator=(other); + KeyEqual::operator=(other); + GrowthPolicy::operator=(other); + + m_buckets = other.m_buckets; + m_overflow_elements = other.m_overflow_elements; + m_first_or_empty_bucket = m_buckets.empty()?static_empty_bucket_ptr(): + m_buckets.data(); + m_nb_elements = other.m_nb_elements; + m_max_load_factor = other.m_max_load_factor; + m_max_load_threshold_rehash = other.m_max_load_threshold_rehash; + m_min_load_threshold_rehash = other.m_min_load_threshold_rehash; + } + + return *this; + } + + hopscotch_hash& operator=(hopscotch_hash&& other) { + other.swap(*this); + other.clear(); + + return *this; + } + + allocator_type get_allocator() const { + return m_buckets.get_allocator(); + } + + + /* + * Iterators + */ + iterator begin() noexcept { + auto begin = m_buckets.begin(); + while(begin != m_buckets.end() && begin->empty()) { + ++begin; + } + + return iterator(begin, m_buckets.end(), m_overflow_elements.begin()); + } + + const_iterator begin() const noexcept { + return cbegin(); + } + + const_iterator cbegin() const noexcept { + auto begin = m_buckets.cbegin(); + while(begin != m_buckets.cend() && begin->empty()) { + ++begin; + } + + return const_iterator(begin, m_buckets.cend(), m_overflow_elements.cbegin()); + } + + iterator end() noexcept { + return iterator(m_buckets.end(), m_buckets.end(), m_overflow_elements.end()); + } + + const_iterator end() const noexcept { + return cend(); + } + + const_iterator cend() const noexcept { + return const_iterator(m_buckets.cend(), m_buckets.cend(), m_overflow_elements.cend()); + } + + + /* + * Capacity + */ + bool empty() const noexcept { + return m_nb_elements == 0; + } + + size_type size() const noexcept { + return m_nb_elements; + } + + size_type max_size() const noexcept { + return hopscotch_bucket::max_size(); + } + + /* + * Modifiers + */ + void clear() noexcept { + for(auto& bucket: m_buckets) { + bucket.clear(); + } + + m_overflow_elements.clear(); + m_nb_elements = 0; + } + + + std::pair insert(const value_type& value) { + return insert_impl(value); + } + + template::value>::type* = nullptr> + std::pair insert(P&& value) { + return insert_impl(value_type(std::forward

(value))); + } + + std::pair insert(value_type&& value) { + return insert_impl(std::move(value)); + } + + + iterator insert(const_iterator hint, const value_type& value) { + if(hint != cend() && compare_keys(KeySelect()(*hint), KeySelect()(value))) { + return mutable_iterator(hint); + } + + return insert(value).first; + } + + template::value>::type* = nullptr> + iterator insert(const_iterator hint, P&& value) { + return emplace_hint(hint, std::forward

(value)); + } + + iterator insert(const_iterator hint, value_type&& value) { + if(hint != cend() && compare_keys(KeySelect()(*hint), KeySelect()(value))) { + return mutable_iterator(hint); + } + + return insert(std::move(value)).first; + } + + + template + void insert(InputIt first, InputIt last) { + if(std::is_base_of::iterator_category>::value) + { + const auto nb_elements_insert = std::distance(first, last); + const std::size_t nb_elements_in_buckets = m_nb_elements - m_overflow_elements.size(); + const std::size_t nb_free_buckets = m_max_load_threshold_rehash - nb_elements_in_buckets; + tsl_assert(m_nb_elements >= m_overflow_elements.size()); + tsl_assert(m_max_load_threshold_rehash >= nb_elements_in_buckets); + + if(nb_elements_insert > 0 && nb_free_buckets < std::size_t(nb_elements_insert)) { + reserve(nb_elements_in_buckets + std::size_t(nb_elements_insert)); + } + } + + for(; first != last; ++first) { + insert(*first); + } + } + + + template + std::pair insert_or_assign(const key_type& k, M&& obj) { + return insert_or_assign_impl(k, std::forward(obj)); + } + + template + std::pair insert_or_assign(key_type&& k, M&& obj) { + return insert_or_assign_impl(std::move(k), std::forward(obj)); + } + + + template + iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj) { + if(hint != cend() && compare_keys(KeySelect()(*hint), k)) { + auto it = mutable_iterator(hint); + it.value() = std::forward(obj); + + return it; + } + + return insert_or_assign(k, std::forward(obj)).first; + } + + template + iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj) { + if(hint != cend() && compare_keys(KeySelect()(*hint), k)) { + auto it = mutable_iterator(hint); + it.value() = std::forward(obj); + + return it; + } + + return insert_or_assign(std::move(k), std::forward(obj)).first; + } + + + template + std::pair emplace(Args&&... args) { + return insert(value_type(std::forward(args)...)); + } + + template + iterator emplace_hint(const_iterator hint, Args&&... args) { + return insert(hint, value_type(std::forward(args)...)); + } + + template + std::pair try_emplace(const key_type& k, Args&&... args) { + return try_emplace_impl(k, std::forward(args)...); + } + + template + std::pair try_emplace(key_type&& k, Args&&... args) { + return try_emplace_impl(std::move(k), std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args) { + if(hint != cend() && compare_keys(KeySelect()(*hint), k)) { + return mutable_iterator(hint); + } + + return try_emplace(k, std::forward(args)...).first; + } + + template + iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args) { + if(hint != cend() && compare_keys(KeySelect()(*hint), k)) { + return mutable_iterator(hint); + } + + return try_emplace(std::move(k), std::forward(args)...).first; + } + + + /** + * Here to avoid `template size_type erase(const K& key)` being used when + * we use an iterator instead of a const_iterator. + */ + iterator erase(iterator pos) { + return erase(const_iterator(pos)); + } + + iterator erase(const_iterator pos) { + const std::size_t ibucket_for_hash = bucket_for_hash(hash_key(pos.key())); + + if(pos.m_buckets_iterator != pos.m_buckets_end_iterator) { + auto it_bucket = m_buckets.begin() + std::distance(m_buckets.cbegin(), pos.m_buckets_iterator); + erase_from_bucket(*it_bucket, ibucket_for_hash); + + return ++iterator(it_bucket, m_buckets.end(), m_overflow_elements.begin()); + } + else { + auto it_next_overflow = erase_from_overflow(pos.m_overflow_iterator, ibucket_for_hash); + return iterator(m_buckets.end(), m_buckets.end(), it_next_overflow); + } + } + + iterator erase(const_iterator first, const_iterator last) { + if(first == last) { + return mutable_iterator(first); + } + + auto to_delete = erase(first); + while(to_delete != last) { + to_delete = erase(to_delete); + } + + return to_delete; + } + + template + size_type erase(const K& key) { + return erase(key, hash_key(key)); + } + + template + size_type erase(const K& key, std::size_t hash) { + const std::size_t ibucket_for_hash = bucket_for_hash(hash); + + hopscotch_bucket* bucket_found = find_in_buckets(key, hash, m_first_or_empty_bucket + ibucket_for_hash); + if(bucket_found != nullptr) { + erase_from_bucket(*bucket_found, ibucket_for_hash); + + return 1; + } + + if((m_first_or_empty_bucket + ibucket_for_hash)->has_overflow()) { + auto it_overflow = find_in_overflow(key); + if(it_overflow != m_overflow_elements.end()) { + erase_from_overflow(it_overflow, ibucket_for_hash); + + return 1; + } + } + + return 0; + } + + void swap(hopscotch_hash& other) { + using std::swap; + + swap(static_cast(*this), static_cast(other)); + swap(static_cast(*this), static_cast(other)); + swap(static_cast(*this), static_cast(other)); + swap(m_buckets, other.m_buckets); + swap(m_overflow_elements, other.m_overflow_elements); + swap(m_first_or_empty_bucket, other.m_first_or_empty_bucket); + swap(m_nb_elements, other.m_nb_elements); + swap(m_max_load_factor, other.m_max_load_factor); + swap(m_max_load_threshold_rehash, other.m_max_load_threshold_rehash); + swap(m_min_load_threshold_rehash, other.m_min_load_threshold_rehash); + } + + + /* + * Lookup + */ + template::value>::type* = nullptr> + typename U::value_type& at(const K& key) { + return at(key, hash_key(key)); + } + + template::value>::type* = nullptr> + typename U::value_type& at(const K& key, std::size_t hash) { + return const_cast(static_cast(this)->at(key, hash)); + } + + + template::value>::type* = nullptr> + const typename U::value_type& at(const K& key) const { + return at(key, hash_key(key)); + } + + template::value>::type* = nullptr> + const typename U::value_type& at(const K& key, std::size_t hash) const { + using T = typename U::value_type; + + const T* value = find_value_impl(key, hash, m_first_or_empty_bucket + bucket_for_hash(hash)); + if(value == nullptr) { + throw std::out_of_range("Couldn't find key."); + } + else { + return *value; + } + } + + + template::value>::type* = nullptr> + typename U::value_type& operator[](K&& key) { + using T = typename U::value_type; + + const std::size_t hash = hash_key(key); + const std::size_t ibucket_for_hash = bucket_for_hash(hash); + + T* value = find_value_impl(key, hash, m_first_or_empty_bucket + ibucket_for_hash); + if(value != nullptr) { + return *value; + } + else { + return insert_impl(ibucket_for_hash, hash, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple()).first.value(); + } + } + + + template + size_type count(const K& key) const { + return count(key, hash_key(key)); + } + + template + size_type count(const K& key, std::size_t hash) const { + return count_impl(key, hash, m_first_or_empty_bucket + bucket_for_hash(hash)); + } + + + template + iterator find(const K& key) { + return find(key, hash_key(key)); + } + + template + iterator find(const K& key, std::size_t hash) { + return find_impl(key, hash, m_first_or_empty_bucket + bucket_for_hash(hash)); + } + + + template + const_iterator find(const K& key) const { + return find(key, hash_key(key)); + } + + template + const_iterator find(const K& key, std::size_t hash) const { + return find_impl(key, hash, m_first_or_empty_bucket + bucket_for_hash(hash)); + } + + + template + std::pair equal_range(const K& key) { + return equal_range(key, hash_key(key)); + } + + template + std::pair equal_range(const K& key, std::size_t hash) { + iterator it = find(key, hash); + return std::make_pair(it, (it == end())?it:std::next(it)); + } + + + template + std::pair equal_range(const K& key) const { + return equal_range(key, hash_key(key)); + } + + template + std::pair equal_range(const K& key, std::size_t hash) const { + const_iterator it = find(key, hash); + return std::make_pair(it, (it == cend())?it:std::next(it)); + } + + /* + * Bucket interface + */ + size_type bucket_count() const { + /* + * So that the last bucket can have NeighborhoodSize neighbors, the size of the bucket array is a little + * bigger than the real number of buckets when not empty. + * We could use some of the buckets at the beginning, but it is faster this way as we avoid extra checks. + */ + if(m_buckets.empty()) { + return 0; + } + + return m_buckets.size() - NeighborhoodSize + 1; + } + + size_type max_bucket_count() const { + const std::size_t max_bucket_count = std::min(GrowthPolicy::max_bucket_count(), m_buckets.max_size()); + return max_bucket_count - NeighborhoodSize + 1; + } + + + /* + * Hash policy + */ + float load_factor() const { + if(bucket_count() == 0) { + return 0; + } + + return float(m_nb_elements)/float(bucket_count()); + } + + float max_load_factor() const { + return m_max_load_factor; + } + + void max_load_factor(float ml) { + m_max_load_factor = std::max(0.1f, std::min(ml, 0.95f)); + m_max_load_threshold_rehash = size_type(float(bucket_count())*m_max_load_factor); + m_min_load_threshold_rehash = size_type(float(bucket_count())*MIN_LOAD_FACTOR_FOR_REHASH); + } + + void rehash(size_type count_) { + count_ = std::max(count_, size_type(std::ceil(float(size())/max_load_factor()))); + rehash_impl(count_); + } + + void reserve(size_type count_) { + rehash(size_type(std::ceil(float(count_)/max_load_factor()))); + } + + + /* + * Observers + */ + hasher hash_function() const { + return static_cast(*this); + } + + key_equal key_eq() const { + return static_cast(*this); + } + + /* + * Other + */ + iterator mutable_iterator(const_iterator pos) { + if(pos.m_buckets_iterator != pos.m_buckets_end_iterator) { + // Get a non-const iterator + auto it = m_buckets.begin() + std::distance(m_buckets.cbegin(), pos.m_buckets_iterator); + return iterator(it, m_buckets.end(), m_overflow_elements.begin()); + } + else { + // Get a non-const iterator + auto it = mutable_overflow_iterator(pos.m_overflow_iterator); + return iterator(m_buckets.end(), m_buckets.end(), it); + } + } + + size_type overflow_size() const noexcept { + return m_overflow_elements.size(); + } + + template::value>::type* = nullptr> + typename U::key_compare key_comp() const { + return m_overflow_elements.key_comp(); + } + + +private: + template + std::size_t hash_key(const K& key) const { + return Hash::operator()(key); + } + + template + bool compare_keys(const K1& key1, const K2& key2) const { + return KeyEqual::operator()(key1, key2); + } + + std::size_t bucket_for_hash(std::size_t hash) const { + const std::size_t bucket = GrowthPolicy::bucket_for_hash(hash); + tsl_assert(bucket < m_buckets.size() || (bucket == 0 && m_buckets.empty())); + + return bucket; + } + + template::value>::type* = nullptr> + void rehash_impl(size_type count_) { + hopscotch_hash new_map = new_hopscotch_hash(count_); + + if(!m_overflow_elements.empty()) { + new_map.m_overflow_elements.swap(m_overflow_elements); + new_map.m_nb_elements += new_map.m_overflow_elements.size(); + + for(const value_type& value : new_map.m_overflow_elements) { + const std::size_t ibucket_for_hash = new_map.bucket_for_hash(new_map.hash_key(KeySelect()(value))); + new_map.m_buckets[ibucket_for_hash].set_overflow(true); + } + } + + try { + const bool use_stored_hash = USE_STORED_HASH_ON_REHASH(new_map.bucket_count()); + for(auto it_bucket = m_buckets.begin(); it_bucket != m_buckets.end(); ++it_bucket) { + if(it_bucket->empty()) { + continue; + } + + const std::size_t hash = use_stored_hash? + it_bucket->truncated_bucket_hash(): + new_map.hash_key(KeySelect()(it_bucket->value())); + const std::size_t ibucket_for_hash = new_map.bucket_for_hash(hash); + + new_map.insert_impl(ibucket_for_hash, hash, std::move(it_bucket->value())); + + + erase_from_bucket(*it_bucket, bucket_for_hash(hash)); + } + } + /* + * The call to insert_impl may throw an exception if an element is added to the overflow + * list. Rollback the elements in this case. + */ + catch(...) { + m_overflow_elements.swap(new_map.m_overflow_elements); + + const bool use_stored_hash = USE_STORED_HASH_ON_REHASH(new_map.bucket_count()); + for(auto it_bucket = new_map.m_buckets.begin(); it_bucket != new_map.m_buckets.end(); ++it_bucket) { + if(it_bucket->empty()) { + continue; + } + + const std::size_t hash = use_stored_hash? + it_bucket->truncated_bucket_hash(): + hash_key(KeySelect()(it_bucket->value())); + const std::size_t ibucket_for_hash = bucket_for_hash(hash); + + // The elements we insert were not in the overflow list before the switch. + // They will not be go in the overflow list if we rollback the switch. + insert_impl(ibucket_for_hash, hash, std::move(it_bucket->value())); + } + + throw; + } + + new_map.swap(*this); + } + + template::value && + !std::is_nothrow_move_constructible::value>::type* = nullptr> + void rehash_impl(size_type count_) { + hopscotch_hash new_map = new_hopscotch_hash(count_); + + const bool use_stored_hash = USE_STORED_HASH_ON_REHASH(new_map.bucket_count()); + for(const hopscotch_bucket& bucket: m_buckets) { + if(bucket.empty()) { + continue; + } + + const std::size_t hash = use_stored_hash? + bucket.truncated_bucket_hash(): + new_map.hash_key(KeySelect()(bucket.value())); + const std::size_t ibucket_for_hash = new_map.bucket_for_hash(hash); + + new_map.insert_impl(ibucket_for_hash, hash, bucket.value()); + } + + for(const value_type& value: m_overflow_elements) { + const std::size_t hash = new_map.hash_key(KeySelect()(value)); + const std::size_t ibucket_for_hash = new_map.bucket_for_hash(hash); + + new_map.insert_impl(ibucket_for_hash, hash, value); + } + + new_map.swap(*this); + } + +#ifdef TSL_NO_RANGE_ERASE_WITH_CONST_ITERATOR + iterator_overflow mutable_overflow_iterator(const_iterator_overflow it) { + return std::next(m_overflow_elements.begin(), std::distance(m_overflow_elements.cbegin(), it)); + } +#else + iterator_overflow mutable_overflow_iterator(const_iterator_overflow it) { + return m_overflow_elements.erase(it, it); + } +#endif + + // iterator is in overflow list + iterator_overflow erase_from_overflow(const_iterator_overflow pos, std::size_t ibucket_for_hash) { +#ifdef TSL_NO_RANGE_ERASE_WITH_CONST_ITERATOR + auto it_next = m_overflow_elements.erase(mutable_overflow_iterator(pos)); +#else + auto it_next = m_overflow_elements.erase(pos); +#endif + m_nb_elements--; + + + // Check if we can remove the overflow flag + tsl_assert(m_buckets[ibucket_for_hash].has_overflow()); + for(const value_type& value: m_overflow_elements) { + const std::size_t bucket_for_value = bucket_for_hash(hash_key(KeySelect()(value))); + if(bucket_for_value == ibucket_for_hash) { + return it_next; + } + } + + m_buckets[ibucket_for_hash].set_overflow(false); + return it_next; + } + + + /** + * bucket_for_value is the bucket in which the value is. + * ibucket_for_hash is the bucket where the value belongs. + */ + void erase_from_bucket(hopscotch_bucket& bucket_for_value, std::size_t ibucket_for_hash) noexcept { + const std::size_t ibucket_for_value = std::distance(m_buckets.data(), &bucket_for_value); + tsl_assert(ibucket_for_value >= ibucket_for_hash); + + bucket_for_value.remove_value(); + m_buckets[ibucket_for_hash].toggle_neighbor_presence(ibucket_for_value - ibucket_for_hash); + m_nb_elements--; + } + + + + template + std::pair insert_or_assign_impl(K&& key, M&& obj) { + auto it = try_emplace_impl(std::forward(key), std::forward(obj)); + if(!it.second) { + it.first.value() = std::forward(obj); + } + + return it; + } + + template + std::pair try_emplace_impl(P&& key, Args&&... args_value) { + const std::size_t hash = hash_key(key); + const std::size_t ibucket_for_hash = bucket_for_hash(hash); + + // Check if already presents + auto it_find = find_impl(key, hash, m_first_or_empty_bucket + ibucket_for_hash); + if(it_find != end()) { + return std::make_pair(it_find, false); + } + + return insert_impl(ibucket_for_hash, hash, std::piecewise_construct, + std::forward_as_tuple(std::forward

(key)), + std::forward_as_tuple(std::forward(args_value)...)); + } + + template + std::pair insert_impl(P&& value) { + const std::size_t hash = hash_key(KeySelect()(value)); + const std::size_t ibucket_for_hash = bucket_for_hash(hash); + + // Check if already presents + auto it_find = find_impl(KeySelect()(value), hash, m_first_or_empty_bucket + ibucket_for_hash); + if(it_find != end()) { + return std::make_pair(it_find, false); + } + + + return insert_impl(ibucket_for_hash, hash, std::forward

(value)); + } + + template + std::pair insert_impl(std::size_t ibucket_for_hash, std::size_t hash, Args&&... value_type_args) { + if((m_nb_elements - m_overflow_elements.size()) >= m_max_load_threshold_rehash) { + rehash(GrowthPolicy::next_bucket_count()); + ibucket_for_hash = bucket_for_hash(hash); + } + + std::size_t ibucket_empty = find_empty_bucket(ibucket_for_hash); + if(ibucket_empty < m_buckets.size()) { + do { + tsl_assert(ibucket_empty >= ibucket_for_hash); + + // Empty bucket is in range of NeighborhoodSize, use it + if(ibucket_empty - ibucket_for_hash < NeighborhoodSize) { + auto it = insert_in_bucket(ibucket_empty, ibucket_for_hash, + hash, std::forward(value_type_args)...); + return std::make_pair(iterator(it, m_buckets.end(), m_overflow_elements.begin()), true); + } + } + // else, try to swap values to get a closer empty bucket + while(swap_empty_bucket_closer(ibucket_empty)); + } + + // Load factor is too low or a rehash will not change the neighborhood, put the value in overflow list + if(size() < m_min_load_threshold_rehash || !will_neighborhood_change_on_rehash(ibucket_for_hash)) { + auto it = insert_in_overflow(ibucket_for_hash, std::forward(value_type_args)...); + return std::make_pair(iterator(m_buckets.end(), m_buckets.end(), it), true); + } + + rehash(GrowthPolicy::next_bucket_count()); + ibucket_for_hash = bucket_for_hash(hash); + + return insert_impl(ibucket_for_hash, hash, std::forward(value_type_args)...); + } + + /* + * Return true if a rehash will change the position of a key-value in the neighborhood of + * ibucket_neighborhood_check. In this case a rehash is needed instead of puting the value in overflow list. + */ + bool will_neighborhood_change_on_rehash(size_t ibucket_neighborhood_check) const { + std::size_t expand_bucket_count = GrowthPolicy::next_bucket_count(); + GrowthPolicy expand_growth_policy(expand_bucket_count); + + const bool use_stored_hash = USE_STORED_HASH_ON_REHASH(expand_bucket_count); + for(size_t ibucket = ibucket_neighborhood_check; + ibucket < m_buckets.size() && (ibucket - ibucket_neighborhood_check) < NeighborhoodSize; + ++ibucket) + { + tsl_assert(!m_buckets[ibucket].empty()); + + const size_t hash = use_stored_hash? + m_buckets[ibucket].truncated_bucket_hash(): + hash_key(KeySelect()(m_buckets[ibucket].value())); + if(bucket_for_hash(hash) != expand_growth_policy.bucket_for_hash(hash)) { + return true; + } + } + + return false; + } + + /* + * Return the index of an empty bucket in m_buckets. + * If none, the returned index equals m_buckets.size() + */ + std::size_t find_empty_bucket(std::size_t ibucket_start) const { + const std::size_t limit = std::min(ibucket_start + MAX_PROBES_FOR_EMPTY_BUCKET, m_buckets.size()); + for(; ibucket_start < limit; ibucket_start++) { + if(m_buckets[ibucket_start].empty()) { + return ibucket_start; + } + } + + return m_buckets.size(); + } + + /* + * Insert value in ibucket_empty where value originally belongs to ibucket_for_hash + * + * Return bucket iterator to ibucket_empty + */ + template + iterator_buckets insert_in_bucket(std::size_t ibucket_empty, std::size_t ibucket_for_hash, + std::size_t hash, Args&&... value_type_args) + { + tsl_assert(ibucket_empty >= ibucket_for_hash ); + tsl_assert(m_buckets[ibucket_empty].empty()); + m_buckets[ibucket_empty].set_value_of_empty_bucket(hopscotch_bucket::truncate_hash(hash), std::forward(value_type_args)...); + + tsl_assert(!m_buckets[ibucket_for_hash].empty()); + m_buckets[ibucket_for_hash].toggle_neighbor_presence(ibucket_empty - ibucket_for_hash); + m_nb_elements++; + + return m_buckets.begin() + ibucket_empty; + } + + template::value>::type* = nullptr> + iterator_overflow insert_in_overflow(std::size_t ibucket_for_hash, Args&&... value_type_args) { + auto it = m_overflow_elements.emplace(m_overflow_elements.end(), std::forward(value_type_args)...); + + m_buckets[ibucket_for_hash].set_overflow(true); + m_nb_elements++; + + return it; + } + + template::value>::type* = nullptr> + iterator_overflow insert_in_overflow(std::size_t ibucket_for_hash, Args&&... value_type_args) { + auto it = m_overflow_elements.emplace(std::forward(value_type_args)...).first; + + m_buckets[ibucket_for_hash].set_overflow(true); + m_nb_elements++; + + return it; + } + + /* + * Try to swap the bucket ibucket_empty_in_out with a bucket preceding it while keeping the neighborhood + * conditions correct. + * + * If a swap was possible, the position of ibucket_empty_in_out will be closer to 0 and true will re returned. + */ + bool swap_empty_bucket_closer(std::size_t& ibucket_empty_in_out) { + tsl_assert(ibucket_empty_in_out >= NeighborhoodSize); + const std::size_t neighborhood_start = ibucket_empty_in_out - NeighborhoodSize + 1; + + for(std::size_t to_check = neighborhood_start; to_check < ibucket_empty_in_out; to_check++) { + neighborhood_bitmap neighborhood_infos = m_buckets[to_check].neighborhood_infos(); + std::size_t to_swap = to_check; + + while(neighborhood_infos != 0 && to_swap < ibucket_empty_in_out) { + if((neighborhood_infos & 1) == 1) { + tsl_assert(m_buckets[ibucket_empty_in_out].empty()); + tsl_assert(!m_buckets[to_swap].empty()); + + m_buckets[to_swap].swap_value_into_empty_bucket(m_buckets[ibucket_empty_in_out]); + + tsl_assert(!m_buckets[to_check].check_neighbor_presence(ibucket_empty_in_out - to_check)); + tsl_assert(m_buckets[to_check].check_neighbor_presence(to_swap - to_check)); + + m_buckets[to_check].toggle_neighbor_presence(ibucket_empty_in_out - to_check); + m_buckets[to_check].toggle_neighbor_presence(to_swap - to_check); + + + ibucket_empty_in_out = to_swap; + + return true; + } + + to_swap++; + neighborhood_infos = neighborhood_bitmap(neighborhood_infos >> 1); + } + } + + return false; + } + + + + template::value>::type* = nullptr> + typename U::value_type* find_value_impl(const K& key, std::size_t hash, hopscotch_bucket* bucket_for_hash) { + return const_cast( + static_cast(this)->find_value_impl(key, hash, bucket_for_hash)); + } + + /* + * Avoid the creation of an iterator to just get the value for operator[] and at() in maps. Faster this way. + * + * Return null if no value for the key (TODO use std::optional when available). + */ + template::value>::type* = nullptr> + const typename U::value_type* find_value_impl(const K& key, std::size_t hash, + const hopscotch_bucket* bucket_for_hash) const + { + const hopscotch_bucket* bucket_found = find_in_buckets(key, hash, bucket_for_hash); + if(bucket_found != nullptr) { + return std::addressof(ValueSelect()(bucket_found->value())); + } + + if(bucket_for_hash->has_overflow()) { + auto it_overflow = find_in_overflow(key); + if(it_overflow != m_overflow_elements.end()) { + return std::addressof(ValueSelect()(*it_overflow)); + } + } + + return nullptr; + } + + template + size_type count_impl(const K& key, std::size_t hash, const hopscotch_bucket* bucket_for_hash) const { + if(find_in_buckets(key, hash, bucket_for_hash) != nullptr) { + return 1; + } + else if(bucket_for_hash->has_overflow() && find_in_overflow(key) != m_overflow_elements.cend()) { + return 1; + } + else { + return 0; + } + } + + template + iterator find_impl(const K& key, std::size_t hash, hopscotch_bucket* bucket_for_hash) { + hopscotch_bucket* bucket_found = find_in_buckets(key, hash, bucket_for_hash); + if(bucket_found != nullptr) { + return iterator(m_buckets.begin() + std::distance(m_buckets.data(), bucket_found), + m_buckets.end(), m_overflow_elements.begin()); + } + + if(!bucket_for_hash->has_overflow()) { + return end(); + } + + return iterator(m_buckets.end(), m_buckets.end(), find_in_overflow(key)); + } + + template + const_iterator find_impl(const K& key, std::size_t hash, const hopscotch_bucket* bucket_for_hash) const { + const hopscotch_bucket* bucket_found = find_in_buckets(key, hash, bucket_for_hash); + if(bucket_found != nullptr) { + return const_iterator(m_buckets.cbegin() + std::distance(m_buckets.data(), bucket_found), + m_buckets.cend(), m_overflow_elements.cbegin()); + } + + if(!bucket_for_hash->has_overflow()) { + return cend(); + } + + + return const_iterator(m_buckets.cend(), m_buckets.cend(), find_in_overflow(key)); + } + + template + hopscotch_bucket* find_in_buckets(const K& key, std::size_t hash, hopscotch_bucket* bucket_for_hash) { + const hopscotch_bucket* bucket_found = + static_cast(this)->find_in_buckets(key, hash, bucket_for_hash); + return const_cast(bucket_found); + } + + + /** + * Return a pointer to the bucket which has the value, nullptr otherwise. + */ + template + const hopscotch_bucket* find_in_buckets(const K& key, std::size_t hash, const hopscotch_bucket* bucket_for_hash) const { + (void) hash; // Avoid warning of unused variable when StoreHash is false; + + // TODO Try to optimize the function. + // I tried to use ffs and __builtin_ffs functions but I could not reduce the time the function + // takes with -march=native + + neighborhood_bitmap neighborhood_infos = bucket_for_hash->neighborhood_infos(); + while(neighborhood_infos != 0) { + if((neighborhood_infos & 1) == 1) { + // Check StoreHash before calling bucket_hash_equal. Functionally it doesn't change anythin. + // If StoreHash is false, bucket_hash_equal is a no-op. Avoiding the call is there to help + // GCC optimizes `hash` parameter away, it seems to not be able to do without this hint. + if((!StoreHash || bucket_for_hash->bucket_hash_equal(hash)) && + compare_keys(KeySelect()(bucket_for_hash->value()), key)) + { + return bucket_for_hash; + } + } + + ++bucket_for_hash; + neighborhood_infos = neighborhood_bitmap(neighborhood_infos >> 1); + } + + return nullptr; + } + + + + template::value>::type* = nullptr> + iterator_overflow find_in_overflow(const K& key) { + return std::find_if(m_overflow_elements.begin(), m_overflow_elements.end(), + [&](const value_type& value) { + return compare_keys(key, KeySelect()(value)); + }); + } + + template::value>::type* = nullptr> + const_iterator_overflow find_in_overflow(const K& key) const { + return std::find_if(m_overflow_elements.cbegin(), m_overflow_elements.cend(), + [&](const value_type& value) { + return compare_keys(key, KeySelect()(value)); + }); + } + + template::value>::type* = nullptr> + iterator_overflow find_in_overflow(const K& key) { + return m_overflow_elements.find(key); + } + + template::value>::type* = nullptr> + const_iterator_overflow find_in_overflow(const K& key) const { + return m_overflow_elements.find(key); + } + + + + template::value>::type* = nullptr> + hopscotch_hash new_hopscotch_hash(size_type bucket_count) { + return hopscotch_hash(bucket_count, static_cast(*this), static_cast(*this), + get_allocator(), m_max_load_factor); + } + + template::value>::type* = nullptr> + hopscotch_hash new_hopscotch_hash(size_type bucket_count) { + return hopscotch_hash(bucket_count, static_cast(*this), static_cast(*this), + get_allocator(), m_max_load_factor, m_overflow_elements.key_comp()); + } + +public: + static const size_type DEFAULT_INIT_BUCKETS_SIZE = 16; + static constexpr float DEFAULT_MAX_LOAD_FACTOR = (NeighborhoodSize <= 30)?0.8f:0.9f; + +private: + static const std::size_t MAX_PROBES_FOR_EMPTY_BUCKET = 12*NeighborhoodSize; + static constexpr float MIN_LOAD_FACTOR_FOR_REHASH = 0.1f; + + static bool USE_STORED_HASH_ON_REHASH(size_type bucket_count) { + (void) bucket_count; + if(StoreHash && is_power_of_two_policy::value) { + tsl_assert(bucket_count > 0); + return (bucket_count - 1) <= std::numeric_limits::max(); + } + else { + return false; + } + } + + /** + * Return an always valid pointer to an static empty hopscotch_bucket. + */ + hopscotch_bucket* static_empty_bucket_ptr() { + static hopscotch_bucket empty_bucket; + return &empty_bucket; + } + +private: + buckets_container_type m_buckets; + overflow_container_type m_overflow_elements; + + /** + * Points to m_buckets.data() if !m_buckets.empty() otherwise points to static_empty_bucket_ptr. + * This variable is useful to avoid the cost of checking if m_buckets is empty when trying + * to find an element. + */ + hopscotch_bucket* m_first_or_empty_bucket; + + size_type m_nb_elements; + + float m_max_load_factor; + + /** + * Max size of the hash table before a rehash occurs automatically to grow the table. + */ + size_type m_max_load_threshold_rehash; + + /** + * Min size of the hash table before a rehash can occurs automatically (except if m_max_load_threshold_rehash os reached). + * If the neighborhood of a bucket is full before the min is reacher, the elements are put into m_overflow_elements. + */ + size_type m_min_load_threshold_rehash; +}; + +} // end namespace detail_hopscotch_hash + + +} // end namespace tsl + +#endif diff --git a/include/tsl/hopscotch_map.h b/include/tsl/hopscotch_map.h new file mode 100644 index 000000000..acd7a7931 --- /dev/null +++ b/include/tsl/hopscotch_map.h @@ -0,0 +1,679 @@ +/** + * MIT License + * + * Copyright (c) 2017 Tessil + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_HOPSCOTCH_MAP_H +#define TSL_HOPSCOTCH_MAP_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hopscotch_hash.h" + + +namespace tsl { + +/** + * Implementation of a hash map using the hopscotch hashing algorithm. + * + * The Key and the value T must be either nothrow move-constructible, copy-constuctible or both. + * + * The size of the neighborhood (NeighborhoodSize) must be > 0 and <= 62 if StoreHash is false. + * When StoreHash is true, 32-bits of the hash will be stored alongside the neighborhood limiting + * the NeighborhoodSize to <= 30. There is no memory usage difference between + * 'NeighborhoodSize 62; StoreHash false' and 'NeighborhoodSize 30; StoreHash true'. + * + * Storing the hash may improve performance on insert during the rehash process if the hash takes time + * to compute. It may also improve read performance if the KeyEqual function takes time (or incurs a cache-miss). + * If used with simple Hash and KeyEqual it may slow things down. + * + * StoreHash can only be set if the GrowthPolicy is set to tsl::power_of_two_growth_policy. + * + * GrowthPolicy defines how the map grows and consequently how a hash value is mapped to a bucket. + * By default the map uses tsl::power_of_two_growth_policy. This policy keeps the number of buckets + * to a power of two and uses a mask to map the hash to a bucket instead of the slow modulo. + * You may define your own growth policy, check tsl::power_of_two_growth_policy for the interface. + * + * If the destructors of Key or T throw an exception, behaviour of the class is undefined. + * + * Iterators invalidation: + * - clear, operator=, reserve, rehash: always invalidate the iterators. + * - insert, emplace, emplace_hint, operator[]: if there is an effective insert, invalidate the iterators + * if a displacement is needed to resolve a collision (which mean that most of the time, + * insert will invalidate the iterators). Or if there is a rehash. + * - erase: iterator on the erased element is the only one which become invalid. + */ +template, + class KeyEqual = std::equal_to, + class Allocator = std::allocator>, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false, + class GrowthPolicy = tsl::hh::power_of_two_growth_policy<2>> +class hopscotch_map { +private: + template + using has_is_transparent = tsl::detail_hopscotch_hash::has_is_transparent; + + class KeySelect { + public: + using key_type = Key; + + const key_type& operator()(const std::pair& key_value) const { + return key_value.first; + } + + key_type& operator()(std::pair& key_value) { + return key_value.first; + } + }; + + class ValueSelect { + public: + using value_type = T; + + const value_type& operator()(const std::pair& key_value) const { + return key_value.second; + } + + value_type& operator()(std::pair& key_value) { + return key_value.second; + } + }; + + + using overflow_container_type = std::list, Allocator>; + using ht = detail_hopscotch_hash::hopscotch_hash, KeySelect, ValueSelect, + Hash, KeyEqual, + Allocator, NeighborhoodSize, + StoreHash, GrowthPolicy, + overflow_container_type>; + +public: + using key_type = typename ht::key_type; + using mapped_type = T; + using value_type = typename ht::value_type; + using size_type = typename ht::size_type; + using difference_type = typename ht::difference_type; + using hasher = typename ht::hasher; + using key_equal = typename ht::key_equal; + using allocator_type = typename ht::allocator_type; + using reference = typename ht::reference; + using const_reference = typename ht::const_reference; + using pointer = typename ht::pointer; + using const_pointer = typename ht::const_pointer; + using iterator = typename ht::iterator; + using const_iterator = typename ht::const_iterator; + + + + /* + * Constructors + */ + hopscotch_map() : hopscotch_map(ht::DEFAULT_INIT_BUCKETS_SIZE) { + } + + explicit hopscotch_map(size_type bucket_count, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : + m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR) + { + } + + hopscotch_map(size_type bucket_count, + const Allocator& alloc) : hopscotch_map(bucket_count, Hash(), KeyEqual(), alloc) + { + } + + hopscotch_map(size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : hopscotch_map(bucket_count, hash, KeyEqual(), alloc) + { + } + + explicit hopscotch_map(const Allocator& alloc) : hopscotch_map(ht::DEFAULT_INIT_BUCKETS_SIZE, alloc) { + } + + template + hopscotch_map(InputIt first, InputIt last, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : hopscotch_map(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + + template + hopscotch_map(InputIt first, InputIt last, + size_type bucket_count, + const Allocator& alloc) : hopscotch_map(first, last, bucket_count, Hash(), KeyEqual(), alloc) + { + } + + template + hopscotch_map(InputIt first, InputIt last, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : hopscotch_map(first, last, bucket_count, hash, KeyEqual(), alloc) + { + } + + hopscotch_map(std::initializer_list init, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : + hopscotch_map(init.begin(), init.end(), bucket_count, hash, equal, alloc) + { + } + + hopscotch_map(std::initializer_list init, + size_type bucket_count, + const Allocator& alloc) : + hopscotch_map(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), alloc) + { + } + + hopscotch_map(std::initializer_list init, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : + hopscotch_map(init.begin(), init.end(), bucket_count, hash, KeyEqual(), alloc) + { + } + + + hopscotch_map& operator=(std::initializer_list ilist) { + m_ht.clear(); + + m_ht.reserve(ilist.size()); + m_ht.insert(ilist.begin(), ilist.end()); + + return *this; + } + + allocator_type get_allocator() const { return m_ht.get_allocator(); } + + + /* + * Iterators + */ + iterator begin() noexcept { return m_ht.begin(); } + const_iterator begin() const noexcept { return m_ht.begin(); } + const_iterator cbegin() const noexcept { return m_ht.cbegin(); } + + iterator end() noexcept { return m_ht.end(); } + const_iterator end() const noexcept { return m_ht.end(); } + const_iterator cend() const noexcept { return m_ht.cend(); } + + + /* + * Capacity + */ + bool empty() const noexcept { return m_ht.empty(); } + size_type size() const noexcept { return m_ht.size(); } + size_type max_size() const noexcept { return m_ht.max_size(); } + + /* + * Modifiers + */ + void clear() noexcept { m_ht.clear(); } + + + + + std::pair insert(const value_type& value) { + return m_ht.insert(value); + } + + template::value>::type* = nullptr> + std::pair insert(P&& value) { + return m_ht.insert(std::forward

(value)); + } + + std::pair insert(value_type&& value) { + return m_ht.insert(std::move(value)); + } + + + iterator insert(const_iterator hint, const value_type& value) { + return m_ht.insert(hint, value); + } + + template::value>::type* = nullptr> + iterator insert(const_iterator hint, P&& value) { + return m_ht.insert(hint, std::forward

(value)); + } + + iterator insert(const_iterator hint, value_type&& value) { + return m_ht.insert(hint, std::move(value)); + } + + + template + void insert(InputIt first, InputIt last) { + m_ht.insert(first, last); + } + + void insert(std::initializer_list ilist) { + m_ht.insert(ilist.begin(), ilist.end()); + } + + + + + template + std::pair insert_or_assign(const key_type& k, M&& obj) { + return m_ht.insert_or_assign(k, std::forward(obj)); + } + + template + std::pair insert_or_assign(key_type&& k, M&& obj) { + return m_ht.insert_or_assign(std::move(k), std::forward(obj)); + } + + template + iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj) { + return m_ht.insert_or_assign(hint, k, std::forward(obj)); + } + + template + iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj) { + return m_ht.insert_or_assign(hint, std::move(k), std::forward(obj)); + } + + + + + /** + * Due to the way elements are stored, emplace will need to move or copy the key-value once. + * The method is equivalent to insert(value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + std::pair emplace(Args&&... args) { + return m_ht.emplace(std::forward(args)...); + } + + + + + /** + * Due to the way elements are stored, emplace_hint will need to move or copy the key-value once. + * The method is equivalent to insert(hint, value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + iterator emplace_hint(const_iterator hint, Args&&... args) { + return m_ht.emplace_hint(hint, std::forward(args)...); + } + + + + + template + std::pair try_emplace(const key_type& k, Args&&... args) { + return m_ht.try_emplace(k, std::forward(args)...); + } + + template + std::pair try_emplace(key_type&& k, Args&&... args) { + return m_ht.try_emplace(std::move(k), std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args) { + return m_ht.try_emplace(hint, k, std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args) { + return m_ht.try_emplace(hint, std::move(k), std::forward(args)...); + } + + + + + iterator erase(iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator first, const_iterator last) { return m_ht.erase(first, last); } + size_type erase(const key_type& key) { return m_ht.erase(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + size_type erase(const key_type& key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + size_type erase(const K& key) { return m_ht.erase(key); } + + /** + * @copydoc erase(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + template::value>::type* = nullptr> + size_type erase(const K& key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + + + + void swap(hopscotch_map& other) { other.m_ht.swap(m_ht); } + + /* + * Lookup + */ + T& at(const Key& key) { return m_ht.at(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + T& at(const Key& key, std::size_t precalculated_hash) { return m_ht.at(key, precalculated_hash); } + + + const T& at(const Key& key) const { return m_ht.at(key); } + + /** + * @copydoc at(const Key& key, std::size_t precalculated_hash) + */ + const T& at(const Key& key, std::size_t precalculated_hash) const { return m_ht.at(key, precalculated_hash); } + + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + T& at(const K& key) { return m_ht.at(key); } + + /** + * @copydoc at(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + T& at(const K& key, std::size_t precalculated_hash) { return m_ht.at(key, precalculated_hash); } + + + /** + * @copydoc at(const K& key) + */ + template::value>::type* = nullptr> + const T& at(const K& key) const { return m_ht.at(key); } + + /** + * @copydoc at(const K& key, std::size_t precalculated_hash) + */ + template::value>::type* = nullptr> + const T& at(const K& key, std::size_t precalculated_hash) const { return m_ht.at(key, precalculated_hash); } + + + + + T& operator[](const Key& key) { return m_ht[key]; } + T& operator[](Key&& key) { return m_ht[std::move(key)]; } + + + + + size_type count(const Key& key) const { return m_ht.count(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + size_type count(const Key& key, std::size_t precalculated_hash) const { + return m_ht.count(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + size_type count(const K& key) const { return m_ht.count(key); } + + /** + * @copydoc count(const K& key) const + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + size_type count(const K& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } + + + + + iterator find(const Key& key) { return m_ht.find(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + iterator find(const Key& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + const_iterator find(const Key& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const Key& key, std::size_t precalculated_hash) + */ + const_iterator find(const Key& key, std::size_t precalculated_hash) const { + return m_ht.find(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + iterator find(const K& key) { return m_ht.find(key); } + + /** + * @copydoc find(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + iterator find(const K& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + /** + * @copydoc find(const K& key) + */ + template::value>::type* = nullptr> + const_iterator find(const K& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + const_iterator find(const K& key, std::size_t precalculated_hash) const { + return m_ht.find(key, precalculated_hash); + } + + + + + std::pair equal_range(const Key& key) { return m_ht.equal_range(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + std::pair equal_range(const Key& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key) { return m_ht.equal_range(key); } + + + /** + * @copydoc equal_range(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * @copydoc equal_range(const K& key) + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const K& key, std::size_t precalculated_hash) + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + + + + /* + * Bucket interface + */ + size_type bucket_count() const { return m_ht.bucket_count(); } + size_type max_bucket_count() const { return m_ht.max_bucket_count(); } + + + /* + * Hash policy + */ + float load_factor() const { return m_ht.load_factor(); } + float max_load_factor() const { return m_ht.max_load_factor(); } + void max_load_factor(float ml) { m_ht.max_load_factor(ml); } + + void rehash(size_type count_) { m_ht.rehash(count_); } + void reserve(size_type count_) { m_ht.reserve(count_); } + + + /* + * Observers + */ + hasher hash_function() const { return m_ht.hash_function(); } + key_equal key_eq() const { return m_ht.key_eq(); } + + /* + * Other + */ + + /** + * Convert a const_iterator to an iterator. + */ + iterator mutable_iterator(const_iterator pos) { + return m_ht.mutable_iterator(pos); + } + + size_type overflow_size() const noexcept { return m_ht.overflow_size(); } + + friend bool operator==(const hopscotch_map& lhs, const hopscotch_map& rhs) { + if(lhs.size() != rhs.size()) { + return false; + } + + for(const auto& element_lhs : lhs) { + const auto it_element_rhs = rhs.find(element_lhs.first); + if(it_element_rhs == rhs.cend() || element_lhs.second != it_element_rhs->second) { + return false; + } + } + + return true; + } + + friend bool operator!=(const hopscotch_map& lhs, const hopscotch_map& rhs) { + return !operator==(lhs, rhs); + } + + friend void swap(hopscotch_map& lhs, hopscotch_map& rhs) { + lhs.swap(rhs); + } + + + +private: + ht m_ht; +}; + + +/** + * Same as `tsl::hopscotch_map`. + */ +template, + class KeyEqual = std::equal_to, + class Allocator = std::allocator>, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false> +using hopscotch_pg_map = hopscotch_map; + +} // end namespace tsl + +#endif diff --git a/include/tsl/hopscotch_set.h b/include/tsl/hopscotch_set.h new file mode 100644 index 000000000..4013d3381 --- /dev/null +++ b/include/tsl/hopscotch_set.h @@ -0,0 +1,525 @@ +/** + * MIT License + * + * Copyright (c) 2017 Tessil + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_HOPSCOTCH_SET_H +#define TSL_HOPSCOTCH_SET_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hopscotch_hash.h" + + +namespace tsl { + +/** + * Implementation of a hash set using the hopscotch hashing algorithm. + * + * The Key must be either nothrow move-constructible, copy-constuctible or both. + * + * The size of the neighborhood (NeighborhoodSize) must be > 0 and <= 62 if StoreHash is false. + * When StoreHash is true, 32-bits of the hash will be stored alongside the neighborhood limiting + * the NeighborhoodSize to <= 30. There is no memory usage difference between + * 'NeighborhoodSize 62; StoreHash false' and 'NeighborhoodSize 30; StoreHash true'. + * + * Storing the hash may improve performance on insert during the rehash process if the hash takes time + * to compute. It may also improve read performance if the KeyEqual function takes time (or incurs a cache-miss). + * If used with simple Hash and KeyEqual it may slow things down. + * + * StoreHash can only be set if the GrowthPolicy is set to tsl::power_of_two_growth_policy. + * + * GrowthPolicy defines how the set grows and consequently how a hash value is mapped to a bucket. + * By default the set uses tsl::power_of_two_growth_policy. This policy keeps the number of buckets + * to a power of two and uses a mask to set the hash to a bucket instead of the slow modulo. + * You may define your own growth policy, check tsl::power_of_two_growth_policy for the interface. + * + * If the destructor of Key throws an exception, behaviour of the class is undefined. + * + * Iterators invalidation: + * - clear, operator=, reserve, rehash: always invalidate the iterators. + * - insert, emplace, emplace_hint, operator[]: if there is an effective insert, invalidate the iterators + * if a displacement is needed to resolve a collision (which mean that most of the time, + * insert will invalidate the iterators). Or if there is a rehash. + * - erase: iterator on the erased element is the only one which become invalid. + */ +template, + class KeyEqual = std::equal_to, + class Allocator = std::allocator, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false, + class GrowthPolicy = tsl::hh::power_of_two_growth_policy<2>> +class hopscotch_set { +private: + template + using has_is_transparent = tsl::detail_hopscotch_hash::has_is_transparent; + + class KeySelect { + public: + using key_type = Key; + + const key_type& operator()(const Key& key) const { + return key; + } + + key_type& operator()(Key& key) { + return key; + } + }; + + + using overflow_container_type = std::list; + using ht = detail_hopscotch_hash::hopscotch_hash; + +public: + using key_type = typename ht::key_type; + using value_type = typename ht::value_type; + using size_type = typename ht::size_type; + using difference_type = typename ht::difference_type; + using hasher = typename ht::hasher; + using key_equal = typename ht::key_equal; + using allocator_type = typename ht::allocator_type; + using reference = typename ht::reference; + using const_reference = typename ht::const_reference; + using pointer = typename ht::pointer; + using const_pointer = typename ht::const_pointer; + using iterator = typename ht::iterator; + using const_iterator = typename ht::const_iterator; + + + /* + * Constructors + */ + hopscotch_set() : hopscotch_set(ht::DEFAULT_INIT_BUCKETS_SIZE) { + } + + explicit hopscotch_set(size_type bucket_count, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : + m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR) + { + } + + hopscotch_set(size_type bucket_count, + const Allocator& alloc) : hopscotch_set(bucket_count, Hash(), KeyEqual(), alloc) + { + } + + hopscotch_set(size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : hopscotch_set(bucket_count, hash, KeyEqual(), alloc) + { + } + + explicit hopscotch_set(const Allocator& alloc) : hopscotch_set(ht::DEFAULT_INIT_BUCKETS_SIZE, alloc) { + } + + template + hopscotch_set(InputIt first, InputIt last, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : hopscotch_set(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + + template + hopscotch_set(InputIt first, InputIt last, + size_type bucket_count, + const Allocator& alloc) : hopscotch_set(first, last, bucket_count, Hash(), KeyEqual(), alloc) + { + } + + template + hopscotch_set(InputIt first, InputIt last, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : hopscotch_set(first, last, bucket_count, hash, KeyEqual(), alloc) + { + } + + hopscotch_set(std::initializer_list init, + size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, + const Hash& hash = Hash(), + const KeyEqual& equal = KeyEqual(), + const Allocator& alloc = Allocator()) : + hopscotch_set(init.begin(), init.end(), bucket_count, hash, equal, alloc) + { + } + + hopscotch_set(std::initializer_list init, + size_type bucket_count, + const Allocator& alloc) : + hopscotch_set(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), alloc) + { + } + + hopscotch_set(std::initializer_list init, + size_type bucket_count, + const Hash& hash, + const Allocator& alloc) : + hopscotch_set(init.begin(), init.end(), bucket_count, hash, KeyEqual(), alloc) + { + } + + + hopscotch_set& operator=(std::initializer_list ilist) { + m_ht.clear(); + + m_ht.reserve(ilist.size()); + m_ht.insert(ilist.begin(), ilist.end()); + + return *this; + } + + allocator_type get_allocator() const { return m_ht.get_allocator(); } + + + /* + * Iterators + */ + iterator begin() noexcept { return m_ht.begin(); } + const_iterator begin() const noexcept { return m_ht.begin(); } + const_iterator cbegin() const noexcept { return m_ht.cbegin(); } + + iterator end() noexcept { return m_ht.end(); } + const_iterator end() const noexcept { return m_ht.end(); } + const_iterator cend() const noexcept { return m_ht.cend(); } + + + /* + * Capacity + */ + bool empty() const noexcept { return m_ht.empty(); } + size_type size() const noexcept { return m_ht.size(); } + size_type max_size() const noexcept { return m_ht.max_size(); } + + /* + * Modifiers + */ + void clear() noexcept { m_ht.clear(); } + + + + + std::pair insert(const value_type& value) { return m_ht.insert(value); } + std::pair insert(value_type&& value) { return m_ht.insert(std::move(value)); } + + iterator insert(const_iterator hint, const value_type& value) { return m_ht.insert(hint, value); } + iterator insert(const_iterator hint, value_type&& value) { return m_ht.insert(hint, std::move(value)); } + + template + void insert(InputIt first, InputIt last) { m_ht.insert(first, last); } + void insert(std::initializer_list ilist) { m_ht.insert(ilist.begin(), ilist.end()); } + + + + + /** + * Due to the way elements are stored, emplace will need to move or copy the key-value once. + * The method is equivalent to insert(value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + std::pair emplace(Args&&... args) { return m_ht.emplace(std::forward(args)...); } + + + + + /** + * Due to the way elements are stored, emplace_hint will need to move or copy the key-value once. + * The method is equivalent to insert(hint, value_type(std::forward(args)...)); + * + * Mainly here for compatibility with the std::unordered_map interface. + */ + template + iterator emplace_hint(const_iterator hint, Args&&... args) { + return m_ht.emplace_hint(hint, std::forward(args)...); + } + + + + + iterator erase(iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator first, const_iterator last) { return m_ht.erase(first, last); } + size_type erase(const key_type& key) { return m_ht.erase(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + size_type erase(const key_type& key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + size_type erase(const K& key) { return m_ht.erase(key); } + + /** + * @copydoc erase(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. + */ + template::value>::type* = nullptr> + size_type erase(const K& key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + + + + void swap(hopscotch_set& other) { other.m_ht.swap(m_ht); } + + + /* + * Lookup + */ + size_type count(const Key& key) const { return m_ht.count(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + size_type count(const Key& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + size_type count(const K& key) const { return m_ht.count(key); } + + /** + * @copydoc count(const K& key) const + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + size_type count(const K& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } + + + + + iterator find(const Key& key) { return m_ht.find(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + iterator find(const Key& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + const_iterator find(const Key& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const Key& key, std::size_t precalculated_hash) + */ + const_iterator find(const Key& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + iterator find(const K& key) { return m_ht.find(key); } + + /** + * @copydoc find(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + iterator find(const K& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } + + /** + * @copydoc find(const K& key) + */ + template::value>::type* = nullptr> + const_iterator find(const K& key) const { return m_ht.find(key); } + + /** + * @copydoc find(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + const_iterator find(const K& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } + + + + + std::pair equal_range(const Key& key) { return m_ht.equal_range(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + std::pair equal_range(const Key& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) + */ + std::pair equal_range(const Key& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. + * If so, K must be hashable and comparable to Key. + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key) { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const K& key) + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same + * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * @copydoc equal_range(const K& key) + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key) const { return m_ht.equal_range(key); } + + /** + * @copydoc equal_range(const K& key, std::size_t precalculated_hash) + */ + template::value>::type* = nullptr> + std::pair equal_range(const K& key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + + + + /* + * Bucket interface + */ + size_type bucket_count() const { return m_ht.bucket_count(); } + size_type max_bucket_count() const { return m_ht.max_bucket_count(); } + + + /* + * Hash policy + */ + float load_factor() const { return m_ht.load_factor(); } + float max_load_factor() const { return m_ht.max_load_factor(); } + void max_load_factor(float ml) { m_ht.max_load_factor(ml); } + + void rehash(size_type count_) { m_ht.rehash(count_); } + void reserve(size_type count_) { m_ht.reserve(count_); } + + + /* + * Observers + */ + hasher hash_function() const { return m_ht.hash_function(); } + key_equal key_eq() const { return m_ht.key_eq(); } + + + /* + * Other + */ + + /** + * Convert a const_iterator to an iterator. + */ + iterator mutable_iterator(const_iterator pos) { + return m_ht.mutable_iterator(pos); + } + + size_type overflow_size() const noexcept { return m_ht.overflow_size(); } + + friend bool operator==(const hopscotch_set& lhs, const hopscotch_set& rhs) { + if(lhs.size() != rhs.size()) { + return false; + } + + for(const auto& element_lhs : lhs) { + const auto it_element_rhs = rhs.find(element_lhs); + if(it_element_rhs == rhs.cend()) { + return false; + } + } + + return true; + } + + friend bool operator!=(const hopscotch_set& lhs, const hopscotch_set& rhs) { + return !operator==(lhs, rhs); + } + + friend void swap(hopscotch_set& lhs, hopscotch_set& rhs) { + lhs.swap(rhs); + } + +private: + ht m_ht; +}; + + +/** + * Same as `tsl::hopscotch_set`. + */ +template, + class KeyEqual = std::equal_to, + class Allocator = std::allocator, + unsigned int NeighborhoodSize = 62, + bool StoreHash = false> +using hopscotch_pg_set = hopscotch_set; + +} // end namespace tsl + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f891ccc47..121f15f5e 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -156,6 +156,9 @@ HAVE_SSTREAM=1 ) target_compile_options(salmon_core PUBLIC "$<$:${TGT_DEBUG_FLAGS}>") target_compile_options(salmon_core PUBLIC "$<$:${TGT_RELEASE_FLAGS}>") +if(NOT CONDA_BUILD) + set_property(TARGET salmon_core PROPERTY INTERPROCEDURAL_OPTIMIZATION True) +endif() # Build the Alevin library add_library(alevin_core STATIC ${ALEVIN_LIB_SRCS}) @@ -168,9 +171,17 @@ span_FEATURE_MAKE_SPAN_TO_STD=14 target_compile_options(alevin_core PUBLIC "$<$:${TGT_DEBUG_FLAGS}>") target_compile_options(alevin_core PUBLIC "$<$:${TGT_RELEASE_FLAGS}>") +if(NOT CONDA_BUILD) + set_property(TARGET alevin_core PROPERTY INTERPROCEDURAL_OPTIMIZATION True) +endif() + # Build the salmon executable add_executable(salmon ${SALMON_MAIN_SRCS} ${SALMON_ALIGN_SRCS}) +if(NOT CONDA_BUILD) + set_property(TARGET salmon PROPERTY INTERPROCEDURAL_OPTIMIZATION True) +endif() + add_executable(unitTests ${UNIT_TESTS_SRCS}) #add_executable(salmon-read ${SALMON_READ_SRCS}) diff --git a/src/SalmonQuantify.cpp b/src/SalmonQuantify.cpp index fefabf3b4..fcca1e896 100644 --- a/src/SalmonQuantify.cpp +++ b/src/SalmonQuantify.cpp @@ -116,7 +116,8 @@ #include "SingleAlignmentFormatter.hpp" #include "ksw2pp/KSW2Aligner.hpp" #include "metro/metrohash64.h" -#include "tsl/robin_map.h" +#include "tsl/hopscotch_map.h" +//#include "tsl/robin_map.h" //#include "TextBootstrapWriter.hpp" /****** QUASI MAPPING DECLARATIONS *********/ @@ -762,8 +763,9 @@ namespace salmon { } } -using AlnCacheMap = //tsl::robin_map; - std::unordered_map; +using AlnCacheMap = tsl::hopscotch_map; + //tsl::robin_map; + //std::unordered_map; inline int32_t getAlnScore( ksw2pp::KSW2Aligner& aligner, @@ -961,8 +963,8 @@ void processReadsQuasi( memset(&ez, 0, sizeof(ksw_extz_t)); size_t numDropped{0}; - AlnCacheMap alnCacheLeft; alnCacheLeft.reserve(16); - AlnCacheMap alnCacheRight; alnCacheRight.reserve(16); + AlnCacheMap alnCacheLeft; alnCacheLeft.reserve(32); + AlnCacheMap alnCacheRight; alnCacheRight.reserve(32); auto rg = parser->getReadGroup(); while (parser->refill(rg)) { From 6c0114ea95a857e182ad7e7accfaef02195b959b Mon Sep 17 00:00:00 2001 From: Rob Patro Date: Fri, 3 Aug 2018 22:47:09 -0400 Subject: [PATCH 3/4] Bump cmake req to 3.9 * test for and enable LTO if supported * compile ksw2 with O3 * don't run eff length through barrier function for unexpressed txps. --- CMakeLists.txt | 4 +++- src/CMakeLists.txt | 16 +++++++++++----- src/CollapsedEMOptimizer.cpp | 3 ++- src/SalmonQuantify.cpp | 6 ++---- src/SalmonUtils.cpp | 33 +++++++++++++++++++-------------- 5 files changed, 37 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a98093a41..41c3e4d87 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.1) +cmake_minimum_required (VERSION 3.9) enable_testing() @@ -76,6 +76,8 @@ else () set(GCCVERSION "5.2") endif() +include(CheckIPOSupported) + set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 121f15f5e..ba2055258 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -91,9 +91,9 @@ add_library(ksw2pp_sse2 OBJECT ${KSW2PP_ADVANCED_LIB_SRCS}) add_library(ksw2pp_sse4 OBJECT ${KSW2PP_ADVANCED_LIB_SRCS}) add_library(ksw2pp_basic OBJECT ${KSW2PP_BASIC_LIB_SRCS}) -set_target_properties(ksw2pp_sse2 PROPERTIES COMPILE_FLAGS "-msse2 -mno-sse4.1") +set_target_properties(ksw2pp_sse2 PROPERTIES COMPILE_FLAGS "-O3 -msse2 -mno-sse4.1") set_target_properties(ksw2pp_sse2 PROPERTIES COMPILE_DEFINITIONS "KSW_CPU_DISPATCH;KSW_SSE2_ONLY;HAVE_KALLOC") -set_target_properties(ksw2pp_sse4 PROPERTIES COMPILE_FLAGS "-msse4.1") +set_target_properties(ksw2pp_sse4 PROPERTIES COMPILE_FLAGS "-O3 -msse4.1") set_target_properties(ksw2pp_sse4 PROPERTIES COMPILE_DEFINITIONS "KSW_CPU_DISPATCH;HAVE_KALLOC") set_target_properties(ksw2pp_basic PROPERTIES COMPILE_DEFINITIONS "KSW_CPU_DISPATCH;HAVE_KALLOC") @@ -103,6 +103,9 @@ set_target_properties(ksw2pp_sse4 PROPERTIES INCLUDE_DIRECTORIES ${GAT_SOURCE_DI # Build the ksw2pp library add_library(ksw2pp STATIC $ $ $) set_target_properties(ksw2pp PROPERTIES COMPILE_DEFINITIONS "KSW_CPU_DISPATCH;HAVE_KALLOC") +if(NOT CONDA_BUILD AND HAS_IPO) + set_property(TARGET ksw2pp PROPERTY INTERPROCEDURAL_OPTIMIZATION True) +endif() set ( UNIT_TESTS_SRCS ${GAT_SOURCE_DIR}/tests/UnitTests.cpp @@ -147,6 +150,9 @@ endif() set (TGT_RELEASE_FLAGS "${TGT_COMPILE_FLAGS};${TGT_WARN_FLAGS}") set (TGT_DEBUG_FLAGS "-g;${TGT_COMPILE_FLAGS};${TGT_WARN_FLAGS}") +# check if we know how to do IPO +check_ipo_supported(RESULT HAS_IPOHAS_IPO) + # Build the Salmon library add_library(salmon_core STATIC ${SALMON_LIB_SRCS} ) target_compile_definitions(salmon_core PUBLIC @@ -156,7 +162,7 @@ HAVE_SSTREAM=1 ) target_compile_options(salmon_core PUBLIC "$<$:${TGT_DEBUG_FLAGS}>") target_compile_options(salmon_core PUBLIC "$<$:${TGT_RELEASE_FLAGS}>") -if(NOT CONDA_BUILD) +if(NOT CONDA_BUILD AND HAS_IPO) set_property(TARGET salmon_core PROPERTY INTERPROCEDURAL_OPTIMIZATION True) endif() @@ -171,14 +177,14 @@ span_FEATURE_MAKE_SPAN_TO_STD=14 target_compile_options(alevin_core PUBLIC "$<$:${TGT_DEBUG_FLAGS}>") target_compile_options(alevin_core PUBLIC "$<$:${TGT_RELEASE_FLAGS}>") -if(NOT CONDA_BUILD) +if(NOT CONDA_BUILD AND HAS_IPO) set_property(TARGET alevin_core PROPERTY INTERPROCEDURAL_OPTIMIZATION True) endif() # Build the salmon executable add_executable(salmon ${SALMON_MAIN_SRCS} ${SALMON_ALIGN_SRCS}) -if(NOT CONDA_BUILD) +if(NOT CONDA_BUILD AND HAS_IPO) set_property(TARGET salmon PROPERTY INTERPROCEDURAL_OPTIMIZATION True) endif() diff --git a/src/CollapsedEMOptimizer.cpp b/src/CollapsedEMOptimizer.cpp index 846f12064..8b7aa4916 100644 --- a/src/CollapsedEMOptimizer.cpp +++ b/src/CollapsedEMOptimizer.cpp @@ -559,6 +559,7 @@ bool CollapsedEMOptimizer::gatherBootstraps( VecT alphasPrime(transcripts.size(), 0.0); VecT expTheta(transcripts.size()); Eigen::VectorXd effLens(transcripts.size()); + double minAlpha = 1e-8; bool scaleCounts = (!sopt.useQuasi and !sopt.allowOrphans); @@ -619,7 +620,7 @@ bool CollapsedEMOptimizer::gatherBootstraps( size_t itNum{0}; // EM termination criterion, adopted from Bray et al. 2016 - double minAlpha = 1e-8; + //double minAlpha = 1e-8; double cutoff = minAlpha; // Since we will use the same weights and transcript groups for each diff --git a/src/SalmonQuantify.cpp b/src/SalmonQuantify.cpp index fcca1e896..e6f4a4498 100644 --- a/src/SalmonQuantify.cpp +++ b/src/SalmonQuantify.cpp @@ -117,8 +117,6 @@ #include "ksw2pp/KSW2Aligner.hpp" #include "metro/metrohash64.h" #include "tsl/hopscotch_map.h" -//#include "tsl/robin_map.h" -//#include "TextBootstrapWriter.hpp" /****** QUASI MAPPING DECLARATIONS *********/ using MateStatus = rapmap::utils::MateStatus; @@ -940,8 +938,8 @@ void processReadsQuasi( auto* qmLog = salmonOpts.qmLog.get(); bool writeQuasimappings = (qmLog != nullptr); - std::string rc1; rc1.reserve(300); - std::string rc2; rc2.reserve(300); + std::string rc1; rc1.reserve(64); + std::string rc2; rc2.reserve(64); // TODO : further investigation of bandwidth and dropoff using ksw2pp::KSW2Aligner; diff --git a/src/SalmonUtils.cpp b/src/SalmonUtils.cpp index e8c7f4fd3..2da137f84 100644 --- a/src/SalmonUtils.cpp +++ b/src/SalmonUtils.cpp @@ -2445,7 +2445,7 @@ int contextSize = outsideContext + insideContext; // lengths we'll consider for this transcript. int32_t locFLDLow = (refLen < cdfMaxArg) ? 1 : fldLow; int32_t locFLDHigh = (refLen < cdfMaxArg) ? cdfMaxArg : fldHigh; - + bool wasProcessed{false}; if (alphas[it] >= minAlpha // available[it] and unprocessedLen > 0 and cdfMaxVal > minCDFMass) { @@ -2614,25 +2614,30 @@ int contextSize = outsideContext + insideContext; effLength += (flWeight * flMassTotal); fl += gcSamp; } + wasProcessed = true; } // for the processed transcript - // throw caution to the wind - double thresh = noThreshold ? 1.0 : unprocessedLen; - - if (noThreshold) { - if (unprocessedLen > 0.0 and effLength > thresh) { - effLensOut(it) = effLength; + if (wasProcessed) { + // throw caution to the wind + double thresh = noThreshold ? 1.0 : unprocessedLen; + if (noThreshold) { + if (unprocessedLen > 0.0 and effLength > thresh) { + effLensOut(it) = effLength; + } else { + effLensOut(it) = effLensIn(it); + } } else { - effLensOut(it) = effLensIn(it); + double offset = std::max(1.0, thresh); + double effLengthNoBias = static_cast(elen); + auto barrierLength = [effLengthNoBias, offset](double x) -> double { + return std::max(x, std::min(effLengthNoBias, offset)); + }; + effLensOut(it) = barrierLength(effLength); } } else { - double offset = std::max(1.0, thresh); - double effLengthNoBias = static_cast(elen); - auto barrierLength = [effLengthNoBias, offset](double x) -> double { - return std::max(x, std::min(effLengthNoBias, offset)); - }; - effLensOut(it) = barrierLength(effLength); + effLensOut(it) = static_cast(elen); } + } } // end parallel_for lambda ); From fa29de22ad3bf8d3fcddb10565d7af3d9eb6de73 Mon Sep 17 00:00:00 2001 From: Rob Patro Date: Sat, 4 Aug 2018 12:54:56 -0400 Subject: [PATCH 4/4] prep for 0.11.2 --- doc/source/salmon.rst | 10 ++++---- include/ksw2pp/KSW2Aligner.hpp | 7 +++++ scripts/fetchRapMap.sh | 6 ++--- src/SalmonQuantify.cpp | 38 +++++++++++++++++++++------ src/SalmonUtils.cpp | 4 +++ src/ksw2pp/KSW2Aligner.cpp | 47 +++++++++++++++++++++++++++++++--- 6 files changed, 92 insertions(+), 20 deletions(-) diff --git a/doc/source/salmon.rst b/doc/source/salmon.rst index 5c702c98a..4a60c1fe5 100644 --- a/doc/source/salmon.rst +++ b/doc/source/salmon.rst @@ -428,11 +428,11 @@ traditional rich equivalence classes. We recommend 4 as a reasonable parameter for this option (it is what was used in the range-factorization paper). """""""""""""" -``--useVBOpt`` +``--useEM`` """""""""""""" -Use the variational Bayesian EM algorithm rather than the "standard" -EM algorithm to optimize abundance estimates. The details of the VBEM +Use the "standard" EM algorithm to optimize abundance estimates +instead of the variational Bayesian EM algorithm. The details of the VBEM algorithm can be found in [#salmon]_. While both the standard EM and the VBEM produce accurate abundance estimates, there are some trade-offs between the approaches. Specifically, the sparsity of @@ -465,8 +465,8 @@ using VB optimization. of the different algorithms is an ongoing area of research. However, preliminary testing suggests that the sparsity-inducing effect of running the VBEM with a small prior may lead, in general, to more accurate estimates (the current testing was - performed mostly through simulation). If these results persist through more - thorough testing, the VBEM may become the default inference mode in future versions of Salmon. + performed mostly through simulation). Hence, the VBEM is the default, and the + standard EM algorithm is accessed via the `--useEM` flag. """"""""""""""""""" diff --git a/include/ksw2pp/KSW2Aligner.hpp b/include/ksw2pp/KSW2Aligner.hpp index be262f12c..636f2fd01 100644 --- a/include/ksw2pp/KSW2Aligner.hpp +++ b/include/ksw2pp/KSW2Aligner.hpp @@ -48,6 +48,13 @@ class KSW2Aligner { KSW2Aligner(int8_t match = 2, int8_t mismatch = -4); KSW2Aligner(std::vector mat); + int transformSequenceKSW2(const char* const queryOriginal, const int queryLength, + std::vector& queryTransformed); + + int transformSequencesKSW2(const char* const queryOriginal, const int queryLength, + const char* const targetOriginal, const int targetLength, + std::vector& queryTransformed, + std::vector& targetTransformed); /** * Variants of the operator that require both an explicit type tag to * determine the type of alignment to perform, as well as a pointer to diff --git a/scripts/fetchRapMap.sh b/scripts/fetchRapMap.sh index a6ea50205..a22cd9a74 100755 --- a/scripts/fetchRapMap.sh +++ b/scripts/fetchRapMap.sh @@ -22,11 +22,11 @@ if [ -d ${INSTALL_DIR}/src/rapmap ] ; then rm -fr ${INSTALL_DIR}/src/rapmap fi -#SVER=salmon-v0.11.1 -SVER=develop-salmon +SVER=salmon-v0.11.2 +#SVER=develop-salmon #SVER=pe-chaining -EXPECTED_SHA256=489c365f859b647564374830325e2a9407a8d5d066b2c4e89f3f671c3a39221b +EXPECTED_SHA256=ba6fcff2e13c06a63972c5a2da97b712b9fbc4c3cb70fd77ed04234c98b8bc89 mkdir -p ${EXTERNAL_DIR} diff --git a/src/SalmonQuantify.cpp b/src/SalmonQuantify.cpp index e6f4a4498..ed8ca9dcf 100644 --- a/src/SalmonQuantify.cpp +++ b/src/SalmonQuantify.cpp @@ -1118,11 +1118,10 @@ void processReadsQuasi( auto* r2 = rp.second.seq.data(); auto l1 = static_cast(rp.first.seq.length()); auto l2 = static_cast(rp.second.seq.length()); - rapmap::utils::reverseRead(rp.first.seq, rc1); - rapmap::utils::reverseRead(rp.second.seq, rc2); - // we will not break the const promise - char* r1rc = const_cast(rc1.data()); - char* r2rc = const_cast(rc2.data()); + // We compute the reverse complements below only if we + // need them and don't have them. + char* r1rc = nullptr; + char* r2rc = nullptr; int32_t bestScore{-1}; std::vector scores(jointHits.size(), bestScore); size_t idx{0}; @@ -1138,6 +1137,14 @@ void processReadsQuasi( const uint32_t buf{8}; if (h.mateStatus == rapmap::utils::MateStatus::PAIRED_END_PAIRED) { + if (!h.fwd and !r1rc) { + rapmap::utils::reverseRead(rp.first.seq, rc1); + r1rc = const_cast(rc1.data()); + } + if (!h.mateIsFwd and !r2rc) { + rapmap::utils::reverseRead(rp.second.seq, rc2); + r2rc = const_cast(rc2.data()); + } auto* r1ptr = h.fwd ? r1 : r1rc; auto* r2ptr = h.mateIsFwd ? r2 : r2rc; @@ -1153,6 +1160,10 @@ void processReadsQuasi( score = s1 + s2; } } else if (h.mateStatus == rapmap::utils::MateStatus::PAIRED_END_LEFT) { + if (!h.fwd and !r1rc) { + rapmap::utils::reverseRead(rp.first.seq, rc1); + r1rc = const_cast(rc1.data()); + } auto* rptr = h.fwd ? r1 : r1rc; int32_t s = @@ -1163,6 +1174,10 @@ void processReadsQuasi( score = s; } } else if (h.mateStatus == rapmap::utils::MateStatus::PAIRED_END_RIGHT) { + if (!h.fwd and !r2rc) { + rapmap::utils::reverseRead(rp.second.seq, rc2); + r2rc = const_cast(rc2.data()); + } auto* rptr = h.fwd ? r2 : r2rc; int32_t s = @@ -1595,9 +1610,8 @@ void processReadsQuasi( alnCache.clear(); auto* r1 = rp.seq.data(); auto l1 = static_cast(rp.seq.length()); - rapmap::utils::reverseRead(rp.seq, rc1); - // we will not break the const promise - char* r1rc = const_cast(rc1.data()); + + char* r1rc = nullptr; int32_t bestScore{std::numeric_limits::min()}; std::vector scores(jointHits.size(), bestScore); size_t idx{0}; @@ -1611,6 +1625,14 @@ void processReadsQuasi( const int32_t tlen = static_cast(t.RefLength); const uint32_t buf{8}; + // compute the reverse complement only if we + // need it and don't have it + if (!h.fwd and !r1rc) { + rapmap::utils::reverseRead(rp.seq, rc1); + // we will not break the const promise + r1rc = const_cast(rc1.data()); + } + auto* rptr = h.fwd ? r1 : r1rc; int32_t s = getAlnScore(aligner, ez, h.pos, rptr, l1, tseq, tlen, a, b, maxReadScore, h.chainStatus.getLeft(), buf, alnCache); diff --git a/src/SalmonUtils.cpp b/src/SalmonUtils.cpp index 2da137f84..6782eb09f 100644 --- a/src/SalmonUtils.cpp +++ b/src/SalmonUtils.cpp @@ -2666,6 +2666,10 @@ void aggregateEstimatesToGeneLevel(TranscriptGeneMap& tgm, auto logger = spdlog::get("jointLog"); + logger->info("NOTE: We recommend using tximport (https://bioconductor.org/packages/release/bioc/html/tximport.html) " + "for aggregating transcript-level salmon abundance estimates to the gene level. It is more versatile, " + "exposes more features, and allows considering multi-sample information during aggregation."); + constexpr double minTPM = std::numeric_limits::denorm_min(); std::ifstream expFile(inputPath.string()); diff --git a/src/ksw2pp/KSW2Aligner.cpp b/src/ksw2pp/KSW2Aligner.cpp index 8d5ab3ea4..ebc67a126 100644 --- a/src/ksw2pp/KSW2Aligner.cpp +++ b/src/ksw2pp/KSW2Aligner.cpp @@ -127,8 +127,47 @@ KSW2Aligner::KSW2Aligner(std::vector mat) { * length - 1]. * @return Alphabet length - number of letters in recognized alphabet. */ -static int -transformSequencesDNA(const char* const queryOriginal, const int queryLength, +int +KSW2Aligner::transformSequenceKSW2(const char* const queryOriginal, const int queryLength, + std::vector& queryTransformed) { + // Alphabet is constructed from letters that are present in sequences. + // Each letter is assigned an ordinal number, starting from 0 up to + // alphabetLength - 1, and new query and target are created in which letters + // are replaced with their ordinal numbers. This query and target are used in + // all the calculations later. + queryTransformed.resize(queryLength, 0); + + int i = 0; + for (int i = 0; i < queryLength; i++) { + uint8_t c = static_cast(queryOriginal[i]); + queryTransformed[i] = seq_nt4_table_loc[c]; + } + return 4; +} + +/** + * from https://github.com/rob-p/edlib/blob/read-aligner/edlib/src/edlib.cpp + * Takes char query and char target, recognizes alphabet and transforms them + * into unsigned char sequences where elements in sequences are not any more + * letters of alphabet, but their index in alphabet. Most of internal edlib + * functions expect such transformed sequences. This function will allocate + * queryTransformed and targetTransformed, so make sure to free them when done. + * Example: + * Original sequences: "ACT" and "CGT". + * Alphabet would be recognized as ['A', 'C', 'T', 'G']. Alphabet length = 4. + * Transformed sequences: [0, 1, 2] and [1, 3, 2]. + * @param [in] queryOriginal + * @param [in] queryLength + * @param [in] targetOriginal + * @param [in] targetLength + * @param [out] queryTransformed It will contain values in range [0, alphabet + * length - 1]. + * @param [out] targetTransformed It will contain values in range [0, alphabet + * length - 1]. + * @return Alphabet length - number of letters in recognized alphabet. + */ +int +KSW2Aligner::transformSequencesKSW2(const char* const queryOriginal, const int queryLength, const char* const targetOriginal, const int targetLength, std::vector& queryTransformed, std::vector& targetTransformed) { @@ -172,7 +211,7 @@ int KSW2Aligner::operator()(const char* const queryOriginal, // auto ez = &result_; auto qlen = queryLength; auto tlen = targetLength; - int asize = transformSequencesDNA(queryOriginal, queryLength, targetOriginal, + int asize = transformSequencesKSW2(queryOriginal, queryLength, targetOriginal, targetLength, query_, target_); (void)asize; int8_t q = config_.gapo; @@ -231,7 +270,7 @@ int KSW2Aligner::operator()(const char* const queryOriginal, // auto ez = &result_; auto qlen = queryLength; auto tlen = targetLength; - int asize = transformSequencesDNA(queryOriginal, queryLength, targetOriginal, + int asize = transformSequencesKSW2(queryOriginal, queryLength, targetOriginal, targetLength, query_, target_); (void)asize; int q = config_.gapo;