diff --git a/CMakeLists.txt b/CMakeLists.txt index bb52c1b..7a6dce1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,8 +44,12 @@ if(PROJECT_IS_TOP_LEVEL AND BUILD_TESTING) if(NOT WIN32) hashmap_executable(HashMapBenchmark src/HashMapBenchmark.cpp) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$") - target_compile_options(HashMapBenchmark PRIVATE -msse4.2) + if(NOT CMAKE_CROSSCOMPILING) + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-march=native" HASHMAP_HAS_MARCH_NATIVE) + if(HASHMAP_HAS_MARCH_NATIVE) + target_compile_options(HashMapBenchmark PRIVATE -march=native) + endif() endif() option(HASHMAP_BENCHMARK_WITH_ABSL "Use installed Abseil when available" ON) @@ -67,6 +71,16 @@ if(PROJECT_IS_TOP_LEVEL AND BUILD_TESTING) endif() endif() + option(HASHMAP_BENCHMARK_WITH_BOOST "Use installed Boost.Unordered when available" ON) + if(HASHMAP_BENCHMARK_WITH_BOOST) + find_path(HASHMAP_BOOST_INCLUDE_DIR NAMES boost/unordered/unordered_flat_map.hpp) + mark_as_advanced(HASHMAP_BOOST_INCLUDE_DIR) + if(HASHMAP_BOOST_INCLUDE_DIR) + target_include_directories(HashMapBenchmark SYSTEM PRIVATE "${HASHMAP_BOOST_INCLUDE_DIR}") + target_compile_definitions(HashMapBenchmark PRIVATE HASHMAP_BENCHMARK_HAS_BOOST) + endif() + endif() + add_test(NAME HashMapBenchmarkSmoke COMMAND HashMapBenchmark -t 4 -c 1000 -i 1000) endif() endif() diff --git a/README.md b/README.md index f032f5f..cddc99c 100644 --- a/README.md +++ b/README.md @@ -110,15 +110,17 @@ and deleted. The benchmark uses `std::chrono::steady_clock` and requires no benchmark framework or vendored dependencies. CMake enables comparisons against installed -Abseil and sparsehash when found. Set `HASHMAP_BENCHMARK_WITH_ABSL=OFF` or -`HASHMAP_BENCHMARK_WITH_SPARSEHASH=OFF` to disable either comparison. Abseil must -provide its CMake package; sparsehash must provide `google/dense_hash_map`. -No dependencies are downloaded during configuration. +Abseil, sparsehash, and Boost.Unordered when found. Set +`HASHMAP_BENCHMARK_WITH_ABSL=OFF`, `HASHMAP_BENCHMARK_WITH_SPARSEHASH=OFF`, or +`HASHMAP_BENCHMARK_WITH_BOOST=OFF` to disable any comparison. Abseil must +provide its CMake package; sparsehash must provide `google/dense_hash_map`; +Boost must provide `boost/unordered/unordered_flat_map.hpp`. No dependencies are +downloaded during configuration. The benchmark is built on Linux and macOS; Windows builds the example and tests. -All containers in a run use the same hash: hardware CRC32 on x86-64 and -MurmurHash3's `fmix64` integer mixer on ARM and other architectures. Hash choices -differ across architectures, so compare containers within the same run. +Each container uses its default hash (`HashMap` automatically mixes hashes that +do not define `is_avalanching`), and `google::dense_hash_map` uses `HashMap`'s +mixer. I ran this benchmark on the following configuration: diff --git a/include/rigtorp/HashMap.h b/include/rigtorp/HashMap.h index c336032..ccea046 100644 --- a/include/rigtorp/HashMap.h +++ b/include/rigtorp/HashMap.h @@ -26,6 +26,7 @@ probing. #include #include +#include #include #include #include @@ -33,6 +34,10 @@ probing. #include #include +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) +#include +#endif + namespace rigtorp { namespace detail { @@ -53,6 +58,39 @@ namespace detail { } #undef RIGTORP_HASHMAP_NOINLINE + +template +inline constexpr bool is_avalanching_v = + requires { typename Hash::is_avalanching; }; + +inline size_t mix_hash(size_t h) noexcept { + if constexpr (sizeof(size_t) >= 8) { +#if defined(__SIZEOF_INT128__) + __extension__ using uint128_t = unsigned __int128; + const uint128_t r = + static_cast(h) * UINT64_C(0x9ddfea08eb382d69); + return static_cast(r) ^ static_cast(r >> 64); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned __int64 hi = 0; + const unsigned __int64 lo = _umul128(h, 0x9ddfea08eb382d69ULL, &hi); + return static_cast(lo ^ hi); +#else + h ^= h >> 30; + h *= 0xbf58476d1ce4e5b9ULL; + h ^= h >> 27; + h *= 0x94d049bb133111ebULL; + h ^= h >> 31; + return h; +#endif + } else { + h ^= h >> 16; + h *= 0x85ebca6bU; + h ^= h >> 13; + h *= 0xc2b2ae35U; + h ^= h >> 16; + return h; + } +} } // namespace detail template , @@ -350,8 +388,12 @@ class HashMap { template size_t key_to_idx(const K &key) const noexcept(noexcept(hasher()(key))) { + size_t h = hasher()(key); + if constexpr (!detail::is_avalanching_v) { + h = detail::mix_hash(h); + } const size_t mask = buckets_.size() - 1; - return hasher()(key) & mask; + return h & mask; } size_t probe_next(size_t idx) const noexcept { diff --git a/src/HashMapBenchmark.cpp b/src/HashMapBenchmark.cpp index 8267e66..f33d816 100644 --- a/src/HashMapBenchmark.cpp +++ b/src/HashMapBenchmark.cpp @@ -1,16 +1,19 @@ // © 2017-2020 Erik Rigtorp // SPDX-License-Identifier: MIT -#if defined(__x86_64__) || defined(_M_X64) -#include // _mm_crc32_u64 -#endif - +#include #include #include +#include #include -#include +#include +#include +#include +#include +#include #include #include +#include #if defined(HASHMAP_BENCHMARK_HAS_SPARSEHASH) #include @@ -20,6 +23,11 @@ #include #endif +#if defined(HASHMAP_BENCHMARK_HAS_BOOST) +#include +#include +#endif + #include #if __has_include() @@ -75,134 +83,521 @@ template using huge_page_allocator = std::allocator; using namespace std::chrono; using namespace rigtorp; -int main(int argc, char *argv[]) { - (void)argc, (void)argv; +namespace { - size_t count = 10000000; - size_t iters = 100000000; - int type = -1; +inline size_t g_alloc_bytes = 0; +inline size_t g_peak_bytes = 0; - int opt; - while ((opt = getopt(argc, argv, "i:c:t:")) != -1) { - switch (opt) { - case 'i': - iters = std::stoul(optarg); - break; - case 'c': - count = std::stoul(optarg); - break; - case 't': - type = std::stoi(optarg); - break; - default: - goto usage; +struct memory_scope { + memory_scope() noexcept { + g_alloc_bytes = 0; + g_peak_bytes = 0; + } + static size_t current() noexcept { return g_alloc_bytes; } + static size_t peak() noexcept { return g_peak_bytes; } +}; + +template class BaseAlloc> +struct tracking_allocator { + using value_type = T; + using is_always_equal = std::true_type; + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + + template struct rebind { + using other = tracking_allocator; + }; + + tracking_allocator() = default; + template + constexpr tracking_allocator( + const tracking_allocator &) noexcept {} + + T *allocate(std::size_t n) { + T *p = BaseAlloc{}.allocate(n); + g_alloc_bytes += n * sizeof(T); + if (g_alloc_bytes > g_peak_bytes) { + g_peak_bytes = g_alloc_bytes; } + return p; } - if (optind != argc) { - usage: - std::cerr << "HashMapBenchmark © 2020 Erik Rigtorp \n" - "usage: HashMapBenchmark [-c count] [-i iters] [-t 1|2|3|4]\n" - << std::endl; - exit(1); + void deallocate(T *p, std::size_t n) noexcept { + g_alloc_bytes -= n * sizeof(T); + BaseAlloc{}.deallocate(p, n); } +}; - using key = size_t; - struct value { - char buf[24]; - }; +template class BaseAlloc> +bool operator==(const tracking_allocator &, + const tracking_allocator &) noexcept { + return true; +} + +template class BaseAlloc> +bool operator!=(const tracking_allocator &, + const tracking_allocator &) noexcept { + return false; +} + +inline uint64_t splitmix64(uint64_t &state) noexcept { + state += UINT64_C(0x9e3779b97f4a7c15); + uint64_t z = state; + z = (z ^ (z >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); + z = (z ^ (z >> 27)) * UINT64_C(0x94d049bb133111eb); + return z ^ (z >> 31); +} - struct hash { - size_t operator()(size_t h) const noexcept { -#if defined(__x86_64__) || defined(_M_X64) - return _mm_crc32_u64(0, h); +inline size_t fast_range(uint64_t r, size_t n) noexcept { +#if defined(__SIZEOF_INT128__) + __extension__ using uint128_t = unsigned __int128; + return static_cast((static_cast(r) * n) >> 64); #else - // MurmurHash3's public-domain fmix64 finalizer, by Austin Appleby: - // https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp - uint64_t x = h; - x ^= x >> 33; - x *= UINT64_C(0xff51afd7ed558ccd); - x ^= x >> 33; - x *= UINT64_C(0xc4ceb9fe1a85ec53); - x ^= x >> 33; - return static_cast(x); + return static_cast(r % n); #endif +} + +template +void run_timed(const std::string &prefix, const char *workload, + const char *name, size_t ops, Fn &&fn) { + auto start = steady_clock::now(); + fn(); + auto stop = steady_clock::now(); + auto ns = duration_cast(stop - start).count(); + std::cout << prefix << workload << " / " << name << ": " + << ns / static_cast(ops) << " ns/op" << std::endl; +} + +template struct key_traits; + +template <> struct key_traits { + static constexpr const char *label = "int"; + static size_t empty_key() noexcept { return 0; } + static size_t deleted_key() noexcept { return 1; } + static size_t make_present(size_t idx) noexcept { return idx + 2; } + static size_t make_missing(size_t idx, size_t count) noexcept { + return count + idx + 2; + } + static size_t value_tag(size_t k) noexcept { return k; } +}; + +template <> struct key_traits { + static constexpr const char *label = "string"; + static std::string empty_key() { return {}; } + static std::string deleted_key() { return "\x01"; } + static std::string format_key(size_t id) { + // 24-byte string to exceed typical SSO buffers. + std::string s = "key_00000000000000000000"; + for (size_t pos = s.size(); pos > 4 && id > 0; --pos) { + s[pos - 1] = static_cast('0' + (id % 10)); + id /= 10; } - }; + return s; + } + static std::string make_present(size_t idx) { return format_key(idx + 1); } + static std::string make_missing(size_t idx, size_t count) { + return format_key(count + idx + 1); + } + static size_t value_tag(const std::string &k) noexcept { return k.size(); } +}; - auto b = [&](const char *n, auto &m) { - std::minstd_rand gen(0); - std::uniform_int_distribution ud(2, count); +#if defined(HASHMAP_BENCHMARK_HAS_SPARSEHASH) +template struct dense_hash { + size_t operator()(const Key &k) const noexcept { + return rigtorp::detail::mix_hash(std::hash{}(k)); + } +}; +#endif +template class BaseAlloc> +void run_benchmarks(size_t count, size_t iters, int type, + const std::string &workload, const std::string &prefix) { + struct value { + Key next; + char buf[16]; + }; + + std::vector order(count); + std::vector erase_order(count); + std::vector miss_keys; + for (size_t i = 0; i < count; ++i) { + order[i] = key_traits::make_present(i); + } + { + uint64_t rng = 0; + for (size_t i = count; i > 1; --i) { + const size_t j = fast_range(splitmix64(rng), i); + std::swap(order[i - 1], order[j]); + } + } + erase_order = order; + { + uint64_t rng = 1; + for (size_t i = count; i > 1; --i) { + const size_t j = fast_range(splitmix64(rng), i); + std::swap(erase_order[i - 1], erase_order[j]); + } + } + if constexpr (!std::is_same_v) { + miss_keys.resize(count); for (size_t i = 0; i < count; ++i) { - const key val = ud(gen); - m.insert({val, {}}); - } - - auto start = steady_clock::now(); - for (size_t i = 0; i < iters; ++i) { - const key val = ud(gen); - const auto it = m.find(val); - if (it == m.end()) { - m.insert({val, {}}); - } else { - m.erase(it); + miss_keys[i] = key_traits::make_missing(i, count); + } + } + + auto bench_map = [&](const char *name, auto make_map) { + if (workload == "all" || workload == "churn") { + auto m = make_map(count); + uint64_t rng = 0; + for (size_t i = 0; i < count; ++i) { + const size_t idx = fast_range(splitmix64(rng), count); + if constexpr (std::is_same_v) { + const Key k = key_traits::make_present(idx); + m.insert({k, {k, {}}}); + } else { + m.insert({order[idx], {order[idx], {}}}); + } } + run_timed(prefix, "churn", name, iters, [&] { + for (size_t i = 0; i < iters; ++i) { + const size_t idx = fast_range(splitmix64(rng), count); + if constexpr (std::is_same_v) { + const Key k = key_traits::make_present(idx); + const auto it = m.find(k); + if (it == m.end()) { + m.insert({k, {k, {}}}); + } else { + m.erase(it); + } + } else { + const Key &k = order[idx]; + const auto it = m.find(k); + if (it == m.end()) { + m.insert({k, {k, {}}}); + } else { + m.erase(it); + } + } + } + }); } - auto stop = steady_clock::now(); - auto duration = stop - start; - - nanoseconds max = {}; - for (size_t i = 0; i < iters; ++i) { - const key val = ud(gen); - auto iter_start = steady_clock::now(); - const auto it = m.find(val); - if (it == m.end()) { - m.insert({val, {}}); - } else { - m.erase(it); + + if (workload == "all" || workload == "hit-throughput") { + auto m = make_map(count); + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); } - auto iter_stop = steady_clock::now(); - max = std::max(max, iter_stop - iter_start); + uint64_t rng = 0; + size_t sum = 0; + run_timed(prefix, "hit-throughput", name, iters, [&] { + for (size_t i = 0; i < iters; ++i) { + const size_t idx = fast_range(splitmix64(rng), count); + if constexpr (std::is_same_v) { + const Key k = key_traits::make_present(idx); + sum += key_traits::value_tag(m.find(k)->second.next); + } else { + sum += key_traits::value_tag(m.find(order[idx])->second.next); + } + } + }); + volatile size_t sink = sum; + (void)sink; } - std::cout << n << ": mean " - << duration_cast(duration).count() / - static_cast(iters) - << " ns/iter, max " << max.count() << " ns/iter" << std::endl; + if (workload == "all" || workload == "hit-latency") { + auto m = make_map(count); + for (size_t i = 0; i < count; ++i) { + const Key &next_k = order[(i + 1) % count]; + m.insert({order[i], {next_k, {}}}); + } + Key k = order[0]; + run_timed(prefix, "hit-latency", name, iters, [&] { + for (size_t i = 0; i < iters; ++i) { + k = m.find(k)->second.next; + } + }); + volatile size_t sink = key_traits::value_tag(k); + (void)sink; + } + + if (workload == "all" || workload == "miss") { + auto m = make_map(count); + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); + } + uint64_t rng = 0; + size_t hits = 0; + run_timed(prefix, "miss", name, iters, [&] { + for (size_t i = 0; i < iters; ++i) { + const size_t idx = fast_range(splitmix64(rng), count); + if constexpr (std::is_same_v) { + const Key k = key_traits::make_missing(idx, count); + if (m.find(k) != m.end()) { + ++hits; + } + } else { + if (m.find(miss_keys[idx]) != m.end()) { + ++hits; + } + } + } + }); + volatile size_t sink = hits; + (void)sink; + } + + if (workload == "all" || workload == "insert") { + auto m = make_map(0); + run_timed(prefix, "insert", name, count, [&] { + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); + } + }); + } + + if (workload == "all" || workload == "insert-reserved") { + auto m = make_map(count); + run_timed(prefix, "insert-reserved", name, count, [&] { + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); + } + }); + } + + if (workload == "all" || workload == "insert-dup") { + auto m = make_map(count); + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); + } + run_timed(prefix, "insert-dup", name, count, [&] { + for (size_t i = 0; i < count; ++i) { + m.insert({erase_order[i], {erase_order[i], {}}}); + } + }); + } + + if (workload == "all" || workload == "erase") { + auto m = make_map(count); + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); + } + run_timed(prefix, "erase", name, count, [&] { + for (size_t i = 0; i < count; ++i) { + m.erase(erase_order[i]); + } + }); + } + + if (workload == "all" || workload == "iterate") { + auto m = make_map(count); + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); + } + const size_t passes = + std::max(1, iters / std::max(1, count)); + size_t sum = 0; + run_timed(prefix, "iterate", name, passes * count, [&] { + for (size_t p = 0; p < passes; ++p) { + for (const auto &kv : m) { + sum += key_traits::value_tag(kv.second.next); + } + } + }); + volatile size_t sink = sum; + (void)sink; + } + + if (workload == "all" || workload == "memory") { + memory_scope scope; + auto m = make_map(count); + for (size_t i = 0; i < count; ++i) { + m.insert({order[i], {order[i], {}}}); + } + const double bytes_per_elem = + static_cast(scope.current()) / static_cast(count); + const double mib = + static_cast(scope.current()) / (1024.0 * 1024.0); + std::cout << prefix << "memory / " << name << ": " << std::fixed + << std::setprecision(1) << bytes_per_elem << " B/elem (" << mib + << " MiB)" << std::endl; + } }; if (type == -1 || type == 1) { - HashMap, - huge_page_allocator>> - hm(2 * count, 0); - b("HashMap", hm); + bench_map("HashMap", [](size_t reserve_count) { + HashMap, std::equal_to<>, + tracking_allocator, BaseAlloc>> + hm(2, key_traits::empty_key()); + if (reserve_count > 0) { + hm.reserve(reserve_count); + } + return hm; + }); } #if defined(HASHMAP_BENCHMARK_HAS_SPARSEHASH) if (type == -1 || type == 2) { - // Couldn't get it to work with the huge_page_allocator - google::dense_hash_map hm(count); - hm.set_empty_key(0); - hm.set_deleted_key(1); - b("google::dense_hash_map", hm); + bench_map("google::dense_hash_map", [](size_t reserve_count) { + // Couldn't get it to work with the huge_page_allocator + google::dense_hash_map< + Key, value, dense_hash, std::equal_to, + tracking_allocator, std::allocator>> + hm(reserve_count); + hm.set_empty_key(key_traits::empty_key()); + hm.set_deleted_key(key_traits::deleted_key()); + return hm; + }); } #endif #if defined(HASHMAP_BENCHMARK_HAS_ABSL) if (type == -1 || type == 3) { - absl::flat_hash_map, - huge_page_allocator>> - hm; - hm.reserve(count); - b("absl::flat_hash_map", hm); + bench_map("absl::flat_hash_map", [](size_t reserve_count) { + absl::flat_hash_map< + Key, value, absl::Hash, std::equal_to<>, + tracking_allocator, BaseAlloc>> + hm; + if (reserve_count > 0) { + hm.reserve(reserve_count); + } + return hm; + }); } #endif if (type == -1 || type == 4) { - std::unordered_map hm; - hm.reserve(count); - b("std::unordered_map", hm); + bench_map("std::unordered_map", [](size_t reserve_count) { + std::unordered_map< + Key, value, std::hash, std::equal_to, + tracking_allocator, std::allocator>> + hm; + if (reserve_count > 0) { + hm.reserve(reserve_count); + } + return hm; + }); + } + +#if defined(HASHMAP_BENCHMARK_HAS_BOOST) + if (type == -1 || type == 5) { + bench_map("boost::unordered_flat_map", [](size_t reserve_count) { + boost::unordered_flat_map< + Key, value, boost::hash, std::equal_to<>, + tracking_allocator, BaseAlloc>> + hm; + if (reserve_count > 0) { + hm.reserve(reserve_count); + } + return hm; + }); + } + + if (type == -1 || type == 6) { + bench_map("boost::unordered_map", [](size_t reserve_count) { + boost::unordered_map< + Key, value, boost::hash, std::equal_to, + tracking_allocator, std::allocator>> + hm; + if (reserve_count > 0) { + hm.reserve(reserve_count); + } + return hm; + }); + } +#endif +} + +template