From 9d46be270a78d7856c900f47fd7832937019970f Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 31 May 2026 16:41:56 -0400 Subject: [PATCH 01/20] Start implementing blocked leaf pages. --- conan.lock | 10 +- conanfile.py | 1 + src/turtle_kv/core/edit_view.hpp | 8 + src/turtle_kv/core/key_view.hpp | 9 + src/turtle_kv/core/packed_key_value_slot.hpp | 184 ++++++++++++++++++ src/turtle_kv/core/value_view.hpp | 10 + src/turtle_kv/import/buffer.hpp | 5 +- src/turtle_kv/mem_table/mem_table.test.cpp | 6 +- src/turtle_kv/tree/packed_leaf_block.hpp | 175 +++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.ipp | 100 ++++++++++ src/turtle_kv/tree/packed_leaf_block.test.cpp | 121 ++++++++++++ src/turtle_kv/tree/packed_leaf_page2.hpp | 37 ++++ src/turtle_kv/tree/packed_node_page.cpp | 6 +- 13 files changed, 658 insertions(+), 14 deletions(-) create mode 100644 src/turtle_kv/tree/packed_leaf_block.hpp create mode 100644 src/turtle_kv/tree/packed_leaf_block.ipp create mode 100644 src/turtle_kv/tree/packed_leaf_block.test.cpp create mode 100644 src/turtle_kv/tree/packed_leaf_page2.hpp diff --git a/conan.lock b/conan.lock index 13b562b..1607f64 100644 --- a/conan.lock +++ b/conan.lock @@ -2,7 +2,8 @@ "version": "0.5", "requires": [ "abseil/20250127.0", - "batteries/0.70.2", + "artc/0.0.1.dev", + "batteries/0.70.4.dev2", "boost/1.88.0", "bzip2/1.0.8", "cli11/2.5.0", @@ -19,7 +20,7 @@ "openssl/3.6.0", "pcg-cpp/cci.20220409", "protobuf/3.21.12", - "vqf/0.2.5", + "vqf/0.2.6", "xxhash/0.8.3", "yaml-cpp/0.9.0", "zlib/1.3.1" @@ -32,8 +33,7 @@ "ninja/1.13.2" ], "python_requires": [ - "cor_recipe_utils/0.18.2", - "cor_recipe_utils/0.8.7" + "cor_recipe_utils/0.19.1" ], "overrides": { "libunwind/[>=1.8 <2]": [ @@ -52,7 +52,7 @@ "boost/1.88.0" ], "batteries/[>=0.60.2 <2]": [ - "batteries/0.70.2" + "batteries/0.70.4.dev2" ] }, "config_requires": [] diff --git a/conanfile.py b/conanfile.py index 31cc052..6892cf6 100644 --- a/conanfile.py +++ b/conanfile.py @@ -85,6 +85,7 @@ def requirements(self): } self.requires("abseil/20250127.0", **VISIBLE, **OVERRIDE) + self.requires("artc/[>=0.0.1 <1]") self.requires("batteries/[>=0.70.2 <1]", **VISIBLE, **OVERRIDE) self.requires("boost/1.88.0", **VISIBLE, **OVERRIDE) self.requires("glog/0.7.1", **VISIBLE) diff --git a/src/turtle_kv/core/edit_view.hpp b/src/turtle_kv/core/edit_view.hpp index b5e6f62..9fc9c20 100644 --- a/src/turtle_kv/core/edit_view.hpp +++ b/src/turtle_kv/core/edit_view.hpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #pragma once #include diff --git a/src/turtle_kv/core/key_view.hpp b/src/turtle_kv/core/key_view.hpp index 7c91fa8..bbf16b6 100644 --- a/src/turtle_kv/core/key_view.hpp +++ b/src/turtle_kv/core/key_view.hpp @@ -85,4 +85,13 @@ inline usize packed_key_data_size(const KeyView& key) return key.size(); } +template +concept HasKeyView = requires(const T& obj) { + { get_key(obj) } -> std::convertible_to; +}; + +static_assert(HasKeyView); +static_assert(HasKeyView); +static_assert(HasKeyView); + } // namespace turtle_kv diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index 9bc08d2..0f0d2d2 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -17,8 +17,146 @@ #include #include +#include + namespace turtle_kv { +struct PackedKeyValueSlot; + +using PackedKeyValueSlotPtr = llfs::PackedPointer; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedKeyValueSlot { + little_u16 key_size; + char key_data_[0]; + + //----- --- -- - - - - + // u8 key_bytes[this->key_size] + //----- --- -- - - - - + // u8 op_code + // u8 value_bytes[this->item_size - offsetof(this->value_bytes)] + //----- --- -- - - - - + + //+++++++++++-+-+--+----- --- -- - - - - + + PackedKeyValueSlot(const PackedKeyValueSlot&) = delete; + PackedKeyValueSlot& operator=(const PackedKeyValueSlot&) = delete; + + //+++++++++++-+-+--+----- --- -- - - - - + + usize slot_size(const PackedKeyValueSlotPtr* p_this) const noexcept + { + const PackedKeyValueSlotPtr* const p_next = p_this + 1; + return usize{p_next->offset.value()} - usize{p_this->offset.value()} + + sizeof(PackedKeyValueSlotPtr); + } + + const char* key_data() const noexcept + { + return this->key_data_; + } + + KeyView key_view() const noexcept + { + return KeyView{this->key_data_, this->key_size}; + } + + const char* value_data() const noexcept + { + return this->key_data() + (this->key_size + 1); + } + + const char* value_data_end(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_data_end(/*size_of_slot=*/this->slot_size(p_this)); + } + + const char* value_data_end(usize size_of_slot) const noexcept + { + return reinterpret_cast(this) + size_of_slot; + } + + usize value_size(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_size(/*size_of_slot=*/this->slot_size(p_this)); + } + + usize value_size(usize size_of_slot) const noexcept + { + return this->value_data_end(size_of_slot) - this->value_data(); + } + + ValueView::OpCode value_op_code() const noexcept + { + return static_cast(this->key_data_[this->key_size]); + } + + ValueView value_view(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_view(/*size_of_slot=*/this->slot_size(p_this)); + } + + ValueView value_view(usize size_of_slot) const noexcept + { + return ValueView::from_packed( + this->value_op_code(), + std::string_view{this->value_data(), this->value_size(size_of_slot)}); + } +}; + +inline KeyView get_key(const PackedKeyValueSlot& packed_slot) noexcept +{ + return packed_slot.key_view(); +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// TODO [tastolfi 2026-05-31] use 16-bit pointer tagging to store slot_size with the pointer in one +// 64-bit word. +// +struct PackedKeyValueSlotRef { + const PackedKeyValueSlot* p_slot; + usize slot_size; +}; + +inline KeyView get_key(const PackedKeyValueSlotRef& slot_ref) noexcept +{ + return slot_ref.p_slot->key_view(); +} + +inline ValueView get_value(const PackedKeyValueSlotRef& slot_ref) noexcept +{ + return slot_ref.p_slot->value_view(slot_ref.slot_size); +} + +inline const PackedKeyValueSlotRef& to_key_value_slot_ref(const PackedKeyValueSlotRef& ref) noexcept +{ + return ref; +} + +inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr* pp_slot) noexcept +{ + return PackedKeyValueSlotRef{ + .p_slot = pp_slot->get(), + .slot_size = pp_slot->get()->slot_size(pp_slot), + }; +} + +inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffer) noexcept +{ + return PackedKeyValueSlotRef{ + .p_slot = static_cast(slot_buffer.data()), + .slot_size = slot_buffer.size(), + }; +} + +template +concept ConvertibleToKeyValueSlotRef = requires(const T& obj) { + { to_key_value_slot_ref(obj) } -> std::convertible_to; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Returns the size required (in bytes) to pack a slot with the passed key and * value. */ @@ -30,6 +168,21 @@ inline usize packed_key_value_slot_size(const KeyView& key, const ValueView& val + value.size(); } +template +inline usize packed_key_value_slot_size(const T& obj) noexcept +{ + return to_key_value_slot_ref(obj).slot_size; +} + +template + requires HasKeyView && HasValueView +inline usize packed_key_value_slot_size(const T& obj) noexcept +{ + return packed_key_value_slot_size(get_key(obj), get_value(obj)); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Serializes the passed key and value into the destination buffer. */ inline std::pair pack_key_value_slot(const KeyView& key, @@ -72,6 +225,37 @@ inline std::pair pack_key_value_slot(const KeyView& key, })); } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief + */ +template + requires HasKeyView && HasValueView +inline usize pack_key_value_slot(const T& src, void* dst) noexcept +{ + const KeyView& key = get_key(src); + const ValueView& value = get_value(src); + const usize slot_size = packed_key_value_slot_size(key, value); + + pack_key_value_slot(key, value, MutableBuffer{dst, slot_size}); + + return slot_size; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief + */ +template +inline usize pack_key_value_slot(const T& src, void* dst) noexcept +{ + const PackedKeyValueSlotRef& slot_ref = to_key_value_slot_ref(src); + std::memcpy(dst, slot_ref.p_slot, slot_ref.slot_size); + return slot_ref.slot_size; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Unpacks a key/value pair from the passed packed slot buffer. */ inline StatusOr> unpack_key_value_slot(ConstBuffer payload) diff --git a/src/turtle_kv/core/value_view.hpp b/src/turtle_kv/core/value_view.hpp index e79673f..3fb4d7b 100644 --- a/src/turtle_kv/core/value_view.hpp +++ b/src/turtle_kv/core/value_view.hpp @@ -404,4 +404,14 @@ inline bool decays_to_item(const ValueView& value) return false; } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// + +template +concept HasValueView = requires(const T& obj) { + { get_value(obj) } -> std::convertible_to; +}; + +static_assert(HasValueView); + } // namespace turtle_kv diff --git a/src/turtle_kv/import/buffer.hpp b/src/turtle_kv/import/buffer.hpp index 8640034..0d712e7 100644 --- a/src/turtle_kv/import/buffer.hpp +++ b/src/turtle_kv/import/buffer.hpp @@ -6,14 +6,13 @@ namespace turtle_kv { +using batt::advance_pointer; using batt::buffer_from_struct; +using batt::byte_distance; using batt::ConstBuffer; using batt::make_buffer; using batt::mutable_buffer_from_struct; using batt::MutableBuffer; using batt::resize_buffer; -using llfs::advance_pointer; -using llfs::byte_distance; - } // namespace turtle_kv diff --git a/src/turtle_kv/mem_table/mem_table.test.cpp b/src/turtle_kv/mem_table/mem_table.test.cpp index 4ad2647..121ea0d 100644 --- a/src/turtle_kv/mem_table/mem_table.test.cpp +++ b/src/turtle_kv/mem_table/mem_table.test.cpp @@ -377,9 +377,9 @@ TEST_F(MemTableTest, PutGet) // TEST_F(MemTableTest, PutUntilFull) { - usize total_key_bytes = 0; - usize total_value_bytes = 0; - usize put_count = 0; + [[maybe_unused]] usize total_key_bytes = 0; + [[maybe_unused]] usize total_value_bytes = 0; + [[maybe_unused]] usize put_count = 0; for (;;) { KeyView key = this->make_random_key(); diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp new file mode 100644 index 0000000..7339cd7 --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -0,0 +1,175 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_HPP + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return get_key(*p_kv); +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlock { + static constexpr u32 kMagic = 0x7370b49full; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u32 magic; // +4 = 4 + little_u16 shared_prefix_size; // +2 = 6 + PackedKeyValueSlotPtr items_[1]; // +2 = 8 + + //+++++++++++-+-+--+----- --- -- - - - - + + static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept + { + BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); + + const auto* block = static_cast(buffer.data()); + + BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); + + return *block; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + usize item_count() const noexcept + { + return this->items_end() - this->items_begin(); + } + + KeyView key_at(usize i) const noexcept + { + return this->items_[i]->key_view(); + } + + ValueView value_at(usize i) const noexcept + { + return this->items_[i]->value_view(&this->items_[i]); + } + + EditView edit_at(usize i) const noexcept + { + auto& packed = *this->items_[i]; + return EditView{packed.key_view(), packed.value_view(&this->items_[i])}; + } + + Optional item_at(usize i) const noexcept + { + return to_item_view(this->edit_at(i)); + } + + const PackedKeyValueSlotPtr& front_item() const noexcept + { + return this->items_[0]; + } + + const PackedKeyValueSlotPtr& back_item() const noexcept + { + return this->items_[this->item_count() - 1]; + } + + const PackedKeyValueSlotPtr* items_begin() const noexcept + { + return this->items_; + } + + const PackedKeyValueSlotPtr* items_end() const noexcept + { + return ((const PackedKeyValueSlotPtr*)this->items_[0].get()) - 1; + } + + Slice items_slice() const noexcept + { + return as_slice(this->items_begin(), this->items_end()); + } + + KeyView min_key() const noexcept + { + return get_key(this->front_item()); + } + + KeyView max_key() const noexcept + { + return get_key(this->back_item()); + } + + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept + { + auto [first, last] = + std::equal_range(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); + + if (first == last) { + return nullptr; + } + return std::addressof(*first); + } + + const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept + { + return std::lower_bound(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); + } +}; + +static_assert(sizeof(PackedLeafBlock) == 8); + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlockStats { + usize block_size; + usize item_count; + usize item_slot_bytes; + usize item_ptr_bytes; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static PackedLeafBlockStats from(const RangeT& src, usize dst_size) noexcept; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ()))>> +StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept; + +} // namespace turtle_kv + +#include "packed_leaf_block.ipp" diff --git a/src/turtle_kv/tree/packed_leaf_block.ipp b/src/turtle_kv/tree/packed_leaf_block.ipp new file mode 100644 index 0000000..ec4fb9c --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block.ipp @@ -0,0 +1,100 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_IPP + +#include "packed_leaf_block.hpp" + +#include + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +/*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, + usize dst_size) noexcept +{ + PackedLeafBlockStats stats{ + .block_size = 0, + .item_count = 0, + .item_slot_bytes = 0, + .item_ptr_bytes = 0, + }; + + if (dst_size < sizeof(PackedLeafBlock)) { + return stats; + } + stats.block_size = dst_size; + usize offset = 0; + dst_size -= sizeof(PackedLeafBlock); + offset += sizeof(PackedLeafBlock); + + for (const auto& src_item : src) { + const usize slot_size = packed_key_value_slot_size(src_item); + const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); + if (dst_size < total_item_size) { + break; + } + stats.item_count += 1; + stats.item_slot_bytes += slot_size; + stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); + dst_size -= total_item_size; + offset += total_item_size; + + if constexpr (false) { + LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) + << BATT_INSPECT(stats.item_count); + } + } + + return stats; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept +{ + if (dst.size() < sizeof(PackedLeafBlock)) { + return {batt::StatusCode::kResourceExhausted}; + } + + auto stats = PackedLeafBlockStats::from(src, dst.size()); + if (stats.block_size != dst.size()) { + return {batt::StatusCode::kResourceExhausted}; + } + + PackedLeafBlock* block = static_cast(dst.data()); + { + block->magic = PackedLeafBlock::kMagic; + block->items_[0].offset = + byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes)); + } + + PackedKeyValueSlotPtr* pp_slot = block->items_; + void* p_slot = const_cast(pp_slot->get()); + + IterT src_iter = std::begin(src); + const IterT src_end = std::next(src_iter, stats.item_count); + for (; src_iter != src_end; ++src_iter) { + const usize slot_size = pack_key_value_slot(*src_iter, p_slot); + p_slot = advance_pointer(p_slot, slot_size); + ++pp_slot; + pp_slot->offset = byte_distance(pp_slot, p_slot); + } + + return {src_iter}; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/packed_leaf_block.test.cpp new file mode 100644 index 0000000..2277269 --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block.test.cpp @@ -0,0 +1,121 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::KeyView; +using turtle_kv::ValueView; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +std::string_view random_str(std::default_random_engine& rng, + usize min_size, + usize max_size, + StableStringStore& strings) noexcept +{ + std::uniform_int_distribution pick_size{min_size, max_size}; + std::uniform_int_distribution pick_char{'a', 'z'}; + + const usize n = pick_size(rng); + MutableBuffer buf = strings.allocate(n); + char* chars = static_cast(buf.data()); + + for (usize i = 0; i < n; ++i, ++chars) { + *chars = pick_char(rng); + } + + return std::string_view{static_cast(buf.data()), buf.size()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(TreePackedLeafBlockTest, Random) +{ + const usize kNumSeeds = 1000; + const usize kMinKeySize = 4; + const usize kMaxKeySize = 48; + const usize kMinValueSize = 0; + const usize kMaxValueSize = 200; + const usize kBlockSize = 8192; + + for (usize seed = 0; seed < kNumSeeds; ++seed) { + std::default_random_engine rng{seed}; + + StableStringStore strings; + std::unordered_set used_keys; + std::vector src_edits; + + // Generate enough random edits to fill a block. + // + usize src_size = 0; + while (src_size < kBlockSize) { + std::string_view key = random_str(rng, kMinKeySize, kMaxKeySize, strings); + if (used_keys.count(key)) { + continue; + } + used_keys.insert(key); + std::string_view value = random_str(rng, kMinValueSize, kMaxValueSize, strings); + EditView edit{key, ValueView::from_str(value)}; + src_edits.push_back(edit); + src_size += key.size() + value.size(); + } + std::sort(src_edits.begin(), src_edits.end(), turtle_kv::KeyOrder{}); + + // Pack a block. + // + std::array block_buffer; + block_buffer.fill('!'); + auto dst_buffer = MutableBuffer{block_buffer.data(), kBlockSize}; + + StatusOr::const_iterator> consumed_src_end = + turtle_kv::pack_leaf_block(src_edits, dst_buffer); + + ASSERT_TRUE(consumed_src_end.ok()); + + for (usize i = kBlockSize; i < kBlockSize * 2; ++i) { + ASSERT_EQ(block_buffer[i], '!') << BATT_INSPECT(i); + } + + const usize packed_count = *consumed_src_end - src_edits.begin(); + const auto& packed_block = turtle_kv::PackedLeafBlock::view_of(dst_buffer); + + usize found_count = 0; + for (const EditView& src_edit : src_edits) { + auto* found_ptr = packed_block.find_key(get_key(src_edit)); + if (found_count < packed_count) { + ASSERT_NE(found_ptr, nullptr) + << BATT_INSPECT(src_edit) << BATT_INSPECT(found_count) << BATT_INSPECT(packed_count); + ++found_count; + } else { + ASSERT_EQ(found_ptr, nullptr); + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_page2.hpp b/src/turtle_kv/tree/packed_leaf_page2.hpp new file mode 100644 index 0000000..22f925f --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_page2.hpp @@ -0,0 +1,37 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_PAGE2_HPP + +#include + +#include + +#include +#include + +namespace turtle_kv { + +struct PackedLeafPageHeader2 { + static constexpr u64 kMagic = 0x6456beb7f9558445ull; + + //+++++++++++-+-+--+----- --- -- - - - - + + u32 key_count; // +4 = 12 + u32 total_packed_size; // +4 = 16 + llfs::PackedPointer> items; // +4 = 20 + u8 pad_[12]; // +12 = 32 +#if 0 + u32 index_step; // +4 = 16 + u32 index_size; // +4 = 20 + llfs::PackedPointer trie_index; // +4 = 32 +#endif +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_node_page.cpp b/src/turtle_kv/tree/packed_node_page.cpp index bff419e..6dbdd78 100644 --- a/src/turtle_kv/tree/packed_node_page.cpp +++ b/src/turtle_kv/tree/packed_node_page.cpp @@ -270,8 +270,8 @@ StatusOr> PackedNodePage::create_piecewise_filter(usize lev if (filter_data.values.empty()) { // Entire segment is live. // - live_ranges.emplace_back( - Interval{PiecewiseFilter::kMinLowerBound, PiecewiseFilter::kMaxUpperBound}); + live_ranges.emplace_back(Interval{PiecewiseFilter::kMinLowerBound, + PiecewiseFilter::kMaxUpperBound}); } else { live_ranges.emplace_back( Interval{PiecewiseFilter::kMinLowerBound, filter_data.values[i].value()}); @@ -496,4 +496,4 @@ std::function PackedNodePage::dump() const }; } -} // namespace turtle_kv \ No newline at end of file +} // namespace turtle_kv From d797ed7db883672ad7da03387ae51cf22ecdf56a Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Fri, 5 Jun 2026 16:08:26 -0400 Subject: [PATCH 02/20] upgraded requirements, wip packed_blocked_leaf_page --- conanfile.py | 12 +-- cor.yml | 2 +- src/CMakeLists.txt | 1 + src/turtle_kv/core/key_range.hpp | 23 +--- src/turtle_kv/core/packed_key_value_slot.hpp | 10 ++ src/turtle_kv/mem_table/mem_table.hpp | 2 +- src/turtle_kv/mem_table/mem_table.ipp | 4 +- .../tree/packed_blocked_leaf_page.hpp | 57 ++++++++++ .../tree/packed_blocked_leaf_page.ipp | 94 ++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.hpp | 61 ++++------- src/turtle_kv/tree/packed_leaf_block.ipp | 100 +++++++++++++++++- src/turtle_kv/tree/packed_leaf_block.test.cpp | 88 +++++++++++++-- src/turtle_kv/tree/packed_leaf_page2.hpp | 37 ------- src/turtle_kv/util/art.hpp | 2 +- 14 files changed, 376 insertions(+), 117 deletions(-) create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.hpp create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.ipp delete mode 100644 src/turtle_kv/tree/packed_leaf_page2.hpp diff --git a/conanfile.py b/conanfile.py index 6892cf6..5a1febd 100644 --- a/conanfile.py +++ b/conanfile.py @@ -16,7 +16,7 @@ class TurtleKvRecipe(ConanFile): name = "turtle_kv" - python_requires = "cor_recipe_utils/0.19.1" + python_requires = "cor_recipe_utils/0.21.4.dev2+g938c9a386" python_requires_extend = "cor_recipe_utils.ConanFileBase" settings = "os", "compiler", "build_type", "arch" @@ -84,19 +84,19 @@ def requirements(self): "force": True, } - self.requires("abseil/20250127.0", **VISIBLE, **OVERRIDE) + self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) self.requires("artc/[>=0.0.1 <1]") self.requires("batteries/[>=0.70.2 <1]", **VISIBLE, **OVERRIDE) - self.requires("boost/1.88.0", **VISIBLE, **OVERRIDE) - self.requires("glog/0.7.1", **VISIBLE) + self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) + self.requires("glog/[>=0.7.1 <1]", **VISIBLE) self.requires("llfs/[>=0.44.0 <1]", **VISIBLE) - self.requires("pcg-cpp/cci.20220409", **VISIBLE) + self.requires("pcg-cpp/[>=cci.20220409]", **VISIBLE) self.requires("yaml-cpp/[>=0.9.0 <1]") self.requires("zlib/1.3.1", **OVERRIDE) # boost/1.88.0 and ninja/1.13.2 depend (exactly) on libbacktrace/cci.20210118 # - self.requires("libbacktrace/[>=cci.20240730]", **OVERRIDE) + self.requires("libbacktrace/[>=cci.20210118]") if platform.system() == "Linux": if self.options.with_keyvcr: diff --git a/cor.yml b/cor.yml index 2a24d1b..c37142c 100644 --- a/cor.yml +++ b/cor.yml @@ -1,3 +1,3 @@ cor: cli: - version: 0.19.1 + version: 0.21.4.dev2+g938c9a386 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8631e97..ab268b2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,7 @@ endif () target_link_libraries( turtle_kv PUBLIC + artc::artc abseil::abseil batteries::batteries boost::boost diff --git a/src/turtle_kv/core/key_range.hpp b/src/turtle_kv/core/key_range.hpp index 16febe1..2e1c198 100644 --- a/src/turtle_kv/core/key_range.hpp +++ b/src/turtle_kv/core/key_range.hpp @@ -34,27 +34,12 @@ inline CInterval get_key_range(const Chunk& chunk) }; } -inline CInterval get_key_range(const EditView& edit) +template +inline CInterval get_key_range(const T& has_key_view) { return CInterval{ - .lower_bound = get_key(edit), - .upper_bound = get_key(edit), - }; -} - -inline CInterval get_key_range(const ItemView& item) -{ - return CInterval{ - .lower_bound = get_key(item), - .upper_bound = get_key(item), - }; -} - -inline CInterval get_key_range(const KeyView& key) -{ - return CInterval{ - .lower_bound = key, - .upper_bound = key, + .lower_bound = get_key(has_key_view), + .upper_bound = get_key(has_key_view), }; } diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index 0f0d2d2..ce42137 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -150,6 +150,16 @@ inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffe }; } +inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return get_key(*p_kv); +} + +inline ValueView get_value(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return p_kv->value_view(std::addressof(p_kv)); +} + template concept ConvertibleToKeyValueSlotRef = requires(const T& obj) { { to_key_value_slot_ref(obj) } -> std::convertible_to; diff --git a/src/turtle_kv/mem_table/mem_table.hpp b/src/turtle_kv/mem_table/mem_table.hpp index 780c7b4..fda4452 100644 --- a/src/turtle_kv/mem_table/mem_table.hpp +++ b/src/turtle_kv/mem_table/mem_table.hpp @@ -488,7 +488,7 @@ class BasicMemTable::PerOpStorageContext // One thread will acquire a lock, others will block at this point. // - absl::MutexLock lock{&this->mem_table_.block_list_mutex_}; + absl::MutexLock lock{this->mem_table_.block_list_mutex_}; // If there are no block buffers attached to the MemTable, then we may just have to wait until // the checkpoint update pipeline catches up. If there are block buffers attached, then its diff --git a/src/turtle_kv/mem_table/mem_table.ipp b/src/turtle_kv/mem_table/mem_table.ipp index 4f5930b..2812e28 100644 --- a/src/turtle_kv/mem_table/mem_table.ipp +++ b/src/turtle_kv/mem_table/mem_table.ipp @@ -395,7 +395,7 @@ void BasicMemTable::handle_external_cache_alloc(i6 this->allocation_tracker_.allocate_external(cache_alloc_delta, overcommit); { - absl::MutexLock lock{&this->block_list_mutex_}; + absl::MutexLock lock{this->block_list_mutex_}; BATT_CHECK(this->cache_alloc_in_progress_); this->total_cache_alloc_.subsume(std::move(alloc)); this->cache_alloc_in_progress_ = false; @@ -417,7 +417,7 @@ void BasicMemTable::handle_external_cache_alloc(i6 } else if (cache_alloc_delta < 0) { StatusOr alloc_to_release; { - absl::MutexLock lock{&this->block_list_mutex_}; + absl::MutexLock lock{this->block_list_mutex_}; BATT_CHECK(this->cache_alloc_in_progress_); alloc_to_release = this->total_cache_alloc_.split(-cache_alloc_delta); this->cache_alloc_in_progress_ = false; diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp new file mode 100644 index 0000000..985f5af --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -0,0 +1,57 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP + +#include + +#include + +#include + +#include +#include + +#include +#include + +namespace turtle_kv { + +struct PackedBlockedLeafPage { + static constexpr u64 kMagic = 0x6456beb7f9558445ull; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u64 magic; + little_u32 item_count; + little_u32 total_packed_size; + little_u32 blocks_per_trie_key; + little_u32 block_size_bytes; + little_u32 block_count; + little_u32 block0_byte_offset_in_page; + + /** \brief Pointer to array that stores, for each block, the starting item index relative to the + * entire leaf. + */ + llfs::PackedPointer> block_starting_item; + + /** \brief Pointer to packed ART index. + */ + llfs::PackedPointer art_block_index; + + //+++++++++++-+-+--+----- --- -- - - - - +}; + +template +StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, + MutableBuffer dst_buffer) noexcept; + +} // namespace turtle_kv + +#include "packed_blocked_leaf_page.ipp" diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp new file mode 100644 index 0000000..6ec3864 --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp @@ -0,0 +1,94 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, + MutableBuffer dst_buffer) noexcept +{ + const usize block_size = 8192; + const usize item_count = std::size(src_items); + + //+++++++++++-+-+--+----- --- -- - - - - + // Calculate the number of blocks needed and how many items in each one. + // + SmallVec block_stats; + { + auto src_iter = std::begin(src_items); + const auto src_end = std::end(src_items); + usize blocks_size_remaining = dst_buffer.size() - block_size; + for (;;) { + if (src_iter == src_end) { + break; + } + BATT_CHECK_LT(src_iter, src_end); + + BATT_ASSIGN_OK_RESULT(auto stats, + PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), + blocks_size_remaining)); + + blocks_size_remaining -= block_size; + src_iter = std::next(src_iter, stats.item_count); + } + } + const usize block_count = block_stats.size(); + const usize block_starting_item_array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; + + //+++++++++++-+-+--+----- --- -- - - - - + // Initialize the leaf header. + // + MutableBuffer dst_remaining = dst_buffer; + dst_remaining += sizeof(llfs::PackedPageHeader); + + auto* leaf_header = static_cast(dst_remaining.data()); + { + leaf_header->magic = PackedBlockedLeafPage::kMagic; + leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); + leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] + leaf_header->blocks_per_trie_key = 0; // TODO [tastolfi 2026-06-01] + leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); + leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); + leaf_header->block0_byte_offset_in_page = 0; // TODO [tastolfi 2026-06-01] + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack `block_starting_item` array. + // + + { + auto* block_starting_item = static_cast*>(dst_remaining.data()); + dst_remaining += block_starting_item_array_size; + + block_starting_item->initialize(block_stats.size()); + + little_u32* block_start = block_starting_item->data(); + u32 item_i = 0; + for (const PackedLeafBlockStats& stats : block_stats) { + *block_start = item_i; + item_i += stats.item_count; + ++block_start; + } + + leaf_header->block_starting_item.reset_unsafe(block_starting_item); + } +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp index 7339cd7..12f125e 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -27,13 +28,6 @@ namespace turtle_kv { -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept -{ - return get_key(*p_kv); -} - //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- // struct PackedLeafBlock { @@ -47,16 +41,10 @@ struct PackedLeafBlock { //+++++++++++-+-+--+----- --- -- - - - - - static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept - { - BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); - - const auto* block = static_cast(buffer.data()); - - BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); - - return *block; - } + /** \brief Returns the passed buffer's memory region, validated as a PackedLeafBlock and cast to + * `const PackedLeafBlock &`. + */ + static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept; //+++++++++++-+-+--+----- --- -- - - - - @@ -111,6 +99,9 @@ struct PackedLeafBlock { return as_slice(this->items_begin(), this->items_end()); } + Slice items_slice(Optional key_lower_bound, + Optional key_upper_bound) const noexcept; + KeyView min_key() const noexcept { return get_key(this->front_item()); @@ -121,31 +112,19 @@ struct PackedLeafBlock { return get_key(this->back_item()); } - const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept + KeyView shared_key_prefix() const noexcept { - auto [first, last] = - std::equal_range(this->items_begin(), - this->items_end(), - key, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; - }); - - if (first == last) { - return nullptr; - } - return std::addressof(*first); + return this->min_key().substr(0, this->shared_prefix_size); } - const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept - { - return std::lower_bound(this->items_begin(), - this->items_end(), - key, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; - }); - } + /** \brief Returns an iterator to the given key in this block if found or nullptr if not found. + */ + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; + + /** \brief Returns an iterator to the first item in this block whose key is not less than `key`; + * if all keys in the block are less than `key`, returns `this->items_end()`. + */ + const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept; }; static_assert(sizeof(PackedLeafBlock) == 8); @@ -168,7 +147,9 @@ struct PackedLeafBlockStats { // template ()))>> -StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept; +StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& stats = None) noexcept; } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.ipp b/src/turtle_kv/tree/packed_leaf_block.ipp index ec4fb9c..fe2f31f 100644 --- a/src/turtle_kv/tree/packed_leaf_block.ipp +++ b/src/turtle_kv/tree/packed_leaf_block.ipp @@ -11,8 +11,14 @@ #include "packed_leaf_block.hpp" +#include + #include +#include + +#include +#include #include #include @@ -64,13 +70,18 @@ template //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept +StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& opt_stats) noexcept { if (dst.size() < sizeof(PackedLeafBlock)) { return {batt::StatusCode::kResourceExhausted}; } - auto stats = PackedLeafBlockStats::from(src, dst.size()); + PackedLeafBlockStats stats = opt_stats.or_else([&] { + return PackedLeafBlockStats::from(src, dst.size()); + }); + if (stats.block_size != dst.size()) { return {batt::StatusCode::kResourceExhausted}; } @@ -82,6 +93,8 @@ StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes)); } + //----- --- -- - - - - + // Pack all slot data. PackedKeyValueSlotPtr* pp_slot = block->items_; void* p_slot = const_cast(pp_slot->get()); @@ -94,7 +107,90 @@ StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept pp_slot->offset = byte_distance(pp_slot, p_slot); } + //----- --- -- - - - - + // Set the common prefix. + // + block->shared_prefix_size = + BATT_CHECKED_CAST(u16, + llfs::find_common_prefix(0, block->min_key(), block->max_key()).size()); + return {src_iter}; } +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// struct PackedLeafBlock + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ const PackedLeafBlock& PackedLeafBlock::view_of(const ConstBuffer& buffer) noexcept +{ + BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); + + const auto* block = static_cast(buffer.data()); + + BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); + + return *block; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedLeafBlock::find_key(const KeyView& key) const noexcept +{ + const auto convert_result = [](auto&& first_last_pair) -> const PackedKeyValueSlotPtr* { + if (first_last_pair.first == first_last_pair.second) { + return nullptr; + } + return first_last_pair.first; + }; + + if (this->shared_prefix_size > 0) { + const usize prefix_size = this->shared_prefix_size; + if (key.size() < prefix_size) { + return nullptr; + } + auto order = batt::compare(key.substr(0, prefix_size), this->shared_key_prefix()); + if (order != batt::Order::Equal) { + return nullptr; + } + + return convert_result( + std::equal_range(this->items_begin(), this->items_end(), key, KeySuffixOrder{prefix_size})); + } + + return convert_result(std::equal_range(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == + batt::Order::Less; + })); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedLeafBlock::lower_bound(const KeyView& key) const noexcept +{ + return std::lower_bound(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Slice PackedLeafBlock::items_slice( + Optional key_lower_bound, + Optional key_upper_bound) const noexcept +{ + auto [first, last] = std::equal_range(this->items_begin(), + this->items_end(), + Interval{key_lower_bound.or_else(global_min_key), + key_upper_bound.or_else(global_max_key)}, + ExtendedKeyRangeOrder{}); + return as_slice(first, last); +} + } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/packed_leaf_block.test.cpp index 2277269..8656778 100644 --- a/src/turtle_kv/tree/packed_leaf_block.test.cpp +++ b/src/turtle_kv/tree/packed_leaf_block.test.cpp @@ -33,18 +33,25 @@ using turtle_kv::ValueView; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // +template std::string_view random_str(std::default_random_engine& rng, + SizeDistribution&& pick_size, usize min_size, usize max_size, - StableStringStore& strings) noexcept + StableStringStore& strings, + std::string_view prefix = "") noexcept { - std::uniform_int_distribution pick_size{min_size, max_size}; std::uniform_int_distribution pick_char{'a', 'z'}; - const usize n = pick_size(rng); - MutableBuffer buf = strings.allocate(n); + const usize n = min_size + std::min(pick_size(rng), max_size - min_size); + MutableBuffer buf = strings.allocate(prefix.size() + n); char* chars = static_cast(buf.data()); + if (!prefix.empty()) { + std::memcpy(chars, prefix.data(), prefix.size()); + chars += prefix.size(); + } + for (usize i = 0; i < n; ++i, ++chars) { *chars = pick_char(rng); } @@ -57,16 +64,28 @@ std::string_view random_str(std::default_random_engine& rng, TEST(TreePackedLeafBlockTest, Random) { const usize kNumSeeds = 1000; + const usize kNumNotFoundQueries = 100; + const usize kNumLowerBoundQueries = 500; + const usize kMinPrefixSize = 0; + const usize kMaxPrefixSize = 8; const usize kMinKeySize = 4; const usize kMaxKeySize = 48; const usize kMinValueSize = 0; const usize kMaxValueSize = 200; const usize kBlockSize = 8192; + std::geometric_distribution pick_prefix_size{0.5}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMaxValueSize}; + for (usize seed = 0; seed < kNumSeeds; ++seed) { std::default_random_engine rng{seed}; StableStringStore strings; + + std::string_view prefix = + random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + std::unordered_set used_keys; std::vector src_edits; @@ -74,14 +93,17 @@ TEST(TreePackedLeafBlockTest, Random) // usize src_size = 0; while (src_size < kBlockSize) { - std::string_view key = random_str(rng, kMinKeySize, kMaxKeySize, strings); + std::string_view key = + random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); if (used_keys.count(key)) { continue; } used_keys.insert(key); - std::string_view value = random_str(rng, kMinValueSize, kMaxValueSize, strings); - EditView edit{key, ValueView::from_str(value)}; - src_edits.push_back(edit); + + std::string_view value = + random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + src_edits.push_back(EditView{key, ValueView::from_str(value)}); src_size += key.size() + value.size(); } std::sort(src_edits.begin(), src_edits.end(), turtle_kv::KeyOrder{}); @@ -104,6 +126,8 @@ TEST(TreePackedLeafBlockTest, Random) const usize packed_count = *consumed_src_end - src_edits.begin(); const auto& packed_block = turtle_kv::PackedLeafBlock::view_of(dst_buffer); + ASSERT_EQ(packed_block.shared_prefix_size.value(), prefix.size()); + usize found_count = 0; for (const EditView& src_edit : src_edits) { auto* found_ptr = packed_block.find_key(get_key(src_edit)); @@ -111,10 +135,58 @@ TEST(TreePackedLeafBlockTest, Random) ASSERT_NE(found_ptr, nullptr) << BATT_INSPECT(src_edit) << BATT_INSPECT(found_count) << BATT_INSPECT(packed_count); ++found_count; + + ASSERT_EQ(get_key(*found_ptr), get_key(src_edit)); + ASSERT_EQ(get_value(*found_ptr), get_value(src_edit)); } else { ASSERT_EQ(found_ptr, nullptr); } } + + for (usize i = 0; i < kNumNotFoundQueries; ++i) { + std::string_view key; + for (;;) { + key = random_str(rng, + pick_key_size, + kMinKeySize + prefix.size(), + kMaxKeySize + prefix.size(), + strings); + if (!used_keys.count(key)) { + break; + } + } + + ASSERT_EQ(packed_block.find_key(key), nullptr); + } + + for (usize i = 0; i < kNumLowerBoundQueries; ++i) { + std::string_view key = (i % 2) ? random_str(rng, + pick_key_size, + kMinKeySize + prefix.size(), + kMaxKeySize + prefix.size(), + strings) + : random_str(rng, // + pick_key_size, + kMinKeySize, + kMaxKeySize, + strings, + prefix); + + const auto expected_iter = + std::lower_bound(src_edits.begin(), src_edits.end(), key, turtle_kv::KeyOrder{}); + + const usize expected_i = std::distance(src_edits.begin(), expected_iter); + + const auto actual_iter = packed_block.lower_bound(key); + + const usize actual_i = std::distance(packed_block.items_begin(), actual_iter); + + if (expected_i >= packed_count) { + ASSERT_EQ(actual_i, packed_count); + } else { + ASSERT_EQ(actual_i, expected_i); + } + } } } diff --git a/src/turtle_kv/tree/packed_leaf_page2.hpp b/src/turtle_kv/tree/packed_leaf_page2.hpp deleted file mode 100644 index 22f925f..0000000 --- a/src/turtle_kv/tree/packed_leaf_page2.hpp +++ /dev/null @@ -1,37 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_LEAF_PAGE2_HPP - -#include - -#include - -#include -#include - -namespace turtle_kv { - -struct PackedLeafPageHeader2 { - static constexpr u64 kMagic = 0x6456beb7f9558445ull; - - //+++++++++++-+-+--+----- --- -- - - - - - - u32 key_count; // +4 = 12 - u32 total_packed_size; // +4 = 16 - llfs::PackedPointer> items; // +4 = 20 - u8 pad_[12]; // +12 = 32 -#if 0 - u32 index_step; // +4 = 16 - u32 index_size; // +4 = 20 - llfs::PackedPointer trie_index; // +4 = 32 -#endif -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art.hpp b/src/turtle_kv/util/art.hpp index b9903cd..d4c050b 100644 --- a/src/turtle_kv/util/art.hpp +++ b/src/turtle_kv/util/art.hpp @@ -948,7 +948,7 @@ class ARTBase ~MemoryContext() noexcept { if (this->art_) { - absl::MutexLock lock{&this->art_->mutex_}; + absl::MutexLock lock{this->art_->mutex_}; for (auto& p_ex : this->thread_extents_) { this->art_->extents_.emplace_back(std::move(p_ex)); } From 5da7474f6bd2066429b79062d474c51ec90ffe31 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Fri, 5 Jun 2026 17:05:25 -0400 Subject: [PATCH 03/20] wip packed blocked leaf page --- conan.lock | 4 +- conanfile.py | 2 +- cor.yml | 2 +- .../tree/packed_blocked_leaf_page.hpp | 2 +- .../tree/packed_blocked_leaf_page.ipp | 54 +++++++++++++++++-- 5 files changed, 56 insertions(+), 8 deletions(-) diff --git a/conan.lock b/conan.lock index 1607f64..7d61ffc 100644 --- a/conan.lock +++ b/conan.lock @@ -11,7 +11,7 @@ "glog/0.7.1", "gtest/1.17.0", "keyvcr/0.2.2", - "libbacktrace/cci.20240730", + "libbacktrace/cci.20210118", "libfuse/3.16.2", "libpfm4/4.13.0", "libunwind/1.8.1", @@ -56,4 +56,4 @@ ] }, "config_requires": [] -} \ No newline at end of file +} diff --git a/conanfile.py b/conanfile.py index 5a1febd..f2165c9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -16,7 +16,7 @@ class TurtleKvRecipe(ConanFile): name = "turtle_kv" - python_requires = "cor_recipe_utils/0.21.4.dev2+g938c9a386" + python_requires = "cor_recipe_utils/0.21.4.dev3+g0d8231b80" python_requires_extend = "cor_recipe_utils.ConanFileBase" settings = "os", "compiler", "build_type", "arch" diff --git a/cor.yml b/cor.yml index c37142c..0d48710 100644 --- a/cor.yml +++ b/cor.yml @@ -1,3 +1,3 @@ cor: cli: - version: 0.21.4.dev2+g938c9a386 + version: 0.21.4.dev3+g0d8231b80 diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp index 985f5af..2bf678e 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -31,7 +31,7 @@ struct PackedBlockedLeafPage { big_u64 magic; little_u32 item_count; little_u32 total_packed_size; - little_u32 blocks_per_trie_key; + little_u32 blocks_per_art_key; little_u32 block_size_bytes; little_u32 block_count; little_u32 block0_byte_offset_in_page; diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp index 6ec3864..dd2ebc3 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp @@ -15,6 +15,10 @@ #include +#include + +#include + namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -62,8 +66,8 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it { leaf_header->magic = PackedBlockedLeafPage::kMagic; leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); - leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] - leaf_header->blocks_per_trie_key = 0; // TODO [tastolfi 2026-06-01] + leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] + leaf_header->blocks_per_art_key = 0; // TODO [tastolfi 2026-06-01] leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); leaf_header->block0_byte_offset_in_page = 0; // TODO [tastolfi 2026-06-01] @@ -72,7 +76,6 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it //+++++++++++-+-+--+----- --- -- - - - - // Pack `block_starting_item` array. // - { auto* block_starting_item = static_cast*>(dst_remaining.data()); dst_remaining += block_starting_item_array_size; @@ -89,6 +92,51 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it leaf_header->block_starting_item.reset_unsafe(block_starting_item); } + + //+++++++++++-+-+--+----- --- -- - - - - + // Calculate blocks_per_art_key based on available space. + // + const usize space_for_art = (dst_remaining.size() & ~(block_size - 1)) - block_size * block_count; + SmallVec art_keys; + usize blocks_per_art_key = 1; + const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); + for (;;) { + art_keys.clear(); + auto items = std::begin(src_items); + for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { + const usize item_i = block_starting_item[block_i]; + art_keys.emplace_back(get_key(*(items + item_i))); + } + + using artc::packed::PackedARTBuilder; + + batt::StableStringStore string_store; + + BATT_ASSIGN_OK_RESULT(auto art_builder, + PackedARTBuilder::from_items(art_keys.begin(), + art_keys.end(), + BATT_OVERLOADS_OF(get_key), + string_store)); + + if (art_builder.get_packed_size() > space_for_art) { + ++blocks_per_art_key; + continue; + } + + MutableBuffer art_buffer{dst_remaining.data(), art_builder.get_packed_size()}; + dst_remaining += art_buffer.size(); + BATT_CHECK_GE(dst_remaining.size(), block_size * block_count); + + BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); + + leaf_header->art_block_index.reset_unsafe(art_root); + break; + } + + // Shift the remaining buffer forward so it aligns with the nearest block boundary. + // + const usize offset_for_block_align = dst_remaining.size() & (block_size - 1); + dst_remaining += } } // namespace turtle_kv From 4572d2a55e0d005a0a47a11cc45f6b1c9e395de8 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sat, 6 Jun 2026 15:46:28 -0400 Subject: [PATCH 04/20] pack_blocked_leaf_page nominally working --- .../script/uniform_key_distribution.hpp | 8 +- .../tree/packed_blocked_leaf_page.cpp | 13 ++ .../tree/packed_blocked_leaf_page.hpp | 78 ++++++-- .../tree/packed_blocked_leaf_page.ipp | 82 +++++++-- .../tree/packed_blocked_leaf_page.test.cpp | 166 ++++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.hpp | 21 ++- src/turtle_kv/tree/packed_leaf_block.ipp | 15 +- src/turtle_kv/tree/packed_leaf_block.test.cpp | 38 +--- src/turtle_kv/tree/random_str.hpp | 48 +++++ 9 files changed, 400 insertions(+), 69 deletions(-) create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.cpp create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp create mode 100644 src/turtle_kv/tree/random_str.hpp diff --git a/src/turtle_kv/script/uniform_key_distribution.hpp b/src/turtle_kv/script/uniform_key_distribution.hpp index 67fd211..700f1d4 100644 --- a/src/turtle_kv/script/uniform_key_distribution.hpp +++ b/src/turtle_kv/script/uniform_key_distribution.hpp @@ -37,7 +37,9 @@ inline constexpr std::array kHashSeeds = { class UniformInsertKeyDistribution : public KeyDistribution { public: - explicit UniformInsertKeyDistribution(usize key_size) noexcept : key_buffer_(key_size) + explicit UniformInsertKeyDistribution(usize key_size, usize seed = 0) noexcept + : next_ordinal_{seed} + , key_buffer_(key_size) { } @@ -48,7 +50,7 @@ class UniformInsertKeyDistribution : public KeyDistribution std::pair get_next(KeySet& inserted_keys) override { - return inserted_keys.create_key(this->format_key(this->count_.fetch_add(1))); + return inserted_keys.create_key(this->format_key(this->next_ordinal_.fetch_add(1))); } //+++++++++++-+-+--+----- --- -- - - - - @@ -77,7 +79,7 @@ class UniformInsertKeyDistribution : public KeyDistribution //+++++++++++-+-+--+----- --- -- - - - - - std::atomic count_{0}; + std::atomic next_ordinal_{0}; SmallVec key_buffer_; }; diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.cpp new file mode 100644 index 0000000..25ef37d --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.cpp @@ -0,0 +1,13 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// + +namespace turtle_kv { +} diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp index 2bf678e..ed02a63 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -16,11 +16,14 @@ #include #include +#include #include #include #include +#include + namespace turtle_kv { struct PackedBlockedLeafPage { @@ -28,29 +31,82 @@ struct PackedBlockedLeafPage { //+++++++++++-+-+--+----- --- -- - - - - - big_u64 magic; - little_u32 item_count; - little_u32 total_packed_size; - little_u32 blocks_per_art_key; - little_u32 block_size_bytes; - little_u32 block_count; - little_u32 block0_byte_offset_in_page; + template + static usize packed_edit_size(const EditT& edit) noexcept + { + return PackedLeafBlock::packed_edit_size(edit); + } + + static usize estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u64 magic; // +8 -> 8 + little_u32 item_count; // +4 -> 12 + little_u32 total_packed_size; // +4 -> 16 + little_u32 blocks_per_art_key; // +4 -> 20 + little_u32 block_size_bytes; // +4 -> 24 + little_u32 block_count; // +4 -> 28 + llfs::PackedPointer block0; // +4 -> 32 /** \brief Pointer to array that stores, for each block, the starting item index relative to the * entire leaf. */ - llfs::PackedPointer> block_starting_item; + llfs::PackedPointer> block_starting_item; // +4 -> 36 /** \brief Pointer to packed ART index. */ - llfs::PackedPointer art_block_index; + llfs::PackedPointer art_block_index; // +4 -> 40 + + u8 pad_[24]; //+++++++++++-+-+--+----- --- -- - - - - }; +static_assert(sizeof(PackedBlockedLeafPage) == 64); + template -StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, - MutableBuffer dst_buffer) noexcept; +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline /*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept +{ + const usize space_after_header = + leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); + + const usize max_block_count = space_after_header / block_size; + + const usize block_starts_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; + + const usize space_after_block_starts = space_after_header - block_starts_size; + + const usize max_art_size = max_key_size * max_block_count * 2; + + const usize space_after_art = space_after_block_starts - max_art_size; + + BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; + const usize space_for_blocks = space_after_art & ~(block_size - 1); + const usize block_count = space_for_blocks / block_size; + + const usize max_wasted_per_block = max_edit_size - 1; + const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; + + const usize final_estimate = block_count * min_block_capacity; + + BATT_CHECK_GT(leaf_size, final_estimate); + + return final_estimate; +} } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp index dd2ebc3..3d52be1 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp @@ -13,8 +13,6 @@ #include -#include - #include #include @@ -24,10 +22,10 @@ namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, - MutableBuffer dst_buffer) noexcept +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept { - const usize block_size = 8192; const usize item_count = std::size(src_items); //+++++++++++-+-+--+----- --- -- - - - - @@ -44,17 +42,18 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it } BATT_CHECK_LT(src_iter, src_end); - BATT_ASSIGN_OK_RESULT(auto stats, - PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), - blocks_size_remaining)); + if (blocks_size_remaining < block_size) { + return {batt::StatusCode::kResourceExhausted}; + } + + auto& stats = block_stats.emplace_back( + PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), block_size)); blocks_size_remaining -= block_size; src_iter = std::next(src_iter, stats.item_count); } } const usize block_count = block_stats.size(); - const usize block_starting_item_array_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; //+++++++++++-+-+--+----- --- -- - - - - // Initialize the leaf header. @@ -63,20 +62,26 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it dst_remaining += sizeof(llfs::PackedPageHeader); auto* leaf_header = static_cast(dst_remaining.data()); + dst_remaining += sizeof(PackedBlockedLeafPage); { leaf_header->magic = PackedBlockedLeafPage::kMagic; leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); - leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] - leaf_header->blocks_per_art_key = 0; // TODO [tastolfi 2026-06-01] + leaf_header->total_packed_size = 0; + leaf_header->blocks_per_art_key = 0; leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); - leaf_header->block0_byte_offset_in_page = 0; // TODO [tastolfi 2026-06-01] + leaf_header->block0.offset = 0; + leaf_header->block_starting_item.offset = 0; + leaf_header->art_block_index.offset = 0; } //+++++++++++-+-+--+----- --- -- - - - - // Pack `block_starting_item` array. // { + const usize block_starting_item_array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; + auto* block_starting_item = static_cast*>(dst_remaining.data()); dst_remaining += block_starting_item_array_size; @@ -92,19 +97,20 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it leaf_header->block_starting_item.reset_unsafe(block_starting_item); } + const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); //+++++++++++-+-+--+----- --- -- - - - - // Calculate blocks_per_art_key based on available space. // - const usize space_for_art = (dst_remaining.size() & ~(block_size - 1)) - block_size * block_count; + const usize space_for_art = dst_remaining.size() - block_size * block_count; SmallVec art_keys; usize blocks_per_art_key = 1; - const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); for (;;) { art_keys.clear(); auto items = std::begin(src_items); for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { const usize item_i = block_starting_item[block_i]; + BATT_CHECK_LT(item_i, item_count); art_keys.emplace_back(get_key(*(items + item_i))); } @@ -112,6 +118,9 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it batt::StableStringStore string_store; + BATT_DEBUG_INFO(BATT_INSPECT_RANGE(art_keys) + << BATT_INSPECT(block_count) << BATT_INSPECT(blocks_per_art_key)); + BATT_ASSIGN_OK_RESULT(auto art_builder, PackedARTBuilder::from_items(art_keys.begin(), art_keys.end(), @@ -130,13 +139,54 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); leaf_header->art_block_index.reset_unsafe(art_root); + leaf_header->blocks_per_art_key = BATT_CHECKED_CAST(u32, blocks_per_art_key); break; } + //+++++++++++-+-+--+----- --- -- - - - - // Shift the remaining buffer forward so it aligns with the nearest block boundary. // const usize offset_for_block_align = dst_remaining.size() & (block_size - 1); - dst_remaining += + dst_remaining += offset_for_block_align; + BATT_CHECK_LE(block_size * block_count, dst_remaining.size()); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack the blocks. + // + leaf_header->block0.reset_unsafe(static_cast(dst_remaining.data())); + { + auto src_iter = std::begin(src_items); + const auto src_end = std::end(src_items); + usize block_i = 0; + for (const PackedLeafBlockStats& stats : block_stats) { + BATT_DEBUG_INFO(BATT_INSPECT(block_i) << BATT_INSPECT(stats)); + + BATT_CHECK_NE(src_iter, src_end); + auto src_block_items = std::ranges::subrange(src_iter, std::next(src_iter, stats.item_count)); + + BATT_CHECK_GE(dst_remaining.size(), block_size); + MutableBuffer dst_block_buffer{dst_remaining.data(), block_size}; + + auto block_end_iter = + BATT_OK_RESULT_OR_PANIC(pack_leaf_block(src_block_items, dst_block_buffer, stats)); + + BATT_CHECK_EQ(block_end_iter, std::end(src_block_items)); + + src_iter = block_end_iter; + dst_remaining += block_size; + ++block_i; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Fill in remaining header fields. + // + leaf_header->total_packed_size = BATT_CHECKED_CAST(u32, dst_buffer.size() - dst_remaining.size()); + + //+++++++++++-+-+--+----- --- -- - - - - + // Success! (nothing succeeds like it) + // + return leaf_header; } } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp new file mode 100644 index 0000000..1361f2c --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp @@ -0,0 +1,166 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include "random_str.hpp" + +#include + +#include +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; +using namespace batt::constants; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::KeyOrder; +using turtle_kv::KeyView; +using turtle_kv::pack_blocked_leaf_page; +using turtle_kv::PackedBlockedLeafPage; +using turtle_kv::random_str; +using turtle_kv::ValueView; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// Plan: +// 1. For different random seeds: +// - generate random set of prefixes (~10% of total keys) +// - generate keys using prefixes, with random values +// - sort +// - pack leaf; verify: +// a. all packed keys present and have right values +// b. any unpacked keys at end missing +// c. randomly generated non-present keys not found +// +TEST(TreePackedBlockedLeafPageTest, Random) +{ + const usize kNumSeeds = 1000; + const usize kLeafPageSize = 1 * kMiB; + const usize kNumPrefixes = 1000; + const usize kMinPrefixSize = 0; + const usize kMaxPrefixSize = 8; + const usize kMinKeySize = 4; + const usize kMaxKeySize = 48; + const usize kMinValueSize = 0; + const usize kMaxValueSize = 200; + const usize kBlockSize = 8192; + + BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); + + std::geometric_distribution pick_prefix_size{0.5}; + std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + + for (usize seed = 0; seed < kNumSeeds; ++seed) { + std::default_random_engine rng{seed}; + + StableStringStore strings; + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate prefixes + // + std::vector prefixes; + { + std::unordered_set used_prefixes; + while (prefixes.size() < kNumPrefixes) { + std::string_view prefix = + random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + + if (used_prefixes.count(prefix)) { + continue; + } + prefixes.push_back(prefix); + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate edits. + // + std::vector edits; + { + usize max_edit_size = 0; + usize max_key_size = 0; + usize total_edits_size = 0; + + std::unordered_set used_keys; + for (;;) { + std::string_view prefix = prefixes[pick_prefix(rng)]; + + std::string_view key = + random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); + + if (used_keys.count(key)) { + continue; + } + + std::string_view value = + random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + EditView edit{key, ValueView::from_str(value)}; + + const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); + + const usize new_max_edit_size = std::max(max_edit_size, edit_size); + const usize new_max_key_size = std::max(max_key_size, key.size()); + + const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, + kBlockSize, + new_max_key_size, + new_max_edit_size); + + // Stop as soon as adding the next key would exceed the estimated space. + // + if (edit_size + total_edits_size > space_available) { + break; + } + + edits.push_back(edit); + total_edits_size += edit_size; + max_edit_size = new_max_edit_size; + max_key_size = new_max_key_size; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Sort edits by key. + // + std::sort(edits.begin(), edits.end(), KeyOrder{}); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack a blocked leaf page. + // + using StorageUnit = std::aligned_storage_t<4096, 4096>; + std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); + ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); + + MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; + + StatusOr packed_leaf = + pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); + + ASSERT_TRUE(packed_leaf.ok()) << BATT_INSPECT(packed_leaf.status()); + } +} + +} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp index 12f125e..6485e00 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -23,6 +23,7 @@ #include #include +#include #include @@ -41,6 +42,20 @@ struct PackedLeafBlock { //+++++++++++-+-+--+----- --- -- - - - - + template + static usize packed_edit_size(const EditT& edit) noexcept + { + const usize slot_size = packed_key_value_slot_size(edit); + const usize edit_size = slot_size + sizeof(PackedKeyValueSlotPtr); + + return edit_size; + } + + static constexpr usize capacity(usize block_size) noexcept + { + return block_size - std::min(block_size, sizeof(PackedLeafBlock)); + } + /** \brief Returns the passed buffer's memory region, validated as a PackedLeafBlock and cast to * `const PackedLeafBlock &`. */ @@ -140,9 +155,13 @@ struct PackedLeafBlockStats { //+++++++++++-+-+--+----- --- -- - - - - template - static PackedLeafBlockStats from(const RangeT& src, usize dst_size) noexcept; + static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; }; +BATT_OBJECT_PRINT_IMPL((inline), + PackedLeafBlockStats, + (block_size, item_count, item_slot_bytes, item_ptr_bytes)) + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -/*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, - usize dst_size) noexcept +inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, + usize dst_size) noexcept { PackedLeafBlockStats stats{ .block_size = 0, @@ -70,9 +70,9 @@ template //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -StatusOr pack_leaf_block(const RangeT& src, - MutableBuffer dst, - const Optional& opt_stats) noexcept +inline StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& opt_stats) noexcept { if (dst.size() < sizeof(PackedLeafBlock)) { return {batt::StatusCode::kResourceExhausted}; @@ -122,7 +122,8 @@ StatusOr pack_leaf_block(const RangeT& src, //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -/*static*/ const PackedLeafBlock& PackedLeafBlock::view_of(const ConstBuffer& buffer) noexcept +inline /*static*/ const PackedLeafBlock& PackedLeafBlock::view_of( + const ConstBuffer& buffer) noexcept { BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); @@ -181,7 +182,7 @@ inline const PackedKeyValueSlotPtr* PackedLeafBlock::lower_bound(const KeyView& //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PackedLeafBlock::items_slice( +inline Slice PackedLeafBlock::items_slice( Optional key_lower_bound, Optional key_upper_bound) const noexcept { diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/packed_leaf_block.test.cpp index 8656778..73a9e1f 100644 --- a/src/turtle_kv/tree/packed_leaf_block.test.cpp +++ b/src/turtle_kv/tree/packed_leaf_block.test.cpp @@ -13,6 +13,8 @@ #include #include +#include "random_str.hpp" + #include #include @@ -28,37 +30,11 @@ using batt::StableStringStore; using batt::StatusOr; using turtle_kv::EditView; +using turtle_kv::KeyOrder; using turtle_kv::KeyView; +using turtle_kv::random_str; using turtle_kv::ValueView; -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -std::string_view random_str(std::default_random_engine& rng, - SizeDistribution&& pick_size, - usize min_size, - usize max_size, - StableStringStore& strings, - std::string_view prefix = "") noexcept -{ - std::uniform_int_distribution pick_char{'a', 'z'}; - - const usize n = min_size + std::min(pick_size(rng), max_size - min_size); - MutableBuffer buf = strings.allocate(prefix.size() + n); - char* chars = static_cast(buf.data()); - - if (!prefix.empty()) { - std::memcpy(chars, prefix.data(), prefix.size()); - chars += prefix.size(); - } - - for (usize i = 0; i < n; ++i, ++chars) { - *chars = pick_char(rng); - } - - return std::string_view{static_cast(buf.data()), buf.size()}; -} - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // TEST(TreePackedLeafBlockTest, Random) @@ -76,7 +52,7 @@ TEST(TreePackedLeafBlockTest, Random) std::geometric_distribution pick_prefix_size{0.5}; std::geometric_distribution pick_key_size{0.7}; - std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMaxValueSize}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; for (usize seed = 0; seed < kNumSeeds; ++seed) { std::default_random_engine rng{seed}; @@ -106,7 +82,7 @@ TEST(TreePackedLeafBlockTest, Random) src_edits.push_back(EditView{key, ValueView::from_str(value)}); src_size += key.size() + value.size(); } - std::sort(src_edits.begin(), src_edits.end(), turtle_kv::KeyOrder{}); + std::sort(src_edits.begin(), src_edits.end(), KeyOrder{}); // Pack a block. // @@ -173,7 +149,7 @@ TEST(TreePackedLeafBlockTest, Random) prefix); const auto expected_iter = - std::lower_bound(src_edits.begin(), src_edits.end(), key, turtle_kv::KeyOrder{}); + std::lower_bound(src_edits.begin(), src_edits.end(), key, KeyOrder{}); const usize expected_i = std::distance(src_edits.begin(), expected_iter); diff --git a/src/turtle_kv/tree/random_str.hpp b/src/turtle_kv/tree/random_str.hpp new file mode 100644 index 0000000..5145d56 --- /dev/null +++ b/src/turtle_kv/tree/random_str.hpp @@ -0,0 +1,48 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include + +#include + +#include +#include +#include +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +std::string_view random_str(std::default_random_engine& rng, + SizeDistribution&& pick_size, + usize min_size, + usize max_size, + batt::StableStringStore& strings, + std::string_view prefix = "") noexcept +{ + std::uniform_int_distribution pick_char{'a', 'z'}; + + const usize n = min_size + std::min(pick_size(rng), max_size - min_size); + batt::MutableBuffer buf = strings.allocate(prefix.size() + n); + char* chars = static_cast(buf.data()); + + if (!prefix.empty()) { + std::memcpy(chars, prefix.data(), prefix.size()); + chars += prefix.size(); + } + + for (usize i = 0; i < n; ++i, ++chars) { + *chars = pick_char(rng); + } + + return std::string_view{static_cast(buf.data()), buf.size()}; +} + +} // namespace turtle_kv From 203fb60d59551df995a82380e2e6859cc845f483 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 7 Jun 2026 12:24:25 -0400 Subject: [PATCH 05/20] Add item seq to PackedBlockedLeafPage. --- .../tree/packed_blocked_leaf_page.hpp | 32 +++++++++ .../tree/packed_blocked_leaf_page.test.cpp | 30 +++++++- src/turtle_kv/tree/packed_leaf_block.hpp | 68 +++++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.ipp | 10 ++- 4 files changed, 136 insertions(+), 4 deletions(-) diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp index ed02a63..5aa3008 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -23,6 +23,9 @@ #include #include +#include + +#include namespace turtle_kv { @@ -64,6 +67,35 @@ struct PackedBlockedLeafPage { u8 pad_[24]; //+++++++++++-+-+--+----- --- -- - - - - + + PackedLeafBlock::Iterator blocks_begin() const noexcept + { + return PackedLeafBlock::Iterator{this->block0.get(), (isize)this->block_size_bytes.value()}; + } + + PackedLeafBlock::Iterator blocks_end() const noexcept + { + return this->blocks_begin() + this->block_count; + } + + auto blocks() const noexcept + { + return std::ranges::subrange(this->blocks_begin(), + this->blocks_end()); + } + + auto blocks_seq() const noexcept + { + return batt::as_seq(this->blocks()); + } + + auto items_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map([](const PackedLeafBlock& block) { + return batt::as_seq(block.items_slice()); + }) | + batt::seq::flatten(); + } }; static_assert(sizeof(PackedBlockedLeafPage) == 64); diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp index 1361f2c..ec26d26 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp @@ -36,6 +36,7 @@ using batt::StatusOr; using turtle_kv::EditView; using turtle_kv::KeyOrder; using turtle_kv::KeyView; +using turtle_kv::Optional; using turtle_kv::pack_blocked_leaf_page; using turtle_kv::PackedBlockedLeafPage; using turtle_kv::random_str; @@ -72,6 +73,9 @@ TEST(TreePackedBlockedLeafPageTest, Random) std::geometric_distribution pick_key_size{0.7}; std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + usize total_keys = 0; + usize total_bytes = 0; + for (usize seed = 0; seed < kNumSeeds; ++seed) { std::default_random_engine rng{seed}; @@ -135,6 +139,9 @@ TEST(TreePackedBlockedLeafPageTest, Random) break; } + ++total_keys; + total_bytes += edit_size; + edits.push_back(edit); total_edits_size += edit_size; max_edit_size = new_max_edit_size; @@ -156,11 +163,30 @@ TEST(TreePackedBlockedLeafPageTest, Random) MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; - StatusOr packed_leaf = + StatusOr status_or_packed_leaf = pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); - ASSERT_TRUE(packed_leaf.ok()) << BATT_INSPECT(packed_leaf.status()); + ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); + + const PackedBlockedLeafPage& packed_leaf = **status_or_packed_leaf; + + //+++++++++++-+-+--+----- --- -- - - - - + // + // + { + auto packed_items = packed_leaf.items_seq(); + using Item = decltype(*packed_items.peek()); + for (const EditView& edit : edits) { + Optional next_packed = packed_items.next(); + + ASSERT_TRUE(next_packed.has_value()); + ASSERT_EQ(get_key(*next_packed), get_key(edit)); + ASSERT_EQ(get_value(*next_packed), get_value(edit)); + } + } } + + std::cerr << BATT_INSPECT(total_keys) << BATT_INSPECT(total_bytes) << std::endl; } } // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp index 6485e00..78b55ad 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -24,6 +24,9 @@ #include #include +#include + +#include #include @@ -36,6 +39,10 @@ struct PackedLeafBlock { //+++++++++++-+-+--+----- --- -- - - - - + class Iterator; + + //+++++++++++-+-+--+----- --- -- - - - - + big_u32 magic; // +4 = 4 little_u16 shared_prefix_size; // +2 = 6 PackedKeyValueSlotPtr items_[1]; // +2 = 8 @@ -144,6 +151,67 @@ struct PackedLeafBlock { static_assert(sizeof(PackedLeafBlock) == 8); +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class PackedLeafBlock::Iterator + : public boost::iterator_facade< // + PackedLeafBlock::Iterator, // <- Derived + const PackedLeafBlock, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedLeafBlock&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = Iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedLeafBlock; + using reference = const PackedLeafBlock&; + + Iterator() = default; + + explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept + : block_{block} + , block_size_{block_size} + { + } + + reference dereference() const + { + return *this->block_; + } + + bool equal(const Self& other) const + { + return this->block_ == other.block_ && this->block_size_ == other.block_size_; + } + + void increment() + { + this->advance(1); + } + + void decrement() + { + this->advance(-1); + } + + void advance(isize delta) + { + this->block_ = static_cast( + advance_pointer(this->block_, delta * this->block_size_)); + } + + isize distance_to(const Self& other) const + { + return (byte_distance(this->block_, other.block_)) / this->block_size_; + } + + private: + const PackedLeafBlock* block_ = nullptr; + isize block_size_ = 0; +}; + //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- // struct PackedLeafBlockStats { diff --git a/src/turtle_kv/tree/packed_leaf_block.ipp b/src/turtle_kv/tree/packed_leaf_block.ipp index d418c82..10c90fb 100644 --- a/src/turtle_kv/tree/packed_leaf_block.ipp +++ b/src/turtle_kv/tree/packed_leaf_block.ipp @@ -89,22 +89,28 @@ inline StatusOr pack_leaf_block(const RangeT& src, PackedLeafBlock* block = static_cast(dst.data()); { block->magic = PackedLeafBlock::kMagic; - block->items_[0].offset = - byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes)); + block->items_[0].offset = BATT_CHECKED_CAST( + u32, + byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes))); } //----- --- -- - - - - // Pack all slot data. + // PackedKeyValueSlotPtr* pp_slot = block->items_; void* p_slot = const_cast(pp_slot->get()); + void* const dst_end = advance_pointer(dst.data(), dst.size()); IterT src_iter = std::begin(src); const IterT src_end = std::next(src_iter, stats.item_count); for (; src_iter != src_end; ++src_iter) { const usize slot_size = pack_key_value_slot(*src_iter, p_slot); p_slot = advance_pointer(p_slot, slot_size); + BATT_CHECK_LE(p_slot, dst_end); ++pp_slot; pp_slot->offset = byte_distance(pp_slot, p_slot); + + BATT_CHECK_EQ((void*)pp_slot->get(), (void*)p_slot); } //----- --- -- - - - - From 511d2fe3f0df2d8ebdfbe5083b11325e1d6cbb0e Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Wed, 24 Jun 2026 10:45:37 -0400 Subject: [PATCH 06/20] Refactor. --- src/turtle_kv/core/packed_key_value_slot.hpp | 5 + .../core/packed_key_value_slot_slice.hpp | 36 ++ src/turtle_kv/tree/in_memory_node.test.cpp | 11 +- .../tree/in_memory_node_segmented_level.cpp | 2 + .../leaf/blocked_leaf_page_loader.concept.hpp | 30 ++ .../tree/leaf/packed_blocked_leaf_page.cpp | 49 +++ .../tree/leaf/packed_blocked_leaf_page.hpp | 337 ++++++++++++++++ .../{ => leaf}/packed_blocked_leaf_page.ipp | 138 ++++++- ...packed_blocked_leaf_page.item_iterator.hpp | 183 +++++++++ ..._blocked_leaf_page.sharded_live_ranges.hpp | 56 +++ ..._blocked_leaf_page.sharded_live_ranges.ipp | 170 ++++++++ .../leaf/packed_blocked_leaf_page.test.cpp | 366 ++++++++++++++++++ .../tree/{ => leaf}/packed_leaf_block.hpp | 91 +---- .../tree/{ => leaf}/packed_leaf_block.ipp | 43 +- .../tree/leaf/packed_leaf_block.iterator.hpp | 104 +++++ .../{ => leaf}/packed_leaf_block.test.cpp | 10 +- .../tree/leaf/packed_leaf_block_stats.hpp | 38 ++ .../tree/leaf/packed_leaf_block_stats.ipp | 56 +++ .../tree/packed_blocked_leaf_page.cpp | 13 - .../tree/packed_blocked_leaf_page.hpp | 145 ------- .../tree/packed_blocked_leaf_page.test.cpp | 192 --------- .../tree/packed_leaf_block_scanner.hpp | 75 ++++ src/turtle_kv/tree/packed_node_page.cpp | 9 + src/turtle_kv/tree/testing/fake_segment.hpp | 11 +- .../util/packed_piecewise_filter_view.hpp | 305 +++++++++++++++ src/turtle_kv/util/piecewise_filter.hpp | 48 ++- src/turtle_kv/util/piecewise_filter.ipp | 132 ++++--- .../util/piecewise_filter.live_subranges.hpp | 88 +++++ src/turtle_kv/util/piecewise_filter.test.cpp | 75 +++- ...piecewise_filter_storage_model.concept.hpp | 60 +++ 30 files changed, 2316 insertions(+), 562 deletions(-) create mode 100644 src/turtle_kv/core/packed_key_value_slot_slice.hpp create mode 100644 src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp rename src/turtle_kv/tree/{ => leaf}/packed_blocked_leaf_page.ipp (59%) create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp rename src/turtle_kv/tree/{ => leaf}/packed_leaf_block.hpp (68%) rename src/turtle_kv/tree/{ => leaf}/packed_leaf_block.ipp (81%) create mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp rename src/turtle_kv/tree/{ => leaf}/packed_leaf_block.test.cpp (96%) create mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp delete mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.cpp delete mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.hpp delete mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp create mode 100644 src/turtle_kv/tree/packed_leaf_block_scanner.hpp create mode 100644 src/turtle_kv/util/packed_piecewise_filter_view.hpp create mode 100644 src/turtle_kv/util/piecewise_filter.live_subranges.hpp create mode 100644 src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index ce42137..342f315 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -142,6 +142,11 @@ inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr* }; } +inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr& p_slot_ref) noexcept +{ + return to_key_value_slot_ref(std::addressof(p_slot_ref)); +} + inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffer) noexcept { return PackedKeyValueSlotRef{ diff --git a/src/turtle_kv/core/packed_key_value_slot_slice.hpp b/src/turtle_kv/core/packed_key_value_slot_slice.hpp new file mode 100644 index 0000000..7f2ee39 --- /dev/null +++ b/src/turtle_kv/core/packed_key_value_slot_slice.hpp @@ -0,0 +1,36 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_CORE_PACKED_KEY_VALUE_SLOT_SLICE_HPP + +#include + +#include + +#include + +namespace turtle_kv { + +using PackedKeyValueSlotSlice = std::variant< // + Slice, + Slice>; + +struct ToPackedKeyValueSlotSlice { + PackedKeyValueSlotSlice operator()(const Slice& ref_slice) + { + return PackedKeyValueSlotSlice{ref_slice}; + } + + PackedKeyValueSlotSlice operator()(const Slice& ptr_slice) + { + return PackedKeyValueSlotSlice{ptr_slice}; + } +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/in_memory_node.test.cpp b/src/turtle_kv/tree/in_memory_node.test.cpp index 22b3d56..52a5dd9 100644 --- a/src/turtle_kv/tree/in_memory_node.test.cpp +++ b/src/turtle_kv/tree/in_memory_node.test.cpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #include // #include @@ -12,9 +20,10 @@ #include #include +#include #include -#include +#include #include #include diff --git a/src/turtle_kv/tree/in_memory_node_segmented_level.cpp b/src/turtle_kv/tree/in_memory_node_segmented_level.cpp index bba1672..19edf8d 100644 --- a/src/turtle_kv/tree/in_memory_node_segmented_level.cpp +++ b/src/turtle_kv/tree/in_memory_node_segmented_level.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp new file mode 100644 index 0000000..f9832cf --- /dev/null +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp @@ -0,0 +1,30 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_BLOCKED_LEAF_PAGE_LOADER_CONCEPT_HPP + +#include + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template +concept BlockedLeafPageLoader = requires(T& loader, llfs::PageId page_id, BlockIndex block_i) { + loader.release_block(page_id, block_i); + { loader.load_block(page_id, block_i) } -> std::convertible_to>; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp new file mode 100644 index 0000000..5e61632 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp @@ -0,0 +1,49 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include "packed_blocked_leaf_page.hpp" +// + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept +{ + const usize space_after_header = + leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); + + const usize max_block_count = space_after_header / block_size; + + const usize block_starts_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; + + const usize space_after_block_starts = space_after_header - block_starts_size; + + const usize max_art_size = max_key_size * max_block_count * 2; + + const usize space_after_art = space_after_block_starts - max_art_size; + + BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; + const usize space_for_blocks = space_after_art & ~(block_size - 1); + const usize block_count = space_for_blocks / block_size; + + const usize max_wasted_per_block = max_edit_size - 1; + const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; + + const usize final_estimate = block_count * min_block_capacity; + + BATT_CHECK_GT(leaf_size, final_estimate); + + return final_estimate; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp new file mode 100644 index 0000000..b043193 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp @@ -0,0 +1,337 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP + +#include "packed_leaf_block.hpp" +#include "packed_leaf_block.iterator.hpp" + +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +#include + +namespace turtle_kv { + +// Forward-declaration. +// +struct PackedBlockedLeafPage; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Packs a blocked leaf page with the passed block size, containing the passed key/value + * pairs, into the passed buffer. + */ +template +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Header for a packed leaf page with blocked structure. + */ +struct PackedBlockedLeafPage // +{ + /** \brief Must be the first 8 bytes of the header. \see PackedBlockedLeagPage::magic + */ + static constexpr u64 kMagic = 0x6456beb7f9558445ull; + + //+++++++++++-+-+--+----- --- -- - - - - + + using BlockIterator = PackedLeafBlock::Iterator; + class ItemIterator; + + using BlockItemsSeq = PackedLeafBlock::BlockItemsSeq; + + struct ItemsSeqFromBlock { + BlockItemsSeq operator()(const PackedLeafBlock& block) const + { + return block.items_seq(); + } + }; + + using BlocksSeq = batt::SubRangeSeq>; + using ItemsSeq = batt::seq::Flatten>; + + struct SlotSliceFromBlock { + PackedKeyValueSlotSlice operator()(const PackedLeafBlock& block) const + { + return {block.items_slice()}; + } + }; + + using SlotSliceSeq = batt::seq::Map; + + template FilterModelT> + class ShardedLiveRanges; + + class HeaderShardView; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static usize packed_edit_size(const EditT& edit) noexcept + { + return PackedLeafBlock::packed_edit_size(edit); + } + + static usize estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept; + + /** \brief Returns the passed buffer's memory region, validated as a PackedBlockedLeafPage and + * cast to `const PackedBlockedLeafPage &`. + */ + static const PackedBlockedLeafPage& view_of(const ConstBuffer& buffer) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u64 magic; // +8 -> 8 + little_u32 total_packed_size; // +4 -> 12 + little_u32 blocks_per_art_key; // +4 -> 16 + little_u32 block_size_bytes; // +4 -> 20 + llfs::PackedPointer block0; // +4 -> 24 + + /** \brief Pointer to array that stores, for each block, the starting item index relative to the + * entire leaf. + */ + llfs::PackedPointer> block_starting_item; // +4 -> 28 + + /** \brief Pointer to packed ART index. + */ + llfs::PackedPointer art_block_index; // +4 -> 32 + + //+++++++++++-+-+--+----- --- -- - - - - + + llfs::PageId page_id() const noexcept + { + return (reinterpret_cast(this) - 1)->page_id.unpack(); + } + + Optional page_shard_id_for_block(llfs::PageCache& page_cache, + usize i, + llfs::PageId leaf_page_id) const noexcept + { + const usize block_begin_offset = this->block_page_offset(i); + const usize block_end_offset = block_begin_offset + this->block_size_bytes; + + return page_cache.page_shard_id_for(leaf_page_id, + Interval{block_begin_offset, block_end_offset}); + } + + Optional page_shard_id_for_block(llfs::PageCache& page_cache, + usize i) const noexcept + { + return this->page_shard_id_for_block(page_cache, i, this->page_id()); + } + + usize min_header_shard_size() const noexcept + { + return this->block_page_offset(0); + } + + //----- --- -- - - - - + + usize block_page_offset(usize i) const noexcept + { + return sizeof(llfs::PackedPageHeader) + offsetof(PackedBlockedLeafPage, block0) + + this->block0.offset + i * this->block_size_bytes; + } + + usize block_count() const noexcept + { + return this->block_starting_item->size() - 1; + } + + BlockIterator blocks_begin() const noexcept + { + return BlockIterator{this->block0.get(), (isize)this->block_size_bytes.value()}; + } + + const PackedLeafBlock& blocks_front() const + { + return *this->blocks_begin(); + } + + BlockIterator blocks_end() const noexcept + { + return this->blocks_begin() + this->block_count(); + } + + const PackedLeafBlock& blocks_back() const + { + return *(this->blocks_begin() + (this->block_count() - 1)); + } + + auto blocks() const noexcept + { + return std::ranges::subrange(this->blocks_begin(), this->blocks_end()); + } + + const PackedLeafBlock& block_at(usize block_i) const noexcept + { + return *(this->blocks_begin() + block_i); + } + + BlocksSeq blocks_seq() const noexcept + { + return batt::as_seq(this->blocks()); + } + + Interval item_index_range_of_block(usize i) const noexcept + { + return Interval{ + (*this->block_starting_item)[i].value(), + (*this->block_starting_item)[i + 1].value(), + }; + } + + /** \brief Returns the index of the block that would contain the given key, if it is present in + * this page. + * + * Always returns a valid block index (i.e., less-than this->block_count()) + */ + usize find_block_index_containing_key(const KeyView& key) const noexcept; + + /** \brief Returns a block iterator to the block that would contain the given key, if it is + * present in this page. + * + * Always returns a valid block iterator. + */ + BlockIterator find_block_containing_key(const KeyView& key) const noexcept; + + //----- --- -- - - - - + + /** \brief Returns the number of key/value pairs in this page. + */ + usize item_count() const noexcept + { + return this->block_starting_item->back(); + } + + /** \brief Returns a sequence of all items in the page, in key order. + */ + ItemsSeq items_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map(ItemsSeqFromBlock{}) | batt::seq::flatten(); + } + + /** \brief Returns an item iterator to the first item in the page. + */ + ItemIterator items_begin() const noexcept; + + /** \brief Returns an item iterator one-past the last item in the page. + */ + ItemIterator items_end() const noexcept; + + /** \brief Returns an item iterator to the i-th item in the page. + */ + ItemIterator item_at(usize i) const noexcept; + + /** \brief Returns an iterator to the given key in this page if found or nullptr if not found. + */ + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; + + /** \brief Returns an iterator to the first item in this page whose key is not less than `key`; + * if all keys in the page are less than `key`, returns `this->items_end()`. + */ + ItemIterator lower_bound(const KeyView& key) const noexcept; + + //----- --- -- - - - - + + KeyView min_key() const noexcept + { + return this->blocks_front().min_key(); + } + + KeyView max_key() const noexcept + { + return this->blocks_back().max_key(); + } + + SlotSliceSeq slot_slice_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map(SlotSliceFromBlock{}); + } + + template FilterModelT> + ShardedLiveRanges sharded_live_ranges( + const BasicPiecewiseFilter& filter, + const Interval& subrange) const noexcept; +}; + +static_assert(sizeof(PackedBlockedLeafPage) == 32); + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief A view of the header prefix of a PackedBlockedLeafPage. + */ +class PackedBlockedLeafPage::HeaderShardView +{ + public: + using Self = HeaderShardView; + + //+++++++++++-+-+--+----- --- -- - - - - + + static Self view_of(const ConstBuffer& buffer) noexcept + { + const PackedBlockedLeafPage& leaf = PackedBlockedLeafPage::view_of(buffer); + BATT_CHECK_GE(buffer.size(), leaf.min_header_shard_size()); + + return Self{leaf, buffer.size()}; + } + + //+++++++++++-+-+--+----- --- -- - - - - + +#if 0 + Seq load_slices(llfs::PageLoader& loader, + Optional first_key, + Optional last_key, + Optional first_index, + Optional last_index, + const PiecewiseFilter& filter); +#endif + + //+++++++++++-+-+--+----- --- -- - - - - + private: + explicit HeaderShardView(const PackedBlockedLeafPage& leaf, usize header_shard_size) noexcept + : leaf_{&leaf} + , header_shard_size_{header_shard_size} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + const PackedBlockedLeafPage* leaf_; + usize header_shard_size_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp similarity index 59% rename from src/turtle_kv/tree/packed_blocked_leaf_page.ipp rename to src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp index 3d52be1..9d415f0 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp @@ -7,13 +7,15 @@ //+++++++++++-+-+--+----- --- -- - - - - #pragma once -#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_IPP #include "packed_blocked_leaf_page.hpp" +#include "packed_blocked_leaf_page.item_iterator.hpp" #include #include +#include #include @@ -65,11 +67,9 @@ StatusOr pack_blocked_leaf_page(const usize block_size, dst_remaining += sizeof(PackedBlockedLeafPage); { leaf_header->magic = PackedBlockedLeafPage::kMagic; - leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); leaf_header->total_packed_size = 0; leaf_header->blocks_per_art_key = 0; leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); - leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); leaf_header->block0.offset = 0; leaf_header->block_starting_item.offset = 0; leaf_header->art_block_index.offset = 0; @@ -80,12 +80,12 @@ StatusOr pack_blocked_leaf_page(const usize block_size, // { const usize block_starting_item_array_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; + sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); auto* block_starting_item = static_cast*>(dst_remaining.data()); dst_remaining += block_starting_item_array_size; - block_starting_item->initialize(block_stats.size()); + block_starting_item->initialize(block_count + 1); little_u32* block_start = block_starting_item->data(); u32 item_i = 0; @@ -94,6 +94,7 @@ StatusOr pack_blocked_leaf_page(const usize block_size, item_i += stats.item_count; ++block_start; } + *block_start = item_count; leaf_header->block_starting_item.reset_unsafe(block_starting_item); } @@ -105,13 +106,25 @@ StatusOr pack_blocked_leaf_page(const usize block_size, const usize space_for_art = dst_remaining.size() - block_size * block_count; SmallVec art_keys; usize blocks_per_art_key = 1; + //----- --- -- - - - - + const auto items = std::begin(src_items); + const auto key_at = [&items](usize i) { + return get_key(*(items + i)); + }; + //----- --- -- - - - - for (;;) { art_keys.clear(); - auto items = std::begin(src_items); for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { const usize item_i = block_starting_item[block_i]; BATT_CHECK_LT(item_i, item_count); - art_keys.emplace_back(get_key(*(items + item_i))); + BATT_CHECK_GT(item_i, 0); + + KeyView k0 = key_at(item_i - 1); + KeyView k1 = key_at(item_i); + KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); + KeyView min_k1{k1.data(), common_prefix.size() + 1}; + + art_keys.emplace_back(min_k1); } using artc::packed::PackedARTBuilder; @@ -189,4 +202,115 @@ StatusOr pack_blocked_leaf_page(const usize block_size, return leaf_header; } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ const PackedBlockedLeafPage& PackedBlockedLeafPage::view_of( + const ConstBuffer& buffer) noexcept +{ + BATT_CHECK_GT(buffer.size(), sizeof(PackedBlockedLeafPage) + sizeof(llfs::PackedPageHeader)); + + auto* packed = static_cast( + advance_pointer(buffer.data(), sizeof(llfs::PackedPageHeader))); + + BATT_CHECK_EQ(packed->magic, PackedBlockedLeafPage::kMagic); + + return *packed; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_begin() const noexcept +{ + auto first_block = this->blocks_begin(); + return ItemIterator{first_block, first_block->items_begin()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_end() const noexcept +{ + auto last_block = this->blocks_end(); + return ItemIterator{last_block, last_block->items_begin()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::item_at(usize i) const noexcept +{ + const llfs::PackedArray& starts = *this->block_starting_item; + + BATT_CHECK_NE(starts.size(), 0); + BATT_CHECK_EQ(starts.front(), 0); + + const auto iter = std::prev(std::upper_bound(starts.begin(), starts.end(), i)); + const isize item_pos_in_block = i - *iter; + const isize block_i = std::distance(starts.begin(), iter); + auto block_iter = this->blocks_begin() + block_i; + + return ItemIterator{block_iter, block_iter->items_begin() + item_pos_in_block}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline usize PackedBlockedLeafPage::find_block_index_containing_key( + const KeyView& key) const noexcept +{ + using artc::packed::find_lower_bound_rank; + using artc::packed::LowerBoundRank; + + LowerBoundRank result = find_lower_bound_rank(this->art_block_index.get(), key); + + const usize part_i = result.exact ? (result.rank + 1) : result.rank; + const usize block_i = part_i * this->blocks_per_art_key; + + BATT_CHECK_LT(block_i, this->block_count()); + + return block_i; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedLeafBlock::Iterator PackedBlockedLeafPage::find_block_containing_key( + const KeyView& key) const noexcept +{ + return this->blocks_begin() + this->find_block_index_containing_key(key); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedBlockedLeafPage::find_key( + const KeyView& key) const noexcept +{ + return this->find_block_containing_key(key)->find_key(key); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::lower_bound( + const KeyView& key) const noexcept +{ + auto block_iter = this->find_block_containing_key(key); + + const PackedKeyValueSlotPtr* p_slot = block_iter->lower_bound(key); + if (p_slot == block_iter->items_end()) { + ++block_iter; + return ItemIterator{block_iter, block_iter->items_begin()}; + } + + return ItemIterator{block_iter, p_slot}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline PackedBlockedLeafPage::ShardedLiveRanges +PackedBlockedLeafPage::sharded_live_ranges(const BasicPiecewiseFilter& filter, + const Interval& subrange) const noexcept +{ + return ShardedLiveRanges{ + this->block_starting_item.get(), + filter.live_subranges_of(subrange), + }; +} + } // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp new file mode 100644 index 0000000..664adc6 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp @@ -0,0 +1,183 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCK_LEAF_PAGE_ITEM_ITERATOR_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Iterator over the items in a blocked leaf page. + */ +class PackedBlockedLeafPage::ItemIterator + : public boost::iterator_facade< // + PackedBlockedLeafPage::ItemIterator, // <- Derived + const PackedKeyValueSlotPtr, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedKeyValueSlotPtr&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = ItemIterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedKeyValueSlotPtr; + using reference = value_type&; + + ItemIterator() = default; + + explicit ItemIterator(BlockIterator block_iter, const PackedKeyValueSlotPtr* slot) noexcept + : block_iter_{block_iter} + , slot_{slot} + { + } + + reference dereference() const + { + return *this->slot_; + } + + bool equal(const Self& other) const + { + return this->block_iter_ == other.block_iter_ && this->slot_ == other.slot_; + } + + void increment() + { + ++this->slot_; + if (this->slot_ == this->block_iter_->items_end()) { + ++this->block_iter_; + this->slot_ = this->block_iter_->items_begin(); + } + } + + void decrement() + { + if (this->slot_ == this->block_iter_->items_begin()) { + --this->block_iter_; + this->slot_ = std::prev(this->block_iter_->items_end()); + } else { + --this->slot_; + } + } + + void advance(isize delta) + { + if (delta == 0) { + return; + } + + isize pos_in_block = this->get_item_pos_in_block(); + + if (delta > 0) { + // Keep stepping through the page one block at a time until we reduce delta to zero. + // + while (delta != 0) { + // Figure out where the current slot is in the current block. + // + const isize remaining_in_block = this->get_remaining_in_block(pos_in_block); + BATT_CHECK_GT(remaining_in_block, 0); + + // If the remaining delta is inside the block, advance the slot pointer and we are done! + // + if (delta < remaining_in_block) { + this->slot_ += delta; + break; + } + // Else reduce delta by the number of slots after this one in the current block. + // + delta -= remaining_in_block; + + // Advance to the next block, resetting the slot pointer. + // + ++this->block_iter_; + this->slot_ = this->block_iter_->items_begin(); + pos_in_block = 0; + } + + } else { // delta < 0 + + delta = -delta; + while (delta != 0) { + BATT_CHECK_GE(pos_in_block, 0); + + // If the remaining delta is inside the block, update the slot pointer and we are done! + // + if (delta <= pos_in_block) { + this->slot_ -= delta; + break; + } + // Else reduce delta by the number of slots before this one in the current block, plus one + // for the current slot. + // + delta -= (pos_in_block + 1); + + // Move to the last item of the previous block. + // + --this->block_iter_; + this->slot_ = std::prev(this->block_iter_->items_end()); + pos_in_block = this->block_iter_->item_count() - 1; + } + } + } + + isize distance_to(const Self& other) const + { + if (this->block_iter_ == other.block_iter_) { + return std::distance(this->slot_, other.slot_); + } + + // Step forward counting items in each block until we reach the same block. + // + if (this->block_iter_ < other.block_iter_) { + isize delta = this->get_remaining_in_block(); + for (auto iter = std::next(this->block_iter_); iter != other.block_iter_; ++iter) { + delta += iter->item_count(); + } + delta += other.get_item_pos_in_block(); + return delta; + } + // Else step backward. + // + isize delta = this->get_item_pos_in_block(); + for (auto iter = std::prev(this->block_iter_); iter != other.block_iter_; --iter) { + delta += iter->item_count(); + } + delta += other.get_remaining_in_block(); + return -delta; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + isize get_item_pos_in_block() const noexcept + { + return std::distance(this->block_iter_->items_begin(), this->slot_); + } + + isize get_remaining_in_block(isize pos_in_block) const noexcept + { + return this->block_iter_->item_count() - pos_in_block; + } + + isize get_remaining_in_block() const noexcept + { + return this->get_remaining_in_block(this->get_item_pos_in_block()); + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + BlockIterator block_iter_; + const PackedKeyValueSlotPtr* slot_ = nullptr; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp new file mode 100644 index 0000000..d2a9353 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp @@ -0,0 +1,56 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template FilterModelT> +class PackedBlockedLeafPage::ShardedLiveRanges +{ + public: + using Item = std::pair /*live_item_range*/>; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit ShardedLiveRanges( + const llfs::PackedArray* block_starts, + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + Optional peek(); + + Optional next(); + + //+++++++++++-+-+--+----- --- -- - - - - + private: + void advance(); + + usize get_block_count() const noexcept; + + Interval get_block_range(usize block_i) const noexcept; + + void clear_current_range(); + + //+++++++++++-+-+--+----- --- -- - - - - + + const llfs::PackedArray* block_starts_; + usize block_index_; + BasicPiecewiseFilter::LiveSubranges filter_live_ranges_; + Interval current_range_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp new file mode 100644 index 0000000..9192cb2 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp @@ -0,0 +1,170 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_IPP + +#include "packed_blocked_leaf_page.sharded_live_ranges.hpp" + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline /*explicit*/ PackedBlockedLeafPage::ShardedLiveRanges::ShardedLiveRanges( + const llfs::PackedArray* block_starts, + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept + : block_starts_{block_starts} + , block_index_{0} + , filter_live_ranges_{std::move(filter_live_ranges)} + , current_range_{0, 0} +{ + this->advance(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline auto PackedBlockedLeafPage::ShardedLiveRanges::peek() -> Optional +{ + if (this->current_range_.empty()) { + return None; + } + return std::make_pair(this->block_index_, this->current_range_); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline auto PackedBlockedLeafPage::ShardedLiveRanges::next() -> Optional +{ + Optional item = this->peek(); + if (item) { + this->advance(); + // std::cerr << ".. " << BATT_INSPECT(this->current_range_) << std::endl; + } + return item; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline void PackedBlockedLeafPage::ShardedLiveRanges::advance() +{ + this->current_range_.lower_bound = this->current_range_.upper_bound; + + Optional> filter_range = this->filter_live_ranges_.peek(); + if (!filter_range) { + return; + } + + // Consume the current range from both current and filter. + // + BATT_CHECK_LE(this->current_range_.upper_bound, filter_range->upper_bound); + filter_range->lower_bound = std::max(filter_range->lower_bound, // + this->current_range_.upper_bound); + + // If the filter range has been consumed, move to the next filter range. + // + if (filter_range->empty()) { + this->filter_live_ranges_.next(); + filter_range = this->filter_live_ranges_.peek(); + + // Once we run out of filter live ranges, we are done. + // + if (!filter_range) { + return; + } + } + + // std::cerr << ".. " << BATT_INSPECT(filter_range) << std::endl; + + const usize block_count = this->get_block_count(); + BATT_CHECK_LT(this->block_index_, block_count); + const usize blocks_remaining = block_count - this->block_index_; + const usize max_probe_steps = BATT_CHECKED_CAST(usize, batt::log2_ceil(blocks_remaining)); + const usize linear_probe_end = this->block_index_ + max_probe_steps; + bool tried_binary_search = false; + + while (this->block_index_ < block_count) { + // Test the intersection of the current block's range with the current filter range; + // if they intersect, then stop here. + // + this->current_range_ = this->get_block_range(this->block_index_) // + .intersection_with(*filter_range); + + if (!this->current_range_.empty()) { + return; + } + + // If we can, continue the linear probe. + // + ++this->block_index_; + if (this->block_index_ <= linear_probe_end) { + continue; + } + + // The binary search fall-back *must* succeed! If we ever find we are about to try it a + // second time, panic. + // + BATT_CHECK(!tried_binary_search); + + // Fall-back to binary search. + // + auto indices = boost::irange(this->block_index_, block_count); + auto iter = + std::lower_bound(indices.begin(), + indices.end(), + *filter_range, + [this](usize i, const Interval& range) { + return Interval::LinearOrder{}(this->get_block_range(i), range); + }); + + // If the first block that might intersect with the filter range is beyond the end of the + // blocks, then we are done. + // + if (iter == indices.end()) { + this->clear_current_range(); + return; + } + + this->block_index_ = *iter; + tried_binary_search = true; + } +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline usize PackedBlockedLeafPage::ShardedLiveRanges::get_block_count() + const noexcept +{ + return this->block_starts_->size() - 1; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline Interval PackedBlockedLeafPage::ShardedLiveRanges::get_block_range( + usize block_i) const noexcept +{ + return Interval{ + (*this->block_starts_)[block_i], + (*this->block_starts_)[block_i + 1], + }; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline void PackedBlockedLeafPage::ShardedLiveRanges::clear_current_range() +{ + this->current_range_.lower_bound = this->current_range_.upper_bound; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp new file mode 100644 index 0000000..fd9ca6f --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -0,0 +1,366 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; +using namespace batt::constants; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::Interval; +using turtle_kv::KeyOrder; +using turtle_kv::KeyView; +using turtle_kv::Optional; +using turtle_kv::pack_blocked_leaf_page; +using turtle_kv::PackedBlockedLeafPage; +using turtle_kv::PackedKeyValueSlotPtr; +using turtle_kv::PiecewiseFilter; +using turtle_kv::random_str; +using turtle_kv::ValueView; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// Plan: +// 1. For different random seeds: +// - generate random set of prefixes (~10% of total keys) +// - generate keys using prefixes, with random values +// - sort +// - pack leaf; verify: +// a. all packed keys present and have right values +// b. any unpacked keys at end missing +// c. randomly generated non-present keys not found +// +TEST(TreePackedBlockedLeafPageTest, Random) +{ + const usize kNumSeeds = 10000; + const usize kLeafPageSize = 1 * kMiB; + const usize kNumPrefixes = 1000; + const usize kMinPrefixSize = 0; + const usize kMaxPrefixSize = 8; + const usize kMinKeySize = 4; + const usize kMaxKeySize = 48; + const usize kMinValueSize = 0; + const usize kMaxValueSize = 200; + const usize kBlockSize = 8192; + + BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); + + std::uniform_int_distribution pick_pct{0, 99}; + std::geometric_distribution pick_prefix_size{0.5}; + std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + + for (usize seed = 803; seed < kNumSeeds; ++seed) { + LOG(INFO) << BATT_INSPECT(seed); + + std::default_random_engine rng{seed}; + + StableStringStore strings; + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate prefixes + // + std::vector prefixes; + { + std::unordered_set used_prefixes; + while (prefixes.size() < kNumPrefixes) { + std::string_view prefix = + random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + + if (used_prefixes.count(prefix)) { + continue; + } + prefixes.push_back(prefix); + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate edits. + // + std::vector edits; + { + usize max_edit_size = 0; + usize max_key_size = 0; + usize total_edits_size = 0; + + std::unordered_set used_keys; + for (;;) { + std::string_view prefix = prefixes[pick_prefix(rng)]; + + std::string_view key = + random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); + + if (used_keys.count(key)) { + continue; + } + used_keys.insert(key); + + std::string_view value = + random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + EditView edit{key, ValueView::from_str(value)}; + + const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); + + const usize new_max_edit_size = std::max(max_edit_size, edit_size); + const usize new_max_key_size = std::max(max_key_size, key.size()); + + const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, + kBlockSize, + new_max_key_size, + new_max_edit_size); + + // Stop as soon as adding the next key would exceed the estimated space. + // + if (edit_size + total_edits_size > space_available) { + break; + } + + edits.push_back(edit); + total_edits_size += edit_size; + max_edit_size = new_max_edit_size; + max_key_size = new_max_key_size; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Sort edits by key. + // + std::sort(edits.begin(), edits.end(), KeyOrder{}); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack a blocked leaf page. + // + using StorageUnit = std::aligned_storage_t<4096, 4096>; + std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); + ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); + + MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; + + StatusOr status_or_packed_leaf = + pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); + + ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); + + const PackedBlockedLeafPage& packed_leaf = PackedBlockedLeafPage::view_of(leaf_buffer); + + ASSERT_EQ(&packed_leaf, *status_or_packed_leaf); + ASSERT_EQ(packed_leaf.min_key(), get_key(edits.front())); + ASSERT_EQ(packed_leaf.max_key(), get_key(edits.back())); + + //+++++++++++-+-+--+----- --- -- - - - - + // Scan over all items in the packed leaf to make sure they are all there. + // + { + PackedBlockedLeafPage::ItemIterator item_iter = packed_leaf.items_begin(); + PackedBlockedLeafPage::ItemIterator items_end = packed_leaf.items_end(); + + std::vector> past_items; + + auto packed_items = packed_leaf.items_seq(); + using Item = decltype(*packed_items.peek()); + Optional prev_key; + Optional prev_item_iter; + + isize position = 0; + + for (const EditView& edit : edits) { + Optional next_packed = packed_items.next(); + + if (prev_key) { + ASSERT_GT(get_key(edit), *prev_key); + } + prev_key = get_key(edit); + + ASSERT_TRUE(next_packed.has_value()); + ASSERT_EQ(get_key(*next_packed), get_key(edit)); + ASSERT_EQ(get_value(*next_packed), get_value(edit)); + + // Test PackedBlockedLeafPage::find_key. + // + const PackedKeyValueSlotPtr* found = packed_leaf.find_key(get_key(edit)); + + ASSERT_NE(found, nullptr); + ASSERT_EQ(found, std::addressof(*item_iter)); + ASSERT_EQ(get_key(*found), get_key(edit)); + ASSERT_EQ(get_value(*found), get_value(edit)) + << BATT_INSPECT_STR(get_key(*found)) << BATT_INSPECT(edit); + + // Test PackedBlockedLeafPage::lower_bound. + // + { + PackedBlockedLeafPage::ItemIterator lb_iter = packed_leaf.lower_bound(get_key(edit)); + + ASSERT_NE(lb_iter, items_end); + ASSERT_EQ(get_key(*lb_iter), get_key(edit)); + ASSERT_EQ(get_value(*lb_iter), get_value(edit)); + } + + ASSERT_EQ(packed_leaf.item_at(position), item_iter); + + ASSERT_NE(item_iter, items_end); + ASSERT_LT(item_iter, items_end); + ASSERT_EQ(std::distance(packed_leaf.items_begin(), item_iter), position); + + if (pick_pct(rng) < 1) { + past_items.push_back(std::make_pair(item_iter, position)); + } + + for (const auto& [past_iter, past_position] : past_items) { + ASSERT_EQ(std::distance(past_iter, item_iter), position - past_position); + ASSERT_EQ(std::distance(item_iter, past_iter), past_position - position); + ASSERT_EQ(past_iter + (position - past_position), item_iter); + ASSERT_EQ(item_iter - (position - past_position), past_iter); + ASSERT_LE(past_iter, item_iter); + ASSERT_GE(item_iter, past_iter) << BATT_INSPECT(position) << BATT_INSPECT(past_position); + } + + if (prev_item_iter) { + ASSERT_EQ(std::next(*prev_item_iter), item_iter); + ASSERT_EQ(*prev_item_iter, std::prev(item_iter)); + } + + prev_item_iter = item_iter; + ++item_iter; + ++position; + } + ASSERT_FALSE(packed_items.peek().has_value()); + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Test ShardedLiveRanges. + // + { + for (usize j = 0; j < 1000; ++j) { + // Drop up to 64 sub-ranges of the leaf. + // + for (usize drop_count = 0; drop_count < 64; ++drop_count) { + std::vector> dropped_ranges; + PiecewiseFilter leaf_filter; + + usize drops_remaining = drop_count; + const u32 item_count = packed_leaf.item_count(); + + u32 next_droppable = 0; + u32 items_dropped = 0; + + for (usize drop_i = 0; drop_i < drop_count; ++drop_i) { + BATT_CHECK_GE(next_droppable, 0); + BATT_CHECK_LT(next_droppable, item_count); + + std::uniform_int_distribution pick_lower_bound{ + next_droppable, + item_count - (drops_remaining * 2 - 1), + }; + const u32 lower_bound_i = pick_lower_bound(rng); + + std::uniform_int_distribution pick_upper_bound{ + lower_bound_i + 1, + item_count - (drops_remaining * 2 - 2), + }; + const u32 upper_bound_i = pick_upper_bound(rng); + + BATT_CHECK_LT(lower_bound_i, upper_bound_i); + BATT_CHECK_GE(lower_bound_i, next_droppable); + + items_dropped += upper_bound_i - lower_bound_i; + + const usize live_count_before = leaf_filter.live().size(); + //----- --- -- - - - - + dropped_ranges.push_back(Interval{lower_bound_i, upper_bound_i}); + leaf_filter.drop_index_range(Interval{lower_bound_i, upper_bound_i}); + //----- --- -- - - - - + const usize live_count_after = leaf_filter.live().size(); + + if (lower_bound_i == 0) { + ASSERT_EQ(live_count_after, live_count_before); + } else { + ASSERT_EQ(live_count_after, live_count_before + 1); + } + + --drops_remaining; + next_droppable = upper_bound_i + 1; + } + + // Verify the number of expected live items. + // + const u32 expected_live_count = item_count - items_dropped; + + if (drop_count > 1) { + ASSERT_GT(expected_live_count, 0) + << BATT_INSPECT_RANGE(dropped_ranges) << BATT_INSPECT(drop_count) + << BATT_INSPECT(item_count); + } + + u32 actual_live_count = 0; + u32 next_possible_live = 0; + u32 next_possible_block = 0; + + packed_leaf.sharded_live_ranges(leaf_filter, Interval{0, item_count}) | + batt::seq::for_each([&](const std::pair>& live_pair) { + const auto [block_index, live_range] = live_pair; + + // std::cerr << BATT_INSPECT(block_index) << BATT_INSPECT(live_range) << std::endl; + + BATT_CHECK_GE(block_index, next_possible_block); + BATT_CHECK_LT(block_index, packed_leaf.block_count()); + BATT_CHECK_GE(live_range.lower_bound, next_possible_live) + << BATT_INSPECT(j) << BATT_INSPECT(drop_count) << BATT_INSPECT(live_range) + << BATT_INSPECT(item_count); + BATT_CHECK_LT(live_range.lower_bound, live_range.upper_bound); + BATT_CHECK_LE(live_range.upper_bound, item_count); + + const Interval block_range = + packed_leaf.item_index_range_of_block(block_index); + + BATT_CHECK_GE(live_range.lower_bound, block_range.lower_bound); + BATT_CHECK_LE(live_range.upper_bound, block_range.upper_bound); + + next_possible_live = live_range.upper_bound; + next_possible_block = block_index; + + actual_live_count += live_range.size(); + }); + + ASSERT_EQ(actual_live_count, expected_live_count) + << BATT_INSPECT(j) << BATT_INSPECT(drop_count); + } + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.hpp similarity index 68% rename from src/turtle_kv/tree/packed_leaf_block.hpp rename to src/turtle_kv/tree/leaf/packed_leaf_block.hpp index 78b55ad..997f0ef 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.hpp @@ -9,6 +9,8 @@ #pragma once #define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_HPP +#include "packed_leaf_block_stats.hpp" + #include #include #include @@ -23,11 +25,8 @@ #include #include -#include #include -#include - #include namespace turtle_kv { @@ -41,6 +40,8 @@ struct PackedLeafBlock { class Iterator; + using BlockItemsSeq = batt::SubRangeSeq>; + //+++++++++++-+-+--+----- --- -- - - - - big_u32 magic; // +4 = 4 @@ -106,6 +107,11 @@ struct PackedLeafBlock { return this->items_[this->item_count() - 1]; } + BlockItemsSeq items_seq() const noexcept + { + return batt::as_seq(this->items_slice()); + } + const PackedKeyValueSlotPtr* items_begin() const noexcept { return this->items_; @@ -151,85 +157,6 @@ struct PackedLeafBlock { static_assert(sizeof(PackedLeafBlock) == 8); -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -class PackedLeafBlock::Iterator - : public boost::iterator_facade< // - PackedLeafBlock::Iterator, // <- Derived - const PackedLeafBlock, // <- Value - std::random_access_iterator_tag, // <- CategoryOrTraversal - const PackedLeafBlock&, // <- Reference - isize // <- Difference - > -{ - public: - using Self = Iterator; - using iterator_category = std::random_access_iterator_tag; - using value_type = const PackedLeafBlock; - using reference = const PackedLeafBlock&; - - Iterator() = default; - - explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept - : block_{block} - , block_size_{block_size} - { - } - - reference dereference() const - { - return *this->block_; - } - - bool equal(const Self& other) const - { - return this->block_ == other.block_ && this->block_size_ == other.block_size_; - } - - void increment() - { - this->advance(1); - } - - void decrement() - { - this->advance(-1); - } - - void advance(isize delta) - { - this->block_ = static_cast( - advance_pointer(this->block_, delta * this->block_size_)); - } - - isize distance_to(const Self& other) const - { - return (byte_distance(this->block_, other.block_)) / this->block_size_; - } - - private: - const PackedLeafBlock* block_ = nullptr; - isize block_size_ = 0; -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -struct PackedLeafBlockStats { - usize block_size; - usize item_count; - usize item_slot_bytes; - usize item_ptr_bytes; - - //+++++++++++-+-+--+----- --- -- - - - - - - template - static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; -}; - -BATT_OBJECT_PRINT_IMPL((inline), - PackedLeafBlockStats, - (block_size, item_count, item_slot_bytes, item_ptr_bytes)) - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template @@ -25,48 +26,6 @@ namespace turtle_kv { -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, - usize dst_size) noexcept -{ - PackedLeafBlockStats stats{ - .block_size = 0, - .item_count = 0, - .item_slot_bytes = 0, - .item_ptr_bytes = 0, - }; - - if (dst_size < sizeof(PackedLeafBlock)) { - return stats; - } - stats.block_size = dst_size; - usize offset = 0; - dst_size -= sizeof(PackedLeafBlock); - offset += sizeof(PackedLeafBlock); - - for (const auto& src_item : src) { - const usize slot_size = packed_key_value_slot_size(src_item); - const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); - if (dst_size < total_item_size) { - break; - } - stats.item_count += 1; - stats.item_slot_bytes += slot_size; - stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); - dst_size -= total_item_size; - offset += total_item_size; - - if constexpr (false) { - LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) - << BATT_INSPECT(stats.item_count); - } - } - - return stats; -} - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp new file mode 100644 index 0000000..ade706d --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp @@ -0,0 +1,104 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_ITERATOR_HPP + +#include "packed_leaf_block.hpp" + +#include +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class PackedLeafBlock::Iterator + : public boost::iterator_facade< // + PackedLeafBlock::Iterator, // <- Derived + const PackedLeafBlock, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedLeafBlock&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = Iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedLeafBlock; + using reference = const PackedLeafBlock&; + + //+++++++++++-+-+--+----- --- -- - - - -- + + Iterator() = default; + + explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept + : block_{block} + , block_size_{block_size} + { + } + + //+++++++++++-+-+--+----- --- -- - - - -- + + reference dereference() const + { + return *this->block_; + } + + bool equal(const Self& other) const + { + return this->block_ == other.block_ && this->block_size_ == other.block_size_; + } + + void increment() + { + this->advance(1); + } + + void decrement() + { + this->advance(-1); + } + + void advance(isize delta) + { + this->block_ = static_cast( + advance_pointer(this->block_, delta * this->block_size_)); + } + + isize distance_to(const Self& other) const + { + return (byte_distance(this->block_, other.block_)) / this->block_size_; + } + + //+++++++++++-+-+--+----- --- -- - - - -- + + const PackedLeafBlock* block() const noexcept + { + return this->block_; + } + + usize block_size() const noexcept + { + return static_cast(this->block_size_); + } + + isize block_isize() const noexcept + { + return this->block_size_; + } + + //+++++++++++-+-+--+----- --- -- - - - -- + private: + const PackedLeafBlock* block_ = nullptr; + isize block_size_ = 0; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp similarity index 96% rename from src/turtle_kv/tree/packed_leaf_block.test.cpp rename to src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp index 73a9e1f..f19fefc 100644 --- a/src/turtle_kv/tree/packed_leaf_block.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp @@ -6,14 +6,14 @@ // //+++++++++++-+-+--+----- --- -- - - - - -#include +#include // -#include +#include #include #include -#include "random_str.hpp" +#include #include @@ -119,6 +119,8 @@ TEST(TreePackedLeafBlockTest, Random) } } + // Run empty queries. + // for (usize i = 0; i < kNumNotFoundQueries; ++i) { std::string_view key; for (;;) { @@ -135,6 +137,8 @@ TEST(TreePackedLeafBlockTest, Random) ASSERT_EQ(packed_block.find_key(key), nullptr); } + // Run lower bound queries. + // for (usize i = 0; i < kNumLowerBoundQueries; ++i) { std::string_view key = (i % 2) ? random_str(rng, pick_key_size, diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp new file mode 100644 index 0000000..2d8360f --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp @@ -0,0 +1,38 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_HPP + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlockStats { + usize block_size; + usize item_count; + usize item_slot_bytes; + usize item_ptr_bytes; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; +}; + +BATT_OBJECT_PRINT_IMPL((inline), + PackedLeafBlockStats, + (block_size, item_count, item_slot_bytes, item_ptr_bytes)) + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp new file mode 100644 index 0000000..a9b9bf7 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp @@ -0,0 +1,56 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_IPP + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, + usize dst_size) noexcept +{ + PackedLeafBlockStats stats{ + .block_size = 0, + .item_count = 0, + .item_slot_bytes = 0, + .item_ptr_bytes = 0, + }; + + if (dst_size < sizeof(PackedLeafBlock)) { + return stats; + } + stats.block_size = dst_size; + usize offset = 0; + dst_size -= sizeof(PackedLeafBlock); + offset += sizeof(PackedLeafBlock); + + for (const auto& src_item : src) { + const usize slot_size = packed_key_value_slot_size(src_item); + const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); + if (dst_size < total_item_size) { + break; + } + stats.item_count += 1; + stats.item_slot_bytes += slot_size; + stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); + dst_size -= total_item_size; + offset += total_item_size; + + if constexpr (false) { + LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) + << BATT_INSPECT(stats.item_count); + } + } + + return stats; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.cpp deleted file mode 100644 index 25ef37d..0000000 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.cpp +++ /dev/null @@ -1,13 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include -// - -namespace turtle_kv { -} diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp deleted file mode 100644 index 5aa3008..0000000 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ /dev/null @@ -1,145 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP - -#include - -#include - -#include - -#include -#include -#include - -#include -#include - -#include -#include - -#include - -namespace turtle_kv { - -struct PackedBlockedLeafPage { - static constexpr u64 kMagic = 0x6456beb7f9558445ull; - - //+++++++++++-+-+--+----- --- -- - - - - - - template - static usize packed_edit_size(const EditT& edit) noexcept - { - return PackedLeafBlock::packed_edit_size(edit); - } - - static usize estimate_capacity(usize leaf_size, - usize block_size, - usize max_key_size, - usize max_edit_size) noexcept; - - //+++++++++++-+-+--+----- --- -- - - - - - - big_u64 magic; // +8 -> 8 - little_u32 item_count; // +4 -> 12 - little_u32 total_packed_size; // +4 -> 16 - little_u32 blocks_per_art_key; // +4 -> 20 - little_u32 block_size_bytes; // +4 -> 24 - little_u32 block_count; // +4 -> 28 - llfs::PackedPointer block0; // +4 -> 32 - - /** \brief Pointer to array that stores, for each block, the starting item index relative to the - * entire leaf. - */ - llfs::PackedPointer> block_starting_item; // +4 -> 36 - - /** \brief Pointer to packed ART index. - */ - llfs::PackedPointer art_block_index; // +4 -> 40 - - u8 pad_[24]; - - //+++++++++++-+-+--+----- --- -- - - - - - - PackedLeafBlock::Iterator blocks_begin() const noexcept - { - return PackedLeafBlock::Iterator{this->block0.get(), (isize)this->block_size_bytes.value()}; - } - - PackedLeafBlock::Iterator blocks_end() const noexcept - { - return this->blocks_begin() + this->block_count; - } - - auto blocks() const noexcept - { - return std::ranges::subrange(this->blocks_begin(), - this->blocks_end()); - } - - auto blocks_seq() const noexcept - { - return batt::as_seq(this->blocks()); - } - - auto items_seq() const noexcept - { - return this->blocks_seq() | batt::seq::map([](const PackedLeafBlock& block) { - return batt::as_seq(block.items_slice()); - }) | - batt::seq::flatten(); - } -}; - -static_assert(sizeof(PackedBlockedLeafPage) == 64); - -template -StatusOr pack_blocked_leaf_page(const usize block_size, - const ItemRangeT& src_items, - const MutableBuffer& dst_buffer) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline /*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, - usize block_size, - usize max_key_size, - usize max_edit_size) noexcept -{ - const usize space_after_header = - leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); - - const usize max_block_count = space_after_header / block_size; - - const usize block_starts_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; - - const usize space_after_block_starts = space_after_header - block_starts_size; - - const usize max_art_size = max_key_size * max_block_count * 2; - - const usize space_after_art = space_after_block_starts - max_art_size; - - BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; - const usize space_for_blocks = space_after_art & ~(block_size - 1); - const usize block_count = space_for_blocks / block_size; - - const usize max_wasted_per_block = max_edit_size - 1; - const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; - - const usize final_estimate = block_count * min_block_capacity; - - BATT_CHECK_GT(leaf_size, final_estimate); - - return final_estimate; -} - -} // namespace turtle_kv - -#include "packed_blocked_leaf_page.ipp" diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp deleted file mode 100644 index ec26d26..0000000 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp +++ /dev/null @@ -1,192 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include -// -#include - -#include -#include - -#include "random_str.hpp" - -#include - -#include -#include - -#include -#include -#include - -namespace { - -using namespace batt::int_types; -using namespace batt::constants; - -using batt::MutableBuffer; -using batt::StableStringStore; -using batt::StatusOr; - -using turtle_kv::EditView; -using turtle_kv::KeyOrder; -using turtle_kv::KeyView; -using turtle_kv::Optional; -using turtle_kv::pack_blocked_leaf_page; -using turtle_kv::PackedBlockedLeafPage; -using turtle_kv::random_str; -using turtle_kv::ValueView; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// Plan: -// 1. For different random seeds: -// - generate random set of prefixes (~10% of total keys) -// - generate keys using prefixes, with random values -// - sort -// - pack leaf; verify: -// a. all packed keys present and have right values -// b. any unpacked keys at end missing -// c. randomly generated non-present keys not found -// -TEST(TreePackedBlockedLeafPageTest, Random) -{ - const usize kNumSeeds = 1000; - const usize kLeafPageSize = 1 * kMiB; - const usize kNumPrefixes = 1000; - const usize kMinPrefixSize = 0; - const usize kMaxPrefixSize = 8; - const usize kMinKeySize = 4; - const usize kMaxKeySize = 48; - const usize kMinValueSize = 0; - const usize kMaxValueSize = 200; - const usize kBlockSize = 8192; - - BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); - - std::geometric_distribution pick_prefix_size{0.5}; - std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; - std::geometric_distribution pick_key_size{0.7}; - std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - - usize total_keys = 0; - usize total_bytes = 0; - - for (usize seed = 0; seed < kNumSeeds; ++seed) { - std::default_random_engine rng{seed}; - - StableStringStore strings; - - //+++++++++++-+-+--+----- --- -- - - - - - // Generate prefixes - // - std::vector prefixes; - { - std::unordered_set used_prefixes; - while (prefixes.size() < kNumPrefixes) { - std::string_view prefix = - random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); - - if (used_prefixes.count(prefix)) { - continue; - } - prefixes.push_back(prefix); - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Generate edits. - // - std::vector edits; - { - usize max_edit_size = 0; - usize max_key_size = 0; - usize total_edits_size = 0; - - std::unordered_set used_keys; - for (;;) { - std::string_view prefix = prefixes[pick_prefix(rng)]; - - std::string_view key = - random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); - - if (used_keys.count(key)) { - continue; - } - - std::string_view value = - random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); - - EditView edit{key, ValueView::from_str(value)}; - - const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); - - const usize new_max_edit_size = std::max(max_edit_size, edit_size); - const usize new_max_key_size = std::max(max_key_size, key.size()); - - const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, - kBlockSize, - new_max_key_size, - new_max_edit_size); - - // Stop as soon as adding the next key would exceed the estimated space. - // - if (edit_size + total_edits_size > space_available) { - break; - } - - ++total_keys; - total_bytes += edit_size; - - edits.push_back(edit); - total_edits_size += edit_size; - max_edit_size = new_max_edit_size; - max_key_size = new_max_key_size; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Sort edits by key. - // - std::sort(edits.begin(), edits.end(), KeyOrder{}); - - //+++++++++++-+-+--+----- --- -- - - - - - // Pack a blocked leaf page. - // - using StorageUnit = std::aligned_storage_t<4096, 4096>; - std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); - ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); - - MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; - - StatusOr status_or_packed_leaf = - pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); - - ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); - - const PackedBlockedLeafPage& packed_leaf = **status_or_packed_leaf; - - //+++++++++++-+-+--+----- --- -- - - - - - // - // - { - auto packed_items = packed_leaf.items_seq(); - using Item = decltype(*packed_items.peek()); - for (const EditView& edit : edits) { - Optional next_packed = packed_items.next(); - - ASSERT_TRUE(next_packed.has_value()); - ASSERT_EQ(get_key(*next_packed), get_key(edit)); - ASSERT_EQ(get_value(*next_packed), get_value(edit)); - } - } - } - - std::cerr << BATT_INSPECT(total_keys) << BATT_INSPECT(total_bytes) << std::endl; -} - -} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block_scanner.hpp b/src/turtle_kv/tree/packed_leaf_block_scanner.hpp new file mode 100644 index 0000000..5f5d81a --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block_scanner.hpp @@ -0,0 +1,75 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_SCANNER_HPP + +#include + +#include + +#include + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +concept PackedLeafBlockProvider = requires(T& provider, usize block_index) { + { provider.get_block(block_index) } -> std::convertible_to>; +}; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template +class PackedLeafBlockScanner +{ + public: + class Impl + { + public: + using Item = StatusOr; + + Optional poll() noexcept + { + } + + Optional next() noexcept + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + void advance() noexcept + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + PackedBlockedLeafPage::HeaderShardView header_; + + BlockProviderT& provider_; + + PiecewiseFilter& filter_; + + usize block_index_; + + Optional> block_; + }; + + //+++++++++++-+-+--+----- --- -- - - - - + + private: + Impl* impl_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_node_page.cpp b/src/turtle_kv/tree/packed_node_page.cpp index 6dbdd78..904b83e 100644 --- a/src/turtle_kv/tree/packed_node_page.cpp +++ b/src/turtle_kv/tree/packed_node_page.cpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #include // @@ -8,6 +16,7 @@ #include #include +#include #include diff --git a/src/turtle_kv/tree/testing/fake_segment.hpp b/src/turtle_kv/tree/testing/fake_segment.hpp index 8297a28..860f722 100644 --- a/src/turtle_kv/tree/testing/fake_segment.hpp +++ b/src/turtle_kv/tree/testing/fake_segment.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -102,11 +103,11 @@ struct FakeSegment { { const bool inactive = this->active_pivots_.is_empty(); if (inactive) { - Slice> live_ranges = this->filter_.live(); - BATT_CHECK_EQ(live_ranges.size(), 1) << BATT_INSPECT(live_ranges); - BATT_CHECK_EQ(live_ranges[0].upper_bound, PiecewiseFilter::kMaxUpperBound) - << BATT_INSPECT(live_ranges); - } + Slice> live_ranges = this->filter_.live(); + BATT_CHECK_EQ(live_ranges.size(), 1) << BATT_INSPECT(live_ranges); + BATT_CHECK_EQ(live_ranges[0].upper_bound, PiecewiseFilter::kMaxUpperBound) + << BATT_INSPECT(live_ranges); + } return inactive; } diff --git a/src/turtle_kv/util/packed_piecewise_filter_view.hpp b/src/turtle_kv/util/packed_piecewise_filter_view.hpp new file mode 100644 index 0000000..069cc11 --- /dev/null +++ b/src/turtle_kv/util/packed_piecewise_filter_view.hpp @@ -0,0 +1,305 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PACKED_PIECEWISE_FILTER_VIEW_HPP + +#include "piecewise_filter_storage_model.concept.hpp" + +#include +#include +#include + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Read-only model of PiecewiseFilterStorageModel for packed filters. + * + * Packed piecewise filters are represented as an array of integers, which are the boundaries + * between live and dropped intervals, plus an additional boolean/bit denoting whether the interval + * from the global minimum to the first stored boundary is live or dropped (`start_is_live`). + * + * The global minimum and maximum bounds are never stored in the packed representation. Instead, + * the minimum is implied via the `start_is_live` bit, and the maximum by whether the number of + * stored bounds (plus the implicit first bound, if start_is_live == true) is even or odd. If it is + * odd, then it is implied that there is a final bound equal to the global maximum. + * + * Examples: + * + * Live Intervals: {[0, 10), [20, 30), [40, 50)} + * Packed: start_is_live=1, {10, 20, 30, 40, 50} + * + * Live Intervals: {[0, 10), [20, 30), [40, +inf)} + * Packed: start_is_live=1, {10, 20, 30, 40} + * + * Live Intervals: {[10, 20), [30, 40), [50, 60)} + * Packed: start_is_live=0, {10, 20, 30, 40, 50, 60} + * + * Live Intervals: {[10, 20), [30, 40), [50, +inf)} + * Packed: start_is_live=0, {10, 20, 30, 40, 50} + */ +class PackedPiecewiseFilterView +{ + public: + //----- --- -- - - - - + + // Forward-declaration; the type returned by this->begin(), this->end() + // + class const_iterator; + + /** \brief The boundary integer type. Must be unsigned. + */ + using OffsetT = const little_u32; + + /** \brief The live range type; what `iterator` iterates over. + */ + using value_type = Interval; + + /** \brief Non-const iterator aliases const_iterator, since this storage model is read-only. + */ + using iterator = const_iterator; + + // Forward-declaration; defined below. + // + friend const Slice& as_const_slice(const PackedPiecewiseFilterView& view); + + //----- --- -- - - - - + + /** \brief Constructs an PackedPiecewiseFilterView representing the live interval [0, +inf). + */ + PackedPiecewiseFilterView() = default; + + /** \brief Destructs the PackedPiecewiseFilterView. + */ + ~PackedPiecewiseFilterView() = default; + + /** \brief PackedPiecewiseFilterView is copy constructible. + */ + PackedPiecewiseFilterView(const PackedPiecewiseFilterView&) = default; + + /** \brief PackedPiecewiseFilterView is copy assignable. + */ + PackedPiecewiseFilterView& operator=(const PackedPiecewiseFilterView&) = default; + + /** \brief Constructs PackedPiecewiseFilterView from the packed data in the arguments. + * + * See the class-level description for details on what `values` and `start_is_live` represent. + */ + explicit PackedPiecewiseFilterView(const Slice& values, + bool start_is_live) noexcept + : values_{values} + , implicit_first_{start_is_live ? 1 : 0} + , size_{(this->implicit_first_ + this->values_.size() + 1) & ~i32{1}} + { + } + + //----- --- -- - - - - + + /** \brief Returns an iterator to the first live interval in this filter. + */ + const_iterator begin() const noexcept; + + /** \brief Returns an iterator to one past the last live interval in this filter. + */ + const_iterator end() const noexcept; + + /** \brief Returns the number of live intervals in the filter; same as + * `std::distance(this->begin(), this->end())`. + */ + usize size() const noexcept; + + /** \brief Returns true iff `this->size() == 0`. + */ + bool empty() const noexcept; + + /** \brief Returns the `i`-th live interval in the filter. Behavior is undefined if `i` is not + * less than `this->size()`. + */ + Interval operator[](isize i) const noexcept; + + //----- --- -- - - - - + private: + /** \brief Points at the stored boundaries, as described in the class-level doc. + */ + Slice values_; + + /** \brief Set to 1 if `start_is_live`, else 0. + */ + i32 implicit_first_ = 1; + + /** \brief The number of live intervals in the filter. + */ + i32 size_ = 1; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Returns a const reference to the stored values referenced by `view`. + */ +inline const Slice& as_const_slice(const PackedPiecewiseFilterView& view) +{ + return view.values_; +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Read-only, random access iterator over the live intervals of a packed piecewise filter. + */ +class PackedPiecewiseFilterView::const_iterator + : public boost::iterator_facade< // + PackedPiecewiseFilterView::const_iterator, // <- Derived + Interval, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + Interval, // <- Reference + isize // <- Difference + > +{ + public: + using Self = const_iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = Interval; + using reference = Interval; + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Constructs an invalid iterator. + */ + const_iterator() noexcept : view_{nullptr}, pos_{0} + { + } + + /** \brief Constructs an iterator to the `pos`-th live interval of `view`. + * + * `view` must remain in-scope while this object exists. + */ + const_iterator(const PackedPiecewiseFilterView* view, isize pos) noexcept : view_{view}, pos_{pos} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Returns the live interval at the current position. + */ + reference dereference() const + { + return (*this->view_)[this->pos_]; + } + + /** \brief Returns true iff this iterator points to the same live interval of the same filter as + * `other`. + */ + bool equal(const Self& other) const + { + return this->view_ == other.view_ && this->pos_ == other.pos_; + } + + /** \brief Moves this iterator forward by one. + */ + void increment() + { + ++this->pos_; + } + + /** \brief Moves this iterator backward by one. + */ + void decrement() + { + --this->pos_; + } + + /** \brief Moves this iterator by `delta`. + */ + void advance(isize delta) + { + this->pos_ += delta; + } + + /** \brief Returns the number of steps required to advance this iterator so it is equivalent to + * `other`. Will panic if this and other do not point at the same filter view. + */ + isize distance_to(const Self& other) const + { + BATT_CHECK_EQ(this->view_, other.view_); + return other.pos_ - this->pos_; + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + /** \brief Pointer to the filter view over which we are iterating. + */ + const PackedPiecewiseFilterView* view_; + + /** \brief The (logical) position of this iterator within `view_`. + */ + isize pos_; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline auto PackedPiecewiseFilterView::begin() const noexcept -> const_iterator +{ + return const_iterator{this, 0}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline auto PackedPiecewiseFilterView::end() const noexcept -> const_iterator +{ + return const_iterator{this, static_cast(this->size())}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline usize PackedPiecewiseFilterView::size() const noexcept +{ + return this->size_; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline bool PackedPiecewiseFilterView::empty() const noexcept +{ + return this->size_ == 0; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline Interval PackedPiecewiseFilterView::operator[](isize i) const noexcept +{ + // Cached for brevity below. + // + const isize n = this->values_.size(); + + // The index within this->values_ of the i-th live interval's lower bound. + // May be negative if the first boundary is implicit (0) + // + const isize j0 = i * 2 - this->implicit_first_; + + // The index within this->values_ of the i-th live interval's upper bound. + // May be past the end of this->values_ if the last boundary is implicit (+inf) + // + const isize j1 = j0 + 1; + + const u32 lower_bound = (j0 < 0) ? std::numeric_limits::min() : this->values_[j0].value(); + const u32 upper_bound = (j1 < n) ? this->values_[j1].value() : std::numeric_limits::max(); + + return Interval{lower_bound, upper_bound}; +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- + +static_assert(PiecewiseFilterStorageModel); + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.hpp b/src/turtle_kv/util/piecewise_filter.hpp index f524319..143dc43 100644 --- a/src/turtle_kv/util/piecewise_filter.hpp +++ b/src/turtle_kv/util/piecewise_filter.hpp @@ -9,6 +9,8 @@ #pragma once #define TURTLE_KV_UTIL_PIECEWISE_FILTER_HPP +#include "piecewise_filter_storage_model.concept.hpp" + #include #include #include @@ -26,11 +28,15 @@ namespace turtle_kv { /** \brief A representation of a filtered range of items. */ -template -class PiecewiseFilter +template ModelT> +class BasicPiecewiseFilter : private ModelT { public: - using Self = PiecewiseFilter; + using Self = BasicPiecewiseFilter; + + using ConstIterator = typename ModelT::const_iterator; + + class LiveSubranges; static_assert(std::is_integral::value && std::is_unsigned::value, "Offset must be an unsigned integer type!"); @@ -46,14 +52,15 @@ class PiecewiseFilter /** \brief Creates and returns a PiecewiseFilter instance from a range of intervals that contain * the live item indexes. */ - static StatusOr from_live(const Slice>& live); + static StatusOr from_live(const Slice>& live) + requires PiecewiseFilterMutableStorageModel; //+++++++++++-+-+--+----- --- -- - - - - /** \brief Constructs a default instance of a PiecewiseFilter object, initialized with no item * range and filtered items. */ - PiecewiseFilter() noexcept; + BasicPiecewiseFilter() noexcept; //+++++++++++-+-+--+----- --- -- - - - - @@ -63,7 +70,8 @@ class PiecewiseFilter * * \return The new dropped interval that coincides with `i`. */ - Interval drop_index_range(Interval i); + Interval drop_index_range(Interval i) + requires PiecewiseFilterMutableStorageModel; /** \brief Returns whether or not the item at index `i` has been filtered out. * @@ -99,7 +107,13 @@ class PiecewiseFilter /** \brief Merges two filters in place, taking the union of the live intervals. */ - void merge(const PiecewiseFilter& other); + void merge(const Self& other) + requires PiecewiseFilterMutableStorageModel; + + /** \brief Returns a seq of Interval that is the intersection of `i` and the live ranges + * of this filter. + */ + LiveSubranges live_subranges_of(Interval i) const; /** \brief Validate the state of the live intervals. */ @@ -109,11 +123,22 @@ class PiecewiseFilter //+++++++++++-+-+--+----- --- -- - - - - private: - /** \brief The range of filtered out item indexes. - */ - SmallVec, 64> live_; + BATT_ALWAYS_INLINE ModelT& live_() noexcept + { + return *this; + } + + BATT_ALWAYS_INLINE const ModelT& live_() const noexcept + { + return *this; + } }; +template +using PiecewiseFilter = BasicPiecewiseFilter, 64>>; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// template inline Interval drop_item_range(PiecewiseFilter& filter, const Slice& items, @@ -127,6 +152,5 @@ inline Interval drop_item_range(PiecewiseFilter& filter, return filter.drop_index_range(Interval{start_i, end_i}); } -} // namespace turtle_kv -#include +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.ipp b/src/turtle_kv/util/piecewise_filter.ipp index 3456043..b892275 100644 --- a/src/turtle_kv/util/piecewise_filter.ipp +++ b/src/turtle_kv/util/piecewise_filter.ipp @@ -11,20 +11,19 @@ #include "piecewise_filter.hpp" -#include - namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -/*static*/ StatusOr> PiecewiseFilter::from_live( - const Slice>& live) +template ModelT> +/*static*/ StatusOr> +BasicPiecewiseFilter::from_live(const Slice>& live) + requires PiecewiseFilterMutableStorageModel { - PiecewiseFilter filter; - filter.live_.clear(); + Self filter; - filter.live_.insert(filter.live_.end(), live.begin(), live.end()); + filter.live_().clear(); + filter.live_().insert(filter.live_().end(), live.begin(), live.end()); if (!filter.check_invariants()) { return Status{::batt::StatusCode::kInvalidArgument}; @@ -35,16 +34,16 @@ template //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -PiecewiseFilter::PiecewiseFilter() noexcept - : live_{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} +template ModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter() noexcept + : ModelT{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} { } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -bool PiecewiseFilter::check_invariants() const +template ModelT> +bool BasicPiecewiseFilter::check_invariants() const { Optional prev_upper_bound = None; @@ -52,7 +51,7 @@ bool PiecewiseFilter::check_invariants() const // - all intervals are in non-decreasing order // - no intervals overlap or are adjacent (i.e., prev.upper_bound == next.lower_bound) // - for (const Interval& range : this->live_) { + for (const Interval& range : this->live_()) { // If a range has the minimum lower bound, it must be the first. // if (range.lower_bound == Self::kMinLowerBound && prev_upper_bound) { @@ -80,15 +79,16 @@ bool PiecewiseFilter::check_invariants() const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Interval PiecewiseFilter::drop_index_range(Interval to_drop) +template ModelT> +Interval BasicPiecewiseFilter::drop_index_range(Interval to_drop) + requires PiecewiseFilterMutableStorageModel { if (to_drop.empty()) { return to_drop; } - auto [first, last] = std::equal_range(this->live_.begin(), - this->live_.end(), + auto [first, last] = std::equal_range(this->live_().begin(), + this->live_().end(), to_drop, typename Interval::LinearOrder{}); @@ -98,7 +98,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // the return value bounds correctly. // if (first == last || to_drop.lower_bound < first->lower_bound) { - if (first != this->live_.begin()) { + if (first != this->live_().begin()) { // We are starting in a live interval gap (dropped region), so we extend to the previous live // interval's upper bound. // @@ -109,7 +109,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t } if (first == last || to_drop.upper_bound >= std::prev(last)->upper_bound) { - if (last != this->live_.end()) { + if (last != this->live_().end()) { // We are ending in a live interval gap, so we extend to the next live interval's start. // dropped.upper_bound = last->lower_bound; @@ -164,7 +164,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // Process all overlapping intervals with `to_drop`. // - while (first != this->live_.end()) { + while (first != this->live_().end()) { if (first->lower_bound >= to_drop.upper_bound) { // Interval is entirely after `to_drop`, so there is nothing left to process. // @@ -177,7 +177,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // Interval right_half{to_drop.upper_bound, first->upper_bound}; first->upper_bound = to_drop.lower_bound; - this->live_.insert(std::next(first), right_half); + this->live_().insert(std::next(first), right_half); return dropped; } else { // Cases 2b and 4. @@ -193,7 +193,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t } else { // Case 1. // - first = this->live_.erase(first); + first = this->live_().erase(first); } } @@ -202,41 +202,41 @@ Interval PiecewiseFilter::drop_index_range(Interval t //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Slice> PiecewiseFilter::live() const +template ModelT> +Slice> BasicPiecewiseFilter::live() const { - return as_const_slice(this->live_); + return as_const_slice(this->live_()); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -bool PiecewiseFilter::live_at_index(OffsetT i) const +template ModelT> +bool BasicPiecewiseFilter::live_at_index(OffsetT i) const { return this->live_lower_bound(i) == i; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -OffsetT PiecewiseFilter::live_lower_bound(OffsetT i) const +template ModelT> +OffsetT BasicPiecewiseFilter::live_lower_bound(OffsetT i) const { // Compute the live interval which could contain `i`. // - auto iter = std::lower_bound(this->live_.begin(), - this->live_.end(), + auto iter = std::lower_bound(this->live_().begin(), + this->live_().end(), i, typename Interval::LinearOrder{}); // Check if current interval contains `i`. // - if (iter != this->live_.end() && iter->contains(i)) { + if (iter != this->live_().end() && iter->contains(i)) { return i; } // `i` is in a dropped range, so we return the start of the next live interval. // - if (iter != this->live_.end()) { + if (iter != this->live_().end()) { return iter->lower_bound; } @@ -247,22 +247,22 @@ OffsetT PiecewiseFilter::live_lower_bound(OffsetT i) const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Interval PiecewiseFilter::find_live_range(Interval i) const +template ModelT> +Interval BasicPiecewiseFilter::find_live_range(Interval i) const { OffsetT start_i = i.lower_bound; OffsetT end_i = i.upper_bound; BATT_CHECK_LE(start_i, end_i); - auto iter = std::lower_bound(this->live_.begin(), - this->live_.end(), + auto iter = std::lower_bound(this->live_().begin(), + this->live_().end(), start_i, typename Interval::LinearOrder{}); // Check if current interval contains or starts at `start_i`. // - if (iter != this->live_.end()) { + if (iter != this->live_().end()) { if (iter->contains(start_i)) { OffsetT live_end = std::min(end_i, iter->upper_bound); return Interval{start_i, live_end}; @@ -283,25 +283,26 @@ Interval PiecewiseFilter::find_live_range(Interval i) //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -void PiecewiseFilter::merge(const PiecewiseFilter& other) +template ModelT> +void BasicPiecewiseFilter::merge(const Self& other) + requires PiecewiseFilterMutableStorageModel { // If other has no live intervals, we are done. // - if (other.live_.empty()) { + if (other.live_().empty()) { return; } // If this has no live intervals, copy from other. // - if (this->live_.empty()) { - this->live_.insert(this->live_.end(), other.live_.begin(), other.live_.end()); + if (this->live_().empty()) { + this->live_().insert(this->live_().end(), other.live_().begin(), other.live_().end()); BATT_CHECK(this->check_invariants()); return; } SmallVec, 64> merged_intervals; - merged_intervals.reserve(this->live_.size() + other.live_.size()); + merged_intervals.reserve(this->live_().size() + other.live_().size()); usize i = 0; usize j = 0; @@ -329,42 +330,57 @@ void PiecewiseFilter::merge(const PiecewiseFilter& other) // Merge the live intervals arrays. // - while (i < this->live_.size() && j < other.live_.size()) { - if (this->live_[i].lower_bound <= other.live_[j].lower_bound) { - add_interval(this->live_[i]); + while (i < this->live_().size() && j < other.live_().size()) { + if (this->live_()[i].lower_bound <= other.live_()[j].lower_bound) { + add_interval(this->live_()[i]); ++i; } else { - add_interval(other.live_[j]); + add_interval(other.live_()[j]); ++j; } } - // Add remaining intervals from this->live_. + // Add remaining intervals.. // - while (i < this->live_.size()) { - add_interval(this->live_[i]); + while (i < this->live_().size()) { + add_interval(this->live_()[i]); ++i; } // Add remaining intervals from other.live_. // - while (j < other.live_.size()) { - add_interval(other.live_[j]); + while (j < other.live_().size()) { + add_interval(other.live_()[j]); ++j; } - this->live_ = std::move(merged_intervals); + this->live_() = std::move(merged_intervals); BATT_CHECK(this->check_invariants()); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -SmallFn PiecewiseFilter::dump() const +template ModelT> +SmallFn BasicPiecewiseFilter::dump() const { return [this](std::ostream& out) { - out << batt::dump_range(this->live_); + out << batt::dump_range(this->live_()); }; } + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::live_subranges_of(Interval query_range) const + -> LiveSubranges +{ + const auto [first, last] = std::equal_range(this->live_().begin(), + this->live_().end(), + query_range, + typename Interval::LinearOrder{}); + + return LiveSubranges{query_range, std::ranges::subrange(first, last)}; +} + } // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.live_subranges.hpp b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp new file mode 100644 index 0000000..c2bf50c --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp @@ -0,0 +1,88 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_LIVE_SUBRANGES_HPP + +#include "piecewise_filter.hpp" + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief A (batt) Seq over the sub-ranges of a PiecewiseFilter which match some interval (the + * `query_range`). + */ +template ModelT> +class BasicPiecewiseFilter::LiveSubranges +{ + public: + using Iterator = PiecewiseFilter::ConstIterator; + using Item = Interval; + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Constructs a LiveSubranges seq containing the passed range of live intervals (`match`), + * with the first and last element clamped to the `query_range`. + * + * `match` *must* not extend more than one live interval past `query_range` at the front or back. + */ + explicit LiveSubranges(Interval query_range, + std::ranges::subrange match) noexcept + : clamp_lower_{query_range.lower_bound} + , clamp_upper_{query_range.upper_bound} + , match_{match} + { + } + + /** \brief Returns the current live subrange, or None if the seq has been fully consumed. + */ + Optional peek() + { + if (this->match_.empty()) { + return None; + } + Interval item = this->match_.front(); + if (this->clamp_lower_) { + item.lower_bound = std::max(item.lower_bound, *this->clamp_lower_); + } + if (this->match_.size() == 1 && this->clamp_upper_) { + item.upper_bound = std::min(item.upper_bound, *this->clamp_upper_); + } + return item; + } + + /** \brief Returns the current live subrange, or None if the seq has been fully consumed, + * consuming the returned item. + */ + Optional next() + { + Optional item = this->peek(); + if (item) { + this->clamp_lower_ = None; + this->match_.advance(1); + } + return item; + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + /** \brief The lower bound to which to clamp the first interval of `this->match_`. + */ + Optional clamp_lower_; + + /** \brief The upper bound to which to clamp the last interval of `this->match_`. + */ + Optional clamp_upper_; + + /** \brief The current subrange of the live intervals in the filter. + */ + std::ranges::subrange match_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index d4c6ee4..fe4d3de 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -13,8 +13,15 @@ #include #include +#include +#include + #include +#include +#include +#include + #include #include #include @@ -39,6 +46,7 @@ using turtle_kv::drop_item_range; using llfs::KeyRangeOrder; +using batt::mask_from_interval; using batt::StableStringStore; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -138,7 +146,7 @@ TEST(PiecewiseFilterTest, QueryTest) auto iter = live_items.lower_bound(start_i); Interval expected_range; - + if (iter == live_items.end() || *iter >= end_i) { expected_range = Interval{end_i, end_i}; } else { @@ -284,4 +292,67 @@ TEST(PiecewiseFilterTest, KeyQueryTest) EXPECT_TRUE(filter.check_invariants()); } } -} // namespace \ No newline at end of file + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(PiecewiseFilterTest, LiveSubranges) +{ + std::uniform_int_distribution pick_bound{0, 64}; + + const auto pick_interval = [&](auto& rng) { + Interval i{pick_bound(rng), pick_bound(rng)}; + if (i.upper_bound < i.lower_bound) { + std::swap(i.lower_bound, i.upper_bound); + } + return i; + }; + + std::array, 1> init_live{{{0, 64}}}; + + const usize n_seeds = 10000000; + const usize n_drops = 10; + const usize n_queries = 15; + const usize first_seed = 0; + + for (usize seed_i = first_seed; seed_i < first_seed + n_seeds; ++seed_i) { + std::default_random_engine rng{seed_i}; + + PiecewiseFilter filter = + BATT_OK_RESULT_OR_PANIC(PiecewiseFilter::from_live(batt::as_slice(init_live))); + + const auto query_as_bits = [&](Interval query) { + u64 bits = 0; + filter.live_subranges_of(query) | batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + + u64 filter_state = ~u64{0}; + + for (usize i = 0; i < n_drops; ++i) { + const Interval drop_interval = pick_interval(rng); + const u64 drop_mask = mask_from_interval(drop_interval); + filter_state &= ~drop_mask; + filter.drop_index_range(drop_interval); + + if constexpr (false) { + std::cerr << BATT_INSPECT(std::bitset<64>{filter_state}) << BATT_INSPECT(filter.dump()) + << std::endl; + } + + for (usize j = 0; j < n_queries; ++j) { + const Interval query_interval = pick_interval(rng); + const u64 query_mask = mask_from_interval(query_interval); + const u64 expected_bits = query_mask & filter_state; + const u64 actual_bits = query_as_bits(query_interval); + + ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{actual_bits}) + << BATT_INSPECT(seed_i) << BATT_INSPECT(query_interval) + << BATT_INSPECT(query_interval.size()) << BATT_INSPECT(std::bitset<64>{query_mask}); + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp new file mode 100644 index 0000000..6dd8701 --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp @@ -0,0 +1,60 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_STORAGE_MODEL_CONCEPT_HPP + +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace turtle_kv { + +template +concept PiecewiseFilterStorageModel = requires(const T& model, + T& src, + T& dst, + Interval interval, + usize i, + std::ostream& out) { + typename T::value_type; + typename T::iterator; + typename T::const_iterator; + + { model.begin() }; + { model.end() }; + { std::begin(model) } -> std::same_as; + { std::end(model) } -> std::same_as; + { as_const_slice(model) }; + { model.empty() } -> std::convertible_to; + { model.size() } -> std::convertible_to; + { *model.begin() } -> std::convertible_to&>; + { model[i] } -> std::convertible_to&>; + { dst = std::move(src) }; + { out << batt::dump_range(model) } -> std::same_as; +}; + +template +concept PiecewiseFilterMutableStorageModel = + PiecewiseFilterStorageModel && + requires(T model, T other, Interval interval, usize i, std::ostream& out) { + { model.clear() }; + { model.erase(model.end()) }; + { model.insert(model.end(), interval) }; + { model.insert(model.end(), model.begin(), model.end()) }; + { model[i] = interval }; + }; + +} // namespace turtle_kv From 4cec2199a4394db49dfe468e14d62eae54a43501 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Wed, 24 Jun 2026 15:47:42 -0400 Subject: [PATCH 07/20] Update requirements. --- conan.lock | 9 +++------ conanfile.py | 6 +++--- .../tree/leaf/packed_blocked_leaf_page.test.cpp | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/conan.lock b/conan.lock index b1f39f1..1cb73b8 100644 --- a/conan.lock +++ b/conan.lock @@ -2,8 +2,8 @@ "version": "0.5", "requires": [ "abseil/20250127.0", - "artc/0.0.1.dev", - "batteries/0.71.1", + "artc/0.1.0", + "batteries/0.72.0", "boost/1.88.0", "bzip2/1.0.8", "cli11/2.5.0", @@ -16,7 +16,7 @@ "libpfm4/4.13.0", "libunwind/1.8.1", "liburing/2.11", - "llfs/0.47.1", + "llfs/0.47.2", "openssl/3.6.0", "pcg-cpp/cci.20220409", "protobuf/3.21.12", @@ -50,9 +50,6 @@ ], "boost/[>=1.84.0 <2]": [ "boost/1.88.0" - ], - "batteries/[>=0.60.2 <2]": [ - "batteries/0.70.4.dev2" ] }, "config_requires": [] diff --git a/conanfile.py b/conanfile.py index 8f6b9f6..3474dd2 100644 --- a/conanfile.py +++ b/conanfile.py @@ -89,11 +89,11 @@ def requirements(self): } self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) - self.requires("artc/[>=0.0.1 <1]") - self.requires("batteries/[>=0.71.1 <1]", **VISIBLE, **OVERRIDE) + self.requires("artc/[>=0.1.0 <1]") + self.requires("batteries/[>=0.72.0 <1]", **VISIBLE, **OVERRIDE) self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) self.requires("glog/[>=0.7.1 <1]", **VISIBLE) - self.requires("llfs/[>=0.47.0 <1]", **VISIBLE) + self.requires("llfs/[>=0.47.2 <1]", **VISIBLE) self.requires("pcg-cpp/[>=cci.20220409]", **VISIBLE) self.requires("yaml-cpp/[>=0.9.0 <1]") self.requires("zlib/1.3.1", **OVERRIDE) diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index fd9ca6f..3b4add5 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -64,7 +64,7 @@ using turtle_kv::ValueView; // TEST(TreePackedBlockedLeafPageTest, Random) { - const usize kNumSeeds = 10000; + const usize kNumSeeds = 100; const usize kLeafPageSize = 1 * kMiB; const usize kNumPrefixes = 1000; const usize kMinPrefixSize = 0; @@ -84,7 +84,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; for (usize seed = 803; seed < kNumSeeds; ++seed) { - LOG(INFO) << BATT_INSPECT(seed); + LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); std::default_random_engine rng{seed}; From 5240d180ea61ce962b8a612ea87137e1bc205a9d Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 28 Jun 2026 11:11:08 -0400 Subject: [PATCH 08/20] Refactor PiecewiseFilter test utils. --- src/turtle_kv/import/interval.hpp | 3 - .../leaf/packed_blocked_leaf_page.test.cpp | 60 ++------ src/turtle_kv/util/piecewise_filter.test.cpp | 43 +++--- src/turtle_kv/util/piecewise_filter.test.hpp | 129 ++++++++++++++++++ ...piecewise_filter_storage_model.concept.hpp | 1 - 5 files changed, 165 insertions(+), 71 deletions(-) create mode 100644 src/turtle_kv/util/piecewise_filter.test.hpp diff --git a/src/turtle_kv/import/interval.hpp b/src/turtle_kv/import/interval.hpp index 53167ea..33cfe08 100644 --- a/src/turtle_kv/import/interval.hpp +++ b/src/turtle_kv/import/interval.hpp @@ -2,9 +2,6 @@ #include -#include -#include - namespace turtle_kv { using batt::BasicInterval; diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index 3b4add5..03c926e 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include @@ -64,7 +65,9 @@ using turtle_kv::ValueView; // TEST(TreePackedBlockedLeafPageTest, Random) { - const usize kNumSeeds = 100; + const usize kFirstSeed = 0; + const usize kNumSeeds = 1000; + const usize kLastSeed = kFirstSeed + kNumSeeds; const usize kLeafPageSize = 1 * kMiB; const usize kNumPrefixes = 1000; const usize kMinPrefixSize = 0; @@ -83,7 +86,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) std::geometric_distribution pick_key_size{0.7}; std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - for (usize seed = 803; seed < kNumSeeds; ++seed) { + for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); std::default_random_engine rng{seed}; @@ -264,56 +267,21 @@ TEST(TreePackedBlockedLeafPageTest, Random) // Test ShardedLiveRanges. // { - for (usize j = 0; j < 1000; ++j) { + for (usize j = 0; j < 10000; ++j) { // Drop up to 64 sub-ranges of the leaf. // for (usize drop_count = 0; drop_count < 64; ++drop_count) { - std::vector> dropped_ranges; PiecewiseFilter leaf_filter; - usize drops_remaining = drop_count; - const u32 item_count = packed_leaf.item_count(); - - u32 next_droppable = 0; + std::vector> dropped_ranges; u32 items_dropped = 0; + const u32 item_count = packed_leaf.item_count(); - for (usize drop_i = 0; drop_i < drop_count; ++drop_i) { - BATT_CHECK_GE(next_droppable, 0); - BATT_CHECK_LT(next_droppable, item_count); - - std::uniform_int_distribution pick_lower_bound{ - next_droppable, - item_count - (drops_remaining * 2 - 1), - }; - const u32 lower_bound_i = pick_lower_bound(rng); - - std::uniform_int_distribution pick_upper_bound{ - lower_bound_i + 1, - item_count - (drops_remaining * 2 - 2), - }; - const u32 upper_bound_i = pick_upper_bound(rng); - - BATT_CHECK_LT(lower_bound_i, upper_bound_i); - BATT_CHECK_GE(lower_bound_i, next_droppable); - - items_dropped += upper_bound_i - lower_bound_i; - - const usize live_count_before = leaf_filter.live().size(); - //----- --- -- - - - - - dropped_ranges.push_back(Interval{lower_bound_i, upper_bound_i}); - leaf_filter.drop_index_range(Interval{lower_bound_i, upper_bound_i}); - //----- --- -- - - - - - const usize live_count_after = leaf_filter.live().size(); - - if (lower_bound_i == 0) { - ASSERT_EQ(live_count_after, live_count_before); - } else { - ASSERT_EQ(live_count_after, live_count_before + 1); - } - - --drops_remaining; - next_droppable = upper_bound_i + 1; - } + std::tie(items_dropped, dropped_ranges) = + turtle_kv::testing::drop_n_disjoint_intervals_from(&leaf_filter, + drop_count, + Interval{0, item_count}, + rng); // Verify the number of expected live items. // @@ -333,8 +301,6 @@ TEST(TreePackedBlockedLeafPageTest, Random) batt::seq::for_each([&](const std::pair>& live_pair) { const auto [block_index, live_range] = live_pair; - // std::cerr << BATT_INSPECT(block_index) << BATT_INSPECT(live_range) << std::endl; - BATT_CHECK_GE(block_index, next_possible_block); BATT_CHECK_LT(block_index, packed_leaf.block_count()); BATT_CHECK_GE(live_range.lower_bound, next_possible_live) diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index fe4d3de..59b8d0b 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -15,9 +15,11 @@ #include #include +#include #include +#include #include #include #include @@ -40,6 +42,7 @@ using turtle_kv::PiecewiseFilter; using turtle_kv::Slice; using turtle_kv::Status; using turtle_kv::StatusOr; +using turtle_kv::testing::drop_n_disjoint_intervals_from; using turtle_kv::testing::RandomStringGenerator; using turtle_kv::drop_item_range; @@ -310,35 +313,35 @@ TEST(PiecewiseFilterTest, LiveSubranges) std::array, 1> init_live{{{0, 64}}}; const usize n_seeds = 10000000; - const usize n_drops = 10; + const usize n_drops = 32; const usize n_queries = 15; const usize first_seed = 0; + PiecewiseFilter filter; + + const auto query_as_bits = [&](Interval query) { + u64 bits = 0; + filter.live_subranges_of(query) | batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + for (usize seed_i = first_seed; seed_i < first_seed + n_seeds; ++seed_i) { std::default_random_engine rng{seed_i}; - PiecewiseFilter filter = - BATT_OK_RESULT_OR_PANIC(PiecewiseFilter::from_live(batt::as_slice(init_live))); + for (usize i = 0; i < n_drops; ++i) { + BATT_DEBUG_INFO(BATT_INSPECT(i) << BATT_INSPECT(seed_i)); - const auto query_as_bits = [&](Interval query) { - u64 bits = 0; - filter.live_subranges_of(query) | batt::seq::for_each([&bits](const Interval& live) { - bits |= mask_from_interval(live); - }); - return bits; - }; + filter = BATT_OK_RESULT_OR_PANIC(PiecewiseFilter::from_live(batt::as_slice(init_live))); + u64 filter_state = ~u64{0}; - u64 filter_state = ~u64{0}; + std::vector> dropped_ranges = + drop_n_disjoint_intervals_from(&filter, i, init_live[0], rng).second; - for (usize i = 0; i < n_drops; ++i) { - const Interval drop_interval = pick_interval(rng); - const u64 drop_mask = mask_from_interval(drop_interval); - filter_state &= ~drop_mask; - filter.drop_index_range(drop_interval); - - if constexpr (false) { - std::cerr << BATT_INSPECT(std::bitset<64>{filter_state}) << BATT_INSPECT(filter.dump()) - << std::endl; + for (const Interval& drop_interval : dropped_ranges) { + const u64 drop_mask = mask_from_interval(drop_interval); + filter_state &= ~drop_mask; } for (usize j = 0; j < n_queries; ++j) { diff --git a/src/turtle_kv/util/piecewise_filter.test.hpp b/src/turtle_kv/util/piecewise_filter.test.hpp new file mode 100644 index 0000000..32f71e6 --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter.test.hpp @@ -0,0 +1,129 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_TEST_HPP + +#include "piecewise_filter.hpp" +#include "piecewise_filter_storage_model.concept.hpp" + +#include + +#include +#include + +#include +#include + +namespace turtle_kv { +namespace testing { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Randomly drops `n` ranges within the specified live range of the passed filter. + * + * Requires that: + * - `drop_within` must be live in `filter` + * - `drop_within.size()` must be large enough to fit `n` disjoint intervals + * + * \return a pair of { total offset size dropped, vector of the dropped intervals } + */ +template ModelT> +inline std::pair>> drop_n_disjoint_intervals_from( + BasicPiecewiseFilter* filter, + usize n, + const Interval& drop_within, + Rng& rng) +{ + constexpr bool debug = false; + + if constexpr (debug) { + std::cerr << BATT_INSPECT(n) << std::endl; + } + + OffsetT dropped_total_size = 0; + std::vector> dropped_ranges; + + if (n == 0) { + return std::make_pair(dropped_total_size, dropped_ranges); + } + + BATT_CHECK_LE(n * 2 - 1, drop_within.size()); + BATT_CHECK_EQ(filter->live().empty(), false); + BATT_CHECK_EQ(filter->find_live_range(drop_within), drop_within); + + usize drops_remaining = n; + OffsetT next_droppable = drop_within.lower_bound; + const OffsetT live_lower_bound = filter->live().front().lower_bound; + const OffsetT live_upper_bound = filter->live().back().upper_bound; + + BATT_DEBUG_INFO(BATT_INSPECT(dropped_total_size) + << BATT_INSPECT_RANGE(dropped_ranges) << BATT_INSPECT(drop_within) + << BATT_INSPECT(n) << BATT_INSPECT(drops_remaining) + << BATT_INSPECT(next_droppable) << BATT_INSPECT(live_lower_bound) + << BATT_INSPECT(live_upper_bound)); + + if constexpr (debug) { + std::cerr << BATT_INSPECT_RANGE(filter->live()) << std::endl; + } + + for (usize drop_i = 0; drop_i < n; ++drop_i) { + BATT_CHECK_GE(next_droppable, 0); + BATT_CHECK_LT(next_droppable, drop_within.upper_bound); + + std::uniform_int_distribution pick_lower_bound{ + next_droppable, + drop_within.upper_bound - (drops_remaining * 2 - 1), + }; + const OffsetT lower_bound_i = pick_lower_bound(rng); + + std::uniform_int_distribution pick_upper_bound{ + lower_bound_i + 1, + drop_within.upper_bound - (drops_remaining * 2 - 2), + }; + const OffsetT upper_bound_i = pick_upper_bound(rng); + + BATT_CHECK_LT(lower_bound_i, upper_bound_i); + BATT_CHECK_GE(lower_bound_i, next_droppable); + + dropped_total_size += upper_bound_i - lower_bound_i; + + const usize live_count_before = filter->live().size(); + //----- --- -- - - - - + dropped_ranges.push_back(Interval{lower_bound_i, upper_bound_i}); + if constexpr (debug) { + std::cerr << " dropping: " << lower_bound_i << ".." << upper_bound_i << std::endl; + } + filter->drop_index_range(Interval{lower_bound_i, upper_bound_i}); + //----- --- -- - - - - + const usize live_count_after = filter->live().size(); + + if constexpr (debug) { + std::cerr << BATT_INSPECT_RANGE(filter->live()) << std::endl; + } + + if (lower_bound_i == live_lower_bound && upper_bound_i == live_upper_bound) { + BATT_CHECK_EQ(live_count_after + 1, live_count_before); + + } else if ((lower_bound_i == live_lower_bound && upper_bound_i != live_upper_bound) || + (upper_bound_i == live_upper_bound && lower_bound_i != live_lower_bound)) { + BATT_CHECK_EQ(live_count_after, live_count_before); + + } else { + BATT_CHECK_EQ(live_count_after, live_count_before + 1); + } + + --drops_remaining; + next_droppable = upper_bound_i + 1; + } + + return std::make_pair(dropped_total_size, dropped_ranges); +} + +} // namespace testing +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp index 6dd8701..5a105df 100644 --- a/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp +++ b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp @@ -17,7 +17,6 @@ #include #include #include -#include #include namespace turtle_kv { From aabd297f590c4d3d3afcd1b2bb7ef3bc918f6539 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 28 Jun 2026 12:10:33 -0400 Subject: [PATCH 09/20] Temporary refactor in preparation to switch to artc::ART --- src/turtle_kv/util/art.hpp | 1582 +---------------- src/turtle_kv/util/art.test.cpp | 61 +- src/turtle_kv/util/art_base.hpp | 928 ++++++++++ src/turtle_kv/util/art_bit_ops.hpp | 51 + src/turtle_kv/util/art_default_inserters.hpp | 66 + src/turtle_kv/util/art_metrics.hpp | 69 + src/turtle_kv/util/art_mutex.hpp | 27 + src/turtle_kv/util/art_scanner.hpp | 362 ++++ .../util/detail/scanner_item_storage_base.hpp | 75 + .../detail/scanner_value_storage_base.hpp | 74 + 10 files changed, 1709 insertions(+), 1586 deletions(-) create mode 100644 src/turtle_kv/util/art_base.hpp create mode 100644 src/turtle_kv/util/art_bit_ops.hpp create mode 100644 src/turtle_kv/util/art_default_inserters.hpp create mode 100644 src/turtle_kv/util/art_metrics.hpp create mode 100644 src/turtle_kv/util/art_mutex.hpp create mode 100644 src/turtle_kv/util/art_scanner.hpp create mode 100644 src/turtle_kv/util/detail/scanner_item_storage_base.hpp create mode 100644 src/turtle_kv/util/detail/scanner_value_storage_base.hpp diff --git a/src/turtle_kv/util/art.hpp b/src/turtle_kv/util/art.hpp index d4c050b..460ecdd 100644 --- a/src/turtle_kv/util/art.hpp +++ b/src/turtle_kv/util/art.hpp @@ -1,1093 +1,34 @@ #pragma once -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include // SSE2 -#include // MMX -#include // SSE3 - -#ifdef __AVX512F__ -#include // AVX512 (AVX, AVX2, FMA) -#endif - -namespace turtle_kv { - -/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns one of: {4, - * 5, 6, 7}. - */ -inline usize index_of(u8 key_byte, const std::array& keys) -{ - __m64 pattern = _mm_set1_pi8((char)key_byte); - u64 extended = *((const u32*)keys.data()); - __m64 values = _mm_cvtsi64_m64(extended); - __m64 result = _m_pcmpeqb(pattern, values); - - return ((__builtin_ffsll((i64)result) - 1) >> 3) & 7; -} - -/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns 31. - */ -inline usize index_of(u8 key_byte, const std::array& keys) -{ - __m128i pattern = _mm_set1_epi8((char)key_byte); - __m128i values = _mm_lddqu_si128((const __m128i*)keys.data()); - -#ifndef __AVX512F__ - int result = _mm_movemask_epi8(_mm_cmpeq_epi8(pattern, values)); -#else - __mmask16 result = _mm_cmpeq_epi8_mask(pattern, values); -#endif - - return (__builtin_ffs(result) - 1) & 31; -} - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -class ARTBase -{ - public: - static constexpr usize kMaxKeyLen = 64; - - /** \brief Tag type indicating that a new object should not be initialized by the ctor. - */ - struct NoInit { - }; - - struct Metrics { - CountMetric construct_count; - CountMetric destruct_count; - FastCountMetric insert_count; - FastCountMetric byte_alloc_count; - FastCountMetric byte_free_count; - - /** \brief Resets all metrics to initial values. - */ - void reset() - { - this->construct_count.reset(); - this->destruct_count.reset(); - this->insert_count.reset(); - this->byte_alloc_count.reset(); - this->byte_free_count.reset(); - } - - //----- --- -- - - - - - - double bytes_per_instance() const - { - return (double)this->byte_alloc_count.get() / (double)this->construct_count.get(); - } - - double average_item_count() const - { - return (double)this->insert_count.get() / (double)this->construct_count.get(); - } - - double bytes_per_insert() const - { - return (double)this->byte_alloc_count.get() / (double)this->insert_count.get(); - } - - /** \brief Returns an estimate of the number of active instances (ART objects). - */ - u64 instance_count() const - { - // Must be in this order! - // - const u64 observed_destruct_count = this->destruct_count.get(); - const u64 observed_construct_count = this->construct_count.get(); - - return observed_construct_count - observed_destruct_count; - } - - /** \brief Returns an estimate of the current number of bytes in use. - */ - u64 bytes_in_use() const - { - // Must be in this order! - // - const u64 observed_free_count = this->byte_free_count.get(); - const u64 observed_alloc_count = this->byte_alloc_count.get(); - - return observed_alloc_count - observed_free_count; - } - }; - - static Metrics& default_metrics() - { - static Metrics m_; - return m_; - } - - enum struct Synchronized { - kFalse = 0, - kTrue = 1, - kDynamic = 2, - }; - - struct Node4; - struct Node16; - struct Node48; - struct Node256; - struct LeafNode; - - enum struct NodeType : u8 { - kLeafNode = 0, - kNode4 = 1, - kNode16 = 2, - kNode48 = 3, - kNode256 = 4, - kNodeBase = 5, - }; - - //----- --- -- - - - - - - static constexpr usize sizeof_value(batt::StaticType) - { - return 0; - } - - template - static constexpr usize sizeof_value(batt::StaticType) - { - return sizeof(ValueT); - } - - //----- --- -- - - - - - - template - static void* uninitialized_value(NodeT* node) - { - return node + 1; - } - - template - static ValueT* mutable_value(NodeT* node, batt::StaticType /**/ = {}) - { - return reinterpret_cast(node + 1); - } - - template - static const ValueT* const_value(const NodeT* node, batt::StaticType /**/ = {}) - { - return reinterpret_cast(node + 1); - } - - //----- --- -- - - - - - - template - static void* construct_value_copy_node(FromNodeT*, ToNodeT* to_node, batt::StaticType) - { - to_node->set_terminal(); - return nullptr; - } - - template - static void* construct_value_copy_addr(FromNodeT*, void*, batt::StaticType) - { - return nullptr; - } - - //----- --- -- - - - - - - template - static ValueT* construct_value_copy_node(FromNodeT* from_node, - ToNodeT* to_node, - batt::StaticType type_of_value) - { - to_node->set_terminal(); - return ARTBase::construct_value_copy_addr(from_node, - ARTBase::uninitialized_value(to_node), - type_of_value); - } - - template - static ValueT* construct_value_copy_addr(FromNodeT* from_node, - void* to_address, - batt::StaticType type_of_value) - { - return new (to_address) ValueT{*ARTBase::const_value(from_node, type_of_value)}; - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // Node class hierarchy: - // - // ┌────────┐ - // │NodeBase│ - // └────────┘ - // △ - // ┌───────────────┤ - // │ │ - // ┌───────────────┐ │ - // │GrowableNode│ │ - // └───────────────┘ │ - // △ │ - // ┌─────────┴────────────┐ └────────┐ - // │ │ │ - // ┌──────────────────────┐┌────────────────────┐ │ - // │IndirectIndexedNode││DirectIndexedNode│ │ - // └──────────────────────┘└────────────────────┘ │ - // △ △ │ - // ┌─────┴───────┐ │ │ - // │ │ │ │ - // ┌─────┐ ┌──────┐ ┌──────┐ ┌───────┐ - // │Node4│ │Node16│ │Node48│ │Node256│ - // └─────┘ └──────┘ └──────┘ └───────┘ - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // Node memory layout: - // - // ┌────────────┬──────────┬──────────────┬───────────────┐ - // │ key prefix │ NodeBase │ (impl) ... │ ValueT │ - // └────────────┴──────────┴──────────────┴───────────────┘ - // ◀──────────▶ ◀───────────────────────▶ ◀─────────────▶ - // variable sizeof(NodeT) sizeof(ValueT) - // length - // - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - - struct NodeBase { - using Self = NodeBase; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr u8 kFlagTerminal = 0x80; - static constexpr u8 kFlagObsolete = 0x40; - - //+++++++++++-+-+--+----- --- -- - - - - - - const NodeType node_type; - - u8 flags_; - u8 prefix_len_; - u8 branch_count_; - SeqMutex mutex_; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit NodeBase(NodeType t) noexcept - : node_type{t} - , flags_{0} - , prefix_len_{0} - , branch_count_{0} - { - } - - explicit NodeBase(NodeType t, ARTBase::NoInit) noexcept : node_type{t} - { - } - - NodeBase(const NodeBase&) = delete; - NodeBase& operator=(const NodeBase&) = delete; - - template - void visit(CaseFns&&... case_fns); - - bool is_terminal() const - { - return (this->flags_ & kFlagTerminal) != 0; - } - - void set_terminal() - { - this->flags_ |= kFlagTerminal; - } - - bool is_obsolete() const - { - return (this->flags_ & kFlagObsolete) != 0; - } - - void set_obsolete() - { - this->flags_ |= kFlagObsolete; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->flags_ = that.flags_; - this->branch_count_ = that.branch_count_; - this->set_prefix(that.prefix() + prefix_offset, that.prefix_len_ - prefix_offset); - } - - const char* prefix() const - { - return (const char*)((((std::uintptr_t)this) - this->prefix_len_) & ~std::uintptr_t{7}); - } - - void set_prefix(const char* data, usize len) - { - this->prefix_len_ = len; - if (len) { - __builtin_memcpy((char*)this->prefix(), data, len); - } - } - }; - - struct LeafNode : NodeBase { - using Self = LeafNode; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - explicit LeafNode() noexcept : Super{NodeType::kLeafNode} - { - } - - explicit LeafNode(NoInit no_init) noexcept : Super{NodeType::kLeafNode, no_init} - { - } - - static usize add_branch() - { - BATT_PANIC() << "not supported!"; - return 0; - } - - static void set_branch_index(u8 key_byte [[maybe_unused]], usize index [[maybe_unused]]) - { - BATT_PANIC() << "not supported!"; - } - - static void set_branch_pointer(usize index [[maybe_unused]], NodeBase* child [[maybe_unused]]) - { - BATT_PANIC() << "not supported!"; - } - - static constexpr usize max_branch_count() - { - return 0; - } - - static constexpr usize branch_count() - { - return 0; - } - - static constexpr usize index_of_branch(u8 key_byte [[maybe_unused]]) - { - return 0; - } - - static NodeBase*& get_branch_ref(usize i [[maybe_unused]]) - { - static NodeBase* null_ = nullptr; - return null_; - } - - //+++++++++++-+-+--+----- --- -- - - - - - - struct ScanState { - explicit ScanState(Self&, ByteInt /*min_key*/) noexcept - { - } - - static constexpr ByteInt get_key_byte() - { - return ByteInt::from_char('\0'); - } - - static constexpr NodeBase* get_branch() - { - return nullptr; - } - - static constexpr bool is_done() - { - return true; - } - - static constexpr void advance() - { - } - }; - }; - - struct BranchView { - NodeBase** p_ptr; - NodeBase* ptr; - - //+++++++++++-+-+--+----- --- -- - - - - - - BranchView() noexcept : p_ptr{nullptr}, ptr{nullptr} - { - } - - explicit BranchView(NodeBase*& branch) noexcept : p_ptr{&branch}, ptr{branch} - { - } - - void load(NodeBase*& branch) - { - this->p_ptr = &branch; - this->ptr = branch; - } - - template - NodeT* store(NodeT* new_ptr) - { - static_assert(std::is_base_of_v); - - *this->p_ptr = new_ptr; - this->ptr = new_ptr; - return new_ptr; - } - - NodeBase* reload() - { - this->ptr = *this->p_ptr; - return this->ptr; - } - }; - - static constexpr NodeType node_type_from_branch_count(usize branch_count) - { - if (branch_count == 4) { - return NodeType::kNode4; - } else if (branch_count == 16) { - return NodeType::kNode16; - } else if (branch_count == 48) { - return NodeType::kNode48; - } else { - return NodeType::kNode256; - } - } - - static_assert(sizeof(NodeBase) == 8); - - using BranchIndex = u8; - - static constexpr BranchIndex kInvalidBranchIndex = u8{255}; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct GrowableNode : NodeBase { - using Self = GrowableNode; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branches_; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit GrowableNode() noexcept : NodeBase{node_type_from_branch_count(kBranchCount)} - { - } - - explicit GrowableNode(NoInit no_init) noexcept - : NodeBase{node_type_from_branch_count(kBranchCount), no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - Derived* derived() - { - return (Derived*)this; - } - - //----- --- -- - - - - - - usize branch_count() const - { - return this->branch_count_; - } - - usize add_branch() - { - const usize i = this->branch_count_; - ++this->branch_count_; - return i; - } - - NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE - { - return this->branches_[i]; - } - - void set_branch_pointer(usize i, NodeBase* child) BATT_ALWAYS_INLINE - { - this->branches_[i] = child; - } - - static constexpr usize max_branch_count() - { - return kBranchCount; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - __builtin_memcpy(this->branches_.data(), - that.branches_.data(), - this->branch_count() * sizeof(NodeBase*)); - } - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct IndirectIndexedNode : GrowableNode> { - using Self = IndirectIndexedNode; - using Super = GrowableNode; - using NoInit = ARTBase::NoInit; - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) \ - if (bit_i == 64) { \ - break; \ - } \ - key_byte = ByteInt::from_i32(key_byte_offset + bit_i); \ - branch = branch_for_byte[key_byte.to_i32()]; \ - if (branch) { \ - this->sorted_branches_[this->branch_count_] = branch; \ - this->sorted_keys_[this->branch_count_] = key_byte; \ - ++this->branch_count_; \ - } \ - bit_i = next_bit(word_val, bit_i) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(word_i, key_byte_offset) \ - word_val = key_bitmap[word_i]; \ - for (;;) { \ - i32 bit_i = first_bit(word_val); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset); \ - break; \ - } - - struct ScanState { - Self& self_; - usize branch_count_; - usize i_; - std::array sorted_branches_; - std::array sorted_keys_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept - : self_{self} - , branch_count_{0} - , i_{0} - { - std::array branch_for_byte; - std::array key_bitmap = {0, 0, 0, 0}; - - const usize n_branches = this->self_.branch_count(); - - for (usize i = 0; i < n_branches; ++i) { - const ByteInt key_byte = ByteInt::from_u8(this->self_.key[i]); - if (key_byte < min_key) { - continue; - } - branch_for_byte[key_byte.to_i32()] = this->self_.branches_[i]; - key_bitmap[(key_byte.to_i32() >> 6) & 3] |= (u64{1} << (key_byte.to_i32() & 0x3f)); - } - - u64 word_val; - ByteInt key_byte; - NodeBase* branch; - - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(0, 0) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(1, 64) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(2, 128) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(3, 192) - } - - ByteInt get_key_byte() const - { - return this->sorted_keys_[this->i_]; - } - - NodeBase* get_branch() const - { - return this->sorted_branches_[this->i_]; - } - - bool is_done() const - { - return this->i_ >= this->branch_count_; - } - - void advance() - { - ++this->i_; - } - }; - -#undef TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1 - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array key; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit IndirectIndexedNode() noexcept : Super{} - { - } - - explicit IndirectIndexedNode(NoInit no_init) noexcept : Super{no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - usize index_of_branch(u8 key_byte) - { - return index_of(key_byte, this->key); - } - - void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE - { - this->key[i] = key_byte; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - __builtin_memcpy(this->key.data(), that.key.data(), this->branch_count()); - } - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct DirectIndexedNode : GrowableNode> { - using Self = DirectIndexedNode; - using Super = GrowableNode; - using NoInit = ARTBase::NoInit; - - struct ScanState { - Self& self_; - ByteInt key_byte_; - usize branch_i_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept - : self_{self} - , key_byte_{min_key} - , branch_i_{kInvalidBranchIndex} - { - this->skip_invalid_branches(); - } - - ByteInt get_key_byte() const - { - return this->key_byte_; - } - - NodeBase* get_branch() const - { - return this->self_.branches_[this->branch_i_]; - } - - bool is_done() const - { - return this->key_byte_ >= ByteInt::from_i32(256); - } - - void advance() - { - ++this->key_byte_; - this->skip_invalid_branches(); - } - - void skip_invalid_branches() - { - while (!this->is_done()) { - this->branch_i_ = this->self_.branch_for_key[this->key_byte_.to_i32()]; - if (this->branch_i_ != kInvalidBranchIndex) { - break; - } - ++this->key_byte_; - } - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branch_for_key; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit DirectIndexedNode() noexcept : Super{} - { - this->branch_for_key.fill(kInvalidBranchIndex); - } - - explicit DirectIndexedNode(NoInit no_init) noexcept : Super{no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - usize index_of_branch(u8 key_byte) - { - return this->branch_for_key[key_byte]; - } - - void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE - { - this->branch_for_key[key_byte] = i; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - this->branch_for_key = that.branch_for_key; - } - }; - - struct Node4 : IndirectIndexedNode<4> { - using IndirectIndexedNode<4>::IndirectIndexedNode; - }; - - struct Node16 : IndirectIndexedNode<16> { - using IndirectIndexedNode<16>::IndirectIndexedNode; - }; - - struct Node48 : DirectIndexedNode<48> { - using DirectIndexedNode<48>::DirectIndexedNode; - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - struct Node256 : NodeBase { - using Self = Node256; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - struct ScanState { - Self& self_; - ByteInt key_byte_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept : self_{self}, key_byte_{min_key} - { - this->skip_null_branches(); - } - - ByteInt get_key_byte() const - { - return this->key_byte_; - } - - NodeBase* get_branch() const - { - return this->self_.branches_[this->key_byte_.to_i32()]; - } - - bool is_done() const - { - return this->key_byte_ >= ByteInt::from_i32(256); - } - - void advance() - { - ++this->key_byte_; - this->skip_null_branches(); - } - - private: - void skip_null_branches() - { - while (!this->is_done() && this->get_branch() == nullptr) { - ++this->key_byte_; - } - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branches_; - - //+++++++++++-+-+--+----- --- -- - - - - - - Node256() noexcept : Super{NodeType::kNode256} - { - this->branches_.fill(nullptr); - } - - explicit Node256(NoInit no_init) noexcept : Super{NodeType::kNode256, no_init} - { - } - - Node256(const Node256&) = delete; - Node256& operator=(const Node256&) = delete; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr usize branch_count() - { - return 256; - } - - usize add_branch() - { - BATT_PANIC() << "Node256::add_branch is illegal!"; - BATT_UNREACHABLE(); - } - - static constexpr usize max_branch_count() - { - return 256; - } - - usize index_of_branch(u8 key_byte) const - { - return key_byte; - } - - void set_branch_index(u8, usize) - { - } - - NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE - { - return this->branches_[i]; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - this->branches_ = that.branches_; - } - }; - - //----- --- -- - - - - - - static_assert(sizeof(Node4) == 48); - static_assert(sizeof(Node4) % 8 == 0); - static_assert(alignof(Node4) >= 8); - - static_assert(sizeof(Node16) == 152); - static_assert(sizeof(Node16) % 8 == 0); - static_assert(alignof(Node16) >= 8); - - static_assert(sizeof(Node48) == 648); - static_assert(sizeof(Node48) % 8 == 0); - static_assert(alignof(Node48) >= 8); - - static_assert(sizeof(Node256) == 2056); - static_assert(sizeof(Node256) % 8 == 0); - static_assert(alignof(Node256) >= 8); - - static constexpr usize kExtentSize = 64 * kKiB; - static constexpr usize kExtentAlign = 4096; - - using ExtentStorageT = std::aligned_storage_t; - - static_assert(sizeof(ExtentStorageT) == kExtentSize); - - //----- --- -- - - - - - - struct MemoryContext { - ARTBase* art_{nullptr}; - std::vector> thread_extents_; - u8* data_{nullptr}; - usize in_use_{sizeof(ExtentStorageT)}; - - //+++++++++++-+-+--+----- --- -- - - - - - - ~MemoryContext() noexcept - { - if (this->art_) { - absl::MutexLock lock{this->art_->mutex_}; - for (auto& p_ex : this->thread_extents_) { - this->art_->extents_.emplace_back(std::move(p_ex)); - } - } - } - - void* alloc(usize n, ARTBase* art) - { - this->art_ = art; - - const usize in_use_prior = this->in_use_; - if (in_use_prior + n <= kExtentSize) { - this->in_use_ += n; - return this->data_ + in_use_prior; - } +#include "art_base.hpp" +#include "art_bit_ops.hpp" +#include "art_default_inserters.hpp" - this->art_->metrics_.byte_alloc_count.add(sizeof(ExtentStorageT)); +#include - this->thread_extents_.emplace_back(std::make_unique()); - u8* start = reinterpret_cast(this->thread_extents_.back().get()); - this->data_ = start; - this->in_use_ = 0; - - return this->alloc(n, art); - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit ARTBase() noexcept : ARTBase{ARTBase::default_metrics()} - { - // No update of construct count because we delegate to general-case ctor. - } - - explicit ARTBase(Metrics& metrics) noexcept : metrics_{metrics} - { - this->metrics_.construct_count.add(1); - } - - ~ARTBase() noexcept - { - this->metrics_.destruct_count.add(1); - } - - //+++++++++++-+-+--+----- --- -- - - - - - protected: - /** \brief RAII (guard) class that updates byte_free_count metric at the right moment during - * destruction of the ART (see comment in data member declarations below). - */ - struct ExtentMetricsUpdateGuard { - ARTBase& art_base_; - - //----- --- -- - - - - - - explicit ExtentMetricsUpdateGuard(ARTBase& art_base) noexcept : art_base_{art_base} - { - } - - ~ExtentMetricsUpdateGuard() noexcept - { - this->art_base_.metrics_.byte_free_count.add(this->art_base_.extents_.size() * - sizeof(ExtentStorageT)); - } - }; - - void* alloc_storage(usize n, usize pre) - { - const usize pad = (pre + 7) & ~usize{7}; - char* const ptr = (char*)this->per_thread_memory_context_.get().alloc(n + pad, this); - return ptr + pad; - } - - Metrics& metrics_; - absl::Mutex mutex_; - std::vector> extents_; - // - // Must be placed exactly here, so it will be destructed after the ScopedSlot but before extents_. - ExtentMetricsUpdateGuard guard_{*this}; - // - ObjectThreadStorage::ScopedSlot per_thread_memory_context_; -}; - -namespace detail { - -template -struct DefaultCopyInserter { - const ValueT& copy_from_; - - explicit DefaultCopyInserter(const ValueT& copy_from) noexcept : copy_from_{copy_from} - { - } - - Status insert_new(void* copy_to) - { - new (copy_to) ValueT{this->copy_from_}; - return OkStatus(); - } - - Status update_existing(ValueT* copy_to) - { - *copy_to = this->copy_from_; - return OkStatus(); - } -}; - -template -struct DefaultMoveInserter { - ValueT&& move_from_; - - explicit DefaultMoveInserter(ValueT&& move_from) noexcept : move_from_{move_from} - { - } - - Status insert_new(void* move_to) - { - new (move_to) ValueT{std::move(this->move_from_)}; - return OkStatus(); - } +#include +#include - Status update_existing(ValueT* move_to) - { - *move_to = std::move(this->move_from_); - return OkStatus(); - } -}; +#include +#include +#include +#include +#include +#include +#include +#include -struct DefaultVoidInserter { - BATT_ALWAYS_INLINE Status insert_new(void*) - { - return OkStatus(); - } +#include +#include +#include - BATT_ALWAYS_INLINE Status update_existing(void*) - { - return OkStatus(); - } -}; +#include +#include +#include +#include +#include -} // namespace detail +namespace turtle_kv { //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- // @@ -1132,7 +73,7 @@ class ART : public ARTBase { static_assert(std::is_same_v); - this->insert(key, detail::DefaultVoidInserter{}).IgnoreError(); + this->insert(key, DefaultVoidInserter{}).IgnoreError(); } bool contains(std::string_view key); @@ -1245,480 +186,6 @@ inline void ARTBase::NodeBase::visit(CaseFns&&... case_fns) } } -namespace detail { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of(usize node_prefix_len, - NodeT* node, - AlignedStorageT* storage, - std::integral_constant, - const Optional&, - void* value_storage_addr, - batt::StaticType type_of_value) -{ - NodeT& node_view = *(new (storage) NodeT{ARTBase::NoInit{}}); - - // Retry the node read until we get a consistent view. - // - for (;;) { - SeqMutex::ReadLock read_lock{node->mutex_}; - node_view.assign_from(*node, /*prefix_offset=*/node_prefix_len); - if (node->is_terminal()) { - ARTBase::construct_value_copy_addr(node, value_storage_addr, type_of_value); - } - if (!read_lock.changed()) { - break; - } - } - - return node_view; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of(usize, - NodeT* node, - AlignedStorageT*, - std::integral_constant, - const Optional& /*sync*/, - const void* /*value_storage_addr*/, - batt::StaticType /*type_of_value*/) -{ - return *node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of( - usize node_prefix_len, - NodeT* node, - AlignedStorageT* storage, - std::integral_constant, - const Optional& sync, - void* value_storage_addr, - batt::StaticType type_of_value) -{ - if (sync.value_or(true)) { - return scanner_view_of( - node_prefix_len, - node, - storage, - std::integral_constant{}, - sync, - value_storage_addr, - type_of_value); - } - return scanner_view_of( - node_prefix_len, - node, - storage, - std::integral_constant{}, - sync, - value_storage_addr, - type_of_value); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -struct ValueStorageBase { - std::aligned_storage_t value_storage_; - - template - std::conditional_t, const void*, void*> value_storage_address( - NodeT* node, - const Optional& sync) - { - if (kSynchronized == ARTBase::Synchronized::kTrue || sync.value_or(true)) { - return &this->value_storage_; - } - return node + 1; - } -}; - -//----- --- -- - - - - - -template -struct ValueStorageBase { - template - const void* value_storage_address(const NodeT* node, const Optional&) const - { - return node + 1; - } -}; - -//----- --- -- - - - - - -template <> -struct ValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//----- --- -- - - - - - -template <> -struct ValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//----- --- -- - - - - - -template <> -struct ValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Base class for scanner; contains storage for the item at the current scanner position. - */ -template -struct ItemStorageBase; - -/** \brief General case (kValuesOnly == false) for scanner item storage. - */ -template -class ItemStorageBase - : public ValueStorageBase -{ - public: - std::array::kMaxKeyLen> key_buffer_; - usize key_len_ = 0; - - //----- --- -- - - - - - - void append_key(usize prefix_len, const char* suffix_data, usize suffix_len) BATT_ALWAYS_INLINE - { - __builtin_memcpy(this->key_buffer_.data() + prefix_len, suffix_data, suffix_len); - } - - void append_key_byte(usize prefix_len, const ByteInt& suffix_byte) BATT_ALWAYS_INLINE - { - this->key_buffer_[prefix_len] = suffix_byte.to_char(); - } - - void set_key_len(usize len) BATT_ALWAYS_INLINE - { - this->key_len_ = len; - } - - std::string_view get_key() const - { - return std::string_view{this->key_buffer_.data(), this->key_len_}; - } -}; - -/** \brief kValuesOnly == true case; no key-related data members. - */ -template -class ItemStorageBase - : public ValueStorageBase -{ - public: - void append_key(usize, const char*, usize) BATT_ALWAYS_INLINE - { - // nothing to do. - } - - void append_key_byte(usize, const ByteInt&) BATT_ALWAYS_INLINE - { - // nothing to do. - } - - void set_key_len(usize) BATT_ALWAYS_INLINE - { - // nothing to do. - } -}; - -} // namespace detail - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Scanner for an ART. - * - * \tparam ValueT The value type stored in the scanned ART - * \tparam kSynchronized (true, false, dynmamic) The concurrency control for this scanner - * \tparam kValuesOnly When true, the scanner does not build/store key (path) information as it is - * traversing items -- only values are available - */ -template -template -class ART::Scanner : public detail::ItemStorageBase -{ - public: - using LeafNode = ARTBase::LeafNode; - using Node4 = ARTBase::Node4; - using Node16 = ARTBase::Node16; - using Node48 = ARTBase::Node48; - using Node256 = ARTBase::Node256; - - using NodeScanState = std::variant; - - static constexpr usize kMaxDepth = ART::kMaxKeyLen; - - using SyncType = std::integral_constant; - - using Value = std::conditional_t, - struct get_value_Not_Supported_If_ValueT_Is_Void, - ValueT>; - - //+++++++++++-+-+--+----- --- -- - - - - - - static_assert(sizeof(Node256) > sizeof(Node48)); - static_assert(sizeof(Node256) > sizeof(Node16)); - static_assert(sizeof(Node256) > sizeof(Node4)); - static_assert(sizeof(Node256) > sizeof(LeafNode)); - - struct Frame { - static constexpr usize kStorageSize = - ((kSynchronized == ARTBase::Synchronized::kFalse) ? 1 : sizeof(Node256)); - - std::aligned_storage_t node_storage_; - NodeScanState scan_state_; - usize key_prefix_len_; - std::string_view lower_bound_key_; - ByteInt min_key_byte_; - - explicit Frame(usize key_prefix_len, std::string_view lower_bound_key) noexcept - : scan_state_{None} - , key_prefix_len_{key_prefix_len} - , lower_bound_key_{lower_bound_key} - , min_key_byte_{ByteInt::from_i32(0)} - { - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - private: - std::aligned_storage_t stack_storage_; - Frame* end_ = reinterpret_cast(&this->stack_storage_); - usize depth_ = 0; - bool have_item_ = false; - ValueT* next_value_ = nullptr; - Optional synchronized_; - - //+++++++++++-+-+--+----- --- -- - - - - - - /** Resets the "have item" state of the scanner; after calling, have item will be false. - */ - void reset_item() BATT_ALWAYS_INLINE - { - this->have_item_ = false; - if (!std::is_same_v) { - this->next_value_ = nullptr; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - public: - explicit Scanner(ART& art, - std::string_view lower_bound_key, - Optional synchronized = None) noexcept - : synchronized_{synchronized} - { - NodeBase* root = nullptr; - for (;;) { - SeqMutex::ReadLock root_read_lock{art.super_root_.mutex_}; - root = art.root_; - if (!root_read_lock.changed()) { - break; - } - } - - if (root) { - root->visit([&](auto* node) { - this->enter(node, /*key_prefix_len=*/0, lower_bound_key); - }); - - if (!this->have_item_) { - this->advance(); - } - } - } - - ~Scanner() noexcept - { - } - - bool is_synchronized() const - { - if (kSynchronized == ARTBase::Synchronized::kFalse) { - return false; - } - if (kSynchronized == ARTBase::Synchronized::kTrue) { - return true; - } - return this->synchronized_.value_or(true); - } - - template - void enter(NodeT* node, usize key_prefix_len, std::string_view lower_bound_key) - { - Frame* top = new (this->end_) Frame{key_prefix_len, lower_bound_key}; - ++this->depth_; - ++this->end_; - - // Node prefix is immutable, so we don't need synchronization. - // - const char* const node_prefix = node->prefix(); - const usize node_prefix_len = node->prefix_len_; - - // We need to create a copy of the node data to protect against data races. - // - NodeT& node_view = - detail::scanner_view_of(node_prefix_len, - node, - &top->node_storage_, - SyncType{}, - this->synchronized_, - this->value_storage_address(node, this->synchronized_), - batt::StaticType{}); - - // Compare the lower bound key to the current node prefix. - // - const usize compare_len = std::min(node_prefix_len, top->lower_bound_key_.size()); - if (compare_len) { - const i32 order = __builtin_memcmp(node_prefix, top->lower_bound_key_.data(), compare_len); - - // If all keys in this subtree come before the lower bound, then there is nothing to do. - // - if (order < 0) { - --this->depth_; - --this->end_; - return; - } - - // If the node prefix is a prefix of the lower bound key, then drop the prefix from the lower - // bound; otherwise the node prefix comes *after* the lower bound, so we can safely ignore the - // lower bound for the rest of the recursion. - // - if (order == 0 && compare_len == node_prefix_len) { - top->lower_bound_key_.remove_prefix(compare_len); - } else { - top->lower_bound_key_ = {}; - } - } - - // Set bounds for branch visitation. - // - top->min_key_byte_ = [&]() -> ByteInt { - if (top->lower_bound_key_.empty()) { - return ByteInt::from_i32(0); - } - const ByteInt next_char = ByteInt::from_char(top->lower_bound_key_.front()); - top->lower_bound_key_.remove_prefix(1); - return next_char; - }(); - - // Append the node prefix to the buffer. - // - if (node_prefix_len) { - this->append_key(top->key_prefix_len_, node_prefix, node_prefix_len); - top->key_prefix_len_ += node_prefix_len; - } - - // If the current node is a key-terminal, emit the contents of the buffer. - // - if (node_view.is_terminal()) { - this->have_item_ = true; - this->set_key_len(top->key_prefix_len_); - if (!std::is_same_v) { - this->next_value_ = (ValueT*)(this->value_storage_address(&node_view, this->synchronized_)); - } - } else { - this->reset_item(); - } - - [[maybe_unused]] auto& scan_state_impl = - top->scan_state_.template emplace(node_view, top->min_key_byte_); - } - - bool is_done() const - { - return this->depth_ == 0; - } - - // get_key() const member function is inherited from ItemStorageBase, if kValuesOnly is false. - - const Value& get_value() const - { - static_assert(!std::is_same_v); - return *this->next_value_; - } - - void advance() - { - this->reset_item(); - - for (;;) { - if (this->depth_ == 0) { - return; - } - - Frame* top = this->end_ - 1; - - batt::case_of( - top->scan_state_, - [](batt::NoneType&) { - BATT_PANIC() << "empty Scanner stack frame!"; - }, - [&](auto& scan_state) - -> std::enable_if_t< - !std::is_same_v, batt::NoneType>> { - //----- --- -- - - - - - if (scan_state.is_done()) { - --this->depth_; - --this->end_; - return; - } - - const ByteInt key_byte = scan_state.get_key_byte(); - NodeBase* const child = scan_state.get_branch(); - - this->append_key_byte(top->key_prefix_len_, key_byte); - - if (key_byte == top->min_key_byte_) { - child->visit([&](auto* child_node) { - this->enter(child_node, top->key_prefix_len_ + 1, top->lower_bound_key_); - }); - } else { - child->visit([&](auto* child_node) { - this->enter(child_node, top->key_prefix_len_ + 1, std::string_view{}); - }); - } - - scan_state.advance(); - }); - - if (this->have_item_) { - return; - } - } - } -}; - //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -1740,3 +207,4 @@ inline void ART::scan(std::string_view lower_bound_key, const Fn& fn) } // namespace turtle_kv #include +#include diff --git a/src/turtle_kv/util/art.test.cpp b/src/turtle_kv/util/art.test.cpp index 91cb16e..64bdfd8 100644 --- a/src/turtle_kv/util/art.test.cpp +++ b/src/turtle_kv/util/art.test.cpp @@ -21,8 +21,10 @@ namespace { using namespace batt::int_types; +using turtle_kv::ART; using turtle_kv::ARTBase; using turtle_kv::ByteInt; +using turtle_kv::DefaultCopyInserter; using turtle_kv::LatencyMetric; using turtle_kv::LatencyTimer; using turtle_kv::None; @@ -31,7 +33,7 @@ using turtle_kv::Optional; using turtle_kv::Status; using turtle_kv::testing::RandomStringGenerator; -using ART = turtle_kv::ART; +using ARTSet = ART; struct BigUInt64KeyGenerator { template @@ -110,35 +112,35 @@ TEST(ArtTest, OverlappingKeyPrefix) const std::string key3 = "application"; const std::string key4 = "applesauce"; - turtle_kv::ART art; + ART art; EXPECT_FALSE(art.contains(key1)); EXPECT_FALSE(art.contains(key2)); EXPECT_FALSE(art.contains(key3)); EXPECT_FALSE(art.contains(key4)); - BATT_CHECK_OK(art.insert(key1, turtle_kv::detail::DefaultCopyInserter{1})); + BATT_CHECK_OK(art.insert(key1, DefaultCopyInserter{1})); EXPECT_TRUE(art.contains(key1)); EXPECT_FALSE(art.contains(key2)); EXPECT_FALSE(art.contains(key3)); EXPECT_FALSE(art.contains(key4)); - BATT_CHECK_OK(art.insert(key2, turtle_kv::detail::DefaultCopyInserter{2})); + BATT_CHECK_OK(art.insert(key2, DefaultCopyInserter{2})); EXPECT_TRUE(art.contains(key1)); EXPECT_TRUE(art.contains(key2)); EXPECT_FALSE(art.contains(key3)); EXPECT_FALSE(art.contains(key4)); - BATT_CHECK_OK(art.insert(key3, turtle_kv::detail::DefaultCopyInserter{3})); + BATT_CHECK_OK(art.insert(key3, DefaultCopyInserter{3})); EXPECT_TRUE(art.contains(key1)); EXPECT_TRUE(art.contains(key2)); EXPECT_TRUE(art.contains(key3)); EXPECT_FALSE(art.contains(key4)); - BATT_CHECK_OK(art.insert(key4, turtle_kv::detail::DefaultCopyInserter{4})); + BATT_CHECK_OK(art.insert(key4, DefaultCopyInserter{4})); EXPECT_TRUE(art.contains(key1)); EXPECT_TRUE(art.contains(key2)); @@ -168,10 +170,10 @@ void run_put_contains_test() keys.emplace_back(generate_key(rng)); } - ART index; + ARTSet index; { - ART::Scanner scanner{index, ""}; + ARTSet::Scanner scanner{index, ""}; EXPECT_TRUE(scanner.is_done()); } @@ -253,7 +255,7 @@ void run_put_contains_test() { LatencyTimer timer{scanner_latency}; - ART::Scanner scanner{index, lower_bound_key}; + ARTSet::Scanner scanner{index, lower_bound_key}; while (!scanner.is_done() && actual_result.size() < scan_length) { actual_result.emplace_back(scanner.get_key()); @@ -272,7 +274,7 @@ void run_put_contains_test() { LatencyTimer timer{scanner_nosync_latency}; - ART::Scanner scanner{index, lower_bound_key}; + ARTSet::Scanner scanner{index, lower_bound_key}; while (!scanner.is_done() && actual_result.size() < scan_length) { actual_result.emplace_back(scanner.get_key()); @@ -290,8 +292,9 @@ void run_put_contains_test() { auto start_time = std::chrono::steady_clock::now(); - ART::Scanner scanner{index, - lower_bound_key}; + ARTSet::Scanner scanner{ + index, + lower_bound_key}; while (!scanner.is_done() && items_found < scan_length) { ++items_found; @@ -309,8 +312,9 @@ void run_put_contains_test() { auto start_time = std::chrono::steady_clock::now(); - ART::Scanner scanner{index, - lower_bound_key}; + ARTSet::Scanner scanner{ + index, + lower_bound_key}; while (!scanner.is_done() && items_found < scan_length) { ++items_found; @@ -330,7 +334,7 @@ void run_put_contains_test() usize count = 0; { LatencyTimer timer{item_latency, num_keys}; - ART::Scanner scanner{index, std::string_view{}}; + ARTSet::Scanner scanner{index, std::string_view{}}; while (!scanner.is_done()) { ++count; scanner.advance(); @@ -344,7 +348,7 @@ void run_put_contains_test() usize count = 0; { LatencyTimer timer{item_nosync_latency, num_keys}; - ART::Scanner scanner{index, std::string_view{}}; + ARTSet::Scanner scanner{index, std::string_view{}}; while (!scanner.is_done()) { ++count; scanner.advance(); @@ -363,7 +367,7 @@ void run_put_contains_test() << BATT_INSPECT(nokeys_item_latency) << std::endl << BATT_INSPECT(nosync_nokeys_item_latency) << std::endl << BATT_INSPECT(sort_latency) << std::endl - << BATT_INSPECT(ART::default_metrics().bytes_per_insert()) << std::endl; + << BATT_INSPECT(ARTSet::default_metrics().bytes_per_insert()) << std::endl; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -400,7 +404,7 @@ TEST(ArtTest, WideKeySet) } } - ART index; + ARTSet index; for (const std::string& key : keys) { EXPECT_FALSE(index.contains(key)); @@ -432,7 +436,7 @@ TEST(ArtTest, SingleThreadTest) LatencyMetric insert_latency; for (usize trial = 0; trial < 3; ++trial) { - ART index; + ARTSet index; { LatencyTimer timer{insert_latency, num_keys}; for (std::string_view s : keys) { @@ -444,7 +448,7 @@ TEST(ArtTest, SingleThreadTest) } } std::cerr << BATT_INSPECT(insert_latency) << std::endl - << BATT_INSPECT(ART::default_metrics().bytes_per_insert()) << std::endl; + << BATT_INSPECT(ARTSet::default_metrics().bytes_per_insert()) << std::endl; } } @@ -564,21 +568,21 @@ struct TestIntInserter { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -void insert_key(turtle_kv::ART& art, const std::string& key) +void insert_key(ART& art, const std::string& key) { art.insert(key); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -void insert_key(turtle_kv::ART& art, const std::string& key) +void insert_key(ART& art, const std::string& key) { BATT_CHECK_OK(art.insert(key, TestStringViewInserter{.src = key})); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -void insert_key(turtle_kv::ART& art, const std::string& key) +void insert_key(ART& art, const std::string& key) { BATT_CHECK_GE(key.size(), sizeof(usize)); BATT_CHECK_OK(art.insert(key, TestIntInserter{.src = *((const usize*)(key.data()))})); @@ -609,7 +613,7 @@ void run_benchmark_test() for (usize n_threads = 1; n_threads <= std::thread::hardware_concurrency(); ++n_threads) { std::atomic round{-1}; std::atomic pending{0}; - std::atomic*> p_index{nullptr}; + std::atomic*> p_index{nullptr}; std::atomic p_keys{nullptr}; std::atomic n_keys{0}; std::vector threads; @@ -636,7 +640,7 @@ void run_benchmark_test() BATT_CHECK_EQ(r, round.load()); VLOG(1) << "thread " << i << " starting round " << r; - turtle_kv::ART& index = *p_index.load(); + ART& index = *p_index.load(); const std::string* keys = p_keys.load(); const usize n = n_keys.load(); @@ -706,7 +710,7 @@ void run_benchmark_test() // Run the benchmark repeatedly `n_rounds` times. // for (int r = 0; r < n_rounds; ++r) { - turtle_kv::ART index; + ART index; // Set `pending` and `p_index`; the threads will not start working until `round` is updated. // @@ -811,8 +815,7 @@ void run_benchmark_test() } } - std::cerr << BATT_INSPECT(turtle_kv::ART::default_metrics().bytes_per_insert()) - << std::endl; + std::cerr << BATT_INSPECT(ART::default_metrics().bytes_per_insert()) << std::endl; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -861,7 +864,7 @@ TEST(ArtTest, ValuePutGetScan) } } - turtle_kv::ART art; + ART art; const auto check_key_by_index = [&art, &keys](usize query_i, usize expected_i) { ASSERT_TRUE(art.contains(keys[query_i])); diff --git a/src/turtle_kv/util/art_base.hpp b/src/turtle_kv/util/art_base.hpp new file mode 100644 index 0000000..afa6bc4 --- /dev/null +++ b/src/turtle_kv/util/art_base.hpp @@ -0,0 +1,928 @@ +#pragma once +#define TURTLE_KV_UTIL_ART_BASE_HPP + +#include "art_bit_ops.hpp" +#include "art_metrics.hpp" +#include "art_mutex.hpp" +#include "byte_int.hpp" +#include "seq_mutex.hpp" + +#include +#include + +#include +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class ARTBase +{ + public: + static constexpr usize kMaxKeyLen = 64; + + /** \brief Tag type indicating that a new object should not be initialized by the ctor. + */ + struct NoInit { + }; + + using Metrics = ARTMetrics; + + static Metrics& default_metrics() + { + static Metrics m_; + return m_; + } + + enum struct Synchronized { + kFalse = 0, + kTrue = 1, + kDynamic = 2, + }; + + struct Node4; + struct Node16; + struct Node48; + struct Node256; + struct LeafNode; + + enum struct NodeType : u8 { + kLeafNode = 0, + kNode4 = 1, + kNode16 = 2, + kNode48 = 3, + kNode256 = 4, + kNodeBase = 5, + }; + + //----- --- -- - - - - + + static constexpr usize sizeof_value(batt::StaticType) + { + return 0; + } + + template + static constexpr usize sizeof_value(batt::StaticType) + { + return sizeof(ValueT); + } + + //----- --- -- - - - - + + template + static void* uninitialized_value(NodeT* node) + { + return node + 1; + } + + template + static ValueT* mutable_value(NodeT* node, batt::StaticType /**/ = {}) + { + return reinterpret_cast(node + 1); + } + + template + static const ValueT* const_value(const NodeT* node, batt::StaticType /**/ = {}) + { + return reinterpret_cast(node + 1); + } + + //----- --- -- - - - - + + template + static void* construct_value_copy_node(FromNodeT*, ToNodeT* to_node, batt::StaticType) + { + to_node->set_terminal(); + return nullptr; + } + + template + static void* construct_value_copy_addr(FromNodeT*, void*, batt::StaticType) + { + return nullptr; + } + + //----- --- -- - - - - + + template + static ValueT* construct_value_copy_node(FromNodeT* from_node, + ToNodeT* to_node, + batt::StaticType type_of_value) + { + to_node->set_terminal(); + return ARTBase::construct_value_copy_addr(from_node, + ARTBase::uninitialized_value(to_node), + type_of_value); + } + + template + static ValueT* construct_value_copy_addr(FromNodeT* from_node, + void* to_address, + batt::StaticType type_of_value) + { + return new (to_address) ValueT{*ARTBase::const_value(from_node, type_of_value)}; + } + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // Node class hierarchy: + // + // ┌────────┐ + // │NodeBase│ + // └────────┘ + // △ + // ┌───────────────┤ + // │ │ + // ┌───────────────┐ │ + // │GrowableNode│ │ + // └───────────────┘ │ + // △ │ + // ┌─────────┴────────────┐ └────────┐ + // │ │ │ + // ┌──────────────────────┐┌────────────────────┐ │ + // │IndirectIndexedNode││DirectIndexedNode│ │ + // └──────────────────────┘└────────────────────┘ │ + // △ △ │ + // ┌─────┴───────┐ │ │ + // │ │ │ │ + // ┌─────┐ ┌──────┐ ┌──────┐ ┌───────┐ + // │Node4│ │Node16│ │Node48│ │Node256│ + // └─────┘ └──────┘ └──────┘ └───────┘ + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // Node memory layout: + // + // ┌────────────┬──────────┬──────────────┬───────────────┐ + // │ key prefix │ NodeBase │ (impl) ... │ ValueT │ + // └────────────┴──────────┴──────────────┴───────────────┘ + // ◀──────────▶ ◀───────────────────────▶ ◀─────────────▶ + // variable sizeof(NodeT) sizeof(ValueT) + // length + // + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + + struct NodeBase { + using Self = NodeBase; + + //+++++++++++-+-+--+----- --- -- - - - - + + static constexpr u8 kFlagTerminal = 0x80; + static constexpr u8 kFlagObsolete = 0x40; + + //+++++++++++-+-+--+----- --- -- - - - - + + const NodeType node_type; + + u8 flags_; + u8 prefix_len_; + u8 branch_count_; + SeqMutex mutex_; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit NodeBase(NodeType t) noexcept + : node_type{t} + , flags_{0} + , prefix_len_{0} + , branch_count_{0} + { + } + + explicit NodeBase(NodeType t, ARTBase::NoInit) noexcept : node_type{t} + { + } + + NodeBase(const NodeBase&) = delete; + NodeBase& operator=(const NodeBase&) = delete; + + template + void visit(CaseFns&&... case_fns); + + bool is_terminal() const + { + return (this->flags_ & kFlagTerminal) != 0; + } + + void set_terminal() + { + this->flags_ |= kFlagTerminal; + } + + bool is_obsolete() const + { + return (this->flags_ & kFlagObsolete) != 0; + } + + void set_obsolete() + { + this->flags_ |= kFlagObsolete; + } + + void assign_from(const Self& that, usize prefix_offset = 0) + { + this->flags_ = that.flags_; + this->branch_count_ = that.branch_count_; + this->set_prefix(that.prefix() + prefix_offset, that.prefix_len_ - prefix_offset); + } + + const char* prefix() const + { + return (const char*)((((std::uintptr_t)this) - this->prefix_len_) & ~std::uintptr_t{7}); + } + + void set_prefix(const char* data, usize len) + { + this->prefix_len_ = len; + if (len) { + __builtin_memcpy((char*)this->prefix(), data, len); + } + } + }; + + struct LeafNode : NodeBase { + using Self = LeafNode; + using Super = NodeBase; + using NoInit = ARTBase::NoInit; + + explicit LeafNode() noexcept : Super{NodeType::kLeafNode} + { + } + + explicit LeafNode(NoInit no_init) noexcept : Super{NodeType::kLeafNode, no_init} + { + } + + static usize add_branch() + { + BATT_PANIC() << "not supported!"; + return 0; + } + + static void set_branch_index(u8 key_byte [[maybe_unused]], usize index [[maybe_unused]]) + { + BATT_PANIC() << "not supported!"; + } + + static void set_branch_pointer(usize index [[maybe_unused]], NodeBase* child [[maybe_unused]]) + { + BATT_PANIC() << "not supported!"; + } + + static constexpr usize max_branch_count() + { + return 0; + } + + static constexpr usize branch_count() + { + return 0; + } + + static constexpr usize index_of_branch(u8 key_byte [[maybe_unused]]) + { + return 0; + } + + static NodeBase*& get_branch_ref(usize i [[maybe_unused]]) + { + static NodeBase* null_ = nullptr; + return null_; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + struct ScanState { + explicit ScanState(Self&, ByteInt /*min_key*/) noexcept + { + } + + static constexpr ByteInt get_key_byte() + { + return ByteInt::from_char('\0'); + } + + static constexpr NodeBase* get_branch() + { + return nullptr; + } + + static constexpr bool is_done() + { + return true; + } + + static constexpr void advance() + { + } + }; + }; + + struct BranchView { + NodeBase** p_ptr; + NodeBase* ptr; + + //+++++++++++-+-+--+----- --- -- - - - - + + BranchView() noexcept : p_ptr{nullptr}, ptr{nullptr} + { + } + + explicit BranchView(NodeBase*& branch) noexcept : p_ptr{&branch}, ptr{branch} + { + } + + void load(NodeBase*& branch) + { + this->p_ptr = &branch; + this->ptr = branch; + } + + template + NodeT* store(NodeT* new_ptr) + { + static_assert(std::is_base_of_v); + + *this->p_ptr = new_ptr; + this->ptr = new_ptr; + return new_ptr; + } + + NodeBase* reload() + { + this->ptr = *this->p_ptr; + return this->ptr; + } + }; + + static constexpr NodeType node_type_from_branch_count(usize branch_count) + { + if (branch_count == 4) { + return NodeType::kNode4; + } else if (branch_count == 16) { + return NodeType::kNode16; + } else if (branch_count == 48) { + return NodeType::kNode48; + } else { + return NodeType::kNode256; + } + } + + static_assert(sizeof(NodeBase) == 8); + + using BranchIndex = u8; + + static constexpr BranchIndex kInvalidBranchIndex = u8{255}; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + template + struct GrowableNode : NodeBase { + using Self = GrowableNode; + using Super = NodeBase; + using NoInit = ARTBase::NoInit; + + //+++++++++++-+-+--+----- --- -- - - - - + + std::array branches_; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit GrowableNode() noexcept : NodeBase{node_type_from_branch_count(kBranchCount)} + { + } + + explicit GrowableNode(NoInit no_init) noexcept + : NodeBase{node_type_from_branch_count(kBranchCount), no_init} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + Derived* derived() + { + return (Derived*)this; + } + + //----- --- -- - - - - + + usize branch_count() const + { + return this->branch_count_; + } + + usize add_branch() + { + const usize i = this->branch_count_; + ++this->branch_count_; + return i; + } + + NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE + { + return this->branches_[i]; + } + + void set_branch_pointer(usize i, NodeBase* child) BATT_ALWAYS_INLINE + { + this->branches_[i] = child; + } + + static constexpr usize max_branch_count() + { + return kBranchCount; + } + + void assign_from(const Self& that, usize prefix_offset = 0) + { + this->Super::assign_from(static_cast(that), prefix_offset); + __builtin_memcpy(this->branches_.data(), + that.branches_.data(), + this->branch_count() * sizeof(NodeBase*)); + } + }; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + template + struct IndirectIndexedNode : GrowableNode> { + using Self = IndirectIndexedNode; + using Super = GrowableNode; + using NoInit = ARTBase::NoInit; + +#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) \ + if (bit_i == 64) { \ + break; \ + } \ + key_byte = ByteInt::from_i32(key_byte_offset + bit_i); \ + branch = branch_for_byte[key_byte.to_i32()]; \ + if (branch) { \ + this->sorted_branches_[this->branch_count_] = branch; \ + this->sorted_keys_[this->branch_count_] = key_byte; \ + ++this->branch_count_; \ + } \ + bit_i = next_bit(word_val, bit_i) + +#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset); \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) + +#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset); \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) + +#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset); \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) + +#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset); \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) + +#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset); \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) + +#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset) \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset); \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) + +#define TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(word_i, key_byte_offset) \ + word_val = key_bitmap[word_i]; \ + for (;;) { \ + i32 bit_i = first_bit(word_val); \ + TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset); \ + break; \ + } + + struct ScanState { + Self& self_; + usize branch_count_; + usize i_; + std::array sorted_branches_; + std::array sorted_keys_; + + //----- --- -- - - - - + + explicit ScanState(Self& self, ByteInt min_key) noexcept + : self_{self} + , branch_count_{0} + , i_{0} + { + std::array branch_for_byte; + std::array key_bitmap = {0, 0, 0, 0}; + + const usize n_branches = this->self_.branch_count(); + + for (usize i = 0; i < n_branches; ++i) { + const ByteInt key_byte = ByteInt::from_u8(this->self_.key[i]); + if (key_byte < min_key) { + continue; + } + branch_for_byte[key_byte.to_i32()] = this->self_.branches_[i]; + key_bitmap[(key_byte.to_i32() >> 6) & 3] |= (u64{1} << (key_byte.to_i32() & 0x3f)); + } + + u64 word_val; + ByteInt key_byte; + NodeBase* branch; + + TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(0, 0) + TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(1, 64) + TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(2, 128) + TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(3, 192) + } + + ByteInt get_key_byte() const + { + return this->sorted_keys_[this->i_]; + } + + NodeBase* get_branch() const + { + return this->sorted_branches_[this->i_]; + } + + bool is_done() const + { + return this->i_ >= this->branch_count_; + } + + void advance() + { + ++this->i_; + } + }; + +#undef TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP +#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64 +#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32 +#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16 +#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8 +#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4 +#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2 +#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1 + + //+++++++++++-+-+--+----- --- -- - - - - + + std::array key; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit IndirectIndexedNode() noexcept : Super{} + { + } + + explicit IndirectIndexedNode(NoInit no_init) noexcept : Super{no_init} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + usize index_of_branch(u8 key_byte) + { + return index_of(key_byte, this->key); + } + + void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE + { + this->key[i] = key_byte; + } + + void assign_from(const Self& that, usize prefix_offset = 0) + { + this->Super::assign_from(static_cast(that), prefix_offset); + __builtin_memcpy(this->key.data(), that.key.data(), this->branch_count()); + } + }; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + template + struct DirectIndexedNode : GrowableNode> { + using Self = DirectIndexedNode; + using Super = GrowableNode; + using NoInit = ARTBase::NoInit; + + struct ScanState { + Self& self_; + ByteInt key_byte_; + usize branch_i_; + + //----- --- -- - - - - + + explicit ScanState(Self& self, ByteInt min_key) noexcept + : self_{self} + , key_byte_{min_key} + , branch_i_{kInvalidBranchIndex} + { + this->skip_invalid_branches(); + } + + ByteInt get_key_byte() const + { + return this->key_byte_; + } + + NodeBase* get_branch() const + { + return this->self_.branches_[this->branch_i_]; + } + + bool is_done() const + { + return this->key_byte_ >= ByteInt::from_i32(256); + } + + void advance() + { + ++this->key_byte_; + this->skip_invalid_branches(); + } + + void skip_invalid_branches() + { + while (!this->is_done()) { + this->branch_i_ = this->self_.branch_for_key[this->key_byte_.to_i32()]; + if (this->branch_i_ != kInvalidBranchIndex) { + break; + } + ++this->key_byte_; + } + } + }; + + //+++++++++++-+-+--+----- --- -- - - - - + + std::array branch_for_key; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit DirectIndexedNode() noexcept : Super{} + { + this->branch_for_key.fill(kInvalidBranchIndex); + } + + explicit DirectIndexedNode(NoInit no_init) noexcept : Super{no_init} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + usize index_of_branch(u8 key_byte) + { + return this->branch_for_key[key_byte]; + } + + void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE + { + this->branch_for_key[key_byte] = i; + } + + void assign_from(const Self& that, usize prefix_offset = 0) + { + this->Super::assign_from(static_cast(that), prefix_offset); + this->branch_for_key = that.branch_for_key; + } + }; + + struct Node4 : IndirectIndexedNode<4> { + using IndirectIndexedNode<4>::IndirectIndexedNode; + }; + + struct Node16 : IndirectIndexedNode<16> { + using IndirectIndexedNode<16>::IndirectIndexedNode; + }; + + struct Node48 : DirectIndexedNode<48> { + using DirectIndexedNode<48>::DirectIndexedNode; + }; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + struct Node256 : NodeBase { + using Self = Node256; + using Super = NodeBase; + using NoInit = ARTBase::NoInit; + + struct ScanState { + Self& self_; + ByteInt key_byte_; + + //----- --- -- - - - - + + explicit ScanState(Self& self, ByteInt min_key) noexcept : self_{self}, key_byte_{min_key} + { + this->skip_null_branches(); + } + + ByteInt get_key_byte() const + { + return this->key_byte_; + } + + NodeBase* get_branch() const + { + return this->self_.branches_[this->key_byte_.to_i32()]; + } + + bool is_done() const + { + return this->key_byte_ >= ByteInt::from_i32(256); + } + + void advance() + { + ++this->key_byte_; + this->skip_null_branches(); + } + + private: + void skip_null_branches() + { + while (!this->is_done() && this->get_branch() == nullptr) { + ++this->key_byte_; + } + } + }; + + //+++++++++++-+-+--+----- --- -- - - - - + + std::array branches_; + + //+++++++++++-+-+--+----- --- -- - - - - + + Node256() noexcept : Super{NodeType::kNode256} + { + this->branches_.fill(nullptr); + } + + explicit Node256(NoInit no_init) noexcept : Super{NodeType::kNode256, no_init} + { + } + + Node256(const Node256&) = delete; + Node256& operator=(const Node256&) = delete; + + //+++++++++++-+-+--+----- --- -- - - - - + + static constexpr usize branch_count() + { + return 256; + } + + usize add_branch() + { + BATT_PANIC() << "Node256::add_branch is illegal!"; + BATT_UNREACHABLE(); + } + + static constexpr usize max_branch_count() + { + return 256; + } + + usize index_of_branch(u8 key_byte) const + { + return key_byte; + } + + void set_branch_index(u8, usize) + { + } + + NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE + { + return this->branches_[i]; + } + + void assign_from(const Self& that, usize prefix_offset = 0) + { + this->Super::assign_from(static_cast(that), prefix_offset); + this->branches_ = that.branches_; + } + }; + + //----- --- -- - - - - + + static_assert(sizeof(Node4) == 48); + static_assert(sizeof(Node4) % 8 == 0); + static_assert(alignof(Node4) >= 8); + + static_assert(sizeof(Node16) == 152); + static_assert(sizeof(Node16) % 8 == 0); + static_assert(alignof(Node16) >= 8); + + static_assert(sizeof(Node48) == 648); + static_assert(sizeof(Node48) % 8 == 0); + static_assert(alignof(Node48) >= 8); + + static_assert(sizeof(Node256) == 2056); + static_assert(sizeof(Node256) % 8 == 0); + static_assert(alignof(Node256) >= 8); + + static constexpr usize kExtentSize = 64 * kKiB; + static constexpr usize kExtentAlign = 4096; + + using ExtentStorageT = std::aligned_storage_t; + + static_assert(sizeof(ExtentStorageT) == kExtentSize); + + //----- --- -- - - - - + + struct MemoryContext { + ARTBase* art_{nullptr}; + std::vector> thread_extents_; + u8* data_{nullptr}; + usize in_use_{sizeof(ExtentStorageT)}; + + //+++++++++++-+-+--+----- --- -- - - - - + + ~MemoryContext() noexcept + { + if (this->art_) { + ARTMutexLock lock{this->art_->mutex_}; + for (auto& p_ex : this->thread_extents_) { + this->art_->extents_.emplace_back(std::move(p_ex)); + } + } + } + + void* alloc(usize n, ARTBase* art) + { + this->art_ = art; + + const usize in_use_prior = this->in_use_; + if (in_use_prior + n <= kExtentSize) { + this->in_use_ += n; + return this->data_ + in_use_prior; + } + + this->art_->metrics_.byte_alloc_count.add(sizeof(ExtentStorageT)); + + this->thread_extents_.emplace_back(std::make_unique()); + u8* start = reinterpret_cast(this->thread_extents_.back().get()); + this->data_ = start; + this->in_use_ = 0; + + return this->alloc(n, art); + } + }; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit ARTBase() noexcept : ARTBase{ARTBase::default_metrics()} + { + // No update of construct count because we delegate to general-case ctor. + } + + explicit ARTBase(Metrics& metrics) noexcept : metrics_{metrics} + { + this->metrics_.construct_count.add(1); + } + + ~ARTBase() noexcept + { + this->metrics_.destruct_count.add(1); + } + + //+++++++++++-+-+--+----- --- -- - - - - + protected: + /** \brief RAII (guard) class that updates byte_free_count metric at the right moment during + * destruction of the ART (see comment in data member declarations below). + */ + struct ExtentMetricsUpdateGuard { + ARTBase& art_base_; + + //----- --- -- - - - - + + explicit ExtentMetricsUpdateGuard(ARTBase& art_base) noexcept : art_base_{art_base} + { + } + + ~ExtentMetricsUpdateGuard() noexcept + { + this->art_base_.metrics_.byte_free_count.add(this->art_base_.extents_.size() * + sizeof(ExtentStorageT)); + } + }; + + void* alloc_storage(usize n, usize pre) + { + const usize pad = (pre + 7) & ~usize{7}; + char* const ptr = (char*)this->per_thread_memory_context_.get().alloc(n + pad, this); + return ptr + pad; + } + + Metrics& metrics_; + ARTMutex mutex_; + std::vector> extents_; + // + // Must be placed exactly here, so it will be destructed after the ScopedSlot but before extents_. + ExtentMetricsUpdateGuard guard_{*this}; + // + batt::ObjectThreadStorage::ScopedSlot per_thread_memory_context_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_bit_ops.hpp b/src/turtle_kv/util/art_bit_ops.hpp new file mode 100644 index 0000000..c6fe312 --- /dev/null +++ b/src/turtle_kv/util/art_bit_ops.hpp @@ -0,0 +1,51 @@ +#pragma once +#define TURTLE_KV_ART_BIT_OPS_HPP + +#include + +#include +#include + +#include // SSE2 +#include // MMX +#include // SSE3 + +#ifdef __AVX512F__ +#include // AVX512 (AVX, AVX2, FMA) +#endif + +namespace turtle_kv { + +using batt::first_bit; +using batt::next_bit; + +/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns one of: {4, + * 5, 6, 7}. + */ +inline usize index_of(u8 key_byte, const std::array& keys) +{ + __m64 pattern = _mm_set1_pi8((char)key_byte); + u64 extended = *((const u32*)keys.data()); + __m64 values = _mm_cvtsi64_m64(extended); + __m64 result = _m_pcmpeqb(pattern, values); + + return ((__builtin_ffsll((i64)result) - 1) >> 3) & 7; +} + +/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns 31. + */ +inline usize index_of(u8 key_byte, const std::array& keys) +{ + __m128i pattern = _mm_set1_epi8((char)key_byte); + __m128i values = _mm_lddqu_si128((const __m128i*)keys.data()); + +#ifndef __AVX512F__ + int result = _mm_movemask_epi8(_mm_cmpeq_epi8(pattern, values)); +#else + __mmask16 result = _mm_cmpeq_epi8_mask(pattern, values); +#endif + + return (__builtin_ffs(result) - 1) & 31; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_default_inserters.hpp b/src/turtle_kv/util/art_default_inserters.hpp new file mode 100644 index 0000000..a383155 --- /dev/null +++ b/src/turtle_kv/util/art_default_inserters.hpp @@ -0,0 +1,66 @@ +#pragma once +#define TURTLE_KV_UTIL_ART_DEFAULT_INSERTERS_HPP + +#include + +#include + +#include + +namespace turtle_kv { + +template +struct DefaultCopyInserter { + const ValueT& copy_from_; + + explicit DefaultCopyInserter(const ValueT& copy_from) noexcept : copy_from_{copy_from} + { + } + + Status insert_new(void* copy_to) + { + new (copy_to) ValueT{this->copy_from_}; + return OkStatus(); + } + + Status update_existing(ValueT* copy_to) + { + *copy_to = this->copy_from_; + return OkStatus(); + } +}; + +template +struct DefaultMoveInserter { + ValueT&& move_from_; + + explicit DefaultMoveInserter(ValueT&& move_from) noexcept : move_from_{move_from} + { + } + + Status insert_new(void* move_to) + { + new (move_to) ValueT{std::move(this->move_from_)}; + return OkStatus(); + } + + Status update_existing(ValueT* move_to) + { + *move_to = std::move(this->move_from_); + return OkStatus(); + } +}; + +struct DefaultVoidInserter { + BATT_ALWAYS_INLINE Status insert_new(void*) + { + return OkStatus(); + } + + BATT_ALWAYS_INLINE Status update_existing(void*) + { + return OkStatus(); + } +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_metrics.hpp b/src/turtle_kv/util/art_metrics.hpp new file mode 100644 index 0000000..9ac8178 --- /dev/null +++ b/src/turtle_kv/util/art_metrics.hpp @@ -0,0 +1,69 @@ +#pragma once +#define TURTLE_KV_UTIL_ART_METRICS_HPP + +#include +#include + +namespace turtle_kv { + +struct ARTMetrics { + CountMetric construct_count; + CountMetric destruct_count; + FastCountMetric insert_count; + FastCountMetric byte_alloc_count; + FastCountMetric byte_free_count; + + /** \brief Resets all metrics to initial values. + */ + void reset() + { + this->construct_count.reset(); + this->destruct_count.reset(); + this->insert_count.reset(); + this->byte_alloc_count.reset(); + this->byte_free_count.reset(); + } + + //----- --- -- - - - - + + double bytes_per_instance() const + { + return (double)this->byte_alloc_count.get() / (double)this->construct_count.get(); + } + + double average_item_count() const + { + return (double)this->insert_count.get() / (double)this->construct_count.get(); + } + + double bytes_per_insert() const + { + return (double)this->byte_alloc_count.get() / (double)this->insert_count.get(); + } + + /** \brief Returns an estimate of the number of active instances (ART objects). + */ + u64 instance_count() const + { + // Must be in this order! + // + const u64 observed_destruct_count = this->destruct_count.get(); + const u64 observed_construct_count = this->construct_count.get(); + + return observed_construct_count - observed_destruct_count; + } + + /** \brief Returns an estimate of the current number of bytes in use. + */ + u64 bytes_in_use() const + { + // Must be in this order! + // + const u64 observed_free_count = this->byte_free_count.get(); + const u64 observed_alloc_count = this->byte_alloc_count.get(); + + return observed_alloc_count - observed_free_count; + } +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_mutex.hpp b/src/turtle_kv/util/art_mutex.hpp new file mode 100644 index 0000000..2bfe3ae --- /dev/null +++ b/src/turtle_kv/util/art_mutex.hpp @@ -0,0 +1,27 @@ +#pragma once +#define TURTLE_KV_UTIL_ART_MUTEX_HPP + +#define ART_USE_ABSEIL_MUTEX 1 +#define ART_USE_STD_MUTEX 0 + +#if ART_USE_ABSEIL_MUTEX +#include +#endif + +#if ART_USE_STD_MUTEX +#include +#endif + +namespace turtle_kv { + +#if ART_USE_ABSEIL_MUTEX +using ARTMutex = absl::Mutex; +using ARTMutexLock = absl::MutexLock; +#endif + +#if ART_USE_STD_MUTEX +using ARTMutex = std::mutex; +using ARTMutexLock = std::unique_lock; +#endif + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_scanner.hpp b/src/turtle_kv/util/art_scanner.hpp new file mode 100644 index 0000000..c18599a --- /dev/null +++ b/src/turtle_kv/util/art_scanner.hpp @@ -0,0 +1,362 @@ +#pragma once +#define TURTLE_KV_UTIL_ART_SCANNER_HPP + +#include "art_base.hpp" + +#include "detail/scanner_item_storage_base.hpp" +#include "detail/scanner_value_storage_base.hpp" + +#include + +namespace turtle_kv { +namespace detail { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +NodeT& scanner_view_of(usize node_prefix_len, + NodeT* node, + AlignedStorageT* storage, + std::integral_constant, + const Optional&, + void* value_storage_addr, + batt::StaticType type_of_value) +{ + NodeT& node_view = *(new (storage) NodeT{ARTBase::NoInit{}}); + + // Retry the node read until we get a consistent view. + // + for (;;) { + SeqMutex::ReadLock read_lock{node->mutex_}; + node_view.assign_from(*node, /*prefix_offset=*/node_prefix_len); + if (node->is_terminal()) { + ARTBase::construct_value_copy_addr(node, value_storage_addr, type_of_value); + } + if (!read_lock.changed()) { + break; + } + } + + return node_view; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +NodeT& scanner_view_of(usize, + NodeT* node, + AlignedStorageT*, + std::integral_constant, + const Optional& /*sync*/, + const void* /*value_storage_addr*/, + batt::StaticType /*type_of_value*/) +{ + return *node; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +NodeT& scanner_view_of( + usize node_prefix_len, + NodeT* node, + AlignedStorageT* storage, + std::integral_constant, + const Optional& sync, + void* value_storage_addr, + batt::StaticType type_of_value) +{ + if (sync.value_or(true)) { + return scanner_view_of( + node_prefix_len, + node, + storage, + std::integral_constant{}, + sync, + value_storage_addr, + type_of_value); + } + return scanner_view_of( + node_prefix_len, + node, + storage, + std::integral_constant{}, + sync, + value_storage_addr, + type_of_value); +} + +} // namespace detail + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Scanner for an ART. + * + * \tparam ValueT The value type stored in the scanned ART + * \tparam kSynchronized (true, false, dynmamic) The concurrency control for this scanner + * \tparam kValuesOnly When true, the scanner does not build/store key (path) information as it is + * traversing items -- only values are available + */ +template +template +class ART::Scanner + : public detail::ScannerItemStorageBase +{ + public: + using LeafNode = ARTBase::LeafNode; + using Node4 = ARTBase::Node4; + using Node16 = ARTBase::Node16; + using Node48 = ARTBase::Node48; + using Node256 = ARTBase::Node256; + + using NodeScanState = std::variant; + + static constexpr usize kMaxDepth = ART::kMaxKeyLen; + + using SyncType = std::integral_constant; + + using Value = std::conditional_t, + struct get_value_Not_Supported_If_ValueT_Is_Void, + ValueT>; + + //+++++++++++-+-+--+----- --- -- - - - - + + static_assert(sizeof(Node256) > sizeof(Node48)); + static_assert(sizeof(Node256) > sizeof(Node16)); + static_assert(sizeof(Node256) > sizeof(Node4)); + static_assert(sizeof(Node256) > sizeof(LeafNode)); + + struct Frame { + static constexpr usize kStorageSize = + ((kSynchronized == ARTBase::Synchronized::kFalse) ? 1 : sizeof(Node256)); + + std::aligned_storage_t node_storage_; + NodeScanState scan_state_; + usize key_prefix_len_; + std::string_view lower_bound_key_; + ByteInt min_key_byte_; + + explicit Frame(usize key_prefix_len, std::string_view lower_bound_key) noexcept + : scan_state_{None} + , key_prefix_len_{key_prefix_len} + , lower_bound_key_{lower_bound_key} + , min_key_byte_{ByteInt::from_i32(0)} + { + } + }; + + //+++++++++++-+-+--+----- --- -- - - - - + private: + std::aligned_storage_t stack_storage_; + Frame* end_ = reinterpret_cast(&this->stack_storage_); + usize depth_ = 0; + bool have_item_ = false; + ValueT* next_value_ = nullptr; + Optional synchronized_; + + //+++++++++++-+-+--+----- --- -- - - - - + + /** Resets the "have item" state of the scanner; after calling, have item will be false. + */ + void reset_item() BATT_ALWAYS_INLINE + { + this->have_item_ = false; + if (!std::is_same_v) { + this->next_value_ = nullptr; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + public: + explicit Scanner(ART& art, + std::string_view lower_bound_key, + Optional synchronized = None) noexcept + : synchronized_{synchronized} + { + NodeBase* root = nullptr; + for (;;) { + SeqMutex::ReadLock root_read_lock{art.super_root_.mutex_}; + root = art.root_; + if (!root_read_lock.changed()) { + break; + } + } + + if (root) { + root->visit([&](auto* node) { + this->enter(node, /*key_prefix_len=*/0, lower_bound_key); + }); + + if (!this->have_item_) { + this->advance(); + } + } + } + + ~Scanner() noexcept + { + } + + bool is_synchronized() const + { + if (kSynchronized == ARTBase::Synchronized::kFalse) { + return false; + } + if (kSynchronized == ARTBase::Synchronized::kTrue) { + return true; + } + return this->synchronized_.value_or(true); + } + + template + void enter(NodeT* node, usize key_prefix_len, std::string_view lower_bound_key) + { + Frame* top = new (this->end_) Frame{key_prefix_len, lower_bound_key}; + ++this->depth_; + ++this->end_; + + // Node prefix is immutable, so we don't need synchronization. + // + const char* const node_prefix = node->prefix(); + const usize node_prefix_len = node->prefix_len_; + + // We need to create a copy of the node data to protect against data races. + // + NodeT& node_view = + detail::scanner_view_of(node_prefix_len, + node, + &top->node_storage_, + SyncType{}, + this->synchronized_, + this->value_storage_address(node, this->synchronized_), + batt::StaticType{}); + + // Compare the lower bound key to the current node prefix. + // + const usize compare_len = std::min(node_prefix_len, top->lower_bound_key_.size()); + if (compare_len) { + const i32 order = __builtin_memcmp(node_prefix, top->lower_bound_key_.data(), compare_len); + + // If all keys in this subtree come before the lower bound, then there is nothing to do. + // + if (order < 0) { + --this->depth_; + --this->end_; + return; + } + + // If the node prefix is a prefix of the lower bound key, then drop the prefix from the lower + // bound; otherwise the node prefix comes *after* the lower bound, so we can safely ignore the + // lower bound for the rest of the recursion. + // + if (order == 0 && compare_len == node_prefix_len) { + top->lower_bound_key_.remove_prefix(compare_len); + } else { + top->lower_bound_key_ = {}; + } + } + + // Set bounds for branch visitation. + // + top->min_key_byte_ = [&]() -> ByteInt { + if (top->lower_bound_key_.empty()) { + return ByteInt::from_i32(0); + } + const ByteInt next_char = ByteInt::from_char(top->lower_bound_key_.front()); + top->lower_bound_key_.remove_prefix(1); + return next_char; + }(); + + // Append the node prefix to the buffer. + // + if (node_prefix_len) { + this->append_key(top->key_prefix_len_, node_prefix, node_prefix_len); + top->key_prefix_len_ += node_prefix_len; + } + + // If the current node is a key-terminal, emit the contents of the buffer. + // + if (node_view.is_terminal()) { + this->have_item_ = true; + this->set_key_len(top->key_prefix_len_); + if (!std::is_same_v) { + this->next_value_ = (ValueT*)(this->value_storage_address(&node_view, this->synchronized_)); + } + } else { + this->reset_item(); + } + + [[maybe_unused]] auto& scan_state_impl = + top->scan_state_.template emplace(node_view, top->min_key_byte_); + } + + bool is_done() const + { + return this->depth_ == 0; + } + + // get_key() const member function is inherited from ItemStorageBase, if kValuesOnly is false. + + const Value& get_value() const + { + static_assert(!std::is_same_v); + return *this->next_value_; + } + + void advance() + { + this->reset_item(); + + for (;;) { + if (this->depth_ == 0) { + return; + } + + Frame* top = this->end_ - 1; + + batt::case_of( + top->scan_state_, + [](batt::NoneType&) { + BATT_PANIC() << "empty Scanner stack frame!"; + }, + [&](auto& scan_state) + -> std::enable_if_t< + !std::is_same_v, batt::NoneType>> { + //----- --- -- - - - - + if (scan_state.is_done()) { + --this->depth_; + --this->end_; + return; + } + + const ByteInt key_byte = scan_state.get_key_byte(); + NodeBase* const child = scan_state.get_branch(); + + this->append_key_byte(top->key_prefix_len_, key_byte); + + if (key_byte == top->min_key_byte_) { + child->visit([&](auto* child_node) { + this->enter(child_node, top->key_prefix_len_ + 1, top->lower_bound_key_); + }); + } else { + child->visit([&](auto* child_node) { + this->enter(child_node, top->key_prefix_len_ + 1, std::string_view{}); + }); + } + + scan_state.advance(); + }); + + if (this->have_item_) { + return; + } + } + } +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/detail/scanner_item_storage_base.hpp b/src/turtle_kv/util/detail/scanner_item_storage_base.hpp new file mode 100644 index 0000000..1b9d498 --- /dev/null +++ b/src/turtle_kv/util/detail/scanner_item_storage_base.hpp @@ -0,0 +1,75 @@ +#pragma once +#define TURTLE_KV_UTIL_DETAIL_SCANNER_ITEM_STORAGE_BASE_HPP + +#include "scanner_value_storage_base.hpp" + +#include + +namespace turtle_kv { +namespace detail { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Base class for scanner; contains storage for the item at the current scanner position. + */ +template +class ScannerItemStorageBase; + +/** \brief General case (kValuesOnly == false) for scanner item storage. + */ +template +class ScannerItemStorageBase + : public ScannerValueStorageBase +{ + public: + std::array key_buffer_; + usize key_len_ = 0; + + //----- --- -- - - - - + + void append_key(usize prefix_len, const char* suffix_data, usize suffix_len) BATT_ALWAYS_INLINE + { + __builtin_memcpy(this->key_buffer_.data() + prefix_len, suffix_data, suffix_len); + } + + void append_key_byte(usize prefix_len, const ByteInt& suffix_byte) BATT_ALWAYS_INLINE + { + this->key_buffer_[prefix_len] = suffix_byte.to_char(); + } + + void set_key_len(usize len) BATT_ALWAYS_INLINE + { + this->key_len_ = len; + } + + std::string_view get_key() const + { + return std::string_view{this->key_buffer_.data(), this->key_len_}; + } +}; + +/** \brief kValuesOnly == true case; no key-related data members. + */ +template +class ScannerItemStorageBase + : public ScannerValueStorageBase +{ + public: + void append_key(usize, const char*, usize) BATT_ALWAYS_INLINE + { + // nothing to do. + } + + void append_key_byte(usize, const ByteInt&) BATT_ALWAYS_INLINE + { + // nothing to do. + } + + void set_key_len(usize) BATT_ALWAYS_INLINE + { + // nothing to do. + } +}; + +} // namespace detail +} // namespace turtle_kv diff --git a/src/turtle_kv/util/detail/scanner_value_storage_base.hpp b/src/turtle_kv/util/detail/scanner_value_storage_base.hpp new file mode 100644 index 0000000..7f6238f --- /dev/null +++ b/src/turtle_kv/util/detail/scanner_value_storage_base.hpp @@ -0,0 +1,74 @@ +#pragma once +#define TURTLE_KV_UTIL_DETAIL_SCANNER_VALUE_STORAGE_BASE_HPP + +#include + +#include + +namespace turtle_kv { +namespace detail { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +struct ScannerValueStorageBase { + std::aligned_storage_t value_storage_; + + template + std::conditional_t, const void*, void*> value_storage_address( + NodeT* node, + const Optional& sync) + { + if (kSynchronized == ARTBase::Synchronized::kTrue || sync.value_or(true)) { + return &this->value_storage_; + } + return node + 1; + } +}; + +//----- --- -- - - - - + +template +struct ScannerValueStorageBase { + template + const void* value_storage_address(const NodeT* node, const Optional&) const + { + return node + 1; + } +}; + +//----- --- -- - - - - + +template <> +struct ScannerValueStorageBase { + template + void* value_storage_address(const NodeT*, const Optional&) const + { + return nullptr; + } +}; + +//----- --- -- - - - - + +template <> +struct ScannerValueStorageBase { + template + void* value_storage_address(const NodeT*, const Optional&) const + { + return nullptr; + } +}; + +//----- --- -- - - - - + +template <> +struct ScannerValueStorageBase { + template + void* value_storage_address(const NodeT*, const Optional&) const + { + return nullptr; + } +}; + +} // namespace detail +} // namespace turtle_kv From 5a31253e9e82fe0f9607b7b61d5c616cf8248590 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 28 Jun 2026 12:21:15 -0400 Subject: [PATCH 10/20] wip - Remove turtle's ART; use artc's --- src/turtle_kv/kv_store.cpp | 9 +- src/turtle_kv/kv_store_scanner.cpp | 46 +- src/turtle_kv/kv_store_scanner.hpp | 42 +- src/turtle_kv/mem_table/mem_table.hpp | 13 +- .../leaf/packed_blocked_leaf_page.test.cpp | 2 +- src/turtle_kv/util/art.hpp | 210 ---- src/turtle_kv/util/art.ipp | 706 ------------- src/turtle_kv/util/art.test.cpp | 921 ----------------- src/turtle_kv/util/art_base.hpp | 928 ------------------ src/turtle_kv/util/art_bit_ops.hpp | 51 - src/turtle_kv/util/art_default_inserters.hpp | 66 -- src/turtle_kv/util/art_metrics.hpp | 69 -- src/turtle_kv/util/art_mutex.hpp | 27 - src/turtle_kv/util/art_scanner.hpp | 362 ------- .../util/detail/scanner_item_storage_base.hpp | 75 -- .../detail/scanner_value_storage_base.hpp | 74 -- 16 files changed, 58 insertions(+), 3543 deletions(-) delete mode 100644 src/turtle_kv/util/art.hpp delete mode 100644 src/turtle_kv/util/art.ipp delete mode 100644 src/turtle_kv/util/art.test.cpp delete mode 100644 src/turtle_kv/util/art_base.hpp delete mode 100644 src/turtle_kv/util/art_bit_ops.hpp delete mode 100644 src/turtle_kv/util/art_default_inserters.hpp delete mode 100644 src/turtle_kv/util/art_metrics.hpp delete mode 100644 src/turtle_kv/util/art_mutex.hpp delete mode 100644 src/turtle_kv/util/art_scanner.hpp delete mode 100644 src/turtle_kv/util/detail/scanner_item_storage_base.hpp delete mode 100644 src/turtle_kv/util/detail/scanner_value_storage_base.hpp diff --git a/src/turtle_kv/kv_store.cpp b/src/turtle_kv/kv_store.cpp index 6e07922..d2f52fd 100644 --- a/src/turtle_kv/kv_store.cpp +++ b/src/turtle_kv/kv_store.cpp @@ -474,7 +474,7 @@ KVStore::~KVStore() noexcept LOG_IF(INFO, show_metrics) << BATT_INSPECT(merge_compactor.average_bytes_per_compaction()); } { - auto& art = ARTBase::default_metrics(); + auto& art = artc::ARTBase::default_metrics(); LOG_IF(INFO, show_metrics) << BATT_INSPECT(art.byte_alloc_count) << BATT_INSPECT(art.construct_count) << BATT_INSPECT(art.destruct_count) @@ -728,7 +728,6 @@ StatusOr KVStore::put(const KeyView& key, return Status{batt::StatusCode::kUnavailable}; } - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // Status KVStore::put(const KeyView& key, const ValueView& value) noexcept /*override*/ @@ -1351,10 +1350,10 @@ using CheckpointEvent = llfs::PackedVariant; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Status KVStore::sync(Optional upper_bound, Optional write_options) noexcept +Status KVStore::sync(Optional upper_bound, + Optional write_options) noexcept { - EditOffset target = - upper_bound ? *upper_bound : this->change_log_writer_->next_edit_offset(); + EditOffset target = upper_bound ? *upper_bound : this->change_log_writer_->next_edit_offset(); bool urgent = write_options && write_options->urgent_sync ? true : false; diff --git a/src/turtle_kv/kv_store_scanner.cpp b/src/turtle_kv/kv_store_scanner.cpp index 4736ff5..4dcc8f4 100644 --- a/src/turtle_kv/kv_store_scanner.cpp +++ b/src/turtle_kv/kv_store_scanner.cpp @@ -11,17 +11,17 @@ namespace { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -KeyView art_scanner_get_key(ART::Scanner& scanner) +template +KeyView art_scanner_get_key(artc::ART::Scanner& scanner) { return scanner.get_key(); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -KeyView art_scanner_get_key(ART::Scanner& scanner) +template +KeyView art_scanner_get_key(artc::ART::Scanner& scanner) { return scanner.get_value().key_view(); } @@ -138,12 +138,12 @@ Status KVStoreScanner::start() // Delta case : single ART index for keys and values // - auto& art_scanner = - *(new (p_mem) ART::Scanner{ - delta_mem_table.art_index(), - this->min_key_, - }); + auto& art_scanner = *(new ( + p_mem) artc::ART::Scanner{ + delta_mem_table.art_index(), + this->min_key_, + }); ++p_mem; if (!art_scanner.is_done()) { @@ -525,10 +525,10 @@ Status KVStoreScanner::set_next_item() // /*explicit*/ KVStoreScanner::ScanLevel::ScanLevel( ActiveMemTableValueTag, - ART::Scanner& - art_scanner) noexcept + artc::ART::Scanner& art_scanner) noexcept : key{art_scanner_get_key(art_scanner)} - , state_impl{MemTableValueScanState{ + , state_impl{MemTableValueScanState{ .art_scanner_ = &art_scanner, }} { @@ -538,10 +538,10 @@ Status KVStoreScanner::set_next_item() // /*explicit*/ KVStoreScanner::ScanLevel::ScanLevel( DeltaMemTableValueTag, - ART::Scanner& - art_scanner) noexcept + artc::ART::Scanner& art_scanner) noexcept : key{art_scanner_get_key(art_scanner)} - , state_impl{MemTableValueScanState{ + , state_impl{MemTableValueScanState{ .art_scanner_ = &art_scanner, }} { @@ -566,11 +566,11 @@ EditView KVStoreScanner::ScanLevel::item() const BATT_PANIC() << "illegal state"; BATT_UNREACHABLE(); }, - [](const MemTableValueScanState& state) -> EditView { + [](const MemTableValueScanState& state) -> EditView { const MemTableValueEntry& entry = state.art_scanner_->get_value(); return EditView{entry.key_view(), entry.value_view()}; }, - [](const MemTableValueScanState& state) -> EditView { + [](const MemTableValueScanState& state) -> EditView { const MemTableValueEntry& entry = state.art_scanner_->get_value(); return EditView{entry.key_view(), entry.value_view()}; }, @@ -598,10 +598,10 @@ ValueView KVStoreScanner::ScanLevel::value() const BATT_PANIC() << "illegal state"; BATT_UNREACHABLE(); }, - [](const MemTableValueScanState& state) -> ValueView { + [](const MemTableValueScanState& state) -> ValueView { return state.art_scanner_->get_value().value_view(); }, - [](const MemTableValueScanState& state) -> ValueView { + [](const MemTableValueScanState& state) -> ValueView { return state.art_scanner_->get_value().value_view(); }, [](const Slice& state) -> ValueView { @@ -660,10 +660,10 @@ bool KVStoreScanner::ScanLevel::advance() BATT_PANIC() << "illegal state"; BATT_UNREACHABLE(); }, - [this](MemTableValueScanState& state) -> bool { + [this](MemTableValueScanState& state) -> bool { return scan_level_mem_table_advance_impl(this, state); }, - [this](MemTableValueScanState& state) -> bool { + [this](MemTableValueScanState& state) -> bool { return scan_level_mem_table_advance_impl(this, state); }, [this](Slice& state) -> bool { diff --git a/src/turtle_kv/kv_store_scanner.hpp b/src/turtle_kv/kv_store_scanner.hpp index e5ad4c6..dbb1471 100644 --- a/src/turtle_kv/kv_store_scanner.hpp +++ b/src/turtle_kv/kv_store_scanner.hpp @@ -26,9 +26,10 @@ #include #include -#include #include +#include + #include #include @@ -102,17 +103,17 @@ class KVStoreScanner //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // - template + template struct MemTableScanState { MemTable* mem_table_; - ART::Scanner* art_scanner_; + artc::ART::Scanner* art_scanner_; }; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // - template + template struct MemTableValueScanState { - ART::Scanner* art_scanner_; + artc::ART::Scanner* art_scanner_; }; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -163,8 +164,8 @@ class KVStoreScanner KeyView key; std::variant, - MemTableValueScanState, + MemTableValueScanState, + MemTableValueScanState, Slice, TreeLevelScanState, TreeLevelScanShardedState, @@ -181,13 +182,13 @@ class KVStoreScanner explicit ScanLevel( ActiveMemTableValueTag, - ART::Scanner& - art_scanner) noexcept; + artc::ART::Scanner& art_scanner) noexcept; explicit ScanLevel( DeltaMemTableValueTag, - ART::Scanner& - art_scanner) noexcept; + artc::ART::Scanner& art_scanner) noexcept; explicit ScanLevel(const Slice& edit_view_slice) noexcept; @@ -327,12 +328,14 @@ class KVStoreScanner //+++++++++++-+-+--+----- --- -- - - - - using DeltaMemTableScannerStorage = std::aligned_storage_t< - /*size=*/std::max(sizeof(ART::Scanner), - sizeof(ART::Scanner)), - /*align=*/std::max(alignof(ART::Scanner), - alignof(ART::Scanner))>; + /*size=*/std::max( + sizeof(artc::ART::Scanner), + sizeof(artc::ART::Scanner)), + /*align=*/std::max( + alignof(artc::ART::Scanner), + alignof(artc::ART::Scanner))>; batt::Toggle::Reader state_reader_; llfs::PageLoader& page_loader_; @@ -344,8 +347,9 @@ class KVStoreScanner bool needs_resume_; Optional next_item_; Status status_; - Optional::Scanner> mem_table_scanner_; - Optional::Scanner> + Optional::Scanner> mem_table_scanner_; + Optional::Scanner> mem_table_value_scanner_; std::array static_delta_storage_; DeltaMemTableScannerStorage* delta_storage_; diff --git a/src/turtle_kv/mem_table/mem_table.hpp b/src/turtle_kv/mem_table/mem_table.hpp index ccf03bb..ace3433 100644 --- a/src/turtle_kv/mem_table/mem_table.hpp +++ b/src/turtle_kv/mem_table/mem_table.hpp @@ -29,7 +29,6 @@ #include #include -#include #include #include @@ -39,6 +38,8 @@ #include #include +#include + #include #include @@ -216,7 +217,7 @@ class BasicMemTable : public MemTableBase //+++++++++++-+-+--+----- --- -- - - - - - ART& art_index() + artc::ART& art_index() { return this->art_index_; } @@ -338,11 +339,11 @@ class BasicMemTable : public MemTableBase // Diagnostic metrics for `this->art_index_`. // - ARTBase::Metrics art_metrics_; + artc::ARTBase::Metrics art_metrics_; // In-memory index used for scans and point queries. // - ART art_index_; + artc::ART art_index_; // Tracks the maximum observed key-value pair size (in bytes); this is used to estimate the // worst-case space wasted in a future batch. @@ -559,8 +560,8 @@ class BasicMemTable::BatchCompactor public: using Self = BatchCompactor; - using ARTScanner = - ART::Scanner; + using ARTScanner = artc::ART::Scanner; //+++++++++++-+-+--+----- --- -- - - - - diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index 03c926e..c3805f4 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -267,7 +267,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) // Test ShardedLiveRanges. // { - for (usize j = 0; j < 10000; ++j) { + for (usize j = 0; j < 1000; ++j) { // Drop up to 64 sub-ranges of the leaf. // for (usize drop_count = 0; drop_count < 64; ++drop_count) { diff --git a/src/turtle_kv/util/art.hpp b/src/turtle_kv/util/art.hpp deleted file mode 100644 index 460ecdd..0000000 --- a/src/turtle_kv/util/art.hpp +++ /dev/null @@ -1,210 +0,0 @@ -#pragma once - -#include "art_base.hpp" -#include "art_bit_ops.hpp" -#include "art_default_inserters.hpp" - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -template -class ART : public ARTBase -{ - public: - using Self = ART; - using Super = ARTBase; - - using value_type = ValueT; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr usize kValueStorageSize = Super::sizeof_value(batt::StaticType{}); - - //+++++++++++-+-+--+----- --- -- - - - - - - template - class Scanner; - - //+++++++++++-+-+--+----- --- -- - - - - - - ART() noexcept - { - } - - explicit ART(ARTBase::Metrics& metrics) noexcept : ARTBase{metrics} - { - } - - ~ART() noexcept - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - template - Status insert(std::string_view key, InserterT&& inserter); - - BATT_ALWAYS_INLINE void insert(std::string_view key) - { - static_assert(std::is_same_v); - - this->insert(key, DefaultVoidInserter{}).IgnoreError(); - } - - bool contains(std::string_view key); - - const ValueT* unsynchronized_find(std::string_view key); - - Optional find(std::string_view key); - - template - void scan(std::string_view lower_bound_key, const Fn& fn); - - /** \brief Returns true iff the container is empty. - */ - bool empty() - { - BranchView branch; - for (;;) { - SeqMutex::ReadLock root_read_lock{this->super_root_.mutex_}; - branch.load(this->root_); - if (!root_read_lock.changed()) { - break; - } - } - return branch.ptr == nullptr; - } - - //+++++++++++-+-+--+----- --- -- - - - - - private: - using SmallestParentNode = Node4; - - //+++++++++++-+-+--+----- --- -- - - - - - - template >> - NodeBase* add_child(NodeT* node, u8 key_byte, NodeBase* child); - - NodeBase* add_child(Node256* node, u8 key_byte, NodeBase* child); - - template - LeafNode* add_child_leaf(NodeT* node, u8 key_byte, const char* new_key_data, usize new_key_len); - - LeafNode* make_leaf_node(const char* prefix, usize prefix_len); - - Node4* make_parent_node(const char* prefix, usize prefix_len); - - Node4* grow_node(LeafNode* old_node); - - Node16* grow_node(Node4* old_node); - - Node48* grow_node(Node16* old_node); - - Node256* grow_node(Node48* old_node); - - Node256* grow_node(Node256*); - - LeafNode* clone_node(LeafNode* orig_node, usize prefix_offset); - - Node4* clone_node(Node4* orig_node, usize prefix_offset); - - Node16* clone_node(Node16* orig_node, usize prefix_offset); - - Node48* clone_node(Node48* orig_node, usize prefix_offset); - - Node256* clone_node(Node256* orig_node, usize prefix_offset); - - template - void find_impl(std::string_view key, batt::StaticType, NodeCallbackFn&& node_callback); - - //+++++++++++-+-+--+----- --- -- - - - - - - NodeBase super_root_{NodeType::kNodeBase}; - NodeBase* root_ = nullptr; -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline void ARTBase::NodeBase::visit(CaseFns&&... case_fns) -{ - auto visitor = batt::make_case_of_visitor(BATT_FORWARD(case_fns)...); - - const NodeType observed = this->node_type; - - switch (observed) { - case NodeType::kLeafNode: - visitor(static_cast(this)); - break; - - case NodeType::kNode4: - visitor(static_cast(this)); - break; - - case NodeType::kNode16: - visitor(static_cast(this)); - break; - - case NodeType::kNode48: - visitor(static_cast(this)); - break; - - case NodeType::kNode256: - visitor(static_cast(this)); - break; - - case NodeType::kNodeBase: // fall-through - default: - BATT_PANIC() << "Bad node type: " << (int)observed; - BATT_UNREACHABLE(); - } -} - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline void ART::scan(std::string_view lower_bound_key, const Fn& fn) -{ - Scanner scanner{*this, lower_bound_key}; - - while (!scanner.is_done()) { - if (!fn(scanner.get_key())) { - return; - } - scanner.advance(); - } -} - -} // namespace turtle_kv - -#include -#include diff --git a/src/turtle_kv/util/art.ipp b/src/turtle_kv/util/art.ipp deleted file mode 100644 index e9276bb..0000000 --- a/src/turtle_kv/util/art.ipp +++ /dev/null @@ -1,706 +0,0 @@ -#pragma once - -#include - -namespace turtle_kv { - -namespace detail { - -inline usize find_common_prefix_len(const char* data0, usize size0, const char* data1, usize size1) -{ - usize n = 0; - usize common_size = std::min(size0, size1); - while (common_size && *data0 == *data1) { - ++n; - --common_size; - ++data0; - ++data1; - } - return n; -} - -} // namespace detail - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline Status ART::insert(std::string_view key, InserterT&& inserter) -{ - bool reset = true; - - const char* key_data = nullptr; - usize key_len = 0; - BranchView branch; - NodeBase* parent = nullptr; - - this->metrics_.insert_count.add(1); - - for (;;) { - if (reset) { - reset = false; - - key_data = key.data(); - key_len = key.size(); - - for (;;) { - SeqMutex::ReadLock root_read_lock{this->super_root_.mutex_}; - branch.load(this->root_); - if (!root_read_lock.changed()) { - break; - } - } - - if (branch.ptr == nullptr) { - SeqMutex::WriteLock root_write_lock{this->super_root_.mutex_}; - if (branch.reload() == nullptr) { - LeafNode* new_node = branch.store(this->make_leaf_node(key_data, key_len)); - Status status = inserter.insert_new(ARTBase::uninitialized_value(new_node)); - if (status.ok()) { - new_node->set_terminal(); - } - return status; - } - } - - parent = &this->super_root_; - } - - Status status = OkStatus(); - bool done = false; - - branch.ptr->visit([&](auto* node) { - SeqMutex::ReadLock node_read_lock{node->mutex_}; - - const char* const node_prefix = node->prefix(); - const usize node_prefix_len = node->prefix_len_; - const bool node_is_terminal = node->is_terminal(); - - if (node_read_lock.changed()) { - reset = true; - return; - } - - const usize common_len = - detail::find_common_prefix_len(node_prefix, node_prefix_len, key_data, key_len); - - // If the key matches the prefix of the current node only partially, then we must split the - // prefix and insert a new parent node. - // - if (common_len != node_prefix_len) { - //----- --- -- - - - - - SeqMutex::WriteLock parent_write_lock{parent->mutex_}; - if (parent->is_obsolete()) { - reset = true; - return; - } - - //----- --- -- - - - - - SeqMutex::WriteLock node_write_lock{node->mutex_}; - - if (node != branch.reload() || node_prefix != node->prefix() || - node_prefix_len != node->prefix_len_ || node->is_obsolete()) { - reset = true; - return; - } - - SmallestParentNode* new_parent = this->make_parent_node(node_prefix, common_len); - auto* new_node = this->clone_node(node, /*prefix_offset=*/(common_len + 1)); - - this->add_child(new_parent, /*key_byte=*/node_prefix[common_len], new_node); - - if (common_len < key_len) { - LeafNode* new_child_leaf = this->add_child_leaf(new_parent, - /*key_byte=*/key_data[common_len], - key_data + (common_len + 1), - key_len - (common_len + 1)); - status = inserter.insert_new(ARTBase::uninitialized_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - - } else { - if (new_parent->is_terminal()) { - status = inserter.update_existing(ARTBase::mutable_value(new_parent)); - } else { - status = inserter.insert_new(ARTBase::uninitialized_value(new_parent)); - if (status.ok()) { - new_parent->set_terminal(); - } - } - } - - node->set_obsolete(); - branch.store(new_parent); - done = true; - return; - } - - // If the common prefix is exactly the length of this node's prefix, then the search key is - // fully consumed exactly at this node; just update the node and we are done! - // - if (key_len == common_len) { - if (!node_is_terminal || !std::is_same_v) { - //----- --- -- - - - - - SeqMutex::WriteLock node_write_lock{node->mutex_}; - - // Now that we are holding the lock, check if anything has changed, and retry if so. - // - if (node->is_obsolete() || node_prefix != node->prefix() || - node_prefix_len != node->prefix_len_) { - reset = true; - return; - } - - // If the node was already terminal, then update (in the case of ValueT=void, this is a - // no-op; we would have short-circuited at the immediately enclosing conditional), - // otherwise mark the node as storing a value, and call InserterT::insert_new. - // - if (!node_is_terminal) { - status = inserter.insert_new(uninitialized_value(node)); - if (status.ok()) { - node->set_terminal(); - } - } else { - status = inserter.update_existing(ARTBase::mutable_value(node)); - } - } - done = true; - return; - } - - const u8 key_byte = key_data[common_len]; - const char* const new_key_data = key_data + (common_len + 1); - const usize new_key_len = key_len - (common_len + 1); - - const usize observed_branch_count = std::min(node->branch_count(), node->max_branch_count()); - BranchView next; - { - const usize i = node->index_of_branch(key_byte); - if (i < observed_branch_count) { - next.load(node->get_branch_ref(i)); - } - } - - if (node_read_lock.changed()) { - reset = true; - return; - } - - // The node has not changed, and there *is* a branch for the current key byte; step into that - // subtree and continue in the outer loop. - // - if (next.p_ptr != nullptr && next.ptr != nullptr) { - parent = node; - branch = next; - key_data = new_key_data; - key_len = new_key_len; - return; - } - - // There is no branch on the current node for the current key byte. - // - Optional::WriteLock> parent_write_lock; - Optional::WriteLock> node_write_lock; - - // If we can't add a branch because the observed branch count is the maximum, then grow the - // node. This requires a lock on the current node and its parent. - // - if (next.p_ptr == nullptr && observed_branch_count == node->max_branch_count()) { - //----- --- -- - - - - - parent_write_lock.emplace(parent->mutex_); - if (parent->is_obsolete()) { - reset = true; - return; - } - - //----- --- -- - - - - - node_write_lock.emplace(node->mutex_); - if (node->is_obsolete()) { - reset = true; - return; - } - - BATT_CHECK_EQ(observed_branch_count, node->branch_count()); - - auto* new_node = this->grow_node(node); - LeafNode* new_child_leaf = - this->add_child_leaf(new_node, key_byte, new_key_data, new_key_len); - status = inserter.insert_new(ARTBase::mutable_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - - node->set_obsolete(); - branch.store(new_node); - - } else { - //----- --- -- - - - - - // It looks like there *may* be room for an extra branch on the current node, so try to add - // one. This requires only a lock on the current node. - // - node_write_lock.emplace(node->mutex_); - if (node->is_obsolete()) { - reset = true; - return; - } - - if (next.p_ptr == nullptr) { - if (observed_branch_count != node->branch_count()) { - reset = true; - return; - } - LeafNode* new_child_leaf = - this->add_child_leaf(node, key_byte, new_key_data, new_key_len); - status = inserter.insert_new(uninitialized_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - - } else { - if (next.reload() != nullptr) { - reset = true; - return; - } - LeafNode* new_child_leaf = this->make_leaf_node(new_key_data, new_key_len); - next.store(new_child_leaf); - status = inserter.insert_new(uninitialized_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - } - } - - done = true; - }); - - if (done) { - return status; - } - } -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -void ART::find_impl(std::string_view key, - batt::StaticType, - NodeCallbackFn&& node_callback) -{ - bool reset = true; - - const char* key_data = nullptr; - usize key_len = 0; - BranchView branch; - - for (;;) { - if (reset) { - reset = false; - - key_data = key.data(); - key_len = key.size(); - - for (;;) { - NodeLockT root_read_lock{this->super_root_.mutex_}; - branch.load(this->root_); - if (!root_read_lock.changed()) { - break; - } - } - if (branch.ptr == nullptr) { - return; // find_impl - } - } - - bool done = false; - - branch.ptr->visit([&](auto* node) { - NodeLockT node_read_lock{node->mutex_}; - - const char* const node_prefix = node->prefix(); - const usize node_prefix_len = node->prefix_len_; - const bool node_is_terminal = node->is_terminal(); - - if (node_read_lock.changed()) { - reset = true; - return; // visit - } - - const usize common_len = - detail::find_common_prefix_len(node_prefix, node_prefix_len, key_data, key_len); - - // Mismatch in the middle of the prefix means the branch we would have taken isn't there. Not - // found. - // - if (common_len != node_prefix_len) { - done = true; - return; // visit - } - - // If the search key is fully consumed, we are done; set `result` if the node is marked as - // terminal. - // - if (common_len == key_len) { - if (node_is_terminal) { - node_callback(node); - if (std::is_same_v || !node_read_lock.changed()) { - done = true; - } else { - node_callback((decltype(node))nullptr); - reset = true; - } - } else { - done = true; - } - return; // visit - } - - const u8 key_byte = key_data[common_len]; - - const usize observed_branch_count = std::min(node->branch_count(), node->max_branch_count()); - BranchView next; - { - const usize i = node->index_of_branch(key_byte); - if (i < observed_branch_count) { - next.load(node->get_branch_ref(i)); - } - } - - if (node_read_lock.changed()) { - reset = true; - return; // visit - } - - if (next.p_ptr == nullptr || next.ptr == nullptr) { - done = true; - return; // visit - } - - key_data += (common_len + 1); - key_len -= (common_len + 1); - branch = next; - }); - - if (done) { - return; // find_impl - } - } - BATT_UNREACHABLE(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline bool ART::contains(std::string_view key) -{ - bool found = false; - - this->find_impl(key, batt::StaticType::ReadLock>{}, [&found](auto* node) { - found = (node != nullptr); - }); - - return found; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::unsynchronized_find(std::string_view key) -> const ValueT* -{ - const ValueT* p_value = nullptr; - - this->find_impl(key, batt::StaticType::NullLock>{}, [&p_value](auto* node) { - if (node == nullptr) { - p_value = nullptr; - } else { - p_value = ARTBase::const_value(node); - } - }); - - return p_value; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::find(std::string_view key) -> Optional -{ - Optional value_copy; - - this->find_impl(key, batt::StaticType::ReadLock>{}, [&value_copy](auto* node) { - if (node == nullptr) { - value_copy = None; - } else { - value_copy.emplace(*ARTBase::const_value(node)); - } - }); - - return value_copy; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline auto ART::add_child(NodeT* node, u8 key_byte, NodeBase* child) -> NodeBase* -{ - const usize i = node->add_branch(); - node->set_branch_index(key_byte, i); - node->set_branch_pointer(i, child); - return child; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::add_child(Node256* node, u8 key_byte, NodeBase* child) -> NodeBase* -{ - const usize i = key_byte; - BATT_CHECK_EQ(node->branches_[i], nullptr); - node->branches_[i] = child; - return child; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline auto ART::add_child_leaf(NodeT* node, - u8 key_byte, - const char* new_key_data, - usize new_key_len) -> LeafNode* -{ - LeafNode* new_child = this->make_leaf_node(new_key_data, new_key_len); - this->add_child(node, key_byte, new_child); - return new_child; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::make_leaf_node(const char* prefix, usize prefix_len) -> LeafNode* -{ - LeafNode* new_node = - new (this->alloc_storage(sizeof(LeafNode) + kValueStorageSize, prefix_len)) LeafNode{}; - - new_node->set_prefix(prefix, prefix_len); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::make_parent_node(const char* prefix, usize prefix_len) -> Node4* -{ - Node4* new_node = - new (this->alloc_storage(sizeof(Node4) + kValueStorageSize, prefix_len)) Node4{}; - - new_node->set_prefix(prefix, prefix_len); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(LeafNode* old_node) -> Node4* -{ - Node4* new_node = - new (this->alloc_storage(sizeof(Node4) + kValueStorageSize, old_node->prefix_len_)) Node4{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(Node4* old_node) -> Node16* -{ - Node16* new_node = - new (this->alloc_storage(sizeof(Node16) + kValueStorageSize, old_node->prefix_len_)) Node16{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - new_node->branch_count_ = old_node->branch_count_; - - std::copy(old_node->key.begin(), old_node->key.end(), new_node->key.begin()); - std::copy(old_node->branches_.begin(), old_node->branches_.end(), new_node->branches_.begin()); - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(Node16* old_node) -> Node48* -{ - Node48* new_node = - new (this->alloc_storage(sizeof(Node48) + kValueStorageSize, old_node->prefix_len_)) Node48{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - new_node->branch_count_ = old_node->branch_count_; - - for (usize i = 0; i < new_node->branch_count_; ++i) { - new_node->branch_for_key[old_node->key[i]] = i; - new_node->branches_[i] = old_node->branches_[i]; - } - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -auto ART::grow_node(Node48* old_node) -> Node256* -{ - Node256* new_node = - new (this->alloc_storage(sizeof(Node256) + kValueStorageSize, old_node->prefix_len_)) - Node256{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - - for (usize key_byte = 0; key_byte < 256; ++key_byte) { - const BranchIndex i = old_node->branch_for_key[key_byte]; - new_node->branches_[key_byte] = (i == kInvalidBranchIndex) ? nullptr : old_node->branches_[i]; - } - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(Node256*) -> Node256* -{ - BATT_PANIC() << "Node256 can not grow larger!"; - BATT_UNREACHABLE(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(LeafNode* orig_node, usize prefix_offset) -> LeafNode* -{ - LeafNode* new_node = new (this->alloc_storage(sizeof(LeafNode) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) - LeafNode{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node4* orig_node, usize prefix_offset) -> Node4* -{ - Node4* new_node = - new (this->alloc_storage(sizeof(Node4) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) Node4{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node16* orig_node, usize prefix_offset) -> Node16* -{ - Node16* new_node = - new (this->alloc_storage(sizeof(Node16) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) Node16{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node48* orig_node, usize prefix_offset) -> Node48* -{ - Node48* new_node = - new (this->alloc_storage(sizeof(Node48) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) Node48{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node256* orig_node, usize prefix_offset) -> Node256* -{ - Node256* new_node = new (this->alloc_storage(sizeof(Node256) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) - Node256{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art.test.cpp b/src/turtle_kv/util/art.test.cpp deleted file mode 100644 index 64bdfd8..0000000 --- a/src/turtle_kv/util/art.test.cpp +++ /dev/null @@ -1,921 +0,0 @@ -#include -// -#include - -#include -#include - -#include - -#include - -#include - -#include - -#include -#include -#include - -namespace { - -using namespace batt::int_types; - -using turtle_kv::ART; -using turtle_kv::ARTBase; -using turtle_kv::ByteInt; -using turtle_kv::DefaultCopyInserter; -using turtle_kv::LatencyMetric; -using turtle_kv::LatencyTimer; -using turtle_kv::None; -using turtle_kv::OkStatus; -using turtle_kv::Optional; -using turtle_kv::Status; -using turtle_kv::testing::RandomStringGenerator; - -using ARTSet = ART; - -struct BigUInt64KeyGenerator { - template - std::string operator()(Rng& rng) const - { - std::uniform_int_distribution pick_n{u64{0}, ~u64{0}}; - - std::string s; - s.resize(sizeof(llfs::big_u64)); - *((llfs::big_u64*)s.data()) = pick_n(rng); - - // depth == 1 will be Node256, but not full. - // - if ((s[0] & 7) == 0) { - s[0] += 1; - } - - // depth == 2 will max out at Node48. - // - s[1] &= 31; - - // depth == 3 will max out at Node16. - // - s[2] &= 15; - - // depth == 4 will max out at Node16, with space. - // - s[3] &= 7; - - // depth == 5 will max out at Node4. - // - s[4] &= 3; - - // depth == 6 will max out at Node4, with space. - // - s[5] &= 1; - - return s; - } -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, ByteInt) -{ - char a, b, c; - - a = 255; - b = 1; - c = 160; - - EXPECT_EQ(ByteInt::from_char(a).to_i32(), 255); - EXPECT_EQ(ByteInt::from_char(b).to_i32(), 1); - EXPECT_EQ(ByteInt::from_char(c).to_i32(), 160); - EXPECT_EQ(ByteInt::from_i32(-1).to_i32(), -1); - EXPECT_LT(ByteInt::from_i32(-1), ByteInt::from_char(a)); - EXPECT_LT(ByteInt::from_i32(-1), ByteInt::from_char(b)); - EXPECT_LT(ByteInt::from_i32(-1), ByteInt::from_char(c)); - - ByteInt i = ByteInt::from_i32(255); - - EXPECT_EQ(i, ByteInt::from_char(a)); - - ++i; - - EXPECT_EQ(i.to_i32(), 256); - EXPECT_EQ(i.to_char(), 0); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, OverlappingKeyPrefix) -{ - const std::string key1 = "app"; - const std::string key2 = "apple"; - const std::string key3 = "application"; - const std::string key4 = "applesauce"; - - ART art; - - EXPECT_FALSE(art.contains(key1)); - EXPECT_FALSE(art.contains(key2)); - EXPECT_FALSE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key1, DefaultCopyInserter{1})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_FALSE(art.contains(key2)); - EXPECT_FALSE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key2, DefaultCopyInserter{2})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_TRUE(art.contains(key2)); - EXPECT_FALSE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key3, DefaultCopyInserter{3})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_TRUE(art.contains(key2)); - EXPECT_TRUE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key4, DefaultCopyInserter{4})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_TRUE(art.contains(key2)); - EXPECT_TRUE(art.contains(key3)); - EXPECT_TRUE(art.contains(key4)); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -void run_put_contains_test() -{ - ARTBase::default_metrics().reset(); - - const usize num_keys = 1e5; - const usize num_scans = 10000; - const usize max_scan_length = 100; - - std::default_random_engine rng{/*seed=*/1}; - KeyGeneratorT generate_key; - std::uniform_int_distribution pick_scan_length{1, max_scan_length}; - - std::vector keys; - std::unordered_set inserted; - - for (usize i = 0; i < num_keys; ++i) { - keys.emplace_back(generate_key(rng)); - } - - ARTSet index; - - { - ARTSet::Scanner scanner{index, ""}; - EXPECT_TRUE(scanner.is_done()); - } - - usize i = 0; - for (const std::string& key : keys) { - if (inserted.count(key)) { - continue; - } - - inserted.emplace(key); - - EXPECT_FALSE(index.contains(key)); - - index.insert(key); - - EXPECT_TRUE(index.contains(key)) << BATT_INSPECT(i) << BATT_INSPECT_STR(key); - - ++i; - } - - for (const std::string& key : keys) { - EXPECT_TRUE(index.contains(key)); - } - - LatencyMetric sort_latency; - - std::vector sorted_keys; - for (const std::string& key : keys) { - sorted_keys.emplace_back(key); - } - { - LatencyTimer timer{sort_latency, sorted_keys.size()}; - std::sort(sorted_keys.begin(), sorted_keys.end()); - } - - LatencyMetric scan_latency; - LatencyMetric scanner_latency; - LatencyMetric scanner_nosync_latency; - LatencyMetric nokeys_scan_latency; - LatencyMetric nokeys_item_latency; - LatencyMetric nosync_nokeys_scan_latency; - LatencyMetric nosync_nokeys_item_latency; - - for (usize i = 0; i < num_scans; ++i) { - const std::string lower_bound_key = generate_key(rng); - const usize scan_length = pick_scan_length(rng); - - std::vector expected_result; - for (auto iter = std::lower_bound(sorted_keys.begin(), sorted_keys.end(), lower_bound_key); - iter != sorted_keys.end() && expected_result.size() < scan_length; - ++iter) { - expected_result.emplace_back(*iter); - } - - for (usize j = 0; j < 3; ++j) { - { - std::vector actual_result; - { - LatencyTimer timer{scan_latency}; - index.scan(lower_bound_key, [&actual_result, scan_length](const std::string_view& key) { - actual_result.emplace_back(key); - return actual_result.size() < scan_length; - }); - } - - EXPECT_EQ(expected_result.size(), actual_result.size()); - - BATT_CHECK_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << std::endl - << BATT_INSPECT_RANGE_PRETTY(expected_result) - << BATT_INSPECT_RANGE_PRETTY(actual_result); - - ASSERT_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << BATT_INSPECT(i); - } - - { - std::vector actual_result; - { - LatencyTimer timer{scanner_latency}; - - ARTSet::Scanner scanner{index, lower_bound_key}; - - while (!scanner.is_done() && actual_result.size() < scan_length) { - actual_result.emplace_back(scanner.get_key()); - scanner.advance(); - } - } - - EXPECT_EQ(expected_result.size(), actual_result.size()); - - ASSERT_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << BATT_INSPECT(i); - } - - { - std::vector actual_result; - { - LatencyTimer timer{scanner_nosync_latency}; - - ARTSet::Scanner scanner{index, lower_bound_key}; - - while (!scanner.is_done() && actual_result.size() < scan_length) { - actual_result.emplace_back(scanner.get_key()); - scanner.advance(); - } - } - - EXPECT_EQ(expected_result.size(), actual_result.size()); - - ASSERT_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << BATT_INSPECT(i); - } - { - usize items_found = 0; - { - auto start_time = std::chrono::steady_clock::now(); - - ARTSet::Scanner scanner{ - index, - lower_bound_key}; - - while (!scanner.is_done() && items_found < scan_length) { - ++items_found; - scanner.advance(); - } - - nokeys_scan_latency.update(start_time, 1); - nokeys_item_latency.update(start_time, items_found); - } - EXPECT_EQ(expected_result.size(), items_found); - } - - { - usize items_found = 0; - { - auto start_time = std::chrono::steady_clock::now(); - - ARTSet::Scanner scanner{ - index, - lower_bound_key}; - - while (!scanner.is_done() && items_found < scan_length) { - ++items_found; - scanner.advance(); - } - - nosync_nokeys_scan_latency.update(start_time, 1); - nosync_nokeys_item_latency.update(start_time, items_found); - } - EXPECT_EQ(expected_result.size(), items_found); - } - } - } - - LatencyMetric item_latency; - { - usize count = 0; - { - LatencyTimer timer{item_latency, num_keys}; - ARTSet::Scanner scanner{index, std::string_view{}}; - while (!scanner.is_done()) { - ++count; - scanner.advance(); - } - } - EXPECT_EQ(count, num_keys); - } - - LatencyMetric item_nosync_latency; - { - usize count = 0; - { - LatencyTimer timer{item_nosync_latency, num_keys}; - ARTSet::Scanner scanner{index, std::string_view{}}; - while (!scanner.is_done()) { - ++count; - scanner.advance(); - } - } - EXPECT_EQ(count, num_keys); - } - - std::cerr << BATT_INSPECT(scan_latency) << std::endl - << BATT_INSPECT(scanner_latency) << std::endl - << BATT_INSPECT(scanner_nosync_latency) << std::endl - << BATT_INSPECT(nokeys_scan_latency) << std::endl - << BATT_INSPECT(nosync_nokeys_scan_latency) << std::endl - << BATT_INSPECT(item_latency) << std::endl - << BATT_INSPECT(item_nosync_latency) << std::endl - << BATT_INSPECT(nokeys_item_latency) << std::endl - << BATT_INSPECT(nosync_nokeys_item_latency) << std::endl - << BATT_INSPECT(sort_latency) << std::endl - << BATT_INSPECT(ARTSet::default_metrics().bytes_per_insert()) << std::endl; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, PutContainsTest_Key24) -{ - run_put_contains_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, PutContainsTest_Key8) -{ - char a[2] = {0, 0}; - a[0] -= 1; - char b[2] = {1, 0}; - - BATT_CHECK_LT(std::string_view{b}, std::string_view{a}); - - run_put_contains_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, WideKeySet) -{ - ARTBase::default_metrics().reset(); - - std::vector keys; - for (u16 first = 0; first < 256; ++first) { - for (u16 second = 0; second < 256; ++second) { - char data[2] = {(char)first, (char)second}; - keys.emplace_back(data, 2); - } - } - - ARTSet index; - - for (const std::string& key : keys) { - EXPECT_FALSE(index.contains(key)); - index.insert(key); - EXPECT_TRUE(index.contains(key)); - } - - for (const std::string& key : keys) { - EXPECT_TRUE(index.contains(key)); - } -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, SingleThreadTest) -{ - ARTBase::default_metrics().reset(); - - for (const usize num_keys : {1e5, 1e6, 1e7}) { - std::vector keys; - { - std::default_random_engine rng{/*seed=*/1}; - RandomStringGenerator generate_key; - - for (usize i = 0; i < num_keys; ++i) { - keys.emplace_back(generate_key(rng)); - } - } - - LatencyMetric insert_latency; - for (usize trial = 0; trial < 3; ++trial) { - ARTSet index; - { - LatencyTimer timer{insert_latency, num_keys}; - for (std::string_view s : keys) { - index.insert(s); - } - } - for (const std::string& key : keys) { - ASSERT_TRUE(index.contains(key)); - } - } - std::cerr << BATT_INSPECT(insert_latency) << std::endl - << BATT_INSPECT(ARTSet::default_metrics().bytes_per_insert()) << std::endl; - } -} - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- - -usize slow_index_of(u8 key_byte, const std::array& keys) -{ - for (usize i = 0; i < keys.size(); ++i) { - if (keys[i] == key_byte) { - return i; - } - } - return 7; -} - -usize slow_index_of(u8 key_byte, const std::array& keys) -{ - for (usize i = 0; i < keys.size(); ++i) { - if (keys[i] == key_byte) { - return i; - } - } - return 31; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, SimdSearch) -{ - using turtle_kv::index_of; - - std::uniform_int_distribution pick_byte{0, 255}; - - // array case - { - // hand-written test cases - { - std::array keys = {2, 0, 3, 1}; - - EXPECT_EQ(index_of(0, keys), 1); - EXPECT_EQ(index_of(1, keys), 3); - EXPECT_EQ(index_of(2, keys), 0); - EXPECT_EQ(index_of(3, keys), 2); - EXPECT_EQ(index_of(4, keys) & 4, 4); - EXPECT_EQ(index_of(50, keys) & 4, 4); - EXPECT_EQ(index_of(99, keys) & 4, 4); - } - - // randomly generated cases - // - std::default_random_engine rng{/*seed=*/1}; - - for (usize i = 0; i < 1000; ++i) { - std::array keys; - keys.fill(0); - for (u8& k : keys) { - k = pick_byte(rng); - } - for (u16 p = 0; p < 256; ++p) { - EXPECT_EQ(index_of((u8)p, keys) & 4, slow_index_of((u8)p, keys) & 4); - } - } - } - - // array case - { - std::default_random_engine rng{/*seed=*/1}; - - for (usize i = 0; i < 100000; ++i) { - std::array keys; - keys.fill(0); - for (u8& k : keys) { - k = pick_byte(rng); - } - for (u16 p = 0; p < 256; ++p) { - EXPECT_EQ(index_of((u8)p, keys), slow_index_of((u8)p, keys)); - } - } - } -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -struct TestStringViewInserter { - const std::string_view& src; - - Status insert_new(void* dst) const - { - new (dst) std::string_view{this->src}; - return OkStatus(); - } - - Status update_existing(std::string_view* dst) const - { - *dst = this->src; - return OkStatus(); - } -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -struct TestIntInserter { - usize src; - - Status insert_new(void* dst) const - { - *((usize*)dst) = this->src; - return OkStatus(); - } - - Status update_existing(usize* dst) const - { - *dst = this->src; - return OkStatus(); - } -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void insert_key(ART& art, const std::string& key) -{ - art.insert(key); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void insert_key(ART& art, const std::string& key) -{ - BATT_CHECK_OK(art.insert(key, TestStringViewInserter{.src = key})); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void insert_key(ART& art, const std::string& key) -{ - BATT_CHECK_GE(key.size(), sizeof(usize)); - BATT_CHECK_OK(art.insert(key, TestIntInserter{.src = *((const usize*)(key.data()))})); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -void run_benchmark_test() -{ - ARTBase::default_metrics().reset(); - - const int n_rounds = 3; - const int n_stages_per_round = 2; // insert and query - - const std::array data_set_sizes = { - 300 * 1000, - 200 * 1000, - 100 * 1000, - }; - - std::cerr << "threads"; - for (usize n_items : data_set_sizes) { - std::cerr << ",puts/sec (N=" << n_items << "),get/sec (N=" << n_items << ")"; - } - std::cerr << std::endl; - - for (usize n_threads = 1; n_threads <= std::thread::hardware_concurrency(); ++n_threads) { - std::atomic round{-1}; - std::atomic pending{0}; - std::atomic*> p_index{nullptr}; - std::atomic p_keys{nullptr}; - std::atomic n_keys{0}; - std::vector threads; - std::vector op_count(n_threads); - std::vector>> stop_round(n_threads); - - for (auto& b : stop_round) { - b->store(false); - } - - std::cerr << n_threads; - - for (usize i = 0; i < n_threads; ++i) { - threads.emplace_back([&, i] { - std::default_random_engine rng{std::random_device{}()}; - - const int n_loops = n_rounds * (int)data_set_sizes.size() * n_stages_per_round; - - for (int r = 0; r < n_loops; ++r) { - VLOG(1) << "thread " << i << " waiting for round " << r; - while (round.load() < r) { - continue; - } - BATT_CHECK_EQ(r, round.load()); - - VLOG(1) << "thread " << i << " starting round " << r; - ART& index = *p_index.load(); - const std::string* keys = p_keys.load(); - const usize n = n_keys.load(); - - if ((r % 2) == 0) { - for (usize j = i; j < n; j += n_threads) { - insert_key(index, keys[j]); - } - } else { - std::atomic& stop = *stop_round[i]; - std::uniform_int_distribution pick_i{0, n - 1}; - i64 count = 0; - while (stop.load() == false) { - const usize k = pick_i(rng); - ASSERT_TRUE(index.contains(keys[k])); - ++count; - } - op_count[i] = count; - } - - pending.fetch_sub(1); - pending.notify_one(); - VLOG(1) << "thread " << i << " finished round " << r; - } - }); - } - - auto on_scope_exit = batt::finally([&] { - for (std::thread& t : threads) { - t.join(); - } - std::cerr << std::endl; - }); - - usize size_i = 0; - for (const usize num_keys : data_set_sizes) { - //----- --- -- - - - - - // Generate random key set. - // - std::vector keys; - { - std::default_random_engine rng{/*seed=*/1}; - RandomStringGenerator generate_key; - - for (usize i = 0; i < num_keys; ++i) { - keys.emplace_back(generate_key(rng)); - } - } - - //----- --- -- - - - - - // Initialize the thread-shared state. - // - n_keys.store(num_keys); - p_keys.store(keys.data()); - auto on_scope_exit2 = batt::finally([&] { - n_keys.store(0); - p_keys.store(nullptr); - }); - - //----- --- -- - - - - - // Create a metric for each latency we want to measure. - // - LatencyMetric insert_latency; - LatencyMetric st_query_latency; - LatencyMetric mt_query_latency; - - //----- --- -- - - - - - // Run the benchmark repeatedly `n_rounds` times. - // - for (int r = 0; r < n_rounds; ++r) { - ART index; - - // Set `pending` and `p_index`; the threads will not start working until `round` is updated. - // - pending.store(n_threads); - p_index.store(&index); - auto on_scope_exit3 = batt::finally([&] { - p_index.store(nullptr); - }); - - // Stage 0: inserts - { - LatencyTimer timer{insert_latency, num_keys}; - - const int next_round = (r + size_i * n_rounds) * 2; - - //----- --- -- - - - - - // Ready, set, go! - // - round.store(next_round); - //----- --- -- - - - - - - for (;;) { - const auto observed = pending.load(); - if (observed > 0) { - pending.wait(observed); - continue; - } - break; - } - } - - pending.store(n_threads); - for (auto& b : stop_round) { - b->store(false); - } - - // Stage 1: queries - { - const auto start_time = std::chrono::steady_clock::now(); - - const int next_round = (r + size_i * n_rounds) * 2 + 1; - - //----- --- -- - - - - - // Ready, set, go! - // - round.store(next_round); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - for (auto& b : stop_round) { - b->store(true); - } - //----- --- -- - - - - - - for (;;) { - const auto observed = pending.load(); - if (observed > 0) { - pending.wait(observed); - continue; - } - break; - } - - const auto end_time = std::chrono::steady_clock::now(); - - const double elapsed_nanos = - std::chrono::duration_cast(end_time - start_time).count(); - - double total_count = 0; - for (double c : op_count) { - total_count += c; - } - BATT_CHECK_GT(total_count, 0); - BATT_CHECK_GT(elapsed_nanos, 0); - - mt_query_latency.update_nanos(elapsed_nanos, total_count); - } - - //----- --- -- - - - - - // After multi-threaded part of this round is done. - // - usize found_count = 0; - auto start_time = std::chrono::steady_clock::now(); - for (const std::string& key : keys) { - ASSERT_TRUE(index.contains(key)) << BATT_INSPECT_STR(key) << BATT_INSPECT(found_count); - ++found_count; - } - st_query_latency.update(start_time, found_count); - ASSERT_EQ(found_count, keys.size()); - } - std::cerr << "," << insert_latency.rate_per_second(false) << "," - << mt_query_latency.rate_per_second(false); - ++size_i; - } - - if (n_threads >= 4) { - n_threads += 1; - if (n_threads >= 8) { - n_threads += 2; - if (n_threads >= 16) { - n_threads += 4; - } - } - } - } - - std::cerr << BATT_INSPECT(ART::default_metrics().bytes_per_insert()) << std::endl; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, MultiThreadTest_Void) -{ - run_benchmark_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, MultiThreadTest_StringView) -{ - run_benchmark_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, MultiThreadTest_Int) -{ - run_benchmark_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, ValuePutGetScan) -{ - ARTBase::default_metrics().reset(); - - const usize n_keys = 100 * 1000; - std::default_random_engine rng{std::random_device{}()}; - RandomStringGenerator generate_key; - - std::vector keys; - keys.resize(n_keys); - - std::unordered_set unique_keys; - - for (std::string& k : keys) { - for (;;) { - k = generate_key(rng); - if (!unique_keys.count(k)) { - unique_keys.emplace(k); - break; - } - } - } - - ART art; - - const auto check_key_by_index = [&art, &keys](usize query_i, usize expected_i) { - ASSERT_TRUE(art.contains(keys[query_i])); - - const std::string_view* p_value = art.unsynchronized_find(keys[query_i]); - ASSERT_NE(p_value, nullptr); - ASSERT_THAT(*p_value, ::testing::StrEq(keys[expected_i])); - - Optional value_copy = art.find(keys[query_i]); - ASSERT_TRUE(value_copy); - ASSERT_THAT(*value_copy, ::testing::StrEq(keys[expected_i])); - }; - - for (usize i = 0; i < n_keys; ++i) { - Status status = art.insert(keys[i], - TestStringViewInserter{ - .src = keys[i / 2], - }); - - ASSERT_TRUE(status.ok()) << BATT_INSPECT(status); - - for (usize j = i - std::min(i, 25); j <= i; ++j) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(j, j / 2)); - } - } - - // Check all values once more to make sure nothing that was inserted earlier was messed up by a - // later insertion or update. - // - for (usize i = 0; i < n_keys; ++i) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(i, i / 2)); - } - - // Now update all keys and verify them. - // - for (usize i = 0; i < n_keys; ++i) { - Status status = art.insert(keys[i], - TestStringViewInserter{ - .src = keys[i], - }); - - ASSERT_TRUE(status.ok()) << BATT_INSPECT(status); - - for (usize j = i - std::min(i, 25); j <= i; ++j) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(j, j)); - } - } - - for (usize i = 0; i < n_keys; ++i) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(i, i)); - } -} - -} // namespace diff --git a/src/turtle_kv/util/art_base.hpp b/src/turtle_kv/util/art_base.hpp deleted file mode 100644 index afa6bc4..0000000 --- a/src/turtle_kv/util/art_base.hpp +++ /dev/null @@ -1,928 +0,0 @@ -#pragma once -#define TURTLE_KV_UTIL_ART_BASE_HPP - -#include "art_bit_ops.hpp" -#include "art_metrics.hpp" -#include "art_mutex.hpp" -#include "byte_int.hpp" -#include "seq_mutex.hpp" - -#include -#include - -#include -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -class ARTBase -{ - public: - static constexpr usize kMaxKeyLen = 64; - - /** \brief Tag type indicating that a new object should not be initialized by the ctor. - */ - struct NoInit { - }; - - using Metrics = ARTMetrics; - - static Metrics& default_metrics() - { - static Metrics m_; - return m_; - } - - enum struct Synchronized { - kFalse = 0, - kTrue = 1, - kDynamic = 2, - }; - - struct Node4; - struct Node16; - struct Node48; - struct Node256; - struct LeafNode; - - enum struct NodeType : u8 { - kLeafNode = 0, - kNode4 = 1, - kNode16 = 2, - kNode48 = 3, - kNode256 = 4, - kNodeBase = 5, - }; - - //----- --- -- - - - - - - static constexpr usize sizeof_value(batt::StaticType) - { - return 0; - } - - template - static constexpr usize sizeof_value(batt::StaticType) - { - return sizeof(ValueT); - } - - //----- --- -- - - - - - - template - static void* uninitialized_value(NodeT* node) - { - return node + 1; - } - - template - static ValueT* mutable_value(NodeT* node, batt::StaticType /**/ = {}) - { - return reinterpret_cast(node + 1); - } - - template - static const ValueT* const_value(const NodeT* node, batt::StaticType /**/ = {}) - { - return reinterpret_cast(node + 1); - } - - //----- --- -- - - - - - - template - static void* construct_value_copy_node(FromNodeT*, ToNodeT* to_node, batt::StaticType) - { - to_node->set_terminal(); - return nullptr; - } - - template - static void* construct_value_copy_addr(FromNodeT*, void*, batt::StaticType) - { - return nullptr; - } - - //----- --- -- - - - - - - template - static ValueT* construct_value_copy_node(FromNodeT* from_node, - ToNodeT* to_node, - batt::StaticType type_of_value) - { - to_node->set_terminal(); - return ARTBase::construct_value_copy_addr(from_node, - ARTBase::uninitialized_value(to_node), - type_of_value); - } - - template - static ValueT* construct_value_copy_addr(FromNodeT* from_node, - void* to_address, - batt::StaticType type_of_value) - { - return new (to_address) ValueT{*ARTBase::const_value(from_node, type_of_value)}; - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // Node class hierarchy: - // - // ┌────────┐ - // │NodeBase│ - // └────────┘ - // △ - // ┌───────────────┤ - // │ │ - // ┌───────────────┐ │ - // │GrowableNode│ │ - // └───────────────┘ │ - // △ │ - // ┌─────────┴────────────┐ └────────┐ - // │ │ │ - // ┌──────────────────────┐┌────────────────────┐ │ - // │IndirectIndexedNode││DirectIndexedNode│ │ - // └──────────────────────┘└────────────────────┘ │ - // △ △ │ - // ┌─────┴───────┐ │ │ - // │ │ │ │ - // ┌─────┐ ┌──────┐ ┌──────┐ ┌───────┐ - // │Node4│ │Node16│ │Node48│ │Node256│ - // └─────┘ └──────┘ └──────┘ └───────┘ - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // Node memory layout: - // - // ┌────────────┬──────────┬──────────────┬───────────────┐ - // │ key prefix │ NodeBase │ (impl) ... │ ValueT │ - // └────────────┴──────────┴──────────────┴───────────────┘ - // ◀──────────▶ ◀───────────────────────▶ ◀─────────────▶ - // variable sizeof(NodeT) sizeof(ValueT) - // length - // - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - - struct NodeBase { - using Self = NodeBase; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr u8 kFlagTerminal = 0x80; - static constexpr u8 kFlagObsolete = 0x40; - - //+++++++++++-+-+--+----- --- -- - - - - - - const NodeType node_type; - - u8 flags_; - u8 prefix_len_; - u8 branch_count_; - SeqMutex mutex_; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit NodeBase(NodeType t) noexcept - : node_type{t} - , flags_{0} - , prefix_len_{0} - , branch_count_{0} - { - } - - explicit NodeBase(NodeType t, ARTBase::NoInit) noexcept : node_type{t} - { - } - - NodeBase(const NodeBase&) = delete; - NodeBase& operator=(const NodeBase&) = delete; - - template - void visit(CaseFns&&... case_fns); - - bool is_terminal() const - { - return (this->flags_ & kFlagTerminal) != 0; - } - - void set_terminal() - { - this->flags_ |= kFlagTerminal; - } - - bool is_obsolete() const - { - return (this->flags_ & kFlagObsolete) != 0; - } - - void set_obsolete() - { - this->flags_ |= kFlagObsolete; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->flags_ = that.flags_; - this->branch_count_ = that.branch_count_; - this->set_prefix(that.prefix() + prefix_offset, that.prefix_len_ - prefix_offset); - } - - const char* prefix() const - { - return (const char*)((((std::uintptr_t)this) - this->prefix_len_) & ~std::uintptr_t{7}); - } - - void set_prefix(const char* data, usize len) - { - this->prefix_len_ = len; - if (len) { - __builtin_memcpy((char*)this->prefix(), data, len); - } - } - }; - - struct LeafNode : NodeBase { - using Self = LeafNode; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - explicit LeafNode() noexcept : Super{NodeType::kLeafNode} - { - } - - explicit LeafNode(NoInit no_init) noexcept : Super{NodeType::kLeafNode, no_init} - { - } - - static usize add_branch() - { - BATT_PANIC() << "not supported!"; - return 0; - } - - static void set_branch_index(u8 key_byte [[maybe_unused]], usize index [[maybe_unused]]) - { - BATT_PANIC() << "not supported!"; - } - - static void set_branch_pointer(usize index [[maybe_unused]], NodeBase* child [[maybe_unused]]) - { - BATT_PANIC() << "not supported!"; - } - - static constexpr usize max_branch_count() - { - return 0; - } - - static constexpr usize branch_count() - { - return 0; - } - - static constexpr usize index_of_branch(u8 key_byte [[maybe_unused]]) - { - return 0; - } - - static NodeBase*& get_branch_ref(usize i [[maybe_unused]]) - { - static NodeBase* null_ = nullptr; - return null_; - } - - //+++++++++++-+-+--+----- --- -- - - - - - - struct ScanState { - explicit ScanState(Self&, ByteInt /*min_key*/) noexcept - { - } - - static constexpr ByteInt get_key_byte() - { - return ByteInt::from_char('\0'); - } - - static constexpr NodeBase* get_branch() - { - return nullptr; - } - - static constexpr bool is_done() - { - return true; - } - - static constexpr void advance() - { - } - }; - }; - - struct BranchView { - NodeBase** p_ptr; - NodeBase* ptr; - - //+++++++++++-+-+--+----- --- -- - - - - - - BranchView() noexcept : p_ptr{nullptr}, ptr{nullptr} - { - } - - explicit BranchView(NodeBase*& branch) noexcept : p_ptr{&branch}, ptr{branch} - { - } - - void load(NodeBase*& branch) - { - this->p_ptr = &branch; - this->ptr = branch; - } - - template - NodeT* store(NodeT* new_ptr) - { - static_assert(std::is_base_of_v); - - *this->p_ptr = new_ptr; - this->ptr = new_ptr; - return new_ptr; - } - - NodeBase* reload() - { - this->ptr = *this->p_ptr; - return this->ptr; - } - }; - - static constexpr NodeType node_type_from_branch_count(usize branch_count) - { - if (branch_count == 4) { - return NodeType::kNode4; - } else if (branch_count == 16) { - return NodeType::kNode16; - } else if (branch_count == 48) { - return NodeType::kNode48; - } else { - return NodeType::kNode256; - } - } - - static_assert(sizeof(NodeBase) == 8); - - using BranchIndex = u8; - - static constexpr BranchIndex kInvalidBranchIndex = u8{255}; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct GrowableNode : NodeBase { - using Self = GrowableNode; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branches_; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit GrowableNode() noexcept : NodeBase{node_type_from_branch_count(kBranchCount)} - { - } - - explicit GrowableNode(NoInit no_init) noexcept - : NodeBase{node_type_from_branch_count(kBranchCount), no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - Derived* derived() - { - return (Derived*)this; - } - - //----- --- -- - - - - - - usize branch_count() const - { - return this->branch_count_; - } - - usize add_branch() - { - const usize i = this->branch_count_; - ++this->branch_count_; - return i; - } - - NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE - { - return this->branches_[i]; - } - - void set_branch_pointer(usize i, NodeBase* child) BATT_ALWAYS_INLINE - { - this->branches_[i] = child; - } - - static constexpr usize max_branch_count() - { - return kBranchCount; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - __builtin_memcpy(this->branches_.data(), - that.branches_.data(), - this->branch_count() * sizeof(NodeBase*)); - } - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct IndirectIndexedNode : GrowableNode> { - using Self = IndirectIndexedNode; - using Super = GrowableNode; - using NoInit = ARTBase::NoInit; - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) \ - if (bit_i == 64) { \ - break; \ - } \ - key_byte = ByteInt::from_i32(key_byte_offset + bit_i); \ - branch = branch_for_byte[key_byte.to_i32()]; \ - if (branch) { \ - this->sorted_branches_[this->branch_count_] = branch; \ - this->sorted_keys_[this->branch_count_] = key_byte; \ - ++this->branch_count_; \ - } \ - bit_i = next_bit(word_val, bit_i) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(word_i, key_byte_offset) \ - word_val = key_bitmap[word_i]; \ - for (;;) { \ - i32 bit_i = first_bit(word_val); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset); \ - break; \ - } - - struct ScanState { - Self& self_; - usize branch_count_; - usize i_; - std::array sorted_branches_; - std::array sorted_keys_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept - : self_{self} - , branch_count_{0} - , i_{0} - { - std::array branch_for_byte; - std::array key_bitmap = {0, 0, 0, 0}; - - const usize n_branches = this->self_.branch_count(); - - for (usize i = 0; i < n_branches; ++i) { - const ByteInt key_byte = ByteInt::from_u8(this->self_.key[i]); - if (key_byte < min_key) { - continue; - } - branch_for_byte[key_byte.to_i32()] = this->self_.branches_[i]; - key_bitmap[(key_byte.to_i32() >> 6) & 3] |= (u64{1} << (key_byte.to_i32() & 0x3f)); - } - - u64 word_val; - ByteInt key_byte; - NodeBase* branch; - - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(0, 0) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(1, 64) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(2, 128) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(3, 192) - } - - ByteInt get_key_byte() const - { - return this->sorted_keys_[this->i_]; - } - - NodeBase* get_branch() const - { - return this->sorted_branches_[this->i_]; - } - - bool is_done() const - { - return this->i_ >= this->branch_count_; - } - - void advance() - { - ++this->i_; - } - }; - -#undef TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1 - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array key; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit IndirectIndexedNode() noexcept : Super{} - { - } - - explicit IndirectIndexedNode(NoInit no_init) noexcept : Super{no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - usize index_of_branch(u8 key_byte) - { - return index_of(key_byte, this->key); - } - - void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE - { - this->key[i] = key_byte; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - __builtin_memcpy(this->key.data(), that.key.data(), this->branch_count()); - } - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct DirectIndexedNode : GrowableNode> { - using Self = DirectIndexedNode; - using Super = GrowableNode; - using NoInit = ARTBase::NoInit; - - struct ScanState { - Self& self_; - ByteInt key_byte_; - usize branch_i_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept - : self_{self} - , key_byte_{min_key} - , branch_i_{kInvalidBranchIndex} - { - this->skip_invalid_branches(); - } - - ByteInt get_key_byte() const - { - return this->key_byte_; - } - - NodeBase* get_branch() const - { - return this->self_.branches_[this->branch_i_]; - } - - bool is_done() const - { - return this->key_byte_ >= ByteInt::from_i32(256); - } - - void advance() - { - ++this->key_byte_; - this->skip_invalid_branches(); - } - - void skip_invalid_branches() - { - while (!this->is_done()) { - this->branch_i_ = this->self_.branch_for_key[this->key_byte_.to_i32()]; - if (this->branch_i_ != kInvalidBranchIndex) { - break; - } - ++this->key_byte_; - } - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branch_for_key; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit DirectIndexedNode() noexcept : Super{} - { - this->branch_for_key.fill(kInvalidBranchIndex); - } - - explicit DirectIndexedNode(NoInit no_init) noexcept : Super{no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - usize index_of_branch(u8 key_byte) - { - return this->branch_for_key[key_byte]; - } - - void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE - { - this->branch_for_key[key_byte] = i; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - this->branch_for_key = that.branch_for_key; - } - }; - - struct Node4 : IndirectIndexedNode<4> { - using IndirectIndexedNode<4>::IndirectIndexedNode; - }; - - struct Node16 : IndirectIndexedNode<16> { - using IndirectIndexedNode<16>::IndirectIndexedNode; - }; - - struct Node48 : DirectIndexedNode<48> { - using DirectIndexedNode<48>::DirectIndexedNode; - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - struct Node256 : NodeBase { - using Self = Node256; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - struct ScanState { - Self& self_; - ByteInt key_byte_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept : self_{self}, key_byte_{min_key} - { - this->skip_null_branches(); - } - - ByteInt get_key_byte() const - { - return this->key_byte_; - } - - NodeBase* get_branch() const - { - return this->self_.branches_[this->key_byte_.to_i32()]; - } - - bool is_done() const - { - return this->key_byte_ >= ByteInt::from_i32(256); - } - - void advance() - { - ++this->key_byte_; - this->skip_null_branches(); - } - - private: - void skip_null_branches() - { - while (!this->is_done() && this->get_branch() == nullptr) { - ++this->key_byte_; - } - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branches_; - - //+++++++++++-+-+--+----- --- -- - - - - - - Node256() noexcept : Super{NodeType::kNode256} - { - this->branches_.fill(nullptr); - } - - explicit Node256(NoInit no_init) noexcept : Super{NodeType::kNode256, no_init} - { - } - - Node256(const Node256&) = delete; - Node256& operator=(const Node256&) = delete; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr usize branch_count() - { - return 256; - } - - usize add_branch() - { - BATT_PANIC() << "Node256::add_branch is illegal!"; - BATT_UNREACHABLE(); - } - - static constexpr usize max_branch_count() - { - return 256; - } - - usize index_of_branch(u8 key_byte) const - { - return key_byte; - } - - void set_branch_index(u8, usize) - { - } - - NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE - { - return this->branches_[i]; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - this->branches_ = that.branches_; - } - }; - - //----- --- -- - - - - - - static_assert(sizeof(Node4) == 48); - static_assert(sizeof(Node4) % 8 == 0); - static_assert(alignof(Node4) >= 8); - - static_assert(sizeof(Node16) == 152); - static_assert(sizeof(Node16) % 8 == 0); - static_assert(alignof(Node16) >= 8); - - static_assert(sizeof(Node48) == 648); - static_assert(sizeof(Node48) % 8 == 0); - static_assert(alignof(Node48) >= 8); - - static_assert(sizeof(Node256) == 2056); - static_assert(sizeof(Node256) % 8 == 0); - static_assert(alignof(Node256) >= 8); - - static constexpr usize kExtentSize = 64 * kKiB; - static constexpr usize kExtentAlign = 4096; - - using ExtentStorageT = std::aligned_storage_t; - - static_assert(sizeof(ExtentStorageT) == kExtentSize); - - //----- --- -- - - - - - - struct MemoryContext { - ARTBase* art_{nullptr}; - std::vector> thread_extents_; - u8* data_{nullptr}; - usize in_use_{sizeof(ExtentStorageT)}; - - //+++++++++++-+-+--+----- --- -- - - - - - - ~MemoryContext() noexcept - { - if (this->art_) { - ARTMutexLock lock{this->art_->mutex_}; - for (auto& p_ex : this->thread_extents_) { - this->art_->extents_.emplace_back(std::move(p_ex)); - } - } - } - - void* alloc(usize n, ARTBase* art) - { - this->art_ = art; - - const usize in_use_prior = this->in_use_; - if (in_use_prior + n <= kExtentSize) { - this->in_use_ += n; - return this->data_ + in_use_prior; - } - - this->art_->metrics_.byte_alloc_count.add(sizeof(ExtentStorageT)); - - this->thread_extents_.emplace_back(std::make_unique()); - u8* start = reinterpret_cast(this->thread_extents_.back().get()); - this->data_ = start; - this->in_use_ = 0; - - return this->alloc(n, art); - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit ARTBase() noexcept : ARTBase{ARTBase::default_metrics()} - { - // No update of construct count because we delegate to general-case ctor. - } - - explicit ARTBase(Metrics& metrics) noexcept : metrics_{metrics} - { - this->metrics_.construct_count.add(1); - } - - ~ARTBase() noexcept - { - this->metrics_.destruct_count.add(1); - } - - //+++++++++++-+-+--+----- --- -- - - - - - protected: - /** \brief RAII (guard) class that updates byte_free_count metric at the right moment during - * destruction of the ART (see comment in data member declarations below). - */ - struct ExtentMetricsUpdateGuard { - ARTBase& art_base_; - - //----- --- -- - - - - - - explicit ExtentMetricsUpdateGuard(ARTBase& art_base) noexcept : art_base_{art_base} - { - } - - ~ExtentMetricsUpdateGuard() noexcept - { - this->art_base_.metrics_.byte_free_count.add(this->art_base_.extents_.size() * - sizeof(ExtentStorageT)); - } - }; - - void* alloc_storage(usize n, usize pre) - { - const usize pad = (pre + 7) & ~usize{7}; - char* const ptr = (char*)this->per_thread_memory_context_.get().alloc(n + pad, this); - return ptr + pad; - } - - Metrics& metrics_; - ARTMutex mutex_; - std::vector> extents_; - // - // Must be placed exactly here, so it will be destructed after the ScopedSlot but before extents_. - ExtentMetricsUpdateGuard guard_{*this}; - // - batt::ObjectThreadStorage::ScopedSlot per_thread_memory_context_; -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_bit_ops.hpp b/src/turtle_kv/util/art_bit_ops.hpp deleted file mode 100644 index c6fe312..0000000 --- a/src/turtle_kv/util/art_bit_ops.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once -#define TURTLE_KV_ART_BIT_OPS_HPP - -#include - -#include -#include - -#include // SSE2 -#include // MMX -#include // SSE3 - -#ifdef __AVX512F__ -#include // AVX512 (AVX, AVX2, FMA) -#endif - -namespace turtle_kv { - -using batt::first_bit; -using batt::next_bit; - -/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns one of: {4, - * 5, 6, 7}. - */ -inline usize index_of(u8 key_byte, const std::array& keys) -{ - __m64 pattern = _mm_set1_pi8((char)key_byte); - u64 extended = *((const u32*)keys.data()); - __m64 values = _mm_cvtsi64_m64(extended); - __m64 result = _m_pcmpeqb(pattern, values); - - return ((__builtin_ffsll((i64)result) - 1) >> 3) & 7; -} - -/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns 31. - */ -inline usize index_of(u8 key_byte, const std::array& keys) -{ - __m128i pattern = _mm_set1_epi8((char)key_byte); - __m128i values = _mm_lddqu_si128((const __m128i*)keys.data()); - -#ifndef __AVX512F__ - int result = _mm_movemask_epi8(_mm_cmpeq_epi8(pattern, values)); -#else - __mmask16 result = _mm_cmpeq_epi8_mask(pattern, values); -#endif - - return (__builtin_ffs(result) - 1) & 31; -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_default_inserters.hpp b/src/turtle_kv/util/art_default_inserters.hpp deleted file mode 100644 index a383155..0000000 --- a/src/turtle_kv/util/art_default_inserters.hpp +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once -#define TURTLE_KV_UTIL_ART_DEFAULT_INSERTERS_HPP - -#include - -#include - -#include - -namespace turtle_kv { - -template -struct DefaultCopyInserter { - const ValueT& copy_from_; - - explicit DefaultCopyInserter(const ValueT& copy_from) noexcept : copy_from_{copy_from} - { - } - - Status insert_new(void* copy_to) - { - new (copy_to) ValueT{this->copy_from_}; - return OkStatus(); - } - - Status update_existing(ValueT* copy_to) - { - *copy_to = this->copy_from_; - return OkStatus(); - } -}; - -template -struct DefaultMoveInserter { - ValueT&& move_from_; - - explicit DefaultMoveInserter(ValueT&& move_from) noexcept : move_from_{move_from} - { - } - - Status insert_new(void* move_to) - { - new (move_to) ValueT{std::move(this->move_from_)}; - return OkStatus(); - } - - Status update_existing(ValueT* move_to) - { - *move_to = std::move(this->move_from_); - return OkStatus(); - } -}; - -struct DefaultVoidInserter { - BATT_ALWAYS_INLINE Status insert_new(void*) - { - return OkStatus(); - } - - BATT_ALWAYS_INLINE Status update_existing(void*) - { - return OkStatus(); - } -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_metrics.hpp b/src/turtle_kv/util/art_metrics.hpp deleted file mode 100644 index 9ac8178..0000000 --- a/src/turtle_kv/util/art_metrics.hpp +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once -#define TURTLE_KV_UTIL_ART_METRICS_HPP - -#include -#include - -namespace turtle_kv { - -struct ARTMetrics { - CountMetric construct_count; - CountMetric destruct_count; - FastCountMetric insert_count; - FastCountMetric byte_alloc_count; - FastCountMetric byte_free_count; - - /** \brief Resets all metrics to initial values. - */ - void reset() - { - this->construct_count.reset(); - this->destruct_count.reset(); - this->insert_count.reset(); - this->byte_alloc_count.reset(); - this->byte_free_count.reset(); - } - - //----- --- -- - - - - - - double bytes_per_instance() const - { - return (double)this->byte_alloc_count.get() / (double)this->construct_count.get(); - } - - double average_item_count() const - { - return (double)this->insert_count.get() / (double)this->construct_count.get(); - } - - double bytes_per_insert() const - { - return (double)this->byte_alloc_count.get() / (double)this->insert_count.get(); - } - - /** \brief Returns an estimate of the number of active instances (ART objects). - */ - u64 instance_count() const - { - // Must be in this order! - // - const u64 observed_destruct_count = this->destruct_count.get(); - const u64 observed_construct_count = this->construct_count.get(); - - return observed_construct_count - observed_destruct_count; - } - - /** \brief Returns an estimate of the current number of bytes in use. - */ - u64 bytes_in_use() const - { - // Must be in this order! - // - const u64 observed_free_count = this->byte_free_count.get(); - const u64 observed_alloc_count = this->byte_alloc_count.get(); - - return observed_alloc_count - observed_free_count; - } -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_mutex.hpp b/src/turtle_kv/util/art_mutex.hpp deleted file mode 100644 index 2bfe3ae..0000000 --- a/src/turtle_kv/util/art_mutex.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once -#define TURTLE_KV_UTIL_ART_MUTEX_HPP - -#define ART_USE_ABSEIL_MUTEX 1 -#define ART_USE_STD_MUTEX 0 - -#if ART_USE_ABSEIL_MUTEX -#include -#endif - -#if ART_USE_STD_MUTEX -#include -#endif - -namespace turtle_kv { - -#if ART_USE_ABSEIL_MUTEX -using ARTMutex = absl::Mutex; -using ARTMutexLock = absl::MutexLock; -#endif - -#if ART_USE_STD_MUTEX -using ARTMutex = std::mutex; -using ARTMutexLock = std::unique_lock; -#endif - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art_scanner.hpp b/src/turtle_kv/util/art_scanner.hpp deleted file mode 100644 index c18599a..0000000 --- a/src/turtle_kv/util/art_scanner.hpp +++ /dev/null @@ -1,362 +0,0 @@ -#pragma once -#define TURTLE_KV_UTIL_ART_SCANNER_HPP - -#include "art_base.hpp" - -#include "detail/scanner_item_storage_base.hpp" -#include "detail/scanner_value_storage_base.hpp" - -#include - -namespace turtle_kv { -namespace detail { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of(usize node_prefix_len, - NodeT* node, - AlignedStorageT* storage, - std::integral_constant, - const Optional&, - void* value_storage_addr, - batt::StaticType type_of_value) -{ - NodeT& node_view = *(new (storage) NodeT{ARTBase::NoInit{}}); - - // Retry the node read until we get a consistent view. - // - for (;;) { - SeqMutex::ReadLock read_lock{node->mutex_}; - node_view.assign_from(*node, /*prefix_offset=*/node_prefix_len); - if (node->is_terminal()) { - ARTBase::construct_value_copy_addr(node, value_storage_addr, type_of_value); - } - if (!read_lock.changed()) { - break; - } - } - - return node_view; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of(usize, - NodeT* node, - AlignedStorageT*, - std::integral_constant, - const Optional& /*sync*/, - const void* /*value_storage_addr*/, - batt::StaticType /*type_of_value*/) -{ - return *node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of( - usize node_prefix_len, - NodeT* node, - AlignedStorageT* storage, - std::integral_constant, - const Optional& sync, - void* value_storage_addr, - batt::StaticType type_of_value) -{ - if (sync.value_or(true)) { - return scanner_view_of( - node_prefix_len, - node, - storage, - std::integral_constant{}, - sync, - value_storage_addr, - type_of_value); - } - return scanner_view_of( - node_prefix_len, - node, - storage, - std::integral_constant{}, - sync, - value_storage_addr, - type_of_value); -} - -} // namespace detail - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Scanner for an ART. - * - * \tparam ValueT The value type stored in the scanned ART - * \tparam kSynchronized (true, false, dynmamic) The concurrency control for this scanner - * \tparam kValuesOnly When true, the scanner does not build/store key (path) information as it is - * traversing items -- only values are available - */ -template -template -class ART::Scanner - : public detail::ScannerItemStorageBase -{ - public: - using LeafNode = ARTBase::LeafNode; - using Node4 = ARTBase::Node4; - using Node16 = ARTBase::Node16; - using Node48 = ARTBase::Node48; - using Node256 = ARTBase::Node256; - - using NodeScanState = std::variant; - - static constexpr usize kMaxDepth = ART::kMaxKeyLen; - - using SyncType = std::integral_constant; - - using Value = std::conditional_t, - struct get_value_Not_Supported_If_ValueT_Is_Void, - ValueT>; - - //+++++++++++-+-+--+----- --- -- - - - - - - static_assert(sizeof(Node256) > sizeof(Node48)); - static_assert(sizeof(Node256) > sizeof(Node16)); - static_assert(sizeof(Node256) > sizeof(Node4)); - static_assert(sizeof(Node256) > sizeof(LeafNode)); - - struct Frame { - static constexpr usize kStorageSize = - ((kSynchronized == ARTBase::Synchronized::kFalse) ? 1 : sizeof(Node256)); - - std::aligned_storage_t node_storage_; - NodeScanState scan_state_; - usize key_prefix_len_; - std::string_view lower_bound_key_; - ByteInt min_key_byte_; - - explicit Frame(usize key_prefix_len, std::string_view lower_bound_key) noexcept - : scan_state_{None} - , key_prefix_len_{key_prefix_len} - , lower_bound_key_{lower_bound_key} - , min_key_byte_{ByteInt::from_i32(0)} - { - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - private: - std::aligned_storage_t stack_storage_; - Frame* end_ = reinterpret_cast(&this->stack_storage_); - usize depth_ = 0; - bool have_item_ = false; - ValueT* next_value_ = nullptr; - Optional synchronized_; - - //+++++++++++-+-+--+----- --- -- - - - - - - /** Resets the "have item" state of the scanner; after calling, have item will be false. - */ - void reset_item() BATT_ALWAYS_INLINE - { - this->have_item_ = false; - if (!std::is_same_v) { - this->next_value_ = nullptr; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - public: - explicit Scanner(ART& art, - std::string_view lower_bound_key, - Optional synchronized = None) noexcept - : synchronized_{synchronized} - { - NodeBase* root = nullptr; - for (;;) { - SeqMutex::ReadLock root_read_lock{art.super_root_.mutex_}; - root = art.root_; - if (!root_read_lock.changed()) { - break; - } - } - - if (root) { - root->visit([&](auto* node) { - this->enter(node, /*key_prefix_len=*/0, lower_bound_key); - }); - - if (!this->have_item_) { - this->advance(); - } - } - } - - ~Scanner() noexcept - { - } - - bool is_synchronized() const - { - if (kSynchronized == ARTBase::Synchronized::kFalse) { - return false; - } - if (kSynchronized == ARTBase::Synchronized::kTrue) { - return true; - } - return this->synchronized_.value_or(true); - } - - template - void enter(NodeT* node, usize key_prefix_len, std::string_view lower_bound_key) - { - Frame* top = new (this->end_) Frame{key_prefix_len, lower_bound_key}; - ++this->depth_; - ++this->end_; - - // Node prefix is immutable, so we don't need synchronization. - // - const char* const node_prefix = node->prefix(); - const usize node_prefix_len = node->prefix_len_; - - // We need to create a copy of the node data to protect against data races. - // - NodeT& node_view = - detail::scanner_view_of(node_prefix_len, - node, - &top->node_storage_, - SyncType{}, - this->synchronized_, - this->value_storage_address(node, this->synchronized_), - batt::StaticType{}); - - // Compare the lower bound key to the current node prefix. - // - const usize compare_len = std::min(node_prefix_len, top->lower_bound_key_.size()); - if (compare_len) { - const i32 order = __builtin_memcmp(node_prefix, top->lower_bound_key_.data(), compare_len); - - // If all keys in this subtree come before the lower bound, then there is nothing to do. - // - if (order < 0) { - --this->depth_; - --this->end_; - return; - } - - // If the node prefix is a prefix of the lower bound key, then drop the prefix from the lower - // bound; otherwise the node prefix comes *after* the lower bound, so we can safely ignore the - // lower bound for the rest of the recursion. - // - if (order == 0 && compare_len == node_prefix_len) { - top->lower_bound_key_.remove_prefix(compare_len); - } else { - top->lower_bound_key_ = {}; - } - } - - // Set bounds for branch visitation. - // - top->min_key_byte_ = [&]() -> ByteInt { - if (top->lower_bound_key_.empty()) { - return ByteInt::from_i32(0); - } - const ByteInt next_char = ByteInt::from_char(top->lower_bound_key_.front()); - top->lower_bound_key_.remove_prefix(1); - return next_char; - }(); - - // Append the node prefix to the buffer. - // - if (node_prefix_len) { - this->append_key(top->key_prefix_len_, node_prefix, node_prefix_len); - top->key_prefix_len_ += node_prefix_len; - } - - // If the current node is a key-terminal, emit the contents of the buffer. - // - if (node_view.is_terminal()) { - this->have_item_ = true; - this->set_key_len(top->key_prefix_len_); - if (!std::is_same_v) { - this->next_value_ = (ValueT*)(this->value_storage_address(&node_view, this->synchronized_)); - } - } else { - this->reset_item(); - } - - [[maybe_unused]] auto& scan_state_impl = - top->scan_state_.template emplace(node_view, top->min_key_byte_); - } - - bool is_done() const - { - return this->depth_ == 0; - } - - // get_key() const member function is inherited from ItemStorageBase, if kValuesOnly is false. - - const Value& get_value() const - { - static_assert(!std::is_same_v); - return *this->next_value_; - } - - void advance() - { - this->reset_item(); - - for (;;) { - if (this->depth_ == 0) { - return; - } - - Frame* top = this->end_ - 1; - - batt::case_of( - top->scan_state_, - [](batt::NoneType&) { - BATT_PANIC() << "empty Scanner stack frame!"; - }, - [&](auto& scan_state) - -> std::enable_if_t< - !std::is_same_v, batt::NoneType>> { - //----- --- -- - - - - - if (scan_state.is_done()) { - --this->depth_; - --this->end_; - return; - } - - const ByteInt key_byte = scan_state.get_key_byte(); - NodeBase* const child = scan_state.get_branch(); - - this->append_key_byte(top->key_prefix_len_, key_byte); - - if (key_byte == top->min_key_byte_) { - child->visit([&](auto* child_node) { - this->enter(child_node, top->key_prefix_len_ + 1, top->lower_bound_key_); - }); - } else { - child->visit([&](auto* child_node) { - this->enter(child_node, top->key_prefix_len_ + 1, std::string_view{}); - }); - } - - scan_state.advance(); - }); - - if (this->have_item_) { - return; - } - } - } -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/detail/scanner_item_storage_base.hpp b/src/turtle_kv/util/detail/scanner_item_storage_base.hpp deleted file mode 100644 index 1b9d498..0000000 --- a/src/turtle_kv/util/detail/scanner_item_storage_base.hpp +++ /dev/null @@ -1,75 +0,0 @@ -#pragma once -#define TURTLE_KV_UTIL_DETAIL_SCANNER_ITEM_STORAGE_BASE_HPP - -#include "scanner_value_storage_base.hpp" - -#include - -namespace turtle_kv { -namespace detail { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Base class for scanner; contains storage for the item at the current scanner position. - */ -template -class ScannerItemStorageBase; - -/** \brief General case (kValuesOnly == false) for scanner item storage. - */ -template -class ScannerItemStorageBase - : public ScannerValueStorageBase -{ - public: - std::array key_buffer_; - usize key_len_ = 0; - - //----- --- -- - - - - - - void append_key(usize prefix_len, const char* suffix_data, usize suffix_len) BATT_ALWAYS_INLINE - { - __builtin_memcpy(this->key_buffer_.data() + prefix_len, suffix_data, suffix_len); - } - - void append_key_byte(usize prefix_len, const ByteInt& suffix_byte) BATT_ALWAYS_INLINE - { - this->key_buffer_[prefix_len] = suffix_byte.to_char(); - } - - void set_key_len(usize len) BATT_ALWAYS_INLINE - { - this->key_len_ = len; - } - - std::string_view get_key() const - { - return std::string_view{this->key_buffer_.data(), this->key_len_}; - } -}; - -/** \brief kValuesOnly == true case; no key-related data members. - */ -template -class ScannerItemStorageBase - : public ScannerValueStorageBase -{ - public: - void append_key(usize, const char*, usize) BATT_ALWAYS_INLINE - { - // nothing to do. - } - - void append_key_byte(usize, const ByteInt&) BATT_ALWAYS_INLINE - { - // nothing to do. - } - - void set_key_len(usize) BATT_ALWAYS_INLINE - { - // nothing to do. - } -}; - -} // namespace detail -} // namespace turtle_kv diff --git a/src/turtle_kv/util/detail/scanner_value_storage_base.hpp b/src/turtle_kv/util/detail/scanner_value_storage_base.hpp deleted file mode 100644 index 7f6238f..0000000 --- a/src/turtle_kv/util/detail/scanner_value_storage_base.hpp +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once -#define TURTLE_KV_UTIL_DETAIL_SCANNER_VALUE_STORAGE_BASE_HPP - -#include - -#include - -namespace turtle_kv { -namespace detail { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -struct ScannerValueStorageBase { - std::aligned_storage_t value_storage_; - - template - std::conditional_t, const void*, void*> value_storage_address( - NodeT* node, - const Optional& sync) - { - if (kSynchronized == ARTBase::Synchronized::kTrue || sync.value_or(true)) { - return &this->value_storage_; - } - return node + 1; - } -}; - -//----- --- -- - - - - - -template -struct ScannerValueStorageBase { - template - const void* value_storage_address(const NodeT* node, const Optional&) const - { - return node + 1; - } -}; - -//----- --- -- - - - - - -template <> -struct ScannerValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//----- --- -- - - - - - -template <> -struct ScannerValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//----- --- -- - - - - - -template <> -struct ScannerValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -} // namespace detail -} // namespace turtle_kv From d75a71262f7338689f4793698cb5d8c66f7de1f3 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 28 Jun 2026 12:55:57 -0400 Subject: [PATCH 11/20] Test tuning. --- conan.lock | 4 ++-- conanfile.py | 2 +- src/turtle_kv/tree/in_memory_node.cpp | 2 ++ src/turtle_kv/tree/in_memory_node_merged_level.cpp | 2 ++ src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp | 2 +- src/turtle_kv/util/piecewise_filter.test.cpp | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/conan.lock b/conan.lock index 1cb73b8..d85e847 100644 --- a/conan.lock +++ b/conan.lock @@ -1,8 +1,8 @@ { "version": "0.5", "requires": [ - "abseil/20250127.0", - "artc/0.1.0", + "abseil/20260107.1", + "artc/0.2.1", "batteries/0.72.0", "boost/1.88.0", "bzip2/1.0.8", diff --git a/conanfile.py b/conanfile.py index 3474dd2..186c7f2 100644 --- a/conanfile.py +++ b/conanfile.py @@ -89,7 +89,7 @@ def requirements(self): } self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) - self.requires("artc/[>=0.1.0 <1]") + self.requires("artc/[>=0.2.1 <1]") self.requires("batteries/[>=0.72.0 <1]", **VISIBLE, **OVERRIDE) self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) self.requires("glog/[>=0.7.1 <1]", **VISIBLE) diff --git a/src/turtle_kv/tree/in_memory_node.cpp b/src/turtle_kv/tree/in_memory_node.cpp index 536e0c3..e0bcd71 100644 --- a/src/turtle_kv/tree/in_memory_node.cpp +++ b/src/turtle_kv/tree/in_memory_node.cpp @@ -15,6 +15,8 @@ #include #include +#include + #include #include diff --git a/src/turtle_kv/tree/in_memory_node_merged_level.cpp b/src/turtle_kv/tree/in_memory_node_merged_level.cpp index 7a67284..a28be9f 100644 --- a/src/turtle_kv/tree/in_memory_node_merged_level.cpp +++ b/src/turtle_kv/tree/in_memory_node_merged_level.cpp @@ -16,6 +16,8 @@ #include +#include + #include namespace turtle_kv { diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index c3805f4..cc7a5c8 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -66,7 +66,7 @@ using turtle_kv::ValueView; TEST(TreePackedBlockedLeafPageTest, Random) { const usize kFirstSeed = 0; - const usize kNumSeeds = 1000; + const usize kNumSeeds = 250; const usize kLastSeed = kFirstSeed + kNumSeeds; const usize kLeafPageSize = 1 * kMiB; const usize kNumPrefixes = 1000; diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index 59b8d0b..7fa32a8 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -312,7 +312,7 @@ TEST(PiecewiseFilterTest, LiveSubranges) std::array, 1> init_live{{{0, 64}}}; - const usize n_seeds = 10000000; + const usize n_seeds = 100000; const usize n_drops = 32; const usize n_queries = 15; const usize first_seed = 0; From 1fe1427eb4d252465c2b2cfed51aeecb570e08ce Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Tue, 7 Jul 2026 10:32:29 -0400 Subject: [PATCH 12/20] Refactor with PackedPiecewiseFilter --- src/turtle_kv/tree/in_memory_node.cpp | 4 +- src/turtle_kv/tree/packed_node_page.cpp | 134 ++--------------- src/turtle_kv/tree/packed_node_page.hpp | 9 +- .../util/packed_piecewise_filter_view.hpp | 49 +++--- src/turtle_kv/util/piecewise_filter.hpp | 36 ++++- src/turtle_kv/util/piecewise_filter.ipp | 58 +++++++ .../util/piecewise_filter.live_subranges.hpp | 2 +- src/turtle_kv/util/piecewise_filter.test.cpp | 141 ++++++++++-------- src/turtle_kv/util/piecewise_filter.test.hpp | 118 +++++++++++++++ 9 files changed, 331 insertions(+), 220 deletions(-) diff --git a/src/turtle_kv/tree/in_memory_node.cpp b/src/turtle_kv/tree/in_memory_node.cpp index e0bcd71..a3a712b 100644 --- a/src/turtle_kv/tree/in_memory_node.cpp +++ b/src/turtle_kv/tree/in_memory_node.cpp @@ -118,8 +118,8 @@ using PackedSegment = PackedUpdateBuffer::Segment; segment.page_id_slot = llfs::PageIdSlot::from_page_id(packed_segment.leaf_page_id.unpack()); segment.active_pivots = packed_segment.active_pivots.unpack(); - BATT_ASSIGN_OK_RESULT(segment.filter, - packed_node.create_piecewise_filter(level_i, segment_i)); + segment.filter = + PiecewiseFilter{packed_node.get_packed_filter(level_i, segment_i)}; segment.check_invariants(__FILE__, __LINE__); } diff --git a/src/turtle_kv/tree/packed_node_page.cpp b/src/turtle_kv/tree/packed_node_page.cpp index 904b83e..c469d6f 100644 --- a/src/turtle_kv/tree/packed_node_page.cpp +++ b/src/turtle_kv/tree/packed_node_page.cpp @@ -222,9 +222,7 @@ StatusOr PackedNodePage::find_key(KeyQuery& query) const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -PackedNodePage::UpdateBuffer::SegmentFilterData PackedNodePage::get_segment_filter_values( - usize level_i, - usize segment_i) const +PackedPiecewiseFilter PackedNodePage::get_packed_filter(usize level_i, usize segment_i) const { const usize i = [&]() -> usize { if (this->is_size_tiered()) { @@ -256,52 +254,9 @@ PackedNodePage::UpdateBuffer::SegmentFilterData PackedNodePage::get_segment_filt bool start_live = (segment.filter_start.value() & PackedNodePage::kSegmentStartsLive) != 0; - return PackedNodePage::UpdateBuffer::SegmentFilterData{ + return PackedPiecewiseFilter{PackedPiecewiseFilterStorage{ as_const_slice(packed_filters.data() + filter_start_i, packed_filters.data() + filter_end_i), - start_live}; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -StatusOr> PackedNodePage::create_piecewise_filter(usize level_i, - usize segment_i) const -{ - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - this->get_segment_filter_values(level_i, segment_i); - - SmallVec, 64> live_ranges; - u32 i = 0; - - // If the first item at index 0 is live, add the corresponding interval first since the - // serialized version of the filter doesn't store index 0. - // - if (filter_data.start_is_live) { - if (filter_data.values.empty()) { - // Entire segment is live. - // - live_ranges.emplace_back(Interval{PiecewiseFilter::kMinLowerBound, - PiecewiseFilter::kMaxUpperBound}); - } else { - live_ranges.emplace_back( - Interval{PiecewiseFilter::kMinLowerBound, filter_data.values[i].value()}); - i++; - } - } - - for (; i + 1 < filter_data.values.size(); i += 2) { - live_ranges.emplace_back( - Interval{filter_data.values[i].value(), filter_data.values[i + 1].value()}); - } - - // If there's one unpaired value left, it's a lower_bound whose upper_bound (kMaxUpperBound) - // was omitted. - // - if (i < filter_data.values.size()) { - live_ranges.emplace_back( - Interval{filter_data.values[i].value(), PiecewiseFilter::kMaxUpperBound}); - } - - return PiecewiseFilter::from_live(as_slice(live_ranges)); + start_live}}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -339,7 +294,11 @@ StatusOr PackedNodePage::UpdateBuffer::Segment::load_leaf_page bool PackedNodePage::UpdateBuffer::Segment::is_index_filtered(const SegmentedLevel& level, u32 index) const { - return !(this->live_lower_bound(level, index) == index); + const usize segment_i = std::distance(level.segments_slice.begin(), this); + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return !filter.live_at_index(index); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -348,34 +307,10 @@ u32 PackedNodePage::UpdateBuffer::Segment::live_lower_bound(const SegmentedLevel u32 item_i) const { const usize segment_i = std::distance(level.segments_slice.begin(), this); - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - level.packed_node_->get_segment_filter_values(level.level_i_, segment_i); - - const Slice filter_values = filter_data.values; - - if (filter_data.values.empty()) { - BATT_CHECK(filter_data.start_is_live); - return item_i; - } - - auto iter = std::upper_bound(filter_values.begin(), filter_values.end(), item_i); - - usize previous_cut_points = std::distance(filter_data.values.begin(), iter); - - bool is_live = (previous_cut_points % 2 == 0) == filter_data.start_is_live; - - // If we're already in an unfiltered region, just return the index. Otherwise, our upper bound - // is the next unfiltered index. - // - if (is_live) { - return item_i; - } - - if (iter != filter_values.end()) { - return iter->value(); - } - - return PiecewiseFilter::kMaxUpperBound; + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return filter.live_lower_bound(item_i); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -384,48 +319,11 @@ Interval PackedNodePage::UpdateBuffer::Segment::get_live_item_range( const SegmentedLevel& level, Interval i) const { - u32 start_i = i.lower_bound; - u32 end_i = i.upper_bound; - - BATT_CHECK_LT(start_i, end_i); - const usize segment_i = std::distance(level.segments_slice.begin(), this); - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - level.packed_node_->get_segment_filter_values(level.level_i_, segment_i); - - const Slice filter_values = filter_data.values; - - if (filter_data.values.empty()) { - BATT_CHECK(filter_data.start_is_live); - return i; - } - - auto iter = std::upper_bound(filter_values.begin(), filter_values.end(), start_i); - - usize previous_cut_points = std::distance(filter_data.values.begin(), iter); - - bool is_live = (previous_cut_points % 2 == 0) == filter_data.start_is_live; - - if (!is_live) { - if (iter == filter_values.end()) { - return Interval{end_i, end_i}; - } - - start_i = iter->value(); - if (start_i >= end_i) { - return Interval{end_i, end_i}; - } - - ++iter; - } - - if (iter != filter_values.end()) { - end_i = std::min(end_i, iter->value()); - } - - BATT_CHECK_LT(start_i, end_i) << BATT_INSPECT(i); - - return Interval{start_i, end_i}; + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return filter.find_live_range(i); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // diff --git a/src/turtle_kv/tree/packed_node_page.hpp b/src/turtle_kv/tree/packed_node_page.hpp index 1e7cc66..5f4e92e 100644 --- a/src/turtle_kv/tree/packed_node_page.hpp +++ b/src/turtle_kv/tree/packed_node_page.hpp @@ -121,11 +121,6 @@ struct PackedNodePage { struct UpdateBuffer { struct SegmentedLevel; - struct SegmentFilterData { - Slice values; - bool start_is_live; - }; - struct Segment { llfs::PackedPageId leaf_page_id; // +8 -> 8 PackedActivePivotsSet64 active_pivots; // +8 -> 16 @@ -387,9 +382,7 @@ struct PackedNodePage { StatusOr find_key_in_level(usize level_i, KeyQuery& query, i32 key_pivot_i) const; - UpdateBuffer::SegmentFilterData get_segment_filter_values(usize level_i, usize segment_i) const; - - StatusOr> create_piecewise_filter(usize level_i, usize segment_i) const; + PackedPiecewiseFilter get_packed_filter(usize level_i, usize segment_i) const; //----- --- -- - - - - diff --git a/src/turtle_kv/util/packed_piecewise_filter_view.hpp b/src/turtle_kv/util/packed_piecewise_filter_view.hpp index 069cc11..990088f 100644 --- a/src/turtle_kv/util/packed_piecewise_filter_view.hpp +++ b/src/turtle_kv/util/packed_piecewise_filter_view.hpp @@ -16,6 +16,7 @@ #include #include +#include #include @@ -50,7 +51,7 @@ namespace turtle_kv { * Live Intervals: {[10, 20), [30, 40), [50, +inf)} * Packed: start_is_live=0, {10, 20, 30, 40, 50} */ -class PackedPiecewiseFilterView +class PackedPiecewiseFilterStorage { public: //----- --- -- - - - - @@ -73,35 +74,35 @@ class PackedPiecewiseFilterView // Forward-declaration; defined below. // - friend const Slice& as_const_slice(const PackedPiecewiseFilterView& view); + friend const Slice& as_const_slice(const PackedPiecewiseFilterStorage& view); //----- --- -- - - - - - /** \brief Constructs an PackedPiecewiseFilterView representing the live interval [0, +inf). + /** \brief Constructs an PackedPiecewiseFilterStorage representing the live interval [0, +inf). */ - PackedPiecewiseFilterView() = default; + PackedPiecewiseFilterStorage() = default; - /** \brief Destructs the PackedPiecewiseFilterView. + /** \brief Destructs the PackedPiecewiseFilterStorage. */ - ~PackedPiecewiseFilterView() = default; + ~PackedPiecewiseFilterStorage() = default; - /** \brief PackedPiecewiseFilterView is copy constructible. + /** \brief PackedPiecewiseFilterStorage is copy constructible. */ - PackedPiecewiseFilterView(const PackedPiecewiseFilterView&) = default; + PackedPiecewiseFilterStorage(const PackedPiecewiseFilterStorage&) = default; - /** \brief PackedPiecewiseFilterView is copy assignable. + /** \brief PackedPiecewiseFilterStorage is copy assignable. */ - PackedPiecewiseFilterView& operator=(const PackedPiecewiseFilterView&) = default; + PackedPiecewiseFilterStorage& operator=(const PackedPiecewiseFilterStorage&) = default; - /** \brief Constructs PackedPiecewiseFilterView from the packed data in the arguments. + /** \brief Constructs PackedPiecewiseFilterStorage from the packed data in the arguments. * * See the class-level description for details on what `values` and `start_is_live` represent. */ - explicit PackedPiecewiseFilterView(const Slice& values, + explicit PackedPiecewiseFilterStorage(const Slice& values, bool start_is_live) noexcept : values_{values} , implicit_first_{start_is_live ? 1 : 0} - , size_{(this->implicit_first_ + this->values_.size() + 1) & ~i32{1}} + , size_{(this->implicit_first_ + BATT_CHECKED_CAST(i32, this->values_.size()) + 1) / 2} { } @@ -148,7 +149,7 @@ class PackedPiecewiseFilterView // /** \brief Returns a const reference to the stored values referenced by `view`. */ -inline const Slice& as_const_slice(const PackedPiecewiseFilterView& view) +inline const Slice& as_const_slice(const PackedPiecewiseFilterStorage& view) { return view.values_; } @@ -157,9 +158,9 @@ inline const Slice& as_const_slice(const PackedPiecewiseFilter // /** \brief Read-only, random access iterator over the live intervals of a packed piecewise filter. */ -class PackedPiecewiseFilterView::const_iterator +class PackedPiecewiseFilterStorage::const_iterator : public boost::iterator_facade< // - PackedPiecewiseFilterView::const_iterator, // <- Derived + PackedPiecewiseFilterStorage::const_iterator, // <- Derived Interval, // <- Value std::random_access_iterator_tag, // <- CategoryOrTraversal Interval, // <- Reference @@ -184,7 +185,7 @@ class PackedPiecewiseFilterView::const_iterator * * `view` must remain in-scope while this object exists. */ - const_iterator(const PackedPiecewiseFilterView* view, isize pos) noexcept : view_{view}, pos_{pos} + const_iterator(const PackedPiecewiseFilterStorage* view, isize pos) noexcept : view_{view}, pos_{pos} { } @@ -239,7 +240,7 @@ class PackedPiecewiseFilterView::const_iterator private: /** \brief Pointer to the filter view over which we are iterating. */ - const PackedPiecewiseFilterView* view_; + const PackedPiecewiseFilterStorage* view_; /** \brief The (logical) position of this iterator within `view_`. */ @@ -248,35 +249,35 @@ class PackedPiecewiseFilterView::const_iterator //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline auto PackedPiecewiseFilterView::begin() const noexcept -> const_iterator +inline auto PackedPiecewiseFilterStorage::begin() const noexcept -> const_iterator { return const_iterator{this, 0}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline auto PackedPiecewiseFilterView::end() const noexcept -> const_iterator +inline auto PackedPiecewiseFilterStorage::end() const noexcept -> const_iterator { return const_iterator{this, static_cast(this->size())}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline usize PackedPiecewiseFilterView::size() const noexcept +inline usize PackedPiecewiseFilterStorage::size() const noexcept { return this->size_; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline bool PackedPiecewiseFilterView::empty() const noexcept +inline bool PackedPiecewiseFilterStorage::empty() const noexcept { return this->size_ == 0; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline Interval PackedPiecewiseFilterView::operator[](isize i) const noexcept +inline Interval PackedPiecewiseFilterStorage::operator[](isize i) const noexcept { // Cached for brevity below. // @@ -300,6 +301,6 @@ inline Interval PackedPiecewiseFilterView::operator[](isize i) const noexce //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -static_assert(PiecewiseFilterStorageModel); +static_assert(PiecewiseFilterStorageModel); } // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.hpp b/src/turtle_kv/util/piecewise_filter.hpp index 16a7cfc..9f22ceb 100644 --- a/src/turtle_kv/util/piecewise_filter.hpp +++ b/src/turtle_kv/util/piecewise_filter.hpp @@ -9,6 +9,7 @@ #pragma once #define TURTLE_KV_UTIL_PIECEWISE_FILTER_HPP +#include "packed_piecewise_filter_view.hpp" #include "piecewise_filter_storage_model.concept.hpp" #include @@ -62,7 +63,19 @@ class BasicPiecewiseFilter : private ModelT /** \brief Constructs a default instance of a PiecewiseFilter object, initialized with no item * range and filtered items. */ - BasicPiecewiseFilter() noexcept; + BasicPiecewiseFilter() noexcept + requires PiecewiseFilterMutableStorageModel; + + /** \brief Constructs a BasicPiecewiseFilter directly from a storage model instance. + */ + explicit BasicPiecewiseFilter(const ModelT& model) noexcept; + + /** \brief Constructs a BasicPiecewiseFilter by copying live intervals from a filter with a + * different storage model. + */ + template OtherModelT> + explicit BasicPiecewiseFilter(const BasicPiecewiseFilter& other) + requires PiecewiseFilterMutableStorageModel; //+++++++++++-+-+--+----- --- -- - - - - @@ -105,7 +118,8 @@ class BasicPiecewiseFilter : private ModelT /** \brief Returns a view of the live item intervals. */ - Slice> live() const; + Slice> live() const + requires PiecewiseFilterMutableStorageModel; /** \brief Merges two filters in place, taking the union of the live intervals. */ @@ -117,6 +131,22 @@ class BasicPiecewiseFilter : private ModelT */ LiveSubranges live_subranges_of(Interval i) const; + /** \brief Returns an iterator to the first live interval. + */ + ConstIterator begin() const; + + /** \brief Returns an iterator past the last live interval. + */ + ConstIterator end() const; + + /** \brief Returns the number of live intervals. + */ + usize size() const; + + /** \brief Returns true iff there are no live intervals. + */ + bool empty() const; + /** \brief Validate the state of the live intervals. */ bool check_invariants() const; @@ -139,6 +169,8 @@ class BasicPiecewiseFilter : private ModelT template using PiecewiseFilter = BasicPiecewiseFilter, 64>>; +using PackedPiecewiseFilter = BasicPiecewiseFilter; + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template diff --git a/src/turtle_kv/util/piecewise_filter.ipp b/src/turtle_kv/util/piecewise_filter.ipp index b892275..3848365 100644 --- a/src/turtle_kv/util/piecewise_filter.ipp +++ b/src/turtle_kv/util/piecewise_filter.ipp @@ -36,10 +36,35 @@ BasicPiecewiseFilter::from_live(const Slice ModelT> BasicPiecewiseFilter::BasicPiecewiseFilter() noexcept + requires PiecewiseFilterMutableStorageModel : ModelT{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} { } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter(const ModelT& model) noexcept + : ModelT{model} +{ +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +template OtherModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter( + const BasicPiecewiseFilter& other) + requires PiecewiseFilterMutableStorageModel + : ModelT{} +{ + this->live_().clear(); + + for (auto iter = other.begin(); iter != other.end(); ++iter) { + this->live_().insert(this->live_().end(), *iter); + } +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template ModelT> @@ -204,10 +229,43 @@ Interval BasicPiecewiseFilter::drop_index_range(Interv // template ModelT> Slice> BasicPiecewiseFilter::live() const + requires PiecewiseFilterMutableStorageModel { return as_const_slice(this->live_()); } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::begin() const -> ConstIterator +{ + return this->live_().begin(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::end() const -> ConstIterator +{ + return this->live_().end(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +usize BasicPiecewiseFilter::size() const +{ + return this->live_().size(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +bool BasicPiecewiseFilter::empty() const +{ + return this->live_().empty(); +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template ModelT> diff --git a/src/turtle_kv/util/piecewise_filter.live_subranges.hpp b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp index c2bf50c..a22a0b1 100644 --- a/src/turtle_kv/util/piecewise_filter.live_subranges.hpp +++ b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp @@ -22,7 +22,7 @@ template ModelT> class BasicPiecewiseFilter::LiveSubranges { public: - using Iterator = PiecewiseFilter::ConstIterator; + using Iterator = typename BasicPiecewiseFilter::ConstIterator; using Item = Interval; //+++++++++++-+-+--+----- --- -- - - - - diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index 7fa32a8..7aa134e 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -42,8 +43,14 @@ using turtle_kv::PiecewiseFilter; using turtle_kv::Slice; using turtle_kv::Status; using turtle_kv::StatusOr; +using turtle_kv::testing::build_filter_with_random_drops; using turtle_kv::testing::drop_n_disjoint_intervals_from; +using turtle_kv::testing::get_packed_filter_from_data; +using turtle_kv::testing::PackedFilterData; +using turtle_kv::testing::pack_in_memory_filter; +using turtle_kv::testing::RandomDropResult; using turtle_kv::testing::RandomStringGenerator; +using turtle_kv::testing::verify_filter_queries; using turtle_kv::drop_item_range; @@ -52,6 +59,10 @@ using llfs::KeyRangeOrder; using batt::mask_from_interval; using batt::StableStringStore; +using turtle_kv::PackedPiecewiseFilter; +using turtle_kv::PackedPiecewiseFilterStorage; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // TEST(PiecewiseFilterTest, InvalidFilterTest) @@ -84,89 +95,68 @@ TEST(PiecewiseFilterTest, InvalidFilterTest) // TEST(PiecewiseFilterTest, QueryTest) { - const usize num_items = 10000; + const u32 num_items = 10000; - for (usize seed = 0; seed < 100; ++seed) { + for (u32 seed = 0; seed < 100; ++seed) { std::default_random_engine rng{seed}; - PiecewiseFilter filter; - EXPECT_TRUE(filter.check_invariants()); + auto [filter, live_items] = build_filter_with_random_drops(num_items, rng); - // All items start live. - // - std::set live_items; - for (usize i = 0; i < num_items; ++i) { - live_items.insert(i); - } + EXPECT_TRUE(filter.check_invariants()); - // Drop random intervals. - // - std::uniform_int_distribution pick_num_dropped{100, num_items / 2}; - usize num_intervals_dropped = pick_num_dropped(rng); - for (usize i = 0; i < num_intervals_dropped; ++i) { - std::uniform_int_distribution pick_interval_start{0, num_items - 1}; - usize start_i = pick_interval_start(rng); + verify_filter_queries(filter, live_items, num_items, seed, rng); + } +} - std::uniform_int_distribution pick_interval_end{start_i, num_items}; - usize end_i = pick_interval_end(rng); +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(PiecewiseFilterTest, PackedQueryTest) +{ + const u32 num_items = 10000; - for (usize j = start_i; j < end_i; ++j) { - live_items.erase(j); - } + for (u32 seed = 0; seed < 100; ++seed) { + std::default_random_engine rng{seed}; - Interval new_dropped = filter.drop_index_range(Interval{start_i, end_i}); - EXPECT_LE(new_dropped.lower_bound, start_i) << BATT_INSPECT(seed); - EXPECT_GE(new_dropped.upper_bound, end_i) << BATT_INSPECT(seed); - } + auto [filter, live_items] = build_filter_with_random_drops(num_items, rng); EXPECT_TRUE(filter.check_invariants()); - // Test live_at_index + // Pack the filter and construct a PackedPiecewiseFilter. // - for (usize i = 0; i < num_items; ++i) { - bool expected_live = live_items.count(i) > 0; - bool actual_live = filter.live_at_index(i); - EXPECT_EQ(actual_live, expected_live) << BATT_INSPECT(seed) << BATT_INSPECT(i); - } + PackedFilterData packed_data = pack_in_memory_filter(filter); + PackedPiecewiseFilter packed_filter = get_packed_filter_from_data(packed_data); - // Test live_lower_bound + // Verify the packed filter has the same number of live intervals. // - for (usize i = 0; i < num_items; ++i) { - auto iter = live_items.lower_bound(i); - usize expected = (iter != live_items.end()) ? *iter : num_items; - usize actual = filter.live_lower_bound(i); - EXPECT_EQ(actual, expected) << BATT_INSPECT(seed) << BATT_INSPECT(i); - } + EXPECT_EQ(packed_filter.size(), filter.size()) << BATT_INSPECT(seed); - // Test find_live_range + // Verify the packed filter produces identical intervals. // - for (usize i = 0; i < 100; ++i) { - std::uniform_int_distribution pick_interval_start{0, num_items - 1}; - usize start_i = pick_interval_start(rng); - - std::uniform_int_distribution pick_interval_end{start_i, num_items}; - usize end_i = pick_interval_end(rng); - - auto iter = live_items.lower_bound(start_i); - Interval expected_range; - - if (iter == live_items.end() || *iter >= end_i) { - expected_range = Interval{end_i, end_i}; - } else { - usize first = *iter; - usize last = first + 1; - auto next = std::next(iter); - - while (next != live_items.end() && *next < end_i && *next == last) { - ++last; - ++next; - } - - expected_range = Interval{first, last}; + { + auto mutable_iter = filter.begin(); + auto packed_iter = packed_filter.begin(); + while (mutable_iter != filter.end() && packed_iter != packed_filter.end()) { + EXPECT_EQ(*mutable_iter, *packed_iter) << BATT_INSPECT(seed); + ++mutable_iter; + ++packed_iter; } + EXPECT_EQ(mutable_iter, filter.end()) << BATT_INSPECT(seed); + EXPECT_EQ(packed_iter, packed_filter.end()) << BATT_INSPECT(seed); + } - Interval actual_range = filter.find_live_range(Interval{start_i, end_i}); - EXPECT_EQ(actual_range, expected_range) << BATT_INSPECT(seed); + // Verify queries produce the same results as the in-memory filter. + // + verify_filter_queries(packed_filter, live_items, num_items, seed, rng); + + // Converting packed back to in-memory should produce identical filter. + // + PiecewiseFilter converted_filter{packed_filter}; + EXPECT_TRUE(converted_filter.check_invariants()); + Slice> original_live = filter.live(); + Slice> converted_live = converted_filter.live(); + ASSERT_EQ(original_live.size(), converted_live.size()) << BATT_INSPECT(seed); + for (usize i = 0; i < original_live.size(); ++i) { + EXPECT_EQ(original_live[i], converted_live[i]) << BATT_INSPECT(seed) << BATT_INSPECT(i); } } } @@ -344,6 +334,20 @@ TEST(PiecewiseFilterTest, LiveSubranges) filter_state &= ~drop_mask; } + // Also test via PackedPiecewiseFilter. + // + PackedFilterData packed_data = pack_in_memory_filter(filter); + PackedPiecewiseFilter packed_filter = get_packed_filter_from_data(packed_data); + + const auto packed_query_as_bits = [&](Interval query) { + u64 bits = 0; + packed_filter.live_subranges_of(query) | + batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + for (usize j = 0; j < n_queries; ++j) { const Interval query_interval = pick_interval(rng); const u64 query_mask = mask_from_interval(query_interval); @@ -353,6 +357,13 @@ TEST(PiecewiseFilterTest, LiveSubranges) ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{actual_bits}) << BATT_INSPECT(seed_i) << BATT_INSPECT(query_interval) << BATT_INSPECT(query_interval.size()) << BATT_INSPECT(std::bitset<64>{query_mask}); + + const u64 packed_actual_bits = packed_query_as_bits(query_interval); + + ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{packed_actual_bits}) + << "PackedPiecewiseFilter mismatch: " << BATT_INSPECT(seed_i) + << BATT_INSPECT(query_interval) << BATT_INSPECT(query_interval.size()) + << BATT_INSPECT(std::bitset<64>{query_mask}); } } } diff --git a/src/turtle_kv/util/piecewise_filter.test.hpp b/src/turtle_kv/util/piecewise_filter.test.hpp index 32f71e6..f4058e5 100644 --- a/src/turtle_kv/util/piecewise_filter.test.hpp +++ b/src/turtle_kv/util/piecewise_filter.test.hpp @@ -125,5 +125,123 @@ inline std::pair>> drop_n_disjoint_interv return std::make_pair(dropped_total_size, dropped_ranges); } +struct PackedFilterData { + std::vector values; + bool start_is_live; +}; + +inline PackedFilterData pack_in_memory_filter(const PiecewiseFilter& filter) +{ + PackedFilterData data; + data.start_is_live = false; + Slice> live = filter.live(); + + if (live.empty()) { + return data; + } + + data.start_is_live = (live[0].lower_bound == PiecewiseFilter::kMinLowerBound); + + for (const Interval& range : live) { + if (range.lower_bound != PiecewiseFilter::kMinLowerBound) { + data.values.push_back(range.lower_bound); + } + if (range.upper_bound != PiecewiseFilter::kMaxUpperBound) { + data.values.push_back(range.upper_bound); + } + } + + return data; +} + +inline PackedPiecewiseFilter get_packed_filter_from_data(const PackedFilterData& data) +{ + return PackedPiecewiseFilter{PackedPiecewiseFilterStorage{ + batt::as_const_slice(data.values), data.start_is_live}}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + +struct RandomDropResult { + PiecewiseFilter filter; + std::set live_items; +}; + +inline RandomDropResult build_filter_with_random_drops(u32 num_items, std::default_random_engine& rng) +{ + RandomDropResult result; + for (u32 i = 0; i < num_items; ++i) { + result.live_items.insert(i); + } + + std::uniform_int_distribution pick_num_dropped{100, num_items / 2}; + u32 num_intervals_dropped = pick_num_dropped(rng); + for (u32 i = 0; i < num_intervals_dropped; ++i) { + std::uniform_int_distribution pick_interval_start{0, num_items - 1}; + u32 start_i = pick_interval_start(rng); + + std::uniform_int_distribution pick_interval_end{start_i, num_items}; + u32 end_i = pick_interval_end(rng); + + for (u32 j = start_i; j < end_i; ++j) { + result.live_items.erase(j); + } + + result.filter.drop_index_range(Interval{start_i, end_i}); + } + + return result; +} + +template +inline void verify_filter_queries(const FilterT& filter, + const std::set& live_items, + u32 num_items, + u32 seed, + std::default_random_engine& rng) +{ + for (u32 i = 0; i < num_items; ++i) { + bool expected_live = live_items.count(i) > 0; + bool actual_live = filter.live_at_index(i); + EXPECT_EQ(actual_live, expected_live) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } + + for (u32 i = 0; i < num_items; ++i) { + auto iter = live_items.lower_bound(i); + u32 expected = (iter != live_items.end()) ? *iter : num_items; + u32 actual = filter.live_lower_bound(i); + EXPECT_EQ(actual, expected) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } + + for (u32 i = 0; i < 100; ++i) { + std::uniform_int_distribution pick_interval_start{0, num_items - 1}; + u32 start_i = pick_interval_start(rng); + + std::uniform_int_distribution pick_interval_end{start_i, num_items}; + u32 end_i = pick_interval_end(rng); + + auto iter = live_items.lower_bound(start_i); + Interval expected_range; + + if (iter == live_items.end() || *iter >= end_i) { + expected_range = Interval{end_i, end_i}; + } else { + u32 first = *iter; + u32 last = first + 1; + auto next = std::next(iter); + + while (next != live_items.end() && *next < end_i && *next == last) { + ++last; + ++next; + } + + expected_range = Interval{first, last}; + } + + Interval actual_range = filter.find_live_range(Interval{start_i, end_i}); + EXPECT_EQ(actual_range, expected_range) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } +} + } // namespace testing } // namespace turtle_kv From 93c7c463680421f54b1560a0cdef03e2ee89b14c Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sat, 11 Jul 2026 18:58:04 -0400 Subject: [PATCH 13/20] merge set --- .../tree/merge_set/composite_merge_set.hpp | 22 +++ .../tree/merge_set/empty_merge_set.hpp | 11 ++ .../tree/merge_set/fake_key_value.hpp | 39 ++++++ .../tree/merge_set/fake_key_value_view.hpp | 20 +++ src/turtle_kv/tree/merge_set/fake_leaf.hpp | 49 +++++++ .../tree/merge_set/h_join_merge_set.cpp | 60 +++++++++ .../tree/merge_set/h_join_merge_set.hpp | 21 +++ .../tree/merge_set/in_memory_merge_set.hpp | 60 +++++++++ .../tree/merge_set/in_storage_merge_set.hpp | 50 +++++++ src/turtle_kv/tree/merge_set/merge_set.cpp | 61 +++++++++ src/turtle_kv/tree/merge_set/merge_set.hpp | 49 +++++++ .../tree/merge_set/merge_set.test.cpp | 18 +++ .../tree/merge_set/random_key_value.hpp | 127 ++++++++++++++++++ .../tree/merge_set/v_join_merge_set.hpp | 17 +++ 14 files changed, 604 insertions(+) create mode 100644 src/turtle_kv/tree/merge_set/composite_merge_set.hpp create mode 100644 src/turtle_kv/tree/merge_set/empty_merge_set.hpp create mode 100644 src/turtle_kv/tree/merge_set/fake_key_value.hpp create mode 100644 src/turtle_kv/tree/merge_set/fake_key_value_view.hpp create mode 100644 src/turtle_kv/tree/merge_set/fake_leaf.hpp create mode 100644 src/turtle_kv/tree/merge_set/h_join_merge_set.cpp create mode 100644 src/turtle_kv/tree/merge_set/h_join_merge_set.hpp create mode 100644 src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp create mode 100644 src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp create mode 100644 src/turtle_kv/tree/merge_set/merge_set.cpp create mode 100644 src/turtle_kv/tree/merge_set/merge_set.hpp create mode 100644 src/turtle_kv/tree/merge_set/merge_set.test.cpp create mode 100644 src/turtle_kv/tree/merge_set/random_key_value.hpp create mode 100644 src/turtle_kv/tree/merge_set/v_join_merge_set.hpp diff --git a/src/turtle_kv/tree/merge_set/composite_merge_set.hpp b/src/turtle_kv/tree/merge_set/composite_merge_set.hpp new file mode 100644 index 0000000..1e29afa --- /dev/null +++ b/src/turtle_kv/tree/merge_set/composite_merge_set.hpp @@ -0,0 +1,22 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_COMPOSITE_MERGE_SET_HPP + +#include +#include + +#include +#include + +namespace turtle_kv { +namespace merge_set { + +struct MergeSet; + +struct CompositeMergeSet { + std::vector> components_; + std::string key_lower_bound_; + std::string key_upper_bound_; +}; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/empty_merge_set.hpp b/src/turtle_kv/tree/merge_set/empty_merge_set.hpp new file mode 100644 index 0000000..b80894b --- /dev/null +++ b/src/turtle_kv/tree/merge_set/empty_merge_set.hpp @@ -0,0 +1,11 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_EMPTY_MERGE_SET_HPP + +namespace turtle_kv { +namespace merge_set { + +struct EmptyMergeSet { +}; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/fake_key_value.hpp b/src/turtle_kv/tree/merge_set/fake_key_value.hpp new file mode 100644 index 0000000..d20c00d --- /dev/null +++ b/src/turtle_kv/tree/merge_set/fake_key_value.hpp @@ -0,0 +1,39 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_HPP + +#include + +#include +#include + +namespace turtle_kv { +namespace merge_set { + +struct FakeKeyValue { + std::string key_; + std::string value_; +}; + +inline std::string_view get_key(const FakeKeyValue& view) noexcept +{ + return view.key_; +} + +inline usize packed_sizeof(const FakeKeyValue& kv) noexcept +{ + return kv.key_.size() + kv.value_.size(); +} + +inline std::string get_min_upper_bound(const std::string_view& view) noexcept +{ + std::string s{view}; + if (s.back() == (char)255) { + s += '\0'; + } else { + ++s.back(); + } + return s; +} + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp b/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp new file mode 100644 index 0000000..aa3847f --- /dev/null +++ b/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp @@ -0,0 +1,20 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_VIEW_HPP + +#include + +namespace turtle_kv { +namespace merge_set { + +struct FakeKeyValueView { + std::string_view key_; + std::string_view value_; +}; + +inline const std::string_view& get_key(const FakeKeyValueView& view) noexcept +{ + return view.key_; +} + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/fake_leaf.hpp b/src/turtle_kv/tree/merge_set/fake_leaf.hpp new file mode 100644 index 0000000..df76bc7 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/fake_leaf.hpp @@ -0,0 +1,49 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_HPP + +#include "fake_key_value.hpp" + +#include + +#include +#include +#include + +namespace turtle_kv { +namespace merge_set { + +struct FakeBlock { + std::vector items_; +}; + +struct FakeLeaf { + std::vector block_keys_; + std::vector block_starts_; + std::vector blocks_; + usize block_size_; + + //+++++++++++-+-+--+----- --- -- - - - - + + usize block_count() const noexcept + { + return this->blocks_.size(); + } + + usize block_size() const noexcept + { + return this->block_size_; + } + + usize block_containing_index(usize index) const noexcept + { + BATT_CHECK(!this->block_starts_.empty()); + + const auto second = std::next(this->block_starts_.begin()); + auto iter = std::upper_bound(second, this->block_starts_.end(), index); + + return std::distance(second, iter); + } +}; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp b/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp new file mode 100644 index 0000000..d91ed3d --- /dev/null +++ b/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp @@ -0,0 +1,60 @@ +#include "h_join_merge_set.hpp" +// + +#include "merge_set.hpp" + +namespace turtle_kv { +namespace merge_set { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval HJoinMergeSet::seek_impl( + usize byte_size, + const std::string_view& key_upper_bound) const noexcept +{ + BATT_CHECK(!this->components_.empty()); + + Interval result; + Interval bytes_remaining{byte_size, byte_size}; + + for (const std::unique_ptr& segment : this->components_) { + const Interval segment_byte_size = get_byte_size(*segment); + + if (bytes_remaining.lower_bound != 0) { + if (bytes_remaining.lower_bound > segment_byte_size.upper_bound) { + bytes_remaining.lower_bound -= segment_byte_size.upper_bound; + } else { + result.lower_bound = seek(*segment, bytes_remaining.lower_bound).lower_bound; + bytes_remaining.lower_bound = 0; + } + } + + if (bytes_remaining.upper_bound != 0) { + if (bytes_remaining.upper_bound > segment_byte_size.lower_bound) { + bytes_remaining.upper_bound -= segment_byte_size.lower_bound; + } else { + result.upper_bound = seek(*segment, bytes_remaining.upper_bound).upper_bound; + bytes_remaining.upper_bound = 0; + } + } + + if (bytes_remaining.lower_bound == 0 && bytes_remaining.upper_bound == 0) { + return result; + } + } + + BATT_CHECK_LE(bytes_remaining.lower_bound, bytes_remaining.upper_bound); + + if (bytes_remaining.lower_bound != 0) { + result.lower_bound = key_upper_bound; + } + + if (bytes_remaining.upper_bound != 0) { + result.upper_bound = key_upper_bound; + } + + return result; +} + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp b/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp new file mode 100644 index 0000000..34fcc23 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp @@ -0,0 +1,21 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_H_JOIN_MERGE_SET_HPP + +#include "composite_merge_set.hpp" + +#include + +namespace turtle_kv { +namespace merge_set { + +struct HJoinMergeSet : CompositeMergeSet { + i32 max_depth_; + + //+++++++++++-+-+--+----- --- -- - - - - + + Interval seek_impl(usize byte_size, + const std::string_view& key_upper_bound) const noexcept; +}; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp b/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp new file mode 100644 index 0000000..2ae6306 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp @@ -0,0 +1,60 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_EMPTY_MERGE_SET_HPP + +#include "fake_key_value.hpp" + +#include +#include +#include + +#include +#include + +namespace turtle_kv { +namespace merge_set { + +struct InMemoryMergeSet { + std::shared_ptr> storage_; + Interval key_range_; + Interval index_range_; + //----- --- -- - - - - + std::string key_upper_bound_; + + //+++++++++++-+-+--+----- --- -- - - - - + + Slice live_slice() const noexcept + { + return { + (*this->storage_).data() + this->index_range_.lower_bound, + (*this->storage_).data() + this->index_range_.upper_bound, + }; + } + + Interval seek_impl(u64 byte_size, + const std::string_view& key_upper_bound) const noexcept + { + usize total = 0; + usize index = this->index_range_.lower_bound; + for (const FakeKeyValue& kv : this->live_slice()) { + const usize n = packed_sizeof(kv); + if (total + n > byte_size) { + break; + } + total += n; + ++index; + } + if (index == this->storage_->size()) { + return { + get_key((*this->storage_)[index]), + key_upper_bound, + }; + } + return { + get_key((*this->storage_)[index]), + get_key((*this->storage_)[index + 1]), + }; + } +}; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp b/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp new file mode 100644 index 0000000..4b853c7 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp @@ -0,0 +1,50 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_IN_STORAGE_MERGE_SET_HPP + +#include "fake_leaf.hpp" + +#include + +#include +#include + +#include + +namespace turtle_kv { +namespace merge_set { + +struct InStorageMergeSet { + std::shared_ptr leaf_; + std::string key_upper_bound_; + PiecewiseFilter filter_; + + //+++++++++++-+-+--+----- --- -- - - - - + + Interval seek_impl(u64 byte_size, + const std::string_view& key_upper_bound) const noexcept + { + const usize block_count = this->leaf_->block_count(); + const usize block_size = this->leaf_->block_size(); + + usize live_i = this->filter_.live_lower_bound(0); + usize block_i = this->leaf_->block_containing_index(live_i); + + while (block_i < block_count) { + if (byte_size < block_size) { + break; + } + + byte_size -= block_size; + live_i = this->filter_.live_lower_bound(this->leaf_->block_starts_[block_i + 1]); + block_i = this->leaf_->block_containing_index(live_i); + } + + return { + (block_i + 0 < block_count) ? this->leaf_->block_keys_[block_i + 0] : key_upper_bound, + (block_i + 1 < block_count) ? this->leaf_->block_keys_[block_i + 1] : key_upper_bound, + }; + } +}; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/merge_set.cpp b/src/turtle_kv/tree/merge_set/merge_set.cpp new file mode 100644 index 0000000..8883cdc --- /dev/null +++ b/src/turtle_kv/tree/merge_set/merge_set.cpp @@ -0,0 +1,61 @@ +#include "merge_set.hpp" +// + +#include + +namespace turtle_kv { +namespace merge_set { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval get_depth(const MergeSet& m) noexcept +{ + return batt::case_of( // + m.impl_, + [&](const EmptyMergeSet&) -> Interval { + return {m.depth_, m.depth_}; + }, + [&](const InMemoryMergeSet&) -> Interval { + return {m.depth_, m.depth_ + 1}; + }, + [&](const InStorageMergeSet&) -> Interval { + return {m.depth_, m.depth_ + 1}; + }, + [&](const HJoinMergeSet& h) -> Interval { + return {m.depth_, h.max_depth_}; + }, + [&](const VJoinMergeSet& v) -> Interval { + return {m.depth_, m.depth_ + (i32)v.components_.size()}; + }); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval get_byte_size(const MergeSet& m) noexcept +{ + return m.byte_size_; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval get_key_range(const MergeSet& m) noexcept +{ + return m.key_range_; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval seek(const MergeSet& m, u64 byte_size) noexcept +{ + return batt::case_of( // + m.impl_, + [&](const EmptyMergeSet&) -> Interval { + return m.key_range_; + }, + [&](const auto& impl) -> Interval { + return impl.seek_impl(byte_size, m.key_range_.upper_bound); + }); +} + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/merge_set.hpp b/src/turtle_kv/tree/merge_set/merge_set.hpp new file mode 100644 index 0000000..82b51f1 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/merge_set.hpp @@ -0,0 +1,49 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_MERGE_SET_HPP + +#include "empty_merge_set.hpp" +#include "h_join_merge_set.hpp" +#include "in_memory_merge_set.hpp" +#include "in_storage_merge_set.hpp" +#include "v_join_merge_set.hpp" + +#include + +#include + +namespace turtle_kv { +namespace merge_set { + +struct MergeSet { + using Impl = std::variant< // + EmptyMergeSet, // + InMemoryMergeSet, // + InStorageMergeSet, // + HJoinMergeSet, // + VJoinMergeSet // + >; + + Impl impl_; + i32 depth_; + Interval byte_size_; + Interval key_range_; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval get_depth(const MergeSet& m) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval get_byte_size(const MergeSet& m) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval get_key_range(const MergeSet& m) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval seek(const MergeSet& m, u64 byte_size) noexcept; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/merge_set.test.cpp b/src/turtle_kv/tree/merge_set/merge_set.test.cpp new file mode 100644 index 0000000..2ae4e6d --- /dev/null +++ b/src/turtle_kv/tree/merge_set/merge_set.test.cpp @@ -0,0 +1,18 @@ +#include +// +#include + +#include +#include + +namespace turtle_kv { +namespace merge_set { +namespace { + +TEST(TreeMergeSetMergeSetTest, Test) +{ +} + +} // namespace +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/random_key_value.hpp b/src/turtle_kv/tree/merge_set/random_key_value.hpp new file mode 100644 index 0000000..05ef605 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/random_key_value.hpp @@ -0,0 +1,127 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_RANDOM_KEY_VALUE_HPP + +#include "fake_key_value.hpp" +#include "fake_leaf.hpp" + +#include + +#include + +#include + +namespace turtle_kv { +namespace merge_set { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline std::string random_str(Rng&& rng, PickLen&& pick_len, PickChar&& pick_char) noexcept +{ + const usize len = pick_len(rng); + std::string s(len, '\0'); + for (usize i = 0; i < len; ++i) { + s[i] = (char)pick_char(rng); + } + return s; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline FakeKeyValue random_key_value(Rng&& rng, + PickKeyLen&& pick_key_len, + PickKeyChar&& pick_key_char, + PickValueLen&& pick_value_len, + PickValueChar&& pick_value_char) noexcept +{ + return FakeKeyValue{ + .key_ = random_str(BATT_FORWRD(rng), // + BATT_FORWARD(pick_key_len), + BATT_FORWARD(pick_key_char)), + .value_ = random_str(BATT_FORWRD(rng), // + BATT_FORWARD(pick_value_len), + BATT_FORWARD(pick_value_char)), + }; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline std::vector random_block(usize size_limit, + Rng&& rng, + PickKeyLen&& pick_key_len, + PickKeyChar&& pick_key_char, + PickValueLen&& pick_value_len, + PickValueChar&& pick_value_char) noexcept +{ + std::vector block; + usize size = 0; + for (;;) { + block.emplace_back(random_key_value(BATT_FORWARD(rng), + BATT_FORWARD(pick_key_len), + BATT_FORWARD(pick_key_char), + BATT_FORWARD(pick_value_len), + BATT_FORWARD(pick_value_char))); + size += packed_sizeof(block.back()); + if (size > size_limit) { + block.pop_back(); + break; + } + } + return block; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline FakeLeaf random_leaf(usize block_size_limit, + Rng&& rng, + PickNumBlocks&& pick_num_blocks, + PickKeyLen&& pick_key_len, + PickKeyChar&& pick_key_char, + PickValueLen&& pick_value_len, + PickValueChar&& pick_value_char) noexcept +{ + FakeLeaf leaf; + usize key_count = 0; + + leaf.block_size_ = block_size_limit; + + const usize num_blocks = pick_num_blocks(rng); + for (usize i = 0; i < num_blocks; ++i) { + FakeBlock block{ + .items_ = random_block(block_size_limit, + BATT_FORWARD(rng), + BATT_FORWARD(pick_key_len), + BATT_FORWARD(pick_key_char), + BATT_FORWARD(pick_value_len), + BATT_FORWARD(pick_value_char)), + }; + + leaf.block_starts_.push_back(key_count); + leaf.block_keys_.push_back(std::string{get_key(block.items_.front())}); + leaf.blocks_.emplace_back(std::move(block)); + + key_count += block.items_.size(); + } + leaf.block_starts_.push_back(key_count); + + return leaf; +} + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp b/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp new file mode 100644 index 0000000..a661b24 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp @@ -0,0 +1,17 @@ +#pragma once +#define TURTLE_KV_TREE_MERGE_SET_V_JOIN_MERGE_SET_HPP + +#include "composite_merge_set.hpp" + +namespace turtle_kv { +namespace merge_set { + +struct VJoinMergeSet : CompositeMergeSet { + Interval seek(usize byte_size) const noexcept + { + return {}; + } +}; + +} // namespace merge_set +} // namespace turtle_kv From ac11b8f46b6a3fde3c595c24a16d4c358082cb2b Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Tue, 14 Jul 2026 11:13:32 -0400 Subject: [PATCH 14/20] Start adding blocked leaf scanner --- src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp | 282 ++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp diff --git a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp new file mode 100644 index 0000000..37c0df1 --- /dev/null +++ b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp @@ -0,0 +1,282 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_SCAN_BLOCKED_LEAF_HPP + +#include "packed_blocked_leaf_page.hpp" +#include "packed_blocked_leaf_page.sharded_live_ranges.hpp" +#include "packed_blocked_leaf_page.sharded_live_ranges.ipp" +#include "packed_leaf_block.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Returns a boxed sequence of PackedKeyValueSlotSlice for a blocked leaf page. + * + * If `filter` is nullptr, all items are considered live (a default-constructed PiecewiseFilter is + * used internally). If `min_key` is provided, scanning begins at the first item >= min_key. + * + * On I/O error, the sequence terminates and `status` is set to the error status. + */ +template FilterModelT = SmallVec, 64>> +BoxedSeq scan_blocked_leaf( + llfs::PageId page_id, + usize block_size, + const BasicPiecewiseFilter* filter, + Optional min_key, + llfs::PageLoader& page_loader, + PageSliceStorage& slice_storage, + llfs::PinPageToJob pin_page_to_job, + Status& status) noexcept; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template FilterModelT> +class BlockedLeafScanSeq +{ + public: + using Item = PackedKeyValueSlotSlice; + using Filter = BasicPiecewiseFilter; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit BlockedLeafScanSeq(llfs::PageId page_id, + usize block_size, + const Filter* filter, + Optional min_key, + llfs::PageLoader& page_loader, + PageSliceStorage& slice_storage, + llfs::PinPageToJob pin_page_to_job, + Status& status) noexcept + : page_id_{page_id} + , block_size_{block_size} + , min_key_{min_key} + , page_loader_{page_loader} + , slice_storage_{slice_storage} + , pin_page_to_job_{pin_page_to_job} + , status_{status} + , filter_{filter} + { + this->initialize(); + } + + Optional peek() + { + if (this->done_) { + return None; + } + + if (!this->pending_slice_) { + this->advance(); + } + + return this->pending_slice_; + } + + Optional next() + { + Optional item = this->peek(); + if (item) { + this->pending_slice_ = None; + } + return item; + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + StatusOr load_shard(const Interval& shard_interval, + llfs::LruPriority lru_priority) noexcept + { + llfs::PageCache& page_cache = *this->page_loader_.page_cache(); + + Optional shard_page_id = + page_cache.page_shard_id_for(this->page_id_, shard_interval); + + if (!shard_page_id) { + return {batt::StatusCode::kUnavailable}; + } + + const llfs::PinnedPage* existing = this->slice_storage_.find_pinned_page(*shard_page_id); + if (existing) { + return ConstBuffer{existing->raw_data(), shard_interval.size()}; + } + + BATT_ASSIGN_OK_RESULT( + llfs::PinnedPage pinned_shard, + this->page_loader_.load_page( + *shard_page_id, + llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), + this->pin_page_to_job_, + llfs::OkIfNotFound{false}, + lru_priority})); + + const void* raw_data = pinned_shard.raw_data(); + this->slice_storage_.insert_pinned_page(std::move(pinned_shard)); + + return ConstBuffer{raw_data, shard_interval.size()}; + } + + void initialize() noexcept + { + StatusOr header_buffer = + this->load_shard(Interval{0, this->block_size_}, + llfs::LruPriority{kTrieIndexLruPriority}); + + if (!header_buffer.ok()) { + this->status_ = header_buffer.status(); + this->done_ = true; + return; + } + + this->leaf_ = &PackedBlockedLeafPage::view_of(*header_buffer); + + const usize actual_header_size = this->leaf_->min_header_shard_size(); + if (actual_header_size > this->block_size_) { + StatusOr second_shard = + this->load_shard(Interval{this->block_size_, 2 * this->block_size_}, + llfs::LruPriority{kTrieIndexLruPriority}); + + if (!second_shard.ok()) { + this->status_ = second_shard.status(); + this->done_ = true; + return; + } + } + + const u32 item_count = BATT_CHECKED_CAST(u32, this->leaf_->item_count()); + u32 first_item = 0; + + if (this->min_key_) { + const usize start_block_i = this->leaf_->find_block_index_containing_key(*this->min_key_); + first_item = (*this->leaf_->block_starting_item)[start_block_i]; + } + + if (this->filter_) { + this->live_ranges_.emplace( + this->leaf_->sharded_live_ranges(*this->filter_, Interval{first_item, item_count})); + } else { + this->live_ranges_.emplace( + this->leaf_->sharded_live_ranges(this->pass_through_filter_, + Interval{first_item, item_count})); + } + } + + void advance() noexcept + { + for (;;) { + auto live_pair = this->live_ranges_->next(); + if (!live_pair) { + this->done_ = true; + return; + } + + const auto [block_index, live_item_range] = *live_pair; + + const usize block_offset = this->leaf_->block_page_offset(block_index); + StatusOr block_buffer = + this->load_shard(Interval{block_offset, block_offset + this->block_size_}, + llfs::LruPriority{kLeafLruPriority}); + + if (!block_buffer.ok()) { + this->status_ = block_buffer.status(); + this->done_ = true; + return; + } + + const PackedLeafBlock& block = PackedLeafBlock::view_of(*block_buffer); + + const Interval block_item_range = this->leaf_->item_index_range_of_block(block_index); + const usize local_begin = live_item_range.lower_bound - block_item_range.lower_bound; + const usize local_end = live_item_range.upper_bound - block_item_range.lower_bound; + + const PackedKeyValueSlotPtr* slice_begin = block.items_begin() + local_begin; + const PackedKeyValueSlotPtr* slice_end = block.items_begin() + local_end; + + if (this->min_key_ && !this->min_key_applied_) { + this->min_key_applied_ = true; + slice_begin = std::lower_bound( + slice_begin, slice_end, *this->min_key_, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); + } + + if (slice_begin == slice_end) { + continue; + } + + this->pending_slice_ = PackedKeyValueSlotSlice{as_slice(slice_begin, slice_end)}; + return; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + + llfs::PageId page_id_; + usize block_size_; + Optional min_key_; + llfs::PageLoader& page_loader_; + PageSliceStorage& slice_storage_; + llfs::PinPageToJob pin_page_to_job_; + Status& status_; + + const Filter* filter_; + PiecewiseFilter pass_through_filter_; + + bool done_ = false; + bool min_key_applied_ = false; + + const PackedBlockedLeafPage* leaf_ = nullptr; + + using LiveRanges = PackedBlockedLeafPage::ShardedLiveRanges; + Optional live_ranges_; + + Optional pending_slice_; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +BoxedSeq scan_blocked_leaf( + llfs::PageId page_id, + usize block_size, + const BasicPiecewiseFilter* filter, + Optional min_key, + llfs::PageLoader& page_loader, + PageSliceStorage& slice_storage, + llfs::PinPageToJob pin_page_to_job, + Status& status) noexcept +{ + return BlockedLeafScanSeq{page_id, block_size, filter, min_key, + page_loader, slice_storage, pin_page_to_job, status} + | seq::boxed(); +} + +} // namespace turtle_kv From b236c2f909a1e5701474c2097de8965d60934816 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Tue, 14 Jul 2026 11:44:19 -0400 Subject: [PATCH 15/20] merge_set wip --- src/turtle_kv/core/key_range.hpp | 22 ++++ .../tree/merge_set/composite_merge_set.cpp | 50 ++++++++ .../tree/merge_set/composite_merge_set.hpp | 12 ++ .../tree/merge_set/fake_key_value.hpp | 4 +- .../tree/merge_set/fake_key_value_view.hpp | 6 +- .../tree/merge_set/h_join_merge_set.cpp | 118 ++++++++++++++++-- .../tree/merge_set/h_join_merge_set.hpp | 56 ++++++++- .../tree/merge_set/in_memory_merge_set.hpp | 16 ++- .../tree/merge_set/in_storage_merge_set.hpp | 12 +- src/turtle_kv/tree/merge_set/merge_set.cpp | 55 +++++++- src/turtle_kv/tree/merge_set/merge_set.hpp | 54 +++++++- .../tree/merge_set/v_join_merge_set.cpp | 43 +++++++ .../tree/merge_set/v_join_merge_set.hpp | 25 +++- 13 files changed, 441 insertions(+), 32 deletions(-) create mode 100644 src/turtle_kv/tree/merge_set/composite_merge_set.cpp create mode 100644 src/turtle_kv/tree/merge_set/v_join_merge_set.cpp diff --git a/src/turtle_kv/core/key_range.hpp b/src/turtle_kv/core/key_range.hpp index 2e1c198..5a9b96f 100644 --- a/src/turtle_kv/core/key_range.hpp +++ b/src/turtle_kv/core/key_range.hpp @@ -43,6 +43,8 @@ inline CInterval get_key_range(const T& has_key_view) }; } +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// struct ExtendedKeyRangeOrder : llfs::KeyRangeOrder { //+++++++++++-+-+--+----- --- -- - - - - @@ -74,4 +76,24 @@ struct ExtendedKeyRangeOrder : llfs::KeyRangeOrder { } }; +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct KeyLowerBoundOrder : KeyOrder { + template + bool operator()(const L& l, const R& r) const + { + return KeyOrder::operator()(get_key_range(l).lower_bound, get_key_range(r).lower_bound); + } +}; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct KeyUpperBoundOrder : KeyOrder { + template + bool operator()(const L& l, const R& r) const + { + return KeyOrder::operator()(get_key_range(l).upper_bound, get_key_range(r).upper_bound); + } +}; + } // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/composite_merge_set.cpp b/src/turtle_kv/tree/merge_set/composite_merge_set.cpp new file mode 100644 index 0000000..47967b7 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/composite_merge_set.cpp @@ -0,0 +1,50 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include "composite_merge_set.hpp" +// +#include "h_join_merge_set.hpp" +#include "merge_set.hpp" +#include "v_join_merge_set.hpp" + +namespace turtle_kv { +namespace merge_set { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +void CompositeMergeSet::add(MergeSet&& src) noexcept +{ + this->components_.emplace_back(std::make_unique(std::move(src))); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template T> +T clone_impl(const T& src, batt::StaticType) noexcept +{ + T dst; + + dst.key_lower_bound_ = src.key_lower_bound_; + dst.key_upper_bound_ = src.key_upper_bound_; + dst.components_.reserve(src.components_.size()); + + for (const std::unique_ptr& component : src.components_) { + dst.components_.emplace_back(std::make_unique(clone(*component))); + } + + return dst; +} + +template HJoinMergeSet clone_impl(const HJoinMergeSet& src, + batt::StaticType) noexcept; + +template VJoinMergeSet clone_impl(const VJoinMergeSet& src, + batt::StaticType) noexcept; + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/composite_merge_set.hpp b/src/turtle_kv/tree/merge_set/composite_merge_set.hpp index 1e29afa..077a08f 100644 --- a/src/turtle_kv/tree/merge_set/composite_merge_set.hpp +++ b/src/turtle_kv/tree/merge_set/composite_merge_set.hpp @@ -4,7 +4,10 @@ #include #include +#include + #include +#include #include namespace turtle_kv { @@ -16,7 +19,16 @@ struct CompositeMergeSet { std::vector> components_; std::string key_lower_bound_; std::string key_upper_bound_; + + //+++++++++++-+-+--+----- --- -- - - - - + + void add(MergeSet&& src) noexcept; }; +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template T> +T clone_impl(const T& src, batt::StaticType = {}) noexcept; + } // namespace merge_set } // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/fake_key_value.hpp b/src/turtle_kv/tree/merge_set/fake_key_value.hpp index d20c00d..0e337ee 100644 --- a/src/turtle_kv/tree/merge_set/fake_key_value.hpp +++ b/src/turtle_kv/tree/merge_set/fake_key_value.hpp @@ -1,6 +1,8 @@ #pragma once #define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_HPP +#include + #include #include @@ -14,7 +16,7 @@ struct FakeKeyValue { std::string value_; }; -inline std::string_view get_key(const FakeKeyValue& view) noexcept +inline KeyView get_key(const FakeKeyValue& view) noexcept { return view.key_; } diff --git a/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp b/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp index aa3847f..60f4be3 100644 --- a/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp +++ b/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp @@ -1,17 +1,19 @@ #pragma once #define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_VIEW_HPP +#include + #include namespace turtle_kv { namespace merge_set { struct FakeKeyValueView { - std::string_view key_; + KeyView key_; std::string_view value_; }; -inline const std::string_view& get_key(const FakeKeyValueView& view) noexcept +inline const KeyView& get_key(const FakeKeyValueView& view) noexcept { return view.key_; } diff --git a/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp b/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp index d91ed3d..87a968f 100644 --- a/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp +++ b/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp @@ -3,22 +3,43 @@ #include "merge_set.hpp" +#include + namespace turtle_kv { namespace merge_set { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Interval HJoinMergeSet::seek_impl( - usize byte_size, - const std::string_view& key_upper_bound) const noexcept +CInterval h_get_byte_size_impl(const std::vector>& segments) noexcept { - BATT_CHECK(!this->components_.empty()); + if (segments.empty()) { + return {0, 0}; + } - Interval result; - Interval bytes_remaining{byte_size, byte_size}; + CInterval result{0, 0}; - for (const std::unique_ptr& segment : this->components_) { - const Interval segment_byte_size = get_byte_size(*segment); + for (const std::unique_ptr& p_component : segments) { + const CInterval component_byte_size = get_byte_size(*p_component); + result.lower_bound += component_byte_size.lower_bound; + result.upper_bound += component_byte_size.upper_bound; + } + + return result; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval h_seek_impl(const std::vector>& segments, + usize byte_size, + const KeyView& key_upper_bound) noexcept +{ + BATT_CHECK(!segments.empty()); + + Interval result; + CInterval bytes_remaining{byte_size, byte_size}; + + for (const std::unique_ptr& segment : segments) { + const CInterval segment_byte_size = get_byte_size(*segment); if (bytes_remaining.lower_bound != 0) { if (bytes_remaining.lower_bound > segment_byte_size.upper_bound) { @@ -56,5 +77,86 @@ Interval HJoinMergeSet::seek_impl( return result; } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +std::tuple HJoinMergeSet::split_impl(const MergeSet& m, + const KeyView& split_key) const noexcept +{ + // Find the segment where the split_key lives and split that one. + // + auto eq_it = std::equal_range(this->components_.begin(), + this->components_.end(), + split_key, + ExtendedKeyRangeOrder{}); + + // Base cases. + // + if (eq_it.first == this->components_.end()) { + return {clone(m), MergeSet{}}; + } + + if (eq_it.second == this->components_.begin()) { + return {MergeSet{}, clone(m)}; + } + + HJoinMergeSet before_split_impl; + HJoinMergeSet after_split_impl; + + before_split_impl.key_lower_bound_ = this->key_lower_bound_; + before_split_impl.key_upper_bound_ = split_key; + before_split_impl.max_depth_ = m.depth_; + + after_split_impl.key_lower_bound_ = split_key; + after_split_impl.key_upper_bound_ = this->key_upper_bound_; + after_split_impl.max_depth_ = m.depth_; + + // Copy all segments that are definitely before `split_key` to `before_split`. + // + std::for_each( // + this->components_.begin(), + eq_it.first, + [&before_split_impl](const std::unique_ptr& p_segment) { + before_split_impl.add(clone(*p_segment)); + }); + + // Handle the matched range if non-empty. + // + if (eq_it.first != eq_it.second) { + // If the split key is *not* at the exact start of the matched segment, then split that segment + // and assign the resulting parts accordingly. + // + if (get_key_range(**eq_it.first).lower_bound < split_key) { + MergeSet middle_lower, middle_upper; + + std::tie(middle_lower, middle_upper) = split(**eq_it.first, split_key); + + before_split_impl.add(std::move(middle_lower)); + after_split_impl.add(std::move(middle_upper)); + + } else { + // The split_key is exactly the start of the middle segment; assign it in whole to + // `after_split`. + // + after_split_impl.add(clone(**eq_it.first)); + } + } + + // Copy all segments that are definitely after `split_key` to `after_split`. + // + std::for_each( // + eq_it.second, + this->components_.end(), + [&after_split_impl](const std::unique_ptr& p_segment) { + after_split_impl.add(clone(*p_segment)); + }); + + // Form the output sets. + // + return { + MergeSet{std::move(before_split_impl), m.depth_}, + MergeSet{std::move(after_split_impl), m.depth_}, + }; +} + } // namespace merge_set } // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp b/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp index 34fcc23..f12c335 100644 --- a/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp +++ b/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp @@ -1,20 +1,72 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #pragma once #define TURTLE_KV_TREE_MERGE_SET_H_JOIN_MERGE_SET_HPP #include "composite_merge_set.hpp" +#include + #include namespace turtle_kv { namespace merge_set { +struct MergeSet; + +CInterval get_byte_size(const MergeSet& m) noexcept; +Interval get_depth(const MergeSet& m) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +CInterval h_get_byte_size_impl( + const std::vector>& segments) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval h_seek_impl(const std::vector>& segments, + usize byte_size, + const KeyView& key_upper_bound) noexcept; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// struct HJoinMergeSet : CompositeMergeSet { i32 max_depth_; //+++++++++++-+-+--+----- --- -- - - - - - Interval seek_impl(usize byte_size, - const std::string_view& key_upper_bound) const noexcept; + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + void add(MergeSet&& src) noexcept + { + this->max_depth_ = std::max(this->max_depth_, get_depth(src).upper_bound); + this->CompositeMergeSet::add(std::move(src)); + } + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + Interval seek_impl(usize byte_size, const KeyView& key_upper_bound) const noexcept + { + return h_seek_impl(this->components_, byte_size, key_upper_bound); + } + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + CInterval get_byte_size_impl() const noexcept + { + return h_get_byte_size_impl(this->components_); + } + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + std::tuple split_impl(const MergeSet& m, + const KeyView& split_key) const noexcept; }; } // namespace merge_set diff --git a/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp b/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp index 2ae6306..81b547b 100644 --- a/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp +++ b/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp @@ -13,15 +13,19 @@ namespace turtle_kv { namespace merge_set { +struct MergeSet; + struct InMemoryMergeSet { std::shared_ptr> storage_; - Interval key_range_; + Interval key_range_; Interval index_range_; //----- --- -- - - - - std::string key_upper_bound_; //+++++++++++-+-+--+----- --- -- - - - - + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // Slice live_slice() const noexcept { return { @@ -30,8 +34,9 @@ struct InMemoryMergeSet { }; } - Interval seek_impl(u64 byte_size, - const std::string_view& key_upper_bound) const noexcept + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + Interval seek_impl(u64 byte_size, const KeyView& key_upper_bound) const noexcept { usize total = 0; usize index = this->index_range_.lower_bound; @@ -54,6 +59,11 @@ struct InMemoryMergeSet { get_key((*this->storage_)[index + 1]), }; } + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + std::tuple split_impl(const MergeSet& m, + const KeyView& split_key) const noexcept; }; } // namespace merge_set diff --git a/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp b/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp index 4b853c7..341b8f6 100644 --- a/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp +++ b/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp @@ -13,6 +13,8 @@ namespace turtle_kv { namespace merge_set { +struct MergeSet; + struct InStorageMergeSet { std::shared_ptr leaf_; std::string key_upper_bound_; @@ -20,8 +22,9 @@ struct InStorageMergeSet { //+++++++++++-+-+--+----- --- -- - - - - - Interval seek_impl(u64 byte_size, - const std::string_view& key_upper_bound) const noexcept + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + Interval seek_impl(u64 byte_size, const KeyView& key_upper_bound) const noexcept { const usize block_count = this->leaf_->block_count(); const usize block_size = this->leaf_->block_size(); @@ -44,6 +47,11 @@ struct InStorageMergeSet { (block_i + 1 < block_count) ? this->leaf_->block_keys_[block_i + 1] : key_upper_bound, }; } + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + std::tuple split_impl(const MergeSet& m, + const KeyView& split_key) const noexcept; }; } // namespace merge_set diff --git a/src/turtle_kv/tree/merge_set/merge_set.cpp b/src/turtle_kv/tree/merge_set/merge_set.cpp index 8883cdc..1e33fde 100644 --- a/src/turtle_kv/tree/merge_set/merge_set.cpp +++ b/src/turtle_kv/tree/merge_set/merge_set.cpp @@ -6,6 +6,37 @@ namespace turtle_kv { namespace merge_set { +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +MergeSet clone(const MergeSet& m) noexcept +{ + MergeSet m2; + + m2.impl_ = batt::case_of( // + m.impl_, + [&](const EmptyMergeSet& e) -> MergeSet::Impl { + return e; + }, + [&](const InMemoryMergeSet& i) -> MergeSet::Impl { + return i; + }, + [&](const InStorageMergeSet& i) -> MergeSet::Impl { + return i; + }, + [&](const HJoinMergeSet& h) -> MergeSet::Impl { + return clone_impl(h); + }, + [&](const VJoinMergeSet& v) -> MergeSet::Impl { + return clone_impl(v); + }); + + m2.depth_ = m.depth_; + m2.byte_size_ = m.byte_size_; + m2.key_range_ = m.key_range_; + + return m2; +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // Interval get_depth(const MergeSet& m) noexcept @@ -31,31 +62,45 @@ Interval get_depth(const MergeSet& m) noexcept //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Interval get_byte_size(const MergeSet& m) noexcept +CInterval get_byte_size(const MergeSet& m) noexcept { return m.byte_size_; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Interval get_key_range(const MergeSet& m) noexcept +Interval get_key_range(const MergeSet& m) noexcept { return m.key_range_; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Interval seek(const MergeSet& m, u64 byte_size) noexcept +Interval seek(const MergeSet& m, u64 byte_size) noexcept { return batt::case_of( // m.impl_, - [&](const EmptyMergeSet&) -> Interval { + [&](const EmptyMergeSet&) -> Interval { return m.key_range_; }, - [&](const auto& impl) -> Interval { + [&](const auto& impl) -> Interval { return impl.seek_impl(byte_size, m.key_range_.upper_bound); }); } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +std::tuple split(const MergeSet& m, const KeyView& split_key) noexcept +{ + return batt::case_of( // + m.impl_, + [&](const EmptyMergeSet& e) -> std::tuple { + return std::tuple{e, e}; + }, + [&](const auto& impl) -> std::tuple { + return impl.split_impl(m, split_key); + }); +} + } // namespace merge_set } // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/merge_set.hpp b/src/turtle_kv/tree/merge_set/merge_set.hpp index 82b51f1..bbd29f4 100644 --- a/src/turtle_kv/tree/merge_set/merge_set.hpp +++ b/src/turtle_kv/tree/merge_set/merge_set.hpp @@ -7,8 +7,11 @@ #include "in_storage_merge_set.hpp" #include "v_join_merge_set.hpp" +#include + #include +#include #include namespace turtle_kv { @@ -23,27 +26,68 @@ struct MergeSet { VJoinMergeSet // >; + CInterval byte_size_; + Interval key_range_; Impl impl_; i32 depth_; - Interval byte_size_; - Interval key_range_; + + //+++++++++++-+-+--+----- --- -- - - - - + + MergeSet() noexcept : byte_size_{0, 0}, key_range_{{}, {}}, impl_{EmptyMergeSet{}}, depth_{0} + { + } + + explicit MergeSet(const EmptyMergeSet&) noexcept : MergeSet{} + { + } + + explicit MergeSet(HJoinMergeSet&& impl, i32 depth) noexcept + : byte_size_{impl.get_byte_size_impl()} + , key_range_{impl.key_lower_bound_, impl.key_upper_bound_} + , impl_{std::move(impl)} + , depth_{depth} + { + } + + MergeSet(const MergeSet&) = delete; + MergeSet& operator=(const MergeSet&) = delete; + + MergeSet(MergeSet&&) = default; + MergeSet& operator=(MergeSet&&) = default; }; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // +MergeSet clone(const MergeSet& m) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Returns the level depth range of the passed MergeSet. + */ Interval get_depth(const MergeSet& m) noexcept; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Interval get_byte_size(const MergeSet& m) noexcept; +/** \brief Returns the minimum known bounding range of merged byte size for the passed set. + */ +CInterval get_byte_size(const MergeSet& m) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Returns the minimum known bounding key range of the passed set. + */ +Interval get_key_range(const MergeSet& m) noexcept; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Interval get_key_range(const MergeSet& m) noexcept; +/** \brief Returns the minimum interval in which lies the true upper bound corresponding to the + * specified number of bytes (as measured from the beginning of the final merged version of `m`). + */ +Interval seek(const MergeSet& m, u64 byte_size) noexcept; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Interval seek(const MergeSet& m, u64 byte_size) noexcept; +std::tuple split(const MergeSet& m, const KeyView& split_key) noexcept; } // namespace merge_set } // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/v_join_merge_set.cpp b/src/turtle_kv/tree/merge_set/v_join_merge_set.cpp new file mode 100644 index 0000000..8e79394 --- /dev/null +++ b/src/turtle_kv/tree/merge_set/v_join_merge_set.cpp @@ -0,0 +1,43 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include "v_join_merge_set.hpp" +// + +#include "h_join_merge_set.hpp" +#include "merge_set.hpp" + +namespace turtle_kv { +namespace merge_set { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Interval VJoinMergeSet::seek_impl(usize byte_size, + const KeyView& key_upper_bound) const noexcept +{ + Interval result; + + const std::vector>& levels = this->components_; + + // At the two extremes, the levels could have all the same keys or share none. In the former + // case, we need to take the min/max of the level-wise seek result, and in the latter, we do the + // same thing as for HJoinMergeSet. + // + result = h_seek_impl(levels, byte_size, key_upper_bound); + + for (const std::unique_ptr& level : levels) { + Interval level_result = seek(*level, byte_size); + result.lower_bound = std::min(result.lower_bound, level_result.lower_bound); + result.upper_bound = std::max(result.upper_bound, level_result.upper_bound); + } + + return result; +} + +} // namespace merge_set +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp b/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp index a661b24..bc09100 100644 --- a/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp +++ b/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp @@ -1,16 +1,33 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #pragma once #define TURTLE_KV_TREE_MERGE_SET_V_JOIN_MERGE_SET_HPP #include "composite_merge_set.hpp" +#include + +#include +#include + namespace turtle_kv { namespace merge_set { struct VJoinMergeSet : CompositeMergeSet { - Interval seek(usize byte_size) const noexcept - { - return {}; - } + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + Interval seek_impl(usize byte_size, const KeyView& key_upper_bound) const noexcept; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + // + std::tuple split_impl(const MergeSet& m, + const KeyView& split_key) const noexcept; }; } // namespace merge_set From 37b3d4e4b7ff5f1189e8760ae387831e92ee84cf Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Tue, 14 Jul 2026 14:17:31 -0400 Subject: [PATCH 16/20] Add sketch of new scan_blocked_leaf design. --- src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp | 82 +++++++++++++++---- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp index 37c0df1..1427fd9 100644 --- a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp +++ b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp @@ -130,12 +130,11 @@ class BlockedLeafScanSeq BATT_ASSIGN_OK_RESULT( llfs::PinnedPage pinned_shard, - this->page_loader_.load_page( - *shard_page_id, - llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), - this->pin_page_to_job_, - llfs::OkIfNotFound{false}, - lru_priority})); + this->page_loader_.load_page(*shard_page_id, + llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), + this->pin_page_to_job_, + llfs::OkIfNotFound{false}, + lru_priority})); const void* raw_data = pinned_shard.raw_data(); this->slice_storage_.insert_pinned_page(std::move(pinned_shard)); @@ -221,11 +220,13 @@ class BlockedLeafScanSeq if (this->min_key_ && !this->min_key_applied_) { this->min_key_applied_ = true; - slice_begin = std::lower_bound( - slice_begin, slice_end, *this->min_key_, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; - }); + slice_begin = + std::lower_bound(slice_begin, + slice_end, + *this->min_key_, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); } if (slice_begin == slice_end) { @@ -274,9 +275,62 @@ BoxedSeq scan_blocked_leaf( llfs::PinPageToJob pin_page_to_job, Status& status) noexcept { - return BlockedLeafScanSeq{page_id, block_size, filter, min_key, - page_loader, slice_storage, pin_page_to_job, status} - | seq::boxed(); + return BlockedLeafScanSeq{page_id, + block_size, + filter, + min_key, + page_loader, + slice_storage, + pin_page_to_job, + status} | + seq::boxed(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT, typename BlockLoaderT> +/*BoxedSeq*/ +auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, + BlockLoaderT* block_loader, + const BasicPiecewiseFilter& filter, + const Interval& key_range) noexcept +{ + const Interval index_range = + packed_leaf->get_block_aligned_index_range_for_key_range(key_range); + + return packed_leaf->sharded_live_ranges(filter, index_range) // + | batt::seq::filter_map([block_loader](const std::pair>& params) + -> Optional> { + // Stage 1: filter + // + const Interval& live_item_range = params.second; + if (live_item_range.empty()) { + return None; + } + + // Stage 2: map + // + const u32 block_index = params.first; + StatusOr block = block_loader->load_block(block_index); + if (!block.ok()) { + return block.status(); + } + + // Convert index range -> slot slice + // + PackedKeyValueSlotSlice slice = + packed_leaf->get_slice_within_block(block_index, block, live_item_range); + + // Stage 3 + // Trim the slice down to `key_range`. + // + if ("this is the first or last block in the sequence") { + // trim the slice + } + + return slice; + }) // + ; } } // namespace turtle_kv From 0e54e301c98cb2df849e395c121bbb96a9a768cc Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Tue, 28 Jul 2026 10:56:09 -0400 Subject: [PATCH 17/20] Add BlockedLeafPageLoader, refactor scan_blocked_leaf and pack_blocked_leafPage --- .../tree/leaf/blocked_leaf_page_loader.cpp | 123 +++++++ .../tree/leaf/blocked_leaf_page_loader.hpp | 72 ++++ .../tree/leaf/packed_blocked_leaf_page.hpp | 22 +- .../tree/leaf/packed_blocked_leaf_page.ipp | 226 ++++++++---- ..._blocked_leaf_page.sharded_live_ranges.hpp | 8 +- ..._blocked_leaf_page.sharded_live_ranges.ipp | 8 +- .../leaf/packed_blocked_leaf_page.test.cpp | 11 +- src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp | 346 +++--------------- 8 files changed, 443 insertions(+), 373 deletions(-) create mode 100644 src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp create mode 100644 src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp new file mode 100644 index 0000000..1c09844 --- /dev/null +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp @@ -0,0 +1,123 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include "blocked_leaf_page_loader.hpp" + +#include "packed_leaf_block.ipp" + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +BlockedLeafPageLoader::BlockedLeafPageLoader(llfs::PageLoader& page_loader, + PageSliceStorage& slice_storage, + llfs::PinPageToJob pin_page_to_job, + usize block_size) noexcept + : page_loader_{page_loader} + , slice_storage_{slice_storage} + , pin_page_to_job_{pin_page_to_job} + , block_size_{block_size} + , page_id_{} +{ +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +StatusOr BlockedLeafPageLoader::set_page( + llfs::PageId page_id) noexcept +{ + this->page_id_ = page_id; + this->leaf_ = nullptr; + this->cache_.clear(); + + // Load the first shard to access the fixed header fields. + // + BATT_ASSIGN_OK_RESULT( + ConstBuffer header_buffer, + this->load_shard(Interval{0, this->block_size_}, + llfs::LruPriority{kTrieIndexLruPriority})); + + this->leaf_ = &PackedBlockedLeafPage::view_of(header_buffer); + + // Load additional shards until the full header metadata is covered. + // + const usize header_size = this->leaf_->min_header_shard_size(); + usize loaded = this->block_size_; + + while (loaded < header_size) { + const usize next_end = std::min(loaded + this->block_size_, header_size); + BATT_ASSIGN_OK_RESULT( + ConstBuffer shard_buffer, + this->load_shard(Interval{loaded, next_end}, + llfs::LruPriority{kTrieIndexLruPriority})); + loaded = next_end; + (void)shard_buffer; + } + + this->cache_.resize(this->leaf_->block_count(), None); + + return this->leaf_; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +StatusOr BlockedLeafPageLoader::load_block(u32 block_index) noexcept +{ + BATT_CHECK_NE(this->leaf_, nullptr); + BATT_CHECK_LT(block_index, this->cache_.size()); + + if (this->cache_[block_index]) { + return &PackedLeafBlock::view_of(*this->cache_[block_index]); + } + + const usize block_offset = this->leaf_->block_page_offset(block_index); + BATT_ASSIGN_OK_RESULT( + ConstBuffer block_buffer, + this->load_shard(Interval{block_offset, block_offset + this->block_size_}, + llfs::LruPriority{kLeafLruPriority})); + + this->cache_[block_index] = block_buffer; + + return &PackedLeafBlock::view_of(block_buffer); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +StatusOr BlockedLeafPageLoader::load_shard( + const Interval& shard_interval, + llfs::LruPriority lru_priority) noexcept +{ + llfs::PageCache& page_cache = *this->page_loader_.page_cache(); + + Optional shard_page_id = + page_cache.page_shard_id_for(this->page_id_, shard_interval); + + if (!shard_page_id) { + return {batt::StatusCode::kUnavailable}; + } + + const llfs::PinnedPage* existing = this->slice_storage_.find_pinned_page(*shard_page_id); + if (existing) { + return ConstBuffer{existing->raw_data(), (usize)shard_interval.size()}; + } + + BATT_ASSIGN_OK_RESULT( + llfs::PinnedPage pinned_shard, + this->page_loader_.load_page(*shard_page_id, + llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), + this->pin_page_to_job_, + llfs::OkIfNotFound{false}, + lru_priority})); + + const void* raw_data = pinned_shard.raw_data(); + this->slice_storage_.insert_pinned_page(std::move(pinned_shard)); + + return ConstBuffer{raw_data, (usize)shard_interval.size()}; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp new file mode 100644 index 0000000..130c950 --- /dev/null +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp @@ -0,0 +1,72 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_BLOCKED_LEAF_PAGE_LOADER_HPP + +#include "packed_blocked_leaf_page.hpp" +#include "packed_leaf_block.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Loads header and blocks from a PackedBlockedLeafPage via sharded page views. + * + * Bound to one page at a time. Call `set_page` to move to a new page. + */ +class BlockedLeafPageLoader +{ + public: + explicit BlockedLeafPageLoader(llfs::PageLoader& page_loader, + PageSliceStorage& slice_storage, + llfs::PinPageToJob pin_page_to_job, + usize block_size) noexcept; + + /** \brief Loads the header shard for the given page, clears the block cache, and returns + * the validated PackedBlockedLeafPage pointer. + */ + StatusOr set_page(llfs::PageId page_id) noexcept; + + /** \brief Returns the current leaf pointer, or nullptr if no page is set. + */ + const PackedBlockedLeafPage* leaf() const noexcept + { + return this->leaf_; + } + + /** \brief Loads the block at the given index. Returns a cached result on subsequent calls. + */ + StatusOr load_block(u32 block_index) noexcept; + + private: + StatusOr load_shard(const Interval& shard_interval, + llfs::LruPriority lru_priority) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + llfs::PageLoader& page_loader_; + PageSliceStorage& slice_storage_; + llfs::PinPageToJob pin_page_to_job_; + usize block_size_; + + llfs::PageId page_id_; + const PackedBlockedLeafPage* leaf_ = nullptr; + SmallVec, 256> cache_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp index b043193..bf95e7c 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp @@ -43,15 +43,22 @@ namespace turtle_kv { // struct PackedBlockedLeafPage; +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +struct PackedLeafResult { + PackedBlockedLeafPage* leaf; + usize items_packed; +}; + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // /** \brief Packs a blocked leaf page with the passed block size, containing the passed key/value - * pairs, into the passed buffer. + * pairs, into the passed buffer. Packs as many items as will fit; returns the number packed. */ template -StatusOr pack_blocked_leaf_page(const usize block_size, - const ItemRangeT& src_items, - const MutableBuffer& dst_buffer) noexcept; +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept; //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- // @@ -286,6 +293,13 @@ struct PackedBlockedLeafPage // ShardedLiveRanges sharded_live_ranges( const BasicPiecewiseFilter& filter, const Interval& subrange) const noexcept; + + Interval get_block_aligned_index_range_for_key_range( + const Interval& key_range) const noexcept; + + PackedKeyValueSlotSlice get_slice_within_block(u32 block_index, + const PackedLeafBlock* block, + const Interval& live_item_range) const noexcept; }; static_assert(sizeof(PackedBlockedLeafPage) == 32); diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp index 9d415f0..fe8a557 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp @@ -24,57 +24,154 @@ namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -StatusOr pack_blocked_leaf_page(const usize block_size, - const ItemRangeT& src_items, - const MutableBuffer& dst_buffer) noexcept +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept { - const usize item_count = std::size(src_items); + //+++++++++++-+-+--+----- --- -- - - - - + // Reserve space for the fixed header. + // + MutableBuffer dst_remaining = dst_buffer; + dst_remaining += sizeof(llfs::PackedPageHeader); + + auto* leaf_header = static_cast(dst_remaining.data()); + dst_remaining += sizeof(PackedBlockedLeafPage); + { + leaf_header->magic = PackedBlockedLeafPage::kMagic; + leaf_header->total_packed_size = 0; + leaf_header->blocks_per_art_key = 1; + leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); + leaf_header->block0.offset = 0; + leaf_header->block_starting_item.offset = 0; + leaf_header->art_block_index.offset = 0; + } + + const usize space_after_header = dst_remaining.size(); //+++++++++++-+-+--+----- --- -- - - - - - // Calculate the number of blocks needed and how many items in each one. + // Pack as many blocks as fit considering only block_starting_item array overhead. // SmallVec block_stats; + usize items_packed = 0; + SmallVec cumulative_item_count; { auto src_iter = std::begin(src_items); const auto src_end = std::end(src_items); - usize blocks_size_remaining = dst_buffer.size() - block_size; + + // The block_starting_item array has block_count + 1 total elements. In fixed_array_overhead, + // we account for the size of the packed array header and the size of the sentinel element + // in the array. + // + const usize fixed_array_overhead = sizeof(llfs::PackedArray) + sizeof(little_u32); + for (;;) { if (src_iter == src_end) { break; } BATT_CHECK_LT(src_iter, src_end); - if (blocks_size_remaining < block_size) { - return {batt::StatusCode::kResourceExhausted}; + cumulative_item_count.emplace_back(items_packed); + + // n is the number of blocks if we add one more block to our current total. + // + const usize n = block_stats.size() + 1; + + // array_size is the precomputed fixed_array_overhead + the size of n total block entries. + // + const usize array_size = fixed_array_overhead + n * sizeof(little_u32); + + const usize blocks_data_size = n * block_size; + const usize alignment_waste = block_size - 1; + + if (array_size + alignment_waste + blocks_data_size > space_after_header) { + break; } auto& stats = block_stats.emplace_back( PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), block_size)); - blocks_size_remaining -= block_size; + items_packed += stats.item_count; src_iter = std::next(src_iter, stats.item_count); } } - const usize block_count = block_stats.size(); + + cumulative_item_count.emplace_back(items_packed); + + if (block_stats.empty()) { + return {batt::StatusCode::kResourceExhausted}; + } //+++++++++++-+-+--+----- --- -- - - - - - // Initialize the leaf header. + // Shrink blocks until the ART fits. // - MutableBuffer dst_remaining = dst_buffer; - dst_remaining += sizeof(llfs::PackedPageHeader); + // Compute separator keys and build the ART. If it doesn't fit in the space between + // block_starting_item and the block data region, pop the last block and retry. + // + using artc::packed::PackedARTBuilder; - auto* leaf_header = static_cast(dst_remaining.data()); - dst_remaining += sizeof(PackedBlockedLeafPage); - { - leaf_header->magic = PackedBlockedLeafPage::kMagic; - leaf_header->total_packed_size = 0; - leaf_header->blocks_per_art_key = 0; - leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); - leaf_header->block0.offset = 0; - leaf_header->block_starting_item.offset = 0; - leaf_header->art_block_index.offset = 0; + const auto items_begin = std::begin(src_items); + const auto key_at = [&items_begin](usize i) { + return get_key(*(items_begin + i)); + }; + + SmallVec art_keys; + usize art_packed_size = 0; + + for (;;) { + const usize block_count = block_stats.size(); + + art_keys.clear(); + for (usize block_i = 1; block_i < block_count; ++block_i) { + BATT_CHECK_LT(block_i, cumulative_item_count.size()); + const usize item_i = cumulative_item_count[block_i]; + BATT_CHECK_GT(item_i, 0); + BATT_CHECK_LT(item_i, items_packed); + + KeyView k0 = key_at(item_i - 1); + KeyView k1 = key_at(item_i); + KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); + KeyView min_k1{k1.data(), common_prefix.size() + 1}; + + art_keys.emplace_back(min_k1); + } + + // Build the ART to determine its packed size. + // + batt::StableStringStore string_store; + + BATT_ASSIGN_OK_RESULT(auto art_builder, + PackedARTBuilder::from_items(art_keys.begin(), + art_keys.end(), + BATT_OVERLOADS_OF(get_key), + string_store)); + + art_packed_size = art_builder.get_packed_size(); + + // Check whether everything fits. Total size is the size of the block_starting_item + ART size + // + boundary alignment padding + block data size. + // + const usize array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); + const usize total_needed = + array_size + art_packed_size + (block_size - 1) + block_count * block_size; + + if (total_needed <= space_after_header) { + break; + } + + // Doesn't fit, so pop the last block and retry. + // + items_packed -= block_stats.back().item_count; + block_stats.pop_back(); + cumulative_item_count.pop_back(); + + if (block_stats.empty()) { + return {batt::StatusCode::kResourceExhausted}; + } } + const usize block_count = block_stats.size(); + //+++++++++++-+-+--+----- --- -- - - - - // Pack `block_starting_item` array. // @@ -94,66 +191,29 @@ StatusOr pack_blocked_leaf_page(const usize block_size, item_i += stats.item_count; ++block_start; } - *block_start = item_count; + *block_start = items_packed; leaf_header->block_starting_item.reset_unsafe(block_starting_item); } - const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); //+++++++++++-+-+--+----- --- -- - - - - - // Calculate blocks_per_art_key based on available space. + // Pack the ART. // - const usize space_for_art = dst_remaining.size() - block_size * block_count; - SmallVec art_keys; - usize blocks_per_art_key = 1; - //----- --- -- - - - - - const auto items = std::begin(src_items); - const auto key_at = [&items](usize i) { - return get_key(*(items + i)); - }; - //----- --- -- - - - - - for (;;) { - art_keys.clear(); - for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { - const usize item_i = block_starting_item[block_i]; - BATT_CHECK_LT(item_i, item_count); - BATT_CHECK_GT(item_i, 0); - - KeyView k0 = key_at(item_i - 1); - KeyView k1 = key_at(item_i); - KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); - KeyView min_k1{k1.data(), common_prefix.size() + 1}; - - art_keys.emplace_back(min_k1); - } - - using artc::packed::PackedARTBuilder; - + { batt::StableStringStore string_store; - BATT_DEBUG_INFO(BATT_INSPECT_RANGE(art_keys) - << BATT_INSPECT(block_count) << BATT_INSPECT(blocks_per_art_key)); - BATT_ASSIGN_OK_RESULT(auto art_builder, PackedARTBuilder::from_items(art_keys.begin(), art_keys.end(), BATT_OVERLOADS_OF(get_key), string_store)); - if (art_builder.get_packed_size() > space_for_art) { - ++blocks_per_art_key; - continue; - } - MutableBuffer art_buffer{dst_remaining.data(), art_builder.get_packed_size()}; dst_remaining += art_buffer.size(); - BATT_CHECK_GE(dst_remaining.size(), block_size * block_count); BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); leaf_header->art_block_index.reset_unsafe(art_root); - leaf_header->blocks_per_art_key = BATT_CHECKED_CAST(u32, blocks_per_art_key); - break; } //+++++++++++-+-+--+----- --- -- - - - - @@ -196,10 +256,7 @@ StatusOr pack_blocked_leaf_page(const usize block_size, // leaf_header->total_packed_size = BATT_CHECKED_CAST(u32, dst_buffer.size() - dst_remaining.size()); - //+++++++++++-+-+--+----- --- -- - - - - - // Success! (nothing succeeds like it) - // - return leaf_header; + return PackedLeafResult{leaf_header, items_packed}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -300,6 +357,41 @@ inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::lower_bound( return ItemIterator{block_iter, p_slot}; } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline Interval PackedBlockedLeafPage::get_block_aligned_index_range_for_key_range( + const Interval& key_range) const noexcept +{ + const usize first_block = this->find_block_index_containing_key(key_range.lower_bound); + const usize last_block = this->find_block_index_containing_key(key_range.upper_bound); + + BATT_CHECK_LE(first_block, last_block); + + const u32 range_begin = (*this->block_starting_item)[first_block].value(); + const u32 range_end = (*this->block_starting_item)[last_block + 1].value(); + + return Interval{range_begin, range_end}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedKeyValueSlotSlice PackedBlockedLeafPage::get_slice_within_block( + u32 block_index, + const PackedLeafBlock* block, + const Interval& live_item_range) const noexcept +{ + BATT_CHECK_LT(block_index, this->block_count()); + + const u32 block_start = (*this->block_starting_item)[block_index].value(); + const usize local_begin = live_item_range.lower_bound - block_start; + const usize local_end = live_item_range.upper_bound - block_start; + + BATT_CHECK_LE(local_end, block->item_count()); + + return PackedKeyValueSlotSlice{as_slice(block->items_begin() + local_begin, + block->items_begin() + local_end)}; +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template FilterModelT> diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp index d2a9353..7726b77 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp @@ -21,7 +21,12 @@ template FilterModelT> class PackedBlockedLeafPage::ShardedLiveRanges { public: - using Item = std::pair /*live_item_range*/>; + struct Item { + u32 block_index; + Interval live_item_range; + bool is_first; + bool is_last; + }; //+++++++++++-+-+--+----- --- -- - - - - @@ -51,6 +56,7 @@ class PackedBlockedLeafPage::ShardedLiveRanges usize block_index_; BasicPiecewiseFilter::LiveSubranges filter_live_ranges_; Interval current_range_; + bool is_first_ = true; }; } // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp index 9192cb2..bdb7cf5 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp @@ -35,7 +35,10 @@ inline auto PackedBlockedLeafPage::ShardedLiveRanges::peek() -> Op if (this->current_range_.empty()) { return None; } - return std::make_pair(this->block_index_, this->current_range_); + return Item{BATT_CHECKED_CAST(u32, this->block_index_), + this->current_range_, + this->is_first_, + /*is_last=*/false}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -46,7 +49,8 @@ inline auto PackedBlockedLeafPage::ShardedLiveRanges::next() -> Op Optional item = this->peek(); if (item) { this->advance(); - // std::cerr << ".. " << BATT_INSPECT(this->current_range_) << std::endl; + item->is_last = this->current_range_.empty(); + this->is_first_ = false; } return item; } diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index cc7a5c8..4c6e39c 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -173,14 +174,15 @@ TEST(TreePackedBlockedLeafPageTest, Random) MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; - StatusOr status_or_packed_leaf = + StatusOr status_or_packed_leaf = pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); + ASSERT_EQ(status_or_packed_leaf->items_packed, edits.size()); const PackedBlockedLeafPage& packed_leaf = PackedBlockedLeafPage::view_of(leaf_buffer); - ASSERT_EQ(&packed_leaf, *status_or_packed_leaf); + ASSERT_EQ(&packed_leaf, status_or_packed_leaf->leaf); ASSERT_EQ(packed_leaf.min_key(), get_key(edits.front())); ASSERT_EQ(packed_leaf.max_key(), get_key(edits.back())); @@ -298,8 +300,9 @@ TEST(TreePackedBlockedLeafPageTest, Random) u32 next_possible_block = 0; packed_leaf.sharded_live_ranges(leaf_filter, Interval{0, item_count}) | - batt::seq::for_each([&](const std::pair>& live_pair) { - const auto [block_index, live_range] = live_pair; + batt::seq::for_each([&](const auto& item) { + const auto& block_index = item.block_index; + const auto& live_range = item.live_item_range; BATT_CHECK_GE(block_index, next_possible_block); BATT_CHECK_LT(block_index, packed_leaf.block_count()); diff --git a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp index 1427fd9..b54f411 100644 --- a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp +++ b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp @@ -12,280 +12,16 @@ #include "packed_blocked_leaf_page.hpp" #include "packed_blocked_leaf_page.sharded_live_ranges.hpp" #include "packed_blocked_leaf_page.sharded_live_ranges.ipp" -#include "packed_leaf_block.hpp" #include #include #include #include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include #include namespace turtle_kv { -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Returns a boxed sequence of PackedKeyValueSlotSlice for a blocked leaf page. - * - * If `filter` is nullptr, all items are considered live (a default-constructed PiecewiseFilter is - * used internally). If `min_key` is provided, scanning begins at the first item >= min_key. - * - * On I/O error, the sequence terminates and `status` is set to the error status. - */ -template FilterModelT = SmallVec, 64>> -BoxedSeq scan_blocked_leaf( - llfs::PageId page_id, - usize block_size, - const BasicPiecewiseFilter* filter, - Optional min_key, - llfs::PageLoader& page_loader, - PageSliceStorage& slice_storage, - llfs::PinPageToJob pin_page_to_job, - Status& status) noexcept; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -template FilterModelT> -class BlockedLeafScanSeq -{ - public: - using Item = PackedKeyValueSlotSlice; - using Filter = BasicPiecewiseFilter; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit BlockedLeafScanSeq(llfs::PageId page_id, - usize block_size, - const Filter* filter, - Optional min_key, - llfs::PageLoader& page_loader, - PageSliceStorage& slice_storage, - llfs::PinPageToJob pin_page_to_job, - Status& status) noexcept - : page_id_{page_id} - , block_size_{block_size} - , min_key_{min_key} - , page_loader_{page_loader} - , slice_storage_{slice_storage} - , pin_page_to_job_{pin_page_to_job} - , status_{status} - , filter_{filter} - { - this->initialize(); - } - - Optional peek() - { - if (this->done_) { - return None; - } - - if (!this->pending_slice_) { - this->advance(); - } - - return this->pending_slice_; - } - - Optional next() - { - Optional item = this->peek(); - if (item) { - this->pending_slice_ = None; - } - return item; - } - - //+++++++++++-+-+--+----- --- -- - - - - - private: - StatusOr load_shard(const Interval& shard_interval, - llfs::LruPriority lru_priority) noexcept - { - llfs::PageCache& page_cache = *this->page_loader_.page_cache(); - - Optional shard_page_id = - page_cache.page_shard_id_for(this->page_id_, shard_interval); - - if (!shard_page_id) { - return {batt::StatusCode::kUnavailable}; - } - - const llfs::PinnedPage* existing = this->slice_storage_.find_pinned_page(*shard_page_id); - if (existing) { - return ConstBuffer{existing->raw_data(), shard_interval.size()}; - } - - BATT_ASSIGN_OK_RESULT( - llfs::PinnedPage pinned_shard, - this->page_loader_.load_page(*shard_page_id, - llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), - this->pin_page_to_job_, - llfs::OkIfNotFound{false}, - lru_priority})); - - const void* raw_data = pinned_shard.raw_data(); - this->slice_storage_.insert_pinned_page(std::move(pinned_shard)); - - return ConstBuffer{raw_data, shard_interval.size()}; - } - - void initialize() noexcept - { - StatusOr header_buffer = - this->load_shard(Interval{0, this->block_size_}, - llfs::LruPriority{kTrieIndexLruPriority}); - - if (!header_buffer.ok()) { - this->status_ = header_buffer.status(); - this->done_ = true; - return; - } - - this->leaf_ = &PackedBlockedLeafPage::view_of(*header_buffer); - - const usize actual_header_size = this->leaf_->min_header_shard_size(); - if (actual_header_size > this->block_size_) { - StatusOr second_shard = - this->load_shard(Interval{this->block_size_, 2 * this->block_size_}, - llfs::LruPriority{kTrieIndexLruPriority}); - - if (!second_shard.ok()) { - this->status_ = second_shard.status(); - this->done_ = true; - return; - } - } - - const u32 item_count = BATT_CHECKED_CAST(u32, this->leaf_->item_count()); - u32 first_item = 0; - - if (this->min_key_) { - const usize start_block_i = this->leaf_->find_block_index_containing_key(*this->min_key_); - first_item = (*this->leaf_->block_starting_item)[start_block_i]; - } - - if (this->filter_) { - this->live_ranges_.emplace( - this->leaf_->sharded_live_ranges(*this->filter_, Interval{first_item, item_count})); - } else { - this->live_ranges_.emplace( - this->leaf_->sharded_live_ranges(this->pass_through_filter_, - Interval{first_item, item_count})); - } - } - - void advance() noexcept - { - for (;;) { - auto live_pair = this->live_ranges_->next(); - if (!live_pair) { - this->done_ = true; - return; - } - - const auto [block_index, live_item_range] = *live_pair; - - const usize block_offset = this->leaf_->block_page_offset(block_index); - StatusOr block_buffer = - this->load_shard(Interval{block_offset, block_offset + this->block_size_}, - llfs::LruPriority{kLeafLruPriority}); - - if (!block_buffer.ok()) { - this->status_ = block_buffer.status(); - this->done_ = true; - return; - } - - const PackedLeafBlock& block = PackedLeafBlock::view_of(*block_buffer); - - const Interval block_item_range = this->leaf_->item_index_range_of_block(block_index); - const usize local_begin = live_item_range.lower_bound - block_item_range.lower_bound; - const usize local_end = live_item_range.upper_bound - block_item_range.lower_bound; - - const PackedKeyValueSlotPtr* slice_begin = block.items_begin() + local_begin; - const PackedKeyValueSlotPtr* slice_end = block.items_begin() + local_end; - - if (this->min_key_ && !this->min_key_applied_) { - this->min_key_applied_ = true; - slice_begin = - std::lower_bound(slice_begin, - slice_end, - *this->min_key_, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; - }); - } - - if (slice_begin == slice_end) { - continue; - } - - this->pending_slice_ = PackedKeyValueSlotSlice{as_slice(slice_begin, slice_end)}; - return; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - - llfs::PageId page_id_; - usize block_size_; - Optional min_key_; - llfs::PageLoader& page_loader_; - PageSliceStorage& slice_storage_; - llfs::PinPageToJob pin_page_to_job_; - Status& status_; - - const Filter* filter_; - PiecewiseFilter pass_through_filter_; - - bool done_ = false; - bool min_key_applied_ = false; - - const PackedBlockedLeafPage* leaf_ = nullptr; - - using LiveRanges = PackedBlockedLeafPage::ShardedLiveRanges; - Optional live_ranges_; - - Optional pending_slice_; -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -BoxedSeq scan_blocked_leaf( - llfs::PageId page_id, - usize block_size, - const BasicPiecewiseFilter* filter, - Optional min_key, - llfs::PageLoader& page_loader, - PageSliceStorage& slice_storage, - llfs::PinPageToJob pin_page_to_job, - Status& status) noexcept -{ - return BlockedLeafScanSeq{page_id, - block_size, - filter, - min_key, - page_loader, - slice_storage, - pin_page_to_job, - status} | - seq::boxed(); -} - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template FilterModelT, typename BlockLoaderT> @@ -299,37 +35,57 @@ auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, packed_leaf->get_block_aligned_index_range_for_key_range(key_range); return packed_leaf->sharded_live_ranges(filter, index_range) // - | batt::seq::filter_map([block_loader](const std::pair>& params) - -> Optional> { - // Stage 1: filter - // - const Interval& live_item_range = params.second; - if (live_item_range.empty()) { - return None; - } - - // Stage 2: map - // - const u32 block_index = params.first; - StatusOr block = block_loader->load_block(block_index); - if (!block.ok()) { - return block.status(); - } - - // Convert index range -> slot slice - // - PackedKeyValueSlotSlice slice = - packed_leaf->get_slice_within_block(block_index, block, live_item_range); - - // Stage 3 - // Trim the slice down to `key_range`. - // - if ("this is the first or last block in the sequence") { - // trim the slice - } - - return slice; - }) // + | + batt::seq::filter_map( + [packed_leaf, block_loader, key_range]( + const typename PackedBlockedLeafPage::ShardedLiveRanges::Item& item) + -> Optional> { + if (item.live_item_range.empty()) { + return None; + } + + StatusOr block = block_loader->load_block(item.block_index); + if (!block.ok()) { + return block.status(); + } + + PackedKeyValueSlotSlice slice = + packed_leaf->get_slice_within_block(item.block_index, + *block, + item.live_item_range); + + if (item.is_first || item.is_last) { + auto& ptr_slice = std::get>(slice); + const auto* begin = ptr_slice.begin(); + const auto* end = ptr_slice.end(); + + if (item.is_first) { + begin = + std::lower_bound(begin, + end, + key_range.lower_bound, + [](const PackedKeyValueSlotPtr& slot, const KeyView& key) { + return get_key(slot) < key; + }); + } + if (item.is_last) { + end = + std::lower_bound(begin, + end, + key_range.upper_bound, + [](const PackedKeyValueSlotPtr& slot, const KeyView& key) { + return get_key(slot) < key; + }); + } + + if (begin == end) { + return None; + } + slice = PackedKeyValueSlotSlice{as_slice(begin, end)}; + } + + return slice; + }) // ; } From 81da49551cdc495ca1012b2b684738ef9fb1e0f1 Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Wed, 12 Aug 2026 14:06:06 -0400 Subject: [PATCH 18/20] Review feedback on block loader, scan_blocked_leaf_page, and pack_blocked_leaf_page. Refactor blocked leaf test. --- .../tree/leaf/blocked_leaf_page_loader.cpp | 81 +++--- .../tree/leaf/blocked_leaf_page_loader.hpp | 5 +- .../tree/leaf/packed_blocked_leaf_page.ipp | 124 ++++----- ..._blocked_leaf_page.sharded_live_ranges.hpp | 5 +- ..._blocked_leaf_page.sharded_live_ranges.ipp | 10 +- .../leaf/packed_blocked_leaf_page.test.cpp | 253 +++++++++++++----- src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp | 2 + .../tree/merge_set/composite_merge_set.cpp | 50 ---- .../tree/merge_set/composite_merge_set.hpp | 34 --- .../tree/merge_set/empty_merge_set.hpp | 11 - .../tree/merge_set/fake_key_value.hpp | 41 --- .../tree/merge_set/fake_key_value_view.hpp | 22 -- src/turtle_kv/tree/merge_set/fake_leaf.hpp | 49 ---- .../tree/merge_set/h_join_merge_set.cpp | 162 ----------- .../tree/merge_set/h_join_merge_set.hpp | 73 ----- .../tree/merge_set/in_memory_merge_set.hpp | 70 ----- .../tree/merge_set/in_storage_merge_set.hpp | 58 ---- src/turtle_kv/tree/merge_set/merge_set.cpp | 106 -------- src/turtle_kv/tree/merge_set/merge_set.hpp | 93 ------- .../tree/merge_set/merge_set.test.cpp | 18 -- .../tree/merge_set/random_key_value.hpp | 127 --------- .../tree/merge_set/v_join_merge_set.cpp | 43 --- .../tree/merge_set/v_join_merge_set.hpp | 34 --- .../tree/testing/in_memory_block_loader.hpp | 39 +++ 24 files changed, 329 insertions(+), 1181 deletions(-) delete mode 100644 src/turtle_kv/tree/merge_set/composite_merge_set.cpp delete mode 100644 src/turtle_kv/tree/merge_set/composite_merge_set.hpp delete mode 100644 src/turtle_kv/tree/merge_set/empty_merge_set.hpp delete mode 100644 src/turtle_kv/tree/merge_set/fake_key_value.hpp delete mode 100644 src/turtle_kv/tree/merge_set/fake_key_value_view.hpp delete mode 100644 src/turtle_kv/tree/merge_set/fake_leaf.hpp delete mode 100644 src/turtle_kv/tree/merge_set/h_join_merge_set.cpp delete mode 100644 src/turtle_kv/tree/merge_set/h_join_merge_set.hpp delete mode 100644 src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp delete mode 100644 src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp delete mode 100644 src/turtle_kv/tree/merge_set/merge_set.cpp delete mode 100644 src/turtle_kv/tree/merge_set/merge_set.hpp delete mode 100644 src/turtle_kv/tree/merge_set/merge_set.test.cpp delete mode 100644 src/turtle_kv/tree/merge_set/random_key_value.hpp delete mode 100644 src/turtle_kv/tree/merge_set/v_join_merge_set.cpp delete mode 100644 src/turtle_kv/tree/merge_set/v_join_merge_set.hpp create mode 100644 src/turtle_kv/tree/testing/in_memory_block_loader.hpp diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp index 1c09844..44e0447 100644 --- a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp @@ -35,31 +35,36 @@ StatusOr BlockedLeafPageLoader::set_page( this->leaf_ = nullptr; this->cache_.clear(); - // Load the first shard to access the fixed header fields. + PageSliceReader slice_reader{this->page_loader_, page_id, + llfs::PageSize{BATT_CHECKED_CAST(i32, this->block_size_)}}; + + // Load the first shard to read the fixed header fields. // BATT_ASSIGN_OK_RESULT( ConstBuffer header_buffer, - this->load_shard(Interval{0, this->block_size_}, - llfs::LruPriority{kTrieIndexLruPriority})); + slice_reader.read_slice(Interval{0, this->block_size_}, + this->slice_storage_, + this->pin_page_to_job_, + llfs::LruPriority{kTrieIndexLruPriority})); this->leaf_ = &PackedBlockedLeafPage::view_of(header_buffer); - // Load additional shards until the full header metadata is covered. + // Load the full header (block_starting_item + ART) as a contiguous buffer. // const usize header_size = this->leaf_->min_header_shard_size(); - usize loaded = this->block_size_; - while (loaded < header_size) { - const usize next_end = std::min(loaded + this->block_size_, header_size); + if (header_size > this->block_size_) { BATT_ASSIGN_OK_RESULT( - ConstBuffer shard_buffer, - this->load_shard(Interval{loaded, next_end}, - llfs::LruPriority{kTrieIndexLruPriority})); - loaded = next_end; - (void)shard_buffer; + header_buffer, + slice_reader.read_slice(Interval{0, header_size}, + this->slice_storage_, + this->pin_page_to_job_, + llfs::LruPriority{kTrieIndexLruPriority})); + + this->leaf_ = &PackedBlockedLeafPage::view_of(header_buffer); } - this->cache_.resize(this->leaf_->block_count(), None); + this->cache_.assign(this->leaf_->block_count(), ConstBuffer{}); return this->leaf_; } @@ -71,53 +76,37 @@ StatusOr BlockedLeafPageLoader::load_block(u32 block_ind BATT_CHECK_NE(this->leaf_, nullptr); BATT_CHECK_LT(block_index, this->cache_.size()); - if (this->cache_[block_index]) { - return &PackedLeafBlock::view_of(*this->cache_[block_index]); + if (this->cache_[block_index].data()) { + return &PackedLeafBlock::view_of(this->cache_[block_index]); } - const usize block_offset = this->leaf_->block_page_offset(block_index); - BATT_ASSIGN_OK_RESULT( - ConstBuffer block_buffer, - this->load_shard(Interval{block_offset, block_offset + this->block_size_}, - llfs::LruPriority{kLeafLruPriority})); - - this->cache_[block_index] = block_buffer; - - return &PackedLeafBlock::view_of(block_buffer); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -StatusOr BlockedLeafPageLoader::load_shard( - const Interval& shard_interval, - llfs::LruPriority lru_priority) noexcept -{ llfs::PageCache& page_cache = *this->page_loader_.page_cache(); Optional shard_page_id = - page_cache.page_shard_id_for(this->page_id_, shard_interval); + this->leaf_->page_shard_id_for_block(page_cache, block_index, this->page_id_); if (!shard_page_id) { return {batt::StatusCode::kUnavailable}; } const llfs::PinnedPage* existing = this->slice_storage_.find_pinned_page(*shard_page_id); - if (existing) { - return ConstBuffer{existing->raw_data(), (usize)shard_interval.size()}; + if (!existing) { + BATT_ASSIGN_OK_RESULT( + llfs::PinnedPage pinned_shard, + this->page_loader_.load_page(*shard_page_id, + llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), + this->pin_page_to_job_, + llfs::OkIfNotFound{false}, + llfs::LruPriority{kLeafLruPriority}})); + + this->slice_storage_.insert_pinned_page(std::move(pinned_shard)); + existing = this->slice_storage_.find_pinned_page(*shard_page_id); } - BATT_ASSIGN_OK_RESULT( - llfs::PinnedPage pinned_shard, - this->page_loader_.load_page(*shard_page_id, - llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), - this->pin_page_to_job_, - llfs::OkIfNotFound{false}, - lru_priority})); - - const void* raw_data = pinned_shard.raw_data(); - this->slice_storage_.insert_pinned_page(std::move(pinned_shard)); + ConstBuffer block_buffer{existing->raw_data(), this->block_size_}; + this->cache_[block_index] = block_buffer; - return ConstBuffer{raw_data, (usize)shard_interval.size()}; + return &PackedLeafBlock::view_of(block_buffer); } } // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp index 130c950..b4c1017 100644 --- a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp @@ -17,7 +17,6 @@ #include #include -#include #include #include @@ -54,8 +53,6 @@ class BlockedLeafPageLoader StatusOr load_block(u32 block_index) noexcept; private: - StatusOr load_shard(const Interval& shard_interval, - llfs::LruPriority lru_priority) noexcept; //+++++++++++-+-+--+----- --- -- - - - - @@ -66,7 +63,7 @@ class BlockedLeafPageLoader llfs::PageId page_id_; const PackedBlockedLeafPage* leaf_ = nullptr; - SmallVec, 256> cache_; + SmallVec cache_; }; } // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp index fe8a557..866f0c9 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp @@ -106,6 +106,8 @@ StatusOr pack_blocked_leaf_page(const usize block_size, // // Compute separator keys and build the ART. If it doesn't fit in the space between // block_starting_item and the block data region, pop the last block and retry. + // When the ART successfully fits, pack both the block_starting_item array and ART before + // breaking out of the retry loop. // using artc::packed::PackedARTBuilder; @@ -115,26 +117,23 @@ StatusOr pack_blocked_leaf_page(const usize block_size, }; SmallVec art_keys; - usize art_packed_size = 0; + for (usize block_i = 1; block_i < block_stats.size(); ++block_i) { + BATT_CHECK_LT(block_i, cumulative_item_count.size()); + const usize item_i = cumulative_item_count[block_i]; + BATT_CHECK_GT(item_i, 0); + BATT_CHECK_LT(item_i, items_packed); + + KeyView k0 = key_at(item_i - 1); + KeyView k1 = key_at(item_i); + KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); + KeyView min_k1{k1.data(), common_prefix.size() + 1}; + + art_keys.emplace_back(min_k1); + } for (;;) { const usize block_count = block_stats.size(); - art_keys.clear(); - for (usize block_i = 1; block_i < block_count; ++block_i) { - BATT_CHECK_LT(block_i, cumulative_item_count.size()); - const usize item_i = cumulative_item_count[block_i]; - BATT_CHECK_GT(item_i, 0); - BATT_CHECK_LT(item_i, items_packed); - - KeyView k0 = key_at(item_i - 1); - KeyView k1 = key_at(item_i); - KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); - KeyView min_k1{k1.data(), common_prefix.size() + 1}; - - art_keys.emplace_back(min_k1); - } - // Build the ART to determine its packed size. // batt::StableStringStore string_store; @@ -145,7 +144,7 @@ StatusOr pack_blocked_leaf_page(const usize block_size, BATT_OVERLOADS_OF(get_key), string_store)); - art_packed_size = art_builder.get_packed_size(); + const usize art_packed_size = art_builder.get_packed_size(); // Check whether everything fits. Total size is the size of the block_starting_item + ART size // + boundary alignment padding + block data size. @@ -155,66 +154,60 @@ StatusOr pack_blocked_leaf_page(const usize block_size, const usize total_needed = array_size + art_packed_size + (block_size - 1) + block_count * block_size; - if (total_needed <= space_after_header) { - break; + // ART doesn't fit, so pop a block and retry the loop. + // + if (total_needed > space_after_header) { + items_packed -= block_stats.back().item_count; + block_stats.pop_back(); + cumulative_item_count.pop_back(); + art_keys.pop_back(); + + if (block_stats.empty()) { + return {batt::StatusCode::kResourceExhausted}; + } + continue; } - // Doesn't fit, so pop the last block and retry. + // Pack `block_starting_item` array. // - items_packed -= block_stats.back().item_count; - block_stats.pop_back(); - cumulative_item_count.pop_back(); + { + const usize block_starting_item_array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); + + auto* block_starting_item = + static_cast*>(dst_remaining.data()); + dst_remaining += block_starting_item_array_size; + + block_starting_item->initialize(block_count + 1); + + little_u32* block_start = block_starting_item->data(); + u32 item_i = 0; + for (const PackedLeafBlockStats& stats : block_stats) { + *block_start = item_i; + item_i += stats.item_count; + ++block_start; + } + *block_start = items_packed; - if (block_stats.empty()) { - return {batt::StatusCode::kResourceExhausted}; + leaf_header->block_starting_item.reset_unsafe(block_starting_item); } - } - const usize block_count = block_stats.size(); - - //+++++++++++-+-+--+----- --- -- - - - - - // Pack `block_starting_item` array. - // - { - const usize block_starting_item_array_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); - - auto* block_starting_item = static_cast*>(dst_remaining.data()); - dst_remaining += block_starting_item_array_size; + // Pack the ART. + // + { + MutableBuffer art_buffer{dst_remaining.data(), art_builder.get_packed_size()}; + dst_remaining += art_buffer.size(); - block_starting_item->initialize(block_count + 1); + BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, + art_builder.build(art_buffer)); - little_u32* block_start = block_starting_item->data(); - u32 item_i = 0; - for (const PackedLeafBlockStats& stats : block_stats) { - *block_start = item_i; - item_i += stats.item_count; - ++block_start; + leaf_header->art_block_index.reset_unsafe(art_root); } - *block_start = items_packed; - leaf_header->block_starting_item.reset_unsafe(block_starting_item); + break; } - //+++++++++++-+-+--+----- --- -- - - - - - // Pack the ART. - // - { - batt::StableStringStore string_store; - - BATT_ASSIGN_OK_RESULT(auto art_builder, - PackedARTBuilder::from_items(art_keys.begin(), - art_keys.end(), - BATT_OVERLOADS_OF(get_key), - string_store)); - - MutableBuffer art_buffer{dst_remaining.data(), art_builder.get_packed_size()}; - dst_remaining += art_buffer.size(); - - BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); - - leaf_header->art_block_index.reset_unsafe(art_root); - } + const usize block_count = block_stats.size(); //+++++++++++-+-+--+----- --- -- - - - - // Shift the remaining buffer forward so it aligns with the nearest block boundary. @@ -402,6 +395,7 @@ PackedBlockedLeafPage::sharded_live_ranges(const BasicPiecewiseFilter{ this->block_starting_item.get(), filter.live_subranges_of(subrange), + subrange, }; } diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp index 7726b77..73fab8e 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp @@ -32,7 +32,8 @@ class PackedBlockedLeafPage::ShardedLiveRanges explicit ShardedLiveRanges( const llfs::PackedArray* block_starts, - BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept; + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges, + const Interval& subrange) noexcept; //+++++++++++-+-+--+----- --- -- - - - - @@ -56,7 +57,7 @@ class PackedBlockedLeafPage::ShardedLiveRanges usize block_index_; BasicPiecewiseFilter::LiveSubranges filter_live_ranges_; Interval current_range_; - bool is_first_ = true; + Interval subrange_; }; } // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp index bdb7cf5..4299151 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp @@ -18,11 +18,13 @@ namespace turtle_kv { template FilterModelT> inline /*explicit*/ PackedBlockedLeafPage::ShardedLiveRanges::ShardedLiveRanges( const llfs::PackedArray* block_starts, - BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges, + const Interval& subrange) noexcept : block_starts_{block_starts} , block_index_{0} , filter_live_ranges_{std::move(filter_live_ranges)} , current_range_{0, 0} + , subrange_{subrange} { this->advance(); } @@ -37,8 +39,8 @@ inline auto PackedBlockedLeafPage::ShardedLiveRanges::peek() -> Op } return Item{BATT_CHECKED_CAST(u32, this->block_index_), this->current_range_, - this->is_first_, - /*is_last=*/false}; + (*this->block_starts_)[this->block_index_] == this->subrange_.lower_bound, + (*this->block_starts_)[this->block_index_ + 1] == this->subrange_.upper_bound}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -49,8 +51,6 @@ inline auto PackedBlockedLeafPage::ShardedLiveRanges::next() -> Op Optional item = this->peek(); if (item) { this->advance(); - item->is_last = this->current_range_.empty(); - this->is_first_ = false; } return item; } diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index 4c6e39c..13443d6 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -53,56 +54,45 @@ using turtle_kv::PiecewiseFilter; using turtle_kv::random_str; using turtle_kv::ValueView; -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// Plan: -// 1. For different random seeds: -// - generate random set of prefixes (~10% of total keys) -// - generate keys using prefixes, with random values -// - sort -// - pack leaf; verify: -// a. all packed keys present and have right values -// b. any unpacked keys at end missing -// c. randomly generated non-present keys not found +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- // -TEST(TreePackedBlockedLeafPageTest, Random) +class PackedBlockedLeafPageTest : public ::testing::Test { - const usize kFirstSeed = 0; - const usize kNumSeeds = 250; - const usize kLastSeed = kFirstSeed + kNumSeeds; - const usize kLeafPageSize = 1 * kMiB; - const usize kNumPrefixes = 1000; - const usize kMinPrefixSize = 0; - const usize kMaxPrefixSize = 8; - const usize kMinKeySize = 4; - const usize kMaxKeySize = 48; - const usize kMinValueSize = 0; - const usize kMaxValueSize = 200; - const usize kBlockSize = 8192; - - BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); - - std::uniform_int_distribution pick_pct{0, 99}; - std::geometric_distribution pick_prefix_size{0.5}; - std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; - std::geometric_distribution pick_key_size{0.7}; - std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - - for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { - LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); + public: + static constexpr usize kLeafPageSize = 1 * kMiB; + static constexpr usize kNumPrefixes = 1000; + static constexpr usize kMinPrefixSize = 0; + static constexpr usize kMaxPrefixSize = 8; + static constexpr usize kMinKeySize = 4; + static constexpr usize kMaxKeySize = 48; + static constexpr usize kMinValueSize = 0; + static constexpr usize kMaxValueSize = 200; + static constexpr usize kBlockSize = 8192; + + //+++++++++++-+-+--+----- --- -- - - - - + + void set_seed(usize seed) + { + this->rng_.seed(seed); + this->edits_.clear(); + this->leaf_storage_.clear(); + } - std::default_random_engine rng{seed}; + void generate_edits(StableStringStore& strings) + { + BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); - StableStringStore strings; + std::geometric_distribution pick_prefix_size{0.5}; + std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - //+++++++++++-+-+--+----- --- -- - - - - - // Generate prefixes - // std::vector prefixes; { std::unordered_set used_prefixes; while (prefixes.size() < kNumPrefixes) { std::string_view prefix = - random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + random_str(this->rng_, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); if (used_prefixes.count(prefix)) { continue; @@ -111,10 +101,6 @@ TEST(TreePackedBlockedLeafPageTest, Random) } } - //+++++++++++-+-+--+----- --- -- - - - - - // Generate edits. - // - std::vector edits; { usize max_edit_size = 0; usize max_key_size = 0; @@ -122,10 +108,10 @@ TEST(TreePackedBlockedLeafPageTest, Random) std::unordered_set used_keys; for (;;) { - std::string_view prefix = prefixes[pick_prefix(rng)]; + std::string_view prefix = prefixes[pick_prefix(this->rng_)]; std::string_view key = - random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); + random_str(this->rng_, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); if (used_keys.count(key)) { continue; @@ -133,7 +119,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) used_keys.insert(key); std::string_view value = - random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + random_str(this->rng_, pick_value_size, kMinValueSize, kMaxValueSize, strings); EditView edit{key, ValueView::from_str(value)}; @@ -147,42 +133,76 @@ TEST(TreePackedBlockedLeafPageTest, Random) new_max_key_size, new_max_edit_size); - // Stop as soon as adding the next key would exceed the estimated space. - // if (edit_size + total_edits_size > space_available) { break; } - edits.push_back(edit); + this->edits_.push_back(edit); total_edits_size += edit_size; max_edit_size = new_max_edit_size; max_key_size = new_max_key_size; } } - //+++++++++++-+-+--+----- --- -- - - - - - // Sort edits by key. - // - std::sort(edits.begin(), edits.end(), KeyOrder{}); + std::sort(this->edits_.begin(), this->edits_.end(), KeyOrder{}); + } - //+++++++++++-+-+--+----- --- -- - - - - - // Pack a blocked leaf page. - // + StatusOr pack_leaf() + { using StorageUnit = std::aligned_storage_t<4096, 4096>; - std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); - ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); + this->leaf_storage_.resize(kLeafPageSize / sizeof(StorageUnit)); + BATT_CHECK_EQ(sizeof(StorageUnit) * this->leaf_storage_.size(), kLeafPageSize); + + MutableBuffer leaf_buffer{this->leaf_storage_.data(), kLeafPageSize}; + + BATT_ASSIGN_OK_RESULT(auto result, pack_blocked_leaf_page(kBlockSize, this->edits_, leaf_buffer)); + + if (result.items_packed != this->edits_.size()) { + return {batt::StatusCode::kInternal}; + } + + return &PackedBlockedLeafPage::view_of(leaf_buffer); + } + + MutableBuffer leaf_buffer() const + { + return MutableBuffer{const_cast(static_cast(this->leaf_storage_.data())), + kLeafPageSize}; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + std::default_random_engine rng_; + std::vector edits_; + + private: + using StorageUnit = std::aligned_storage_t<4096, 4096>; + std::vector leaf_storage_; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST_F(PackedBlockedLeafPageTest, Random) +{ + const usize kFirstSeed = 0; + const usize kNumSeeds = 250; + const usize kLastSeed = kFirstSeed + kNumSeeds; - MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; + std::uniform_int_distribution pick_pct{0, 99}; - StatusOr status_or_packed_leaf = - pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); + for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { + LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); + + StableStringStore strings; + this->set_seed(seed); + this->generate_edits(strings); - ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); - ASSERT_EQ(status_or_packed_leaf->items_packed, edits.size()); + StatusOr status_or_leaf = this->pack_leaf(); + ASSERT_TRUE(status_or_leaf.ok()) << BATT_INSPECT(status_or_leaf.status()); - const PackedBlockedLeafPage& packed_leaf = PackedBlockedLeafPage::view_of(leaf_buffer); + const PackedBlockedLeafPage& packed_leaf = **status_or_leaf; + const auto& edits = this->edits_; - ASSERT_EQ(&packed_leaf, status_or_packed_leaf->leaf); ASSERT_EQ(packed_leaf.min_key(), get_key(edits.front())); ASSERT_EQ(packed_leaf.max_key(), get_key(edits.back())); @@ -240,7 +260,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) ASSERT_LT(item_iter, items_end); ASSERT_EQ(std::distance(packed_leaf.items_begin(), item_iter), position); - if (pick_pct(rng) < 1) { + if (pick_pct(this->rng_) < 1) { past_items.push_back(std::make_pair(item_iter, position)); } @@ -283,7 +303,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) turtle_kv::testing::drop_n_disjoint_intervals_from(&leaf_filter, drop_count, Interval{0, item_count}, - rng); + this->rng_); // Verify the number of expected live items. // @@ -332,4 +352,101 @@ TEST(TreePackedBlockedLeafPageTest, Random) } } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST_F(PackedBlockedLeafPageTest, ScanBlockedLeaf) +{ + using turtle_kv::testing::InMemoryBlockLoader; + using turtle_kv::scan_blocked_leaf; + using turtle_kv::PackedKeyValueSlotSlice; + + const usize kFirstSeed = 0; + const usize kNumSeeds = 250; + const usize kLastSeed = kFirstSeed + kNumSeeds; + const usize kTrialsPerSeed = 100; + const usize kMaxDropCount = 500; + + for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { + LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); + + StableStringStore strings; + this->set_seed(seed); + this->generate_edits(strings); + + StatusOr status_or_leaf = this->pack_leaf(); + ASSERT_TRUE(status_or_leaf.ok()) << BATT_INSPECT(status_or_leaf.status()); + + const PackedBlockedLeafPage& packed_leaf = **status_or_leaf; + const auto& edits = this->edits_; + const u32 item_count = packed_leaf.item_count(); + + InMemoryBlockLoader block_loader{&packed_leaf}; + + std::uniform_int_distribution pick_edit{0, edits.size() - 1}; + + for (usize trial = 0; trial < kTrialsPerSeed; ++trial) { + // Pick a random half-open key range [lower_key, upper_key). + // + usize lo_i = pick_edit(this->rng_); + usize hi_i = pick_edit(this->rng_); + if (lo_i > hi_i) { + std::swap(lo_i, hi_i); + } + if (lo_i == hi_i && hi_i + 1 < edits.size()) { + ++hi_i; + } + + KeyView lower_key = get_key(edits[lo_i]); + KeyView upper_key = get_key(edits[hi_i]); + Interval key_range{lower_key, upper_key}; + + // Build a random filter with drops. + // + const usize drop_count = std::uniform_int_distribution{0, kMaxDropCount}(this->rng_); + PiecewiseFilter leaf_filter; + + turtle_kv::testing::drop_n_disjoint_intervals_from(&leaf_filter, + drop_count, + Interval{0, item_count}, + this->rng_); + + // Compute expected results: edits in [lo_i, hi_i) that are live. + // + std::vector> expected; + for (usize i = lo_i; i < hi_i; ++i) { + if (leaf_filter.live_at_index(i)) { + expected.emplace_back(get_key(edits[i]), get_value(edits[i])); + } + } + + // Run scan_blocked_leaf and collect results. + // + std::vector> actual; + + scan_blocked_leaf(&packed_leaf, &block_loader, leaf_filter, key_range) // + | batt::seq::for_each([&](const StatusOr& status_or_slice) { + ASSERT_TRUE(status_or_slice.ok()) << BATT_INSPECT(status_or_slice.status()); + std::visit( + [&](const auto& s) { + for (const auto& slot : s) { + actual.emplace_back(get_key(slot), get_value(slot)); + } + }, + *status_or_slice); + }); + + ASSERT_EQ(actual.size(), expected.size()) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(drop_count) + << BATT_INSPECT(lower_key) << BATT_INSPECT(upper_key); + + for (usize i = 0; i < expected.size(); ++i) { + ASSERT_EQ(actual[i].first, expected[i].first) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(i); + ASSERT_EQ(actual[i].second, expected[i].second) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(i); + } + } + } +} + } // namespace diff --git a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp index b54f411..5f6fce1 100644 --- a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp +++ b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp @@ -31,6 +31,8 @@ auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, const BasicPiecewiseFilter& filter, const Interval& key_range) noexcept { + BATT_CHECK_NOT_NULLPTR(packed_leaf); + const Interval index_range = packed_leaf->get_block_aligned_index_range_for_key_range(key_range); diff --git a/src/turtle_kv/tree/merge_set/composite_merge_set.cpp b/src/turtle_kv/tree/merge_set/composite_merge_set.cpp deleted file mode 100644 index 47967b7..0000000 --- a/src/turtle_kv/tree/merge_set/composite_merge_set.cpp +++ /dev/null @@ -1,50 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include "composite_merge_set.hpp" -// -#include "h_join_merge_set.hpp" -#include "merge_set.hpp" -#include "v_join_merge_set.hpp" - -namespace turtle_kv { -namespace merge_set { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void CompositeMergeSet::add(MergeSet&& src) noexcept -{ - this->components_.emplace_back(std::make_unique(std::move(src))); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template T> -T clone_impl(const T& src, batt::StaticType) noexcept -{ - T dst; - - dst.key_lower_bound_ = src.key_lower_bound_; - dst.key_upper_bound_ = src.key_upper_bound_; - dst.components_.reserve(src.components_.size()); - - for (const std::unique_ptr& component : src.components_) { - dst.components_.emplace_back(std::make_unique(clone(*component))); - } - - return dst; -} - -template HJoinMergeSet clone_impl(const HJoinMergeSet& src, - batt::StaticType) noexcept; - -template VJoinMergeSet clone_impl(const VJoinMergeSet& src, - batt::StaticType) noexcept; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/composite_merge_set.hpp b/src/turtle_kv/tree/merge_set/composite_merge_set.hpp deleted file mode 100644 index 077a08f..0000000 --- a/src/turtle_kv/tree/merge_set/composite_merge_set.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_COMPOSITE_MERGE_SET_HPP - -#include -#include - -#include - -#include -#include -#include - -namespace turtle_kv { -namespace merge_set { - -struct MergeSet; - -struct CompositeMergeSet { - std::vector> components_; - std::string key_lower_bound_; - std::string key_upper_bound_; - - //+++++++++++-+-+--+----- --- -- - - - - - - void add(MergeSet&& src) noexcept; -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template T> -T clone_impl(const T& src, batt::StaticType = {}) noexcept; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/empty_merge_set.hpp b/src/turtle_kv/tree/merge_set/empty_merge_set.hpp deleted file mode 100644 index b80894b..0000000 --- a/src/turtle_kv/tree/merge_set/empty_merge_set.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_EMPTY_MERGE_SET_HPP - -namespace turtle_kv { -namespace merge_set { - -struct EmptyMergeSet { -}; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/fake_key_value.hpp b/src/turtle_kv/tree/merge_set/fake_key_value.hpp deleted file mode 100644 index 0e337ee..0000000 --- a/src/turtle_kv/tree/merge_set/fake_key_value.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_HPP - -#include - -#include - -#include -#include - -namespace turtle_kv { -namespace merge_set { - -struct FakeKeyValue { - std::string key_; - std::string value_; -}; - -inline KeyView get_key(const FakeKeyValue& view) noexcept -{ - return view.key_; -} - -inline usize packed_sizeof(const FakeKeyValue& kv) noexcept -{ - return kv.key_.size() + kv.value_.size(); -} - -inline std::string get_min_upper_bound(const std::string_view& view) noexcept -{ - std::string s{view}; - if (s.back() == (char)255) { - s += '\0'; - } else { - ++s.back(); - } - return s; -} - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp b/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp deleted file mode 100644 index 60f4be3..0000000 --- a/src/turtle_kv/tree/merge_set/fake_key_value_view.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_VIEW_HPP - -#include - -#include - -namespace turtle_kv { -namespace merge_set { - -struct FakeKeyValueView { - KeyView key_; - std::string_view value_; -}; - -inline const KeyView& get_key(const FakeKeyValueView& view) noexcept -{ - return view.key_; -} - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/fake_leaf.hpp b/src/turtle_kv/tree/merge_set/fake_leaf.hpp deleted file mode 100644 index df76bc7..0000000 --- a/src/turtle_kv/tree/merge_set/fake_leaf.hpp +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_FAKE_KEY_VALUE_HPP - -#include "fake_key_value.hpp" - -#include - -#include -#include -#include - -namespace turtle_kv { -namespace merge_set { - -struct FakeBlock { - std::vector items_; -}; - -struct FakeLeaf { - std::vector block_keys_; - std::vector block_starts_; - std::vector blocks_; - usize block_size_; - - //+++++++++++-+-+--+----- --- -- - - - - - - usize block_count() const noexcept - { - return this->blocks_.size(); - } - - usize block_size() const noexcept - { - return this->block_size_; - } - - usize block_containing_index(usize index) const noexcept - { - BATT_CHECK(!this->block_starts_.empty()); - - const auto second = std::next(this->block_starts_.begin()); - auto iter = std::upper_bound(second, this->block_starts_.end(), index); - - return std::distance(second, iter); - } -}; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp b/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp deleted file mode 100644 index 87a968f..0000000 --- a/src/turtle_kv/tree/merge_set/h_join_merge_set.cpp +++ /dev/null @@ -1,162 +0,0 @@ -#include "h_join_merge_set.hpp" -// - -#include "merge_set.hpp" - -#include - -namespace turtle_kv { -namespace merge_set { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -CInterval h_get_byte_size_impl(const std::vector>& segments) noexcept -{ - if (segments.empty()) { - return {0, 0}; - } - - CInterval result{0, 0}; - - for (const std::unique_ptr& p_component : segments) { - const CInterval component_byte_size = get_byte_size(*p_component); - result.lower_bound += component_byte_size.lower_bound; - result.upper_bound += component_byte_size.upper_bound; - } - - return result; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -Interval h_seek_impl(const std::vector>& segments, - usize byte_size, - const KeyView& key_upper_bound) noexcept -{ - BATT_CHECK(!segments.empty()); - - Interval result; - CInterval bytes_remaining{byte_size, byte_size}; - - for (const std::unique_ptr& segment : segments) { - const CInterval segment_byte_size = get_byte_size(*segment); - - if (bytes_remaining.lower_bound != 0) { - if (bytes_remaining.lower_bound > segment_byte_size.upper_bound) { - bytes_remaining.lower_bound -= segment_byte_size.upper_bound; - } else { - result.lower_bound = seek(*segment, bytes_remaining.lower_bound).lower_bound; - bytes_remaining.lower_bound = 0; - } - } - - if (bytes_remaining.upper_bound != 0) { - if (bytes_remaining.upper_bound > segment_byte_size.lower_bound) { - bytes_remaining.upper_bound -= segment_byte_size.lower_bound; - } else { - result.upper_bound = seek(*segment, bytes_remaining.upper_bound).upper_bound; - bytes_remaining.upper_bound = 0; - } - } - - if (bytes_remaining.lower_bound == 0 && bytes_remaining.upper_bound == 0) { - return result; - } - } - - BATT_CHECK_LE(bytes_remaining.lower_bound, bytes_remaining.upper_bound); - - if (bytes_remaining.lower_bound != 0) { - result.lower_bound = key_upper_bound; - } - - if (bytes_remaining.upper_bound != 0) { - result.upper_bound = key_upper_bound; - } - - return result; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -std::tuple HJoinMergeSet::split_impl(const MergeSet& m, - const KeyView& split_key) const noexcept -{ - // Find the segment where the split_key lives and split that one. - // - auto eq_it = std::equal_range(this->components_.begin(), - this->components_.end(), - split_key, - ExtendedKeyRangeOrder{}); - - // Base cases. - // - if (eq_it.first == this->components_.end()) { - return {clone(m), MergeSet{}}; - } - - if (eq_it.second == this->components_.begin()) { - return {MergeSet{}, clone(m)}; - } - - HJoinMergeSet before_split_impl; - HJoinMergeSet after_split_impl; - - before_split_impl.key_lower_bound_ = this->key_lower_bound_; - before_split_impl.key_upper_bound_ = split_key; - before_split_impl.max_depth_ = m.depth_; - - after_split_impl.key_lower_bound_ = split_key; - after_split_impl.key_upper_bound_ = this->key_upper_bound_; - after_split_impl.max_depth_ = m.depth_; - - // Copy all segments that are definitely before `split_key` to `before_split`. - // - std::for_each( // - this->components_.begin(), - eq_it.first, - [&before_split_impl](const std::unique_ptr& p_segment) { - before_split_impl.add(clone(*p_segment)); - }); - - // Handle the matched range if non-empty. - // - if (eq_it.first != eq_it.second) { - // If the split key is *not* at the exact start of the matched segment, then split that segment - // and assign the resulting parts accordingly. - // - if (get_key_range(**eq_it.first).lower_bound < split_key) { - MergeSet middle_lower, middle_upper; - - std::tie(middle_lower, middle_upper) = split(**eq_it.first, split_key); - - before_split_impl.add(std::move(middle_lower)); - after_split_impl.add(std::move(middle_upper)); - - } else { - // The split_key is exactly the start of the middle segment; assign it in whole to - // `after_split`. - // - after_split_impl.add(clone(**eq_it.first)); - } - } - - // Copy all segments that are definitely after `split_key` to `after_split`. - // - std::for_each( // - eq_it.second, - this->components_.end(), - [&after_split_impl](const std::unique_ptr& p_segment) { - after_split_impl.add(clone(*p_segment)); - }); - - // Form the output sets. - // - return { - MergeSet{std::move(before_split_impl), m.depth_}, - MergeSet{std::move(after_split_impl), m.depth_}, - }; -} - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp b/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp deleted file mode 100644 index f12c335..0000000 --- a/src/turtle_kv/tree/merge_set/h_join_merge_set.hpp +++ /dev/null @@ -1,73 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_H_JOIN_MERGE_SET_HPP - -#include "composite_merge_set.hpp" - -#include - -#include - -namespace turtle_kv { -namespace merge_set { - -struct MergeSet; - -CInterval get_byte_size(const MergeSet& m) noexcept; -Interval get_depth(const MergeSet& m) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -CInterval h_get_byte_size_impl( - const std::vector>& segments) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -Interval h_seek_impl(const std::vector>& segments, - usize byte_size, - const KeyView& key_upper_bound) noexcept; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -struct HJoinMergeSet : CompositeMergeSet { - i32 max_depth_; - - //+++++++++++-+-+--+----- --- -- - - - - - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - void add(MergeSet&& src) noexcept - { - this->max_depth_ = std::max(this->max_depth_, get_depth(src).upper_bound); - this->CompositeMergeSet::add(std::move(src)); - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - Interval seek_impl(usize byte_size, const KeyView& key_upper_bound) const noexcept - { - return h_seek_impl(this->components_, byte_size, key_upper_bound); - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - CInterval get_byte_size_impl() const noexcept - { - return h_get_byte_size_impl(this->components_); - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - std::tuple split_impl(const MergeSet& m, - const KeyView& split_key) const noexcept; -}; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp b/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp deleted file mode 100644 index 81b547b..0000000 --- a/src/turtle_kv/tree/merge_set/in_memory_merge_set.hpp +++ /dev/null @@ -1,70 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_EMPTY_MERGE_SET_HPP - -#include "fake_key_value.hpp" - -#include -#include -#include - -#include -#include - -namespace turtle_kv { -namespace merge_set { - -struct MergeSet; - -struct InMemoryMergeSet { - std::shared_ptr> storage_; - Interval key_range_; - Interval index_range_; - //----- --- -- - - - - - std::string key_upper_bound_; - - //+++++++++++-+-+--+----- --- -- - - - - - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - Slice live_slice() const noexcept - { - return { - (*this->storage_).data() + this->index_range_.lower_bound, - (*this->storage_).data() + this->index_range_.upper_bound, - }; - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - Interval seek_impl(u64 byte_size, const KeyView& key_upper_bound) const noexcept - { - usize total = 0; - usize index = this->index_range_.lower_bound; - for (const FakeKeyValue& kv : this->live_slice()) { - const usize n = packed_sizeof(kv); - if (total + n > byte_size) { - break; - } - total += n; - ++index; - } - if (index == this->storage_->size()) { - return { - get_key((*this->storage_)[index]), - key_upper_bound, - }; - } - return { - get_key((*this->storage_)[index]), - get_key((*this->storage_)[index + 1]), - }; - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - std::tuple split_impl(const MergeSet& m, - const KeyView& split_key) const noexcept; -}; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp b/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp deleted file mode 100644 index 341b8f6..0000000 --- a/src/turtle_kv/tree/merge_set/in_storage_merge_set.hpp +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_IN_STORAGE_MERGE_SET_HPP - -#include "fake_leaf.hpp" - -#include - -#include -#include - -#include - -namespace turtle_kv { -namespace merge_set { - -struct MergeSet; - -struct InStorageMergeSet { - std::shared_ptr leaf_; - std::string key_upper_bound_; - PiecewiseFilter filter_; - - //+++++++++++-+-+--+----- --- -- - - - - - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - Interval seek_impl(u64 byte_size, const KeyView& key_upper_bound) const noexcept - { - const usize block_count = this->leaf_->block_count(); - const usize block_size = this->leaf_->block_size(); - - usize live_i = this->filter_.live_lower_bound(0); - usize block_i = this->leaf_->block_containing_index(live_i); - - while (block_i < block_count) { - if (byte_size < block_size) { - break; - } - - byte_size -= block_size; - live_i = this->filter_.live_lower_bound(this->leaf_->block_starts_[block_i + 1]); - block_i = this->leaf_->block_containing_index(live_i); - } - - return { - (block_i + 0 < block_count) ? this->leaf_->block_keys_[block_i + 0] : key_upper_bound, - (block_i + 1 < block_count) ? this->leaf_->block_keys_[block_i + 1] : key_upper_bound, - }; - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - std::tuple split_impl(const MergeSet& m, - const KeyView& split_key) const noexcept; -}; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/merge_set.cpp b/src/turtle_kv/tree/merge_set/merge_set.cpp deleted file mode 100644 index 1e33fde..0000000 --- a/src/turtle_kv/tree/merge_set/merge_set.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "merge_set.hpp" -// - -#include - -namespace turtle_kv { -namespace merge_set { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -MergeSet clone(const MergeSet& m) noexcept -{ - MergeSet m2; - - m2.impl_ = batt::case_of( // - m.impl_, - [&](const EmptyMergeSet& e) -> MergeSet::Impl { - return e; - }, - [&](const InMemoryMergeSet& i) -> MergeSet::Impl { - return i; - }, - [&](const InStorageMergeSet& i) -> MergeSet::Impl { - return i; - }, - [&](const HJoinMergeSet& h) -> MergeSet::Impl { - return clone_impl(h); - }, - [&](const VJoinMergeSet& v) -> MergeSet::Impl { - return clone_impl(v); - }); - - m2.depth_ = m.depth_; - m2.byte_size_ = m.byte_size_; - m2.key_range_ = m.key_range_; - - return m2; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -Interval get_depth(const MergeSet& m) noexcept -{ - return batt::case_of( // - m.impl_, - [&](const EmptyMergeSet&) -> Interval { - return {m.depth_, m.depth_}; - }, - [&](const InMemoryMergeSet&) -> Interval { - return {m.depth_, m.depth_ + 1}; - }, - [&](const InStorageMergeSet&) -> Interval { - return {m.depth_, m.depth_ + 1}; - }, - [&](const HJoinMergeSet& h) -> Interval { - return {m.depth_, h.max_depth_}; - }, - [&](const VJoinMergeSet& v) -> Interval { - return {m.depth_, m.depth_ + (i32)v.components_.size()}; - }); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -CInterval get_byte_size(const MergeSet& m) noexcept -{ - return m.byte_size_; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -Interval get_key_range(const MergeSet& m) noexcept -{ - return m.key_range_; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -Interval seek(const MergeSet& m, u64 byte_size) noexcept -{ - return batt::case_of( // - m.impl_, - [&](const EmptyMergeSet&) -> Interval { - return m.key_range_; - }, - [&](const auto& impl) -> Interval { - return impl.seek_impl(byte_size, m.key_range_.upper_bound); - }); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -std::tuple split(const MergeSet& m, const KeyView& split_key) noexcept -{ - return batt::case_of( // - m.impl_, - [&](const EmptyMergeSet& e) -> std::tuple { - return std::tuple{e, e}; - }, - [&](const auto& impl) -> std::tuple { - return impl.split_impl(m, split_key); - }); -} - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/merge_set.hpp b/src/turtle_kv/tree/merge_set/merge_set.hpp deleted file mode 100644 index bbd29f4..0000000 --- a/src/turtle_kv/tree/merge_set/merge_set.hpp +++ /dev/null @@ -1,93 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_MERGE_SET_HPP - -#include "empty_merge_set.hpp" -#include "h_join_merge_set.hpp" -#include "in_memory_merge_set.hpp" -#include "in_storage_merge_set.hpp" -#include "v_join_merge_set.hpp" - -#include - -#include - -#include -#include - -namespace turtle_kv { -namespace merge_set { - -struct MergeSet { - using Impl = std::variant< // - EmptyMergeSet, // - InMemoryMergeSet, // - InStorageMergeSet, // - HJoinMergeSet, // - VJoinMergeSet // - >; - - CInterval byte_size_; - Interval key_range_; - Impl impl_; - i32 depth_; - - //+++++++++++-+-+--+----- --- -- - - - - - - MergeSet() noexcept : byte_size_{0, 0}, key_range_{{}, {}}, impl_{EmptyMergeSet{}}, depth_{0} - { - } - - explicit MergeSet(const EmptyMergeSet&) noexcept : MergeSet{} - { - } - - explicit MergeSet(HJoinMergeSet&& impl, i32 depth) noexcept - : byte_size_{impl.get_byte_size_impl()} - , key_range_{impl.key_lower_bound_, impl.key_upper_bound_} - , impl_{std::move(impl)} - , depth_{depth} - { - } - - MergeSet(const MergeSet&) = delete; - MergeSet& operator=(const MergeSet&) = delete; - - MergeSet(MergeSet&&) = default; - MergeSet& operator=(MergeSet&&) = default; -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -MergeSet clone(const MergeSet& m) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/** \brief Returns the level depth range of the passed MergeSet. - */ -Interval get_depth(const MergeSet& m) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/** \brief Returns the minimum known bounding range of merged byte size for the passed set. - */ -CInterval get_byte_size(const MergeSet& m) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/** \brief Returns the minimum known bounding key range of the passed set. - */ -Interval get_key_range(const MergeSet& m) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/** \brief Returns the minimum interval in which lies the true upper bound corresponding to the - * specified number of bytes (as measured from the beginning of the final merged version of `m`). - */ -Interval seek(const MergeSet& m, u64 byte_size) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -std::tuple split(const MergeSet& m, const KeyView& split_key) noexcept; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/merge_set.test.cpp b/src/turtle_kv/tree/merge_set/merge_set.test.cpp deleted file mode 100644 index 2ae4e6d..0000000 --- a/src/turtle_kv/tree/merge_set/merge_set.test.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -// -#include - -#include -#include - -namespace turtle_kv { -namespace merge_set { -namespace { - -TEST(TreeMergeSetMergeSetTest, Test) -{ -} - -} // namespace -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/random_key_value.hpp b/src/turtle_kv/tree/merge_set/random_key_value.hpp deleted file mode 100644 index 05ef605..0000000 --- a/src/turtle_kv/tree/merge_set/random_key_value.hpp +++ /dev/null @@ -1,127 +0,0 @@ -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_RANDOM_KEY_VALUE_HPP - -#include "fake_key_value.hpp" -#include "fake_leaf.hpp" - -#include - -#include - -#include - -namespace turtle_kv { -namespace merge_set { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline std::string random_str(Rng&& rng, PickLen&& pick_len, PickChar&& pick_char) noexcept -{ - const usize len = pick_len(rng); - std::string s(len, '\0'); - for (usize i = 0; i < len; ++i) { - s[i] = (char)pick_char(rng); - } - return s; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline FakeKeyValue random_key_value(Rng&& rng, - PickKeyLen&& pick_key_len, - PickKeyChar&& pick_key_char, - PickValueLen&& pick_value_len, - PickValueChar&& pick_value_char) noexcept -{ - return FakeKeyValue{ - .key_ = random_str(BATT_FORWRD(rng), // - BATT_FORWARD(pick_key_len), - BATT_FORWARD(pick_key_char)), - .value_ = random_str(BATT_FORWRD(rng), // - BATT_FORWARD(pick_value_len), - BATT_FORWARD(pick_value_char)), - }; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline std::vector random_block(usize size_limit, - Rng&& rng, - PickKeyLen&& pick_key_len, - PickKeyChar&& pick_key_char, - PickValueLen&& pick_value_len, - PickValueChar&& pick_value_char) noexcept -{ - std::vector block; - usize size = 0; - for (;;) { - block.emplace_back(random_key_value(BATT_FORWARD(rng), - BATT_FORWARD(pick_key_len), - BATT_FORWARD(pick_key_char), - BATT_FORWARD(pick_value_len), - BATT_FORWARD(pick_value_char))); - size += packed_sizeof(block.back()); - if (size > size_limit) { - block.pop_back(); - break; - } - } - return block; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline FakeLeaf random_leaf(usize block_size_limit, - Rng&& rng, - PickNumBlocks&& pick_num_blocks, - PickKeyLen&& pick_key_len, - PickKeyChar&& pick_key_char, - PickValueLen&& pick_value_len, - PickValueChar&& pick_value_char) noexcept -{ - FakeLeaf leaf; - usize key_count = 0; - - leaf.block_size_ = block_size_limit; - - const usize num_blocks = pick_num_blocks(rng); - for (usize i = 0; i < num_blocks; ++i) { - FakeBlock block{ - .items_ = random_block(block_size_limit, - BATT_FORWARD(rng), - BATT_FORWARD(pick_key_len), - BATT_FORWARD(pick_key_char), - BATT_FORWARD(pick_value_len), - BATT_FORWARD(pick_value_char)), - }; - - leaf.block_starts_.push_back(key_count); - leaf.block_keys_.push_back(std::string{get_key(block.items_.front())}); - leaf.blocks_.emplace_back(std::move(block)); - - key_count += block.items_.size(); - } - leaf.block_starts_.push_back(key_count); - - return leaf; -} - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/v_join_merge_set.cpp b/src/turtle_kv/tree/merge_set/v_join_merge_set.cpp deleted file mode 100644 index 8e79394..0000000 --- a/src/turtle_kv/tree/merge_set/v_join_merge_set.cpp +++ /dev/null @@ -1,43 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include "v_join_merge_set.hpp" -// - -#include "h_join_merge_set.hpp" -#include "merge_set.hpp" - -namespace turtle_kv { -namespace merge_set { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -Interval VJoinMergeSet::seek_impl(usize byte_size, - const KeyView& key_upper_bound) const noexcept -{ - Interval result; - - const std::vector>& levels = this->components_; - - // At the two extremes, the levels could have all the same keys or share none. In the former - // case, we need to take the min/max of the level-wise seek result, and in the latter, we do the - // same thing as for HJoinMergeSet. - // - result = h_seek_impl(levels, byte_size, key_upper_bound); - - for (const std::unique_ptr& level : levels) { - Interval level_result = seek(*level, byte_size); - result.lower_bound = std::min(result.lower_bound, level_result.lower_bound); - result.upper_bound = std::max(result.upper_bound, level_result.upper_bound); - } - - return result; -} - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp b/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp deleted file mode 100644 index bc09100..0000000 --- a/src/turtle_kv/tree/merge_set/v_join_merge_set.hpp +++ /dev/null @@ -1,34 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_MERGE_SET_V_JOIN_MERGE_SET_HPP - -#include "composite_merge_set.hpp" - -#include - -#include -#include - -namespace turtle_kv { -namespace merge_set { - -struct VJoinMergeSet : CompositeMergeSet { - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - Interval seek_impl(usize byte_size, const KeyView& key_upper_bound) const noexcept; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - std::tuple split_impl(const MergeSet& m, - const KeyView& split_key) const noexcept; -}; - -} // namespace merge_set -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/testing/in_memory_block_loader.hpp b/src/turtle_kv/tree/testing/in_memory_block_loader.hpp new file mode 100644 index 0000000..9450053 --- /dev/null +++ b/src/turtle_kv/tree/testing/in_memory_block_loader.hpp @@ -0,0 +1,39 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once + +#include +#include + +#include +#include + +namespace turtle_kv { +namespace testing { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class InMemoryBlockLoader +{ + public: + explicit InMemoryBlockLoader(const PackedBlockedLeafPage* leaf) noexcept : leaf_{leaf} + { + } + + StatusOr load_block(u32 block_index) noexcept + { + return &*(this->leaf_->blocks_begin() + block_index); + } + + private: + const PackedBlockedLeafPage* leaf_; +}; + +} // namespace testing +} // namespace turtle_kv From 6cb7e5fa856a62ab929031de64d6b615a1fc845f Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Wed, 12 Aug 2026 14:58:03 -0400 Subject: [PATCH 19/20] Delete unused file --- .../leaf/packed_blocked_leaf_page.test.cpp | 4 +- .../tree/packed_leaf_block_scanner.hpp | 75 ------------------- 2 files changed, 2 insertions(+), 77 deletions(-) delete mode 100644 src/turtle_kv/tree/packed_leaf_block_scanner.hpp diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index 13443d6..a2bcbf5 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -59,6 +59,8 @@ using turtle_kv::ValueView; class PackedBlockedLeafPageTest : public ::testing::Test { public: + using StorageUnit = std::aligned_storage_t<4096, 4096>; + static constexpr usize kLeafPageSize = 1 * kMiB; static constexpr usize kNumPrefixes = 1000; static constexpr usize kMinPrefixSize = 0; @@ -149,7 +151,6 @@ class PackedBlockedLeafPageTest : public ::testing::Test StatusOr pack_leaf() { - using StorageUnit = std::aligned_storage_t<4096, 4096>; this->leaf_storage_.resize(kLeafPageSize / sizeof(StorageUnit)); BATT_CHECK_EQ(sizeof(StorageUnit) * this->leaf_storage_.size(), kLeafPageSize); @@ -176,7 +177,6 @@ class PackedBlockedLeafPageTest : public ::testing::Test std::vector edits_; private: - using StorageUnit = std::aligned_storage_t<4096, 4096>; std::vector leaf_storage_; }; diff --git a/src/turtle_kv/tree/packed_leaf_block_scanner.hpp b/src/turtle_kv/tree/packed_leaf_block_scanner.hpp deleted file mode 100644 index 5f5d81a..0000000 --- a/src/turtle_kv/tree/packed_leaf_block_scanner.hpp +++ /dev/null @@ -1,75 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_SCANNER_HPP - -#include - -#include - -#include - -#include - -#include - -namespace turtle_kv { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -concept PackedLeafBlockProvider = requires(T& provider, usize block_index) { - { provider.get_block(block_index) } -> std::convertible_to>; -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -template -class PackedLeafBlockScanner -{ - public: - class Impl - { - public: - using Item = StatusOr; - - Optional poll() noexcept - { - } - - Optional next() noexcept - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - private: - void advance() noexcept - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - PackedBlockedLeafPage::HeaderShardView header_; - - BlockProviderT& provider_; - - PiecewiseFilter& filter_; - - usize block_index_; - - Optional> block_; - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - private: - Impl* impl_; -}; - -} // namespace turtle_kv From 4262cf87517bcc9ad4a72343e931ec7869556416 Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Fri, 14 Aug 2026 17:06:25 -0400 Subject: [PATCH 20/20] Fix scan_blocked_leaf return type. --- .../leaf/packed_blocked_leaf_page.test.cpp | 53 ++++++++++--------- src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp | 16 +++--- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index a2bcbf5..3be0fba 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -59,8 +59,6 @@ using turtle_kv::ValueView; class PackedBlockedLeafPageTest : public ::testing::Test { public: - using StorageUnit = std::aligned_storage_t<4096, 4096>; - static constexpr usize kLeafPageSize = 1 * kMiB; static constexpr usize kNumPrefixes = 1000; static constexpr usize kMinPrefixSize = 0; @@ -151,6 +149,7 @@ class PackedBlockedLeafPageTest : public ::testing::Test StatusOr pack_leaf() { + using StorageUnit = std::aligned_storage_t<4096, 4096>; this->leaf_storage_.resize(kLeafPageSize / sizeof(StorageUnit)); BATT_CHECK_EQ(sizeof(StorageUnit) * this->leaf_storage_.size(), kLeafPageSize); @@ -177,6 +176,7 @@ class PackedBlockedLeafPageTest : public ::testing::Test std::vector edits_; private: + using StorageUnit = std::aligned_storage_t<4096, 4096>; std::vector leaf_storage_; }; @@ -358,7 +358,6 @@ TEST_F(PackedBlockedLeafPageTest, ScanBlockedLeaf) { using turtle_kv::testing::InMemoryBlockLoader; using turtle_kv::scan_blocked_leaf; - using turtle_kv::PackedKeyValueSlotSlice; const usize kFirstSeed = 0; const usize kNumSeeds = 250; @@ -419,32 +418,34 @@ TEST_F(PackedBlockedLeafPageTest, ScanBlockedLeaf) } } - // Run scan_blocked_leaf and collect results. + // Run scan_blocked_leaf and verify results. // - std::vector> actual; - - scan_blocked_leaf(&packed_leaf, &block_loader, leaf_filter, key_range) // - | batt::seq::for_each([&](const StatusOr& status_or_slice) { - ASSERT_TRUE(status_or_slice.ok()) << BATT_INSPECT(status_or_slice.status()); - std::visit( - [&](const auto& s) { - for (const auto& slot : s) { - actual.emplace_back(get_key(slot), get_value(slot)); - } - }, - *status_or_slice); - }); - - ASSERT_EQ(actual.size(), expected.size()) - << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(drop_count) - << BATT_INSPECT(lower_key) << BATT_INSPECT(upper_key); + usize actual_i = 0; + + auto scan_seq = scan_blocked_leaf(&packed_leaf, &block_loader, leaf_filter, key_range) + | batt::seq::status_ok(); - for (usize i = 0; i < expected.size(); ++i) { - ASSERT_EQ(actual[i].first, expected[i].first) - << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(i); - ASSERT_EQ(actual[i].second, expected[i].second) - << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(i); + for (;;) { + auto slice = scan_seq.next(); + if (!slice) { + break; + } + for (const auto& slot : *slice) { + ASSERT_LT(actual_i, expected.size()) + << BATT_INSPECT(seed) << BATT_INSPECT(trial); + ASSERT_EQ(get_key(slot), expected[actual_i].first) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(actual_i); + ASSERT_EQ(get_value(slot), expected[actual_i].second) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(actual_i); + ++actual_i; + } } + + ASSERT_TRUE(scan_seq.status().ok()) << BATT_INSPECT(scan_seq.status()); + + ASSERT_EQ(actual_i, expected.size()) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(drop_count) + << BATT_INSPECT(lower_key) << BATT_INSPECT(upper_key); } } } diff --git a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp index 5f6fce1..52090b9 100644 --- a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp +++ b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp @@ -14,7 +14,7 @@ #include "packed_blocked_leaf_page.sharded_live_ranges.ipp" #include -#include +#include #include #include @@ -25,7 +25,6 @@ namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template FilterModelT, typename BlockLoaderT> -/*BoxedSeq*/ auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, BlockLoaderT* block_loader, const BasicPiecewiseFilter& filter, @@ -41,7 +40,7 @@ auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, batt::seq::filter_map( [packed_leaf, block_loader, key_range]( const typename PackedBlockedLeafPage::ShardedLiveRanges::Item& item) - -> Optional> { + -> Optional>> { if (item.live_item_range.empty()) { return None; } @@ -51,15 +50,14 @@ auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, return block.status(); } - PackedKeyValueSlotSlice slice = + Slice slice = std::get>( packed_leaf->get_slice_within_block(item.block_index, *block, - item.live_item_range); + item.live_item_range)); if (item.is_first || item.is_last) { - auto& ptr_slice = std::get>(slice); - const auto* begin = ptr_slice.begin(); - const auto* end = ptr_slice.end(); + const auto* begin = slice.begin(); + const auto* end = slice.end(); if (item.is_first) { begin = @@ -83,7 +81,7 @@ auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, if (begin == end) { return None; } - slice = PackedKeyValueSlotSlice{as_slice(begin, end)}; + slice = as_slice(begin, end); } return slice;