diff --git a/conan.lock b/conan.lock index 9aa5830..d85e847 100644 --- a/conan.lock +++ b/conan.lock @@ -1,8 +1,9 @@ { "version": "0.5", "requires": [ - "abseil/20250127.0", - "batteries/0.71.1", + "abseil/20260107.1", + "artc/0.2.1", + "batteries/0.72.0", "boost/1.88.0", "bzip2/1.0.8", "cli11/2.5.0", @@ -10,16 +11,16 @@ "glog/0.7.1", "gtest/1.17.0", "keyvcr/0.2.2", - "libbacktrace/cci.20240730", + "libbacktrace/cci.20210118", "libfuse/3.16.2", "libpfm4/4.13.0", "libunwind/1.8.1", "liburing/2.11", - "llfs/0.47.1", + "llfs/0.47.2", "openssl/3.6.0", "pcg-cpp/cci.20220409", "protobuf/3.21.12", - "vqf/0.2.5", + "vqf/0.2.6", "xxhash/0.8.3", "yaml-cpp/0.9.0", "zlib/1.3.1" @@ -49,10 +50,7 @@ ], "boost/[>=1.84.0 <2]": [ "boost/1.88.0" - ], - "batteries/[>=0.60.2 <2]": [ - "batteries/0.70.2" ] }, "config_requires": [] -} \ No newline at end of file +} diff --git a/conanfile.py b/conanfile.py index d90205b..186c7f2 100644 --- a/conanfile.py +++ b/conanfile.py @@ -16,7 +16,7 @@ class TurtleKvRecipe(ConanFile): name = "turtle_kv" - python_requires = "cor_recipe_utils/0.19.1" + python_requires = "cor_recipe_utils/0.21.4.dev3+g0d8231b80" python_requires_extend = "cor_recipe_utils.ConanFileBase" settings = "os", "compiler", "build_type", "arch" @@ -88,18 +88,19 @@ def requirements(self): "force": True, } - self.requires("abseil/20250127.0", **VISIBLE, **OVERRIDE) - self.requires("batteries/[>=0.71.1 <1]", **VISIBLE, **OVERRIDE) - self.requires("boost/1.88.0", **VISIBLE, **OVERRIDE) - self.requires("glog/0.7.1", **VISIBLE) - self.requires("llfs/[>=0.47.0 <1]", **VISIBLE) - self.requires("pcg-cpp/cci.20220409", **VISIBLE) + self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) + self.requires("artc/[>=0.2.1 <1]") + self.requires("batteries/[>=0.72.0 <1]", **VISIBLE, **OVERRIDE) + self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) + self.requires("glog/[>=0.7.1 <1]", **VISIBLE) + self.requires("llfs/[>=0.47.2 <1]", **VISIBLE) + self.requires("pcg-cpp/[>=cci.20220409]", **VISIBLE) self.requires("yaml-cpp/[>=0.9.0 <1]") self.requires("zlib/1.3.1", **OVERRIDE) # boost/1.88.0 and ninja/1.13.2 depend (exactly) on libbacktrace/cci.20210118 # - self.requires("libbacktrace/[>=cci.20240730]", **OVERRIDE) + self.requires("libbacktrace/[>=cci.20210118]") if platform.system() == "Linux": if self.options.with_keyvcr: diff --git a/cor.yml b/cor.yml index 2a24d1b..0d48710 100644 --- a/cor.yml +++ b/cor.yml @@ -1,3 +1,3 @@ cor: cli: - version: 0.19.1 + version: 0.21.4.dev3+g0d8231b80 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8631e97..ab268b2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,7 @@ endif () target_link_libraries( turtle_kv PUBLIC + artc::artc abseil::abseil batteries::batteries boost::boost diff --git a/src/turtle_kv/core/edit_view.hpp b/src/turtle_kv/core/edit_view.hpp index b5e6f62..9fc9c20 100644 --- a/src/turtle_kv/core/edit_view.hpp +++ b/src/turtle_kv/core/edit_view.hpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #pragma once #include diff --git a/src/turtle_kv/core/key_range.hpp b/src/turtle_kv/core/key_range.hpp index 16febe1..5a9b96f 100644 --- a/src/turtle_kv/core/key_range.hpp +++ b/src/turtle_kv/core/key_range.hpp @@ -34,30 +34,17 @@ inline CInterval get_key_range(const Chunk& chunk) }; } -inline CInterval get_key_range(const EditView& edit) +template +inline CInterval get_key_range(const T& has_key_view) { return CInterval{ - .lower_bound = get_key(edit), - .upper_bound = get_key(edit), - }; -} - -inline CInterval get_key_range(const ItemView& item) -{ - return CInterval{ - .lower_bound = get_key(item), - .upper_bound = get_key(item), - }; -} - -inline CInterval get_key_range(const KeyView& key) -{ - return CInterval{ - .lower_bound = key, - .upper_bound = key, + .lower_bound = get_key(has_key_view), + .upper_bound = get_key(has_key_view), }; } +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// struct ExtendedKeyRangeOrder : llfs::KeyRangeOrder { //+++++++++++-+-+--+----- --- -- - - - - @@ -89,4 +76,24 @@ struct ExtendedKeyRangeOrder : llfs::KeyRangeOrder { } }; +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct KeyLowerBoundOrder : KeyOrder { + template + bool operator()(const L& l, const R& r) const + { + return KeyOrder::operator()(get_key_range(l).lower_bound, get_key_range(r).lower_bound); + } +}; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct KeyUpperBoundOrder : KeyOrder { + template + bool operator()(const L& l, const R& r) const + { + return KeyOrder::operator()(get_key_range(l).upper_bound, get_key_range(r).upper_bound); + } +}; + } // namespace turtle_kv diff --git a/src/turtle_kv/core/key_view.hpp b/src/turtle_kv/core/key_view.hpp index 7c91fa8..bbf16b6 100644 --- a/src/turtle_kv/core/key_view.hpp +++ b/src/turtle_kv/core/key_view.hpp @@ -85,4 +85,13 @@ inline usize packed_key_data_size(const KeyView& key) return key.size(); } +template +concept HasKeyView = requires(const T& obj) { + { get_key(obj) } -> std::convertible_to; +}; + +static_assert(HasKeyView); +static_assert(HasKeyView); +static_assert(HasKeyView); + } // namespace turtle_kv diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index 9bc08d2..342f315 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -17,8 +17,161 @@ #include #include +#include + namespace turtle_kv { +struct PackedKeyValueSlot; + +using PackedKeyValueSlotPtr = llfs::PackedPointer; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedKeyValueSlot { + little_u16 key_size; + char key_data_[0]; + + //----- --- -- - - - - + // u8 key_bytes[this->key_size] + //----- --- -- - - - - + // u8 op_code + // u8 value_bytes[this->item_size - offsetof(this->value_bytes)] + //----- --- -- - - - - + + //+++++++++++-+-+--+----- --- -- - - - - + + PackedKeyValueSlot(const PackedKeyValueSlot&) = delete; + PackedKeyValueSlot& operator=(const PackedKeyValueSlot&) = delete; + + //+++++++++++-+-+--+----- --- -- - - - - + + usize slot_size(const PackedKeyValueSlotPtr* p_this) const noexcept + { + const PackedKeyValueSlotPtr* const p_next = p_this + 1; + return usize{p_next->offset.value()} - usize{p_this->offset.value()} + + sizeof(PackedKeyValueSlotPtr); + } + + const char* key_data() const noexcept + { + return this->key_data_; + } + + KeyView key_view() const noexcept + { + return KeyView{this->key_data_, this->key_size}; + } + + const char* value_data() const noexcept + { + return this->key_data() + (this->key_size + 1); + } + + const char* value_data_end(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_data_end(/*size_of_slot=*/this->slot_size(p_this)); + } + + const char* value_data_end(usize size_of_slot) const noexcept + { + return reinterpret_cast(this) + size_of_slot; + } + + usize value_size(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_size(/*size_of_slot=*/this->slot_size(p_this)); + } + + usize value_size(usize size_of_slot) const noexcept + { + return this->value_data_end(size_of_slot) - this->value_data(); + } + + ValueView::OpCode value_op_code() const noexcept + { + return static_cast(this->key_data_[this->key_size]); + } + + ValueView value_view(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_view(/*size_of_slot=*/this->slot_size(p_this)); + } + + ValueView value_view(usize size_of_slot) const noexcept + { + return ValueView::from_packed( + this->value_op_code(), + std::string_view{this->value_data(), this->value_size(size_of_slot)}); + } +}; + +inline KeyView get_key(const PackedKeyValueSlot& packed_slot) noexcept +{ + return packed_slot.key_view(); +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// TODO [tastolfi 2026-05-31] use 16-bit pointer tagging to store slot_size with the pointer in one +// 64-bit word. +// +struct PackedKeyValueSlotRef { + const PackedKeyValueSlot* p_slot; + usize slot_size; +}; + +inline KeyView get_key(const PackedKeyValueSlotRef& slot_ref) noexcept +{ + return slot_ref.p_slot->key_view(); +} + +inline ValueView get_value(const PackedKeyValueSlotRef& slot_ref) noexcept +{ + return slot_ref.p_slot->value_view(slot_ref.slot_size); +} + +inline const PackedKeyValueSlotRef& to_key_value_slot_ref(const PackedKeyValueSlotRef& ref) noexcept +{ + return ref; +} + +inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr* pp_slot) noexcept +{ + return PackedKeyValueSlotRef{ + .p_slot = pp_slot->get(), + .slot_size = pp_slot->get()->slot_size(pp_slot), + }; +} + +inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr& p_slot_ref) noexcept +{ + return to_key_value_slot_ref(std::addressof(p_slot_ref)); +} + +inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffer) noexcept +{ + return PackedKeyValueSlotRef{ + .p_slot = static_cast(slot_buffer.data()), + .slot_size = slot_buffer.size(), + }; +} + +inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return get_key(*p_kv); +} + +inline ValueView get_value(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return p_kv->value_view(std::addressof(p_kv)); +} + +template +concept ConvertibleToKeyValueSlotRef = requires(const T& obj) { + { to_key_value_slot_ref(obj) } -> std::convertible_to; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Returns the size required (in bytes) to pack a slot with the passed key and * value. */ @@ -30,6 +183,21 @@ inline usize packed_key_value_slot_size(const KeyView& key, const ValueView& val + value.size(); } +template +inline usize packed_key_value_slot_size(const T& obj) noexcept +{ + return to_key_value_slot_ref(obj).slot_size; +} + +template + requires HasKeyView && HasValueView +inline usize packed_key_value_slot_size(const T& obj) noexcept +{ + return packed_key_value_slot_size(get_key(obj), get_value(obj)); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Serializes the passed key and value into the destination buffer. */ inline std::pair pack_key_value_slot(const KeyView& key, @@ -72,6 +240,37 @@ inline std::pair pack_key_value_slot(const KeyView& key, })); } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief + */ +template + requires HasKeyView && HasValueView +inline usize pack_key_value_slot(const T& src, void* dst) noexcept +{ + const KeyView& key = get_key(src); + const ValueView& value = get_value(src); + const usize slot_size = packed_key_value_slot_size(key, value); + + pack_key_value_slot(key, value, MutableBuffer{dst, slot_size}); + + return slot_size; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief + */ +template +inline usize pack_key_value_slot(const T& src, void* dst) noexcept +{ + const PackedKeyValueSlotRef& slot_ref = to_key_value_slot_ref(src); + std::memcpy(dst, slot_ref.p_slot, slot_ref.slot_size); + return slot_ref.slot_size; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Unpacks a key/value pair from the passed packed slot buffer. */ inline StatusOr> unpack_key_value_slot(ConstBuffer payload) diff --git a/src/turtle_kv/core/packed_key_value_slot_slice.hpp b/src/turtle_kv/core/packed_key_value_slot_slice.hpp new file mode 100644 index 0000000..7f2ee39 --- /dev/null +++ b/src/turtle_kv/core/packed_key_value_slot_slice.hpp @@ -0,0 +1,36 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_CORE_PACKED_KEY_VALUE_SLOT_SLICE_HPP + +#include + +#include + +#include + +namespace turtle_kv { + +using PackedKeyValueSlotSlice = std::variant< // + Slice, + Slice>; + +struct ToPackedKeyValueSlotSlice { + PackedKeyValueSlotSlice operator()(const Slice& ref_slice) + { + return PackedKeyValueSlotSlice{ref_slice}; + } + + PackedKeyValueSlotSlice operator()(const Slice& ptr_slice) + { + return PackedKeyValueSlotSlice{ptr_slice}; + } +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/core/value_view.hpp b/src/turtle_kv/core/value_view.hpp index e79673f..3fb4d7b 100644 --- a/src/turtle_kv/core/value_view.hpp +++ b/src/turtle_kv/core/value_view.hpp @@ -404,4 +404,14 @@ inline bool decays_to_item(const ValueView& value) return false; } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// + +template +concept HasValueView = requires(const T& obj) { + { get_value(obj) } -> std::convertible_to; +}; + +static_assert(HasValueView); + } // namespace turtle_kv diff --git a/src/turtle_kv/import/buffer.hpp b/src/turtle_kv/import/buffer.hpp index 8640034..0d712e7 100644 --- a/src/turtle_kv/import/buffer.hpp +++ b/src/turtle_kv/import/buffer.hpp @@ -6,14 +6,13 @@ namespace turtle_kv { +using batt::advance_pointer; using batt::buffer_from_struct; +using batt::byte_distance; using batt::ConstBuffer; using batt::make_buffer; using batt::mutable_buffer_from_struct; using batt::MutableBuffer; using batt::resize_buffer; -using llfs::advance_pointer; -using llfs::byte_distance; - } // namespace turtle_kv diff --git a/src/turtle_kv/import/interval.hpp b/src/turtle_kv/import/interval.hpp index 53167ea..33cfe08 100644 --- a/src/turtle_kv/import/interval.hpp +++ b/src/turtle_kv/import/interval.hpp @@ -2,9 +2,6 @@ #include -#include -#include - namespace turtle_kv { using batt::BasicInterval; diff --git a/src/turtle_kv/import/slice.hpp b/src/turtle_kv/import/slice.hpp index 00a8b22..abf769a 100644 --- a/src/turtle_kv/import/slice.hpp +++ b/src/turtle_kv/import/slice.hpp @@ -6,7 +6,6 @@ namespace turtle_kv { using batt::as_const_slice; using batt::as_range; -using batt::as_seq; using batt::as_slice; using batt::Slice; diff --git a/src/turtle_kv/kv_store.cpp b/src/turtle_kv/kv_store.cpp index 6e07922..d2f52fd 100644 --- a/src/turtle_kv/kv_store.cpp +++ b/src/turtle_kv/kv_store.cpp @@ -474,7 +474,7 @@ KVStore::~KVStore() noexcept LOG_IF(INFO, show_metrics) << BATT_INSPECT(merge_compactor.average_bytes_per_compaction()); } { - auto& art = ARTBase::default_metrics(); + auto& art = artc::ARTBase::default_metrics(); LOG_IF(INFO, show_metrics) << BATT_INSPECT(art.byte_alloc_count) << BATT_INSPECT(art.construct_count) << BATT_INSPECT(art.destruct_count) @@ -728,7 +728,6 @@ StatusOr KVStore::put(const KeyView& key, return Status{batt::StatusCode::kUnavailable}; } - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // Status KVStore::put(const KeyView& key, const ValueView& value) noexcept /*override*/ @@ -1351,10 +1350,10 @@ using CheckpointEvent = llfs::PackedVariant; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Status KVStore::sync(Optional upper_bound, Optional write_options) noexcept +Status KVStore::sync(Optional upper_bound, + Optional write_options) noexcept { - EditOffset target = - upper_bound ? *upper_bound : this->change_log_writer_->next_edit_offset(); + EditOffset target = upper_bound ? *upper_bound : this->change_log_writer_->next_edit_offset(); bool urgent = write_options && write_options->urgent_sync ? true : false; diff --git a/src/turtle_kv/kv_store_scanner.cpp b/src/turtle_kv/kv_store_scanner.cpp index 4736ff5..4dcc8f4 100644 --- a/src/turtle_kv/kv_store_scanner.cpp +++ b/src/turtle_kv/kv_store_scanner.cpp @@ -11,17 +11,17 @@ namespace { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -KeyView art_scanner_get_key(ART::Scanner& scanner) +template +KeyView art_scanner_get_key(artc::ART::Scanner& scanner) { return scanner.get_key(); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -KeyView art_scanner_get_key(ART::Scanner& scanner) +template +KeyView art_scanner_get_key(artc::ART::Scanner& scanner) { return scanner.get_value().key_view(); } @@ -138,12 +138,12 @@ Status KVStoreScanner::start() // Delta case : single ART index for keys and values // - auto& art_scanner = - *(new (p_mem) ART::Scanner{ - delta_mem_table.art_index(), - this->min_key_, - }); + auto& art_scanner = *(new ( + p_mem) artc::ART::Scanner{ + delta_mem_table.art_index(), + this->min_key_, + }); ++p_mem; if (!art_scanner.is_done()) { @@ -525,10 +525,10 @@ Status KVStoreScanner::set_next_item() // /*explicit*/ KVStoreScanner::ScanLevel::ScanLevel( ActiveMemTableValueTag, - ART::Scanner& - art_scanner) noexcept + artc::ART::Scanner& art_scanner) noexcept : key{art_scanner_get_key(art_scanner)} - , state_impl{MemTableValueScanState{ + , state_impl{MemTableValueScanState{ .art_scanner_ = &art_scanner, }} { @@ -538,10 +538,10 @@ Status KVStoreScanner::set_next_item() // /*explicit*/ KVStoreScanner::ScanLevel::ScanLevel( DeltaMemTableValueTag, - ART::Scanner& - art_scanner) noexcept + artc::ART::Scanner& art_scanner) noexcept : key{art_scanner_get_key(art_scanner)} - , state_impl{MemTableValueScanState{ + , state_impl{MemTableValueScanState{ .art_scanner_ = &art_scanner, }} { @@ -566,11 +566,11 @@ EditView KVStoreScanner::ScanLevel::item() const BATT_PANIC() << "illegal state"; BATT_UNREACHABLE(); }, - [](const MemTableValueScanState& state) -> EditView { + [](const MemTableValueScanState& state) -> EditView { const MemTableValueEntry& entry = state.art_scanner_->get_value(); return EditView{entry.key_view(), entry.value_view()}; }, - [](const MemTableValueScanState& state) -> EditView { + [](const MemTableValueScanState& state) -> EditView { const MemTableValueEntry& entry = state.art_scanner_->get_value(); return EditView{entry.key_view(), entry.value_view()}; }, @@ -598,10 +598,10 @@ ValueView KVStoreScanner::ScanLevel::value() const BATT_PANIC() << "illegal state"; BATT_UNREACHABLE(); }, - [](const MemTableValueScanState& state) -> ValueView { + [](const MemTableValueScanState& state) -> ValueView { return state.art_scanner_->get_value().value_view(); }, - [](const MemTableValueScanState& state) -> ValueView { + [](const MemTableValueScanState& state) -> ValueView { return state.art_scanner_->get_value().value_view(); }, [](const Slice& state) -> ValueView { @@ -660,10 +660,10 @@ bool KVStoreScanner::ScanLevel::advance() BATT_PANIC() << "illegal state"; BATT_UNREACHABLE(); }, - [this](MemTableValueScanState& state) -> bool { + [this](MemTableValueScanState& state) -> bool { return scan_level_mem_table_advance_impl(this, state); }, - [this](MemTableValueScanState& state) -> bool { + [this](MemTableValueScanState& state) -> bool { return scan_level_mem_table_advance_impl(this, state); }, [this](Slice& state) -> bool { diff --git a/src/turtle_kv/kv_store_scanner.hpp b/src/turtle_kv/kv_store_scanner.hpp index e5ad4c6..dbb1471 100644 --- a/src/turtle_kv/kv_store_scanner.hpp +++ b/src/turtle_kv/kv_store_scanner.hpp @@ -26,9 +26,10 @@ #include #include -#include #include +#include + #include #include @@ -102,17 +103,17 @@ class KVStoreScanner //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // - template + template struct MemTableScanState { MemTable* mem_table_; - ART::Scanner* art_scanner_; + artc::ART::Scanner* art_scanner_; }; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // - template + template struct MemTableValueScanState { - ART::Scanner* art_scanner_; + artc::ART::Scanner* art_scanner_; }; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -163,8 +164,8 @@ class KVStoreScanner KeyView key; std::variant, - MemTableValueScanState, + MemTableValueScanState, + MemTableValueScanState, Slice, TreeLevelScanState, TreeLevelScanShardedState, @@ -181,13 +182,13 @@ class KVStoreScanner explicit ScanLevel( ActiveMemTableValueTag, - ART::Scanner& - art_scanner) noexcept; + artc::ART::Scanner& art_scanner) noexcept; explicit ScanLevel( DeltaMemTableValueTag, - ART::Scanner& - art_scanner) noexcept; + artc::ART::Scanner& art_scanner) noexcept; explicit ScanLevel(const Slice& edit_view_slice) noexcept; @@ -327,12 +328,14 @@ class KVStoreScanner //+++++++++++-+-+--+----- --- -- - - - - using DeltaMemTableScannerStorage = std::aligned_storage_t< - /*size=*/std::max(sizeof(ART::Scanner), - sizeof(ART::Scanner)), - /*align=*/std::max(alignof(ART::Scanner), - alignof(ART::Scanner))>; + /*size=*/std::max( + sizeof(artc::ART::Scanner), + sizeof(artc::ART::Scanner)), + /*align=*/std::max( + alignof(artc::ART::Scanner), + alignof(artc::ART::Scanner))>; batt::Toggle::Reader state_reader_; llfs::PageLoader& page_loader_; @@ -344,8 +347,9 @@ class KVStoreScanner bool needs_resume_; Optional next_item_; Status status_; - Optional::Scanner> mem_table_scanner_; - Optional::Scanner> + Optional::Scanner> mem_table_scanner_; + Optional::Scanner> mem_table_value_scanner_; std::array static_delta_storage_; DeltaMemTableScannerStorage* delta_storage_; diff --git a/src/turtle_kv/mem_table/mem_table.hpp b/src/turtle_kv/mem_table/mem_table.hpp index c135c2b..ace3433 100644 --- a/src/turtle_kv/mem_table/mem_table.hpp +++ b/src/turtle_kv/mem_table/mem_table.hpp @@ -29,7 +29,6 @@ #include #include -#include #include #include @@ -39,6 +38,8 @@ #include #include +#include + #include #include @@ -216,7 +217,7 @@ class BasicMemTable : public MemTableBase //+++++++++++-+-+--+----- --- -- - - - - - ART& art_index() + artc::ART& art_index() { return this->art_index_; } @@ -338,11 +339,11 @@ class BasicMemTable : public MemTableBase // Diagnostic metrics for `this->art_index_`. // - ARTBase::Metrics art_metrics_; + artc::ARTBase::Metrics art_metrics_; // In-memory index used for scans and point queries. // - ART art_index_; + artc::ART art_index_; // Tracks the maximum observed key-value pair size (in bytes); this is used to estimate the // worst-case space wasted in a future batch. @@ -490,7 +491,7 @@ class BasicMemTable::PerOpStorageContext // One thread will acquire a lock, others will block at this point. // - absl::MutexLock lock{&this->mem_table_.block_list_mutex_}; + absl::MutexLock lock{this->mem_table_.block_list_mutex_}; // If there are no block buffers attached to the MemTable, then we may just have to wait until // the checkpoint update pipeline catches up. If there are block buffers attached, then its @@ -559,8 +560,8 @@ class BasicMemTable::BatchCompactor public: using Self = BatchCompactor; - using ARTScanner = - ART::Scanner; + using ARTScanner = artc::ART::Scanner; //+++++++++++-+-+--+----- --- -- - - - - diff --git a/src/turtle_kv/mem_table/mem_table.ipp b/src/turtle_kv/mem_table/mem_table.ipp index 0ec2d8e..233ce47 100644 --- a/src/turtle_kv/mem_table/mem_table.ipp +++ b/src/turtle_kv/mem_table/mem_table.ipp @@ -395,7 +395,7 @@ void BasicMemTable::handle_external_cache_alloc(i6 this->allocation_tracker_.allocate_external(cache_alloc_delta, overcommit); { - absl::MutexLock lock{&this->block_list_mutex_}; + absl::MutexLock lock{this->block_list_mutex_}; BATT_CHECK(this->cache_alloc_in_progress_); this->total_cache_alloc_.subsume(std::move(alloc)); this->cache_alloc_in_progress_ = false; @@ -417,7 +417,7 @@ void BasicMemTable::handle_external_cache_alloc(i6 } else if (cache_alloc_delta < 0) { StatusOr alloc_to_release; { - absl::MutexLock lock{&this->block_list_mutex_}; + absl::MutexLock lock{this->block_list_mutex_}; BATT_CHECK(this->cache_alloc_in_progress_); alloc_to_release = this->total_cache_alloc_.split(-cache_alloc_delta); this->cache_alloc_in_progress_ = false; diff --git a/src/turtle_kv/mem_table/mem_table.test.cpp b/src/turtle_kv/mem_table/mem_table.test.cpp index 68a684b..a894348 100644 --- a/src/turtle_kv/mem_table/mem_table.test.cpp +++ b/src/turtle_kv/mem_table/mem_table.test.cpp @@ -378,9 +378,9 @@ TEST_F(MemTableTest, PutGet) // TEST_F(MemTableTest, PutUntilFull) { - usize total_key_bytes = 0; - usize total_value_bytes = 0; - usize put_count = 0; + [[maybe_unused]] usize total_key_bytes = 0; + [[maybe_unused]] usize total_value_bytes = 0; + [[maybe_unused]] usize put_count = 0; for (;;) { KeyView key = this->make_random_key(); diff --git a/src/turtle_kv/script/uniform_key_distribution.hpp b/src/turtle_kv/script/uniform_key_distribution.hpp index 67fd211..700f1d4 100644 --- a/src/turtle_kv/script/uniform_key_distribution.hpp +++ b/src/turtle_kv/script/uniform_key_distribution.hpp @@ -37,7 +37,9 @@ inline constexpr std::array kHashSeeds = { class UniformInsertKeyDistribution : public KeyDistribution { public: - explicit UniformInsertKeyDistribution(usize key_size) noexcept : key_buffer_(key_size) + explicit UniformInsertKeyDistribution(usize key_size, usize seed = 0) noexcept + : next_ordinal_{seed} + , key_buffer_(key_size) { } @@ -48,7 +50,7 @@ class UniformInsertKeyDistribution : public KeyDistribution std::pair get_next(KeySet& inserted_keys) override { - return inserted_keys.create_key(this->format_key(this->count_.fetch_add(1))); + return inserted_keys.create_key(this->format_key(this->next_ordinal_.fetch_add(1))); } //+++++++++++-+-+--+----- --- -- - - - - @@ -77,7 +79,7 @@ class UniformInsertKeyDistribution : public KeyDistribution //+++++++++++-+-+--+----- --- -- - - - - - std::atomic count_{0}; + std::atomic next_ordinal_{0}; SmallVec key_buffer_; }; diff --git a/src/turtle_kv/tree/in_memory_node.cpp b/src/turtle_kv/tree/in_memory_node.cpp index 536e0c3..a3a712b 100644 --- a/src/turtle_kv/tree/in_memory_node.cpp +++ b/src/turtle_kv/tree/in_memory_node.cpp @@ -15,6 +15,8 @@ #include #include +#include + #include #include @@ -116,8 +118,8 @@ using PackedSegment = PackedUpdateBuffer::Segment; segment.page_id_slot = llfs::PageIdSlot::from_page_id(packed_segment.leaf_page_id.unpack()); segment.active_pivots = packed_segment.active_pivots.unpack(); - BATT_ASSIGN_OK_RESULT(segment.filter, - packed_node.create_piecewise_filter(level_i, segment_i)); + segment.filter = + PiecewiseFilter{packed_node.get_packed_filter(level_i, segment_i)}; segment.check_invariants(__FILE__, __LINE__); } diff --git a/src/turtle_kv/tree/in_memory_node.test.cpp b/src/turtle_kv/tree/in_memory_node.test.cpp index 22b3d56..52a5dd9 100644 --- a/src/turtle_kv/tree/in_memory_node.test.cpp +++ b/src/turtle_kv/tree/in_memory_node.test.cpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #include // #include @@ -12,9 +20,10 @@ #include #include +#include #include -#include +#include #include #include diff --git a/src/turtle_kv/tree/in_memory_node_merged_level.cpp b/src/turtle_kv/tree/in_memory_node_merged_level.cpp index 7a67284..a28be9f 100644 --- a/src/turtle_kv/tree/in_memory_node_merged_level.cpp +++ b/src/turtle_kv/tree/in_memory_node_merged_level.cpp @@ -16,6 +16,8 @@ #include +#include + #include namespace turtle_kv { diff --git a/src/turtle_kv/tree/in_memory_node_segmented_level.cpp b/src/turtle_kv/tree/in_memory_node_segmented_level.cpp index bba1672..19edf8d 100644 --- a/src/turtle_kv/tree/in_memory_node_segmented_level.cpp +++ b/src/turtle_kv/tree/in_memory_node_segmented_level.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp new file mode 100644 index 0000000..f9832cf --- /dev/null +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp @@ -0,0 +1,30 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_BLOCKED_LEAF_PAGE_LOADER_CONCEPT_HPP + +#include + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template +concept BlockedLeafPageLoader = requires(T& loader, llfs::PageId page_id, BlockIndex block_i) { + loader.release_block(page_id, block_i); + { loader.load_block(page_id, block_i) } -> std::convertible_to>; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp new file mode 100644 index 0000000..b442bfa --- /dev/null +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.cpp @@ -0,0 +1,114 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include "blocked_leaf_page_loader.hpp" + +#include "packed_leaf_block.ipp" + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +BlockedLeafPageLoader::BlockedLeafPageLoader(llfs::PageLoader& page_loader, + PageSliceStorage& slice_storage, + llfs::PinPageToJob pin_page_to_job, + usize block_size) noexcept + : page_loader_{page_loader} + , slice_storage_{slice_storage} + , pin_page_to_job_{pin_page_to_job} + , block_size_{block_size} + , page_id_{} +{ +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +StatusOr BlockedLeafPageLoader::set_page( + llfs::PageId page_id) noexcept +{ + this->page_id_ = page_id; + this->leaf_ = nullptr; + this->cache_.clear(); + + PageSliceReader slice_reader{this->page_loader_, + page_id, + llfs::PageSize{BATT_CHECKED_CAST(i32, this->block_size_)}}; + + // Load the first shard to read the fixed header fields. + // + BATT_ASSIGN_OK_RESULT(ConstBuffer header_buffer, + slice_reader.read_slice(Interval{0, this->block_size_}, + this->slice_storage_, + this->pin_page_to_job_, + llfs::LruPriority{kTrieIndexLruPriority})); + + this->leaf_ = &PackedBlockedLeafPage::view_of(header_buffer); + + // Load the full header (block_starting_item + ART) as a contiguous buffer. + // + const usize header_size = this->leaf_->min_header_shard_size(); + + if (header_size > this->block_size_) { + BATT_ASSIGN_OK_RESULT(header_buffer, + slice_reader.read_slice(Interval{0, header_size}, + this->slice_storage_, + this->pin_page_to_job_, + llfs::LruPriority{kTrieIndexLruPriority})); + + this->leaf_ = &PackedBlockedLeafPage::view_of(header_buffer); + } + + this->cache_.assign(kCacheSlots, CacheSlot{0, ConstBuffer{}}); + + return this->leaf_; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +StatusOr BlockedLeafPageLoader::load_block(u32 block_index) noexcept +{ + BATT_CHECK_NE(this->leaf_, nullptr); + BATT_CHECK_LT(block_index, this->leaf_->block_count()); + + const u32 slot = block_index % kCacheSlots; + const u32 tag = block_index / kCacheSlots; + + if (this->cache_[slot].buffer.data() && this->cache_[slot].tag == tag) { + return &PackedLeafBlock::view_of(this->cache_[slot].buffer); + } + + llfs::PageCache& page_cache = *this->page_loader_.page_cache(); + + Optional shard_page_id = + this->leaf_->page_shard_id_for_block(page_cache, block_index, this->page_id_); + + if (!shard_page_id) { + return {batt::StatusCode::kUnavailable}; + } + + const llfs::PinnedPage* existing = this->slice_storage_.find_pinned_page(*shard_page_id); + if (!existing) { + BATT_ASSIGN_OK_RESULT( + llfs::PinnedPage pinned_shard, + this->page_loader_.load_page(*shard_page_id, + llfs::PageLoadOptions{llfs::ShardedPageView::page_layout_id(), + this->pin_page_to_job_, + llfs::OkIfNotFound{false}, + llfs::LruPriority{kLeafLruPriority}})); + + this->slice_storage_.insert_pinned_page(std::move(pinned_shard)); + existing = this->slice_storage_.find_pinned_page(*shard_page_id); + } + + ConstBuffer block_buffer{existing->raw_data(), this->block_size_}; + this->cache_[slot] = CacheSlot{tag, block_buffer}; + + return &PackedLeafBlock::view_of(block_buffer); +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp new file mode 100644 index 0000000..99d066c --- /dev/null +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.hpp @@ -0,0 +1,75 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_BLOCKED_LEAF_PAGE_LOADER_HPP + +#include "packed_blocked_leaf_page.hpp" +#include "packed_leaf_block.hpp" + +#include +#include + +#include +#include +#include +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Loads header and blocks from a PackedBlockedLeafPage via sharded page views. + * + * Bound to one page at a time. Call `set_page` to move to a new page. + */ +class BlockedLeafPageLoader +{ + public: + explicit BlockedLeafPageLoader(llfs::PageLoader& page_loader, + PageSliceStorage& slice_storage, + llfs::PinPageToJob pin_page_to_job, + usize block_size) noexcept; + + /** \brief Loads the header shard for the given page, clears the block cache, and returns + * the validated PackedBlockedLeafPage pointer. + */ + StatusOr set_page(llfs::PageId page_id) noexcept; + + /** \brief Returns the current leaf pointer, or nullptr if no page is set. + */ + const PackedBlockedLeafPage* leaf() const noexcept + { + return this->leaf_; + } + + /** \brief Loads the block at the given index. Returns a cached result on subsequent calls. + */ + StatusOr load_block(u32 block_index) noexcept; + + private: + static constexpr usize kCacheSlots = 256; + + struct CacheSlot { + u32 tag; + ConstBuffer buffer; + }; + + //+++++++++++-+-+--+----- --- -- - - - - + + llfs::PageLoader& page_loader_; + PageSliceStorage& slice_storage_; + llfs::PinPageToJob pin_page_to_job_; + usize block_size_; + + llfs::PageId page_id_; + const PackedBlockedLeafPage* leaf_ = nullptr; + SmallVec cache_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp new file mode 100644 index 0000000..5e61632 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp @@ -0,0 +1,49 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include "packed_blocked_leaf_page.hpp" +// + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept +{ + const usize space_after_header = + leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); + + const usize max_block_count = space_after_header / block_size; + + const usize block_starts_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; + + const usize space_after_block_starts = space_after_header - block_starts_size; + + const usize max_art_size = max_key_size * max_block_count * 2; + + const usize space_after_art = space_after_block_starts - max_art_size; + + BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; + const usize space_for_blocks = space_after_art & ~(block_size - 1); + const usize block_count = space_for_blocks / block_size; + + const usize max_wasted_per_block = max_edit_size - 1; + const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; + + const usize final_estimate = block_count * min_block_capacity; + + BATT_CHECK_GT(leaf_size, final_estimate); + + return final_estimate; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp new file mode 100644 index 0000000..ffa0ec1 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp @@ -0,0 +1,355 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP + +#include "packed_leaf_block.hpp" +#include "packed_leaf_block.iterator.hpp" + +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include + +#include + +namespace turtle_kv { + +BATT_STRONG_TYPEDEF(u32, LeafItemIndex); + +// Forward-declaration. +// +struct PackedBlockedLeafPage; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +struct PackedLeafResult { + PackedBlockedLeafPage* leaf; + usize items_packed; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Packs a blocked leaf page with the passed block size, containing the passed key/value + * pairs, into the passed buffer. Packs as many items as will fit; returns the number packed. + */ +template +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Header for a packed leaf page with blocked structure. + */ +struct PackedBlockedLeafPage // +{ + /** \brief Must be the first 8 bytes of the header. \see PackedBlockedLeagPage::magic + */ + static constexpr u64 kMagic = 0x6456beb7f9558445ull; + + //+++++++++++-+-+--+----- --- -- - - - - + + using BlockIterator = PackedLeafBlock::Iterator; + class ItemIterator; + + using BlockItemsSeq = PackedLeafBlock::BlockItemsSeq; + + struct ItemsSeqFromBlock { + BlockItemsSeq operator()(const PackedLeafBlock& block) const + { + return block.items_seq(); + } + }; + + using BlocksSeq = batt::SubRangeSeq>; + using ItemsSeq = batt::seq::Flatten>; + + struct SlotSliceFromBlock { + PackedKeyValueSlotSlice operator()(const PackedLeafBlock& block) const + { + return {block.items_slice()}; + } + }; + + using SlotSliceSeq = batt::seq::Map; + + template FilterModelT> + class ShardedLiveRanges; + + class HeaderShardView; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static usize packed_edit_size(const EditT& edit) noexcept + { + return PackedLeafBlock::packed_edit_size(edit); + } + + static usize estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept; + + /** \brief Returns the passed buffer's memory region, validated as a PackedBlockedLeafPage and + * cast to `const PackedBlockedLeafPage &`. + */ + static const PackedBlockedLeafPage& view_of(const ConstBuffer& buffer) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u64 magic; // +8 -> 8 + little_u32 total_packed_size; // +4 -> 12 + little_u32 blocks_per_art_key; // +4 -> 16 + little_u32 block_size_bytes; // +4 -> 20 + llfs::PackedPointer block0; // +4 -> 24 + + /** \brief Pointer to array that stores, for each block, the starting item index relative to the + * entire leaf. + */ + llfs::PackedPointer> block_starting_item; // +4 -> 28 + + /** \brief Pointer to packed ART index. + */ + llfs::PackedPointer art_block_index; // +4 -> 32 + + //+++++++++++-+-+--+----- --- -- - - - - + + llfs::PageId page_id() const noexcept + { + return (reinterpret_cast(this) - 1)->page_id.unpack(); + } + + Optional page_shard_id_for_block(llfs::PageCache& page_cache, + usize i, + llfs::PageId leaf_page_id) const noexcept + { + const usize block_begin_offset = this->block_page_offset(i); + const usize block_end_offset = block_begin_offset + this->block_size_bytes; + + return page_cache.page_shard_id_for(leaf_page_id, + Interval{block_begin_offset, block_end_offset}); + } + + Optional page_shard_id_for_block(llfs::PageCache& page_cache, + usize i) const noexcept + { + return this->page_shard_id_for_block(page_cache, i, this->page_id()); + } + + usize min_header_shard_size() const noexcept + { + return this->block_page_offset(0); + } + + //----- --- -- - - - - + + usize block_page_offset(usize i) const noexcept + { + return sizeof(llfs::PackedPageHeader) + offsetof(PackedBlockedLeafPage, block0) + + this->block0.offset + i * this->block_size_bytes; + } + + usize block_count() const noexcept + { + return this->block_starting_item->size() - 1; + } + + BlockIterator blocks_begin() const noexcept + { + return BlockIterator{this->block0.get(), (isize)this->block_size_bytes.value()}; + } + + const PackedLeafBlock& blocks_front() const + { + return *this->blocks_begin(); + } + + BlockIterator blocks_end() const noexcept + { + return this->blocks_begin() + this->block_count(); + } + + const PackedLeafBlock& blocks_back() const + { + return *(this->blocks_begin() + (this->block_count() - 1)); + } + + auto blocks() const noexcept + { + return std::ranges::subrange(this->blocks_begin(), this->blocks_end()); + } + + const PackedLeafBlock& block_at(usize block_i) const noexcept + { + return *(this->blocks_begin() + block_i); + } + + BlocksSeq blocks_seq() const noexcept + { + return batt::as_seq(this->blocks()); + } + + Interval item_index_range_of_block(usize i) const noexcept + { + return Interval{ + (*this->block_starting_item)[i].value(), + (*this->block_starting_item)[i + 1].value(), + }; + } + + /** \brief Returns the index of the block that would contain the given key, if it is present in + * this page. + * + * Always returns a valid block index (i.e., less-than this->block_count()) + */ + usize find_block_index_containing_key(const KeyView& key) const noexcept; + + /** \brief Returns a block iterator to the block that would contain the given key, if it is + * present in this page. + * + * Always returns a valid block iterator. + */ + BlockIterator find_block_containing_key(const KeyView& key) const noexcept; + + //----- --- -- - - - - + + /** \brief Returns the number of key/value pairs in this page. + */ + usize item_count() const noexcept + { + return this->block_starting_item->back(); + } + + /** \brief Returns a sequence of all items in the page, in key order. + */ + ItemsSeq items_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map(ItemsSeqFromBlock{}) | batt::seq::flatten(); + } + + /** \brief Returns an item iterator to the first item in the page. + */ + ItemIterator items_begin() const noexcept; + + /** \brief Returns an item iterator one-past the last item in the page. + */ + ItemIterator items_end() const noexcept; + + /** \brief Returns an item iterator to the i-th item in the page. + */ + ItemIterator item_at(usize i) const noexcept; + + /** \brief Returns an iterator to the given key in this page if found or nullptr if not found. + */ + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; + + /** \brief Returns an iterator to the first item in this page whose key is not less than `key`; + * if all keys in the page are less than `key`, returns `this->items_end()`. + */ + ItemIterator lower_bound(const KeyView& key) const noexcept; + + //----- --- -- - - - - + + KeyView min_key() const noexcept + { + return this->blocks_front().min_key(); + } + + KeyView max_key() const noexcept + { + return this->blocks_back().max_key(); + } + + SlotSliceSeq slot_slice_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map(SlotSliceFromBlock{}); + } + + template FilterModelT> + ShardedLiveRanges sharded_live_ranges( + const BasicPiecewiseFilter& filter, + const Interval& subrange) const noexcept; + + Interval get_block_aligned_index_range_for_key_range( + const Interval& key_range) const noexcept; + + Slice get_slice_within_block( + u32 block_index, + const PackedLeafBlock* block, + const Interval& live_item_range) const noexcept; +}; + +static_assert(sizeof(PackedBlockedLeafPage) == 32); + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief A view of the header prefix of a PackedBlockedLeafPage. + */ +class PackedBlockedLeafPage::HeaderShardView +{ + public: + using Self = HeaderShardView; + + //+++++++++++-+-+--+----- --- -- - - - - + + static Self view_of(const ConstBuffer& buffer) noexcept + { + const PackedBlockedLeafPage& leaf = PackedBlockedLeafPage::view_of(buffer); + BATT_CHECK_GE(buffer.size(), leaf.min_header_shard_size()); + + return Self{leaf, buffer.size()}; + } + + //+++++++++++-+-+--+----- --- -- - - - - + +#if 0 + Seq load_slices(llfs::PageLoader& loader, + Optional first_key, + Optional last_key, + Optional first_index, + Optional last_index, + const PiecewiseFilter& filter); +#endif + + //+++++++++++-+-+--+----- --- -- - - - - + private: + explicit HeaderShardView(const PackedBlockedLeafPage& leaf, usize header_shard_size) noexcept + : leaf_{&leaf} + , header_shard_size_{header_shard_size} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + const PackedBlockedLeafPage* leaf_; + usize header_shard_size_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp new file mode 100644 index 0000000..7327116 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp @@ -0,0 +1,400 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_IPP + +#include "packed_blocked_leaf_page.hpp" +#include "packed_blocked_leaf_page.item_iterator.hpp" + +#include + +#include +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept +{ + //+++++++++++-+-+--+----- --- -- - - - - + // Reserve space for the fixed header. + // + MutableBuffer dst_remaining = dst_buffer; + dst_remaining += sizeof(llfs::PackedPageHeader); + + auto* leaf_header = static_cast(dst_remaining.data()); + dst_remaining += sizeof(PackedBlockedLeafPage); + { + leaf_header->magic = PackedBlockedLeafPage::kMagic; + leaf_header->total_packed_size = 0; + leaf_header->blocks_per_art_key = 1; + leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); + leaf_header->block0.offset = 0; + leaf_header->block_starting_item.offset = 0; + leaf_header->art_block_index.offset = 0; + } + + const usize space_after_header = dst_remaining.size(); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack as many blocks as fit considering only block_starting_item array overhead. + // + SmallVec block_stats; + usize items_packed = 0; + SmallVec cumulative_item_count; + { + auto src_iter = std::begin(src_items); + const auto src_end = std::end(src_items); + + // The block_starting_item array has block_count + 1 total elements. In fixed_array_overhead, + // we account for the size of the packed array header and the size of the sentinel element + // in the array. + // + const usize fixed_array_overhead = sizeof(llfs::PackedArray) + sizeof(little_u32); + + for (;;) { + if (src_iter == src_end) { + break; + } + BATT_CHECK_LT(src_iter, src_end); + + cumulative_item_count.emplace_back(items_packed); + + // n is the number of blocks if we add one more block to our current total. + // + const usize n = block_stats.size() + 1; + + // array_size is the precomputed fixed_array_overhead + the size of n total block entries. + // + const usize array_size = fixed_array_overhead + n * sizeof(little_u32); + + const usize blocks_data_size = n * block_size; + const usize alignment_waste = block_size - 1; + + if (array_size + alignment_waste + blocks_data_size > space_after_header) { + break; + } + + auto& stats = block_stats.emplace_back( + PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), block_size)); + + items_packed += stats.item_count; + src_iter = std::next(src_iter, stats.item_count); + } + } + + cumulative_item_count.emplace_back(items_packed); + + if (block_stats.empty()) { + return {batt::StatusCode::kResourceExhausted}; + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Shrink blocks until the ART fits. + // + // Compute separator keys and build the ART. If it doesn't fit in the space between + // block_starting_item and the block data region, pop the last block and retry. + // When the ART successfully fits, pack both the block_starting_item array and ART before + // breaking out of the retry loop. + // + using artc::packed::PackedARTBuilder; + + const auto items_begin = std::begin(src_items); + const auto key_at = [&items_begin](usize i) { + return get_key(*(items_begin + i)); + }; + + SmallVec art_keys; + for (usize block_i = 1; block_i < block_stats.size(); ++block_i) { + BATT_CHECK_LT(block_i, cumulative_item_count.size()); + const usize item_i = cumulative_item_count[block_i]; + BATT_CHECK_GT(item_i, 0); + BATT_CHECK_LT(item_i, items_packed); + + KeyView k0 = key_at(item_i - 1); + KeyView k1 = key_at(item_i); + KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); + KeyView min_k1{k1.data(), common_prefix.size() + 1}; + + art_keys.emplace_back(min_k1); + } + + for (;;) { + const usize block_count = block_stats.size(); + + // Build the ART to determine its packed size. + // + batt::StableStringStore string_store; + + BATT_ASSIGN_OK_RESULT(auto art_builder, + PackedARTBuilder::from_items(art_keys.begin(), + art_keys.end(), + BATT_OVERLOADS_OF(get_key), + string_store)); + + const usize art_packed_size = art_builder.get_packed_size(); + + // Check whether everything fits. Total size is the size of the block_starting_item + ART size + // + boundary alignment padding + block data size. + // + const usize array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); + const usize total_needed = + array_size + art_packed_size + (block_size - 1) + block_count * block_size; + + // ART doesn't fit, so pop a block and retry the loop. + // + if (total_needed > space_after_header) { + items_packed -= block_stats.back().item_count; + block_stats.pop_back(); + cumulative_item_count.pop_back(); + art_keys.pop_back(); + + if (block_stats.empty()) { + return {batt::StatusCode::kResourceExhausted}; + } + continue; + } + + // Pack `block_starting_item` array. + // + { + const usize block_starting_item_array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); + + auto* block_starting_item = static_cast*>(dst_remaining.data()); + dst_remaining += block_starting_item_array_size; + + block_starting_item->initialize(block_count + 1); + + little_u32* block_start = block_starting_item->data(); + u32 item_i = 0; + for (const PackedLeafBlockStats& stats : block_stats) { + *block_start = item_i; + item_i += stats.item_count; + ++block_start; + } + *block_start = items_packed; + + leaf_header->block_starting_item.reset_unsafe(block_starting_item); + } + + // Pack the ART. + // + { + MutableBuffer art_buffer{dst_remaining.data(), art_builder.get_packed_size()}; + dst_remaining += art_buffer.size(); + + BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); + + leaf_header->art_block_index.reset_unsafe(art_root); + } + + break; + } + + const usize block_count = block_stats.size(); + + //+++++++++++-+-+--+----- --- -- - - - - + // Shift the remaining buffer forward so it aligns with the nearest block boundary. + // + const usize offset_for_block_align = dst_remaining.size() & (block_size - 1); + dst_remaining += offset_for_block_align; + BATT_CHECK_LE(block_size * block_count, dst_remaining.size()); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack the blocks. + // + leaf_header->block0.reset_unsafe(static_cast(dst_remaining.data())); + { + auto src_iter = std::begin(src_items); + const auto src_end = std::end(src_items); + usize block_i = 0; + for (const PackedLeafBlockStats& stats : block_stats) { + BATT_DEBUG_INFO(BATT_INSPECT(block_i) << BATT_INSPECT(stats)); + + BATT_CHECK_NE(src_iter, src_end); + auto src_block_items = std::ranges::subrange(src_iter, std::next(src_iter, stats.item_count)); + + BATT_CHECK_GE(dst_remaining.size(), block_size); + MutableBuffer dst_block_buffer{dst_remaining.data(), block_size}; + + auto block_end_iter = + BATT_OK_RESULT_OR_PANIC(pack_leaf_block(src_block_items, dst_block_buffer, stats)); + + BATT_CHECK_EQ(block_end_iter, std::end(src_block_items)); + + src_iter = block_end_iter; + dst_remaining += block_size; + ++block_i; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Fill in remaining header fields. + // + leaf_header->total_packed_size = BATT_CHECKED_CAST(u32, dst_buffer.size() - dst_remaining.size()); + + return PackedLeafResult{leaf_header, items_packed}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ const PackedBlockedLeafPage& PackedBlockedLeafPage::view_of( + const ConstBuffer& buffer) noexcept +{ + BATT_CHECK_GT(buffer.size(), sizeof(PackedBlockedLeafPage) + sizeof(llfs::PackedPageHeader)); + + auto* packed = static_cast( + advance_pointer(buffer.data(), sizeof(llfs::PackedPageHeader))); + + BATT_CHECK_EQ(packed->magic, PackedBlockedLeafPage::kMagic); + + return *packed; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_begin() const noexcept +{ + auto first_block = this->blocks_begin(); + return ItemIterator{first_block, first_block->items_begin()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_end() const noexcept +{ + auto last_block = this->blocks_end(); + return ItemIterator{last_block, last_block->items_begin()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::item_at(usize i) const noexcept +{ + const llfs::PackedArray& starts = *this->block_starting_item; + + BATT_CHECK_NE(starts.size(), 0); + BATT_CHECK_EQ(starts.front(), 0); + + const auto iter = std::prev(std::upper_bound(starts.begin(), starts.end(), i)); + const isize item_pos_in_block = i - *iter; + const isize block_i = std::distance(starts.begin(), iter); + auto block_iter = this->blocks_begin() + block_i; + + return ItemIterator{block_iter, block_iter->items_begin() + item_pos_in_block}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline usize PackedBlockedLeafPage::find_block_index_containing_key( + const KeyView& key) const noexcept +{ + using artc::packed::find_lower_bound_rank; + using artc::packed::LowerBoundRank; + + LowerBoundRank result = find_lower_bound_rank(this->art_block_index.get(), key); + + const usize part_i = result.exact ? (result.rank + 1) : result.rank; + const usize block_i = part_i * this->blocks_per_art_key; + + BATT_CHECK_LT(block_i, this->block_count()); + + return block_i; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedLeafBlock::Iterator PackedBlockedLeafPage::find_block_containing_key( + const KeyView& key) const noexcept +{ + return this->blocks_begin() + this->find_block_index_containing_key(key); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedBlockedLeafPage::find_key( + const KeyView& key) const noexcept +{ + return this->find_block_containing_key(key)->find_key(key); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::lower_bound( + const KeyView& key) const noexcept +{ + auto block_iter = this->find_block_containing_key(key); + + const PackedKeyValueSlotPtr* p_slot = block_iter->lower_bound(key); + if (p_slot == block_iter->items_end()) { + ++block_iter; + return ItemIterator{block_iter, block_iter->items_begin()}; + } + + return ItemIterator{block_iter, p_slot}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline Interval PackedBlockedLeafPage::get_block_aligned_index_range_for_key_range( + const Interval& key_range) const noexcept +{ + const usize first_block = this->find_block_index_containing_key(key_range.lower_bound); + const usize last_block = this->find_block_index_containing_key(key_range.upper_bound); + + BATT_CHECK_LE(first_block, last_block); + + const LeafItemIndex range_begin{(*this->block_starting_item)[first_block].value()}; + const LeafItemIndex range_end{(*this->block_starting_item)[last_block + 1].value()}; + + return Interval{range_begin, range_end}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline Slice PackedBlockedLeafPage::get_slice_within_block( + u32 block_index, + const PackedLeafBlock* block, + const Interval& live_item_range) const noexcept +{ + BATT_CHECK_LT(block_index, this->block_count()); + + const u32 block_start = (*this->block_starting_item)[block_index].value(); + const usize local_begin = live_item_range.lower_bound - block_start; + const usize local_end = live_item_range.upper_bound - block_start; + + BATT_CHECK_LE(local_end, block->item_count()); + + return as_slice(block->items_begin() + local_begin, block->items_begin() + local_end); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline PackedBlockedLeafPage::ShardedLiveRanges +PackedBlockedLeafPage::sharded_live_ranges(const BasicPiecewiseFilter& filter, + const Interval& subrange) const noexcept +{ + return ShardedLiveRanges{ + this->block_starting_item.get(), + filter.live_subranges_of( + Interval{subrange.lower_bound.value(), subrange.upper_bound.value()}), + subrange, + }; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp new file mode 100644 index 0000000..664adc6 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp @@ -0,0 +1,183 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCK_LEAF_PAGE_ITEM_ITERATOR_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Iterator over the items in a blocked leaf page. + */ +class PackedBlockedLeafPage::ItemIterator + : public boost::iterator_facade< // + PackedBlockedLeafPage::ItemIterator, // <- Derived + const PackedKeyValueSlotPtr, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedKeyValueSlotPtr&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = ItemIterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedKeyValueSlotPtr; + using reference = value_type&; + + ItemIterator() = default; + + explicit ItemIterator(BlockIterator block_iter, const PackedKeyValueSlotPtr* slot) noexcept + : block_iter_{block_iter} + , slot_{slot} + { + } + + reference dereference() const + { + return *this->slot_; + } + + bool equal(const Self& other) const + { + return this->block_iter_ == other.block_iter_ && this->slot_ == other.slot_; + } + + void increment() + { + ++this->slot_; + if (this->slot_ == this->block_iter_->items_end()) { + ++this->block_iter_; + this->slot_ = this->block_iter_->items_begin(); + } + } + + void decrement() + { + if (this->slot_ == this->block_iter_->items_begin()) { + --this->block_iter_; + this->slot_ = std::prev(this->block_iter_->items_end()); + } else { + --this->slot_; + } + } + + void advance(isize delta) + { + if (delta == 0) { + return; + } + + isize pos_in_block = this->get_item_pos_in_block(); + + if (delta > 0) { + // Keep stepping through the page one block at a time until we reduce delta to zero. + // + while (delta != 0) { + // Figure out where the current slot is in the current block. + // + const isize remaining_in_block = this->get_remaining_in_block(pos_in_block); + BATT_CHECK_GT(remaining_in_block, 0); + + // If the remaining delta is inside the block, advance the slot pointer and we are done! + // + if (delta < remaining_in_block) { + this->slot_ += delta; + break; + } + // Else reduce delta by the number of slots after this one in the current block. + // + delta -= remaining_in_block; + + // Advance to the next block, resetting the slot pointer. + // + ++this->block_iter_; + this->slot_ = this->block_iter_->items_begin(); + pos_in_block = 0; + } + + } else { // delta < 0 + + delta = -delta; + while (delta != 0) { + BATT_CHECK_GE(pos_in_block, 0); + + // If the remaining delta is inside the block, update the slot pointer and we are done! + // + if (delta <= pos_in_block) { + this->slot_ -= delta; + break; + } + // Else reduce delta by the number of slots before this one in the current block, plus one + // for the current slot. + // + delta -= (pos_in_block + 1); + + // Move to the last item of the previous block. + // + --this->block_iter_; + this->slot_ = std::prev(this->block_iter_->items_end()); + pos_in_block = this->block_iter_->item_count() - 1; + } + } + } + + isize distance_to(const Self& other) const + { + if (this->block_iter_ == other.block_iter_) { + return std::distance(this->slot_, other.slot_); + } + + // Step forward counting items in each block until we reach the same block. + // + if (this->block_iter_ < other.block_iter_) { + isize delta = this->get_remaining_in_block(); + for (auto iter = std::next(this->block_iter_); iter != other.block_iter_; ++iter) { + delta += iter->item_count(); + } + delta += other.get_item_pos_in_block(); + return delta; + } + // Else step backward. + // + isize delta = this->get_item_pos_in_block(); + for (auto iter = std::prev(this->block_iter_); iter != other.block_iter_; --iter) { + delta += iter->item_count(); + } + delta += other.get_remaining_in_block(); + return -delta; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + isize get_item_pos_in_block() const noexcept + { + return std::distance(this->block_iter_->items_begin(), this->slot_); + } + + isize get_remaining_in_block(isize pos_in_block) const noexcept + { + return this->block_iter_->item_count() - pos_in_block; + } + + isize get_remaining_in_block() const noexcept + { + return this->get_remaining_in_block(this->get_item_pos_in_block()); + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + BlockIterator block_iter_; + const PackedKeyValueSlotPtr* slot_ = nullptr; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp new file mode 100644 index 0000000..9fb628b --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp @@ -0,0 +1,63 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template FilterModelT> +class PackedBlockedLeafPage::ShardedLiveRanges +{ + public: + struct Item { + u32 block_index; + Interval live_item_range; + bool is_first_block; + bool is_last_block; + }; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit ShardedLiveRanges( + const llfs::PackedArray* block_starts, + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges, + const Interval& subrange) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + Optional peek(); + + Optional next(); + + //+++++++++++-+-+--+----- --- -- - - - - + private: + void advance(); + + usize get_block_count() const noexcept; + + Interval get_block_range(usize block_i) const noexcept; + + void clear_current_range(); + + //+++++++++++-+-+--+----- --- -- - - - - + + const llfs::PackedArray* block_starts_; + usize block_index_; + BasicPiecewiseFilter::LiveSubranges filter_live_ranges_; + Interval current_range_; + Interval subrange_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp new file mode 100644 index 0000000..56b3236 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp @@ -0,0 +1,175 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_IPP + +#include "packed_blocked_leaf_page.sharded_live_ranges.hpp" + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline /*explicit*/ PackedBlockedLeafPage::ShardedLiveRanges::ShardedLiveRanges( + const llfs::PackedArray* block_starts, + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges, + const Interval& subrange) noexcept + : block_starts_{block_starts} + , block_index_{0} + , filter_live_ranges_{std::move(filter_live_ranges)} + , current_range_{0, 0} + , subrange_{subrange} +{ + this->advance(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline auto PackedBlockedLeafPage::ShardedLiveRanges::peek() -> Optional +{ + if (this->current_range_.empty()) { + return None; + } + return Item{BATT_CHECKED_CAST(u32, this->block_index_), + Interval{LeafItemIndex{this->current_range_.lower_bound}, + LeafItemIndex{this->current_range_.upper_bound}}, + (*this->block_starts_)[this->block_index_].value() == this->subrange_.lower_bound.value(), + (*this->block_starts_)[this->block_index_ + 1].value() == this->subrange_.upper_bound.value()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline auto PackedBlockedLeafPage::ShardedLiveRanges::next() -> Optional +{ + Optional item = this->peek(); + if (item) { + this->advance(); + } + return item; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline void PackedBlockedLeafPage::ShardedLiveRanges::advance() +{ + this->current_range_.lower_bound = this->current_range_.upper_bound; + + Optional> filter_range = this->filter_live_ranges_.peek(); + if (!filter_range) { + return; + } + + // Consume the current range from both current and filter. + // + BATT_CHECK_LE(this->current_range_.upper_bound, filter_range->upper_bound); + filter_range->lower_bound = std::max(filter_range->lower_bound, // + this->current_range_.upper_bound); + + // If the filter range has been consumed, move to the next filter range. + // + if (filter_range->empty()) { + this->filter_live_ranges_.next(); + filter_range = this->filter_live_ranges_.peek(); + + // Once we run out of filter live ranges, we are done. + // + if (!filter_range) { + return; + } + } + + // std::cerr << ".. " << BATT_INSPECT(filter_range) << std::endl; + + const usize block_count = this->get_block_count(); + BATT_CHECK_LT(this->block_index_, block_count); + const usize blocks_remaining = block_count - this->block_index_; + const usize max_probe_steps = BATT_CHECKED_CAST(usize, batt::log2_ceil(blocks_remaining)); + const usize linear_probe_end = this->block_index_ + max_probe_steps; + bool tried_binary_search = false; + + while (this->block_index_ < block_count) { + // Test the intersection of the current block's range with the current filter range; + // if they intersect, then stop here. + // + this->current_range_ = this->get_block_range(this->block_index_) // + .intersection_with(*filter_range); + + if (!this->current_range_.empty()) { + return; + } + + // If we can, continue the linear probe. + // + ++this->block_index_; + if (this->block_index_ <= linear_probe_end) { + continue; + } + + // The binary search fall-back *must* succeed! If we ever find we are about to try it a + // second time, panic. + // + BATT_CHECK(!tried_binary_search); + + // Fall-back to binary search. + // + auto indices = boost::irange(this->block_index_, block_count); + auto iter = + std::lower_bound(indices.begin(), + indices.end(), + *filter_range, + [this](usize i, const Interval& range) { + return Interval::LinearOrder{}(this->get_block_range(i), range); + }); + + // If the first block that might intersect with the filter range is beyond the end of the + // blocks, then we are done. + // + if (iter == indices.end()) { + this->clear_current_range(); + return; + } + + this->block_index_ = *iter; + tried_binary_search = true; + } +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline usize PackedBlockedLeafPage::ShardedLiveRanges::get_block_count() + const noexcept +{ + return this->block_starts_->size() - 1; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline Interval PackedBlockedLeafPage::ShardedLiveRanges::get_block_range( + usize block_i) const noexcept +{ + return Interval{ + (*this->block_starts_)[block_i], + (*this->block_starts_)[block_i + 1], + }; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline void PackedBlockedLeafPage::ShardedLiveRanges::clear_current_range() +{ + this->current_range_.lower_bound = this->current_range_.upper_bound; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp new file mode 100644 index 0000000..a35fd6b --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -0,0 +1,455 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; +using namespace batt::constants; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::Interval; +using turtle_kv::KeyOrder; +using turtle_kv::KeyView; +using turtle_kv::LeafItemIndex; +using turtle_kv::Optional; +using turtle_kv::pack_blocked_leaf_page; +using turtle_kv::PackedBlockedLeafPage; +using turtle_kv::PackedKeyValueSlotPtr; +using turtle_kv::PiecewiseFilter; +using turtle_kv::random_str; +using turtle_kv::ValueView; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class PackedBlockedLeafPageTest : public ::testing::Test +{ + public: + static constexpr usize kLeafPageSize = 1 * kMiB; + static constexpr usize kNumPrefixes = 1000; + static constexpr usize kMinPrefixSize = 0; + static constexpr usize kMaxPrefixSize = 8; + static constexpr usize kMinKeySize = 4; + static constexpr usize kMaxKeySize = 48; + static constexpr usize kMinValueSize = 0; + static constexpr usize kMaxValueSize = 200; + static constexpr usize kBlockSize = 8192; + + //+++++++++++-+-+--+----- --- -- - - - - + + void set_seed(usize seed) + { + this->rng_.seed(seed); + this->edits_.clear(); + this->leaf_storage_.clear(); + } + + void generate_edits(StableStringStore& strings) + { + BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); + + std::geometric_distribution pick_prefix_size{0.5}; + std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + + std::vector prefixes; + { + std::unordered_set used_prefixes; + while (prefixes.size() < kNumPrefixes) { + std::string_view prefix = + random_str(this->rng_, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + + if (used_prefixes.count(prefix)) { + continue; + } + prefixes.push_back(prefix); + } + } + + { + usize max_edit_size = 0; + usize max_key_size = 0; + usize total_edits_size = 0; + + std::unordered_set used_keys; + for (;;) { + std::string_view prefix = prefixes[pick_prefix(this->rng_)]; + + std::string_view key = + random_str(this->rng_, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); + + if (used_keys.count(key)) { + continue; + } + used_keys.insert(key); + + std::string_view value = + random_str(this->rng_, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + EditView edit{key, ValueView::from_str(value)}; + + const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); + + const usize new_max_edit_size = std::max(max_edit_size, edit_size); + const usize new_max_key_size = std::max(max_key_size, key.size()); + + const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, + kBlockSize, + new_max_key_size, + new_max_edit_size); + + if (edit_size + total_edits_size > space_available) { + break; + } + + this->edits_.push_back(edit); + total_edits_size += edit_size; + max_edit_size = new_max_edit_size; + max_key_size = new_max_key_size; + } + } + + std::sort(this->edits_.begin(), this->edits_.end(), KeyOrder{}); + } + + StatusOr pack_leaf() + { + using StorageUnit = std::aligned_storage_t<4096, 4096>; + this->leaf_storage_.resize(kLeafPageSize / sizeof(StorageUnit)); + BATT_CHECK_EQ(sizeof(StorageUnit) * this->leaf_storage_.size(), kLeafPageSize); + + MutableBuffer leaf_buffer{this->leaf_storage_.data(), kLeafPageSize}; + + BATT_ASSIGN_OK_RESULT(auto result, + pack_blocked_leaf_page(kBlockSize, this->edits_, leaf_buffer)); + + if (result.items_packed != this->edits_.size()) { + return {batt::StatusCode::kInternal}; + } + + return &PackedBlockedLeafPage::view_of(leaf_buffer); + } + + MutableBuffer leaf_buffer() const + { + return MutableBuffer{const_cast(static_cast(this->leaf_storage_.data())), + kLeafPageSize}; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + std::default_random_engine rng_; + std::vector edits_; + + private: + using StorageUnit = std::aligned_storage_t<4096, 4096>; + std::vector leaf_storage_; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST_F(PackedBlockedLeafPageTest, Random) +{ + const usize kFirstSeed = 0; + const usize kNumSeeds = 250; + const usize kLastSeed = kFirstSeed + kNumSeeds; + + std::uniform_int_distribution pick_pct{0, 99}; + + for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { + LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); + + StableStringStore strings; + this->set_seed(seed); + this->generate_edits(strings); + + StatusOr status_or_leaf = this->pack_leaf(); + ASSERT_TRUE(status_or_leaf.ok()) << BATT_INSPECT(status_or_leaf.status()); + + const PackedBlockedLeafPage& packed_leaf = **status_or_leaf; + const auto& edits = this->edits_; + + ASSERT_EQ(packed_leaf.min_key(), get_key(edits.front())); + ASSERT_EQ(packed_leaf.max_key(), get_key(edits.back())); + + //+++++++++++-+-+--+----- --- -- - - - - + // Scan over all items in the packed leaf to make sure they are all there. + // + { + PackedBlockedLeafPage::ItemIterator item_iter = packed_leaf.items_begin(); + PackedBlockedLeafPage::ItemIterator items_end = packed_leaf.items_end(); + + std::vector> past_items; + + auto packed_items = packed_leaf.items_seq(); + using Item = decltype(*packed_items.peek()); + Optional prev_key; + Optional prev_item_iter; + + isize position = 0; + + for (const EditView& edit : edits) { + Optional next_packed = packed_items.next(); + + if (prev_key) { + ASSERT_GT(get_key(edit), *prev_key); + } + prev_key = get_key(edit); + + ASSERT_TRUE(next_packed.has_value()); + ASSERT_EQ(get_key(*next_packed), get_key(edit)); + ASSERT_EQ(get_value(*next_packed), get_value(edit)); + + // Test PackedBlockedLeafPage::find_key. + // + const PackedKeyValueSlotPtr* found = packed_leaf.find_key(get_key(edit)); + + ASSERT_NE(found, nullptr); + ASSERT_EQ(found, std::addressof(*item_iter)); + ASSERT_EQ(get_key(*found), get_key(edit)); + ASSERT_EQ(get_value(*found), get_value(edit)) + << BATT_INSPECT_STR(get_key(*found)) << BATT_INSPECT(edit); + + // Test PackedBlockedLeafPage::lower_bound. + // + { + PackedBlockedLeafPage::ItemIterator lb_iter = packed_leaf.lower_bound(get_key(edit)); + + ASSERT_NE(lb_iter, items_end); + ASSERT_EQ(get_key(*lb_iter), get_key(edit)); + ASSERT_EQ(get_value(*lb_iter), get_value(edit)); + } + + ASSERT_EQ(packed_leaf.item_at(position), item_iter); + + ASSERT_NE(item_iter, items_end); + ASSERT_LT(item_iter, items_end); + ASSERT_EQ(std::distance(packed_leaf.items_begin(), item_iter), position); + + if (pick_pct(this->rng_) < 1) { + past_items.push_back(std::make_pair(item_iter, position)); + } + + for (const auto& [past_iter, past_position] : past_items) { + ASSERT_EQ(std::distance(past_iter, item_iter), position - past_position); + ASSERT_EQ(std::distance(item_iter, past_iter), past_position - position); + ASSERT_EQ(past_iter + (position - past_position), item_iter); + ASSERT_EQ(item_iter - (position - past_position), past_iter); + ASSERT_LE(past_iter, item_iter); + ASSERT_GE(item_iter, past_iter) << BATT_INSPECT(position) << BATT_INSPECT(past_position); + } + + if (prev_item_iter) { + ASSERT_EQ(std::next(*prev_item_iter), item_iter); + ASSERT_EQ(*prev_item_iter, std::prev(item_iter)); + } + + prev_item_iter = item_iter; + ++item_iter; + ++position; + } + ASSERT_FALSE(packed_items.peek().has_value()); + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Test ShardedLiveRanges. + // + { + for (usize j = 0; j < 1000; ++j) { + // Drop up to 64 sub-ranges of the leaf. + // + for (usize drop_count = 0; drop_count < 64; ++drop_count) { + PiecewiseFilter leaf_filter; + + std::vector> dropped_ranges; + u32 items_dropped = 0; + const u32 item_count = packed_leaf.item_count(); + + std::tie(items_dropped, dropped_ranges) = + turtle_kv::testing::drop_n_disjoint_intervals_from(&leaf_filter, + drop_count, + Interval{0, item_count}, + this->rng_); + + // Verify the number of expected live items. + // + const u32 expected_live_count = item_count - items_dropped; + + if (drop_count > 1) { + ASSERT_GT(expected_live_count, 0) + << BATT_INSPECT_RANGE(dropped_ranges) << BATT_INSPECT(drop_count) + << BATT_INSPECT(item_count); + } + + u32 actual_live_count = 0; + u32 next_possible_live = 0; + u32 next_possible_block = 0; + + packed_leaf.sharded_live_ranges(leaf_filter, + Interval{LeafItemIndex{0}, LeafItemIndex{item_count}}) | + batt::seq::for_each([&](const auto& item) { + const auto& block_index = item.block_index; + const auto& live_range = item.live_item_range; + + BATT_CHECK_GE(block_index, next_possible_block); + BATT_CHECK_LT(block_index, packed_leaf.block_count()); + BATT_CHECK_GE(live_range.lower_bound, next_possible_live) + << BATT_INSPECT(j) << BATT_INSPECT(drop_count) << BATT_INSPECT(live_range) + << BATT_INSPECT(item_count); + BATT_CHECK_LT(live_range.lower_bound, live_range.upper_bound); + BATT_CHECK_LE(live_range.upper_bound, item_count); + + const Interval block_range = + packed_leaf.item_index_range_of_block(block_index); + + BATT_CHECK_GE(live_range.lower_bound, block_range.lower_bound); + BATT_CHECK_LE(live_range.upper_bound, block_range.upper_bound); + + next_possible_live = live_range.upper_bound; + next_possible_block = block_index; + + actual_live_count += live_range.size(); + }); + + ASSERT_EQ(actual_live_count, expected_live_count) + << BATT_INSPECT(j) << BATT_INSPECT(drop_count); + } + } + } + } +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST_F(PackedBlockedLeafPageTest, ScanBlockedLeaf) +{ + using turtle_kv::scan_blocked_leaf; + using turtle_kv::testing::InMemoryBlockLoader; + + const usize kFirstSeed = 0; + const usize kNumSeeds = 250; + const usize kLastSeed = kFirstSeed + kNumSeeds; + const usize kTrialsPerSeed = 100; + const usize kMaxDropCount = 500; + + for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { + LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); + + StableStringStore strings; + this->set_seed(seed); + this->generate_edits(strings); + + StatusOr status_or_leaf = this->pack_leaf(); + ASSERT_TRUE(status_or_leaf.ok()) << BATT_INSPECT(status_or_leaf.status()); + + const PackedBlockedLeafPage& packed_leaf = **status_or_leaf; + const auto& edits = this->edits_; + const u32 item_count = packed_leaf.item_count(); + + InMemoryBlockLoader block_loader{&packed_leaf}; + + std::uniform_int_distribution pick_edit{0, edits.size() - 1}; + + for (usize trial = 0; trial < kTrialsPerSeed; ++trial) { + // Pick a random half-open key range [lower_key, upper_key). + // + usize lo_i = pick_edit(this->rng_); + usize hi_i = pick_edit(this->rng_); + if (lo_i > hi_i) { + std::swap(lo_i, hi_i); + } + if (lo_i == hi_i && hi_i + 1 < edits.size()) { + ++hi_i; + } + + KeyView lower_key = get_key(edits[lo_i]); + KeyView upper_key = get_key(edits[hi_i]); + Interval key_range{lower_key, upper_key}; + + // Build a random filter with drops. + // + const usize drop_count = std::uniform_int_distribution{0, kMaxDropCount}(this->rng_); + PiecewiseFilter leaf_filter; + + turtle_kv::testing::drop_n_disjoint_intervals_from(&leaf_filter, + drop_count, + Interval{0, item_count}, + this->rng_); + + // Compute expected results: edits in [lo_i, hi_i) that are live. + // + std::vector> expected; + for (usize i = lo_i; i < hi_i; ++i) { + if (leaf_filter.live_at_index(i)) { + expected.emplace_back(get_key(edits[i]), get_value(edits[i])); + } + } + + // Run scan_blocked_leaf and verify results. + // + usize actual_i = 0; + + auto scan_seq = scan_blocked_leaf(&packed_leaf, &block_loader, leaf_filter, key_range) | + batt::seq::status_ok(); + + for (;;) { + auto slice = scan_seq.next(); + if (!slice) { + break; + } + for (const auto& slot : *slice) { + ASSERT_LT(actual_i, expected.size()) << BATT_INSPECT(seed) << BATT_INSPECT(trial); + ASSERT_EQ(turtle_kv::get_key(slot), expected[actual_i].first) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(actual_i); + ASSERT_EQ(turtle_kv::get_value(slot), expected[actual_i].second) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(actual_i); + ++actual_i; + } + } + + ASSERT_TRUE(scan_seq.status().ok()) << BATT_INSPECT(scan_seq.status()); + + ASSERT_EQ(actual_i, expected.size()) + << BATT_INSPECT(seed) << BATT_INSPECT(trial) << BATT_INSPECT(drop_count) + << BATT_INSPECT(lower_key) << BATT_INSPECT(upper_key); + } + } +} + +} // namespace diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.hpp new file mode 100644 index 0000000..997f0ef --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.hpp @@ -0,0 +1,170 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_HPP + +#include "packed_leaf_block_stats.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlock { + static constexpr u32 kMagic = 0x7370b49full; + + //+++++++++++-+-+--+----- --- -- - - - - + + class Iterator; + + using BlockItemsSeq = batt::SubRangeSeq>; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u32 magic; // +4 = 4 + little_u16 shared_prefix_size; // +2 = 6 + PackedKeyValueSlotPtr items_[1]; // +2 = 8 + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static usize packed_edit_size(const EditT& edit) noexcept + { + const usize slot_size = packed_key_value_slot_size(edit); + const usize edit_size = slot_size + sizeof(PackedKeyValueSlotPtr); + + return edit_size; + } + + static constexpr usize capacity(usize block_size) noexcept + { + return block_size - std::min(block_size, sizeof(PackedLeafBlock)); + } + + /** \brief Returns the passed buffer's memory region, validated as a PackedLeafBlock and cast to + * `const PackedLeafBlock &`. + */ + static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + usize item_count() const noexcept + { + return this->items_end() - this->items_begin(); + } + + KeyView key_at(usize i) const noexcept + { + return this->items_[i]->key_view(); + } + + ValueView value_at(usize i) const noexcept + { + return this->items_[i]->value_view(&this->items_[i]); + } + + EditView edit_at(usize i) const noexcept + { + auto& packed = *this->items_[i]; + return EditView{packed.key_view(), packed.value_view(&this->items_[i])}; + } + + Optional item_at(usize i) const noexcept + { + return to_item_view(this->edit_at(i)); + } + + const PackedKeyValueSlotPtr& front_item() const noexcept + { + return this->items_[0]; + } + + const PackedKeyValueSlotPtr& back_item() const noexcept + { + return this->items_[this->item_count() - 1]; + } + + BlockItemsSeq items_seq() const noexcept + { + return batt::as_seq(this->items_slice()); + } + + const PackedKeyValueSlotPtr* items_begin() const noexcept + { + return this->items_; + } + + const PackedKeyValueSlotPtr* items_end() const noexcept + { + return ((const PackedKeyValueSlotPtr*)this->items_[0].get()) - 1; + } + + Slice items_slice() const noexcept + { + return as_slice(this->items_begin(), this->items_end()); + } + + Slice items_slice(Optional key_lower_bound, + Optional key_upper_bound) const noexcept; + + KeyView min_key() const noexcept + { + return get_key(this->front_item()); + } + + KeyView max_key() const noexcept + { + return get_key(this->back_item()); + } + + KeyView shared_key_prefix() const noexcept + { + return this->min_key().substr(0, this->shared_prefix_size); + } + + /** \brief Returns an iterator to the given key in this block if found or nullptr if not found. + */ + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; + + /** \brief Returns an iterator to the first item in this block whose key is not less than `key`; + * if all keys in the block are less than `key`, returns `this->items_end()`. + */ + const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept; +}; + +static_assert(sizeof(PackedLeafBlock) == 8); + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ()))>> +StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& stats = None) noexcept; + +} // namespace turtle_kv + +#include "packed_leaf_block.ipp" diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.ipp b/src/turtle_kv/tree/leaf/packed_leaf_block.ipp new file mode 100644 index 0000000..25d842c --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.ipp @@ -0,0 +1,162 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_IPP + +#include "packed_leaf_block.hpp" +#include "packed_leaf_block_stats.ipp" + +#include + +#include +#include + +#include + +#include +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& opt_stats) noexcept +{ + if (dst.size() < sizeof(PackedLeafBlock)) { + return {batt::StatusCode::kResourceExhausted}; + } + + PackedLeafBlockStats stats = opt_stats.or_else([&] { + return PackedLeafBlockStats::from(src, dst.size()); + }); + + if (stats.block_size != dst.size()) { + return {batt::StatusCode::kResourceExhausted}; + } + + PackedLeafBlock* block = static_cast(dst.data()); + { + block->magic = PackedLeafBlock::kMagic; + block->items_[0].offset = BATT_CHECKED_CAST( + u32, + byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes))); + } + + //----- --- -- - - - - + // Pack all slot data. + // + PackedKeyValueSlotPtr* pp_slot = block->items_; + void* p_slot = const_cast(pp_slot->get()); + void* const dst_end = advance_pointer(dst.data(), dst.size()); + + IterT src_iter = std::begin(src); + const IterT src_end = std::next(src_iter, stats.item_count); + for (; src_iter != src_end; ++src_iter) { + const usize slot_size = pack_key_value_slot(*src_iter, p_slot); + p_slot = advance_pointer(p_slot, slot_size); + BATT_CHECK_LE(p_slot, dst_end); + ++pp_slot; + pp_slot->offset = byte_distance(pp_slot, p_slot); + + BATT_CHECK_EQ((void*)pp_slot->get(), (void*)p_slot); + } + + //----- --- -- - - - - + // Set the common prefix. + // + block->shared_prefix_size = + BATT_CHECKED_CAST(u16, + llfs::find_common_prefix(0, block->min_key(), block->max_key()).size()); + + return {src_iter}; +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// struct PackedLeafBlock + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline /*static*/ const PackedLeafBlock& PackedLeafBlock::view_of( + const ConstBuffer& buffer) noexcept +{ + BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); + + const auto* block = static_cast(buffer.data()); + + BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); + + return *block; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedLeafBlock::find_key(const KeyView& key) const noexcept +{ + const auto convert_result = [](auto&& first_last_pair) -> const PackedKeyValueSlotPtr* { + if (first_last_pair.first == first_last_pair.second) { + return nullptr; + } + return first_last_pair.first; + }; + + if (this->shared_prefix_size > 0) { + const usize prefix_size = this->shared_prefix_size; + if (key.size() < prefix_size) { + return nullptr; + } + auto order = batt::compare(key.substr(0, prefix_size), this->shared_key_prefix()); + if (order != batt::Order::Equal) { + return nullptr; + } + + return convert_result( + std::equal_range(this->items_begin(), this->items_end(), key, KeySuffixOrder{prefix_size})); + } + + return convert_result(std::equal_range(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == + batt::Order::Less; + })); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedLeafBlock::lower_bound(const KeyView& key) const noexcept +{ + return std::lower_bound(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline Slice PackedLeafBlock::items_slice( + Optional key_lower_bound, + Optional key_upper_bound) const noexcept +{ + auto [first, last] = std::equal_range(this->items_begin(), + this->items_end(), + Interval{key_lower_bound.or_else(global_min_key), + key_upper_bound.or_else(global_max_key)}, + ExtendedKeyRangeOrder{}); + return as_slice(first, last); +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp new file mode 100644 index 0000000..ade706d --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp @@ -0,0 +1,104 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_ITERATOR_HPP + +#include "packed_leaf_block.hpp" + +#include +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class PackedLeafBlock::Iterator + : public boost::iterator_facade< // + PackedLeafBlock::Iterator, // <- Derived + const PackedLeafBlock, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedLeafBlock&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = Iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedLeafBlock; + using reference = const PackedLeafBlock&; + + //+++++++++++-+-+--+----- --- -- - - - -- + + Iterator() = default; + + explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept + : block_{block} + , block_size_{block_size} + { + } + + //+++++++++++-+-+--+----- --- -- - - - -- + + reference dereference() const + { + return *this->block_; + } + + bool equal(const Self& other) const + { + return this->block_ == other.block_ && this->block_size_ == other.block_size_; + } + + void increment() + { + this->advance(1); + } + + void decrement() + { + this->advance(-1); + } + + void advance(isize delta) + { + this->block_ = static_cast( + advance_pointer(this->block_, delta * this->block_size_)); + } + + isize distance_to(const Self& other) const + { + return (byte_distance(this->block_, other.block_)) / this->block_size_; + } + + //+++++++++++-+-+--+----- --- -- - - - -- + + const PackedLeafBlock* block() const noexcept + { + return this->block_; + } + + usize block_size() const noexcept + { + return static_cast(this->block_size_); + } + + isize block_isize() const noexcept + { + return this->block_size_; + } + + //+++++++++++-+-+--+----- --- -- - - - -- + private: + const PackedLeafBlock* block_ = nullptr; + isize block_size_ = 0; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp b/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp new file mode 100644 index 0000000..f19fefc --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp @@ -0,0 +1,173 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include + +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::KeyOrder; +using turtle_kv::KeyView; +using turtle_kv::random_str; +using turtle_kv::ValueView; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(TreePackedLeafBlockTest, Random) +{ + const usize kNumSeeds = 1000; + const usize kNumNotFoundQueries = 100; + const usize kNumLowerBoundQueries = 500; + const usize kMinPrefixSize = 0; + const usize kMaxPrefixSize = 8; + const usize kMinKeySize = 4; + const usize kMaxKeySize = 48; + const usize kMinValueSize = 0; + const usize kMaxValueSize = 200; + const usize kBlockSize = 8192; + + std::geometric_distribution pick_prefix_size{0.5}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + + for (usize seed = 0; seed < kNumSeeds; ++seed) { + std::default_random_engine rng{seed}; + + StableStringStore strings; + + std::string_view prefix = + random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + + std::unordered_set used_keys; + std::vector src_edits; + + // Generate enough random edits to fill a block. + // + usize src_size = 0; + while (src_size < kBlockSize) { + std::string_view key = + random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); + if (used_keys.count(key)) { + continue; + } + used_keys.insert(key); + + std::string_view value = + random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + src_edits.push_back(EditView{key, ValueView::from_str(value)}); + src_size += key.size() + value.size(); + } + std::sort(src_edits.begin(), src_edits.end(), KeyOrder{}); + + // Pack a block. + // + std::array block_buffer; + block_buffer.fill('!'); + auto dst_buffer = MutableBuffer{block_buffer.data(), kBlockSize}; + + StatusOr::const_iterator> consumed_src_end = + turtle_kv::pack_leaf_block(src_edits, dst_buffer); + + ASSERT_TRUE(consumed_src_end.ok()); + + for (usize i = kBlockSize; i < kBlockSize * 2; ++i) { + ASSERT_EQ(block_buffer[i], '!') << BATT_INSPECT(i); + } + + const usize packed_count = *consumed_src_end - src_edits.begin(); + const auto& packed_block = turtle_kv::PackedLeafBlock::view_of(dst_buffer); + + ASSERT_EQ(packed_block.shared_prefix_size.value(), prefix.size()); + + usize found_count = 0; + for (const EditView& src_edit : src_edits) { + auto* found_ptr = packed_block.find_key(get_key(src_edit)); + if (found_count < packed_count) { + ASSERT_NE(found_ptr, nullptr) + << BATT_INSPECT(src_edit) << BATT_INSPECT(found_count) << BATT_INSPECT(packed_count); + ++found_count; + + ASSERT_EQ(get_key(*found_ptr), get_key(src_edit)); + ASSERT_EQ(get_value(*found_ptr), get_value(src_edit)); + } else { + ASSERT_EQ(found_ptr, nullptr); + } + } + + // Run empty queries. + // + for (usize i = 0; i < kNumNotFoundQueries; ++i) { + std::string_view key; + for (;;) { + key = random_str(rng, + pick_key_size, + kMinKeySize + prefix.size(), + kMaxKeySize + prefix.size(), + strings); + if (!used_keys.count(key)) { + break; + } + } + + ASSERT_EQ(packed_block.find_key(key), nullptr); + } + + // Run lower bound queries. + // + for (usize i = 0; i < kNumLowerBoundQueries; ++i) { + std::string_view key = (i % 2) ? random_str(rng, + pick_key_size, + kMinKeySize + prefix.size(), + kMaxKeySize + prefix.size(), + strings) + : random_str(rng, // + pick_key_size, + kMinKeySize, + kMaxKeySize, + strings, + prefix); + + const auto expected_iter = + std::lower_bound(src_edits.begin(), src_edits.end(), key, KeyOrder{}); + + const usize expected_i = std::distance(src_edits.begin(), expected_iter); + + const auto actual_iter = packed_block.lower_bound(key); + + const usize actual_i = std::distance(packed_block.items_begin(), actual_iter); + + if (expected_i >= packed_count) { + ASSERT_EQ(actual_i, packed_count); + } else { + ASSERT_EQ(actual_i, expected_i); + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp new file mode 100644 index 0000000..2d8360f --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp @@ -0,0 +1,38 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_HPP + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlockStats { + usize block_size; + usize item_count; + usize item_slot_bytes; + usize item_ptr_bytes; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; +}; + +BATT_OBJECT_PRINT_IMPL((inline), + PackedLeafBlockStats, + (block_size, item_count, item_slot_bytes, item_ptr_bytes)) + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp new file mode 100644 index 0000000..a9b9bf7 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp @@ -0,0 +1,56 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_IPP + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, + usize dst_size) noexcept +{ + PackedLeafBlockStats stats{ + .block_size = 0, + .item_count = 0, + .item_slot_bytes = 0, + .item_ptr_bytes = 0, + }; + + if (dst_size < sizeof(PackedLeafBlock)) { + return stats; + } + stats.block_size = dst_size; + usize offset = 0; + dst_size -= sizeof(PackedLeafBlock); + offset += sizeof(PackedLeafBlock); + + for (const auto& src_item : src) { + const usize slot_size = packed_key_value_slot_size(src_item); + const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); + if (dst_size < total_item_size) { + break; + } + stats.item_count += 1; + stats.item_slot_bytes += slot_size; + stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); + dst_size -= total_item_size; + offset += total_item_size; + + if constexpr (false) { + LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) + << BATT_INSPECT(stats.item_count); + } + } + + return stats; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp new file mode 100644 index 0000000..0f0cf00 --- /dev/null +++ b/src/turtle_kv/tree/leaf/scan_blocked_leaf.hpp @@ -0,0 +1,80 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_SCAN_BLOCKED_LEAF_HPP + +#include "packed_blocked_leaf_page.hpp" +#include "packed_blocked_leaf_page.sharded_live_ranges.hpp" +#include "packed_blocked_leaf_page.sharded_live_ranges.ipp" + +#include +#include +#include +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT, typename BlockLoaderT> +auto scan_blocked_leaf(const PackedBlockedLeafPage* packed_leaf, + BlockLoaderT* block_loader, + const BasicPiecewiseFilter& filter, + const Interval& key_range) noexcept +{ + BATT_CHECK_NOT_NULLPTR(packed_leaf); + + const Interval aligned_index_range = + packed_leaf->get_block_aligned_index_range_for_key_range(key_range); + + return packed_leaf->sharded_live_ranges(filter, aligned_index_range) // + | + batt::seq::filter_map( + [packed_leaf, block_loader, key_range]( + const typename PackedBlockedLeafPage::ShardedLiveRanges::Item& item) + -> Optional>> { + if (item.live_item_range.empty()) { + return None; + } + + StatusOr block = block_loader->load_block(item.block_index); + if (!block.ok()) { + return block.status(); + } + + Slice slice = + packed_leaf->get_slice_within_block(item.block_index, + *block, + item.live_item_range); + + if (item.is_first_block || item.is_last_block) { + const auto* begin = slice.begin(); + const auto* end = slice.end(); + + if (item.is_first_block) { + begin = std::lower_bound(begin, end, key_range.lower_bound, llfs::KeyOrder{}); + } + if (item.is_last_block) { + end = std::lower_bound(begin, end, key_range.upper_bound, llfs::KeyOrder{}); + } + + if (begin == end) { + return None; + } + slice = as_slice(begin, end); + } + + return slice; + }) // + ; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_node_page.cpp b/src/turtle_kv/tree/packed_node_page.cpp index bff419e..c469d6f 100644 --- a/src/turtle_kv/tree/packed_node_page.cpp +++ b/src/turtle_kv/tree/packed_node_page.cpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #include // @@ -8,6 +16,7 @@ #include #include +#include #include @@ -213,9 +222,7 @@ StatusOr PackedNodePage::find_key(KeyQuery& query) const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -PackedNodePage::UpdateBuffer::SegmentFilterData PackedNodePage::get_segment_filter_values( - usize level_i, - usize segment_i) const +PackedPiecewiseFilter PackedNodePage::get_packed_filter(usize level_i, usize segment_i) const { const usize i = [&]() -> usize { if (this->is_size_tiered()) { @@ -247,52 +254,9 @@ PackedNodePage::UpdateBuffer::SegmentFilterData PackedNodePage::get_segment_filt bool start_live = (segment.filter_start.value() & PackedNodePage::kSegmentStartsLive) != 0; - return PackedNodePage::UpdateBuffer::SegmentFilterData{ + return PackedPiecewiseFilter{PackedPiecewiseFilterStorage{ as_const_slice(packed_filters.data() + filter_start_i, packed_filters.data() + filter_end_i), - start_live}; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -StatusOr> PackedNodePage::create_piecewise_filter(usize level_i, - usize segment_i) const -{ - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - this->get_segment_filter_values(level_i, segment_i); - - SmallVec, 64> live_ranges; - u32 i = 0; - - // If the first item at index 0 is live, add the corresponding interval first since the - // serialized version of the filter doesn't store index 0. - // - if (filter_data.start_is_live) { - if (filter_data.values.empty()) { - // Entire segment is live. - // - live_ranges.emplace_back( - Interval{PiecewiseFilter::kMinLowerBound, PiecewiseFilter::kMaxUpperBound}); - } else { - live_ranges.emplace_back( - Interval{PiecewiseFilter::kMinLowerBound, filter_data.values[i].value()}); - i++; - } - } - - for (; i + 1 < filter_data.values.size(); i += 2) { - live_ranges.emplace_back( - Interval{filter_data.values[i].value(), filter_data.values[i + 1].value()}); - } - - // If there's one unpaired value left, it's a lower_bound whose upper_bound (kMaxUpperBound) - // was omitted. - // - if (i < filter_data.values.size()) { - live_ranges.emplace_back( - Interval{filter_data.values[i].value(), PiecewiseFilter::kMaxUpperBound}); - } - - return PiecewiseFilter::from_live(as_slice(live_ranges)); + start_live}}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -330,7 +294,11 @@ StatusOr PackedNodePage::UpdateBuffer::Segment::load_leaf_page bool PackedNodePage::UpdateBuffer::Segment::is_index_filtered(const SegmentedLevel& level, u32 index) const { - return !(this->live_lower_bound(level, index) == index); + const usize segment_i = std::distance(level.segments_slice.begin(), this); + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return !filter.live_at_index(index); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -339,34 +307,10 @@ u32 PackedNodePage::UpdateBuffer::Segment::live_lower_bound(const SegmentedLevel u32 item_i) const { const usize segment_i = std::distance(level.segments_slice.begin(), this); - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - level.packed_node_->get_segment_filter_values(level.level_i_, segment_i); - - const Slice filter_values = filter_data.values; - - if (filter_data.values.empty()) { - BATT_CHECK(filter_data.start_is_live); - return item_i; - } - - auto iter = std::upper_bound(filter_values.begin(), filter_values.end(), item_i); - - usize previous_cut_points = std::distance(filter_data.values.begin(), iter); - - bool is_live = (previous_cut_points % 2 == 0) == filter_data.start_is_live; - - // If we're already in an unfiltered region, just return the index. Otherwise, our upper bound - // is the next unfiltered index. - // - if (is_live) { - return item_i; - } - - if (iter != filter_values.end()) { - return iter->value(); - } - - return PiecewiseFilter::kMaxUpperBound; + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return filter.live_lower_bound(item_i); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -375,48 +319,11 @@ Interval PackedNodePage::UpdateBuffer::Segment::get_live_item_range( const SegmentedLevel& level, Interval i) const { - u32 start_i = i.lower_bound; - u32 end_i = i.upper_bound; - - BATT_CHECK_LT(start_i, end_i); - const usize segment_i = std::distance(level.segments_slice.begin(), this); - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - level.packed_node_->get_segment_filter_values(level.level_i_, segment_i); - - const Slice filter_values = filter_data.values; - - if (filter_data.values.empty()) { - BATT_CHECK(filter_data.start_is_live); - return i; - } - - auto iter = std::upper_bound(filter_values.begin(), filter_values.end(), start_i); - - usize previous_cut_points = std::distance(filter_data.values.begin(), iter); - - bool is_live = (previous_cut_points % 2 == 0) == filter_data.start_is_live; - - if (!is_live) { - if (iter == filter_values.end()) { - return Interval{end_i, end_i}; - } - - start_i = iter->value(); - if (start_i >= end_i) { - return Interval{end_i, end_i}; - } - - ++iter; - } - - if (iter != filter_values.end()) { - end_i = std::min(end_i, iter->value()); - } - - BATT_CHECK_LT(start_i, end_i) << BATT_INSPECT(i); - - return Interval{start_i, end_i}; + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return filter.find_live_range(i); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // @@ -496,4 +403,4 @@ std::function PackedNodePage::dump() const }; } -} // namespace turtle_kv \ No newline at end of file +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_node_page.hpp b/src/turtle_kv/tree/packed_node_page.hpp index 1e7cc66..5f4e92e 100644 --- a/src/turtle_kv/tree/packed_node_page.hpp +++ b/src/turtle_kv/tree/packed_node_page.hpp @@ -121,11 +121,6 @@ struct PackedNodePage { struct UpdateBuffer { struct SegmentedLevel; - struct SegmentFilterData { - Slice values; - bool start_is_live; - }; - struct Segment { llfs::PackedPageId leaf_page_id; // +8 -> 8 PackedActivePivotsSet64 active_pivots; // +8 -> 16 @@ -387,9 +382,7 @@ struct PackedNodePage { StatusOr find_key_in_level(usize level_i, KeyQuery& query, i32 key_pivot_i) const; - UpdateBuffer::SegmentFilterData get_segment_filter_values(usize level_i, usize segment_i) const; - - StatusOr> create_piecewise_filter(usize level_i, usize segment_i) const; + PackedPiecewiseFilter get_packed_filter(usize level_i, usize segment_i) const; //----- --- -- - - - - diff --git a/src/turtle_kv/tree/random_str.hpp b/src/turtle_kv/tree/random_str.hpp new file mode 100644 index 0000000..5145d56 --- /dev/null +++ b/src/turtle_kv/tree/random_str.hpp @@ -0,0 +1,48 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include + +#include + +#include +#include +#include +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +std::string_view random_str(std::default_random_engine& rng, + SizeDistribution&& pick_size, + usize min_size, + usize max_size, + batt::StableStringStore& strings, + std::string_view prefix = "") noexcept +{ + std::uniform_int_distribution pick_char{'a', 'z'}; + + const usize n = min_size + std::min(pick_size(rng), max_size - min_size); + batt::MutableBuffer buf = strings.allocate(prefix.size() + n); + char* chars = static_cast(buf.data()); + + if (!prefix.empty()) { + std::memcpy(chars, prefix.data(), prefix.size()); + chars += prefix.size(); + } + + for (usize i = 0; i < n; ++i, ++chars) { + *chars = pick_char(rng); + } + + return std::string_view{static_cast(buf.data()), buf.size()}; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/testing/fake_segment.hpp b/src/turtle_kv/tree/testing/fake_segment.hpp index 8297a28..860f722 100644 --- a/src/turtle_kv/tree/testing/fake_segment.hpp +++ b/src/turtle_kv/tree/testing/fake_segment.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -102,11 +103,11 @@ struct FakeSegment { { const bool inactive = this->active_pivots_.is_empty(); if (inactive) { - Slice> live_ranges = this->filter_.live(); - BATT_CHECK_EQ(live_ranges.size(), 1) << BATT_INSPECT(live_ranges); - BATT_CHECK_EQ(live_ranges[0].upper_bound, PiecewiseFilter::kMaxUpperBound) - << BATT_INSPECT(live_ranges); - } + Slice> live_ranges = this->filter_.live(); + BATT_CHECK_EQ(live_ranges.size(), 1) << BATT_INSPECT(live_ranges); + BATT_CHECK_EQ(live_ranges[0].upper_bound, PiecewiseFilter::kMaxUpperBound) + << BATT_INSPECT(live_ranges); + } return inactive; } diff --git a/src/turtle_kv/tree/testing/in_memory_block_loader.hpp b/src/turtle_kv/tree/testing/in_memory_block_loader.hpp new file mode 100644 index 0000000..9450053 --- /dev/null +++ b/src/turtle_kv/tree/testing/in_memory_block_loader.hpp @@ -0,0 +1,39 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once + +#include +#include + +#include +#include + +namespace turtle_kv { +namespace testing { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class InMemoryBlockLoader +{ + public: + explicit InMemoryBlockLoader(const PackedBlockedLeafPage* leaf) noexcept : leaf_{leaf} + { + } + + StatusOr load_block(u32 block_index) noexcept + { + return &*(this->leaf_->blocks_begin() + block_index); + } + + private: + const PackedBlockedLeafPage* leaf_; +}; + +} // namespace testing +} // namespace turtle_kv diff --git a/src/turtle_kv/util/art.hpp b/src/turtle_kv/util/art.hpp deleted file mode 100644 index b9903cd..0000000 --- a/src/turtle_kv/util/art.hpp +++ /dev/null @@ -1,1742 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include // SSE2 -#include // MMX -#include // SSE3 - -#ifdef __AVX512F__ -#include // AVX512 (AVX, AVX2, FMA) -#endif - -namespace turtle_kv { - -/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns one of: {4, - * 5, 6, 7}. - */ -inline usize index_of(u8 key_byte, const std::array& keys) -{ - __m64 pattern = _mm_set1_pi8((char)key_byte); - u64 extended = *((const u32*)keys.data()); - __m64 values = _mm_cvtsi64_m64(extended); - __m64 result = _m_pcmpeqb(pattern, values); - - return ((__builtin_ffsll((i64)result) - 1) >> 3) & 7; -} - -/** \brief Returns the index of `key_byte` in the array `keys`, if present; else returns 31. - */ -inline usize index_of(u8 key_byte, const std::array& keys) -{ - __m128i pattern = _mm_set1_epi8((char)key_byte); - __m128i values = _mm_lddqu_si128((const __m128i*)keys.data()); - -#ifndef __AVX512F__ - int result = _mm_movemask_epi8(_mm_cmpeq_epi8(pattern, values)); -#else - __mmask16 result = _mm_cmpeq_epi8_mask(pattern, values); -#endif - - return (__builtin_ffs(result) - 1) & 31; -} - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -class ARTBase -{ - public: - static constexpr usize kMaxKeyLen = 64; - - /** \brief Tag type indicating that a new object should not be initialized by the ctor. - */ - struct NoInit { - }; - - struct Metrics { - CountMetric construct_count; - CountMetric destruct_count; - FastCountMetric insert_count; - FastCountMetric byte_alloc_count; - FastCountMetric byte_free_count; - - /** \brief Resets all metrics to initial values. - */ - void reset() - { - this->construct_count.reset(); - this->destruct_count.reset(); - this->insert_count.reset(); - this->byte_alloc_count.reset(); - this->byte_free_count.reset(); - } - - //----- --- -- - - - - - - double bytes_per_instance() const - { - return (double)this->byte_alloc_count.get() / (double)this->construct_count.get(); - } - - double average_item_count() const - { - return (double)this->insert_count.get() / (double)this->construct_count.get(); - } - - double bytes_per_insert() const - { - return (double)this->byte_alloc_count.get() / (double)this->insert_count.get(); - } - - /** \brief Returns an estimate of the number of active instances (ART objects). - */ - u64 instance_count() const - { - // Must be in this order! - // - const u64 observed_destruct_count = this->destruct_count.get(); - const u64 observed_construct_count = this->construct_count.get(); - - return observed_construct_count - observed_destruct_count; - } - - /** \brief Returns an estimate of the current number of bytes in use. - */ - u64 bytes_in_use() const - { - // Must be in this order! - // - const u64 observed_free_count = this->byte_free_count.get(); - const u64 observed_alloc_count = this->byte_alloc_count.get(); - - return observed_alloc_count - observed_free_count; - } - }; - - static Metrics& default_metrics() - { - static Metrics m_; - return m_; - } - - enum struct Synchronized { - kFalse = 0, - kTrue = 1, - kDynamic = 2, - }; - - struct Node4; - struct Node16; - struct Node48; - struct Node256; - struct LeafNode; - - enum struct NodeType : u8 { - kLeafNode = 0, - kNode4 = 1, - kNode16 = 2, - kNode48 = 3, - kNode256 = 4, - kNodeBase = 5, - }; - - //----- --- -- - - - - - - static constexpr usize sizeof_value(batt::StaticType) - { - return 0; - } - - template - static constexpr usize sizeof_value(batt::StaticType) - { - return sizeof(ValueT); - } - - //----- --- -- - - - - - - template - static void* uninitialized_value(NodeT* node) - { - return node + 1; - } - - template - static ValueT* mutable_value(NodeT* node, batt::StaticType /**/ = {}) - { - return reinterpret_cast(node + 1); - } - - template - static const ValueT* const_value(const NodeT* node, batt::StaticType /**/ = {}) - { - return reinterpret_cast(node + 1); - } - - //----- --- -- - - - - - - template - static void* construct_value_copy_node(FromNodeT*, ToNodeT* to_node, batt::StaticType) - { - to_node->set_terminal(); - return nullptr; - } - - template - static void* construct_value_copy_addr(FromNodeT*, void*, batt::StaticType) - { - return nullptr; - } - - //----- --- -- - - - - - - template - static ValueT* construct_value_copy_node(FromNodeT* from_node, - ToNodeT* to_node, - batt::StaticType type_of_value) - { - to_node->set_terminal(); - return ARTBase::construct_value_copy_addr(from_node, - ARTBase::uninitialized_value(to_node), - type_of_value); - } - - template - static ValueT* construct_value_copy_addr(FromNodeT* from_node, - void* to_address, - batt::StaticType type_of_value) - { - return new (to_address) ValueT{*ARTBase::const_value(from_node, type_of_value)}; - } - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // Node class hierarchy: - // - // ┌────────┐ - // │NodeBase│ - // └────────┘ - // △ - // ┌───────────────┤ - // │ │ - // ┌───────────────┐ │ - // │GrowableNode│ │ - // └───────────────┘ │ - // △ │ - // ┌─────────┴────────────┐ └────────┐ - // │ │ │ - // ┌──────────────────────┐┌────────────────────┐ │ - // │IndirectIndexedNode││DirectIndexedNode│ │ - // └──────────────────────┘└────────────────────┘ │ - // △ △ │ - // ┌─────┴───────┐ │ │ - // │ │ │ │ - // ┌─────┐ ┌──────┐ ┌──────┐ ┌───────┐ - // │Node4│ │Node16│ │Node48│ │Node256│ - // └─────┘ └──────┘ └──────┘ └───────┘ - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // Node memory layout: - // - // ┌────────────┬──────────┬──────────────┬───────────────┐ - // │ key prefix │ NodeBase │ (impl) ... │ ValueT │ - // └────────────┴──────────┴──────────────┴───────────────┘ - // ◀──────────▶ ◀───────────────────────▶ ◀─────────────▶ - // variable sizeof(NodeT) sizeof(ValueT) - // length - // - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - - struct NodeBase { - using Self = NodeBase; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr u8 kFlagTerminal = 0x80; - static constexpr u8 kFlagObsolete = 0x40; - - //+++++++++++-+-+--+----- --- -- - - - - - - const NodeType node_type; - - u8 flags_; - u8 prefix_len_; - u8 branch_count_; - SeqMutex mutex_; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit NodeBase(NodeType t) noexcept - : node_type{t} - , flags_{0} - , prefix_len_{0} - , branch_count_{0} - { - } - - explicit NodeBase(NodeType t, ARTBase::NoInit) noexcept : node_type{t} - { - } - - NodeBase(const NodeBase&) = delete; - NodeBase& operator=(const NodeBase&) = delete; - - template - void visit(CaseFns&&... case_fns); - - bool is_terminal() const - { - return (this->flags_ & kFlagTerminal) != 0; - } - - void set_terminal() - { - this->flags_ |= kFlagTerminal; - } - - bool is_obsolete() const - { - return (this->flags_ & kFlagObsolete) != 0; - } - - void set_obsolete() - { - this->flags_ |= kFlagObsolete; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->flags_ = that.flags_; - this->branch_count_ = that.branch_count_; - this->set_prefix(that.prefix() + prefix_offset, that.prefix_len_ - prefix_offset); - } - - const char* prefix() const - { - return (const char*)((((std::uintptr_t)this) - this->prefix_len_) & ~std::uintptr_t{7}); - } - - void set_prefix(const char* data, usize len) - { - this->prefix_len_ = len; - if (len) { - __builtin_memcpy((char*)this->prefix(), data, len); - } - } - }; - - struct LeafNode : NodeBase { - using Self = LeafNode; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - explicit LeafNode() noexcept : Super{NodeType::kLeafNode} - { - } - - explicit LeafNode(NoInit no_init) noexcept : Super{NodeType::kLeafNode, no_init} - { - } - - static usize add_branch() - { - BATT_PANIC() << "not supported!"; - return 0; - } - - static void set_branch_index(u8 key_byte [[maybe_unused]], usize index [[maybe_unused]]) - { - BATT_PANIC() << "not supported!"; - } - - static void set_branch_pointer(usize index [[maybe_unused]], NodeBase* child [[maybe_unused]]) - { - BATT_PANIC() << "not supported!"; - } - - static constexpr usize max_branch_count() - { - return 0; - } - - static constexpr usize branch_count() - { - return 0; - } - - static constexpr usize index_of_branch(u8 key_byte [[maybe_unused]]) - { - return 0; - } - - static NodeBase*& get_branch_ref(usize i [[maybe_unused]]) - { - static NodeBase* null_ = nullptr; - return null_; - } - - //+++++++++++-+-+--+----- --- -- - - - - - - struct ScanState { - explicit ScanState(Self&, ByteInt /*min_key*/) noexcept - { - } - - static constexpr ByteInt get_key_byte() - { - return ByteInt::from_char('\0'); - } - - static constexpr NodeBase* get_branch() - { - return nullptr; - } - - static constexpr bool is_done() - { - return true; - } - - static constexpr void advance() - { - } - }; - }; - - struct BranchView { - NodeBase** p_ptr; - NodeBase* ptr; - - //+++++++++++-+-+--+----- --- -- - - - - - - BranchView() noexcept : p_ptr{nullptr}, ptr{nullptr} - { - } - - explicit BranchView(NodeBase*& branch) noexcept : p_ptr{&branch}, ptr{branch} - { - } - - void load(NodeBase*& branch) - { - this->p_ptr = &branch; - this->ptr = branch; - } - - template - NodeT* store(NodeT* new_ptr) - { - static_assert(std::is_base_of_v); - - *this->p_ptr = new_ptr; - this->ptr = new_ptr; - return new_ptr; - } - - NodeBase* reload() - { - this->ptr = *this->p_ptr; - return this->ptr; - } - }; - - static constexpr NodeType node_type_from_branch_count(usize branch_count) - { - if (branch_count == 4) { - return NodeType::kNode4; - } else if (branch_count == 16) { - return NodeType::kNode16; - } else if (branch_count == 48) { - return NodeType::kNode48; - } else { - return NodeType::kNode256; - } - } - - static_assert(sizeof(NodeBase) == 8); - - using BranchIndex = u8; - - static constexpr BranchIndex kInvalidBranchIndex = u8{255}; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct GrowableNode : NodeBase { - using Self = GrowableNode; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branches_; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit GrowableNode() noexcept : NodeBase{node_type_from_branch_count(kBranchCount)} - { - } - - explicit GrowableNode(NoInit no_init) noexcept - : NodeBase{node_type_from_branch_count(kBranchCount), no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - Derived* derived() - { - return (Derived*)this; - } - - //----- --- -- - - - - - - usize branch_count() const - { - return this->branch_count_; - } - - usize add_branch() - { - const usize i = this->branch_count_; - ++this->branch_count_; - return i; - } - - NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE - { - return this->branches_[i]; - } - - void set_branch_pointer(usize i, NodeBase* child) BATT_ALWAYS_INLINE - { - this->branches_[i] = child; - } - - static constexpr usize max_branch_count() - { - return kBranchCount; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - __builtin_memcpy(this->branches_.data(), - that.branches_.data(), - this->branch_count() * sizeof(NodeBase*)); - } - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct IndirectIndexedNode : GrowableNode> { - using Self = IndirectIndexedNode; - using Super = GrowableNode; - using NoInit = ARTBase::NoInit; - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) \ - if (bit_i == 64) { \ - break; \ - } \ - key_byte = ByteInt::from_i32(key_byte_offset + bit_i); \ - branch = branch_for_byte[key_byte.to_i32()]; \ - if (branch) { \ - this->sorted_branches_[this->branch_count_] = branch; \ - this->sorted_keys_[this->branch_count_] = key_byte; \ - ++this->branch_count_; \ - } \ - bit_i = next_bit(word_val, bit_i) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset) \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32(key_byte_offset) - -#define TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(word_i, key_byte_offset) \ - word_val = key_bitmap[word_i]; \ - for (;;) { \ - i32 bit_i = first_bit(word_val); \ - TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64(key_byte_offset); \ - break; \ - } - - struct ScanState { - Self& self_; - usize branch_count_; - usize i_; - std::array sorted_branches_; - std::array sorted_keys_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept - : self_{self} - , branch_count_{0} - , i_{0} - { - std::array branch_for_byte; - std::array key_bitmap = {0, 0, 0, 0}; - - const usize n_branches = this->self_.branch_count(); - - for (usize i = 0; i < n_branches; ++i) { - const ByteInt key_byte = ByteInt::from_u8(this->self_.key[i]); - if (key_byte < min_key) { - continue; - } - branch_for_byte[key_byte.to_i32()] = this->self_.branches_[i]; - key_bitmap[(key_byte.to_i32() >> 6) & 3] |= (u64{1} << (key_byte.to_i32() & 0x3f)); - } - - u64 word_val; - ByteInt key_byte; - NodeBase* branch; - - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(0, 0) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(1, 64) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(2, 128) - TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP(3, 192) - } - - ByteInt get_key_byte() const - { - return this->sorted_keys_[this->i_]; - } - - NodeBase* get_branch() const - { - return this->sorted_branches_[this->i_]; - } - - bool is_done() const - { - return this->i_ >= this->branch_count_; - } - - void advance() - { - ++this->i_; - } - }; - -#undef TURTLE_KV_ART_SMALL_NODE_OUTER_LOOP -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_64 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_32 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_16 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_8 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_4 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_2 -#undef TURTLE_KV_ART_SMALL_NODE_INNER_LOOP_1 - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array key; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit IndirectIndexedNode() noexcept : Super{} - { - } - - explicit IndirectIndexedNode(NoInit no_init) noexcept : Super{no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - usize index_of_branch(u8 key_byte) - { - return index_of(key_byte, this->key); - } - - void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE - { - this->key[i] = key_byte; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - __builtin_memcpy(this->key.data(), that.key.data(), this->branch_count()); - } - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - template - struct DirectIndexedNode : GrowableNode> { - using Self = DirectIndexedNode; - using Super = GrowableNode; - using NoInit = ARTBase::NoInit; - - struct ScanState { - Self& self_; - ByteInt key_byte_; - usize branch_i_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept - : self_{self} - , key_byte_{min_key} - , branch_i_{kInvalidBranchIndex} - { - this->skip_invalid_branches(); - } - - ByteInt get_key_byte() const - { - return this->key_byte_; - } - - NodeBase* get_branch() const - { - return this->self_.branches_[this->branch_i_]; - } - - bool is_done() const - { - return this->key_byte_ >= ByteInt::from_i32(256); - } - - void advance() - { - ++this->key_byte_; - this->skip_invalid_branches(); - } - - void skip_invalid_branches() - { - while (!this->is_done()) { - this->branch_i_ = this->self_.branch_for_key[this->key_byte_.to_i32()]; - if (this->branch_i_ != kInvalidBranchIndex) { - break; - } - ++this->key_byte_; - } - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branch_for_key; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit DirectIndexedNode() noexcept : Super{} - { - this->branch_for_key.fill(kInvalidBranchIndex); - } - - explicit DirectIndexedNode(NoInit no_init) noexcept : Super{no_init} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - usize index_of_branch(u8 key_byte) - { - return this->branch_for_key[key_byte]; - } - - void set_branch_index(u8 key_byte, usize i) BATT_ALWAYS_INLINE - { - this->branch_for_key[key_byte] = i; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - this->branch_for_key = that.branch_for_key; - } - }; - - struct Node4 : IndirectIndexedNode<4> { - using IndirectIndexedNode<4>::IndirectIndexedNode; - }; - - struct Node16 : IndirectIndexedNode<16> { - using IndirectIndexedNode<16>::IndirectIndexedNode; - }; - - struct Node48 : DirectIndexedNode<48> { - using DirectIndexedNode<48>::DirectIndexedNode; - }; - - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - - // - struct Node256 : NodeBase { - using Self = Node256; - using Super = NodeBase; - using NoInit = ARTBase::NoInit; - - struct ScanState { - Self& self_; - ByteInt key_byte_; - - //----- --- -- - - - - - - explicit ScanState(Self& self, ByteInt min_key) noexcept : self_{self}, key_byte_{min_key} - { - this->skip_null_branches(); - } - - ByteInt get_key_byte() const - { - return this->key_byte_; - } - - NodeBase* get_branch() const - { - return this->self_.branches_[this->key_byte_.to_i32()]; - } - - bool is_done() const - { - return this->key_byte_ >= ByteInt::from_i32(256); - } - - void advance() - { - ++this->key_byte_; - this->skip_null_branches(); - } - - private: - void skip_null_branches() - { - while (!this->is_done() && this->get_branch() == nullptr) { - ++this->key_byte_; - } - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - std::array branches_; - - //+++++++++++-+-+--+----- --- -- - - - - - - Node256() noexcept : Super{NodeType::kNode256} - { - this->branches_.fill(nullptr); - } - - explicit Node256(NoInit no_init) noexcept : Super{NodeType::kNode256, no_init} - { - } - - Node256(const Node256&) = delete; - Node256& operator=(const Node256&) = delete; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr usize branch_count() - { - return 256; - } - - usize add_branch() - { - BATT_PANIC() << "Node256::add_branch is illegal!"; - BATT_UNREACHABLE(); - } - - static constexpr usize max_branch_count() - { - return 256; - } - - usize index_of_branch(u8 key_byte) const - { - return key_byte; - } - - void set_branch_index(u8, usize) - { - } - - NodeBase*& get_branch_ref(usize i) BATT_ALWAYS_INLINE - { - return this->branches_[i]; - } - - void assign_from(const Self& that, usize prefix_offset = 0) - { - this->Super::assign_from(static_cast(that), prefix_offset); - this->branches_ = that.branches_; - } - }; - - //----- --- -- - - - - - - static_assert(sizeof(Node4) == 48); - static_assert(sizeof(Node4) % 8 == 0); - static_assert(alignof(Node4) >= 8); - - static_assert(sizeof(Node16) == 152); - static_assert(sizeof(Node16) % 8 == 0); - static_assert(alignof(Node16) >= 8); - - static_assert(sizeof(Node48) == 648); - static_assert(sizeof(Node48) % 8 == 0); - static_assert(alignof(Node48) >= 8); - - static_assert(sizeof(Node256) == 2056); - static_assert(sizeof(Node256) % 8 == 0); - static_assert(alignof(Node256) >= 8); - - static constexpr usize kExtentSize = 64 * kKiB; - static constexpr usize kExtentAlign = 4096; - - using ExtentStorageT = std::aligned_storage_t; - - static_assert(sizeof(ExtentStorageT) == kExtentSize); - - //----- --- -- - - - - - - struct MemoryContext { - ARTBase* art_{nullptr}; - std::vector> thread_extents_; - u8* data_{nullptr}; - usize in_use_{sizeof(ExtentStorageT)}; - - //+++++++++++-+-+--+----- --- -- - - - - - - ~MemoryContext() noexcept - { - if (this->art_) { - absl::MutexLock lock{&this->art_->mutex_}; - for (auto& p_ex : this->thread_extents_) { - this->art_->extents_.emplace_back(std::move(p_ex)); - } - } - } - - void* alloc(usize n, ARTBase* art) - { - this->art_ = art; - - const usize in_use_prior = this->in_use_; - if (in_use_prior + n <= kExtentSize) { - this->in_use_ += n; - return this->data_ + in_use_prior; - } - - this->art_->metrics_.byte_alloc_count.add(sizeof(ExtentStorageT)); - - this->thread_extents_.emplace_back(std::make_unique()); - u8* start = reinterpret_cast(this->thread_extents_.back().get()); - this->data_ = start; - this->in_use_ = 0; - - return this->alloc(n, art); - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit ARTBase() noexcept : ARTBase{ARTBase::default_metrics()} - { - // No update of construct count because we delegate to general-case ctor. - } - - explicit ARTBase(Metrics& metrics) noexcept : metrics_{metrics} - { - this->metrics_.construct_count.add(1); - } - - ~ARTBase() noexcept - { - this->metrics_.destruct_count.add(1); - } - - //+++++++++++-+-+--+----- --- -- - - - - - protected: - /** \brief RAII (guard) class that updates byte_free_count metric at the right moment during - * destruction of the ART (see comment in data member declarations below). - */ - struct ExtentMetricsUpdateGuard { - ARTBase& art_base_; - - //----- --- -- - - - - - - explicit ExtentMetricsUpdateGuard(ARTBase& art_base) noexcept : art_base_{art_base} - { - } - - ~ExtentMetricsUpdateGuard() noexcept - { - this->art_base_.metrics_.byte_free_count.add(this->art_base_.extents_.size() * - sizeof(ExtentStorageT)); - } - }; - - void* alloc_storage(usize n, usize pre) - { - const usize pad = (pre + 7) & ~usize{7}; - char* const ptr = (char*)this->per_thread_memory_context_.get().alloc(n + pad, this); - return ptr + pad; - } - - Metrics& metrics_; - absl::Mutex mutex_; - std::vector> extents_; - // - // Must be placed exactly here, so it will be destructed after the ScopedSlot but before extents_. - ExtentMetricsUpdateGuard guard_{*this}; - // - ObjectThreadStorage::ScopedSlot per_thread_memory_context_; -}; - -namespace detail { - -template -struct DefaultCopyInserter { - const ValueT& copy_from_; - - explicit DefaultCopyInserter(const ValueT& copy_from) noexcept : copy_from_{copy_from} - { - } - - Status insert_new(void* copy_to) - { - new (copy_to) ValueT{this->copy_from_}; - return OkStatus(); - } - - Status update_existing(ValueT* copy_to) - { - *copy_to = this->copy_from_; - return OkStatus(); - } -}; - -template -struct DefaultMoveInserter { - ValueT&& move_from_; - - explicit DefaultMoveInserter(ValueT&& move_from) noexcept : move_from_{move_from} - { - } - - Status insert_new(void* move_to) - { - new (move_to) ValueT{std::move(this->move_from_)}; - return OkStatus(); - } - - Status update_existing(ValueT* move_to) - { - *move_to = std::move(this->move_from_); - return OkStatus(); - } -}; - -struct DefaultVoidInserter { - BATT_ALWAYS_INLINE Status insert_new(void*) - { - return OkStatus(); - } - - BATT_ALWAYS_INLINE Status update_existing(void*) - { - return OkStatus(); - } -}; - -} // namespace detail - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -template -class ART : public ARTBase -{ - public: - using Self = ART; - using Super = ARTBase; - - using value_type = ValueT; - - //+++++++++++-+-+--+----- --- -- - - - - - - static constexpr usize kValueStorageSize = Super::sizeof_value(batt::StaticType{}); - - //+++++++++++-+-+--+----- --- -- - - - - - - template - class Scanner; - - //+++++++++++-+-+--+----- --- -- - - - - - - ART() noexcept - { - } - - explicit ART(ARTBase::Metrics& metrics) noexcept : ARTBase{metrics} - { - } - - ~ART() noexcept - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - template - Status insert(std::string_view key, InserterT&& inserter); - - BATT_ALWAYS_INLINE void insert(std::string_view key) - { - static_assert(std::is_same_v); - - this->insert(key, detail::DefaultVoidInserter{}).IgnoreError(); - } - - bool contains(std::string_view key); - - const ValueT* unsynchronized_find(std::string_view key); - - Optional find(std::string_view key); - - template - void scan(std::string_view lower_bound_key, const Fn& fn); - - /** \brief Returns true iff the container is empty. - */ - bool empty() - { - BranchView branch; - for (;;) { - SeqMutex::ReadLock root_read_lock{this->super_root_.mutex_}; - branch.load(this->root_); - if (!root_read_lock.changed()) { - break; - } - } - return branch.ptr == nullptr; - } - - //+++++++++++-+-+--+----- --- -- - - - - - private: - using SmallestParentNode = Node4; - - //+++++++++++-+-+--+----- --- -- - - - - - - template >> - NodeBase* add_child(NodeT* node, u8 key_byte, NodeBase* child); - - NodeBase* add_child(Node256* node, u8 key_byte, NodeBase* child); - - template - LeafNode* add_child_leaf(NodeT* node, u8 key_byte, const char* new_key_data, usize new_key_len); - - LeafNode* make_leaf_node(const char* prefix, usize prefix_len); - - Node4* make_parent_node(const char* prefix, usize prefix_len); - - Node4* grow_node(LeafNode* old_node); - - Node16* grow_node(Node4* old_node); - - Node48* grow_node(Node16* old_node); - - Node256* grow_node(Node48* old_node); - - Node256* grow_node(Node256*); - - LeafNode* clone_node(LeafNode* orig_node, usize prefix_offset); - - Node4* clone_node(Node4* orig_node, usize prefix_offset); - - Node16* clone_node(Node16* orig_node, usize prefix_offset); - - Node48* clone_node(Node48* orig_node, usize prefix_offset); - - Node256* clone_node(Node256* orig_node, usize prefix_offset); - - template - void find_impl(std::string_view key, batt::StaticType, NodeCallbackFn&& node_callback); - - //+++++++++++-+-+--+----- --- -- - - - - - - NodeBase super_root_{NodeType::kNodeBase}; - NodeBase* root_ = nullptr; -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline void ARTBase::NodeBase::visit(CaseFns&&... case_fns) -{ - auto visitor = batt::make_case_of_visitor(BATT_FORWARD(case_fns)...); - - const NodeType observed = this->node_type; - - switch (observed) { - case NodeType::kLeafNode: - visitor(static_cast(this)); - break; - - case NodeType::kNode4: - visitor(static_cast(this)); - break; - - case NodeType::kNode16: - visitor(static_cast(this)); - break; - - case NodeType::kNode48: - visitor(static_cast(this)); - break; - - case NodeType::kNode256: - visitor(static_cast(this)); - break; - - case NodeType::kNodeBase: // fall-through - default: - BATT_PANIC() << "Bad node type: " << (int)observed; - BATT_UNREACHABLE(); - } -} - -namespace detail { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of(usize node_prefix_len, - NodeT* node, - AlignedStorageT* storage, - std::integral_constant, - const Optional&, - void* value_storage_addr, - batt::StaticType type_of_value) -{ - NodeT& node_view = *(new (storage) NodeT{ARTBase::NoInit{}}); - - // Retry the node read until we get a consistent view. - // - for (;;) { - SeqMutex::ReadLock read_lock{node->mutex_}; - node_view.assign_from(*node, /*prefix_offset=*/node_prefix_len); - if (node->is_terminal()) { - ARTBase::construct_value_copy_addr(node, value_storage_addr, type_of_value); - } - if (!read_lock.changed()) { - break; - } - } - - return node_view; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of(usize, - NodeT* node, - AlignedStorageT*, - std::integral_constant, - const Optional& /*sync*/, - const void* /*value_storage_addr*/, - batt::StaticType /*type_of_value*/) -{ - return *node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -NodeT& scanner_view_of( - usize node_prefix_len, - NodeT* node, - AlignedStorageT* storage, - std::integral_constant, - const Optional& sync, - void* value_storage_addr, - batt::StaticType type_of_value) -{ - if (sync.value_or(true)) { - return scanner_view_of( - node_prefix_len, - node, - storage, - std::integral_constant{}, - sync, - value_storage_addr, - type_of_value); - } - return scanner_view_of( - node_prefix_len, - node, - storage, - std::integral_constant{}, - sync, - value_storage_addr, - type_of_value); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -struct ValueStorageBase { - std::aligned_storage_t value_storage_; - - template - std::conditional_t, const void*, void*> value_storage_address( - NodeT* node, - const Optional& sync) - { - if (kSynchronized == ARTBase::Synchronized::kTrue || sync.value_or(true)) { - return &this->value_storage_; - } - return node + 1; - } -}; - -//----- --- -- - - - - - -template -struct ValueStorageBase { - template - const void* value_storage_address(const NodeT* node, const Optional&) const - { - return node + 1; - } -}; - -//----- --- -- - - - - - -template <> -struct ValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//----- --- -- - - - - - -template <> -struct ValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//----- --- -- - - - - - -template <> -struct ValueStorageBase { - template - void* value_storage_address(const NodeT*, const Optional&) const - { - return nullptr; - } -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Base class for scanner; contains storage for the item at the current scanner position. - */ -template -struct ItemStorageBase; - -/** \brief General case (kValuesOnly == false) for scanner item storage. - */ -template -class ItemStorageBase - : public ValueStorageBase -{ - public: - std::array::kMaxKeyLen> key_buffer_; - usize key_len_ = 0; - - //----- --- -- - - - - - - void append_key(usize prefix_len, const char* suffix_data, usize suffix_len) BATT_ALWAYS_INLINE - { - __builtin_memcpy(this->key_buffer_.data() + prefix_len, suffix_data, suffix_len); - } - - void append_key_byte(usize prefix_len, const ByteInt& suffix_byte) BATT_ALWAYS_INLINE - { - this->key_buffer_[prefix_len] = suffix_byte.to_char(); - } - - void set_key_len(usize len) BATT_ALWAYS_INLINE - { - this->key_len_ = len; - } - - std::string_view get_key() const - { - return std::string_view{this->key_buffer_.data(), this->key_len_}; - } -}; - -/** \brief kValuesOnly == true case; no key-related data members. - */ -template -class ItemStorageBase - : public ValueStorageBase -{ - public: - void append_key(usize, const char*, usize) BATT_ALWAYS_INLINE - { - // nothing to do. - } - - void append_key_byte(usize, const ByteInt&) BATT_ALWAYS_INLINE - { - // nothing to do. - } - - void set_key_len(usize) BATT_ALWAYS_INLINE - { - // nothing to do. - } -}; - -} // namespace detail - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Scanner for an ART. - * - * \tparam ValueT The value type stored in the scanned ART - * \tparam kSynchronized (true, false, dynmamic) The concurrency control for this scanner - * \tparam kValuesOnly When true, the scanner does not build/store key (path) information as it is - * traversing items -- only values are available - */ -template -template -class ART::Scanner : public detail::ItemStorageBase -{ - public: - using LeafNode = ARTBase::LeafNode; - using Node4 = ARTBase::Node4; - using Node16 = ARTBase::Node16; - using Node48 = ARTBase::Node48; - using Node256 = ARTBase::Node256; - - using NodeScanState = std::variant; - - static constexpr usize kMaxDepth = ART::kMaxKeyLen; - - using SyncType = std::integral_constant; - - using Value = std::conditional_t, - struct get_value_Not_Supported_If_ValueT_Is_Void, - ValueT>; - - //+++++++++++-+-+--+----- --- -- - - - - - - static_assert(sizeof(Node256) > sizeof(Node48)); - static_assert(sizeof(Node256) > sizeof(Node16)); - static_assert(sizeof(Node256) > sizeof(Node4)); - static_assert(sizeof(Node256) > sizeof(LeafNode)); - - struct Frame { - static constexpr usize kStorageSize = - ((kSynchronized == ARTBase::Synchronized::kFalse) ? 1 : sizeof(Node256)); - - std::aligned_storage_t node_storage_; - NodeScanState scan_state_; - usize key_prefix_len_; - std::string_view lower_bound_key_; - ByteInt min_key_byte_; - - explicit Frame(usize key_prefix_len, std::string_view lower_bound_key) noexcept - : scan_state_{None} - , key_prefix_len_{key_prefix_len} - , lower_bound_key_{lower_bound_key} - , min_key_byte_{ByteInt::from_i32(0)} - { - } - }; - - //+++++++++++-+-+--+----- --- -- - - - - - private: - std::aligned_storage_t stack_storage_; - Frame* end_ = reinterpret_cast(&this->stack_storage_); - usize depth_ = 0; - bool have_item_ = false; - ValueT* next_value_ = nullptr; - Optional synchronized_; - - //+++++++++++-+-+--+----- --- -- - - - - - - /** Resets the "have item" state of the scanner; after calling, have item will be false. - */ - void reset_item() BATT_ALWAYS_INLINE - { - this->have_item_ = false; - if (!std::is_same_v) { - this->next_value_ = nullptr; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - public: - explicit Scanner(ART& art, - std::string_view lower_bound_key, - Optional synchronized = None) noexcept - : synchronized_{synchronized} - { - NodeBase* root = nullptr; - for (;;) { - SeqMutex::ReadLock root_read_lock{art.super_root_.mutex_}; - root = art.root_; - if (!root_read_lock.changed()) { - break; - } - } - - if (root) { - root->visit([&](auto* node) { - this->enter(node, /*key_prefix_len=*/0, lower_bound_key); - }); - - if (!this->have_item_) { - this->advance(); - } - } - } - - ~Scanner() noexcept - { - } - - bool is_synchronized() const - { - if (kSynchronized == ARTBase::Synchronized::kFalse) { - return false; - } - if (kSynchronized == ARTBase::Synchronized::kTrue) { - return true; - } - return this->synchronized_.value_or(true); - } - - template - void enter(NodeT* node, usize key_prefix_len, std::string_view lower_bound_key) - { - Frame* top = new (this->end_) Frame{key_prefix_len, lower_bound_key}; - ++this->depth_; - ++this->end_; - - // Node prefix is immutable, so we don't need synchronization. - // - const char* const node_prefix = node->prefix(); - const usize node_prefix_len = node->prefix_len_; - - // We need to create a copy of the node data to protect against data races. - // - NodeT& node_view = - detail::scanner_view_of(node_prefix_len, - node, - &top->node_storage_, - SyncType{}, - this->synchronized_, - this->value_storage_address(node, this->synchronized_), - batt::StaticType{}); - - // Compare the lower bound key to the current node prefix. - // - const usize compare_len = std::min(node_prefix_len, top->lower_bound_key_.size()); - if (compare_len) { - const i32 order = __builtin_memcmp(node_prefix, top->lower_bound_key_.data(), compare_len); - - // If all keys in this subtree come before the lower bound, then there is nothing to do. - // - if (order < 0) { - --this->depth_; - --this->end_; - return; - } - - // If the node prefix is a prefix of the lower bound key, then drop the prefix from the lower - // bound; otherwise the node prefix comes *after* the lower bound, so we can safely ignore the - // lower bound for the rest of the recursion. - // - if (order == 0 && compare_len == node_prefix_len) { - top->lower_bound_key_.remove_prefix(compare_len); - } else { - top->lower_bound_key_ = {}; - } - } - - // Set bounds for branch visitation. - // - top->min_key_byte_ = [&]() -> ByteInt { - if (top->lower_bound_key_.empty()) { - return ByteInt::from_i32(0); - } - const ByteInt next_char = ByteInt::from_char(top->lower_bound_key_.front()); - top->lower_bound_key_.remove_prefix(1); - return next_char; - }(); - - // Append the node prefix to the buffer. - // - if (node_prefix_len) { - this->append_key(top->key_prefix_len_, node_prefix, node_prefix_len); - top->key_prefix_len_ += node_prefix_len; - } - - // If the current node is a key-terminal, emit the contents of the buffer. - // - if (node_view.is_terminal()) { - this->have_item_ = true; - this->set_key_len(top->key_prefix_len_); - if (!std::is_same_v) { - this->next_value_ = (ValueT*)(this->value_storage_address(&node_view, this->synchronized_)); - } - } else { - this->reset_item(); - } - - [[maybe_unused]] auto& scan_state_impl = - top->scan_state_.template emplace(node_view, top->min_key_byte_); - } - - bool is_done() const - { - return this->depth_ == 0; - } - - // get_key() const member function is inherited from ItemStorageBase, if kValuesOnly is false. - - const Value& get_value() const - { - static_assert(!std::is_same_v); - return *this->next_value_; - } - - void advance() - { - this->reset_item(); - - for (;;) { - if (this->depth_ == 0) { - return; - } - - Frame* top = this->end_ - 1; - - batt::case_of( - top->scan_state_, - [](batt::NoneType&) { - BATT_PANIC() << "empty Scanner stack frame!"; - }, - [&](auto& scan_state) - -> std::enable_if_t< - !std::is_same_v, batt::NoneType>> { - //----- --- -- - - - - - if (scan_state.is_done()) { - --this->depth_; - --this->end_; - return; - } - - const ByteInt key_byte = scan_state.get_key_byte(); - NodeBase* const child = scan_state.get_branch(); - - this->append_key_byte(top->key_prefix_len_, key_byte); - - if (key_byte == top->min_key_byte_) { - child->visit([&](auto* child_node) { - this->enter(child_node, top->key_prefix_len_ + 1, top->lower_bound_key_); - }); - } else { - child->visit([&](auto* child_node) { - this->enter(child_node, top->key_prefix_len_ + 1, std::string_view{}); - }); - } - - scan_state.advance(); - }); - - if (this->have_item_) { - return; - } - } - } -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline void ART::scan(std::string_view lower_bound_key, const Fn& fn) -{ - Scanner scanner{*this, lower_bound_key}; - - while (!scanner.is_done()) { - if (!fn(scanner.get_key())) { - return; - } - scanner.advance(); - } -} - -} // namespace turtle_kv - -#include diff --git a/src/turtle_kv/util/art.ipp b/src/turtle_kv/util/art.ipp deleted file mode 100644 index e9276bb..0000000 --- a/src/turtle_kv/util/art.ipp +++ /dev/null @@ -1,706 +0,0 @@ -#pragma once - -#include - -namespace turtle_kv { - -namespace detail { - -inline usize find_common_prefix_len(const char* data0, usize size0, const char* data1, usize size1) -{ - usize n = 0; - usize common_size = std::min(size0, size1); - while (common_size && *data0 == *data1) { - ++n; - --common_size; - ++data0; - ++data1; - } - return n; -} - -} // namespace detail - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline Status ART::insert(std::string_view key, InserterT&& inserter) -{ - bool reset = true; - - const char* key_data = nullptr; - usize key_len = 0; - BranchView branch; - NodeBase* parent = nullptr; - - this->metrics_.insert_count.add(1); - - for (;;) { - if (reset) { - reset = false; - - key_data = key.data(); - key_len = key.size(); - - for (;;) { - SeqMutex::ReadLock root_read_lock{this->super_root_.mutex_}; - branch.load(this->root_); - if (!root_read_lock.changed()) { - break; - } - } - - if (branch.ptr == nullptr) { - SeqMutex::WriteLock root_write_lock{this->super_root_.mutex_}; - if (branch.reload() == nullptr) { - LeafNode* new_node = branch.store(this->make_leaf_node(key_data, key_len)); - Status status = inserter.insert_new(ARTBase::uninitialized_value(new_node)); - if (status.ok()) { - new_node->set_terminal(); - } - return status; - } - } - - parent = &this->super_root_; - } - - Status status = OkStatus(); - bool done = false; - - branch.ptr->visit([&](auto* node) { - SeqMutex::ReadLock node_read_lock{node->mutex_}; - - const char* const node_prefix = node->prefix(); - const usize node_prefix_len = node->prefix_len_; - const bool node_is_terminal = node->is_terminal(); - - if (node_read_lock.changed()) { - reset = true; - return; - } - - const usize common_len = - detail::find_common_prefix_len(node_prefix, node_prefix_len, key_data, key_len); - - // If the key matches the prefix of the current node only partially, then we must split the - // prefix and insert a new parent node. - // - if (common_len != node_prefix_len) { - //----- --- -- - - - - - SeqMutex::WriteLock parent_write_lock{parent->mutex_}; - if (parent->is_obsolete()) { - reset = true; - return; - } - - //----- --- -- - - - - - SeqMutex::WriteLock node_write_lock{node->mutex_}; - - if (node != branch.reload() || node_prefix != node->prefix() || - node_prefix_len != node->prefix_len_ || node->is_obsolete()) { - reset = true; - return; - } - - SmallestParentNode* new_parent = this->make_parent_node(node_prefix, common_len); - auto* new_node = this->clone_node(node, /*prefix_offset=*/(common_len + 1)); - - this->add_child(new_parent, /*key_byte=*/node_prefix[common_len], new_node); - - if (common_len < key_len) { - LeafNode* new_child_leaf = this->add_child_leaf(new_parent, - /*key_byte=*/key_data[common_len], - key_data + (common_len + 1), - key_len - (common_len + 1)); - status = inserter.insert_new(ARTBase::uninitialized_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - - } else { - if (new_parent->is_terminal()) { - status = inserter.update_existing(ARTBase::mutable_value(new_parent)); - } else { - status = inserter.insert_new(ARTBase::uninitialized_value(new_parent)); - if (status.ok()) { - new_parent->set_terminal(); - } - } - } - - node->set_obsolete(); - branch.store(new_parent); - done = true; - return; - } - - // If the common prefix is exactly the length of this node's prefix, then the search key is - // fully consumed exactly at this node; just update the node and we are done! - // - if (key_len == common_len) { - if (!node_is_terminal || !std::is_same_v) { - //----- --- -- - - - - - SeqMutex::WriteLock node_write_lock{node->mutex_}; - - // Now that we are holding the lock, check if anything has changed, and retry if so. - // - if (node->is_obsolete() || node_prefix != node->prefix() || - node_prefix_len != node->prefix_len_) { - reset = true; - return; - } - - // If the node was already terminal, then update (in the case of ValueT=void, this is a - // no-op; we would have short-circuited at the immediately enclosing conditional), - // otherwise mark the node as storing a value, and call InserterT::insert_new. - // - if (!node_is_terminal) { - status = inserter.insert_new(uninitialized_value(node)); - if (status.ok()) { - node->set_terminal(); - } - } else { - status = inserter.update_existing(ARTBase::mutable_value(node)); - } - } - done = true; - return; - } - - const u8 key_byte = key_data[common_len]; - const char* const new_key_data = key_data + (common_len + 1); - const usize new_key_len = key_len - (common_len + 1); - - const usize observed_branch_count = std::min(node->branch_count(), node->max_branch_count()); - BranchView next; - { - const usize i = node->index_of_branch(key_byte); - if (i < observed_branch_count) { - next.load(node->get_branch_ref(i)); - } - } - - if (node_read_lock.changed()) { - reset = true; - return; - } - - // The node has not changed, and there *is* a branch for the current key byte; step into that - // subtree and continue in the outer loop. - // - if (next.p_ptr != nullptr && next.ptr != nullptr) { - parent = node; - branch = next; - key_data = new_key_data; - key_len = new_key_len; - return; - } - - // There is no branch on the current node for the current key byte. - // - Optional::WriteLock> parent_write_lock; - Optional::WriteLock> node_write_lock; - - // If we can't add a branch because the observed branch count is the maximum, then grow the - // node. This requires a lock on the current node and its parent. - // - if (next.p_ptr == nullptr && observed_branch_count == node->max_branch_count()) { - //----- --- -- - - - - - parent_write_lock.emplace(parent->mutex_); - if (parent->is_obsolete()) { - reset = true; - return; - } - - //----- --- -- - - - - - node_write_lock.emplace(node->mutex_); - if (node->is_obsolete()) { - reset = true; - return; - } - - BATT_CHECK_EQ(observed_branch_count, node->branch_count()); - - auto* new_node = this->grow_node(node); - LeafNode* new_child_leaf = - this->add_child_leaf(new_node, key_byte, new_key_data, new_key_len); - status = inserter.insert_new(ARTBase::mutable_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - - node->set_obsolete(); - branch.store(new_node); - - } else { - //----- --- -- - - - - - // It looks like there *may* be room for an extra branch on the current node, so try to add - // one. This requires only a lock on the current node. - // - node_write_lock.emplace(node->mutex_); - if (node->is_obsolete()) { - reset = true; - return; - } - - if (next.p_ptr == nullptr) { - if (observed_branch_count != node->branch_count()) { - reset = true; - return; - } - LeafNode* new_child_leaf = - this->add_child_leaf(node, key_byte, new_key_data, new_key_len); - status = inserter.insert_new(uninitialized_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - - } else { - if (next.reload() != nullptr) { - reset = true; - return; - } - LeafNode* new_child_leaf = this->make_leaf_node(new_key_data, new_key_len); - next.store(new_child_leaf); - status = inserter.insert_new(uninitialized_value(new_child_leaf)); - if (status.ok()) { - new_child_leaf->set_terminal(); - } - } - } - - done = true; - }); - - if (done) { - return status; - } - } -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -void ART::find_impl(std::string_view key, - batt::StaticType, - NodeCallbackFn&& node_callback) -{ - bool reset = true; - - const char* key_data = nullptr; - usize key_len = 0; - BranchView branch; - - for (;;) { - if (reset) { - reset = false; - - key_data = key.data(); - key_len = key.size(); - - for (;;) { - NodeLockT root_read_lock{this->super_root_.mutex_}; - branch.load(this->root_); - if (!root_read_lock.changed()) { - break; - } - } - if (branch.ptr == nullptr) { - return; // find_impl - } - } - - bool done = false; - - branch.ptr->visit([&](auto* node) { - NodeLockT node_read_lock{node->mutex_}; - - const char* const node_prefix = node->prefix(); - const usize node_prefix_len = node->prefix_len_; - const bool node_is_terminal = node->is_terminal(); - - if (node_read_lock.changed()) { - reset = true; - return; // visit - } - - const usize common_len = - detail::find_common_prefix_len(node_prefix, node_prefix_len, key_data, key_len); - - // Mismatch in the middle of the prefix means the branch we would have taken isn't there. Not - // found. - // - if (common_len != node_prefix_len) { - done = true; - return; // visit - } - - // If the search key is fully consumed, we are done; set `result` if the node is marked as - // terminal. - // - if (common_len == key_len) { - if (node_is_terminal) { - node_callback(node); - if (std::is_same_v || !node_read_lock.changed()) { - done = true; - } else { - node_callback((decltype(node))nullptr); - reset = true; - } - } else { - done = true; - } - return; // visit - } - - const u8 key_byte = key_data[common_len]; - - const usize observed_branch_count = std::min(node->branch_count(), node->max_branch_count()); - BranchView next; - { - const usize i = node->index_of_branch(key_byte); - if (i < observed_branch_count) { - next.load(node->get_branch_ref(i)); - } - } - - if (node_read_lock.changed()) { - reset = true; - return; // visit - } - - if (next.p_ptr == nullptr || next.ptr == nullptr) { - done = true; - return; // visit - } - - key_data += (common_len + 1); - key_len -= (common_len + 1); - branch = next; - }); - - if (done) { - return; // find_impl - } - } - BATT_UNREACHABLE(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline bool ART::contains(std::string_view key) -{ - bool found = false; - - this->find_impl(key, batt::StaticType::ReadLock>{}, [&found](auto* node) { - found = (node != nullptr); - }); - - return found; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::unsynchronized_find(std::string_view key) -> const ValueT* -{ - const ValueT* p_value = nullptr; - - this->find_impl(key, batt::StaticType::NullLock>{}, [&p_value](auto* node) { - if (node == nullptr) { - p_value = nullptr; - } else { - p_value = ARTBase::const_value(node); - } - }); - - return p_value; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::find(std::string_view key) -> Optional -{ - Optional value_copy; - - this->find_impl(key, batt::StaticType::ReadLock>{}, [&value_copy](auto* node) { - if (node == nullptr) { - value_copy = None; - } else { - value_copy.emplace(*ARTBase::const_value(node)); - } - }); - - return value_copy; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline auto ART::add_child(NodeT* node, u8 key_byte, NodeBase* child) -> NodeBase* -{ - const usize i = node->add_branch(); - node->set_branch_index(key_byte, i); - node->set_branch_pointer(i, child); - return child; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::add_child(Node256* node, u8 key_byte, NodeBase* child) -> NodeBase* -{ - const usize i = key_byte; - BATT_CHECK_EQ(node->branches_[i], nullptr); - node->branches_[i] = child; - return child; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -template -inline auto ART::add_child_leaf(NodeT* node, - u8 key_byte, - const char* new_key_data, - usize new_key_len) -> LeafNode* -{ - LeafNode* new_child = this->make_leaf_node(new_key_data, new_key_len); - this->add_child(node, key_byte, new_child); - return new_child; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::make_leaf_node(const char* prefix, usize prefix_len) -> LeafNode* -{ - LeafNode* new_node = - new (this->alloc_storage(sizeof(LeafNode) + kValueStorageSize, prefix_len)) LeafNode{}; - - new_node->set_prefix(prefix, prefix_len); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::make_parent_node(const char* prefix, usize prefix_len) -> Node4* -{ - Node4* new_node = - new (this->alloc_storage(sizeof(Node4) + kValueStorageSize, prefix_len)) Node4{}; - - new_node->set_prefix(prefix, prefix_len); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(LeafNode* old_node) -> Node4* -{ - Node4* new_node = - new (this->alloc_storage(sizeof(Node4) + kValueStorageSize, old_node->prefix_len_)) Node4{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(Node4* old_node) -> Node16* -{ - Node16* new_node = - new (this->alloc_storage(sizeof(Node16) + kValueStorageSize, old_node->prefix_len_)) Node16{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - new_node->branch_count_ = old_node->branch_count_; - - std::copy(old_node->key.begin(), old_node->key.end(), new_node->key.begin()); - std::copy(old_node->branches_.begin(), old_node->branches_.end(), new_node->branches_.begin()); - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(Node16* old_node) -> Node48* -{ - Node48* new_node = - new (this->alloc_storage(sizeof(Node48) + kValueStorageSize, old_node->prefix_len_)) Node48{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - new_node->branch_count_ = old_node->branch_count_; - - for (usize i = 0; i < new_node->branch_count_; ++i) { - new_node->branch_for_key[old_node->key[i]] = i; - new_node->branches_[i] = old_node->branches_[i]; - } - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -auto ART::grow_node(Node48* old_node) -> Node256* -{ - Node256* new_node = - new (this->alloc_storage(sizeof(Node256) + kValueStorageSize, old_node->prefix_len_)) - Node256{}; - - new_node->set_prefix(old_node->prefix(), old_node->prefix_len_); - - for (usize key_byte = 0; key_byte < 256; ++key_byte) { - const BranchIndex i = old_node->branch_for_key[key_byte]; - new_node->branches_[key_byte] = (i == kInvalidBranchIndex) ? nullptr : old_node->branches_[i]; - } - - if (old_node->is_terminal()) { - ARTBase::construct_value_copy_node(old_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(old_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::grow_node(Node256*) -> Node256* -{ - BATT_PANIC() << "Node256 can not grow larger!"; - BATT_UNREACHABLE(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(LeafNode* orig_node, usize prefix_offset) -> LeafNode* -{ - LeafNode* new_node = new (this->alloc_storage(sizeof(LeafNode) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) - LeafNode{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node4* orig_node, usize prefix_offset) -> Node4* -{ - Node4* new_node = - new (this->alloc_storage(sizeof(Node4) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) Node4{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node16* orig_node, usize prefix_offset) -> Node16* -{ - Node16* new_node = - new (this->alloc_storage(sizeof(Node16) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) Node16{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node48* orig_node, usize prefix_offset) -> Node48* -{ - Node48* new_node = - new (this->alloc_storage(sizeof(Node48) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) Node48{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline auto ART::clone_node(Node256* orig_node, usize prefix_offset) -> Node256* -{ - Node256* new_node = new (this->alloc_storage(sizeof(Node256) + kValueStorageSize, - (orig_node->prefix_len_ - prefix_offset))) - Node256{ARTBase::NoInit{}}; - - new_node->assign_from(*orig_node, prefix_offset); - - if (orig_node->is_terminal()) { - ARTBase::construct_value_copy_node(orig_node, new_node, batt::StaticType{}); - } - - BATT_CHECK_EQ(orig_node->is_terminal(), new_node->is_terminal()); - - return new_node; -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art.test.cpp b/src/turtle_kv/util/art.test.cpp deleted file mode 100644 index 91cb16e..0000000 --- a/src/turtle_kv/util/art.test.cpp +++ /dev/null @@ -1,918 +0,0 @@ -#include -// -#include - -#include -#include - -#include - -#include - -#include - -#include - -#include -#include -#include - -namespace { - -using namespace batt::int_types; - -using turtle_kv::ARTBase; -using turtle_kv::ByteInt; -using turtle_kv::LatencyMetric; -using turtle_kv::LatencyTimer; -using turtle_kv::None; -using turtle_kv::OkStatus; -using turtle_kv::Optional; -using turtle_kv::Status; -using turtle_kv::testing::RandomStringGenerator; - -using ART = turtle_kv::ART; - -struct BigUInt64KeyGenerator { - template - std::string operator()(Rng& rng) const - { - std::uniform_int_distribution pick_n{u64{0}, ~u64{0}}; - - std::string s; - s.resize(sizeof(llfs::big_u64)); - *((llfs::big_u64*)s.data()) = pick_n(rng); - - // depth == 1 will be Node256, but not full. - // - if ((s[0] & 7) == 0) { - s[0] += 1; - } - - // depth == 2 will max out at Node48. - // - s[1] &= 31; - - // depth == 3 will max out at Node16. - // - s[2] &= 15; - - // depth == 4 will max out at Node16, with space. - // - s[3] &= 7; - - // depth == 5 will max out at Node4. - // - s[4] &= 3; - - // depth == 6 will max out at Node4, with space. - // - s[5] &= 1; - - return s; - } -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, ByteInt) -{ - char a, b, c; - - a = 255; - b = 1; - c = 160; - - EXPECT_EQ(ByteInt::from_char(a).to_i32(), 255); - EXPECT_EQ(ByteInt::from_char(b).to_i32(), 1); - EXPECT_EQ(ByteInt::from_char(c).to_i32(), 160); - EXPECT_EQ(ByteInt::from_i32(-1).to_i32(), -1); - EXPECT_LT(ByteInt::from_i32(-1), ByteInt::from_char(a)); - EXPECT_LT(ByteInt::from_i32(-1), ByteInt::from_char(b)); - EXPECT_LT(ByteInt::from_i32(-1), ByteInt::from_char(c)); - - ByteInt i = ByteInt::from_i32(255); - - EXPECT_EQ(i, ByteInt::from_char(a)); - - ++i; - - EXPECT_EQ(i.to_i32(), 256); - EXPECT_EQ(i.to_char(), 0); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, OverlappingKeyPrefix) -{ - const std::string key1 = "app"; - const std::string key2 = "apple"; - const std::string key3 = "application"; - const std::string key4 = "applesauce"; - - turtle_kv::ART art; - - EXPECT_FALSE(art.contains(key1)); - EXPECT_FALSE(art.contains(key2)); - EXPECT_FALSE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key1, turtle_kv::detail::DefaultCopyInserter{1})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_FALSE(art.contains(key2)); - EXPECT_FALSE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key2, turtle_kv::detail::DefaultCopyInserter{2})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_TRUE(art.contains(key2)); - EXPECT_FALSE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key3, turtle_kv::detail::DefaultCopyInserter{3})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_TRUE(art.contains(key2)); - EXPECT_TRUE(art.contains(key3)); - EXPECT_FALSE(art.contains(key4)); - - BATT_CHECK_OK(art.insert(key4, turtle_kv::detail::DefaultCopyInserter{4})); - - EXPECT_TRUE(art.contains(key1)); - EXPECT_TRUE(art.contains(key2)); - EXPECT_TRUE(art.contains(key3)); - EXPECT_TRUE(art.contains(key4)); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -void run_put_contains_test() -{ - ARTBase::default_metrics().reset(); - - const usize num_keys = 1e5; - const usize num_scans = 10000; - const usize max_scan_length = 100; - - std::default_random_engine rng{/*seed=*/1}; - KeyGeneratorT generate_key; - std::uniform_int_distribution pick_scan_length{1, max_scan_length}; - - std::vector keys; - std::unordered_set inserted; - - for (usize i = 0; i < num_keys; ++i) { - keys.emplace_back(generate_key(rng)); - } - - ART index; - - { - ART::Scanner scanner{index, ""}; - EXPECT_TRUE(scanner.is_done()); - } - - usize i = 0; - for (const std::string& key : keys) { - if (inserted.count(key)) { - continue; - } - - inserted.emplace(key); - - EXPECT_FALSE(index.contains(key)); - - index.insert(key); - - EXPECT_TRUE(index.contains(key)) << BATT_INSPECT(i) << BATT_INSPECT_STR(key); - - ++i; - } - - for (const std::string& key : keys) { - EXPECT_TRUE(index.contains(key)); - } - - LatencyMetric sort_latency; - - std::vector sorted_keys; - for (const std::string& key : keys) { - sorted_keys.emplace_back(key); - } - { - LatencyTimer timer{sort_latency, sorted_keys.size()}; - std::sort(sorted_keys.begin(), sorted_keys.end()); - } - - LatencyMetric scan_latency; - LatencyMetric scanner_latency; - LatencyMetric scanner_nosync_latency; - LatencyMetric nokeys_scan_latency; - LatencyMetric nokeys_item_latency; - LatencyMetric nosync_nokeys_scan_latency; - LatencyMetric nosync_nokeys_item_latency; - - for (usize i = 0; i < num_scans; ++i) { - const std::string lower_bound_key = generate_key(rng); - const usize scan_length = pick_scan_length(rng); - - std::vector expected_result; - for (auto iter = std::lower_bound(sorted_keys.begin(), sorted_keys.end(), lower_bound_key); - iter != sorted_keys.end() && expected_result.size() < scan_length; - ++iter) { - expected_result.emplace_back(*iter); - } - - for (usize j = 0; j < 3; ++j) { - { - std::vector actual_result; - { - LatencyTimer timer{scan_latency}; - index.scan(lower_bound_key, [&actual_result, scan_length](const std::string_view& key) { - actual_result.emplace_back(key); - return actual_result.size() < scan_length; - }); - } - - EXPECT_EQ(expected_result.size(), actual_result.size()); - - BATT_CHECK_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << std::endl - << BATT_INSPECT_RANGE_PRETTY(expected_result) - << BATT_INSPECT_RANGE_PRETTY(actual_result); - - ASSERT_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << BATT_INSPECT(i); - } - - { - std::vector actual_result; - { - LatencyTimer timer{scanner_latency}; - - ART::Scanner scanner{index, lower_bound_key}; - - while (!scanner.is_done() && actual_result.size() < scan_length) { - actual_result.emplace_back(scanner.get_key()); - scanner.advance(); - } - } - - EXPECT_EQ(expected_result.size(), actual_result.size()); - - ASSERT_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << BATT_INSPECT(i); - } - - { - std::vector actual_result; - { - LatencyTimer timer{scanner_nosync_latency}; - - ART::Scanner scanner{index, lower_bound_key}; - - while (!scanner.is_done() && actual_result.size() < scan_length) { - actual_result.emplace_back(scanner.get_key()); - scanner.advance(); - } - } - - EXPECT_EQ(expected_result.size(), actual_result.size()); - - ASSERT_EQ(expected_result, actual_result) - << BATT_INSPECT_STR(lower_bound_key) << BATT_INSPECT(i); - } - { - usize items_found = 0; - { - auto start_time = std::chrono::steady_clock::now(); - - ART::Scanner scanner{index, - lower_bound_key}; - - while (!scanner.is_done() && items_found < scan_length) { - ++items_found; - scanner.advance(); - } - - nokeys_scan_latency.update(start_time, 1); - nokeys_item_latency.update(start_time, items_found); - } - EXPECT_EQ(expected_result.size(), items_found); - } - - { - usize items_found = 0; - { - auto start_time = std::chrono::steady_clock::now(); - - ART::Scanner scanner{index, - lower_bound_key}; - - while (!scanner.is_done() && items_found < scan_length) { - ++items_found; - scanner.advance(); - } - - nosync_nokeys_scan_latency.update(start_time, 1); - nosync_nokeys_item_latency.update(start_time, items_found); - } - EXPECT_EQ(expected_result.size(), items_found); - } - } - } - - LatencyMetric item_latency; - { - usize count = 0; - { - LatencyTimer timer{item_latency, num_keys}; - ART::Scanner scanner{index, std::string_view{}}; - while (!scanner.is_done()) { - ++count; - scanner.advance(); - } - } - EXPECT_EQ(count, num_keys); - } - - LatencyMetric item_nosync_latency; - { - usize count = 0; - { - LatencyTimer timer{item_nosync_latency, num_keys}; - ART::Scanner scanner{index, std::string_view{}}; - while (!scanner.is_done()) { - ++count; - scanner.advance(); - } - } - EXPECT_EQ(count, num_keys); - } - - std::cerr << BATT_INSPECT(scan_latency) << std::endl - << BATT_INSPECT(scanner_latency) << std::endl - << BATT_INSPECT(scanner_nosync_latency) << std::endl - << BATT_INSPECT(nokeys_scan_latency) << std::endl - << BATT_INSPECT(nosync_nokeys_scan_latency) << std::endl - << BATT_INSPECT(item_latency) << std::endl - << BATT_INSPECT(item_nosync_latency) << std::endl - << BATT_INSPECT(nokeys_item_latency) << std::endl - << BATT_INSPECT(nosync_nokeys_item_latency) << std::endl - << BATT_INSPECT(sort_latency) << std::endl - << BATT_INSPECT(ART::default_metrics().bytes_per_insert()) << std::endl; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, PutContainsTest_Key24) -{ - run_put_contains_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, PutContainsTest_Key8) -{ - char a[2] = {0, 0}; - a[0] -= 1; - char b[2] = {1, 0}; - - BATT_CHECK_LT(std::string_view{b}, std::string_view{a}); - - run_put_contains_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, WideKeySet) -{ - ARTBase::default_metrics().reset(); - - std::vector keys; - for (u16 first = 0; first < 256; ++first) { - for (u16 second = 0; second < 256; ++second) { - char data[2] = {(char)first, (char)second}; - keys.emplace_back(data, 2); - } - } - - ART index; - - for (const std::string& key : keys) { - EXPECT_FALSE(index.contains(key)); - index.insert(key); - EXPECT_TRUE(index.contains(key)); - } - - for (const std::string& key : keys) { - EXPECT_TRUE(index.contains(key)); - } -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, SingleThreadTest) -{ - ARTBase::default_metrics().reset(); - - for (const usize num_keys : {1e5, 1e6, 1e7}) { - std::vector keys; - { - std::default_random_engine rng{/*seed=*/1}; - RandomStringGenerator generate_key; - - for (usize i = 0; i < num_keys; ++i) { - keys.emplace_back(generate_key(rng)); - } - } - - LatencyMetric insert_latency; - for (usize trial = 0; trial < 3; ++trial) { - ART index; - { - LatencyTimer timer{insert_latency, num_keys}; - for (std::string_view s : keys) { - index.insert(s); - } - } - for (const std::string& key : keys) { - ASSERT_TRUE(index.contains(key)); - } - } - std::cerr << BATT_INSPECT(insert_latency) << std::endl - << BATT_INSPECT(ART::default_metrics().bytes_per_insert()) << std::endl; - } -} - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- - -usize slow_index_of(u8 key_byte, const std::array& keys) -{ - for (usize i = 0; i < keys.size(); ++i) { - if (keys[i] == key_byte) { - return i; - } - } - return 7; -} - -usize slow_index_of(u8 key_byte, const std::array& keys) -{ - for (usize i = 0; i < keys.size(); ++i) { - if (keys[i] == key_byte) { - return i; - } - } - return 31; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, SimdSearch) -{ - using turtle_kv::index_of; - - std::uniform_int_distribution pick_byte{0, 255}; - - // array case - { - // hand-written test cases - { - std::array keys = {2, 0, 3, 1}; - - EXPECT_EQ(index_of(0, keys), 1); - EXPECT_EQ(index_of(1, keys), 3); - EXPECT_EQ(index_of(2, keys), 0); - EXPECT_EQ(index_of(3, keys), 2); - EXPECT_EQ(index_of(4, keys) & 4, 4); - EXPECT_EQ(index_of(50, keys) & 4, 4); - EXPECT_EQ(index_of(99, keys) & 4, 4); - } - - // randomly generated cases - // - std::default_random_engine rng{/*seed=*/1}; - - for (usize i = 0; i < 1000; ++i) { - std::array keys; - keys.fill(0); - for (u8& k : keys) { - k = pick_byte(rng); - } - for (u16 p = 0; p < 256; ++p) { - EXPECT_EQ(index_of((u8)p, keys) & 4, slow_index_of((u8)p, keys) & 4); - } - } - } - - // array case - { - std::default_random_engine rng{/*seed=*/1}; - - for (usize i = 0; i < 100000; ++i) { - std::array keys; - keys.fill(0); - for (u8& k : keys) { - k = pick_byte(rng); - } - for (u16 p = 0; p < 256; ++p) { - EXPECT_EQ(index_of((u8)p, keys), slow_index_of((u8)p, keys)); - } - } - } -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -struct TestStringViewInserter { - const std::string_view& src; - - Status insert_new(void* dst) const - { - new (dst) std::string_view{this->src}; - return OkStatus(); - } - - Status update_existing(std::string_view* dst) const - { - *dst = this->src; - return OkStatus(); - } -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -struct TestIntInserter { - usize src; - - Status insert_new(void* dst) const - { - *((usize*)dst) = this->src; - return OkStatus(); - } - - Status update_existing(usize* dst) const - { - *dst = this->src; - return OkStatus(); - } -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void insert_key(turtle_kv::ART& art, const std::string& key) -{ - art.insert(key); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void insert_key(turtle_kv::ART& art, const std::string& key) -{ - BATT_CHECK_OK(art.insert(key, TestStringViewInserter{.src = key})); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -void insert_key(turtle_kv::ART& art, const std::string& key) -{ - BATT_CHECK_GE(key.size(), sizeof(usize)); - BATT_CHECK_OK(art.insert(key, TestIntInserter{.src = *((const usize*)(key.data()))})); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -void run_benchmark_test() -{ - ARTBase::default_metrics().reset(); - - const int n_rounds = 3; - const int n_stages_per_round = 2; // insert and query - - const std::array data_set_sizes = { - 300 * 1000, - 200 * 1000, - 100 * 1000, - }; - - std::cerr << "threads"; - for (usize n_items : data_set_sizes) { - std::cerr << ",puts/sec (N=" << n_items << "),get/sec (N=" << n_items << ")"; - } - std::cerr << std::endl; - - for (usize n_threads = 1; n_threads <= std::thread::hardware_concurrency(); ++n_threads) { - std::atomic round{-1}; - std::atomic pending{0}; - std::atomic*> p_index{nullptr}; - std::atomic p_keys{nullptr}; - std::atomic n_keys{0}; - std::vector threads; - std::vector op_count(n_threads); - std::vector>> stop_round(n_threads); - - for (auto& b : stop_round) { - b->store(false); - } - - std::cerr << n_threads; - - for (usize i = 0; i < n_threads; ++i) { - threads.emplace_back([&, i] { - std::default_random_engine rng{std::random_device{}()}; - - const int n_loops = n_rounds * (int)data_set_sizes.size() * n_stages_per_round; - - for (int r = 0; r < n_loops; ++r) { - VLOG(1) << "thread " << i << " waiting for round " << r; - while (round.load() < r) { - continue; - } - BATT_CHECK_EQ(r, round.load()); - - VLOG(1) << "thread " << i << " starting round " << r; - turtle_kv::ART& index = *p_index.load(); - const std::string* keys = p_keys.load(); - const usize n = n_keys.load(); - - if ((r % 2) == 0) { - for (usize j = i; j < n; j += n_threads) { - insert_key(index, keys[j]); - } - } else { - std::atomic& stop = *stop_round[i]; - std::uniform_int_distribution pick_i{0, n - 1}; - i64 count = 0; - while (stop.load() == false) { - const usize k = pick_i(rng); - ASSERT_TRUE(index.contains(keys[k])); - ++count; - } - op_count[i] = count; - } - - pending.fetch_sub(1); - pending.notify_one(); - VLOG(1) << "thread " << i << " finished round " << r; - } - }); - } - - auto on_scope_exit = batt::finally([&] { - for (std::thread& t : threads) { - t.join(); - } - std::cerr << std::endl; - }); - - usize size_i = 0; - for (const usize num_keys : data_set_sizes) { - //----- --- -- - - - - - // Generate random key set. - // - std::vector keys; - { - std::default_random_engine rng{/*seed=*/1}; - RandomStringGenerator generate_key; - - for (usize i = 0; i < num_keys; ++i) { - keys.emplace_back(generate_key(rng)); - } - } - - //----- --- -- - - - - - // Initialize the thread-shared state. - // - n_keys.store(num_keys); - p_keys.store(keys.data()); - auto on_scope_exit2 = batt::finally([&] { - n_keys.store(0); - p_keys.store(nullptr); - }); - - //----- --- -- - - - - - // Create a metric for each latency we want to measure. - // - LatencyMetric insert_latency; - LatencyMetric st_query_latency; - LatencyMetric mt_query_latency; - - //----- --- -- - - - - - // Run the benchmark repeatedly `n_rounds` times. - // - for (int r = 0; r < n_rounds; ++r) { - turtle_kv::ART index; - - // Set `pending` and `p_index`; the threads will not start working until `round` is updated. - // - pending.store(n_threads); - p_index.store(&index); - auto on_scope_exit3 = batt::finally([&] { - p_index.store(nullptr); - }); - - // Stage 0: inserts - { - LatencyTimer timer{insert_latency, num_keys}; - - const int next_round = (r + size_i * n_rounds) * 2; - - //----- --- -- - - - - - // Ready, set, go! - // - round.store(next_round); - //----- --- -- - - - - - - for (;;) { - const auto observed = pending.load(); - if (observed > 0) { - pending.wait(observed); - continue; - } - break; - } - } - - pending.store(n_threads); - for (auto& b : stop_round) { - b->store(false); - } - - // Stage 1: queries - { - const auto start_time = std::chrono::steady_clock::now(); - - const int next_round = (r + size_i * n_rounds) * 2 + 1; - - //----- --- -- - - - - - // Ready, set, go! - // - round.store(next_round); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - for (auto& b : stop_round) { - b->store(true); - } - //----- --- -- - - - - - - for (;;) { - const auto observed = pending.load(); - if (observed > 0) { - pending.wait(observed); - continue; - } - break; - } - - const auto end_time = std::chrono::steady_clock::now(); - - const double elapsed_nanos = - std::chrono::duration_cast(end_time - start_time).count(); - - double total_count = 0; - for (double c : op_count) { - total_count += c; - } - BATT_CHECK_GT(total_count, 0); - BATT_CHECK_GT(elapsed_nanos, 0); - - mt_query_latency.update_nanos(elapsed_nanos, total_count); - } - - //----- --- -- - - - - - // After multi-threaded part of this round is done. - // - usize found_count = 0; - auto start_time = std::chrono::steady_clock::now(); - for (const std::string& key : keys) { - ASSERT_TRUE(index.contains(key)) << BATT_INSPECT_STR(key) << BATT_INSPECT(found_count); - ++found_count; - } - st_query_latency.update(start_time, found_count); - ASSERT_EQ(found_count, keys.size()); - } - std::cerr << "," << insert_latency.rate_per_second(false) << "," - << mt_query_latency.rate_per_second(false); - ++size_i; - } - - if (n_threads >= 4) { - n_threads += 1; - if (n_threads >= 8) { - n_threads += 2; - if (n_threads >= 16) { - n_threads += 4; - } - } - } - } - - std::cerr << BATT_INSPECT(turtle_kv::ART::default_metrics().bytes_per_insert()) - << std::endl; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, MultiThreadTest_Void) -{ - run_benchmark_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, MultiThreadTest_StringView) -{ - run_benchmark_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, MultiThreadTest_Int) -{ - run_benchmark_test(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(ArtTest, ValuePutGetScan) -{ - ARTBase::default_metrics().reset(); - - const usize n_keys = 100 * 1000; - std::default_random_engine rng{std::random_device{}()}; - RandomStringGenerator generate_key; - - std::vector keys; - keys.resize(n_keys); - - std::unordered_set unique_keys; - - for (std::string& k : keys) { - for (;;) { - k = generate_key(rng); - if (!unique_keys.count(k)) { - unique_keys.emplace(k); - break; - } - } - } - - turtle_kv::ART art; - - const auto check_key_by_index = [&art, &keys](usize query_i, usize expected_i) { - ASSERT_TRUE(art.contains(keys[query_i])); - - const std::string_view* p_value = art.unsynchronized_find(keys[query_i]); - ASSERT_NE(p_value, nullptr); - ASSERT_THAT(*p_value, ::testing::StrEq(keys[expected_i])); - - Optional value_copy = art.find(keys[query_i]); - ASSERT_TRUE(value_copy); - ASSERT_THAT(*value_copy, ::testing::StrEq(keys[expected_i])); - }; - - for (usize i = 0; i < n_keys; ++i) { - Status status = art.insert(keys[i], - TestStringViewInserter{ - .src = keys[i / 2], - }); - - ASSERT_TRUE(status.ok()) << BATT_INSPECT(status); - - for (usize j = i - std::min(i, 25); j <= i; ++j) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(j, j / 2)); - } - } - - // Check all values once more to make sure nothing that was inserted earlier was messed up by a - // later insertion or update. - // - for (usize i = 0; i < n_keys; ++i) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(i, i / 2)); - } - - // Now update all keys and verify them. - // - for (usize i = 0; i < n_keys; ++i) { - Status status = art.insert(keys[i], - TestStringViewInserter{ - .src = keys[i], - }); - - ASSERT_TRUE(status.ok()) << BATT_INSPECT(status); - - for (usize j = i - std::min(i, 25); j <= i; ++j) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(j, j)); - } - } - - for (usize i = 0; i < n_keys; ++i) { - ASSERT_NO_FATAL_FAILURE(check_key_by_index(i, i)); - } -} - -} // namespace diff --git a/src/turtle_kv/util/packed_piecewise_filter_view.hpp b/src/turtle_kv/util/packed_piecewise_filter_view.hpp new file mode 100644 index 0000000..990088f --- /dev/null +++ b/src/turtle_kv/util/packed_piecewise_filter_view.hpp @@ -0,0 +1,306 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PACKED_PIECEWISE_FILTER_VIEW_HPP + +#include "piecewise_filter_storage_model.concept.hpp" + +#include +#include +#include + +#include +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Read-only model of PiecewiseFilterStorageModel for packed filters. + * + * Packed piecewise filters are represented as an array of integers, which are the boundaries + * between live and dropped intervals, plus an additional boolean/bit denoting whether the interval + * from the global minimum to the first stored boundary is live or dropped (`start_is_live`). + * + * The global minimum and maximum bounds are never stored in the packed representation. Instead, + * the minimum is implied via the `start_is_live` bit, and the maximum by whether the number of + * stored bounds (plus the implicit first bound, if start_is_live == true) is even or odd. If it is + * odd, then it is implied that there is a final bound equal to the global maximum. + * + * Examples: + * + * Live Intervals: {[0, 10), [20, 30), [40, 50)} + * Packed: start_is_live=1, {10, 20, 30, 40, 50} + * + * Live Intervals: {[0, 10), [20, 30), [40, +inf)} + * Packed: start_is_live=1, {10, 20, 30, 40} + * + * Live Intervals: {[10, 20), [30, 40), [50, 60)} + * Packed: start_is_live=0, {10, 20, 30, 40, 50, 60} + * + * Live Intervals: {[10, 20), [30, 40), [50, +inf)} + * Packed: start_is_live=0, {10, 20, 30, 40, 50} + */ +class PackedPiecewiseFilterStorage +{ + public: + //----- --- -- - - - - + + // Forward-declaration; the type returned by this->begin(), this->end() + // + class const_iterator; + + /** \brief The boundary integer type. Must be unsigned. + */ + using OffsetT = const little_u32; + + /** \brief The live range type; what `iterator` iterates over. + */ + using value_type = Interval; + + /** \brief Non-const iterator aliases const_iterator, since this storage model is read-only. + */ + using iterator = const_iterator; + + // Forward-declaration; defined below. + // + friend const Slice& as_const_slice(const PackedPiecewiseFilterStorage& view); + + //----- --- -- - - - - + + /** \brief Constructs an PackedPiecewiseFilterStorage representing the live interval [0, +inf). + */ + PackedPiecewiseFilterStorage() = default; + + /** \brief Destructs the PackedPiecewiseFilterStorage. + */ + ~PackedPiecewiseFilterStorage() = default; + + /** \brief PackedPiecewiseFilterStorage is copy constructible. + */ + PackedPiecewiseFilterStorage(const PackedPiecewiseFilterStorage&) = default; + + /** \brief PackedPiecewiseFilterStorage is copy assignable. + */ + PackedPiecewiseFilterStorage& operator=(const PackedPiecewiseFilterStorage&) = default; + + /** \brief Constructs PackedPiecewiseFilterStorage from the packed data in the arguments. + * + * See the class-level description for details on what `values` and `start_is_live` represent. + */ + explicit PackedPiecewiseFilterStorage(const Slice& values, + bool start_is_live) noexcept + : values_{values} + , implicit_first_{start_is_live ? 1 : 0} + , size_{(this->implicit_first_ + BATT_CHECKED_CAST(i32, this->values_.size()) + 1) / 2} + { + } + + //----- --- -- - - - - + + /** \brief Returns an iterator to the first live interval in this filter. + */ + const_iterator begin() const noexcept; + + /** \brief Returns an iterator to one past the last live interval in this filter. + */ + const_iterator end() const noexcept; + + /** \brief Returns the number of live intervals in the filter; same as + * `std::distance(this->begin(), this->end())`. + */ + usize size() const noexcept; + + /** \brief Returns true iff `this->size() == 0`. + */ + bool empty() const noexcept; + + /** \brief Returns the `i`-th live interval in the filter. Behavior is undefined if `i` is not + * less than `this->size()`. + */ + Interval operator[](isize i) const noexcept; + + //----- --- -- - - - - + private: + /** \brief Points at the stored boundaries, as described in the class-level doc. + */ + Slice values_; + + /** \brief Set to 1 if `start_is_live`, else 0. + */ + i32 implicit_first_ = 1; + + /** \brief The number of live intervals in the filter. + */ + i32 size_ = 1; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Returns a const reference to the stored values referenced by `view`. + */ +inline const Slice& as_const_slice(const PackedPiecewiseFilterStorage& view) +{ + return view.values_; +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Read-only, random access iterator over the live intervals of a packed piecewise filter. + */ +class PackedPiecewiseFilterStorage::const_iterator + : public boost::iterator_facade< // + PackedPiecewiseFilterStorage::const_iterator, // <- Derived + Interval, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + Interval, // <- Reference + isize // <- Difference + > +{ + public: + using Self = const_iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = Interval; + using reference = Interval; + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Constructs an invalid iterator. + */ + const_iterator() noexcept : view_{nullptr}, pos_{0} + { + } + + /** \brief Constructs an iterator to the `pos`-th live interval of `view`. + * + * `view` must remain in-scope while this object exists. + */ + const_iterator(const PackedPiecewiseFilterStorage* view, isize pos) noexcept : view_{view}, pos_{pos} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Returns the live interval at the current position. + */ + reference dereference() const + { + return (*this->view_)[this->pos_]; + } + + /** \brief Returns true iff this iterator points to the same live interval of the same filter as + * `other`. + */ + bool equal(const Self& other) const + { + return this->view_ == other.view_ && this->pos_ == other.pos_; + } + + /** \brief Moves this iterator forward by one. + */ + void increment() + { + ++this->pos_; + } + + /** \brief Moves this iterator backward by one. + */ + void decrement() + { + --this->pos_; + } + + /** \brief Moves this iterator by `delta`. + */ + void advance(isize delta) + { + this->pos_ += delta; + } + + /** \brief Returns the number of steps required to advance this iterator so it is equivalent to + * `other`. Will panic if this and other do not point at the same filter view. + */ + isize distance_to(const Self& other) const + { + BATT_CHECK_EQ(this->view_, other.view_); + return other.pos_ - this->pos_; + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + /** \brief Pointer to the filter view over which we are iterating. + */ + const PackedPiecewiseFilterStorage* view_; + + /** \brief The (logical) position of this iterator within `view_`. + */ + isize pos_; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline auto PackedPiecewiseFilterStorage::begin() const noexcept -> const_iterator +{ + return const_iterator{this, 0}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline auto PackedPiecewiseFilterStorage::end() const noexcept -> const_iterator +{ + return const_iterator{this, static_cast(this->size())}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline usize PackedPiecewiseFilterStorage::size() const noexcept +{ + return this->size_; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline bool PackedPiecewiseFilterStorage::empty() const noexcept +{ + return this->size_ == 0; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline Interval PackedPiecewiseFilterStorage::operator[](isize i) const noexcept +{ + // Cached for brevity below. + // + const isize n = this->values_.size(); + + // The index within this->values_ of the i-th live interval's lower bound. + // May be negative if the first boundary is implicit (0) + // + const isize j0 = i * 2 - this->implicit_first_; + + // The index within this->values_ of the i-th live interval's upper bound. + // May be past the end of this->values_ if the last boundary is implicit (+inf) + // + const isize j1 = j0 + 1; + + const u32 lower_bound = (j0 < 0) ? std::numeric_limits::min() : this->values_[j0].value(); + const u32 upper_bound = (j1 < n) ? this->values_[j1].value() : std::numeric_limits::max(); + + return Interval{lower_bound, upper_bound}; +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- + +static_assert(PiecewiseFilterStorageModel); + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.hpp b/src/turtle_kv/util/piecewise_filter.hpp index f524319..9f22ceb 100644 --- a/src/turtle_kv/util/piecewise_filter.hpp +++ b/src/turtle_kv/util/piecewise_filter.hpp @@ -9,6 +9,9 @@ #pragma once #define TURTLE_KV_UTIL_PIECEWISE_FILTER_HPP +#include "packed_piecewise_filter_view.hpp" +#include "piecewise_filter_storage_model.concept.hpp" + #include #include #include @@ -17,6 +20,8 @@ #include #include +#include + #include #include @@ -26,11 +31,15 @@ namespace turtle_kv { /** \brief A representation of a filtered range of items. */ -template -class PiecewiseFilter +template ModelT> +class BasicPiecewiseFilter : private ModelT { public: - using Self = PiecewiseFilter; + using Self = BasicPiecewiseFilter; + + using ConstIterator = typename ModelT::const_iterator; + + class LiveSubranges; static_assert(std::is_integral::value && std::is_unsigned::value, "Offset must be an unsigned integer type!"); @@ -46,14 +55,27 @@ class PiecewiseFilter /** \brief Creates and returns a PiecewiseFilter instance from a range of intervals that contain * the live item indexes. */ - static StatusOr from_live(const Slice>& live); + static StatusOr from_live(const Slice>& live) + requires PiecewiseFilterMutableStorageModel; //+++++++++++-+-+--+----- --- -- - - - - /** \brief Constructs a default instance of a PiecewiseFilter object, initialized with no item * range and filtered items. */ - PiecewiseFilter() noexcept; + BasicPiecewiseFilter() noexcept + requires PiecewiseFilterMutableStorageModel; + + /** \brief Constructs a BasicPiecewiseFilter directly from a storage model instance. + */ + explicit BasicPiecewiseFilter(const ModelT& model) noexcept; + + /** \brief Constructs a BasicPiecewiseFilter by copying live intervals from a filter with a + * different storage model. + */ + template OtherModelT> + explicit BasicPiecewiseFilter(const BasicPiecewiseFilter& other) + requires PiecewiseFilterMutableStorageModel; //+++++++++++-+-+--+----- --- -- - - - - @@ -63,7 +85,8 @@ class PiecewiseFilter * * \return The new dropped interval that coincides with `i`. */ - Interval drop_index_range(Interval i); + Interval drop_index_range(Interval i) + requires PiecewiseFilterMutableStorageModel; /** \brief Returns whether or not the item at index `i` has been filtered out. * @@ -95,11 +118,34 @@ class PiecewiseFilter /** \brief Returns a view of the live item intervals. */ - Slice> live() const; + Slice> live() const + requires PiecewiseFilterMutableStorageModel; /** \brief Merges two filters in place, taking the union of the live intervals. */ - void merge(const PiecewiseFilter& other); + void merge(const Self& other) + requires PiecewiseFilterMutableStorageModel; + + /** \brief Returns a seq of Interval that is the intersection of `i` and the live ranges + * of this filter. + */ + LiveSubranges live_subranges_of(Interval i) const; + + /** \brief Returns an iterator to the first live interval. + */ + ConstIterator begin() const; + + /** \brief Returns an iterator past the last live interval. + */ + ConstIterator end() const; + + /** \brief Returns the number of live intervals. + */ + usize size() const; + + /** \brief Returns true iff there are no live intervals. + */ + bool empty() const; /** \brief Validate the state of the live intervals. */ @@ -109,11 +155,24 @@ class PiecewiseFilter //+++++++++++-+-+--+----- --- -- - - - - private: - /** \brief The range of filtered out item indexes. - */ - SmallVec, 64> live_; + BATT_ALWAYS_INLINE ModelT& live_() noexcept + { + return *this; + } + + BATT_ALWAYS_INLINE const ModelT& live_() const noexcept + { + return *this; + } }; +template +using PiecewiseFilter = BasicPiecewiseFilter, 64>>; + +using PackedPiecewiseFilter = BasicPiecewiseFilter; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// template inline Interval drop_item_range(PiecewiseFilter& filter, const Slice& items, @@ -127,6 +186,5 @@ inline Interval drop_item_range(PiecewiseFilter& filter, return filter.drop_index_range(Interval{start_i, end_i}); } -} // namespace turtle_kv -#include +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.ipp b/src/turtle_kv/util/piecewise_filter.ipp index 3456043..3848365 100644 --- a/src/turtle_kv/util/piecewise_filter.ipp +++ b/src/turtle_kv/util/piecewise_filter.ipp @@ -11,20 +11,19 @@ #include "piecewise_filter.hpp" -#include - namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -/*static*/ StatusOr> PiecewiseFilter::from_live( - const Slice>& live) +template ModelT> +/*static*/ StatusOr> +BasicPiecewiseFilter::from_live(const Slice>& live) + requires PiecewiseFilterMutableStorageModel { - PiecewiseFilter filter; - filter.live_.clear(); + Self filter; - filter.live_.insert(filter.live_.end(), live.begin(), live.end()); + filter.live_().clear(); + filter.live_().insert(filter.live_().end(), live.begin(), live.end()); if (!filter.check_invariants()) { return Status{::batt::StatusCode::kInvalidArgument}; @@ -35,16 +34,41 @@ template //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -PiecewiseFilter::PiecewiseFilter() noexcept - : live_{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} +template ModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter() noexcept + requires PiecewiseFilterMutableStorageModel + : ModelT{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} +{ +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter(const ModelT& model) noexcept + : ModelT{model} +{ +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +template OtherModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter( + const BasicPiecewiseFilter& other) + requires PiecewiseFilterMutableStorageModel + : ModelT{} { + this->live_().clear(); + + for (auto iter = other.begin(); iter != other.end(); ++iter) { + this->live_().insert(this->live_().end(), *iter); + } } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -bool PiecewiseFilter::check_invariants() const +template ModelT> +bool BasicPiecewiseFilter::check_invariants() const { Optional prev_upper_bound = None; @@ -52,7 +76,7 @@ bool PiecewiseFilter::check_invariants() const // - all intervals are in non-decreasing order // - no intervals overlap or are adjacent (i.e., prev.upper_bound == next.lower_bound) // - for (const Interval& range : this->live_) { + for (const Interval& range : this->live_()) { // If a range has the minimum lower bound, it must be the first. // if (range.lower_bound == Self::kMinLowerBound && prev_upper_bound) { @@ -80,15 +104,16 @@ bool PiecewiseFilter::check_invariants() const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Interval PiecewiseFilter::drop_index_range(Interval to_drop) +template ModelT> +Interval BasicPiecewiseFilter::drop_index_range(Interval to_drop) + requires PiecewiseFilterMutableStorageModel { if (to_drop.empty()) { return to_drop; } - auto [first, last] = std::equal_range(this->live_.begin(), - this->live_.end(), + auto [first, last] = std::equal_range(this->live_().begin(), + this->live_().end(), to_drop, typename Interval::LinearOrder{}); @@ -98,7 +123,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // the return value bounds correctly. // if (first == last || to_drop.lower_bound < first->lower_bound) { - if (first != this->live_.begin()) { + if (first != this->live_().begin()) { // We are starting in a live interval gap (dropped region), so we extend to the previous live // interval's upper bound. // @@ -109,7 +134,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t } if (first == last || to_drop.upper_bound >= std::prev(last)->upper_bound) { - if (last != this->live_.end()) { + if (last != this->live_().end()) { // We are ending in a live interval gap, so we extend to the next live interval's start. // dropped.upper_bound = last->lower_bound; @@ -164,7 +189,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // Process all overlapping intervals with `to_drop`. // - while (first != this->live_.end()) { + while (first != this->live_().end()) { if (first->lower_bound >= to_drop.upper_bound) { // Interval is entirely after `to_drop`, so there is nothing left to process. // @@ -177,7 +202,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // Interval right_half{to_drop.upper_bound, first->upper_bound}; first->upper_bound = to_drop.lower_bound; - this->live_.insert(std::next(first), right_half); + this->live_().insert(std::next(first), right_half); return dropped; } else { // Cases 2b and 4. @@ -193,7 +218,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t } else { // Case 1. // - first = this->live_.erase(first); + first = this->live_().erase(first); } } @@ -202,41 +227,74 @@ Interval PiecewiseFilter::drop_index_range(Interval t //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Slice> PiecewiseFilter::live() const +template ModelT> +Slice> BasicPiecewiseFilter::live() const + requires PiecewiseFilterMutableStorageModel +{ + return as_const_slice(this->live_()); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::begin() const -> ConstIterator +{ + return this->live_().begin(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::end() const -> ConstIterator +{ + return this->live_().end(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +usize BasicPiecewiseFilter::size() const +{ + return this->live_().size(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +bool BasicPiecewiseFilter::empty() const { - return as_const_slice(this->live_); + return this->live_().empty(); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -bool PiecewiseFilter::live_at_index(OffsetT i) const +template ModelT> +bool BasicPiecewiseFilter::live_at_index(OffsetT i) const { return this->live_lower_bound(i) == i; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -OffsetT PiecewiseFilter::live_lower_bound(OffsetT i) const +template ModelT> +OffsetT BasicPiecewiseFilter::live_lower_bound(OffsetT i) const { // Compute the live interval which could contain `i`. // - auto iter = std::lower_bound(this->live_.begin(), - this->live_.end(), + auto iter = std::lower_bound(this->live_().begin(), + this->live_().end(), i, typename Interval::LinearOrder{}); // Check if current interval contains `i`. // - if (iter != this->live_.end() && iter->contains(i)) { + if (iter != this->live_().end() && iter->contains(i)) { return i; } // `i` is in a dropped range, so we return the start of the next live interval. // - if (iter != this->live_.end()) { + if (iter != this->live_().end()) { return iter->lower_bound; } @@ -247,22 +305,22 @@ OffsetT PiecewiseFilter::live_lower_bound(OffsetT i) const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Interval PiecewiseFilter::find_live_range(Interval i) const +template ModelT> +Interval BasicPiecewiseFilter::find_live_range(Interval i) const { OffsetT start_i = i.lower_bound; OffsetT end_i = i.upper_bound; BATT_CHECK_LE(start_i, end_i); - auto iter = std::lower_bound(this->live_.begin(), - this->live_.end(), + auto iter = std::lower_bound(this->live_().begin(), + this->live_().end(), start_i, typename Interval::LinearOrder{}); // Check if current interval contains or starts at `start_i`. // - if (iter != this->live_.end()) { + if (iter != this->live_().end()) { if (iter->contains(start_i)) { OffsetT live_end = std::min(end_i, iter->upper_bound); return Interval{start_i, live_end}; @@ -283,25 +341,26 @@ Interval PiecewiseFilter::find_live_range(Interval i) //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -void PiecewiseFilter::merge(const PiecewiseFilter& other) +template ModelT> +void BasicPiecewiseFilter::merge(const Self& other) + requires PiecewiseFilterMutableStorageModel { // If other has no live intervals, we are done. // - if (other.live_.empty()) { + if (other.live_().empty()) { return; } // If this has no live intervals, copy from other. // - if (this->live_.empty()) { - this->live_.insert(this->live_.end(), other.live_.begin(), other.live_.end()); + if (this->live_().empty()) { + this->live_().insert(this->live_().end(), other.live_().begin(), other.live_().end()); BATT_CHECK(this->check_invariants()); return; } SmallVec, 64> merged_intervals; - merged_intervals.reserve(this->live_.size() + other.live_.size()); + merged_intervals.reserve(this->live_().size() + other.live_().size()); usize i = 0; usize j = 0; @@ -329,42 +388,57 @@ void PiecewiseFilter::merge(const PiecewiseFilter& other) // Merge the live intervals arrays. // - while (i < this->live_.size() && j < other.live_.size()) { - if (this->live_[i].lower_bound <= other.live_[j].lower_bound) { - add_interval(this->live_[i]); + while (i < this->live_().size() && j < other.live_().size()) { + if (this->live_()[i].lower_bound <= other.live_()[j].lower_bound) { + add_interval(this->live_()[i]); ++i; } else { - add_interval(other.live_[j]); + add_interval(other.live_()[j]); ++j; } } - // Add remaining intervals from this->live_. + // Add remaining intervals.. // - while (i < this->live_.size()) { - add_interval(this->live_[i]); + while (i < this->live_().size()) { + add_interval(this->live_()[i]); ++i; } // Add remaining intervals from other.live_. // - while (j < other.live_.size()) { - add_interval(other.live_[j]); + while (j < other.live_().size()) { + add_interval(other.live_()[j]); ++j; } - this->live_ = std::move(merged_intervals); + this->live_() = std::move(merged_intervals); BATT_CHECK(this->check_invariants()); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -SmallFn PiecewiseFilter::dump() const +template ModelT> +SmallFn BasicPiecewiseFilter::dump() const { return [this](std::ostream& out) { - out << batt::dump_range(this->live_); + out << batt::dump_range(this->live_()); }; } + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::live_subranges_of(Interval query_range) const + -> LiveSubranges +{ + const auto [first, last] = std::equal_range(this->live_().begin(), + this->live_().end(), + query_range, + typename Interval::LinearOrder{}); + + return LiveSubranges{query_range, std::ranges::subrange(first, last)}; +} + } // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.live_subranges.hpp b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp new file mode 100644 index 0000000..a22a0b1 --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp @@ -0,0 +1,88 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_LIVE_SUBRANGES_HPP + +#include "piecewise_filter.hpp" + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief A (batt) Seq over the sub-ranges of a PiecewiseFilter which match some interval (the + * `query_range`). + */ +template ModelT> +class BasicPiecewiseFilter::LiveSubranges +{ + public: + using Iterator = typename BasicPiecewiseFilter::ConstIterator; + using Item = Interval; + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Constructs a LiveSubranges seq containing the passed range of live intervals (`match`), + * with the first and last element clamped to the `query_range`. + * + * `match` *must* not extend more than one live interval past `query_range` at the front or back. + */ + explicit LiveSubranges(Interval query_range, + std::ranges::subrange match) noexcept + : clamp_lower_{query_range.lower_bound} + , clamp_upper_{query_range.upper_bound} + , match_{match} + { + } + + /** \brief Returns the current live subrange, or None if the seq has been fully consumed. + */ + Optional peek() + { + if (this->match_.empty()) { + return None; + } + Interval item = this->match_.front(); + if (this->clamp_lower_) { + item.lower_bound = std::max(item.lower_bound, *this->clamp_lower_); + } + if (this->match_.size() == 1 && this->clamp_upper_) { + item.upper_bound = std::min(item.upper_bound, *this->clamp_upper_); + } + return item; + } + + /** \brief Returns the current live subrange, or None if the seq has been fully consumed, + * consuming the returned item. + */ + Optional next() + { + Optional item = this->peek(); + if (item) { + this->clamp_lower_ = None; + this->match_.advance(1); + } + return item; + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + /** \brief The lower bound to which to clamp the first interval of `this->match_`. + */ + Optional clamp_lower_; + + /** \brief The upper bound to which to clamp the last interval of `this->match_`. + */ + Optional clamp_upper_; + + /** \brief The current subrange of the live intervals in the filter. + */ + std::ranges::subrange match_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index d4c6ee4..7aa134e 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -13,8 +13,18 @@ #include #include +#include +#include +#include +#include + #include +#include +#include +#include +#include + #include #include #include @@ -33,14 +43,26 @@ using turtle_kv::PiecewiseFilter; using turtle_kv::Slice; using turtle_kv::Status; using turtle_kv::StatusOr; +using turtle_kv::testing::build_filter_with_random_drops; +using turtle_kv::testing::drop_n_disjoint_intervals_from; +using turtle_kv::testing::get_packed_filter_from_data; +using turtle_kv::testing::PackedFilterData; +using turtle_kv::testing::pack_in_memory_filter; +using turtle_kv::testing::RandomDropResult; using turtle_kv::testing::RandomStringGenerator; +using turtle_kv::testing::verify_filter_queries; using turtle_kv::drop_item_range; using llfs::KeyRangeOrder; +using batt::mask_from_interval; using batt::StableStringStore; +using turtle_kv::PackedPiecewiseFilter; +using turtle_kv::PackedPiecewiseFilterStorage; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // TEST(PiecewiseFilterTest, InvalidFilterTest) @@ -73,89 +95,68 @@ TEST(PiecewiseFilterTest, InvalidFilterTest) // TEST(PiecewiseFilterTest, QueryTest) { - const usize num_items = 10000; + const u32 num_items = 10000; - for (usize seed = 0; seed < 100; ++seed) { + for (u32 seed = 0; seed < 100; ++seed) { std::default_random_engine rng{seed}; - PiecewiseFilter filter; - EXPECT_TRUE(filter.check_invariants()); + auto [filter, live_items] = build_filter_with_random_drops(num_items, rng); - // All items start live. - // - std::set live_items; - for (usize i = 0; i < num_items; ++i) { - live_items.insert(i); - } + EXPECT_TRUE(filter.check_invariants()); - // Drop random intervals. - // - std::uniform_int_distribution pick_num_dropped{100, num_items / 2}; - usize num_intervals_dropped = pick_num_dropped(rng); - for (usize i = 0; i < num_intervals_dropped; ++i) { - std::uniform_int_distribution pick_interval_start{0, num_items - 1}; - usize start_i = pick_interval_start(rng); + verify_filter_queries(filter, live_items, num_items, seed, rng); + } +} - std::uniform_int_distribution pick_interval_end{start_i, num_items}; - usize end_i = pick_interval_end(rng); +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(PiecewiseFilterTest, PackedQueryTest) +{ + const u32 num_items = 10000; - for (usize j = start_i; j < end_i; ++j) { - live_items.erase(j); - } + for (u32 seed = 0; seed < 100; ++seed) { + std::default_random_engine rng{seed}; - Interval new_dropped = filter.drop_index_range(Interval{start_i, end_i}); - EXPECT_LE(new_dropped.lower_bound, start_i) << BATT_INSPECT(seed); - EXPECT_GE(new_dropped.upper_bound, end_i) << BATT_INSPECT(seed); - } + auto [filter, live_items] = build_filter_with_random_drops(num_items, rng); EXPECT_TRUE(filter.check_invariants()); - // Test live_at_index + // Pack the filter and construct a PackedPiecewiseFilter. // - for (usize i = 0; i < num_items; ++i) { - bool expected_live = live_items.count(i) > 0; - bool actual_live = filter.live_at_index(i); - EXPECT_EQ(actual_live, expected_live) << BATT_INSPECT(seed) << BATT_INSPECT(i); - } + PackedFilterData packed_data = pack_in_memory_filter(filter); + PackedPiecewiseFilter packed_filter = get_packed_filter_from_data(packed_data); - // Test live_lower_bound + // Verify the packed filter has the same number of live intervals. // - for (usize i = 0; i < num_items; ++i) { - auto iter = live_items.lower_bound(i); - usize expected = (iter != live_items.end()) ? *iter : num_items; - usize actual = filter.live_lower_bound(i); - EXPECT_EQ(actual, expected) << BATT_INSPECT(seed) << BATT_INSPECT(i); - } + EXPECT_EQ(packed_filter.size(), filter.size()) << BATT_INSPECT(seed); - // Test find_live_range + // Verify the packed filter produces identical intervals. // - for (usize i = 0; i < 100; ++i) { - std::uniform_int_distribution pick_interval_start{0, num_items - 1}; - usize start_i = pick_interval_start(rng); - - std::uniform_int_distribution pick_interval_end{start_i, num_items}; - usize end_i = pick_interval_end(rng); - - auto iter = live_items.lower_bound(start_i); - Interval expected_range; - - if (iter == live_items.end() || *iter >= end_i) { - expected_range = Interval{end_i, end_i}; - } else { - usize first = *iter; - usize last = first + 1; - auto next = std::next(iter); - - while (next != live_items.end() && *next < end_i && *next == last) { - ++last; - ++next; - } - - expected_range = Interval{first, last}; + { + auto mutable_iter = filter.begin(); + auto packed_iter = packed_filter.begin(); + while (mutable_iter != filter.end() && packed_iter != packed_filter.end()) { + EXPECT_EQ(*mutable_iter, *packed_iter) << BATT_INSPECT(seed); + ++mutable_iter; + ++packed_iter; } + EXPECT_EQ(mutable_iter, filter.end()) << BATT_INSPECT(seed); + EXPECT_EQ(packed_iter, packed_filter.end()) << BATT_INSPECT(seed); + } - Interval actual_range = filter.find_live_range(Interval{start_i, end_i}); - EXPECT_EQ(actual_range, expected_range) << BATT_INSPECT(seed); + // Verify queries produce the same results as the in-memory filter. + // + verify_filter_queries(packed_filter, live_items, num_items, seed, rng); + + // Converting packed back to in-memory should produce identical filter. + // + PiecewiseFilter converted_filter{packed_filter}; + EXPECT_TRUE(converted_filter.check_invariants()); + Slice> original_live = filter.live(); + Slice> converted_live = converted_filter.live(); + ASSERT_EQ(original_live.size(), converted_live.size()) << BATT_INSPECT(seed); + for (usize i = 0; i < original_live.size(); ++i) { + EXPECT_EQ(original_live[i], converted_live[i]) << BATT_INSPECT(seed) << BATT_INSPECT(i); } } } @@ -284,4 +285,88 @@ TEST(PiecewiseFilterTest, KeyQueryTest) EXPECT_TRUE(filter.check_invariants()); } } -} // namespace \ No newline at end of file + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(PiecewiseFilterTest, LiveSubranges) +{ + std::uniform_int_distribution pick_bound{0, 64}; + + const auto pick_interval = [&](auto& rng) { + Interval i{pick_bound(rng), pick_bound(rng)}; + if (i.upper_bound < i.lower_bound) { + std::swap(i.lower_bound, i.upper_bound); + } + return i; + }; + + std::array, 1> init_live{{{0, 64}}}; + + const usize n_seeds = 100000; + const usize n_drops = 32; + const usize n_queries = 15; + const usize first_seed = 0; + + PiecewiseFilter filter; + + const auto query_as_bits = [&](Interval query) { + u64 bits = 0; + filter.live_subranges_of(query) | batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + + for (usize seed_i = first_seed; seed_i < first_seed + n_seeds; ++seed_i) { + std::default_random_engine rng{seed_i}; + + for (usize i = 0; i < n_drops; ++i) { + BATT_DEBUG_INFO(BATT_INSPECT(i) << BATT_INSPECT(seed_i)); + + filter = BATT_OK_RESULT_OR_PANIC(PiecewiseFilter::from_live(batt::as_slice(init_live))); + u64 filter_state = ~u64{0}; + + std::vector> dropped_ranges = + drop_n_disjoint_intervals_from(&filter, i, init_live[0], rng).second; + + for (const Interval& drop_interval : dropped_ranges) { + const u64 drop_mask = mask_from_interval(drop_interval); + filter_state &= ~drop_mask; + } + + // Also test via PackedPiecewiseFilter. + // + PackedFilterData packed_data = pack_in_memory_filter(filter); + PackedPiecewiseFilter packed_filter = get_packed_filter_from_data(packed_data); + + const auto packed_query_as_bits = [&](Interval query) { + u64 bits = 0; + packed_filter.live_subranges_of(query) | + batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + + for (usize j = 0; j < n_queries; ++j) { + const Interval query_interval = pick_interval(rng); + const u64 query_mask = mask_from_interval(query_interval); + const u64 expected_bits = query_mask & filter_state; + const u64 actual_bits = query_as_bits(query_interval); + + ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{actual_bits}) + << BATT_INSPECT(seed_i) << BATT_INSPECT(query_interval) + << BATT_INSPECT(query_interval.size()) << BATT_INSPECT(std::bitset<64>{query_mask}); + + const u64 packed_actual_bits = packed_query_as_bits(query_interval); + + ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{packed_actual_bits}) + << "PackedPiecewiseFilter mismatch: " << BATT_INSPECT(seed_i) + << BATT_INSPECT(query_interval) << BATT_INSPECT(query_interval.size()) + << BATT_INSPECT(std::bitset<64>{query_mask}); + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/util/piecewise_filter.test.hpp b/src/turtle_kv/util/piecewise_filter.test.hpp new file mode 100644 index 0000000..f4058e5 --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter.test.hpp @@ -0,0 +1,247 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_TEST_HPP + +#include "piecewise_filter.hpp" +#include "piecewise_filter_storage_model.concept.hpp" + +#include + +#include +#include + +#include +#include + +namespace turtle_kv { +namespace testing { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Randomly drops `n` ranges within the specified live range of the passed filter. + * + * Requires that: + * - `drop_within` must be live in `filter` + * - `drop_within.size()` must be large enough to fit `n` disjoint intervals + * + * \return a pair of { total offset size dropped, vector of the dropped intervals } + */ +template ModelT> +inline std::pair>> drop_n_disjoint_intervals_from( + BasicPiecewiseFilter* filter, + usize n, + const Interval& drop_within, + Rng& rng) +{ + constexpr bool debug = false; + + if constexpr (debug) { + std::cerr << BATT_INSPECT(n) << std::endl; + } + + OffsetT dropped_total_size = 0; + std::vector> dropped_ranges; + + if (n == 0) { + return std::make_pair(dropped_total_size, dropped_ranges); + } + + BATT_CHECK_LE(n * 2 - 1, drop_within.size()); + BATT_CHECK_EQ(filter->live().empty(), false); + BATT_CHECK_EQ(filter->find_live_range(drop_within), drop_within); + + usize drops_remaining = n; + OffsetT next_droppable = drop_within.lower_bound; + const OffsetT live_lower_bound = filter->live().front().lower_bound; + const OffsetT live_upper_bound = filter->live().back().upper_bound; + + BATT_DEBUG_INFO(BATT_INSPECT(dropped_total_size) + << BATT_INSPECT_RANGE(dropped_ranges) << BATT_INSPECT(drop_within) + << BATT_INSPECT(n) << BATT_INSPECT(drops_remaining) + << BATT_INSPECT(next_droppable) << BATT_INSPECT(live_lower_bound) + << BATT_INSPECT(live_upper_bound)); + + if constexpr (debug) { + std::cerr << BATT_INSPECT_RANGE(filter->live()) << std::endl; + } + + for (usize drop_i = 0; drop_i < n; ++drop_i) { + BATT_CHECK_GE(next_droppable, 0); + BATT_CHECK_LT(next_droppable, drop_within.upper_bound); + + std::uniform_int_distribution pick_lower_bound{ + next_droppable, + drop_within.upper_bound - (drops_remaining * 2 - 1), + }; + const OffsetT lower_bound_i = pick_lower_bound(rng); + + std::uniform_int_distribution pick_upper_bound{ + lower_bound_i + 1, + drop_within.upper_bound - (drops_remaining * 2 - 2), + }; + const OffsetT upper_bound_i = pick_upper_bound(rng); + + BATT_CHECK_LT(lower_bound_i, upper_bound_i); + BATT_CHECK_GE(lower_bound_i, next_droppable); + + dropped_total_size += upper_bound_i - lower_bound_i; + + const usize live_count_before = filter->live().size(); + //----- --- -- - - - - + dropped_ranges.push_back(Interval{lower_bound_i, upper_bound_i}); + if constexpr (debug) { + std::cerr << " dropping: " << lower_bound_i << ".." << upper_bound_i << std::endl; + } + filter->drop_index_range(Interval{lower_bound_i, upper_bound_i}); + //----- --- -- - - - - + const usize live_count_after = filter->live().size(); + + if constexpr (debug) { + std::cerr << BATT_INSPECT_RANGE(filter->live()) << std::endl; + } + + if (lower_bound_i == live_lower_bound && upper_bound_i == live_upper_bound) { + BATT_CHECK_EQ(live_count_after + 1, live_count_before); + + } else if ((lower_bound_i == live_lower_bound && upper_bound_i != live_upper_bound) || + (upper_bound_i == live_upper_bound && lower_bound_i != live_lower_bound)) { + BATT_CHECK_EQ(live_count_after, live_count_before); + + } else { + BATT_CHECK_EQ(live_count_after, live_count_before + 1); + } + + --drops_remaining; + next_droppable = upper_bound_i + 1; + } + + return std::make_pair(dropped_total_size, dropped_ranges); +} + +struct PackedFilterData { + std::vector values; + bool start_is_live; +}; + +inline PackedFilterData pack_in_memory_filter(const PiecewiseFilter& filter) +{ + PackedFilterData data; + data.start_is_live = false; + Slice> live = filter.live(); + + if (live.empty()) { + return data; + } + + data.start_is_live = (live[0].lower_bound == PiecewiseFilter::kMinLowerBound); + + for (const Interval& range : live) { + if (range.lower_bound != PiecewiseFilter::kMinLowerBound) { + data.values.push_back(range.lower_bound); + } + if (range.upper_bound != PiecewiseFilter::kMaxUpperBound) { + data.values.push_back(range.upper_bound); + } + } + + return data; +} + +inline PackedPiecewiseFilter get_packed_filter_from_data(const PackedFilterData& data) +{ + return PackedPiecewiseFilter{PackedPiecewiseFilterStorage{ + batt::as_const_slice(data.values), data.start_is_live}}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + +struct RandomDropResult { + PiecewiseFilter filter; + std::set live_items; +}; + +inline RandomDropResult build_filter_with_random_drops(u32 num_items, std::default_random_engine& rng) +{ + RandomDropResult result; + for (u32 i = 0; i < num_items; ++i) { + result.live_items.insert(i); + } + + std::uniform_int_distribution pick_num_dropped{100, num_items / 2}; + u32 num_intervals_dropped = pick_num_dropped(rng); + for (u32 i = 0; i < num_intervals_dropped; ++i) { + std::uniform_int_distribution pick_interval_start{0, num_items - 1}; + u32 start_i = pick_interval_start(rng); + + std::uniform_int_distribution pick_interval_end{start_i, num_items}; + u32 end_i = pick_interval_end(rng); + + for (u32 j = start_i; j < end_i; ++j) { + result.live_items.erase(j); + } + + result.filter.drop_index_range(Interval{start_i, end_i}); + } + + return result; +} + +template +inline void verify_filter_queries(const FilterT& filter, + const std::set& live_items, + u32 num_items, + u32 seed, + std::default_random_engine& rng) +{ + for (u32 i = 0; i < num_items; ++i) { + bool expected_live = live_items.count(i) > 0; + bool actual_live = filter.live_at_index(i); + EXPECT_EQ(actual_live, expected_live) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } + + for (u32 i = 0; i < num_items; ++i) { + auto iter = live_items.lower_bound(i); + u32 expected = (iter != live_items.end()) ? *iter : num_items; + u32 actual = filter.live_lower_bound(i); + EXPECT_EQ(actual, expected) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } + + for (u32 i = 0; i < 100; ++i) { + std::uniform_int_distribution pick_interval_start{0, num_items - 1}; + u32 start_i = pick_interval_start(rng); + + std::uniform_int_distribution pick_interval_end{start_i, num_items}; + u32 end_i = pick_interval_end(rng); + + auto iter = live_items.lower_bound(start_i); + Interval expected_range; + + if (iter == live_items.end() || *iter >= end_i) { + expected_range = Interval{end_i, end_i}; + } else { + u32 first = *iter; + u32 last = first + 1; + auto next = std::next(iter); + + while (next != live_items.end() && *next < end_i && *next == last) { + ++last; + ++next; + } + + expected_range = Interval{first, last}; + } + + Interval actual_range = filter.find_live_range(Interval{start_i, end_i}); + EXPECT_EQ(actual_range, expected_range) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } +} + +} // namespace testing +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp new file mode 100644 index 0000000..5a105df --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp @@ -0,0 +1,59 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_STORAGE_MODEL_CONCEPT_HPP + +#include +#include + +#include + +#include +#include +#include +#include + +namespace turtle_kv { + +template +concept PiecewiseFilterStorageModel = requires(const T& model, + T& src, + T& dst, + Interval interval, + usize i, + std::ostream& out) { + typename T::value_type; + typename T::iterator; + typename T::const_iterator; + + { model.begin() }; + { model.end() }; + { std::begin(model) } -> std::same_as; + { std::end(model) } -> std::same_as; + { as_const_slice(model) }; + { model.empty() } -> std::convertible_to; + { model.size() } -> std::convertible_to; + { *model.begin() } -> std::convertible_to&>; + { model[i] } -> std::convertible_to&>; + { dst = std::move(src) }; + { out << batt::dump_range(model) } -> std::same_as; +}; + +template +concept PiecewiseFilterMutableStorageModel = + PiecewiseFilterStorageModel && + requires(T model, T other, Interval interval, usize i, std::ostream& out) { + { model.clear() }; + { model.erase(model.end()) }; + { model.insert(model.end(), interval) }; + { model.insert(model.end(), model.begin(), model.end()) }; + { model[i] = interval }; + }; + +} // namespace turtle_kv