diff --git a/btree_test b/btree_test index a1b19bb..7988bbe 100755 Binary files a/btree_test and b/btree_test differ diff --git a/cache_performance_demo b/cache_performance_demo new file mode 100755 index 0000000..7282dba Binary files /dev/null and b/cache_performance_demo differ diff --git a/content_addressable_demo b/content_addressable_demo index e6bc797..8daab3b 100755 Binary files a/content_addressable_demo and b/content_addressable_demo differ diff --git a/content_hash_demo b/content_hash_demo index e377306..8e76b6a 100755 Binary files a/content_hash_demo and b/content_hash_demo differ diff --git a/deduplication_demo b/deduplication_demo index 71f47e3..4c9abb2 100755 Binary files a/deduplication_demo and b/deduplication_demo differ diff --git a/include/btree.h b/include/btree.h index 4aeb694..108e885 100644 --- a/include/btree.h +++ b/include/btree.h @@ -8,6 +8,8 @@ #include "fraction.h" #include "page_manager.h" #include "content_storage.h" +#include "page_cache.h" +#include "writer_queue.h" /* * BTree that stores the BTreeNodes, ensures it is balanced @@ -19,6 +21,8 @@ class BTree { std::shared_ptr> root; int maxKeysPerNode; // Maximum keys in each node ContentStorage content_storage; + PageCache page_cache; + WriterQueue writer_queue; void insertNonFull(std::shared_ptr> root, const KeyType& key, const ValueType& value); void splitChild(std::shared_ptr> parent, int index, std::shared_ptr> child); @@ -30,10 +34,12 @@ class BTree { public: BTree(int maxKeys); + ~BTree(); void insert(const KeyType& key, const ValueType& value); void deleteKey(const KeyType& key); ValueType* search(const KeyType& key); // Public search method void printStorageStats() const; + void flush(); // To flush all pending writes Page findKey(std::shared_ptr> node, const KeyType& key); diff --git a/include/page_cache.h b/include/page_cache.h new file mode 100644 index 0000000..fdb1c24 --- /dev/null +++ b/include/page_cache.h @@ -0,0 +1,56 @@ +#pragma once +#include +#include +#include +#include +#include +#include "page_manager.h" +#include "content_storage.h" + +// with metadata +template +struct CachedPage { + std::shared_ptr> page; + bool is_dirty; + std::chrono::steady_clock::time_point last_accessed; + + CachedPage(std::shared_ptr> p, bool dirty = false) + : page(p), is_dirty(dirty), last_accessed(std::chrono::steady_clock::now()) {} +}; + +template +class PageCache { +private: + // Core cache storage + std::unordered_map> cache; + + // LRU tracking + std::list lru_order; + std::unordered_map::iterator> lru_iterators; + + // For thread safety + mutable std::mutex cache_mutex; + + size_t max_cache_size; + + // Reference to content storage + ContentStorage* content_storage; + + void updateLRU(uint16_t page_id); + void evictLRU(); + void evictIfNeeded(); + +public: + PageCache(ContentStorage* storage, size_t max_size = 100); + ~PageCache(); + + // These are the main cache operations + std::shared_ptr> getPage(uint16_t page_id); + void putPage(uint16_t page_id, std::shared_ptr> page); + void markDirty(uint16_t page_id); + + // Cache management + std::vector>>> getDirtyPages(); + void clearDirtyFlag(uint16_t page_id); + void flushAll(); +}; diff --git a/include/writer_queue.h b/include/writer_queue.h new file mode 100644 index 0000000..7a425ae --- /dev/null +++ b/include/writer_queue.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include "page_manager.h" +#include "content_storage.h" +#include "page_cache.h" + +template +struct WriteRequest { + uint16_t page_id; + std::shared_ptr> page; + std::chrono::steady_clock::time_point timestamp; + + WriteRequest(uint16_t id, std::shared_ptr> p) + : page_id(id), page(p), timestamp(std::chrono::steady_clock::now()) {} +}; + +template +class WriterQueue { +private: + // Queue + std::queue> write_queue; + std::mutex queue_mutex; + std::condition_variable queue_cv; + std::condition_variable empty_cv; + + // Thread + std::vector writer_threads; + std::atomic running; + size_t num_writer_threads; + + // References to other components + ContentStorage* content_storage; + PageCache* page_cache; + + // Configuration + size_t max_queue_size; + std::chrono::milliseconds batch_timeout; + + // Worker thread function + void writerWorker(int worker_id); + + // Batch processing + std::vector> getBatch(size_t max_batch_size); + void processBatch(const std::vector>& batch, int worker_id); + +public: + WriterQueue(ContentStorage* storage, PageCache* cache, + size_t num_threads = 2, size_t max_queue = 1000); + ~WriterQueue(); + + // Queue operations + bool enqueueWrite(uint16_t page_id, std::shared_ptr> page); + void start(); + void stop(); + void waitForEmpty(); +}; diff --git a/makefile b/makefile index b1493b1..f428790 100644 --- a/makefile +++ b/makefile @@ -4,29 +4,34 @@ SRCDIR = src OBJDIR = obj # Source files (only B-tree related files) -SOURCES = src/Btree.cpp src/main.cpp src/page_manager.cpp +SOURCES = src/Btree.cpp src/main.cpp src/page_manager.cpp src/page_cache.cpp src/writer_queue.cpp OBJECTS = $(SOURCES:$(SRCDIR)/%.cpp=$(OBJDIR)/%.o) # Demo source files -DEMO_SOURCES = src/Btree.cpp src/content_hash_demo.cpp src/page_manager.cpp +DEMO_SOURCES = src/Btree.cpp src/content_hash_demo.cpp src/page_manager.cpp src/page_cache.cpp src/writer_queue.cpp DEMO_OBJECTS = $(DEMO_SOURCES:$(SRCDIR)/%.cpp=$(OBJDIR)/%.o) # Content addressable demo -ADDRESSABLE_SOURCES = src/Btree.cpp src/content_addressable_demo.cpp src/page_manager.cpp +ADDRESSABLE_SOURCES = src/Btree.cpp src/content_addressable_demo.cpp src/page_manager.cpp src/page_cache.cpp src/writer_queue.cpp ADDRESSABLE_OBJECTS = $(ADDRESSABLE_SOURCES:$(SRCDIR)/%.cpp=$(OBJDIR)/%.o) # Deduplication demo -DEDUP_SOURCES = src/Btree.cpp src/deduplication_demo.cpp src/page_manager.cpp +DEDUP_SOURCES = src/Btree.cpp src/deduplication_demo.cpp src/page_manager.cpp src/page_cache.cpp src/writer_queue.cpp DEDUP_OBJECTS = $(DEDUP_SOURCES:$(SRCDIR)/%.cpp=$(OBJDIR)/%.o) +# Cache performance demo +CACHE_PERF_SOURCES = src/Btree.cpp src/cache_performance_demo.cpp src/page_manager.cpp src/page_cache.cpp src/writer_queue.cpp +CACHE_PERF_OBJECTS = $(CACHE_PERF_SOURCES:$(SRCDIR)/%.cpp=$(OBJDIR)/%.o) + # Target executables TARGET = btree_test DEMO_TARGET = content_hash_demo ADDRESSABLE_TARGET = content_addressable_demo DEDUP_TARGET = deduplication_demo +CACHE_PERF_TARGET = cache_performance_demo # Default target -all: $(TARGET) $(DEMO_TARGET) $(ADDRESSABLE_TARGET) $(DEDUP_TARGET) +all: $(TARGET) $(DEMO_TARGET) $(ADDRESSABLE_TARGET) $(DEDUP_TARGET) $(CACHE_PERF_TARGET) # Create object directory if it doesn't exist $(OBJDIR): @@ -52,9 +57,13 @@ $(ADDRESSABLE_TARGET): $(ADDRESSABLE_OBJECTS) $(DEDUP_TARGET): $(DEDUP_OBJECTS) $(CXX) $(DEDUP_OBJECTS) -o $(DEDUP_TARGET) +# Link cache performance demo executable +$(CACHE_PERF_TARGET): $(CACHE_PERF_OBJECTS) + $(CXX) $(CACHE_PERF_OBJECTS) -o $(CACHE_PERF_TARGET) + # Clean build files clean: - rm -rf $(OBJDIR) $(TARGET) $(DEMO_TARGET) $(ADDRESSABLE_TARGET) $(DEDUP_TARGET) + rm -rf $(OBJDIR) $(TARGET) $(DEMO_TARGET) $(ADDRESSABLE_TARGET) $(DEDUP_TARGET) $(CACHE_PERF_TARGET) # Run the test run: $(TARGET) @@ -72,6 +81,10 @@ addressable: $(ADDRESSABLE_TARGET) dedup: $(DEDUP_TARGET) ./$(DEDUP_TARGET) +# Run the cache performance demo +cache_perf: $(CACHE_PERF_TARGET) + ./$(CACHE_PERF_TARGET) + # Run all tests (for CI/CD compatibility) tests: $(TARGET) $(DEMO_TARGET) $(ADDRESSABLE_TARGET) $(DEDUP_TARGET) @echo "=== Running Content Hash Demo ===" @@ -87,4 +100,4 @@ tests: $(TARGET) $(DEMO_TARGET) $(ADDRESSABLE_TARGET) $(DEDUP_TARGET) @echo -e "insert 1 apple\ninsert 2 banana\nsearch 1\nsearch 2\nquit" | ./$(TARGET) > /dev/null @echo "All tests passed!" -.PHONY: all clean run demo addressable dedup tests +.PHONY: all clean run demo addressable dedup cache_perf tests diff --git a/src/Btree.cpp b/src/Btree.cpp index 912ca05..4320c92 100644 --- a/src/Btree.cpp +++ b/src/Btree.cpp @@ -6,29 +6,53 @@ * BTree Constructor Implementation */ template -BTree::BTree(int maxKeys) : maxKeysPerNode(maxKeys) { +BTree::BTree(int maxKeys) + : maxKeysPerNode(maxKeys), + page_cache(&content_storage, 50), // Cache up to 50 pages, can change latr + writer_queue(&content_storage, &page_cache, 2) { // 2 writer threads + + writer_queue.start(); // Initially, the tree is empty, so we create a root node // and mark it as a leaf (all data starts at the leaf level in B+ Trees) uint16_t root_id = content_storage.storePage(createPage(true)); - root = content_storage.getPage(root_id); + root = page_cache.getPage(root_id); // Get through cache +} + +/* + * BTree Destructor Implementation to stop writer q and flush pending writes + */ +template +BTree::~BTree() { + writer_queue.stop(); + page_cache.flushAll(); +} + +/* + * Flush pending writes + */ +template +void BTree::flush() { + writer_queue.waitForEmpty(); + page_cache.flushAll(); } /* - * Example Method to Insert Key-Value Pairs (Placeholder) + * Placeholder method to insert key value pairs */ template void BTree::insert(const KeyType& key, const ValueType& value) { if (!root) { // If tree is empty, create a new root uint16_t root_id = content_storage.storePage(createPage(true)); - root = content_storage.getPage(root_id); + root = page_cache.getPage(root_id); } else if (root->keys.size() == maxKeysPerNode) { // Check if the root is full Page new_root_page = createPage(false); new_root_page.children.push_back(root->header.page_id); // Page ID of the old root splitChild(std::make_shared>(new_root_page), 0, root); // Split child bc of overflow - // Store the new root in content storage - uint16_t new_root_id = content_storage.storePage(new_root_page); - root = content_storage.getPage(new_root_id); + // Store the new root using cache and writer queue + page_cache.putPage(new_root_page.header.page_id, std::make_shared>(new_root_page)); + writer_queue.enqueueWrite(new_root_page.header.page_id, std::make_shared>(new_root_page)); + root = page_cache.getPage(new_root_page.header.page_id); } // Now the root is guaranteed to not be empty insertNonFull(root, key, value); // Insert @@ -54,8 +78,8 @@ Page BTree::findKey(std::shared_ptr> idx++; // move to child that might have key } if (idx < node->children.size()) { - // Load child page from content storage - auto child_page = content_storage.getPage(node->children[idx]); + // Load child page from cache + auto child_page = page_cache.getPage(node->children[idx]); if (child_page) { return findKey(child_page, key); } else { @@ -101,9 +125,10 @@ void BTree::insertNonFull(std::shared_ptr> nod } } - // Store the modified page in content storage and update the node reference - uint16_t new_page_id = content_storage.storePage(modified_node); - node = content_storage.getPage(new_page_id); + // Store the modified page using cache and writer queue + page_cache.putPage(modified_node.header.page_id, std::make_shared>(modified_node)); + writer_queue.enqueueWrite(modified_node.header.page_id, std::make_shared>(modified_node)); + node = page_cache.getPage(modified_node.header.page_id); } else { // Find child to descend into @@ -111,8 +136,8 @@ void BTree::insertNonFull(std::shared_ptr> nod i--; i++; - // Load child page from content storage - auto child_page = content_storage.getPage(node->children[i]); + // Load child page from cache + auto child_page = page_cache.getPage(node->children[i]); if (!child_page) { throw std::runtime_error("child page not found"); } @@ -150,9 +175,13 @@ void BTree::splitChild(std::shared_ptr> parent modified_child.children.resize(mid + 1); } - // Store both modified pages in content storage - content_storage.storePage(modified_child); - uint16_t new_child_id = content_storage.storePage(new_child_page); + // Store both modified pages using cache and writer queue + page_cache.putPage(modified_child.header.page_id, std::make_shared>(modified_child)); + writer_queue.enqueueWrite(modified_child.header.page_id, std::make_shared>(modified_child)); + + uint16_t new_child_id = content_storage.storePage(new_child_page); // New page needs ID first + page_cache.putPage(new_child_id, std::make_shared>(new_child_page)); + writer_queue.enqueueWrite(new_child_id, std::make_shared>(new_child_page)); // Update parent parent->children.insert(parent->children.begin() + index + 1, new_child_id); // Insert new child page ID @@ -167,12 +196,12 @@ void BTree::deleteKey(const KeyType& key) { // If root is now empty and has a child, make child the new root if (!root->is_leaf && root->keys.empty()) { - auto child_page = content_storage.getPage(root->children[0]); + auto child_page = page_cache.getPage(root->children[0]); if (child_page) { root = child_page; } else { uint16_t root_id = content_storage.storePage(createPage(true)); - root = content_storage.getPage(root_id); + root = page_cache.getPage(root_id); } } } @@ -198,9 +227,10 @@ void BTree::deleteFromNode(std::shared_ptr> no size_t start_offset = idx * value_size; modified_node.data.erase(modified_node.data.begin() + start_offset, modified_node.data.begin() + start_offset + value_size); - // Store the modified page in content storage and update the node reference - uint16_t new_page_id = content_storage.storePage(modified_node); - node = content_storage.getPage(new_page_id); + // Store the modified page using cache and writer queue + page_cache.putPage(modified_node.header.page_id, std::make_shared>(modified_node)); + writer_queue.enqueueWrite(modified_node.header.page_id, std::make_shared>(modified_node)); + node = page_cache.getPage(modified_node.header.page_id); } else { // Key not found return; @@ -210,8 +240,8 @@ void BTree::deleteFromNode(std::shared_ptr> no idx++; // move to child that might have key } - // Load child page from content storage - auto child_page = content_storage.getPage(node->children[idx]); + // Load child page from cache + auto child_page = page_cache.getPage(node->children[idx]); if (!child_page) { throw std::runtime_error("child page not found"); } @@ -229,8 +259,8 @@ void BTree::deleteFromNode(std::shared_ptr> no template void BTree::borrowFromLeft(std::shared_ptr> parent, int index) { - auto child_page = content_storage.getPage(parent->children[index]); - auto sibling_page = content_storage.getPage(parent->children[index - 1]); + auto child_page = page_cache.getPage(parent->children[index]); + auto sibling_page = page_cache.getPage(parent->children[index - 1]); if (!child_page || !sibling_page) { throw std::runtime_error("child or sibling page not found"); @@ -254,9 +284,11 @@ void BTree::borrowFromLeft(std::shared_ptr> pa modified_sibling.data.resize(modified_sibling.data.size() - value_size); // Remove the last value from sibling parent->keys[index - 1] = modified_child.keys[0]; // Update the parent key - // Store modified pages in content storage - content_storage.storePage(modified_child); - content_storage.storePage(modified_sibling); + // Store modified pages using cache and writer queue + page_cache.putPage(modified_child.header.page_id, std::make_shared>(modified_child)); + writer_queue.enqueueWrite(modified_child.header.page_id, std::make_shared>(modified_child)); + page_cache.putPage(modified_sibling.header.page_id, std::make_shared>(modified_sibling)); + writer_queue.enqueueWrite(modified_sibling.header.page_id, std::make_shared>(modified_sibling)); } else { // If not leaf, borrow the last key and child pointer modified_child.keys.insert(modified_child.keys.begin(), parent->keys[index - 1]); parent->keys[index - 1] = modified_sibling.keys.back(); // Update the parent key diff --git a/src/cache_performance_demo.cpp b/src/cache_performance_demo.cpp new file mode 100644 index 0000000..dc99434 --- /dev/null +++ b/src/cache_performance_demo.cpp @@ -0,0 +1,136 @@ +#include +#include +#include +#include +#include "btree.h" + +int main() { + std::cout << "=== Cache & Writer Queue Performance Demo ===" << std::endl; + + // Create B-tree with small node size to force more operations + BTree tree(3); // 3 keys per node + + // Random test data cuz I was craving fruits + std::vector> test_data; + std::vector fruits = {"apple", "banana", "cherry", "date", "elderberry", + "fig", "grape", "honeydew", "kiwi", "lemon"}; + + for (int i = 1; i <= 50; ++i) { + test_data.emplace_back(i, fruits[i % fruits.size()] + "_" + std::to_string(i)); + } + + std::cout << "\n1. Inserting " << test_data.size() << " key-value pairs" << std::endl; + auto start = std::chrono::high_resolution_clock::now(); + + for (const auto& pair : test_data) { + tree.insert(pair.first, pair.second); + } + + auto end = std::chrono::high_resolution_clock::now(); + auto insert_duration = std::chrono::duration_cast(end - start); + + std::cout << "Insert completed in: " << insert_duration.count() << " microseconds" << std::endl; + + // Test search performance (should benefit from cache) + std::cout << "\n2. Testing search performance..." << std::endl; + start = std::chrono::high_resolution_clock::now(); + + int successful_searches = 0; + for (int i = 1; i <= 50; ++i) { + auto result = tree.search(i); + if (result) { + successful_searches++; + delete result; + } + } + + end = std::chrono::high_resolution_clock::now(); + auto search_duration = std::chrono::duration_cast(end - start); + + std::cout << "Found " << successful_searches << " keys" << std::endl; + std::cout << "Search completed in: " << search_duration.count() << " microseconds" << std::endl; + + // Test repeated searches (should show cache benefits) + std::cout << "\n3. Testing repeated searches (cache should help)..." << std::endl; + start = std::chrono::high_resolution_clock::now(); + + successful_searches = 0; + for (int round = 0; round < 3; ++round) { + for (int i = 1; i <= 20; ++i) { + auto result = tree.search(i); + if (result) { + successful_searches++; + delete result; + } + } + } + + end = std::chrono::high_resolution_clock::now(); + auto repeated_search_duration = std::chrono::duration_cast(end - start); + + std::cout << "Found " << successful_searches << " keys across 3 rounds" << std::endl; + std::cout << "Repeated searches completed in: " << repeated_search_duration.count() << " microseconds" << std::endl; + + // Insert more data to trigger more cache and writer queue activity + std::cout << "\n4. Inserting additional data to stress test cache and writer queue..." << std::endl; + start = std::chrono::high_resolution_clock::now(); + + for (int i = 51; i <= 100; ++i) { + tree.insert(i, "stress_test_" + std::to_string(i)); + } + + end = std::chrono::high_resolution_clock::now(); + auto stress_insert_duration = std::chrono::duration_cast(end - start); + + std::cout << "Stress test insert completed in: " << stress_insert_duration.count() << " microseconds" << std::endl; + + // Flush all pending writes + std::cout << "\n5. Flushing all pending writes..." << std::endl; + start = std::chrono::high_resolution_clock::now(); + + tree.flush(); + + end = std::chrono::high_resolution_clock::now(); + auto flush_duration = std::chrono::duration_cast(end - start); + + std::cout << "Flush completed in: " << flush_duration.count() << " microseconds" << std::endl; + + // Final verification + std::cout << "\n6. Final verification - searching for some keys..." << std::endl; + std::vector test_keys = {1, 25, 50, 75, 100}; + + for (int key : test_keys) { + auto result = tree.search(key); + if (result) { + std::cout << "Key " << key << ": " << *result << std::endl; + delete result; + } else { + std::cout << "Key " << key << ": NOT FOUND" << std::endl; + } + } + + // Show storage stats + std::cout << "\n7. Storage statistics:" << std::endl; + tree.printStorageStats(); + + std::cout << "\n=== Performance Summary ===" << std::endl; + std::cout << "Initial insert (50 items): " << insert_duration.count() << " μs" << std::endl; + std::cout << "Search (50 items): " << search_duration.count() << " μs" << std::endl; + std::cout << "Repeated searches (60 items): " << repeated_search_duration.count() << " μs" << std::endl; + std::cout << "Stress insert (50 items): " << stress_insert_duration.count() << " μs" << std::endl; + std::cout << "Flush time: " << flush_duration.count() << " μs" << std::endl; + + double avg_insert_time = (insert_duration.count() + stress_insert_duration.count()) / 100.0; + double avg_search_time = search_duration.count() / 50.0; + + std::cout << "Average insert time: " << avg_insert_time << " μs per item" << std::endl; + std::cout << "Average search time: " << avg_search_time << " μs per item" << std::endl; + + std::cout << "\n=== Cache & Writer Queue Benefits ===" << std::endl; + std::cout << "✓ Pages are cached for faster repeated access" << std::endl; + std::cout << "✓ Writes are batched and processed in background" << std::endl; + std::cout << "✓ Content-addressable storage still provides deduplication" << std::endl; + std::cout << "✓ Multi-threaded write processing improves throughput" << std::endl; + + return 0; +} diff --git a/src/page_cache.cpp b/src/page_cache.cpp new file mode 100644 index 0000000..1688982 --- /dev/null +++ b/src/page_cache.cpp @@ -0,0 +1,162 @@ +#include "page_cache.h" +#include + +template +PageCache::PageCache(ContentStorage* storage, size_t max_size) + : content_storage(storage), max_cache_size(max_size) { + if (!storage) { // Just include descriptive error messages + throw std::invalid_argument("ContentStorage cannot be null"); + } +} + +template +PageCache::~PageCache() { + flushAll(); +} + +template +void PageCache::updateLRU(uint16_t page_id) { + auto it = lru_iterators.find(page_id); + if (it != lru_iterators.end()) { + lru_order.erase(it->second); + } + + lru_order.push_front(page_id); + lru_iterators[page_id] = lru_order.begin(); +} + +template +void PageCache::evictLRU() { + if (lru_order.empty()) return; + + uint16_t lru_page_id = lru_order.back(); + + auto cache_it = cache.find(lru_page_id); + if (cache_it != cache.end() && cache_it->second.is_dirty) { + // Write back to content storage + content_storage->storePage(*(cache_it->second.page)); + std::cout << "Cache: Writing back dirty page " << lru_page_id << " during eviction" << std::endl; + } + + // Remove from cache and LRU tracking + cache.erase(lru_page_id); + lru_order.pop_back(); + lru_iterators.erase(lru_page_id); +} + +template +void PageCache::evictIfNeeded() { + while (cache.size() >= max_cache_size) { + evictLRU(); + } +} + +template +std::shared_ptr> PageCache::getPage(uint16_t page_id) { + std::lock_guard lock(cache_mutex); + + // Check if page is in cache + auto it = cache.find(page_id); + if (it != cache.end()) { + // Update LRU since cache got hit + updateLRU(page_id); + it->second.last_accessed = std::chrono::steady_clock::now(); + return it->second.page; + } + + // Cache miss so load from content storage + auto page = content_storage->getPage(page_id); + if (!page) { + return nullptr; + } + + // Add to cache + evictIfNeeded(); + cache.emplace(page_id, CachedPage(page, false)); + updateLRU(page_id); + + std::cout << "Cache: Loaded page " << page_id << " from storage" << std::endl; + return page; +} + +template +void PageCache::putPage(uint16_t page_id, std::shared_ptr> page) { + std::lock_guard lock(cache_mutex); + + evictIfNeeded(); + + // Add or update page in cache + auto it = cache.find(page_id); + if (it != cache.end()) { + // Update existing entry + it->second.page = page; + it->second.is_dirty = true; + it->second.last_accessed = std::chrono::steady_clock::now(); + } else { + // Add new entry + cache.emplace(page_id, CachedPage(page, true)); + } + + updateLRU(page_id); + std::cout << "Cache: Stored page " << page_id << " (marked as dirty)" << std::endl; +} + +template +void PageCache::markDirty(uint16_t page_id) { + std::lock_guard lock(cache_mutex); + + auto it = cache.find(page_id); + if (it != cache.end()) { + it->second.is_dirty = true; + updateLRU(page_id); + std::cout << "Cache: Marked page " << page_id << " as dirty" << std::endl; + } +} + +template +std::vector>>> PageCache::getDirtyPages() { + std::lock_guard lock(cache_mutex); + + std::vector>>> dirty_pages; + + for (const auto& entry : cache) { + if (entry.second.is_dirty) { + dirty_pages.emplace_back(entry.first, entry.second.page); + } + } + + return dirty_pages; +} + +template +void PageCache::clearDirtyFlag(uint16_t page_id) { + std::lock_guard lock(cache_mutex); + + auto it = cache.find(page_id); + if (it != cache.end()) { + it->second.is_dirty = false; + } +} + +template +void PageCache::flushAll() { + std::lock_guard lock(cache_mutex); + + std::cout << "Cache: Flushing all dirty pages" << std::endl; + size_t flushed = 0; + + for (auto& entry : cache) { + if (entry.second.is_dirty) { + content_storage->storePage(*(entry.second.page)); + entry.second.is_dirty = false; + flushed++; + } + } + + std::cout << "Cache: Flushed " << flushed << " dirty pages" << std::endl; +} + + + +template class PageCache; +template class PageCache; diff --git a/src/writer_queue.cpp b/src/writer_queue.cpp new file mode 100644 index 0000000..af4bff7 --- /dev/null +++ b/src/writer_queue.cpp @@ -0,0 +1,149 @@ +#include "writer_queue.h" +#include +#include + +template +WriterQueue::WriterQueue(ContentStorage* storage, PageCache* cache, + size_t num_threads, size_t max_queue) + : content_storage(storage), page_cache(cache), running(false), + num_writer_threads(num_threads), max_queue_size(max_queue), + batch_timeout(std::chrono::milliseconds(10)) { + + if (!storage || !cache) { + throw std::invalid_argument("ContentStorage and PageCache cannot be null"); + } +} + +template +WriterQueue::~WriterQueue() { + stop(); +} + +template +void WriterQueue::start() { + if (running.load()) { + return; // Already running + } + + running.store(true); + + // Start writer threads + for (size_t i = 0; i < num_writer_threads; ++i) { + writer_threads.emplace_back(&WriterQueue::writerWorker, this, i); + } + + std::cout << "WriterQueue: Started " << num_writer_threads << " writer threads" << std::endl; +} + +template +void WriterQueue::stop() { + if (!running.load()) { + return; // Already stopped + } + + std::cout << "WriterQueue: Stopping writer threads" << std::endl; + + // Signal threads to stop + running.store(false); + queue_cv.notify_all(); + + // Wait for all threads to finish + for (auto& thread : writer_threads) { + if (thread.joinable()) { + thread.join(); + } + } + + writer_threads.clear(); + std::cout << "WriterQueue: All writer threads stopped" << std::endl; +} + +template +bool WriterQueue::enqueueWrite(uint16_t page_id, std::shared_ptr> page) { + std::unique_lock lock(queue_mutex); + + // Check if queue is full + if (write_queue.size() >= max_queue_size) { + std::cout << "WriterQueue: Queue overflow, dropping write request for page " << page_id << std::endl; + return false; + } + + write_queue.emplace(page_id, page); + + // Notify writer thread + queue_cv.notify_one(); + + return true; +} + +template +void WriterQueue::waitForEmpty() { + std::unique_lock lock(queue_mutex); + empty_cv.wait(lock, [this] { return write_queue.empty(); }); +} + +template +std::vector> WriterQueue::getBatch(size_t max_batch_size) { + std::vector> batch; + std::unique_lock lock(queue_mutex); + + auto timeout = std::chrono::steady_clock::now() + batch_timeout; + queue_cv.wait_until(lock, timeout, [this] { return !write_queue.empty() || !running.load(); }); + + // Collect batch + while (!write_queue.empty() && batch.size() < max_batch_size) { + batch.push_back(std::move(write_queue.front())); + write_queue.pop(); + } + + if (write_queue.empty()) { + empty_cv.notify_all(); + } + + return batch; +} + +template +void WriterQueue::processBatch(const std::vector>& batch, int worker_id) { + if (batch.empty()) return; + + std::cout << "WriterQueue: Worker " << worker_id << " processing batch of " << batch.size() << " pages" << std::endl; + + for (const auto& request : batch) { + try { + // Write to content storage (this is where deduplication happens yayy) + uint16_t stored_page_id = content_storage->storePage(*(request.page)); + + // Clear dirty flag in cache since weve written it + page_cache->clearDirtyFlag(request.page_id); + + } catch (const std::exception& e) { + std::cerr << "WriterQueue: Error writing page " << request.page_id << ": " << e.what() << std::endl; + } + } +} + +template +void WriterQueue::writerWorker(int worker_id) { + std::cout << "WriterQueue: Worker " << worker_id << " started" << std::endl; + + const size_t max_batch_size = 10; + + while (running.load() || !write_queue.empty()) { + auto batch = getBatch(max_batch_size); + + if (!batch.empty()) { + processBatch(batch, worker_id); + } + + // Small delay to prevent busy waiting + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + + std::cout << "WriterQueue: Worker " << worker_id << " finished" << std::endl; +} + + + +template class WriterQueue; +template class WriterQueue;