From f8d3473328495e7f1cdacf351662cfd90a3eca65 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:04:22 +0200 Subject: [PATCH 01/24] fuse: hold DLM grants as a bitmap and drive every path from it The interval tree recorded a grant as a node, and every path that touched the page cache had to keep the tree in step with it: a revoked range was kept so writeback would take the grant again, and each place that dropped folios then had to free the ranges the drop had emptied. That is a second description of the page cache, maintained by hand at every drop site, and wrong wherever a site was missed. Record the grants in a bitmap instead, a bit per page, in fixed shards by offset. Adjacent grants coalesce by setting neighbouring bits, so nothing is allocated or rearranged to record one, and a revoke clears bits rather than splitting nodes. The record then says what the server has given this client and nothing about the page cache: a revoked grant is forgotten, not kept, because writeback holds the range again for every run it sends, and an absent record and a revoked one both make it ask. Every fuse_dlm_ranges_dropped() site goes with it, and so does fuse_dlm_write_grant_exists(): with grants forgotten on revoke there is no cheap answer to "is anything held for write", and the dirty and writeback tags already say what the attribute cache needs. Signed-off-by: Horst Birthelmer --- fs/fuse/dir.c | 36 +- fs/fuse/file.c | 69 +-- fs/fuse/fuse_dlm_cache.c | 877 ++++++++++++++------------------------- fs/fuse/fuse_dlm_cache.h | 111 +++-- fs/fuse/fuse_i.h | 2 +- fs/fuse/inode.c | 67 +-- 6 files changed, 406 insertions(+), 756 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index fe4470e39fc095..2bfb1f810322ee 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -800,15 +800,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, get_fuse_inode(inode)); truncate_pagecache(inode, 0); } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { - /* - * Only when the drop really emptied the mapping, as - * in fuse_open(): a folio that survived still needs - * its record. - */ - if (fuse_open_drop_cache(inode) && - fm->fc->dlm && fm->fc->writeback_cache) - fuse_dlm_ranges_dropped(get_fuse_inode(inode), - 0, U64_MAX); + fuse_open_drop_cache(inode); } } return err; @@ -2235,32 +2227,6 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); - - /* - * The cache above the new size is gone, so the ranges - * describing it have nothing left to say. From the first - * whole page above it: the page holding the new end of the - * file survives the truncate, and so does its record. - */ - if (fc->dlm && fc->writeback_cache) - fuse_dlm_ranges_dropped(fi, PAGE_ALIGN(outarg.attr.size), - U64_MAX); - - /* - * invalidate_inode_pages2() emptied the mapping below the new - * size too (laundering anything dirty first, so those bytes - * are on the server). A revoked range is kept only to make - * writeback take the grant again before sending the folios - * under it, so with those folios gone it describes nothing - * and would sit in the tree unfreed, keeping its neighbours - * from merging. Only when the drop really emptied it: a busy - * folio that survived still needs its record, and a fault - * populating after the check keeps its page visible to it. - */ - if (fc->dlm && fc->writeback_cache && outarg.attr.size && - !filemap_range_has_page(mapping, 0, outarg.attr.size - 1)) - fuse_dlm_ranges_dropped(fi, 0, - PAGE_ALIGN(outarg.attr.size) - 1); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index d269f88628651b..69a09bd12f4a7c 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -342,8 +342,7 @@ int fuse_finish_open(struct inode *inode, struct file *file) } /* - * Drop the page cache an open that did not get FOPEN_KEEP_CACHE must not keep, - * and report whether the mapping came out empty. + * Drop the page cache an open that did not get FOPEN_KEEP_CACHE must not keep. * * Serialised on the mapping's invalidate lock. Every opener runs this same * full-mapping walk and invalidate_inode_pages2_range() takes each folio's @@ -354,13 +353,10 @@ int fuse_finish_open(struct inode *inode, struct file *file) * mapping and the rest fall straight back out of mapping_empty(). * * Holding it exclusive also fences faults for the duration, which is what - * truncate already does across this same walk, and it keeps the emptiness - * test from reading a mapping another opener is halfway through. + * truncate already does across this same walk. */ -bool fuse_open_drop_cache(struct inode *inode) +void fuse_open_drop_cache(struct inode *inode) { - bool emptied; - filemap_invalidate_lock(inode->i_mapping); /* * Write back first: the drop launders whatever it finds dirty a @@ -369,10 +365,7 @@ bool fuse_open_drop_cache(struct inode *inode) */ filemap_write_and_wait(inode->i_mapping); invalidate_inode_pages2(inode->i_mapping); - emptied = !filemap_range_has_page(inode->i_mapping, 0, LLONG_MAX); filemap_invalidate_unlock(inode->i_mapping); - - return emptied; } static void fuse_truncate_update_attr(struct inode *inode, struct file *file) @@ -444,18 +437,7 @@ static int fuse_open(struct inode *inode, struct file *file) fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { - /* - * Only when the drop really emptied the mapping; a - * folio that survived still needs its record. This - * open holds no lock against buffered IO, which can - * repopulate the mapping once the invalidate lock is - * dropped -- but such a folio is populated after the - * drop and keeps its page, so a later reader finds - * both the page and the record it needs. - */ - if (fuse_open_drop_cache(inode) && - fc->dlm && fc->writeback_cache) - fuse_dlm_ranges_dropped(fi, 0, U64_MAX); + fuse_open_drop_cache(inode); } } if (dax_truncate) @@ -4241,29 +4223,9 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, file_update_time(file); } - if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) { + if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) truncate_pagecache_range(inode, offset, offset + length - 1); - /* - * The whole pages inside the hole are gone, so a revoked - * range over them has nothing left to make writeback take - * the grant again for, and would sit in the tree unfreed. - * The pages straddling the ends survive with their punched - * part zeroed, so their record still names real (now zero) - * bytes and stays. Only when the drop really emptied the - * span: a busy folio that survived keeps its record. - */ - if (fm->fc->dlm && fm->fc->writeback_cache) { - uint64_t first = PAGE_ALIGN(offset); - uint64_t last = (uint64_t)(offset + length) & PAGE_MASK; - - if (first < last && - !filemap_range_has_page(inode->i_mapping, first, - last - 1)) - fuse_dlm_ranges_dropped(fi, first, last - 1); - } - } - fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); out: @@ -4367,24 +4329,9 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, if (err) goto out; - { - loff_t lstart = ALIGN_DOWN(pos_out, PAGE_SIZE); - loff_t lend = ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1; - - truncate_inode_pages_range(inode_out->i_mapping, lstart, lend); - - /* - * The record over the dropped span has nothing left to - * describe, and left DIRTY it would make the next partial - * write there keep and flush folio bytes nobody wrote. - * Only when the drop really emptied it: a folio a - * concurrent fault put back keeps its record. - */ - if (fc->dlm && fc->writeback_cache && - !filemap_range_has_page(inode_out->i_mapping, lstart, - lend)) - fuse_dlm_ranges_dropped(fi_out, lstart, lend); - } + truncate_inode_pages_range(inode_out->i_mapping, + ALIGN_DOWN(pos_out, PAGE_SIZE), + ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1); file_update_time(file_out); fuse_write_update_attr(inode_out, pos_out + outarg.size, outarg.size); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 00e5c5f8998420..5976b0f030a98b 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -2,46 +2,33 @@ /* * FUSE page lock cache implementation * - * cache->ranges records the grants the server has given this client. A - * grant still on the wire covers nothing and must not appear there, but - * a revoke has to be able to find it: otherwise a revoke processed - * before the grant is recorded removes nothing, and the grant recorded - * afterwards is never taken back. - * - * A range therefore carries enum fuse_dlm_range_state: - * - * - REQUESTED, on cache->pending, while its FUSE_DLM_WB_LOCK is in - * flight. - * - * - REVOKED, in either place. On cache->pending it is a request a - * revoke overlapped while it was in flight, and - * fuse_dlm_request_commit() drops that grant instead of recording it. - * In cache->ranges it is a grant that was recorded and has since been - * taken away, kept because the page cache under it is still - * described. It covers nothing either way, so the IO paths ask - * again. - * - * - READ or WRITE, in cache->ranges. The only states - * fuse_dlm_range_is_locked() reports as covered; the mode is not a - * separate field, since a range is either not held or held in one - * definite mode. - * - * In-flight requests are kept off the tree so the state is consulted - * only where a request is retired, not by every tree walker. - * - * The record says nothing about the page cache under a range. What is + * The shards record the grants the server has given this client, a bit + * per page. A grant still on the wire covers nothing and must not + * appear there, but a revoke has to be able to find it: otherwise a + * revoke processed before the grant is recorded removes nothing, and the + * grant recorded afterwards is never taken back. A request in flight + * therefore waits on cache->pending, where a revoke marks it killed and + * fuse_dlm_request_commit() drops the grant instead of recording it. + * + * Keeping requests out of the shards leaves every walker looking at + * grants alone. + * + * The record says nothing about the page cache under a page. What is * cached there, and whether the server has seen it, is what the page - * cache itself answers. + * cache itself answers. A revoked grant is therefore forgotten, not + * kept: writeback holds the range again for every run it sends, and an + * absent record and a revoked one both make it ask. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include #include -#include #include #include #include -#include +#include +#include /* @@ -51,81 +38,77 @@ */ #define FUSE_DLM_GRANT_RETRIES 16 -/* Lifecycle of a range; see the file comment above */ -enum fuse_dlm_range_state { - /* FUSE_DLM_WB_LOCK in flight, on cache->pending */ - FUSE_DLM_RANGE_REQUESTED, - /* - * On cache->pending, revoked in flight and the grant must not be - * recorded. In cache->ranges, granted once and taken away, kept to - * describe the page cache under it. Covers nothing either way. - */ - FUSE_DLM_RANGE_REVOKED, - /* Granted shared, in cache->ranges */ - FUSE_DLM_RANGE_READ, - /* Granted exclusive, in cache->ranges */ - FUSE_DLM_RANGE_WRITE, -}; +/* + * How far beyond the requested range a grant is recorded. + * + * A server may grant more than was asked for, and recording the extra is + * what lets the writes that follow skip the round trip entirely. But + * coverage is kept per shard, so the number of records a grant creates + * grows with its size, and outarg is the server's to choose: an + * unbounded grant would be an unbounded amount of work here. Cap it. + * Recording less than the server gave is safe - it only costs a + * re-request - and a cap this size still covers thousands of writes. + */ +#define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) -/* A range of pages with a lock */ +/* A FUSE_DLM_WB_LOCK request in flight, on cache->pending */ struct fuse_dlm_range { - /* Interval tree node; only linked once granted */ - struct rb_node rb; - /* - * The range, as byte offsets, both inclusive. Grants arrive page - * aligned, and a range is split only at the bounds of another, so - * these are page aligned too. - */ + /* The range asked for, as byte offsets, both inclusive */ uint64_t start; uint64_t end; - /* Subtree end value for interval tree */ - uint64_t __subtree_end; - /* Lifecycle and, once granted, the mode; see the enum above */ - enum fuse_dlm_range_state state; - /* Temporary list entry for operations, and the cache->pending link */ + /* A revoke overlapped this request in flight */ + bool killed; + /* The cache->pending link */ struct list_head list; }; -/* The state a grant in @mode is recorded under */ -static inline enum fuse_dlm_range_state -fuse_dlm_granted_state(enum fuse_page_lock_mode mode) +/* + * Bit of the page at @off within its shard. Callers pass page aligned + * bounds: a grant is aligned in __fuse_get_dlm_lock(), a revoke in + * fuse_dlm_revoke_inval_range() and a query in fuse_dlm_lock_is_held(). + */ +static unsigned long fuse_dlm_bit(uint64_t off) { - return mode == FUSE_PAGE_LOCK_READ ? FUSE_DLM_RANGE_READ : - FUSE_DLM_RANGE_WRITE; + return (off & (FUSE_DLM_SHARD_SIZE - 1)) >> PAGE_SHIFT; } /** - * fuse_dlm_state_satisfies - is a range in @held usable for @want - * @held: the state of a range recorded in the tree - * @want: FUSE_DLM_RANGE_READ or FUSE_DLM_RANGE_WRITE + * fuse_dlm_shard_get - the shard covering @off, created if there is none + * @cache: the page cache + * @off: byte offset the caller is about to record a grant at + * + * Only the recording path needs a shard to exist; readers and the revoke + * paths treat a missing one as a region holding no grant. * - * A WRITE grant is exclusive and so covers a READ request; nothing else - * substitutes for anything. The two pending states never appear in the - * tree and cover nothing. + * Return: the shard, or NULL if it could not be allocated. */ -static inline bool fuse_dlm_state_satisfies(enum fuse_dlm_range_state held, - enum fuse_dlm_range_state want) +static struct fuse_dlm_shard *fuse_dlm_shard_get(struct fuse_dlm_cache *cache, + uint64_t off) { - return held == want || - (held == FUSE_DLM_RANGE_WRITE && want == FUSE_DLM_RANGE_READ); -} + unsigned long idx = off >> FUSE_DLM_SHARD_SHIFT; + struct fuse_dlm_shard *shard, *old; -/* Interval tree definitions for page ranges */ -static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) -{ - return range->start; -} + shard = xa_load(&cache->shards, idx); + if (shard) + return shard; -static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) -{ - return range->end; -} + shard = kzalloc(sizeof(*shard), GFP_NOFS); + if (!shard) + return NULL; -INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); + /* + * Two recorders can reach the same empty region at once; the loser + * drops its shard and takes the winner's. + */ + old = xa_cmpxchg(&cache->shards, idx, NULL, shard, GFP_NOFS); + if (old) { + kfree(shard); + /* xa_cmpxchg() returns an errno as an internal entry */ + return xa_is_err(old) ? NULL : old; + } -static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off); + return shard; +} /** * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] @@ -140,296 +123,144 @@ static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off); * The nodes are owned by the threads waiting on their replies: mark * only, never remove or free. * - * Caller holds @cache->lock for write. + * Takes @cache->pending_lock. A request published after this returns is + * one whose FUSE_DLM_WB_LOCK had not been sent when the revoke was + * processed, so the grant it goes on to receive answers a request made + * after the revoke and is recorded, not killed. */ static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, uint64_t start, uint64_t end) { struct fuse_dlm_range *req; + spin_lock(&cache->pending_lock); list_for_each_entry(req, &cache->pending, list) if (req->start <= end && start <= req->end) - req->state = FUSE_DLM_RANGE_REVOKED; + req->killed = true; + spin_unlock(&cache->pending_lock); } /** - * fuse_page_cache_init - Initialize a page cache lock manager - * @cache: The cache to initialize - * - * Initialize a page cache lock manager for a FUSE inode. - * - * Return: 0 on success, negative error code on failure + * fuse_dlm_cache_init - Initialize a page cache lock manager + * @inode: The fuse inode to initialize the cache of */ -int fuse_dlm_cache_init(struct fuse_inode *inode) +void fuse_dlm_cache_init(struct fuse_inode *inode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - if (!cache) - return -EINVAL; - init_rwsem(&cache->lock); - cache->ranges = RB_ROOT_CACHED; + xa_init(&cache->shards); + spin_lock_init(&cache->pending_lock); INIT_LIST_HEAD(&cache->pending); - - return 0; } /** - * fuse_page_cache_destroy - Clean up a page cache lock manager - * @cache: The cache to clean up + * fuse_dlm_cache_release_locks - Clean up a page cache lock manager + * @inode: The fuse inode to clean up the cache of * * Release all locks and free all resources associated with the cache. */ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - struct rb_node *node; - - if (!cache) - return; + struct fuse_dlm_shard *shard; + unsigned long idx; - /* Release all locks */ + /* + * Coverage goes away here, so the whole cache is taken for write: + * see the locking comment on struct fuse_dlm_cache. + */ down_write(&cache->lock); /* * Every grant goes, so every request in flight is revoked. Mark * only; each node is owned by the thread waiting on its reply. */ fuse_dlm_kill_pending(cache, 0, U64_MAX); - while ((node = rb_first_cached(&cache->ranges)) != NULL) { - range = rb_entry(node, struct fuse_dlm_range, rb); - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } + + xa_for_each(&cache->shards, idx, shard) + kfree(shard); + /* + * Leaves the xarray empty and usable: an inode is released again + * on every O_TRUNC open, not only on eviction. + */ + xa_destroy(&cache->shards); up_write(&cache->lock); } /** - * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] - * @cache: The page cache - * @start: Start byte offset - * @end: End byte offset + * fuse_dlm_shard_record - record a grant over [@start, @end] in @shard + * @shard: the shard covering [@start, @end] + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * @mode: the mode it was granted in * - * Return: Pointer to the first overlapping range, or NULL if none found - */ -static struct fuse_dlm_range * -fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) -{ - return fuse_page_it_iter_first(&cache->ranges, start, end); -} - -/** - * fuse_page_try_merge - Try to merge ranges within a specific region - * @cache: The page cache - * @start: Start byte offset - * @end: End byte offset + * A write grant sets both maps, so a read query is answered by @granted + * alone, and a write grant over a page already held for read upgrades it + * by setting the bit the read grant left clear. * - * Attempt to merge ranges within and adjacent to the specified region - * that have the same lock mode. + * @granted is set before @write, so a query racing this sees the page + * covered for read before it sees it covered for write. Either order is + * safe, a bit not yet seen only costing a re-request of a range already + * held, but this one never reports a write grant the read map does not + * back. + * + * [@start, @end] must be page aligned and lie wholly inside @shard. + * Caller holds fuse_dlm_cache.lock for read. */ -static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static void fuse_dlm_shard_record(struct fuse_dlm_shard *shard, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - struct fuse_dlm_range *range, *next; - uint64_t first = start ? start - 1 : start; - uint64_t last = end < U64_MAX ? end + 1 : end; - - if (!cache) - return; + unsigned long bit = fuse_dlm_bit(start); + unsigned long last = fuse_dlm_bit(end); - /* - * Find the first range that might need merging. Directly adjacent - * ranges can merge, hence the region is widened by one unit to each - * side (saturating at the type bounds). This must stay an - * interval-tree lookup: the tree holds every cached grant of the - * inode and strided writers grow it for the lifetime of the file, - * so seeding the merge by walking from the tree minimum would make - * every new grant cost a full scan. - */ - range = fuse_page_it_iter_first(&cache->ranges, first, last); - - /* Try to merge ranges in and around the specified region */ - while (range && range->start <= last) { - /* Get next range before we potentially modify the tree */ - next = NULL; - if (rb_next(&range->rb)) { - next = rb_entry(rb_next(&range->rb), - struct fuse_dlm_range, rb); - } - - /* Merge neighbours the server has given us on the same terms */ - if (next && range->state == next->state && - range->end + 1 == next->start) { - /* Merge ranges: re-insert so __subtree_end is updated */ - fuse_page_it_remove(next, &cache->ranges); - fuse_page_it_remove(range, &cache->ranges); - range->end = next->end; - fuse_page_it_insert(range, &cache->ranges); - kfree(next); - - /* Continue with the same range */ - continue; - } - - /* Move to next range */ - range = next; + for (; bit <= last; bit++) { + set_bit(bit, shard->granted); + if (mode == FUSE_PAGE_LOCK_WRITE) + set_bit(bit, shard->write); } } /** - * fuse_dlm_lock_range_locked - Record a granted range of pages - * @inode: The fuse inode - * @start: Start byte offset - * @end: End byte offset - * @mode: Lock mode (read or write) - * - * Add a locked range on the specified range of pages. - * If parts of the range are already locked, only add the remaining parts. - * For overlapping ranges, handle lock compatibility: - * - READ locks are compatible with existing READ locks - * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * fuse_dlm_record_grant - record a grant across the shards it spans + * @cache: the page cache + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode it was granted in * - * Everything inserted here is READ or WRITE: this runs only after the - * server has answered. + * Each shard is filled in on its own: coverage only grows here, and a + * walker under @cache->lock held for read may see part of the grant + * before the rest, which makes it ask again for a range it already has + * rather than trust one it has not got. * - * Caller holds the cache lock for write. + * Caller holds @cache->lock for read. * - * Return: 0 on success, negative error code on failure + * Return: 0 on success, negative error code on failure. A failure part + * way through leaves the shards already done recorded, which under + * reports the grant and is safe. */ -static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, - enum fuse_page_lock_mode mode) +static int fuse_dlm_record_grant(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *new_range, *next; - enum fuse_dlm_range_state want; - bool covered_to_end = false; - int ret = 0; - LIST_HEAD(to_lock); - LIST_HEAD(to_upgrade); - uint64_t current_start = start; + unsigned long idx, last_idx; - if (!cache || start > end) + if (start > end) return -EINVAL; - /* The state this grant records */ - want = fuse_dlm_granted_state(mode); - - /* - * Ranges are upgraded whole below, so split at the grant bounds - * first: a range extending past the grant would otherwise have its - * uncovered part upgraded with it, recording coverage the server - * never gave. A read range half-covered by a write grant would - * report the other half held for write, and a revoked range - * half-regranted would report its still-revoked bytes as held, so - * writeback would send them without taking the range again. - */ - fuse_dlm_split_at(cache, start); - if (end < U64_MAX) - fuse_dlm_split_at(cache, end + 1); - - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - /* - * A revoked range is covered again by this grant, and a read - * range needs upgrading when a write is granted. - */ - if (range->state == FUSE_DLM_RANGE_REVOKED || - (want == FUSE_DLM_RANGE_WRITE && - range->state != FUSE_DLM_RANGE_WRITE)) - list_add_tail(&range->list, &to_upgrade); - /* If WRITE lock already exists - nothing to do */ - - /* If there's a gap before this range, we need to add the missing range */ - if (current_start < range->start) { - new_range = kmalloc(sizeof(*new_range), GFP_NOFS); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = range->start - 1; - new_range->state = want; - INIT_LIST_HEAD(&new_range->list); - - list_add_tail(&new_range->list, &to_lock); - } + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard; + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); - /* Move current_start past this range */ - if (range->end >= end) - covered_to_end = true; - else - current_start = max(current_start, range->end + 1); + shard = fuse_dlm_shard_get(cache, lo); + if (!shard) + return -ENOMEM; - /* Move to next range */ - range = next; + fuse_dlm_shard_record(shard, lo, hi, mode); } - /* If there's a gap after the last range to the end, extend the range */ - if (!covered_to_end && current_start <= end) { - new_range = kmalloc(sizeof(*new_range), GFP_NOFS); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = end; - new_range->state = want; - INIT_LIST_HEAD(&new_range->list); - - list_add_tail(&new_range->list, &to_lock); - } - - /* Everything on this list is now covered in @want */ - list_for_each_entry(range, &to_upgrade, list) - range->state = want; - - /* Add all new ranges to the tree */ - list_for_each_entry(new_range, &to_lock, list) { - /* Add to interval tree */ - fuse_page_it_insert(new_range, &cache->ranges); - } - - /* Try to merge adjacent ranges with the same mode */ - fuse_dlm_try_merge(cache, start, end); - return 0; - -out_free: - /* Free any ranges we allocated but didn't insert */ - while (!list_empty(&to_lock)) { - new_range = - list_first_entry(&to_lock, struct fuse_dlm_range, list); - list_del(&new_range->list); - kfree(new_range); - } - - /* - * Nothing to undo on @to_upgrade: every goto here is taken before - * the loop above runs, so no state has been changed yet. - */ - return ret; -} - -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) -{ - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - int ret; - - down_write(&cache->lock); - ret = fuse_dlm_lock_range_locked(inode, start, end, mode); - up_write(&cache->lock); - - return ret; } /** @@ -440,8 +271,8 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, * @end: end byte offset being requested (inclusive) * * The mode is not recorded here: until the server answers the range is - * held in neither, and the mode that reaches the tree is the one passed - * to fuse_dlm_request_commit(). + * held in neither, and the mode that reaches the shards is the one + * passed to fuse_dlm_request_commit(). * * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different * threads, so a revoke can be processed before the grant the reply @@ -449,11 +280,18 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, * that revoke a node to mark; without one it removes nothing, and the * grant recorded afterwards is never taken back by any later NOTIFY. * - * The request covers nothing while in flight, so it is kept off the - * tree. @req is reachable only through cache->pending, which both - * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink under - * the cache lock before the caller returns; stack storage is therefore - * fine and nothing is allocated here. + * The request covers nothing while in flight, so it is kept out of the + * shards. @req is reachable only through cache->pending, which both + * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink before + * the caller returns; stack storage is therefore fine and nothing is + * allocated here. + * + * Publishing touches the pending list and nothing else, so it takes + * @cache->pending_lock alone. It deliberately does not take + * @cache->lock: this runs on every cached write that is not already + * covered, and taking the cache rwsem for write here made every writer + * of a file queue behind every other one before its request had even + * been sent. */ void fuse_dlm_request_begin(struct fuse_inode *inode, struct fuse_dlm_range *req, uint64_t start, @@ -461,15 +299,13 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - RB_CLEAR_NODE(&req->rb); req->start = start; req->end = end; - req->state = FUSE_DLM_RANGE_REQUESTED; - /* Nothing reads this while the request is pending; publish it set */ + req->killed = false; - down_write(&cache->lock); + spin_lock(&cache->pending_lock); list_add_tail(&req->list, &cache->pending); - up_write(&cache->lock); + spin_unlock(&cache->pending_lock); } /** @@ -480,9 +316,10 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, * @end: end byte offset the server granted (inclusive) * @mode: the mode that was requested * - * Unlinking @req and recording the grant are one step under the cache - * lock, so a revoke lands either before it and is seen on @req, or after - * it and finds the grant in the tree. + * Unlinking @req and recording the grant are one step under + * @cache->lock held for read, so a revoke - which takes it for write - + * lands either before it and is seen on @req, or after it and finds the + * grant in the tree. * * @req is retired in every case and may be reused. * @@ -497,12 +334,21 @@ int fuse_dlm_request_commit(struct fuse_inode *inode, bool revoked; int ret = 0; - down_write(&cache->lock); + /* + * Read, not write: recording adds coverage. Holding it across the + * unlink and the record is what keeps a revoke from landing + * between them, since the revoke paths take it for write. + */ + down_read(&cache->lock); + + spin_lock(&cache->pending_lock); list_del(&req->list); - revoked = req->state == FUSE_DLM_RANGE_REVOKED; + revoked = req->killed; + spin_unlock(&cache->pending_lock); + if (!revoked) - ret = fuse_dlm_lock_range_locked(inode, start, end, mode); - up_write(&cache->lock); + ret = fuse_dlm_record_grant(cache, start, end, mode); + up_read(&cache->lock); return revoked ? -EAGAIN : ret; } @@ -519,99 +365,9 @@ void fuse_dlm_request_abort(struct fuse_inode *inode, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - down_write(&cache->lock); + spin_lock(&cache->pending_lock); list_del(&req->list); - up_write(&cache->lock); -} - -/** - * fuse_dlm_split_at - make @off start a range - * @cache: The page cache - * @off: byte offset to split at - * - * Splits the range containing @off in two, both halves keeping the state - * of the original, so a revoke can apply to one side only. A no-op when - * @off already starts a range or falls in a gap. - * - * Caller holds @cache->lock for write. - * - * Cannot fail: the split decides which bytes a caller goes on to name, - * and both naming more than was written and lowering more than was sent - * lose data. iomap allocates the state it keeps per folio the same way. - */ -static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off) -{ - struct fuse_dlm_range *range, *tail; - - if (!off) - return; - - range = fuse_page_it_iter_first(&cache->ranges, off, off); - if (!range || range->start == off) - return; - - tail = kmalloc(sizeof(*tail), GFP_NOFS | __GFP_NOFAIL); - - *tail = *range; - INIT_LIST_HEAD(&tail->list); - tail->start = off; - - /* - * Bounds are never edited in place: the interval tree caches a - * subtree end that only insertion recomputes. - */ - fuse_page_it_remove(range, &cache->ranges); - range->end = off - 1; - fuse_page_it_insert(range, &cache->ranges); - fuse_page_it_insert(tail, &cache->ranges); -} - -/** - * fuse_dlm_ranges_dropped - the page cache under [start, end] is gone - * @inode: the fuse inode - * @start: start byte offset (inclusive) - * @end: end byte offset (inclusive) - * - * A revoked range exists only to describe page cache dirtied before the - * grant was taken away. Once that cache is gone the range has nothing - * left to say and is freed; a range still held goes back to describing - * nothing. - * - * The caller must have established that the range really is empty, not - * merely asked for it to be dropped: a folio that survived an - * invalidate is still there, and claiming otherwise would let writeback - * send it with no record of where it came from. - */ -void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, - uint64_t end) -{ - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; - - if (start > end) - return; - - down_write(&cache->lock); - - fuse_dlm_split_at(cache, start); - if (end < U64_MAX) - fuse_dlm_split_at(cache, end + 1); - - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - next = fuse_page_it_iter_next(range, start, end); - - if (range->state == FUSE_DLM_RANGE_REVOKED) { - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } - - range = next; - } - - fuse_dlm_try_merge(cache, start, end); - - up_write(&cache->lock); + spin_unlock(&cache->pending_lock); } /** @@ -620,15 +376,11 @@ void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, * @start: Start byte offset * @end: End byte offset * - * The server has taken [start, end] back. A range that has nothing - * cached under it is removed; one that has is kept and marked - * FUSE_DLM_RANGE_REVOKED, so the page cache it covers stays described. - * Removing it instead would leave a gap, and a gap reads as "no record", - * which is what an untracked range looks like: writeback would then send - * folios dirtied under the grant that was just taken away. - * - * A revoked range covers nothing, so fuse_dlm_range_is_locked() reports - * it uncovered and the IO paths request again. + * The server has taken [start, end] back, so the grants over it are + * removed and the IO paths ask again. Page cache dirtied under a grant + * that has gone is not lost by this: writeback takes the range again for + * every run it sends, and a range it finds unrecorded is a range it asks + * for. * * An inverted range is rejected rather than silently revoking nothing: * the callers revoke coverage, and a revoke that quietly keeps the grant @@ -638,62 +390,86 @@ void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_unlock_range(struct fuse_inode *inode, - uint64_t start, uint64_t end) +int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; + struct fuse_dlm_shard *shard; + unsigned long idx; - if (!cache || start > end) + if (start > end) return -EINVAL; + /* + * Write, not read: this is a path that takes coverage away, and + * the shard walkers rely on that never happening under them. See + * the locking comment on struct fuse_dlm_cache. + */ down_write(&cache->lock); /* - * Before touching the tree, and even when nothing in the tree - * overlaps: a revoke racing an in-flight grant finds no overlap - * because that grant is not recorded yet. + * Before touching any map, and even when nothing is covered: a + * revoke racing an in-flight grant finds nothing set, because that + * grant is not recorded yet. */ fuse_dlm_kill_pending(cache, start, end); - /* Split so the revoked region has its own ranges */ - fuse_dlm_split_at(cache, start); - if (end < U64_MAX) - fuse_dlm_split_at(cache, end + 1); + /* + * Only the regions that hold something, not every index in the + * range: a revoke to EOF runs to U64_MAX and walking that index + * by index would never finish. + */ + xa_for_each_range(&cache->shards, idx, shard, + start >> FUSE_DLM_SHARD_SHIFT, + end >> FUSE_DLM_SHARD_SHIFT) { + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + unsigned long first = fuse_dlm_bit(lo); + unsigned long nbits = fuse_dlm_bit(hi) - first + 1; - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we modify the tree */ - next = fuse_page_it_iter_next(range, start, end); + /* + * Plain, not atomic: @cache->lock is held for write, so no + * reader and no recorder can be looking at these words. + */ + bitmap_clear(shard->granted, first, nbits); + bitmap_clear(shard->write, first, nbits); /* - * A revoked range is kept only to say that the page cache - * under it was dirtied under a grant that has gone, so that - * writeback takes the range again before sending it. With - * nothing cached there it has nothing to say. - * - * A grant with no end is recorded to U64_MAX; the page cache - * is indexed by a signed offset, so ask it about as much of - * that as it can name. + * Keep the shard table to the regions that hold something: + * a file revoked a region at a time would otherwise leave an + * empty shard behind for every one of them. */ - if (filemap_range_has_page(inode->inode.i_mapping, range->start, - min_t(uint64_t, range->end, - LLONG_MAX))) { - range->state = FUSE_DLM_RANGE_REVOKED; - } else { - fuse_page_it_remove(range, &cache->ranges); - kfree(range); + if (bitmap_empty(shard->granted, FUSE_DLM_SHARD_PAGES)) { + xa_erase(&cache->shards, idx); + kfree(shard); } - - range = next; } - fuse_dlm_try_merge(cache, start, end); - up_write(&cache->lock); return 0; } +/* + * Is every page of [@from, @to] covered in @mode? Both bounds are page + * aligned and lie inside @shard. A write grant sets both maps, so a + * read request is answered by @granted alone. + * + * No lock of its own: bits are set atomically and cleared only under + * fuse_dlm_cache.lock held for write, which the caller holds for read. + * A bit set concurrently may be missed, which costs a re-request of a + * range already held. + */ +static bool fuse_dlm_shard_covers(struct fuse_dlm_shard *shard, uint64_t from, + uint64_t to, enum fuse_page_lock_mode mode) +{ + const unsigned long *map = mode == FUSE_PAGE_LOCK_WRITE ? + shard->write : shard->granted; + unsigned long first = fuse_dlm_bit(from); + unsigned long last = fuse_dlm_bit(to); + + return find_next_zero_bit(map, last + 1, first) > last; +} + /** * fuse_dlm_range_is_locked - Check if a byte range is already locked * @inode: The fuse inode @@ -703,106 +479,44 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, * * Return: true if the entire range is locked, false otherwise */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +static bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, + enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - enum fuse_dlm_range_state want; - uint64_t current_start = start; + unsigned long idx, last_idx; + bool covered = true; - if (!cache || start > end) + if (start > end) return false; - /* The state a range has to be in to cover this request */ - want = fuse_dlm_granted_state(mode); - + /* + * Read: coverage is only ever removed under @cache->lock held for + * write, so the set of grants can grow under this walk but never + * shrink. That is what lets the shards be visited one at a time, + * and their maps read without any further lock. The worst a + * concurrent recorder can do is make this report a range uncovered + * that has just become covered, and the caller then asks for a + * grant it already holds. + */ down_read(&cache->lock); - /* Find the first range that overlaps with [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - - /* Check if the entire range is covered */ - while (range && current_start <= end) { - /* - * The held lock must be at least as strong as the one - * requested. A WRITE lock (exclusive) satisfies a READ - * request, so only treat the range as uncovered when the - * held mode is weaker than what we ask for. This avoids - * re-requesting a READ lock for a range we already hold - * a WRITE lock on (e.g. read-after-write). - */ - if (!fuse_dlm_state_satisfies(range->state, want)) { - /* Held lock is weaker than requested */ - up_read(&cache->lock); - return false; - } + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - /* Check if there's a gap before this range */ - if (current_start < range->start) { - /* Found a gap */ - up_read(&cache->lock); - return false; - } - - /* Covered through the end of the requested range? */ - if (range->end >= end) { - up_read(&cache->lock); - return true; - } + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard = xa_load(&cache->shards, idx); + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); - /* Move current_start past this range */ - current_start = range->end + 1; - - /* Get next overlapping range */ - range = fuse_page_it_iter_next(range, start, end); - } - - /* Check if we covered the entire range */ - if (current_start <= end) { - /* There's a gap at the end */ - up_read(&cache->lock); - return false; - } - - up_read(&cache->lock); - return true; -} - -/** - * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere - * @fi: the fuse inode - * - * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully - * covered, this asks whether any part of the file is held exclusively. A - * client that holds a write grant may be sitting on dirty page cache the - * server has not seen, so its mtime and ctime run ahead of anything the - * server can report. - * - * A revoked range is not counted. The state does not keep the mode the - * range was granted in, so a revoked one cannot be told from a read that - * was taken away, and the dirty page cache such a range describes is - * what the caller checks the mapping for instead. - * - * Return: true if at least one recorded range is held for write - */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) -{ - struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; - struct fuse_dlm_range *range; - bool held = false; - - down_read(&cache->lock); - for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; - range = fuse_page_it_iter_next(range, 0, U64_MAX)) { - if (range->state == FUSE_DLM_RANGE_WRITE) { - held = true; + if (!shard || !fuse_dlm_shard_covers(shard, lo, hi, mode)) { + covered = false; break; } } + up_read(&cache->lock); - return held; + return covered; } /** @@ -859,6 +573,7 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, struct fuse_dlm_lock_out outarg; struct fuse_dlm_range req; uint64_t pg_start, pg_end; + uint64_t grant_start, grant_end; int tries = FUSE_DLM_GRANT_RETRIES; int err; @@ -891,6 +606,7 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, } memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); inarg.fh = ff->fh; inarg.start = pg_start; @@ -920,10 +636,10 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, } /* * The range is contended, the same answer a READ gets and - * fuse_do_readfolio() turns into AOP_TRUNCATED_PAGE for its + * fuse_do_readpage() turns into AOP_TRUNCATED_PAGE for its * caller to retry. There is no such convention here, and - * the writeback caller can only redirty the folio it is - * holding, so ask again instead of reporting it. + * the writeback caller loses the folio it is holding on an + * error, so ask again instead of reporting it. */ if (err == -EDEADLK || err == -EAGAIN) goto retry; @@ -940,16 +656,37 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, } /* - * Retire the request and record the grant. A server may grant more - * than was asked for, and recording all of it is what lets later IO - * over the same neighbourhood skip the round trip. - * - * fuse_dlm_kill_pending() matches a request in flight on the bounds - * published to it, so a revoke aimed only at the part beyond them - * marks nothing here. It would have to be a revoke of a sub-range - * the server is granting in the same breath. + * Keep the recorded grant to a bounded distance either side of + * what was asked for; see FUSE_DLM_MAX_EXTRA_GRANT. Both bounds + * stay outside [pg_start, pg_end], so the range this call has to + * cover is still covered. */ - err = fuse_dlm_request_commit(fi, &req, outarg.start, outarg.end, mode); + grant_start = outarg.start; + grant_end = outarg.end; + /* + * Both differences are safe: the check above established + * outarg.start <= pg_start <= pg_end <= outarg.end, and a branch + * is only taken when there is more than the cap to give back, so + * neither adjusted bound can wrap. + */ + if (pg_start - grant_start > FUSE_DLM_MAX_EXTRA_GRANT) + grant_start = pg_start - FUSE_DLM_MAX_EXTRA_GRANT; + if (grant_end - pg_end > FUSE_DLM_MAX_EXTRA_GRANT) + grant_end = pg_end + FUSE_DLM_MAX_EXTRA_GRANT; + + /* + * Align inward. A page is covered only when it is covered whole, + * and the bit helpers take that as given; the bounds themselves are + * the server's to choose, only their superset property having been + * checked. Rounding inward cannot uncover [pg_start, pg_end], + * which is page aligned already. + */ + grant_start = ALIGN(grant_start, PAGE_SIZE); + /* A last-byte offset, so it is the successor that aligns */ + grant_end -= (grant_end + 1) & (PAGE_SIZE - 1); + + /* Retire the request and record the grant */ + err = fuse_dlm_request_commit(fi, &req, grant_start, grant_end, mode); if (err == -EAGAIN) { /* * A revoke overlapping this range was processed while the diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index d1078cd84546d5..c54cad9d000226 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -7,9 +7,12 @@ #define _FS_FUSE_DLM_CACHE_H #include -#include +#include #include +#include #include +#include +#include struct fuse_inode; @@ -28,38 +31,97 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; */ #define FUSE_DLM_GRANT_UNRECORDED 1 +/* + * Coverage is kept per aligned region of the file rather than in one + * structure for the whole inode, because one structure needs one lock + * and every thread writing the file then serialises on it however far + * apart their ranges are. The region wants to be small enough that + * concurrent writers land in different ones and large enough that the + * per-region overhead stays amortised. + */ +#define FUSE_DLM_SHARD_SHIFT 24 +#define FUSE_DLM_SHARD_SIZE (1ULL << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_PAGES (FUSE_DLM_SHARD_SIZE / PAGE_SIZE) + +/* First and last byte offset (both inclusive) covered by shard @idx */ +#define FUSE_DLM_SHARD_FIRST(idx) ((uint64_t)(idx) << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_LAST(idx) (FUSE_DLM_SHARD_FIRST(idx) + \ + FUSE_DLM_SHARD_SIZE - 1) + +/* + * The grants over one region, a bit per page. + * + * @granted says the page is covered, @write that it is covered for + * write. A write grant sets both, so a read request is answered by + * @granted alone and nothing has to compare modes; @write is a subset of + * @granted. + * + * A region is a fixed span of pages, so the maps are a fixed size and + * recording a grant neither allocates nor rearranges anything: adjacent + * grants coalesce because they set neighbouring bits, and a region + * fragmented to the last page costs no more than the two maps it already + * has. + * + * Bits are set with the atomic helpers, since recorders run concurrently + * under fuse_dlm_cache.lock held for read. They are cleared only under + * that lock held for write, which excludes every reader and every + * recorder, so the revoke path uses the plain bulk helpers and a query + * needs no lock of its own. + */ +struct fuse_dlm_shard { + unsigned long granted[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; + unsigned long write[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; +}; + /* * Page cache lock manager. * - * @ranges holds the grants the client has been given, and the ones it - * has had taken back that still describe page cache - * (FUSE_DLM_RANGE_REVOKED). A request still on the wire covers nothing - * and lives on @pending instead, so tree walkers never filter on state. - * See enum fuse_dlm_range_state in fuse_dlm_cache.c. + * The shards hold the grants the client has been given. A request still + * on the wire covers nothing and lives on @pending instead, so the + * shards answer for grants only. See struct fuse_dlm_range in + * fuse_dlm_cache.c. + * + * Locking, outermost first: + * + * @lock rw_semaphore over the whole cache. Taken for read by + * everything that adds or reads coverage, and for write + * only by the paths that take coverage away + * (fuse_dlm_unlock_range, fuse_dlm_cache_release_locks). + * That is the invariant the shard walks rely on: + * **coverage is only ever removed under @lock held for + * write**, so a walker holding it for read sees a set of + * grants that can grow under it but never shrink, and may + * therefore visit shards one at a time and read their + * maps without any further lock. Shards are freed only + * there too, so a shard pointer stays good for as long as + * the read side is held. + * @pending_lock the pending list. Innermost, and the only lock + * fuse_dlm_request_begin() and fuse_dlm_request_abort() + * take at all. */ struct fuse_dlm_cache { - /* Lock protecting the tree and the pending list */ + /* See the locking comment above */ struct rw_semaphore lock; - /* Interval tree of recorded ranges, granted or revoked */ - struct rb_root_cached ranges; /* - * FUSE_DLM_WB_LOCK requests in flight (REQUESTED, or REVOKED once - * a revoke has overlapped one). Owned by the queueing thread; the - * revoke paths mark them only. + * struct fuse_dlm_shard by offset >> FUSE_DLM_SHARD_SHIFT, + * allocated when a region first holds a grant. + */ + struct xarray shards; + /* Protects @pending and the killed flag of everything on it */ + spinlock_t pending_lock; + /* + * FUSE_DLM_WB_LOCK requests in flight. Owned by the queueing + * thread; the revoke paths only mark them killed. */ struct list_head pending; }; /* Initialize a page cache lock manager */ -int fuse_dlm_cache_init(struct fuse_inode *inode); +void fuse_dlm_cache_init(struct fuse_inode *inode); /* Clean up a page cache lock manager */ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); -/* Lock a range of pages */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); - /* * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a * revoke processed while the reply is on the wire can mark it. @req is @@ -88,28 +150,15 @@ void fuse_dlm_request_abort(struct fuse_inode *inode, int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); -/* Check if a page range is already locked */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); - /* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); -/* Is any part of the file held for write? */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); - -/* - * The page cache under [start, end] is gone: free the revoked ranges over - * it. The caller must have established the range really is empty. - */ -void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, - uint64_t end); - /* Hold [start, end] again so writeback can send what it found revoked */ int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, uint64_t start, uint64_t end); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, size_t length, enum fuse_page_lock_mode mode); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 87ea35f48377e3..194ef3c5935ac1 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1311,7 +1311,7 @@ void fuse_file_free(struct fuse_file *ff); int fuse_finish_open(struct inode *inode, struct file *file); /* Drop the page cache an open must not keep; true if it came out empty */ -bool fuse_open_drop_cache(struct inode *inode); +void fuse_open_drop_cache(struct inode *inode); void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, unsigned int flags); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 09671882ac97a6..42a9aed126dd44 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -498,14 +498,14 @@ u32 fuse_get_cache_mask(struct inode *inode) * for exactly what the grant covers: * * - size, when the server reports less than i_size and the tail it does not - * know about, [srv_size, i_size), is entirely under a write grant. Taking - * the server's answer would shrink i_size and have truncate_pagecache() - * throw the unwritten tail away. - * - mtime and ctime, while a write grant covers unwritten data: our writes - * have stamped them locally and the server's stamps predate them. Only - * while the cache is actually dirty, not for as long as the grant lives: - * a grant is held until it is revoked or the inode is evicted, and past - * the writeback the server's stamps are the newer ones. Keeping ours + * know about, [attr->size, i_size), is entirely under a write grant. + * Taking the server's answer would shrink i_size and have + * truncate_pagecache() throw the unwritten tail away. + * - mtime and ctime, while the page cache is dirty or under writeback: our + * writes have stamped them locally and the server's stamps predate them. + * Only while the cache is actually dirty, not for as long as a grant + * lives: a grant is held until it is revoked or the inode is evicted, and + * past the writeback the server's stamps are the newer ones. Keeping ours * beyond that would hide a remote chown or chmod indefinitely. * * A remote truncate cannot slip through. It has to revoke the grant first, @@ -536,11 +536,6 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, * the tail grants itself while cached writes above the new size * are still waiting for writeback. */ - if (!fuse_dlm_write_grant_exists(fi) && - !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) && - !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) - return cache_mask; - if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) cache_mask |= STATX_MTIME | STATX_CTIME; @@ -646,27 +641,6 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr if (inval) invalidate_inode_pages2(inode->i_mapping); - - /* - * The DLM record has to follow the cache out, as on every - * other path that drops it. A revoked range exists only to - * make writeback take the grant again before sending the - * folios under it, so once they are gone it describes - * nothing and would sit in the tree unfreed. The pages above - * the new size are gone unconditionally; the rest only when - * the invalidate really emptied the mapping, so a folio that - * survived (or was faulted back) keeps its record. - */ - if (fc->dlm && fc->writeback_cache) { - if (have_size && oldsize != attr->size) - fuse_dlm_ranges_dropped(fi, - PAGE_ALIGN(attr->size), - U64_MAX); - if (inval && - !filemap_range_has_page(inode->i_mapping, 0, - LLONG_MAX)) - fuse_dlm_ranges_dropped(fi, 0, U64_MAX); - } } if (IS_ENABLED(CONFIG_FUSE_DAX)) @@ -968,8 +942,6 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, { struct fuse_inode *fi; struct inode *inode; - uint64_t pg_first; - uint64_t pg_last; loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; @@ -1014,17 +986,8 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; - /* - * Byte bounds of the same region, and the page aligned form - * the DLM record is told about. A grant is recorded page - * aligned, so the range handed to it has to cover whole - * pages or the record would keep a range the page cache no - * longer backs. - */ + /* Byte bounds of the same region */ end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; - pg_first = (uint64_t)offset & PAGE_MASK; - pg_last = len <= 0 ? U64_MAX : - (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); /* * A data invalidation means another (remote) entity is modifying @@ -1138,18 +1101,6 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pg_end, may_be_dirty); - /* - * A revoked range exists to describe page cache - * dirtied before the grant went; with that cache gone - * it has nothing left to say. Only when it really - * went: an invalidate can leave a busy folio behind, - * and that folio still needs its record. - */ - if (has_pages && - !filemap_range_has_page(inode->i_mapping, offset, - end_byte)) - fuse_dlm_ranges_dropped(fi, pg_first, pg_last); - if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); From ce821f7ad4dd857c952a5a116a6d69c92785b08c Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:08:12 +0200 Subject: [PATCH 02/24] fuse: keep a folio writeback could not send write_cache_pages() takes the dirty flag off a folio before it calls ->writepage, and fuse_launder_folio() does the same for the submit it makes itself, so a folio that comes back unsent has thrown its bytes away unless they are put back. Every path here already puts them back, but none of them asks whether there is still a connection to send them on: after an abort every send fails, and a folio redirtied for a retry that can no longer happen keeps sync() going forever. Put the redirty in one helper and make it ask. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 69a09bd12f4a7c..6cd28dd0f1b894 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3021,6 +3021,30 @@ static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio return wpa; } +/* + * Put a folio writeback could not send back on the dirty list. + * + * write_cache_pages() takes the dirty flag off a folio before it calls + * here, and fuse_launder_folio() does the same for the submit it makes + * itself, so a folio that comes back unsent has thrown its bytes away + * unless they are put back. @wbc is NULL for the launder. + * + * Only while there is a connection left to take the bytes. After an abort + * every send fails, and a folio redirtied for a retry that can no longer + * happen would keep sync() going forever. + */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct writeback_control *wbc, + struct folio *folio) +{ + if (!READ_ONCE(fc->connected)) + return; + + folio_mark_dirty(folio); + if (wbc) + wbc->pages_skipped += folio_nr_pages(folio); +} + static int fuse_writepage_locked(struct folio *folio) { struct address_space *mapping = folio->mapping; @@ -3075,7 +3099,7 @@ static int fuse_writepage_locked(struct folio *folio) * that laundered it then reports the folio busy, as it does for any * folio it cannot free. */ - filemap_dirty_folio(mapping, folio); + fuse_writeback_redirty(fc, NULL, folio); mapping_set_error(mapping, error); return error; } @@ -3186,7 +3210,7 @@ static int fuse_writepages_fill(struct folio *folio, * flag off before calling here, so put it back * rather than drop a write fsync reported done. */ - folio_redirty_for_writepage(wbc, folio); + fuse_writeback_redirty(fc, wbc, folio); goto out_unlock; } } @@ -3225,7 +3249,7 @@ static int fuse_writepages_fill(struct folio *folio, !fuse_dlm_lock_is_held(fi, folio_pos(folio), folio_size(folio), FUSE_PAGE_LOCK_WRITE)) { - folio_redirty_for_writepage(wbc, folio); + fuse_writeback_redirty(fc, wbc, folio); err = 0; goto out_unlock; } @@ -3234,7 +3258,7 @@ static int fuse_writepages_fill(struct folio *folio, folio_pos(folio) + folio_size(folio) - 1); if (err < 0 && err != -ENOSYS) { - folio_redirty_for_writepage(wbc, folio); + fuse_writeback_redirty(fc, wbc, folio); goto out_unlock; } err = 0; @@ -3249,7 +3273,7 @@ static int fuse_writepages_fill(struct folio *folio, err = -ENOMEM; wpa = fuse_writepage_args_setup(folio, data->ff); if (!wpa) { - folio_redirty_for_writepage(wbc, folio); + fuse_writeback_redirty(fc, wbc, folio); goto out_unlock; } fuse_file_get(wpa->ia.ff); From a5c4d268e162aac79873f88fbbcdda917f7b11fe Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:09:40 +0200 Subject: [PATCH 03/24] fuse: keep the local size against an attribute reply already on the wire fuse_attr_cache_mask() decides whether the server's size wins from an i_size it reads on entry, and then sleeps in the grant query before fuse_change_attributes_i() applies the answer under fi->lock. A GETATTR that left while i_size still matched the server's is therefore applied over every extension made since, and i_size drops by exactly the writes in flight. truncate_pagecache() then zeroes the tail of the page holding the new size and drops what is above, which writeback sends as zeros. Move attr_version where fuse_write_end() commits the extension so those replies are dropped, the same reason fuse_write_update_attr() moves it, and count the extension until the folio under it is dirty, for a reply that leaves after it. FUSE_I_SIZE_UNSTABLE cannot serve as the count: it is a single bit and every writer clears it. [hbi: adapted -- the shared-lock write here commits i_size in fuse_write_end() behind the write cursor rather than claiming the whole extension up front, so the count is held across that commit instead of across the write loop.] Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 33 ++++++++++++++++++++++++++++++--- fs/fuse/fuse_i.h | 15 +++++++++++++++ fs/fuse/inode.c | 9 ++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 6cd28dd0f1b894..5770317cdcaca6 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3399,6 +3399,9 @@ static int fuse_write_end(struct file *file, struct address_space *mapping, struct folio *folio, void *fsdata) { struct inode *inode = folio->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + bool extending; /* Haven't copied anything? Skip zeroing, size extending, dirtying. */ if (!copied) @@ -3426,18 +3429,41 @@ static int fuse_write_end(struct file *file, struct address_space *mapping, * beyond-EOF optimisation effective: folios wholly past EOF are zeroed * locally instead of sending the server a read-modify-write READ for * data that does not exist yet. + * + * Count the extension until the folio under it is dirty: in between, + * [old size, pos) is covered by nothing fuse_attr_cache_mask() can + * see, and a reply that leaves in that window would shrink i_size + * back. With i_rwsem held shared several writers sit there at once, + * which is why they are counted rather than flagged. */ - if (pos > inode->i_size) { - struct fuse_inode *fi = get_fuse_inode(inode); + extending = pos > inode->i_size; + if (extending) { + atomic_inc(&fi->size_extenders); spin_lock(&fi->lock); - if (pos > inode->i_size) + if (pos > inode->i_size) { + /* + * Retire the attribute replies already on the wire. + * fuse_attr_cache_mask() decides whether the server's + * size wins from an i_size it read before this commit + * and before it slept in the grant query, so a GETATTR + * that left while i_size still matched the server's is + * applied afterwards and shrinks it back. Moving + * attr_version makes fuse_change_attributes_i() drop + * those replies, which is what fuse_write_update_attr() + * moves it for. + */ + fi->attr_version = atomic64_inc_return(&fc->attr_version); i_size_write(inode, pos); + } spin_unlock(&fi->lock); } folio_mark_dirty(folio); + if (extending) + atomic_dec(&fi->size_extenders); + unlock: folio_unlock(folio); folio_put(folio); @@ -4454,6 +4480,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->write_stream_run = 0; fi->write_stream_next = 0; fi->write_stream_start = 0; + atomic_set(&fi->size_extenders, 0); if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 194ef3c5935ac1..578751035a71c1 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -274,6 +274,21 @@ struct fuse_inode { unsigned int write_stream_run; loff_t write_stream_next; loff_t write_stream_start; + + /* + * Buffered writes that have committed an i_size + * extension and not yet dirtied the folio under + * it. + * + * The DLM path holds i_rwsem shared, so several + * writers extend i_size at once and each one is + * ahead of the server until its bytes are sent. + * While this is non zero the local size wins over + * the server's; see fuse_attr_cache_mask(). + * FUSE_I_SIZE_UNSTABLE cannot serve: it is one bit + * and every writer clears it. + */ + atomic_t size_extenders; }; /* readdir cache (directory only) */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 42a9aed126dd44..1d4c4da481582d 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -548,9 +548,16 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, * truncate them away before they are ever sent. The grant check * alone misses them, because a page-mkwrite grant is never * recorded and a local truncate revokes its own tail grants. + * + * Both miss a write that has committed its extension and not yet + * dirtied the folio under it: nothing is dirty there, and a NOTIFY + * can revoke the grant in between. With i_rwsem held shared several + * writers sit in that window at once, which is why they are counted + * rather than flagged. */ if (have_size && size > (loff_t) attr->size && - (fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + (atomic_read(&fi->size_extenders) || + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, FUSE_PAGE_LOCK_WRITE) || filemap_range_needs_writeback(inode->i_mapping, attr->size, size - 1))) From 96593ba996ab802701f707af86d394d789c6ad13 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:11:52 +0200 Subject: [PATCH 04/24] fuse: take the writeback grant with no folio held fuse_writepages_fill() runs with the folio locked and, once folio_start_writeback() has run for an earlier folio of the same pass, with those under writeback too. Asking the server for a grant there is the ordering Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE- reason.txt exists to forbid: no cluster lock may be taken while a page lock is held. The read path has AOP_TRUNCATED_PAGE to unlock and retry with, and ->writepage has nothing of the sort, so the violation sits where the remedy does not reach. Until now only a writeback driven by a revoke handler skipped, and only because that one deadlocks outright. Skip any folio whose grant has gone: put it back on the dirty list and remember the range. fuse_writepages() takes it back once the pass has let go of every folio, and the pass that follows sends it. A data integrity writeback has no later pass, so it goes round again while anything is left deferred; fsync() and close() would otherwise report bytes written that are still only in the page cache. Not from fuse_launder_folio(), which arrives with the folio locked by folio_unmap_invalidate(), nor from a revoke handler, which would ask for the range it is revoking. There the skip simply stands. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 133 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 106 insertions(+), 27 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5770317cdcaca6..1bf746549cf297 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3062,8 +3062,22 @@ static int fuse_writepage_locked(struct folio *folio) /* * Hold the range again before sending it; see fuse_writepages_fill(). + * + * folio_unmap_invalidate() holds the folio locked here, so a grant + * this folio does not already hold must not be asked for and there is + * no later pass of this submit to defer it to. Leave the folio dirty + * and report no error: the invalidate that laundered it then finds it + * busy, and an ordinary writeback sends it with a grant of its own. */ if (fc->dlm && fc->writeback_cache) { + if (!fuse_dlm_lock_is_held(fi, folio_pos(folio), + folio_size(folio), + FUSE_PAGE_LOCK_WRITE)) { + fuse_file_put(ff, false); + fuse_writeback_redirty(fc, NULL, folio); + return 0; + } + error = fuse_dlm_regrant_range(ff, inode, folio_pos(folio), folio_pos(folio) + folio_size(folio) - 1); @@ -3104,11 +3118,25 @@ static int fuse_writepage_locked(struct folio *folio) return error; } +/* + * How many times a data integrity writeback goes round for folios it had to + * skip. Each pass takes the grants the one before it deferred, so one more + * is normally enough; the cap is there because a revoke can take them again. + */ +#define FUSE_WB_DEFER_PASSES 4 + struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; struct inode *inode; unsigned int max_pages; + /* + * The folios this pass could not send because their grant had gone. + * Taken back in fuse_writepages(), where no folio is held, for the + * pass that follows; see fuse_writepages_fill(). + */ + u64 regrant_start; + u64 regrant_end; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data) @@ -3232,31 +3260,50 @@ static int fuse_writepages_fill(struct folio *folio, * -ENOSYS, which is not a failure. */ if (fc->dlm && fc->writeback_cache) { + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + /* - * Driven by a NOTIFY invalidate. A grant this folio does not - * already hold would have to be asked for from inside the - * handler the server is waiting on, for the range that - * handler is revoking, with the folio locked and under - * writeback. The server cannot answer that until the revoke - * completes, and the revoke cannot complete until this - * returns. + * The folio is locked here, and the folios queued before it + * in this pass are under writeback, so a grant this folio + * does not already hold must not be asked for: + * Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE- + * reason.txt states the rule the read path is built around, + * that no cluster lock may be taken while a page lock is + * held. fuse_do_readpage() has AOP_TRUNCATED_PAGE to unlock + * and retry with; ->writepage has nothing of the sort. * - * Leave the folio dirty instead. An ordinary writeback sends - * it with a grant of its own; nothing is lost and no error is - * recorded for a later fsync to report. + * Skip the folio instead. It goes back on the dirty list, + * the range is remembered for fuse_writepages() to take back + * with no folio held, and the pass that follows sends it. + * Nothing is lost and no error is recorded for a later fsync + * to report. */ - if (fuse_in_notify_ctx() && - !fuse_dlm_lock_is_held(fi, folio_pos(folio), - folio_size(folio), + if (!fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) { fuse_writeback_redirty(fc, wbc, folio); + if (data->regrant_end <= data->regrant_start) { + data->regrant_start = pos; + data->regrant_end = pos + len; + } else { + data->regrant_start = min_t(u64, + data->regrant_start, + pos); + data->regrant_end = max_t(u64, + data->regrant_end, + pos + len); + } err = 0; goto out_unlock; } - err = fuse_dlm_regrant_range(data->ff, inode, folio_pos(folio), - folio_pos(folio) + - folio_size(folio) - 1); + /* + * Held: this walks the record and sends nothing. It stays a + * call rather than the check above so a grant that arrives + * between them is still used. + */ + err = fuse_dlm_regrant_range(data->ff, inode, pos, + pos + len - 1); if (err < 0 && err != -ENOSYS) { fuse_writeback_redirty(fc, wbc, folio); goto out_unlock; @@ -3303,7 +3350,7 @@ static int fuse_writepages(struct address_space *mapping, { struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_fill_wb_data data; + unsigned int tries = FUSE_WB_DEFER_PASSES; int err; err = -EIO; @@ -3314,17 +3361,49 @@ static int fuse_writepages(struct address_space *mapping, fc->num_background >= fc->congestion_threshold) return 0; - data.inode = inode; - data.wpa = NULL; - data.ff = NULL; + /* + * A folio whose grant had gone is skipped and its range taken back + * below, which leaves the folio dirty for a later pass. For a data + * integrity writeback there is no later pass: fsync() and close() + * would report the bytes written while they are still only in the + * page cache. Go round again, now that the grant is held, until + * nothing is left deferred. + */ + do { + struct fuse_fill_wb_data data = { .inode = inode }; + bool regranted = false; + + err = write_cache_pages(mapping, wbc, fuse_writepages_fill, + &data); + if (data.wpa) { + WARN_ON(!data.wpa->ia.ap.num_pages); + fuse_writepages_send(&data); + } - err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); - if (data.wpa) { - WARN_ON(!data.wpa->ia.ap.num_pages); - fuse_writepages_send(&data); - } - if (data.ff) - fuse_file_put(data.ff, false); + /* + * Take back what the folios above had to skip, so the pass + * that follows finds the grant and sends the folios they left + * dirty. With no folio held, which is the whole point. + * + * Not from a revoke handler: it would ask for the very range + * it is revoking, so the folios it skipped stay dirty for an + * ordinary writeback, and every pass here would skip them + * again. + */ + if (data.ff && data.regrant_end > data.regrant_start && + !fuse_in_notify_ctx()) { + fuse_dlm_regrant_range(data.ff, inode, + data.regrant_start, + data.regrant_end - 1); + regranted = true; + } + + if (data.ff) + fuse_file_put(data.ff, false); + + if (err || wbc->sync_mode != WB_SYNC_ALL || !regranted) + break; + } while (--tries); out: return err; From 72940490f7c8f28a15a8b76c91d5e480b19d7bb4 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:12:45 +0200 Subject: [PATCH 05/24] fuse: do not record a grant past what a revoke could mark fuse_dlm_request_begin() publishes the range asked for, and fuse_dlm_kill_pending() can only test that one, but the commit records what the server granted, which may reach FUSE_DLM_MAX_EXTRA_GRANT either side of it. A revoke processed while the request was on the wire and landing in that excess marks nothing: the request does not overlap it, and the shards hold no bit for it yet because the grant is not recorded. The excess is then recorded over the revoked range and no later NOTIFY takes it back, which is the case the pending list exists to catch. Publish the widest bounds the commit could record as well, and separate the two outcomes. A revoke over the range asked for still kills the grant. One over the excess alone leaves the range asked for recorded and drops the excess, which only costs a re-request. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 5976b0f030a98b..2776138fa85609 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -51,13 +51,25 @@ */ #define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) -/* A FUSE_DLM_WB_LOCK request in flight, on cache->pending */ +/* + * A FUSE_DLM_WB_LOCK request in flight, on cache->pending. + * + * Two ranges, because the range asked for and the range that may end up + * recorded are not the same one: the server may grant more, up to + * FUSE_DLM_MAX_EXTRA_GRANT either side. A revoke has to be tested against + * both, and means something different for each. + */ struct fuse_dlm_range { /* The range asked for, as byte offsets, both inclusive */ uint64_t start; uint64_t end; - /* A revoke overlapped this request in flight */ + /* The widest [start, end] fuse_dlm_request_commit() could record */ + uint64_t wide_start; + uint64_t wide_end; + /* A revoke overlapped the range asked for: the grant is dead */ bool killed; + /* A revoke overlapped only the excess: record the asked for range */ + bool clamp; /* The cache->pending link */ struct list_head list; }; @@ -134,9 +146,12 @@ static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, struct fuse_dlm_range *req; spin_lock(&cache->pending_lock); - list_for_each_entry(req, &cache->pending, list) + list_for_each_entry(req, &cache->pending, list) { if (req->start <= end && start <= req->end) req->killed = true; + else if (req->wide_start <= end && start <= req->wide_end) + req->clamp = true; + } spin_unlock(&cache->pending_lock); } @@ -301,7 +316,17 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, req->start = start; req->end = end; + /* + * The bounds __fuse_get_dlm_lock() caps the recorded grant to. A + * revoke between here and the commit must be seen by one of the two + * tests in fuse_dlm_kill_pending(), or it would be recorded over. + */ + req->wide_start = start > FUSE_DLM_MAX_EXTRA_GRANT ? + start - FUSE_DLM_MAX_EXTRA_GRANT : 0; + req->wide_end = U64_MAX - end < FUSE_DLM_MAX_EXTRA_GRANT ? + U64_MAX : end + FUSE_DLM_MAX_EXTRA_GRANT; req->killed = false; + req->clamp = false; spin_lock(&cache->pending_lock); list_add_tail(&req->list, &cache->pending); @@ -321,9 +346,20 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, * lands either before it and is seen on @req, or after it and finds the * grant in the tree. * + * A revoke processed while @req was in flight lands in one of three + * places, and fuse_dlm_kill_pending() has already said which: + * + * - over the range asked for. The grant may predate it and there is no + * way to tell, so nothing is recorded and the caller asks again. + * - over the excess the server volunteered beyond it, and nothing else. + * The range asked for is untouched by it and is recorded; the excess + * is dropped, which only costs a re-request. + * - outside both, which says nothing about this grant. The whole of + * [start, end] is recorded. + * * @req is retired in every case and may be reused. * - * Return: -EAGAIN if a revoke overlapped @req while it was in flight, + * Return: -EAGAIN if a revoke overlapped the range @req asked for, * nothing recorded; otherwise the result of recording the grant. */ int fuse_dlm_request_commit(struct fuse_inode *inode, @@ -331,7 +367,7 @@ int fuse_dlm_request_commit(struct fuse_inode *inode, uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - bool revoked; + bool revoked, clamp; int ret = 0; /* @@ -344,6 +380,11 @@ int fuse_dlm_request_commit(struct fuse_inode *inode, spin_lock(&cache->pending_lock); list_del(&req->list); revoked = req->killed; + clamp = req->clamp; + if (clamp) { + start = req->start; + end = req->end; + } spin_unlock(&cache->pending_lock); if (!revoked) From 4c10c5309abfbfff12186fe316b239ae4c99eff9 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:13:17 +0200 Subject: [PATCH 06/24] fuse: bound the range a writeback pass defers fuse_writepages_fill() unions every folio it had to skip, and fuse_writepages() asks for the lot in one grant. A pass sweeping a large file skips folios gigabytes apart, so the union grows to the whole sweep and the request covers a range nothing wanted. Stop extending at one shard. The folios left out stay dirty and a later pass asks for them. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 1bf746549cf297..74eea1b32e8e71 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3286,12 +3286,22 @@ static int fuse_writepages_fill(struct folio *folio, data->regrant_start = pos; data->regrant_end = pos + len; } else { - data->regrant_start = min_t(u64, - data->regrant_start, - pos); - data->regrant_end = max_t(u64, - data->regrant_end, - pos + len); + u64 st = min_t(u64, data->regrant_start, pos); + u64 en = max_t(u64, data->regrant_end, + pos + len); + + /* + * One shard is as far as a single request is + * worth taking back. A pass sweeping a large + * file skips folios a long way apart, and the + * span between them says nothing about what + * is wanted; the folios left out stay dirty + * and a later pass asks for them. + */ + if (en - st <= FUSE_DLM_SHARD_SIZE) { + data->regrant_start = st; + data->regrant_end = en; + } } err = 0; goto out_unlock; From e06e9330f38b7a5d595fa69d45a1e6dc12c7666e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:15:37 +0200 Subject: [PATCH 07/24] fuse: pin a dlm grant from confirmation to dirty A writer confirms its grant and then copies and dirties, and a revoke landing in between is answered while those bytes are in no page cache and on no wire. The flush the revoke runs cannot find them, so they go out later under a fresh grant, after the server has handed the lock to another node. Add a pin: the range a writer is about to dirty, on a list a revoke walks before it removes anything. A revoke publishes the range it takes away on a second list and waits for the pins overlapping it, and a pin overlapping a published range is refused. Refusal and wait test the same overlap, so the wait converges, and ranges that do not overlap never meet: a notify leaves the rest of the file writable and a write outside the notified range does not hold it up. Both nodes are caller storage, so nothing is allocated to take a pin and the writeback path can take one with a folio held. Confirm the grant under the pin, never before, in the three places bytes become the server's: the cached interior of a chunk, the writethrough edges around their FUSE_WRITE, and the writeback folio until it is under writeback. A grant that has gone is asked for again with the pin dropped, since that request is answered by the server the revoke came from. With the writers fenced, the flush a revoke runs needs a single pass: nothing can turn up dirty behind it. [hbi: adapted -- ->write_begin and ->write_end have nowhere to keep a pin node that spans the pair, so the cached interior is pinned once per chunk rather than once per folio, and fuse_dlm_buffered_write() cuts that interior into shard sized chunks so a large write bounds how long it holds a revoke up.] Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 196 ++++++++++++++++++++++++++++--- fs/fuse/fuse_dlm_cache.c | 246 +++++++++++++++++++++++++++++++++++++++ fs/fuse/fuse_dlm_cache.h | 72 ++++++++++++ fs/fuse/fuse_i.h | 39 ++++++- fs/fuse/inode.c | 42 ++++++- 5 files changed, 572 insertions(+), 23 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 74eea1b32e8e71..92b00ac326d247 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1902,6 +1902,62 @@ static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) return (err < 0 && err != -ENOSYS) ? err : 0; } +/* + * How many times a writer confirms its grant again before giving up on + * the range. A pass costs a round trip only when the grant has gone, + * which is a revoke landing between the request and the confirmation. + */ +#define FUSE_DLM_PIN_RETRIES 16 + +/* + * Pin [@pos, @pos + @len) with the grant over it confirmed, so the bytes + * can be dirtied under a lock that cannot be taken away meanwhile; see + * fuse_dlm_pin(). @pin is the caller's storage for the pin, which it + * drops with fuse_dlm_unpin() once the bytes are dirty. + * + * The grant is asked for again when it has gone, and the pin must not be + * held across that request: it is answered by the server the revoke + * waiting for the pin came from. So confirm and request alternate, and + * no folio may be held here. + */ +static int fuse_dlm_pin_write(struct file *file, struct fuse_dlm_span *pin, + loff_t pos, size_t len) +{ + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + unsigned int tries = FUSE_DLM_PIN_RETRIES; + int err; + + for (;;) { + fuse_dlm_pin(fi, pin, pos, len); + /* + * A server that turned out to have no DLM leaves nothing to + * confirm, and the pin still pairs with the caller's unpin. + */ + if (!fc->dlm || + fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) + return 0; + fuse_dlm_unpin(fi); + + if (!tries--) + return -EIO; + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + if (err < 0 && err != -ENOSYS) + return err; + if (err > 0) { + /* + * Granted but unrecorded, so there is nothing for the + * confirmation above to find. The range is covered + * cluster-wide; pin and proceed. + */ + fuse_dlm_pin(fi, pin, pos, len); + return 0; + } + } +} + /* * Write @len bytes of @from at the current iocb position, either straight * through to the server (@through, for an unaligned edge) or into the page @@ -1910,16 +1966,32 @@ static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) * temporarily capped to @len so the unconsumed tail stays available for the * next chunk. Returns bytes written (< @len means a short write, the caller * stops) or a negative error. + * + * The grant over the chunk is confirmed and pinned across both, so the + * bytes cannot become the server's under a revoke that has already been + * answered. For the cached interior that is one pin for the whole chunk: + * ->write_begin and ->write_end have nowhere to keep a node that spans the + * pair, and generic_perform_write() runs between them. For the edges the + * bytes never enter the page cache at all, so a revoke cannot find them by + * flushing it and the pin has to cover the FUSE_WRITE itself, which is the + * reply the revoke already waits for when the same bytes go through + * writeback. */ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, size_t len, bool through) { + struct file *file = iocb->ki_filp; + struct fuse_dlm_span pin; size_t hidden; ssize_t res; if (!len) return 0; + res = fuse_dlm_pin_write(file, &pin, iocb->ki_pos, len); + if (res) + return res; + /* Cap the iterator to this chunk, keeping the tail for later chunks. */ hidden = iov_iter_count(from) - len; iov_iter_truncate(from, len); @@ -1929,6 +2001,8 @@ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, * (which leave it partly advanced) reexpand to the exact remainder. */ iov_iter_reexpand(from, iov_iter_count(from) + hidden); + fuse_dlm_unpin(get_fuse_inode(file_inode(file))); + return res; } @@ -1957,7 +2031,8 @@ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, /* No whole folio inside the write: nothing cacheable, all through. */ if (mid_end <= mid_start) - return fuse_perform_write(iocb, from, true); + return fuse_dlm_write_chunk(iocb, from, + iov_iter_count(from), true); /* Unaligned head [pos, mid_start): through. */ res = fuse_dlm_write_chunk(iocb, from, mid_start - pos, true); @@ -1967,13 +2042,25 @@ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, if (res < mid_start - pos) return total; - /* Aligned interior [mid_start, mid_end): cached whole folios. */ - res = fuse_dlm_write_chunk(iocb, from, mid_end - mid_start, false); - if (res < 0) - return total ? total : res; - total += res; - if (res < mid_end - mid_start) - return total; + /* + * Aligned interior [mid_start, mid_end): cached whole folios, taken + * one shard at a time. fuse_dlm_write_chunk() pins the grant for + * the length of a chunk and a revoke of the range waits for that, + * so the chunk is what bounds how long a large write holds a notify + * up. + */ + while (mid_start < mid_end) { + size_t chunk = min_t(loff_t, mid_end - mid_start, + FUSE_DLM_SHARD_SIZE); + + res = fuse_dlm_write_chunk(iocb, from, chunk, false); + if (res < 0) + return total ? total : res; + total += res; + if (res < chunk) + return total; + mid_start += chunk; + } /* Unaligned tail [mid_end, end): through. */ res = fuse_dlm_write_chunk(iocb, from, end - mid_end, true); @@ -3054,6 +3141,8 @@ static int fuse_writepage_locked(struct folio *folio) struct fuse_writepage_args *wpa; struct fuse_args_pages *ap; struct fuse_file *ff; + struct fuse_dlm_span pin; + bool pinned = false; int error = -EIO; ff = fuse_write_file_get(fi); @@ -3068,22 +3157,44 @@ static int fuse_writepage_locked(struct folio *folio) * no later pass of this submit to defer it to. Leave the folio dirty * and report no error: the invalidate that laundered it then finds it * busy, and an ordinary writeback sends it with a grant of its own. + * + * The grant is pinned from the moment it is found until the folio is + * under writeback, where the revoke waits for it again. */ if (fc->dlm && fc->writeback_cache) { - if (!fuse_dlm_lock_is_held(fi, folio_pos(folio), - folio_size(folio), - FUSE_PAGE_LOCK_WRITE)) { + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + + /* + * The revoke handler laundering the range it is taking away. + * That lock is still this client's until the handler returns, + * so send without asking: the record has gone already and + * asking would be a round trip for the very range being + * revoked. Only for that range, since the latched path + * launders the whole mapping and the rest of it may be + * covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_dlm_unpin(fi); + pinned = false; + } + + if (!pinned) { fuse_file_put(ff, false); fuse_writeback_redirty(fc, NULL, folio); return 0; } - error = fuse_dlm_regrant_range(ff, inode, folio_pos(folio), - folio_pos(folio) + - folio_size(folio) - 1); + error = fuse_dlm_regrant_range(ff, inode, pos, pos + len - 1); if (error < 0 && error != -ENOSYS) goto err_writepage_args; } +queue: wpa = fuse_writepage_args_setup(folio, ff); error = -ENOMEM; @@ -3094,6 +3205,15 @@ static int fuse_writepage_locked(struct folio *folio) ap->num_pages = 1; folio_start_writeback(folio); + + /* + * Under writeback now, so the flush a revoke runs before it takes the + * grant away waits for these bytes. Nothing further is needed to + * keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); + fuse_writepage_args_page_fill(wpa, folio, 0); spin_lock(&fi->lock); @@ -3104,6 +3224,8 @@ static int fuse_writepage_locked(struct folio *folio) return 0; err_writepage_args: + if (pinned) + fuse_dlm_unpin(fi); fuse_file_put(ff, false); err: /* @@ -3224,6 +3346,8 @@ static int fuse_writepages_fill(struct folio *folio, struct inode *inode = data->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_dlm_span pin; + bool pinned = false; int err; if (!data->ff) { @@ -3263,6 +3387,18 @@ static int fuse_writepages_fill(struct folio *folio, loff_t pos = folio_pos(folio); size_t len = folio_size(folio); + /* + * The revoke handler flushing the range it is taking away. + * That lock is still this client's until the handler returns, + * so send without asking: the record has gone already and + * asking would be a round trip for the very range being + * revoked. Only for that range, since the latched path + * launders the whole mapping and the rest of it may be + * covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + /* * The folio is locked here, and the folios queued before it * in this pass are under writeback, so a grant this folio @@ -3278,9 +3414,19 @@ static int fuse_writepages_fill(struct folio *folio, * with no folio held, and the pass that follows sends it. * Nothing is lost and no error is recorded for a later fsync * to report. + * + * A refused pin is a revoke of this range draining, and + * leaves the folio in the same place for the same reason. A + * revoke elsewhere in the file does not refuse it. */ - if (!fuse_dlm_lock_is_held(fi, pos, len, - FUSE_PAGE_LOCK_WRITE)) { + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_dlm_unpin(fi); + pinned = false; + } + + if (!pinned) { fuse_writeback_redirty(fc, wbc, folio); if (data->regrant_end <= data->regrant_start) { data->regrant_start = pos; @@ -3308,18 +3454,20 @@ static int fuse_writepages_fill(struct folio *folio, } /* - * Held: this walks the record and sends nothing. It stays a - * call rather than the check above so a grant that arrives - * between them is still used. + * Held, and pinned so it stays held: this walks the record + * and sends nothing. It stays a call rather than the check + * above so a grant that arrives between them is still used. */ err = fuse_dlm_regrant_range(data->ff, inode, pos, pos + len - 1); if (err < 0 && err != -ENOSYS) { fuse_writeback_redirty(fc, wbc, folio); + fuse_dlm_unpin(fi); goto out_unlock; } err = 0; } +queue: if (wpa && fuse_writepage_need_send(fc, &folio->page, ap, data, wbc)) { fuse_writepages_send(data); @@ -3331,6 +3479,8 @@ static int fuse_writepages_fill(struct folio *folio, wpa = fuse_writepage_args_setup(folio, data->ff); if (!wpa) { fuse_writeback_redirty(fc, wbc, folio); + if (pinned) + fuse_dlm_unpin(fi); goto out_unlock; } fuse_file_get(wpa->ia.ff); @@ -3340,6 +3490,14 @@ static int fuse_writepages_fill(struct folio *folio, ap->num_pages = 0; } folio_start_writeback(folio); + + /* + * Under writeback now, so the flush a revoke runs before it takes + * the grant away waits for these bytes. Nothing further is needed + * to keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); ap->descs[ap->num_pages].offset = 0; ap->descs[ap->num_pages].length = PAGE_SIZE; ap->pages[ap->num_pages] = &folio->page; diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 2776138fa85609..fb37b05336b12c 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -167,6 +167,252 @@ void fuse_dlm_cache_init(struct fuse_inode *inode) xa_init(&cache->shards); spin_lock_init(&cache->pending_lock); INIT_LIST_HEAD(&cache->pending); + spin_lock_init(&cache->pin_lock); + INIT_LIST_HEAD(&cache->pins); + INIT_LIST_HEAD(&cache->fences); + init_waitqueue_head(&cache->pin_wq); +} + +/* + * Set up @span over the page-aligned range [@offset, @offset + @length), + * the same range a fuse_dlm_lock_is_held() with these arguments asks + * about, so a fence over a page cannot miss a pin on that page. + */ +static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, + size_t length) +{ + span->start = (uint64_t)offset & PAGE_MASK; + span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + span->owner = current; +} + +/* + * Is @span inside the fence the revoke handler running on this task + * published? + * + * Only there is a pin both unnecessary and unsafe: the lock over that + * range is still this client's until the handler returns, and the fence + * the pin would wait on is the handler's own. Outside it the handler is + * an ordinary writer with no claim on the range, and answering as if it + * held a pin would let a revoke of that other range drain without + * waiting for bytes on their way out. The latched path launders the + * whole mapping from inside a fence over one range, so that is reached. + * + * Built the way fuse_dlm_revoke_begin() builds the fence, so this + * answers what the fence list would. + */ +static bool fuse_dlm_in_own_fence(const struct fuse_dlm_span *span) +{ + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + uint64_t start, end; + + if (!ctx) + return false; + + start = (uint64_t) ctx->start & PAGE_MASK; + end = ctx->end >= LLONG_MAX ? U64_MAX : + ((uint64_t) ctx->end | (PAGE_SIZE - 1)); + + return span->start >= start && span->end <= end; +} + +/* + * Does anything on @head share a byte with [@start, @end]? Caller holds + * fuse_dlm_cache.pin_lock. + */ +static bool fuse_dlm_overlaps_locked(struct list_head *head, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_span *span; + + list_for_each_entry(span, head, list) + if (span->start <= end && start <= span->end) + return true; + + return false; +} + +/* fuse_dlm_overlaps_locked() taking the lock itself */ +static bool fuse_dlm_overlaps(struct fuse_dlm_cache *cache, + struct list_head *head, uint64_t start, + uint64_t end) +{ + bool overlap; + + spin_lock(&cache->pin_lock); + overlap = fuse_dlm_overlaps_locked(head, start, end); + spin_unlock(&cache->pin_lock); + + return overlap; +} + +/** + * fuse_dlm_pin - hold the grants over a range until fuse_dlm_unpin() + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes + * + * Waits out a revoke overlapping that range, so it must not be called + * with a folio held: the revoke drops that same page cache once it has + * drained. A revoke elsewhere in the file is not waited for. The caller + * confirms its grant after this returns, never before; a confirmation + * from before the pin says nothing. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + fuse_dlm_span_set(pin, offset, length); + + /* + * The revoke handler driving this inode's page cache, over the + * range it is taking away. Anywhere else it pins like any other + * writer; see fuse_dlm_in_own_fence(). + */ + if (fuse_dlm_in_own_fence(pin)) + return; + + spin_lock(&cache->pin_lock); + while (fuse_dlm_overlaps_locked(&cache->fences, pin->start, pin->end)) { + spin_unlock(&cache->pin_lock); + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->fences, + pin->start, pin->end)); + spin_lock(&cache->pin_lock); + } + /* + * At the head, so fuse_dlm_unpin() drops the innermost pin of a + * task that holds more than one. + */ + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); +} + +/** + * fuse_dlm_trypin - fuse_dlm_pin() for a caller that cannot sleep + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes + * + * For the writeback path, which holds a folio locked and under writeback + * and has nothing to wait with. A refusal means a revoke of this range + * is draining; the caller redirties and the pass that follows sends the + * folio. + * + * Return: true if the range is pinned, false if it is not. + */ +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; + + fuse_dlm_span_set(pin, offset, length); + + /* See fuse_dlm_pin() */ + if (fuse_dlm_in_own_fence(pin)) + return true; + + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); + + return !fenced; +} + +/** + * fuse_dlm_unpin - release the pin this task last took on @inode + * @inode: the fuse inode + * + * Found by owner rather than by node: iomap hands ->put_folio the inode + * and nothing of the iteration, and a task holds one pin at a time. + * + * No shortcut of its own: a revoke handler takes a real pin outside its + * fence and has to drop it, and one that took none has nothing on the + * list for the walk below to find. + */ +void fuse_dlm_unpin(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_span *pin; + bool waiters; + + spin_lock(&cache->pin_lock); + list_for_each_entry(pin, &cache->pins, list) { + if (pin->owner == current) { + list_del(&pin->list); + break; + } + } + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); + + if (waiters) + wake_up_all(&cache->pin_wq); +} + +/** + * fuse_dlm_revoke_begin - fence the writers over a range that have not + * dirtied yet + * @inode: the fuse inode + * @fence: caller-owned storage, live until fuse_dlm_revoke_end() + * @offset: start byte offset being revoked + * @len: length in bytes, or <= 0 for everything from @offset on + * + * Publishes the range, then waits for the pins over it taken before it. + * On return no thread is between confirming a grant on that range and + * dirtying under it, and none can start, so what the caller flushes is + * everything the grants it is about to drop can have produced. A writer + * elsewhere in the file is neither waited for nor held up. + * + * Publishing before waiting is what makes the wait converge: a pin is + * refused on the same overlap this waits on, so nothing admitted after + * this can prolong it. + * + * Revokes on one inode fence independently, each over its own range. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + /* The range fuse_dlm_unlock_range() will be asked to drop */ + fence->start = (uint64_t)offset & PAGE_MASK; + fence->end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + fence->owner = NULL; + + spin_lock(&cache->pin_lock); + list_add(&fence->list, &cache->fences); + spin_unlock(&cache->pin_lock); + + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->pins, fence->start, + fence->end)); +} + +/** + * fuse_dlm_revoke_end - drop the fence fuse_dlm_revoke_begin() published + * @inode: the fuse inode + * @fence: the fence published there + */ +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pin_lock); + list_del(&fence->list); + spin_unlock(&cache->pin_lock); + + wake_up_all(&cache->pin_wq); } /** diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index c54cad9d000226..ec2e82c406a1cb 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -22,6 +23,21 @@ struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * A range held on one of the two lists in struct fuse_dlm_cache: a + * writer between confirming a grant and dirtying under it (@owner set), + * or a revoke taking grants away (@owner NULL). Caller-owned storage, + * live until the matching unpin or revoke end. + */ +struct fuse_dlm_span { + /* Page-aligned byte offsets, both inclusive */ + uint64_t start; + uint64_t end; + /* The pinning task, NULL for a revoke */ + struct task_struct *owner; + struct list_head list; +}; + /* * fuse_get_dlm_lock() result: the server granted the lock but recording * it locally failed, leaving the grant invisible to @@ -98,6 +114,22 @@ struct fuse_dlm_shard { * @pending_lock the pending list. Innermost, and the only lock * fuse_dlm_request_begin() and fuse_dlm_request_abort() * take at all. + * @pin_lock the pin and fence lists. Innermost, taken alone, and + * never held across a sleep. + * + * @lock says what is covered now, which is not enough for a writer: it + * confirms a grant, then copies and dirties, and a revoke landing in + * between sends those bytes out after the server has handed the lock on. + * The pin closes that: a revoke waits for the pins over the range it is + * taking away before it removes anything, so a grant confirmed under a + * pin is still held when the bytes become visible to writeback. + * + * Both sides are ranges rather than a count, so a revoke fences only the + * writers it overlaps and a write outside it runs on. Refusal and wait + * test the same overlap, which is what makes the wait converge: once a + * fence is published no pin that would prolong it is admitted. The + * nodes are caller storage, so nothing is allocated to take a pin and + * the writeback path can take one with a folio held. */ struct fuse_dlm_cache { /* See the locking comment above */ @@ -114,6 +146,17 @@ struct fuse_dlm_cache { * thread; the revoke paths only mark them killed. */ struct list_head pending; + /* Protects @pins and @fences */ + spinlock_t pin_lock; + /* + * Writers between confirming a grant and dirtying under it, each + * over the range it is about to write. See the pin comment above. + */ + struct list_head pins; + /* Revokes in progress, each over the range it takes away */ + struct list_head fences; + /* Both directions: pins draining, and the fences they wait on */ + wait_queue_head_t pin_wq; }; /* Initialize a page cache lock manager */ @@ -150,6 +193,35 @@ void fuse_dlm_request_abort(struct fuse_inode *inode, int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); +/* + * Hold the grants over [@offset, @offset + @length) against revocation + * until fuse_dlm_unpin(), which drops the pin this task last took. @pin + * is caller-owned storage, live until then. fuse_dlm_pin() waits out a + * revoke overlapping that range and must not be called with a folio + * held; fuse_dlm_trypin() never sleeps and fails instead. Neither may + * be held across a DLM request: that request is answered by the server + * the revoke came from. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin(struct fuse_inode *inode); + +/* + * Fence the writers that hold a grant over [@offset, @offset + @len) but + * have not dirtied under it yet, for the duration of a revoke. @len <= 0 + * means to EOF, as in fuse_notify_inval_inode(). @fence is caller-owned + * storage, live until the matching end. Between these the caller may + * drop coverage over that range knowing nothing will be dirtied under + * what it drops, and a write outside it is left alone. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len); +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence); + /* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 578751035a71c1..d828ba425883dd 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -46,11 +46,29 @@ */ extern const char fuse_notify_ctx_key[]; -static inline void *fuse_notify_ctx_enter(void) +/* + * What a task driving page cache work for a NOTIFY invalidate carries. + * + * @key tells it apart from anything else parked in journal_info. The range + * is the one being revoked: the lock over it is still this client's until + * the handler returns, so writeback of it need not ask for a grant, while + * anything outside it must. + */ +struct fuse_notify_ctx { + const char *key; + loff_t start; + loff_t end; /* inclusive; LLONG_MAX to EOF */ +}; + +static inline void *fuse_notify_ctx_enter(struct fuse_notify_ctx *ctx, + loff_t start, loff_t end) { void *old = current->journal_info; - current->journal_info = (void *)fuse_notify_ctx_key; + ctx->key = fuse_notify_ctx_key; + ctx->start = start; + ctx->end = end; + current->journal_info = ctx; return old; } @@ -59,9 +77,24 @@ static inline void fuse_notify_ctx_leave(void *old) current->journal_info = old; } +static inline struct fuse_notify_ctx *fuse_notify_ctx(void) +{ + struct fuse_notify_ctx *ctx = current->journal_info; + + return (ctx && ctx->key == fuse_notify_ctx_key) ? ctx : NULL; +} + static inline bool fuse_in_notify_ctx(void) { - return current->journal_info == (void *)fuse_notify_ctx_key; + return fuse_notify_ctx(); +} + +/* Is [@pos, @pos + @len) the range the revoke in progress is taking away? */ +static inline bool fuse_in_notify_range(loff_t pos, unsigned int len) +{ + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + + return ctx && pos >= ctx->start && pos + len - 1 <= ctx->end; } /** Default max number of pages that can be used in a single read request */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 1d4c4da481582d..8a5e09b46d12ec 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -985,7 +985,10 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * here must not send a DLM request. See * fuse_writepages_fill(). */ - void *notify_ctx = fuse_notify_ctx_enter(); + struct fuse_notify_ctx ctx; + struct fuse_dlm_span fence; + void *notify_ctx; + bool fenced; pg_start = offset >> PAGE_SHIFT; if (len <= 0) @@ -996,6 +999,36 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* Byte bounds of the same region */ end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + notify_ctx = fuse_notify_ctx_enter(&ctx, offset, end_byte); + + /* + * Fence the writers that hold a grant over this range but + * have not dirtied under it yet. Their bytes are in no page + * cache and on no wire, so nothing below can find them, and + * once the grant is gone they would go out behind the + * handover. Published before the flush so what it drains is + * flushed with everything else. + * + * Only the writers over this range: a write elsewhere in the + * file holds a grant this handler does not touch, and neither + * waits for the other. + * + * Entered after fuse_notify_ctx_enter(): the page cache work + * below is this handler's own and must not fence itself. + * + * A fenced writer may be waiting for a FUSE_WRITE reply, so + * this waits on the server the same way the flush below does. + * + * Regular files only: the record shares the readdir cache + * union arm and exists nowhere else. Latched into a local, + * since fc->dlm can be cleared while this runs and the fence + * has to come off the list either way. + */ + fenced = S_ISREG(inode->i_mode) && fc->dlm && + fc->writeback_cache; + if (fenced) + fuse_dlm_revoke_begin(fi, &fence, offset, len); + /* * A data invalidation means another (remote) entity is modifying * the file. Two things happen here: @@ -1065,6 +1098,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * waiting on. do_writepages() runs in this context, * so the grant is asked for before the revoke. * + * One pass is enough: the fence above has drained the + * writers that held a grant without having dirtied + * under it, and refuses new ones, so nothing can turn + * up dirty behind this. + * * Waited out here rather than left to the drop, which * launders when the range may be dirty and so waits * for these same replies. One explicit wait, before @@ -1122,6 +1160,8 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_notify_invalidate_range(inode, pg_start, pg_end, true); } + if (fenced) + fuse_dlm_revoke_end(fi, &fence); fuse_notify_ctx_leave(notify_ctx); } iput(inode); From 7c652b382c65a8e76e6d267cbe7f7ddda3ce3ef3 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:19:18 +0200 Subject: [PATCH 08/24] fuse: fill a folio under a pinned read grant A folio made uptodate from the server is served to every later reader, so the grant it was fetched under has to be held from the confirmation until the bytes are in the page cache. Without that a revoke sweeping the range leaves the fill behind it: the folio stays cached, uncovered, and the server sends no further notify for a lock this client no longer holds. Confirm the grant under a pin, as the write path does. Refused, or not covered, unlock the folio and back off with AOP_TRUNCATED_PAGE: neither the wait nor the grant request may be taken with a page lock held. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 92b00ac326d247..d579b0dbeb5170 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1063,20 +1063,100 @@ static int fuse_do_readpage(struct file *file, struct page *page) return 0; } +/** + * fuse_read_folio_retry - back off a fill with no grant to run under + * @file: file to read through + * @folio: the folio handed over locked, unlocked here + * @pos: byte offset of @folio + * @len: its size in bytes + * + * Neither reason a fill is refused can be dealt with while the folio is + * held: waiting a revoke out would hold the page cache that revoke is + * about to drop, and no grant may be asked for under a page lock at all + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * Unlock, do both, and send the caller round again to find the range + * covered. + * + * Return: AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_retry(struct file *file, struct folio *folio, + loff_t pos, size_t len) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + struct fuse_dlm_span pin; + int err; + + folio_unlock(folio); + + /* Wait the revoke out; what it leaves behind is asked for below */ + fuse_dlm_pin(fi, &pin, pos, len); + fuse_dlm_unpin(fi); + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_READ); + if (err == -ENOSYS) + return AOP_TRUNCATED_PAGE; + if (err < 0) + return err; + /* + * Granted but unrecorded, so the retry finds the range uncovered + * and comes straight back here. Report it rather than spin. + */ + if (err > 0) + return -ENOMEM; + + return AOP_TRUNCATED_PAGE; +} + static int fuse_read_folio(struct file *file, struct folio *folio) { struct page *page = &folio->page; struct inode *inode = page->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + struct fuse_dlm_span pin; + bool pinned = false; int err; err = -EIO; if (fuse_is_bad(inode)) goto out; + /* + * The grant the folio is filled under, held from the confirmation + * until the bytes are in the page cache. What lands here is served + * to every later reader of the file, so it must neither be fetched + * under a grant a revoke has taken away nor be dropped into a range + * a revoke has just swept: such a folio is uptodate and covered by + * nothing, and no further notify comes for a lock this client no + * longer holds. + * + * The pin closes the second, since a revoke over the folio waits + * for the fill and drops the folio after it; the confirmation + * closes the first. Both fail into fuse_read_folio_retry(). + */ + if (fc->dlm && fc->writeback_cache) { + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin(fi); + pinned = false; + } + if (!pinned) + return fuse_read_folio_retry(file, folio, pos, len); + } + err = fuse_do_readpage(file, page); fuse_invalidate_atime(inode); out: unlock_page(page); + /* + * After the unlock, so a revoke draining this pin finds the folio + * it has to drop unlocked and takes it out. + */ + if (pinned) + fuse_dlm_unpin(fi); return err; } From 69571af765d633593b0ec8a2c20acc1c4a70e575 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:19:47 +0200 Subject: [PATCH 09/24] fuse: fill a readahead window under a pinned read grant A readahead reply lands in the page cache from the task that processes it, so the pin over the folios has to span the request: taken before it is sent and dropped once the folios are filled and unlocked. A revoke of the range waits for that and drops the folios after; one already draining refuses the pin and the window goes back unfilled. The node therefore outlives the pinning task, which fuse_dlm_unpin() cannot express. Add a span-owned pin, dropped by node and carrying no owner for the by-owner search to match. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 74 ++++++++++++++++++++++++++++++++++++---- fs/fuse/fuse_dlm_cache.c | 64 +++++++++++++++++++++++++++++++--- fs/fuse/fuse_dlm_cache.h | 30 ++++++++++++---- fs/fuse/fuse_i.h | 9 +++++ 4 files changed, 160 insertions(+), 17 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index d579b0dbeb5170..724097b1860975 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1191,19 +1191,39 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, folio_end_read(folio, !err); folio_put(folio); } + + /* + * Dropped after the folios, which are filled, uptodate and unlocked + * by now: a revoke draining this pin finds them and takes them out. + * Held until here so it cannot have swept before they were there. + */ + if (ia->read.dlm_fi) + fuse_dlm_unpin_span(ia->read.dlm_fi, &ia->read.dlm_pin); + if (ia->ff) fuse_file_put(ia->ff, false); fuse_io_free(ia); } -static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) +/** + * fuse_send_readpages - read a run of pages of a readahead window + * @ia: the request, owning the pages and the pin over them + * @file: file to read through + * + * Return: 0 once the request is on its way or has been completed, + * -EAGAIN when a revoke of the range refused the grant and nothing was + * sent. The pages are given back either way. + */ +static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file) { struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); struct fuse_args_pages *ap = &ia->ap; loff_t pos = page_offset(ap->pages[0]); size_t count = ap->num_pages << PAGE_SHIFT; + unsigned int i; ssize_t res; int err; @@ -1218,6 +1238,29 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) } WARN_ON((loff_t) (pos + count) < 0); + /* + * The grant fuse_readahead() took, confirmed under a pin and held + * until the reply has filled the pages. A revoke of the range + * waits for that, so the reply cannot be fetched under a grant the + * server has since handed on, and cannot land behind a sweep that + * would leave the pages uptodate and covered by nothing. + * + * Refused, or gone since it was asked for: give the pages back + * unfilled rather than serve what no lock covers. The read that + * wanted them comes back through fuse_read_folio(), which asks + * again with no page held. + */ + if (fm->fc->dlm && fm->fc->writeback_cache) { + if (!fuse_dlm_trypin_span(fi, &ia->read.dlm_pin, pos, count)) + goto uncovered; + if (!fuse_dlm_lock_is_held(fi, pos, count, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin_span(fi, &ia->read.dlm_pin); + goto uncovered; + } + ia->read.dlm_fi = fi; + } + fuse_read_args_fill(ia, file, pos, count, FUSE_READ); ia->read.attr_ver = fuse_get_attr_version(fm->fc); if (fm->fc->async_read) { @@ -1225,12 +1268,23 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) ap->args.end = fuse_readpages_end; err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (!err) - return; + return 0; } else { res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } fuse_readpages_end(fm, &ap->args, err); + return 0; + +uncovered: + for (i = 0; i < ap->num_pages; i++) { + struct folio *folio = page_folio(ap->pages[i]); + + folio_end_read(folio, false); + folio_put(folio); + } + fuse_io_free(ia); + return -EAGAIN; } /* @@ -1409,6 +1463,11 @@ static void fuse_readahead(struct readahead_control *rac) * @rac, so the window that gets populated is the window that is * covered. * + * The grant is only asked for here. Confirming it and holding it + * against a revoke is fuse_send_readpages(), one run of pages at a + * time, since that is where the request the reply fills them from + * goes out; a run it declines ends the window. + * * The grant covers the window this call intends rather than the one * it ends up with, since the folios past what mm built are not * allocated yet. readahead_expand() stops at the first folio already @@ -1505,7 +1564,8 @@ static void fuse_readahead(struct readahead_control *rac) ap->descs[i].length = PAGE_SIZE; } ap->num_pages = nr_pages; - fuse_send_readpages(ia, rac->file); + if (fuse_send_readpages(ia, rac->file)) + break; } /* @@ -1548,9 +1608,11 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* * A NOTIFY invalidate racing this read drops the folios it * supersedes, so the read either misses and refetches or returns - * data that was current when it was copied. There is nothing to - * fence: unlike a write, a read leaves nothing behind that could - * reach the server under a grant it no longer holds. + * data that was current when it was copied. What a read does leave + * behind is the page cache it fills, which must not outlast the + * grant it was fetched under; that is fenced where the filling + * happens, in fuse_read_folio() and fuse_send_readpages(), and the + * grant taken here is what they confirm. */ if (fuse_inode_force_dio(inode)) { size_t count = iov_iter_count(to); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index fb37b05336b12c..bd0afb948781f3 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -179,11 +179,11 @@ void fuse_dlm_cache_init(struct fuse_inode *inode) * about, so a fence over a page cannot miss a pin on that page. */ static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, - size_t length) + size_t length, struct task_struct *owner) { span->start = (uint64_t)offset & PAGE_MASK; span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); - span->owner = current; + span->owner = owner; } /* @@ -264,7 +264,7 @@ void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - fuse_dlm_span_set(pin, offset, length); + fuse_dlm_span_set(pin, offset, length, current); /* * The revoke handler driving this inode's page cache, over the @@ -310,7 +310,7 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; bool fenced; - fuse_dlm_span_set(pin, offset, length); + fuse_dlm_span_set(pin, offset, length, current); /* See fuse_dlm_pin() */ if (fuse_dlm_in_own_fence(pin)) @@ -326,6 +326,62 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, return !fenced; } +/** + * fuse_dlm_trypin_span - fuse_dlm_trypin() for a fill that ends elsewhere + * @inode: the fuse inode + * @pin: caller-owned storage, live until fuse_dlm_unpin_span() + * @offset: byte offset the caller is about to fill + * @length: length of the region in bytes + * + * For a read whose reply lands in another task: the node is dropped by + * fuse_dlm_unpin_span() from wherever the fill ends, and carries no + * owner, so a fuse_dlm_unpin() by the task that took it cannot match it + * instead of its own. + * + * Never sleeps, and has no notify-context shortcut: a fill is not + * reached from a revoke handler, and a pin taken there would have to be + * dropped from a task that is not in one. + * + * Return: true if the range is pinned, false if a revoke of it is + * draining. + */ +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; + + fuse_dlm_span_set(pin, offset, length, NULL); + + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); + + return !fenced; +} + +/** + * fuse_dlm_unpin_span - release the pin fuse_dlm_trypin_span() took + * @inode: the fuse inode + * @pin: the node published there + */ +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool waiters; + + spin_lock(&cache->pin_lock); + list_del(&pin->list); + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); + + if (waiters) + wake_up_all(&cache->pin_wq); +} + /** * fuse_dlm_unpin - release the pin this task last took on @inode * @inode: the fuse inode diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index ec2e82c406a1cb..ae6aa2bcbdae93 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -24,10 +24,14 @@ struct fuse_file; enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; /* - * A range held on one of the two lists in struct fuse_dlm_cache: a - * writer between confirming a grant and dirtying under it (@owner set), - * or a revoke taking grants away (@owner NULL). Caller-owned storage, - * live until the matching unpin or revoke end. + * A range held on one of the two lists in struct fuse_dlm_cache: an IO + * between confirming a grant and publishing the page cache it covers, + * or a revoke taking grants away. Caller-owned storage, live until the + * matching unpin or revoke end. + * + * @owner is the pinning task where the pin is dropped by owner, and NULL + * on every fence and on a pin dropped by node because the fill it covers + * ends in another task. */ struct fuse_dlm_span { /* Page-aligned byte offsets, both inclusive */ @@ -122,7 +126,9 @@ struct fuse_dlm_shard { * between sends those bytes out after the server has handed the lock on. * The pin closes that: a revoke waits for the pins over the range it is * taking away before it removes anything, so a grant confirmed under a - * pin is still held when the bytes become visible to writeback. + * pin is still held when the bytes become visible to writeback. A read + * is the same the other way round, a fill landing in a range the revoke + * has already swept staying cached under no grant at all. * * Both sides are ranges rather than a count, so a revoke fences only the * writers it overlaps and a write outside it runs on. Refusal and wait @@ -149,8 +155,9 @@ struct fuse_dlm_cache { /* Protects @pins and @fences */ spinlock_t pin_lock; /* - * Writers between confirming a grant and dirtying under it, each - * over the range it is about to write. See the pin comment above. + * IO between confirming a grant and publishing under it: a writer + * over what it is about to dirty, a read over what it is about to + * fill. See the pin comment above. */ struct list_head pins; /* Revokes in progress, each over the range it takes away */ @@ -208,6 +215,15 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, loff_t offset, size_t length); void fuse_dlm_unpin(struct fuse_inode *inode); +/* + * fuse_dlm_trypin() for a fill whose reply lands in another task: @pin + * is dropped by node rather than by owner, and is live from the request + * until fuse_dlm_unpin_span(). + */ +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin); + /* * Fence the writers that hold a grant over [@offset, @offset + @len) but * have not dirtied under it yet, for the duration of a revoke. @len <= 0 diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index d828ba425883dd..fb1a8c4298a75a 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1330,6 +1330,15 @@ struct fuse_io_args { struct { struct fuse_read_in in; u64 attr_ver; + /* + * The grant the folios are filled under, held + * from the request until the reply has filled + * them; see fuse_send_readpages(). @dlm_fi is + * the inode to drop it on, and NULL when there + * is no pin to drop. + */ + struct fuse_dlm_span dlm_pin; + struct fuse_inode *dlm_fi; } read; struct { struct fuse_write_in in; From 2cc8e67a14f333e4d990b058ca028db94b7f21bf Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:22:56 +0200 Subject: [PATCH 10/24] fuse: ask for the read grant before the folios are locked A read grant may not be requested under a page lock, so every buffered read path asks for one before it enters the page cache: buffered read, splice read and read fault. The window covers what readahead may add beyond the read, bounded by the file. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 70 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 724097b1860975..f431beddebd254 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1063,6 +1063,46 @@ static int fuse_do_readpage(struct file *file, struct page *page) return 0; } +/** + * fuse_read_grant - take the grant a page cache fill runs under + * @file: file to read through + * @pos: byte offset the read starts at + * @count: bytes the read asks for + * + * ->read_folio and ->readahead are entered with the pages they fill + * already locked, and no grant may be asked for under a page lock + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * A read asks here, before anything is locked, and the fill paths only + * confirm what this took. + * + * Readahead fills past the end of the read, so ask for a window beyond + * it as well, bounded by the file since readahead stops there. A run of + * pages no grant covers is given back unfilled and fetched one page at + * a time, so what is asked for here is what readahead is worth. + * + * Return: what fuse_get_dlm_lock() returned, 0 when there is nothing to + * ask for. + */ +static int fuse_read_grant(struct file *file, loff_t pos, size_t count) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + loff_t size = i_size_read(inode); + loff_t ahead = (loff_t)file->f_ra.ra_pages << PAGE_SHIFT; + loff_t end = pos + count; + + if (!fc->writeback_cache || !fc->dlm) + return 0; + + if (end < size) + end += min(ahead, size - end); + + if (end <= pos) + return 0; + + return fuse_get_dlm_lock(file, pos, end - pos, FUSE_PAGE_LOCK_READ); +} + /** * fuse_read_folio_retry - back off a fill with no grant to run under * @file: file to read through @@ -1092,7 +1132,7 @@ static int fuse_read_folio_retry(struct file *file, struct folio *folio, fuse_dlm_pin(fi, &pin, pos, len); fuse_dlm_unpin(fi); - err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_READ); + err = fuse_read_grant(file, pos, len); if (err == -ENOSYS) return AOP_TRUNCATED_PAGE; if (err < 0) @@ -1599,11 +1639,8 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } - /* if we have dlm support acquire a read lock for the area - * we are reading from. */ - if (fc->writeback_cache && fc->dlm) - fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + /* The grant this read and the readahead behind it fill under */ + fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); /* * A NOTIFY invalidate racing this read drops the folios it @@ -2998,8 +3035,10 @@ static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); - else - return filemap_splice_read(in, ppos, pipe, len, flags); + + fuse_read_grant(in, *ppos, len); + + return filemap_splice_read(in, ppos, pipe, len, flags); } static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, @@ -3983,9 +4022,22 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) return VM_FAULT_LOCKED; } +/* + * A read fault fills the page cache through ->read_folio and + * ->readahead, which run with the pages locked. Ask for the grant they + * fill under before filemap_fault() locks any of them. + */ +static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) +{ + fuse_read_grant(vmf->vma->vm_file, (loff_t)vmf->pgoff << PAGE_SHIFT, + PAGE_SIZE); + + return filemap_fault(vmf); +} + static const struct vm_operations_struct fuse_file_vm_ops = { .close = fuse_vma_close, - .fault = filemap_fault, + .fault = fuse_filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = fuse_page_mkwrite, }; From 889385bcb36176be560327284e37be88858a8625 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:23:31 +0200 Subject: [PATCH 11/24] fuse: stop asking for the readahead grant under the folio locks ->readahead runs with every page of the window locked, so the grant request it sent went out under those locks and a revoke of the window had to be given up on. The same goes for the one-request lookahead past the window, which runs from inside ->readahead with the window still in flight. The read now takes the grant before the page cache is entered; fill only what it covers. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 65 +++++++++++++------------------------------------- 1 file changed, 16 insertions(+), 49 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f431beddebd254..0fbcdd42bf5ad8 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1279,8 +1279,8 @@ static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file) WARN_ON((loff_t) (pos + count) < 0); /* - * The grant fuse_readahead() took, confirmed under a pin and held - * until the reply has filled the pages. A revoke of the range + * The grant the read took in fuse_read_grant(), confirmed under a + * pin and held until the reply has filled the pages. A revoke of the range * waits for that, so the reply cannot be fetched under a grant the * server has since handed on, and cannot land behind a sweep that * would leave the pages uptodate and covered by nothing. @@ -1392,19 +1392,11 @@ static void fuse_readahead_lookahead(struct file *file, struct inode *inode, return; /* - * Same grant the window itself takes: folios the server handed out - * no lock for are folios it will not revoke when a remote node - * writes them. + * No grant is asked for here either: this runs from inside + * ->readahead with the window it just sent still locked. What the + * read's grant does not reach fuse_send_readpages() declines, and + * the pages go back unfilled. */ - if (fc->writeback_cache && fc->dlm) { - int err = fuse_get_dlm_lock(file, (loff_t)start << PAGE_SHIFT, - (size_t)nr << PAGE_SHIFT, - FUSE_PAGE_LOCK_READ); - - if (err < 0 && err != -ENOSYS) - return; - } - ia = fuse_io_alloc(NULL, nr); if (!ia) return; @@ -1495,46 +1487,21 @@ static void fuse_readahead(struct readahead_control *rac) } /* - * Readahead fills the page cache past the range the reader locked, - * so take a DLM read grant over the whole window here too. Folios + * Readahead fills the page cache past the range the reader asked + * for, and what lands there has to be covered by a grant: folios * the server handed out no lock for are folios it will not revoke * when a remote node writes them, and a later read would be served - * from stale cache. Take the grant before any folio is pulled off - * @rac, so the window that gets populated is the window that is - * covered. - * - * The grant is only asked for here. Confirming it and holding it - * against a revoke is fuse_send_readpages(), one run of pages at a - * time, since that is where the request the reply fills them from - * goes out; a run it declines ends the window. + * from stale cache. * - * The grant covers the window this call intends rather than the one - * it ends up with, since the folios past what mm built are not - * allocated yet. readahead_expand() stops at the first folio already - * cached, so a short realisation leaves the tail covered but not - * populated. That is the harmless direction: coverage without cached - * data serves nothing stale, and a folio already cached is one an - * earlier grant already covers. + * No grant is asked for here. ->readahead is entered with every + * page of the window already locked, and none may be asked for + * under a page lock; the read this window belongs to took one over + * it in fuse_read_grant(), before the page cache was entered. * - * Speculative pages are not worth serving uncovered: on a failed - * request drop the window and let read_pages() clean up the folios - * left in @rac. A server without DLM support answers -ENOSYS and - * clears fc->dlm, which is not a failure. - * - * The round trip is taken before any folio of the window is locked - * and with nothing fenced out, so it holds up this reader and - * nothing else. + * What that left uncovered fuse_send_readpages() declines, one run + * of pages at a time, and those pages go back unfilled for + * fuse_read_folio() to fetch with no page held. */ - if (fc->writeback_cache && fc->dlm) { - size_t len = (size_t)(target_end - readahead_index(rac)) - << PAGE_SHIFT; - int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), len, - FUSE_PAGE_LOCK_READ); - - if (err < 0 && err != -ENOSYS) - return; - } - for (;;) { struct fuse_io_args *ia; struct fuse_args_pages *ap; From 001714a6732e3df265569e5376f057bf497c6435 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:23:46 +0200 Subject: [PATCH 12/24] fuse: ask for the read grant on a readahead advice POSIX_FADV_WILLNEED and readahead(2) fill through ->readahead, which now fills only what a grant already covers. Take the grant for the advised range first, with no page held. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 0fbcdd42bf5ad8..c7edc0dadab193 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -4833,6 +4834,21 @@ int fuse_migrate_folio(struct address_space *mapping, struct folio *dst, #endif +/* + * POSIX_FADV_WILLNEED, and readahead(2) with it, populate the page cache + * through ->readahead, which runs with the pages locked. Ask for the + * grant that fill needs while nothing is held; a window no grant covers + * is given back unfilled. + */ +static int fuse_fadvise(struct file *file, loff_t offset, loff_t len, + int advice) +{ + if (advice == POSIX_FADV_WILLNEED && offset >= 0 && len > 0) + fuse_read_grant(file, offset, len); + + return generic_fadvise(file, offset, len, advice); +} + static const struct file_operations fuse_file_operations = { .llseek = fuse_file_llseek, .read_iter = fuse_file_read_iter, @@ -4852,6 +4868,7 @@ static const struct file_operations fuse_file_operations = { .poll = fuse_file_poll, .fallocate = fuse_file_fallocate, .copy_file_range = fuse_copy_file_range, + .fadvise = fuse_fadvise, }; static const struct address_space_operations fuse_file_aops = { From 1abdc10ad89e5dee0155078c1a486dc97d4db3a5 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:24:17 +0200 Subject: [PATCH 13/24] fuse: pin a streamed write across the request A streamed write goes to the server out of the caller's pages, so its bytes are in no page cache and a revoke of the range finds nothing to flush. Hold the grant across the FUSE_WRITE, as the writethrough edges do. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index c7edc0dadab193..99e276cb5c3488 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2568,6 +2568,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) generic_perform_write(iocb, from)); } else if (through) { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + struct fuse_dlm_span pin; loff_t pos = iocb->ki_pos; count = iov_iter_count(from); @@ -2588,8 +2589,23 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } } + /* + * These bytes never enter the page cache, so a revoke + * cannot find them by flushing it, and the grant they + * were taken under can be handed on while they are + * still on the wire. Hold it across the FUSE_WRITE, + * as the writethrough edges do; a revoke of the range + * waits for the reply. + */ + err = fuse_dlm_pin_write(file, &pin, pos, count); + if (err) { + written = err; + goto wb_out; + } + written = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); + fuse_dlm_unpin(fi); if (written < 0) { err = written; goto wb_out; From f3fef2b14a67194b0cf7d58d349f28a80e229a87 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 10:24:45 +0200 Subject: [PATCH 14/24] fuse: take no dlm lock for O_DIRECT An O_DIRECT read or write neither fills nor dirties the page cache, so a grant over its range covers nothing and only conflicts with the rest of the cluster. Skip it on both sides. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 99e276cb5c3488..87290b629ee72f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1607,8 +1607,13 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } - /* The grant this read and the readahead behind it fill under */ - fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); + /* + * The grant this read and the readahead behind it fill under. Not + * for O_DIRECT, which takes no DLM lock at all: it fills no page + * cache, and what it reads is the server's to order. + */ + if (!(iocb->ki_flags & IOCB_DIRECT)) + fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); /* * A NOTIFY invalidate racing this read drops the folios it @@ -2433,8 +2438,14 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * Only the append case must wait for the lock: its range * depends on i_size, which is stable only under the exclusive * inode lock. + * + * O_DIRECT takes no DLM lock at all, here or anywhere: it + * dirties no page cache, so there is nothing for a grant to + * cover, and its bytes are the server's to order against the + * rest of the cluster. */ - if (fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + if (fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + !(iocb->ki_flags & IOCB_APPEND)) { dlm_pos = iocb->ki_pos; dlm_len = iov_iter_count(from); @@ -2470,7 +2481,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* note that this small code dup will save us a lot of headache later * when appends are done concurrently without using parallel direct writes */ - if (fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + if (fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + (iocb->ki_flags & IOCB_APPEND)) { /* * An append write lands at the current EOF no matter * what ki_pos holds: generic_write_checks() rewrites @@ -2495,7 +2507,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * generic_write_checks() may have put ki_pos past the granted * range. Re-lock where the write really lands. */ - if (fc->dlm && (iocb->ki_flags & IOCB_APPEND) && + if (fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + (iocb->ki_flags & IOCB_APPEND) && iocb->ki_pos != dlm_pos) { dlm_pos = iocb->ki_pos; dlm_len = iov_iter_count(from); From 07f14d49806a99239f8cf1833496427a9d1a172f Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 11:37:22 +0200 Subject: [PATCH 15/24] fuse: cover the readahead lookahead with the read grant fuse_readahead() rounds the window mm built up to a whole request and fuse_readahead_lookahead() puts one more request past that, so the pages this client fills reach further than the ra_pages the read asked a grant for. Now that neither of them requests a grant of its own, those pages are the ones fuse_send_readpages() declines: the lookahead allocates a window, hands it straight back unfilled, and the chain it exists to keep alive dies at the first hop. Size the grant from the same unit the window is built in, so what readahead may fill is what the read covers. A wider read grant conflicts only with a remote writer; other readers share it. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 87290b629ee72f..5163aef9a81f4f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1064,6 +1064,27 @@ static int fuse_do_readpage(struct file *file, struct page *page) return 0; } +/* + * The unit the readahead window is built in: one request's worth of pages, + * never more than the readahead this fd was allowed. @ra is NULL where + * there is no per-fd state to bound it by. + * + * fuse_readahead() rounds its window up to this and fuse_readahead_lookahead() + * puts one more of them past it, so fuse_read_grant() sizes the grant from it + * as well and the two stay in step. + */ +static unsigned int fuse_readahead_unit(struct fuse_conn *fc, + struct file_ra_state *ra) +{ + unsigned int max_pages = min_t(unsigned int, fc->max_pages, + fc->max_read / PAGE_SIZE); + + if (!ra) + return max_pages; + + return min_t(unsigned int, max_pages, ra->ra_pages); +} + /** * fuse_read_grant - take the grant a page cache fill runs under * @file: file to read through @@ -1088,13 +1109,24 @@ static int fuse_read_grant(struct file *file, loff_t pos, size_t count) { struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); + struct file_ra_state *ra = &file->f_ra; loff_t size = i_size_read(inode); - loff_t ahead = (loff_t)file->f_ra.ra_pages << PAGE_SHIFT; loff_t end = pos + count; + unsigned int unit; + loff_t ahead; if (!fc->writeback_cache || !fc->dlm) return 0; + /* + * What readahead may fill past the read: the window mm builds, at + * most ra_pages wide, plus what this filesystem adds to it -- the + * round up to a whole request, under one unit, and the one request + * fuse_readahead_lookahead() puts past that. + */ + unit = fuse_readahead_unit(fc, ra); + ahead = (loff_t)(ra->ra_pages + 2 * unit) << PAGE_SHIFT; + if (end < size) end += min(ahead, size - end); @@ -1477,12 +1509,11 @@ static void fuse_readahead(struct readahead_control *rac) * negotiated large requests does not get to prefetch beyond the * window the admin allowed. */ - unit = max_pages; + unit = fuse_readahead_unit(fc, rac->ra); target_end = readahead_index(rac) + readahead_count(rac); if (rac->ra) { unsigned int nr = readahead_count(rac); - unit = min_t(unsigned int, max_pages, rac->ra->ra_pages); if (unit > 1 && nr % unit) target_end = readahead_index(rac) + roundup(nr, unit); } From b7d17268c93f33dea034ad79e38edd5cfc200b58 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 13:40:22 +0200 Subject: [PATCH 16/24] fuse: sample the attr version before the compound getattr fuse_file_open() reads the version once the reply is already back, so it is never older than anything that moved while the request was on the wire and fuse_change_attributes_i() applies the attributes however stale they have become. Under DLM that is a size from before the open shrinking i_size back under the writers. Sample it before the request, the way every other caller does. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5163aef9a81f4f..e93a89d0cece82 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -248,7 +248,20 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, if (inode && fc->compound_open_getattr) { struct fuse_attr_out attr_outarg; + u64 attr_version; + /* + * Sampled before the request, the way every other + * caller does it. fuse_change_attributes_i() drops + * a reply whose version is older than the inode's, + * and a version read once the reply is already back + * is never older than anything that moved while it + * was on the wire -- the reply would be applied + * however stale it had become, and under DLM that + * means a size from before the open shrinking + * i_size back under the writers. + */ + attr_version = fuse_get_attr_version(fc); err = fuse_compound_open_getattr(fm, nodeid, open_flags, opcode, ff, &attr_outarg, outargp); @@ -258,7 +271,7 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, fuse_change_attributes(inode, &attr_outarg.attr, NULL, ATTR_TIMEOUT(&attr_outarg), - fuse_get_attr_version(fc)); + attr_version); } if (err == -ENOSYS) { err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); From 4b9596f956396bf506559c8045a3e29efcb1466c Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 13:40:22 +0200 Subject: [PATCH 17/24] fuse: hold the size against any cached tail filemap_range_needs_writeback() covers the tail only while it is dirty. A reply that is merely behind always has those folios here but not always dirty: writeback can clean one while the reply that would move the server's size is still on the wire, and the beyond-EOF shortcut in fuse_write_begin() zeroes a folio over data the server does hold. Test for any cached folio instead, and ask the lock tree last since that query sleeps. A remote truncate still gets through, as it revokes first and the revoke drops the range. Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 56 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 8a5e09b46d12ec..d86c3f285e1e3d 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -498,9 +498,15 @@ u32 fuse_get_cache_mask(struct inode *inode) * for exactly what the grant covers: * * - size, when the server reports less than i_size and the tail it does not - * know about, [attr->size, i_size), is entirely under a write grant. - * Taking the server's answer would shrink i_size and have - * truncate_pagecache() throw the unwritten tail away. + * know about, [attr->size, i_size), is under a write grant or still + * cached. Taking the server's answer would shrink i_size, and a shrink + * is destructive here in a way it is not upstream, where the writeback + * cache always keeps STATX_SIZE and this never runs: truncate_pagecache() + * throws the tail away dirty or not, and fuse_send_writepage() crops the + * requests already queued over it, ending their writeback as if they had + * been sent. Neither reports an error, so fsync() succeeds over the + * hole. Upstream only lowers i_size under fuse_set_nowrite(), which is + * what makes both safe there. * - mtime and ctime, while the page cache is dirty or under writeback: our * writes have stamped them locally and the server's stamps predate them. * Only while the cache is actually dirty, not for as long as a grant @@ -509,9 +515,9 @@ u32 fuse_get_cache_mask(struct inode *inode) * beyond that would hide a remote chown or chmod indefinitely. * * A remote truncate cannot slip through. It has to revoke the grant first, - * and the revoke launders the tail and drops the grant, so by the time the - * smaller size is reported neither check holds and the server's answer is - * applied as usual. A grant the server made but that could not be recorded + * and the revoke launders the tail, drops the grant and drops the folios, so + * by the time the smaller size is reported none of the checks hold and the + * server's answer is applied as usual. A grant the server made but that could not be recorded * (FUSE_DLM_GRANT_UNRECORDED) is invisible to the lock tree and falls back to * trusting the server, as before. * @@ -542,25 +548,37 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, /* * The local size stays authoritative while the extension is - * covered by a write grant, and also while anything in - * [attr->size, size) is dirty or under writeback: those bytes - * exist only here, and taking the server's smaller size would - * truncate them away before they are ever sent. The grant check - * alone misses them, because a page-mkwrite grant is never + * covered by a write grant, and also while [attr->size, size) is + * cached at all: taking the server's smaller size would truncate + * those folios away, and lowering i_size crops the writeback + * requests already queued over them (fuse_send_writepage()), both + * of which throw bytes away without reporting an error. The grant + * check alone misses them, because a page-mkwrite grant is never * recorded and a local truncate revokes its own tail grants. * - * Both miss a write that has committed its extension and not yet - * dirtied the folio under it: nothing is dirty there, and a NOTIFY - * can revoke the grant in between. With i_rwsem held shared several - * writers sit in that window at once, which is why they are counted - * rather than flagged. + * Any folio, not just a dirty one. A reply that is merely behind + * -- the server has not seen the tail yet -- always has those + * folios here, but not always dirty: writeback can have cleaned + * one while the reply that would move the server's size was still + * on the wire, and the beyond-EOF shortcut in fuse_write_begin() + * turns a short i_size into a folio zeroed over data the server + * does hold. A remote truncate is not caught by mistake: it has + * to revoke first, and the revoke drops the range, so by the time + * the smaller size is reported there is nothing cached there and + * the shrink applies. + * + * All three miss a write that has committed its extension and not + * yet dirtied the folio under it: nothing is cached there, and a + * NOTIFY can revoke the grant in between. With i_rwsem held + * shared several writers sit in that window at once, which is why + * they are counted rather than flagged. Cheapest test first: the + * grant query sleeps. */ if (have_size && size > (loff_t) attr->size && (atomic_read(&fi->size_extenders) || + filemap_range_has_page(inode->i_mapping, attr->size, size - 1) || fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, - FUSE_PAGE_LOCK_WRITE) || - filemap_range_needs_writeback(inode->i_mapping, attr->size, - size - 1))) + FUSE_PAGE_LOCK_WRITE))) cache_mask |= STATX_SIZE; return cache_mask; From 9da7db72ae9574027eb233e3570075d89fafe553 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 13:40:23 +0200 Subject: [PATCH 18/24] fuse: crop the queued writepages against a high water mark fuse_send_writepage() crops a queued request to i_size and ends the writeback on everything past it without sending it. The folios come back clean with their bytes gone and nothing reports an error, so a later fsync() succeeds over the hole. That is what a truncate wants and silent data loss for every other shrink, which upstream never sees because a writeback mount keeps STATX_SIZE. With DLM the server's size is applied, and a reply behind the local writers crops queued data away. Crop against fi->wb_crop instead. A truncate lowers it under the freeze; otherwise it follows i_size back down only once no request is left to protect. Signed-off-by: Horst Birthelmer --- fs/fuse/dir.c | 1 + fs/fuse/file.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ fs/fuse/fuse_i.h | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 2bfb1f810322ee..78736a1636809b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2205,6 +2205,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, i_size_write(inode, outarg.attr.size); if (is_truncate) { + fuse_writeback_crop_truncated(inode, outarg.attr.size); /* NOTE: this may release/reacquire fi->lock */ __fuse_release_nowrite(inode); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e93a89d0cece82..93e2ed62542881 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -390,6 +390,7 @@ static void fuse_truncate_update_attr(struct inode *inode, struct file *file) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); i_size_write(inode, 0); + fuse_writeback_crop_truncated(inode, 0); spin_unlock(&fi->lock); file_update_time(file); fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); @@ -3179,10 +3180,41 @@ __releases(fi->lock) __acquires(fi->lock) { struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); loff_t crop = i_size_read(inode); struct fuse_writepage_args *wpa; + /* + * fuse_send_writepage() crops a queued request to the size passed + * here and ends the writeback on everything past it without ever + * sending it -- the folios come back clean with their bytes gone + * and nothing reports an error, so a later fsync() succeeds over + * the hole. That is right after a truncate, where those bytes are + * meant to disappear, and it is silent data loss for any other + * shrink. + * + * Upstream there is no other shrink on a writeback mount: + * fuse_get_cache_mask() keeps STATX_SIZE, so i_size never takes + * the server's answer and only a truncate lowers it, under + * fuse_set_nowrite(). With DLM the server's size is applied + * (fuse_attr_cache_mask()), and a reply that is merely behind the + * local writers -- the normal state of a shared file being written + * -- would crop gigabytes of queued data away. + * + * So crop against the high water mark, which a truncate moves back + * down (fuse_writeback_crop_truncated(), called under the freeze) + * and which follows i_size again as soon as no request is left to + * protect (fuse_writepage_end()). In the steady state it is + * i_size, including the partial folio at EOF that the crop exists + * to clip. + */ + if (fc->writeback_cache) { + if (crop > fi->wb_crop) + fi->wb_crop = crop; + crop = fi->wb_crop; + } + while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) { wpa = list_entry(fi->queued_writes.next, struct fuse_writepage_args, queue_entry); @@ -3212,6 +3244,17 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, spin_lock(&fi->lock); fi->writectr--; fuse_writepage_finish(wpa); + /* + * Nothing sent and nothing queued: the crop has no request left to + * protect, so let it follow i_size again rather than stay at a + * high water mark this inode may never reach a second time -- a + * file shrunk by another node and written again would otherwise + * have its last folio sent whole instead of clipped at EOF. See + * fuse_flush_writepages(). + */ + if (fc->writeback_cache && fi->writectr == 0 && + list_empty(&fi->queued_writes)) + fi->wb_crop = i_size_read(inode); spin_unlock(&fi->lock); fuse_writepage_free(wpa); } @@ -4979,6 +5022,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->write_stream_next = 0; fi->write_stream_start = 0; atomic_set(&fi->size_extenders, 0); + fi->wb_crop = 0; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index fb1a8c4298a75a..d5f5f58ed61078 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -322,6 +322,23 @@ struct fuse_inode { * and every writer clears it. */ atomic_t size_extenders; + + /* + * High water mark of i_size, and the size the + * queued writepage requests are cropped against. + * A truncate lowers it + * (fuse_writeback_crop_truncated()); otherwise it + * only follows i_size back down once no request is + * left to protect (fuse_writepage_end()). + * + * i_size itself cannot serve: with DLM the server's + * size is applied to it (fuse_attr_cache_mask()), + * and a reply that is only behind the local writers + * would have fuse_send_writepage() crop away bytes + * on their way out. Protected by fi->lock; + * writeback-cache regular files only. + */ + loff_t wb_crop; }; /* readdir cache (directory only) */ @@ -1604,6 +1621,23 @@ void fuse_flush_writepages(struct inode *inode); void fuse_set_nowrite(struct inode *inode); void fuse_release_nowrite(struct inode *inode); +/* + * A truncate has lowered i_size under fuse_set_nowrite(). That shrink is + * the authoritative one: the bytes above it are gone, and the writepage + * requests still standing over them are meant to be cropped away rather + * than kept. Let the crop follow i_size back down. + * + * Every other shrink -- under DLM, any attribute reply that is merely + * behind the local writers -- must not, which is what fi->wb_crop is for. + * Called under fi->lock; see fuse_flush_writepages(). + */ +static inline void fuse_writeback_crop_truncated(struct inode *inode, + loff_t size) +{ + if (get_fuse_conn(inode)->writeback_cache && S_ISREG(inode->i_mode)) + get_fuse_inode(inode)->wb_crop = size; +} + /** * Scan all fuse_mounts belonging to fc to find the first where * ilookup5() returns a result. Return that result and the From 464fd181dbcb6b751b37cb4f7d178c30b13ca6bc Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 13:40:23 +0200 Subject: [PATCH 19/24] fuse: invalidate the mapping through the writeback aware helper invalidate_inode_pages2() launders a dirty mapping one folio per round trip, and it waits for replies fuse_flush_writepages() holds back while writepages are frozen. fuse_notify_invalidate_range() already writes the range back in batched writeback and substitutes a plain drop under the freeze. Use it for the attribute change path too, and rename it as it is no longer notify only. Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d86c3f285e1e3d..36041927fd8a02 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -584,6 +584,10 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, return cache_mask; } +/* Defined below, next to the NOTIFY paths that are its other caller. */ +static void fuse_invalidate_mapping_range(struct inode *inode, pgoff_t start, + pgoff_t end, bool may_be_dirty); + static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, u64 attr_valid, u64 attr_version, u64 evict_ctr) @@ -664,8 +668,16 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr inval = true; } + /* + * Through the helper rather than invalidate_inode_pages2(): + * it writes the range back in batched writeback first, where + * the bare invalidate would launder it a folio per round + * trip, and it substitutes a plain drop while writepages are + * frozen, where laundering would wait for a reply that + * fuse_flush_writepages() is holding back. + */ if (inval) - invalidate_inode_pages2(inode->i_mapping); + fuse_invalidate_mapping_range(inode, 0, -1, true); } if (IS_ENABLED(CONFIG_FUSE_DAX)) @@ -933,8 +945,8 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, * fuse_do_setattr() invalidates the mapping after releasing the freeze, the * O_TRUNC open path calls truncate_pagecache(). */ -static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, - pgoff_t end, bool may_be_dirty) +static void fuse_invalidate_mapping_range(struct inode *inode, pgoff_t start, + pgoff_t end, bool may_be_dirty) { struct fuse_inode *fi = get_fuse_inode(inode); loff_t last; @@ -1128,7 +1140,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * writeback to block on. Either way a server that * revokes from a thread it also needs to answer * FUSE_WRITE on deadlocks here, the same contract - * fuse_notify_invalidate_range() states for a frozen + * fuse_invalidate_mapping_range() states for a frozen * inode. The error is left to the mapping, where * fsync collects it. */ @@ -1158,11 +1170,12 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * cached there. */ if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1, true); + fuse_invalidate_mapping_range(inode, 0, -1, + true); else if (has_pages) - fuse_notify_invalidate_range(inode, pg_start, - pg_end, - may_be_dirty); + fuse_invalidate_mapping_range(inode, pg_start, + pg_end, + may_be_dirty); if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", @@ -1175,8 +1188,8 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - fuse_notify_invalidate_range(inode, pg_start, pg_end, - true); + fuse_invalidate_mapping_range(inode, pg_start, pg_end, + true); } if (fenced) fuse_dlm_revoke_end(fi, &fence); From 7b93219fa7ecda08c0297e52d1b7c9ae7b905f42 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 13:40:23 +0200 Subject: [PATCH 20/24] fuse: keep the data cache on a size change under dlm A size that differs from the cached one is upstream's evidence that another client wrote the file. Under DLM the evidence is redundant: a remote write has to revoke the grant covering it, and the revoke invalidates the range on its way through. It is also not affordable. Another node extends a shared file, the GETATTR that fuse_cache_write_iter() takes for STATX_MODE returns the larger size, and fuse_attr_cache_mask() holds STATX_SIZE only for the opposite direction, so a write that wanted the mode bits launders the whole mapping with the writer parked behind it. Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 36041927fd8a02..9117ba8eed2dce 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -652,7 +652,34 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr if (have_size && oldsize != attr->size) { truncate_pagecache(inode, attr->size); - if (!fc->explicit_inval_data) + /* + * A size that differs from the cached one is + * upstream's evidence that another client wrote + * the file, and the data cache is dropped on the + * strength of it. Under DLM that evidence is + * redundant: a remote write cannot land without + * revoking the grant covering it, and the revoke + * invalidates the range on its way through + * (fuse_reverse_inval_inode()). Coherency is the + * grant's job here, not the size delta's. + * + * It is also not affordable. A shared file reaches + * this on every write: another node extends it, the + * GETATTR that fuse_cache_write_iter() takes for + * STATX_MODE returns the larger size, and + * fuse_attr_cache_mask() holds STATX_SIZE only for + * the opposite direction -- a local size ahead of + * the server's. So a write that wanted the mode + * bits invalidates the whole mapping, and on a + * writeback mapping that is a laundering pass, not + * a drop: every dirty folio in the file goes out + * one FUSE_WRITE at a time, with the writer parked + * behind all of it. + * + * The size itself is still taken below. Only the + * data cache is kept. + */ + if (!fc->explicit_inval_data && !fc->dlm) inval = true; } else if (have_mtime && fc->auto_inval_data) { struct timespec64 new_mtime = { From ea9aefb94781dee9e861a6d8d6f4093b39c9857e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 23:19:05 +0200 Subject: [PATCH 21/24] fuse: keep the cache when an attribute reply shrinks the size fuse_attr_cache_mask() decides whether the server's smaller size may be taken, then sleeps in the grant query before its answer is used. A write below EOF extends nothing, so it bumps neither fi->attr_version, which would drop the reply, nor fi->size_extenders, which would hold the size. Folios dirtied in the doomed range during that window are invisible to all three legs. Re-testing closer to the truncate is not enough on its own. truncate_pagecache() runs after fi->lock is dropped and holds nothing a writer holds, so a folio dirtied between the decision and the walk, or during the walk, is discarded anyway, with no error to report it: the bytes are gone and a later fsync() succeeds over the hole. A real truncate may discard the cache because fuse_set_nowrite() and i_rwsem hold the writers off; an attribute reply holds off nothing. So keep the folios, and re-test the size under fi->lock where nothing sleeps between the answer and its use. If the size was wrong the folios are written back and the size recovers, and fuse_flush_writepages() crops against a high water mark so nothing is clipped meanwhile. If it was right, the revoke that had to precede it already laundered and dropped the range, so there is nothing left to discard. Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 9117ba8eed2dce..1c86892e891d2b 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -626,6 +626,30 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr evict_ctr); oldsize = inode->i_size; + /* + * fuse_attr_cache_mask() answered before the grant query slept, and a + * write below EOF bumps neither fi->attr_version nor + * fi->size_extenders: it extends nothing, so the version check cannot + * drop the reply and the count cannot hold the size. Folios can have + * been dirtied in the doomed range since the answer, and + * truncate_pagecache() below throws them away with no error to report + * it. + * + * Re-test here instead, where nothing sleeps between the answer and + * the truncate acting on it, and keep the local size when the range + * still holds bytes the server has not seen. A remote truncate is + * unaffected: it revokes first, and the revoke launders and drops the + * range, so there is nothing here to find. + */ + if (have_size && !(cache_mask & STATX_SIZE) && fc->dlm && + fc->writeback_cache && S_ISREG(inode->i_mode) && + (loff_t) attr->size < oldsize && + filemap_range_needs_writeback(inode->i_mapping, attr->size, + oldsize - 1)) { + cache_mask |= STATX_SIZE; + attr->size = oldsize; + } + /* * In case of writeback_cache enabled, the cached writes beyond EOF * extend local i_size without keeping userspace server in sync. So, @@ -651,7 +675,23 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr bool have_mtime = !sx || (sx->mask & STATX_MTIME); if (have_size && oldsize != attr->size) { - truncate_pagecache(inode, attr->size); + /* + * Not under DLM. This runs after fi->lock is + * dropped and takes nothing a writer holds, so a + * folio dirtied between the decision and the walk, + * or during it, is discarded with no error to report + * it. A real truncate may do this because + * fuse_set_nowrite() and i_rwsem hold the writers + * off; an attribute reply holds off nothing. + * + * Keeping the folios costs nothing either way. If + * the size was wrong they are written back and the + * size recovers; if it was right, the revoke that had + * to precede it already dropped the range and there + * is nothing here to discard. + */ + if (!(fc->dlm && fc->writeback_cache)) + truncate_pagecache(inode, attr->size); /* * A size that differs from the cached one is * upstream's evidence that another client wrote From 34b300607ea02eee598e4b7e3489b7d0e2fae73f Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 21:06:42 +0200 Subject: [PATCH 22/24] fuse: do not accept a short writeback write A server may acknowledge fewer bytes than the FUSE_WRITE carried. The write path acts on that: fuse_send_write() returns the short count and fuse_perform_write() stops there. Writeback had no caller to report it to and ended every folio clean, so the bytes past the reply were dropped with nothing recording it and a later fsync() succeeded over the hole. Redirty the folios the reply does not reach, so the bytes survive for another pass, and set the error on the mapping: the redirty lands after the wait an fsync() in progress has already done, and that fsync would otherwise return success over them once more. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 93e2ed62542881..9c83098d7feca1 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3109,6 +3109,11 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) kfree(wpa); } +/* Defined below, with the writeback paths that are its other callers. */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct writeback_control *wbc, + struct folio *folio); + static void fuse_writepage_finish(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -3231,6 +3236,29 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, struct inode *inode = wpa->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_args_pages *ap = &wpa->ia.ap; + + /* + * A server that acknowledges fewer bytes than it was sent has not + * taken the rest. The write path acts on that (fuse_send_write() + * returns the short count and fuse_perform_write() stops on it); + * writeback had no caller to report it to and ended every folio + * clean, so the tail was dropped with nothing to say so and a later + * fsync() succeeded over the hole. + * + * Put the folios the reply does not reach back on the dirty list so + * the bytes survive for another pass, and record the error: the + * redirty lands after the wait an fsync() in progress already did, + * which would otherwise return success over them again. + */ + if (!error && wpa->ia.write.out.size < wpa->ia.write.in.size) { + unsigned int i = wpa->ia.write.out.size >> PAGE_SHIFT; + + for (; i < ap->num_pages; i++) + fuse_writeback_redirty(fc, NULL, + page_folio(ap->pages[i])); + error = -EIO; + } mapping_set_error(inode->i_mapping, error); /* From 881356988d6ebfe289be47a6fc20714887f28b70 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 23:59:35 +0200 Subject: [PATCH 23/24] fuse: compare the alignment bound in page indices fuse_writepage_need_send() tests the next alignment against wbc->range_end, a page index against a byte offset. It also reads that field for a cyclic writeback, which does not set it: write_cache_pages() runs to the end of the mapping there and leaves range_end at zero, so the test is true for every aligned index and each folio is sent as a request of its own. Take the bound in page indices, and to the end of the mapping when the pass is cyclic. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 9c83098d7feca1..903def0b3d7ccd 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3618,9 +3618,18 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, /* Reached alignment */ if (fc->alignment_pages && !(page->index % fc->alignment_pages)) { - /* we are at a point where we would write aligned - * check if we potentially could reach the next alignment */ - if (page->index + fc->alignment_pages > wbc->range_end) + /* + * We are at a point where we would write aligned, so check + * whether the next alignment is still inside the range this + * pass covers. Both sides in page indices: wbc->range_end is + * a byte offset, and it carries nothing at all for a cyclic + * writeback, where write_cache_pages() runs to the end of the + * mapping instead. + */ + pgoff_t last = wbc->range_cyclic ? (pgoff_t) -1 : + (pgoff_t) (wbc->range_end >> PAGE_SHIFT); + + if (page->index + fc->alignment_pages > last) return true; if (ap->num_pages + fc->alignment_pages > fc->max_pages) From dce32262157adc3a9f3b0a733919a041f8a99f6d Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sun, 6 Sep 2026 00:00:02 +0200 Subject: [PATCH 24/24] fuse: read the write stream marks once fuse_writeback_kick_stream() reads fi->write_stream_start twice in the same decision, once to round down into the start of the range and again to compare against its end. A writer moving the mark in between inverts the range the kick is given. Racing writers are meant to cost a kick rather than correctness, which needs each mark read once into a local. Take the end from this write's own position rather than reading back the mark just stored, and annotate the accesses. Still no lock: the run is a hint, and nothing waits for it. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 903def0b3d7ccd..48a09dd42efa3a 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2336,7 +2336,10 @@ static bool fuse_write_stream_update(struct fuse_inode *fi, size_t len) * A hint only: nothing waits for it, and a chunk that is partly dirty or * already written back sends what it has. The run is read and written * without the inode lock, which the DLM path holds shared, so writers - * landing on it together cost a kick, not correctness. + * landing on it together cost a kick, not correctness. Each mark is read + * once into a local for that to hold: reading write_stream_start twice, + * to round down and again to compare, lets a writer moving it in between + * invert the range the kick is given. */ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, size_t len, bool stream) @@ -2344,25 +2347,27 @@ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct fuse_inode *fi = get_fuse_inode(inode); - loff_t chunk, start, end; + loff_t chunk, run, start, end; bool sequential; - sequential = pos == fi->write_stream_next; - fi->write_stream_next = pos + (loff_t)len; + sequential = pos == READ_ONCE(fi->write_stream_next); + WRITE_ONCE(fi->write_stream_next, pos + (loff_t)len); if (!sequential || !stream) { /* Nothing behind this write is known to be finished */ - fi->write_stream_start = pos; + WRITE_ONCE(fi->write_stream_start, pos); return; } + run = READ_ONCE(fi->write_stream_start); chunk = fuse_write_chunk_size(get_fuse_conn(inode)); - start = round_down(fi->write_stream_start, chunk); - end = round_down(fi->write_stream_next, chunk); + start = round_down(run, chunk); + /* This write's own end, not the mark another writer may have moved */ + end = round_down(pos + (loff_t)len, chunk); /* No bound crossed, so nothing has been left complete */ - if (end <= fi->write_stream_start) + if (end <= run) return; - fi->write_stream_start = end; + WRITE_ONCE(fi->write_stream_start, end); /* * What filemap_fdatawrite_range_kick() does upstream: a range write * with no integrity, which this kernel has no helper for.