diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index bc3e495d619d06..59c11ece2f95e5 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2074,15 +2074,32 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. + * + * Revoke and drop under the coherency gate write side, + * like the NOTIFY invalidate path: a gate reader that + * already re-validated its grant must not have the + * lock tree and the cache yanked mid-hold, or it + * would repopulate the truncated range trusting a + * grant that no longer exists. Waiting for gate + * readers here is safe: we hold i_rwsem exclusive, so + * no gate holder can be waiting on it (the write path + * takes i_rwsem before the gate, the read path never + * takes it). */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); i_size_write(inode, 0); truncate_pagecache(inode, 0); + if (wb_sem) + percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2186,11 +2203,23 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + + /* + * Revoke and drop under the coherency gate write side; see + * the atomic-O_TRUNC branch above. i_rwsem is held + * exclusive here as well (setattr), so waiting out gate + * readers cannot deadlock. + */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); + if (wb_sem) + percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 55ab5e9cf61715..854f08303d6f66 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -414,6 +414,18 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, if (likely(fi)) { spin_lock(&fi->lock); list_del(&ff->write_entry); + /* + * Leave forced direct IO mode once the last writer is gone: with + * no local writer left there is no cached-write contention with + * the remote modifier that triggered the switch. Restore + * FUSE_I_CACHE_IO_MODE for any frozen cached opens. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + list_empty(&fi->write_files)) { + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + } spin_unlock(&fi->lock); } spin_lock(&fc->lock); @@ -450,9 +462,24 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_release_args *ra = ff->release_args; int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE; + bool was_force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); fuse_prepare_release(fi, ff, open_flags, opcode, false); + /* + * If this release dropped the last writer, fuse_prepare_release() + * cleared the forced-direct-IO latch (under fi->lock). Drop any clean + * folios a read racing the latch may have repopulated so they cannot be + * served stale once caching mode resumes. No inode lock or + * wb_inval_rwsem: release may run on the fuse server thread (async fput + * from aio completion), where blocking on a contended inode lock could + * stall the connection. Writes were routed direct while latched, so + * only clean folios exist and this invalidate is server-free; the last + * writer is gone, so no forced-dio writer can race the drop. + */ + if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state)) + invalidate_inode_pages2(inode->i_mapping); + if (ra && ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id); @@ -1095,10 +1122,23 @@ static void fuse_readahead(struct readahead_control *rac) } } +static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); + +/* + * Bound on re-requesting a revoked DLM grant before a cached read is + * served unlocked; see fuse_cache_read_iter(). + */ +#define FUSE_DLM_READ_RETRIES 3 + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { - struct inode *inode = iocb->ki_filp->f_mapping->host; + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + ssize_t res; + int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1113,7 +1153,69 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } - return generic_file_read_iter(iocb, to); + /* if we have dlm support acquire a read lock for the area + * we are reading from. */ + if (fc->writeback_cache && fc->dlm) + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); + + /* + * Fence the cache-serving read against a NOTIFY invalidate so we never + * hand back a folio the server has just superseded. The gate read side + * is per-CPU cheap; the NOTIFY holds the write side with priority. + * Re-check the forced-DIO latch under it: if a storm latched us while we + * waited on a pending writer, reroute to direct like the buffered write + * path, so we do not repopulate the cache the latch just dropped. + * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + */ + if (wb_sem) { + int tries = FUSE_DLM_READ_RETRIES; + +retry: + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + return fuse_direct_read_iter(iocb, to); + } + /* + * The DLM lock was requested before entering the gate, and + * the NOTIFY invalidate we may just have waited on revokes + * locks under the gate write side. Re-check the grant here + * and re-request with the gate dropped, so a + * FUSE_DLM_WB_LOCK round trip never parks a pending + * invalidate behind our own gate hold. Once the check + * passes the lock cannot go away for the rest of the gate + * hold. A failed or unrecorded request falls through + * unlocked, as before: the retry is taken even then (the + * latch must be re-checked under the re-entered gate), so + * lock_err has to stay sticky across it -- seeded by the + * pre-gate request above -- or a grant that failed would + * be re-requested forever. The retry is also bounded: a + * remote writer can revoke each successful grant before + * the gate is re-entered, and a reader-only inode has no + * force-DIO latch to end such a storm, so after + * FUSE_DLM_READ_RETRIES re-requests the read is served + * unlocked rather than looping without bound. + */ + if (!lock_err && fc->dlm && tries-- > 0 && + !fuse_dlm_lock_is_held(fi, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ)) { + percpu_up_read(wb_sem); + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); + goto retry; + } + } + + res = generic_file_read_iter(iocb, to); + + if (wb_sem) + percpu_up_read(wb_sem); + + return res; } static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, @@ -1400,9 +1502,15 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); + bool force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); - /* Server side has to advise that it supports parallel dio writes. */ - if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) + /* + * Server side has to advise that it supports parallel dio writes. + * When the inode is latched into forced direct IO, parallel writes are + * used unconditionally: the page cache has been flushed and is bypassed + * for this inode. + */ + if (!force_dio && !(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) return true; /* @@ -1413,7 +1521,7 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from return true; /* shared locks are not allowed with parallel page cache IO */ - if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) + if (!force_dio && test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) return true; /* Parallel dio beyond EOF is not supported, at least for now. */ @@ -1424,7 +1532,7 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from } static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, - bool *exclusive) + bool *exclusive, bool *uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1440,17 +1548,31 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, * should be performed only after taking shared inode lock. * Previous past eof check was without inode lock and might * have raced, so check it again. + * + * Under the forced-dio latch the cached/uncached accounting is + * bypassed (the latch guarantees the cache is flushed and not + * repopulated), so no reference is taken. Report back through + * *uncached whether one actually was, so fuse_dio_unlock() can + * drop it based on that decision rather than re-testing the + * latch, which the notify-storm path can toggle mid-write. */ - if (fuse_io_past_eof(iocb, from) || - fuse_inode_uncached_io_start(fi) != 0) { + if (fuse_io_past_eof(iocb, from)) { inode_unlock_shared(inode); inode_lock(inode); *exclusive = true; + } else if (!test_bit(FUSE_I_FORCE_DIO, &fi->state)) { + if (fuse_inode_uncached_io_start(fi) != 0) { + inode_unlock_shared(inode); + inode_lock(inode); + *exclusive = true; + } else { + *uncached = true; + } } } } -static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) +static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1459,11 +1581,34 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) inode_unlock(inode); } else { /* Allow opens in caching mode after last parallel dio end */ - fuse_inode_uncached_io_end(fi); + if (uncached) + fuse_inode_uncached_io_end(fi); inode_unlock_shared(inode); } } +static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); + +/* + * Request the DLM write lock covering a cached write. -ENOSYS cleared + * fc->dlm: the server has no DLM, proceed as a plain cached write. Any + * other failure means the cache would be dirtied without DLM coverage - + * the caller must fail the write instead. A granted-but-unrecorded + * lock (positive return) is covered cluster-wide; proceed, but flag it + * so the in-gate re-validation skips a check an invisible grant could + * never pass. + */ +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, + bool *unrecorded) +{ + int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + + if (err < 0 && err != -ENOSYS) + return err; + *unrecorded = err > 0; + return 0; +} + static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1472,6 +1617,15 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = mapping->host; ssize_t err; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + bool wb_guard = false; + bool dlm_unrecorded = false; + loff_t dlm_pos = 0; + size_t dlm_len = 0; + + if (fuse_inode_force_dio(inode)) + return fuse_direct_write_iter(iocb, from); if (fc->writeback_cache) { /* Update size (EOF optimization) and mode (SUID clearing) */ @@ -1486,23 +1640,130 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto writethrough; } - /* if we have dlm support acquire the lock for the area - * we are writing into */ - if (fc->dlm) { - /* note that a file opened with O_APPEND will have relative values - * in ki_pos. This code is here for convenience and for libfuse overlay test. - * Filesystems should handle O_APPEND with 'direct io' to additionally - * get the performance benefits of 'parallel direct writes'. */ - loff_t pos = file->f_flags & O_APPEND ? i_size_read(inode) + iocb->ki_pos : iocb->ki_pos; - size_t length = iov_iter_count(from); - fuse_get_dlm_write_lock(file, pos, length); + /* + * Request the DLM write lock before taking i_rwsem: the request + * is an unbounded cluster round trip, and holding the + * writer-priority rwsem across it would park a truncate -- and + * behind it every later writer -- for the duration. The + * grant-to-use window this leaves open is closed by the in-gate + * re-validation below. Only the append case must wait for the + * lock: its range depends on i_size, which is stable only under + * the exclusive inode lock. + */ + if (fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + dlm_pos = iocb->ki_pos; + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + return err; + } + + /* + * generic_file_write_iter() is open-coded from here so the gate + * can be taken inside the inode lock. The order matters: every + * gate holder must take i_rwsem first and the gate second, so + * that a writer holding i_rwsem exclusive can wait out gate + * readers without deadlocking. + */ + inode_lock(inode); + + /* note that this small code dup will save us a lot of headache later + * when appends are done concurrently without using parallel direct writes */ + if (fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + /* + * An append write lands at the current EOF no matter + * what ki_pos holds: generic_write_checks() rewrites + * ki_pos to i_size for IOCB_APPEND, and i_size is stable + * here because append writes hold the inode lock + * exclusive. Lock where the data will land. + */ + dlm_pos = i_size_read(inode); + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + goto wb_out; } - return generic_file_write_iter(iocb, from); + + /* + * Hold the coherency gate (wb_inval_rwsem) for read across the + * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- + * which takes the write side (blocking, with priority) around + * its invalidate + latch set -- cannot strand the folios we are + * about to write. Re-check the latch under it (it may have been + * set while we blocked on the inode lock) and re-route to the + * direct path if set. Then re-validate the DLM grant: the + * request above ran outside the gate, and the invalidate we may + * just have waited on revokes grants under the gate write side. + */ + wb_guard = !!wb_sem; + if (wb_guard) { +retry: + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + inode_unlock(inode); + return fuse_direct_write_iter(iocb, from); + } + if (fc->dlm && !dlm_unrecorded && + !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, + FUSE_PAGE_LOCK_WRITE)) { + percpu_up_read(wb_sem); + err = fuse_cache_wr_dlm_lock(file, dlm_pos, + dlm_len, + &dlm_unrecorded); + if (err) { + /* The gate is already dropped; funnel + * the failure through the one audited + * exit. */ + wb_guard = false; + goto wb_out; + } + goto retry; + } + } + + err = generic_write_checks(iocb, from); + if (err > 0) + err = __generic_file_write_iter(iocb, from); + +wb_out: + if (wb_guard) + percpu_up_read(wb_sem); + inode_unlock(inode); + + if (err > 0) + err = generic_write_sync(iocb, err); + + return err; } writethrough: inode_lock(inode); + /* + * The forced-direct-IO latch feature is active under writeback+dlm; + * hold the coherency gate (wb_inval_rwsem) for read across the + * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- which takes + * the write side (blocking, with priority) around its invalidate + latch + * set -- cannot strand the folios we are about to write. Re-check the + * latch under it and re-route to the direct path if set. No DLM + * re-validation here: this path never requested a DLM lock (the + * writeback branch above returns before reaching it). + */ + wb_guard = !!wb_sem; + if (wb_guard) { + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + inode_unlock(inode); + return fuse_direct_write_iter(iocb, from); + } + } + err = generic_write_checks(iocb, from); if (err <= 0) goto out; @@ -1525,6 +1786,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from); } out: + if (wb_guard) + percpu_up_read(wb_sem); inode_unlock(inode); if (written > 0) written = generic_write_sync(iocb, written); @@ -1744,10 +2007,11 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); struct address_space *mapping = inode->i_mapping; loff_t pos = iocb->ki_pos; + bool exclusive = false; + bool uncached = false; ssize_t res; - bool exclusive; - fuse_dio_lock(iocb, from, &exclusive); + fuse_dio_lock(iocb, from, &exclusive, &uncached); res = generic_write_checks(iocb, from); if (res > 0) { if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { @@ -1768,7 +2032,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) (pos + res - 1) >> PAGE_SHIFT); } } - fuse_dio_unlock(iocb, exclusive); + fuse_dio_unlock(iocb, exclusive, uncached); return res; } @@ -1785,7 +2049,7 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) if (FUSE_IS_DAX(inode)) return fuse_dax_read_iter(iocb, to); - if (!(ff->open_flags & FOPEN_DIRECT_IO)) + if (!(ff->open_flags & FOPEN_DIRECT_IO) && !fuse_inode_force_dio(inode)) return fuse_cache_read_iter(iocb, to); else return fuse_direct_read_iter(iocb, to); @@ -1803,7 +2067,7 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (FUSE_IS_DAX(inode)) return fuse_dax_write_iter(iocb, from); - if (!(ff->open_flags & FOPEN_DIRECT_IO)) + if (!(ff->open_flags & FOPEN_DIRECT_IO) && !fuse_inode_force_dio(inode)) return fuse_cache_write_iter(iocb, from); else return fuse_direct_write_iter(iocb, from); @@ -2359,7 +2623,7 @@ static void fuse_vma_close(struct vm_area_struct *vma) /** * Request a DLM lock from the FUSE server. * - * This routine is similar to fuse_get_dlm_write_lock(), but it + * This routine is similar to fuse_get_dlm_lock(), but it * does not cache the DLM lock in the kernel. */ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t length) @@ -2469,6 +2733,29 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) if (FUSE_IS_DAX(file_inode(file))) return fuse_dax_mmap(file, vma); + /* + * If the inode was latched into forced direct IO after a remote-modify + * notification, a mapping needs the page cache, so revert to caching + * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * under mmap_lock and the buffered write path holds both across a fault + * on the user buffer (which takes mmap_lock), so taking either here + * would invert lock order (ABBA). Clearing the latch and dropping the + * cache is sufficient -- writers re-check the latch and route to cached + * IO once it is clear, and in-flight parallel dio drains itself. Cached + * opens frozen while latched are still counted in iocachectr, so restore + * FUSE_I_CACHE_IO_MODE for them. + */ + if (fuse_inode_force_dio(file_inode(file))) { + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + + spin_lock(&fi->lock); + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + spin_unlock(&fi->lock); + invalidate_inode_pages2(file->f_mapping); + } + /* * FOPEN_DIRECT_IO handling is special compared to O_DIRECT, * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP. @@ -3274,6 +3561,7 @@ static const struct address_space_operations fuse_file_aops = { void fuse_init_file_inode(struct inode *inode, unsigned int flags) { struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); inode->i_fop = &fuse_file_operations; inode->i_data.a_ops = &fuse_file_aops; @@ -3285,6 +3573,25 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); + /* + * Coherency gate for the forced-direct-IO feature; only writeback+dlm + * regular files need it. A percpu_rw_semaphore embeds per-CPU state, + * so allocate it out of line and only when the mount can use it rather + * than paying it on every inode. On failure leave it NULL: the gate + * stays inactive (best-effort invalidate) and the inode is still usable. + */ + fi->wb_inval_rwsem = NULL; + if (fc->writeback_cache && fc->dlm) { + struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); + + if (sem && percpu_init_rwsem(sem)) { + kfree(sem); + sem = NULL; + } + fi->wb_inval_rwsem = sem; + } + fi->notify_stamp = jiffies; + fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index d765dd8018cc6a..960d51e7836a3c 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -31,6 +31,12 @@ struct fuse_dlm_range { #define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ #define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ +/* + * Bound on re-requesting a grant whose recording lost against a + * concurrent revoke; see fuse_get_dlm_lock(). + */ +#define FUSE_DLM_RECORD_TRIES 3 + /* Interval tree definitions for page ranges */ static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) { @@ -63,6 +69,7 @@ int fuse_dlm_cache_init(struct fuse_inode *inode) init_rwsem(&cache->lock); cache->ranges = RB_ROOT_CACHED; + cache->revoke_gen = 0; return 0; } @@ -84,6 +91,7 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) /* Release all locks */ down_write(&cache->lock); + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); while ((node = rb_first_cached(&cache->ranges)) != NULL) { range = rb_entry(node, struct fuse_dlm_range, rb); fuse_page_it_remove(range, &cache->ranges); @@ -120,26 +128,25 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, uint64_t end) { struct fuse_dlm_range *range, *next; - struct rb_node *node; + uint64_t first = start ? start - 1 : start; + uint64_t last = end < U64_MAX ? end + 1 : end; if (!cache) return; - /* Find the first range that might need merging */ - range = NULL; - node = rb_first_cached(&cache->ranges); - while (node) { - range = rb_entry(node, struct fuse_dlm_range, rb); - if (range->end >= start - 1) - break; - node = rb_next(node); - } - - if (!range || range->start > end + 1) - return; + /* + * Find the first range that might need merging. Directly adjacent + * ranges can merge, hence the region is widened by one unit to each + * side (saturating at the type bounds). This must stay an + * interval-tree lookup: the tree holds every cached grant of the + * inode and strided writers grow it for the lifetime of the file, + * so seeding the merge by walking from the tree minimum would make + * every new grant cost a full scan. + */ + range = fuse_page_it_iter_first(&cache->ranges, first, last); /* Try to merge ranges in and around the specified region */ - while (range && range->start <= end + 1) { + while (range && range->start <= last) { /* Get next range before we potentially modify the tree */ next = NULL; if (rb_next(&range->rb)) { @@ -150,11 +157,11 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, /* Try to merge with next range if adjacent and same mode */ if (next && range->mode == next->mode && range->end + 1 == next->start) { - /* Merge ranges */ - range->end = next->end; - - /* Remove next from tree */ + /* Merge ranges: re-insert so __subtree_end is updated */ fuse_page_it_remove(next, &cache->ranges); + fuse_page_it_remove(range, &cache->ranges); + range->end = next->end; + fuse_page_it_insert(range, &cache->ranges); kfree(next); /* Continue with the same range */ @@ -167,11 +174,13 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, } /** - * fuse_dlm_lock_range - Lock a range of pages + * __fuse_dlm_lock_range - Lock a range of pages * @cache: The page cache * @start: Start page offset * @end: End page offset * @mode: Lock mode (read or write) + * @genp: If non-NULL, the revocation generation sampled before the grant + * was requested; recording fails with -EAGAIN if it has moved * * Add a locked range on the specified range of pages. * If parts of the range are already locked, only add the remaining parts. @@ -182,12 +191,14 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + const uint64_t *genp) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; int lock_mode; + bool covered_to_end = false; int ret = 0; LIST_HEAD(to_lock); LIST_HEAD(to_upgrade); @@ -202,6 +213,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, down_write(&cache->lock); + /* + * A revoke was processed after @genp was sampled; the grant this + * record carries may be the very one it targeted (a revoke of a + * not-yet-recorded grant removes nothing and would never be + * retried). Refuse, the caller re-requests. + */ + if (genp && cache->revoke_gen != *genp) { + up_write(&cache->lock); + return -EAGAIN; + } + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -233,14 +255,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, } /* Move current_start past this range */ - current_start = max(current_start, range->end + 1); + if (range->end >= end) + covered_to_end = true; + else + current_start = max(current_start, range->end + 1); /* Move to next range */ range = next; } /* If there's a gap after the last range to the end, extend the range */ - if (current_start <= end) { + if (!covered_to_end && current_start <= end) { new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); if (!new_range) { ret = -ENOMEM; @@ -294,6 +319,35 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, return ret; } +int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, NULL); +} + +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, &gen); +} + +/** + * fuse_dlm_revoke_gen - sample the revocation generation + * @inode: the fuse inode + * + * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The + * reply and a NOTIFY revoke can be serviced on different threads, so a + * revoke may be processed between the reply arriving and its grant + * being recorded. fuse_dlm_lock_range_gen() re-checks the generation + * under the cache lock and refuses to record a grant such a revoke may + * have already killed. + */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +{ + return READ_ONCE(inode->dlm_locked_areas.revoke_gen); +} + /** * fuse_dlm_punch_hole - Punch a hole in a locked range * @cache: The page cache @@ -322,13 +376,17 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, /* If the hole is at the beginning of the range */ if (start == range->start) { + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } /* If the hole is at the end of the range */ if (end == range->end) { + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } @@ -362,8 +420,12 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, * @start: Start page offset * @end: End page offset * - * Release locks on the specified range of pages. - * Note that if start and end are set to zero the cache is destroyed. + * Release locks on the specified range of pages. An inverted range is + * rejected rather than silently removing nothing: the callers revoke + * coverage, and a revoke that quietly keeps the grant alive would let + * the re-validating IO paths trust a lock the server has taken away. + * To drop every grant use fuse_dlm_cache_release_locks() (there is no + * in-band sentinel range for it). * * Return: 0 on success, negative error code on failure */ @@ -374,16 +436,19 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, struct fuse_dlm_range *range, *next; int ret = 0; - if (!cache) + if (!cache || start > end) return -EINVAL; - if (start == 0 && end == 0) { - fuse_dlm_cache_release_locks(inode); - return 0; - } - down_write(&cache->lock); + /* + * Unconditional, even when nothing overlaps: the revoke racing + * with an in-flight grant finds an empty tree precisely because + * the grant is not recorded yet, and the bump is what makes the + * recording side notice (see fuse_dlm_lock_range_gen()). + */ + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -400,10 +465,14 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, break; } else if (start > range->start) { /* Adjust the end of the range */ + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); } else if (end < range->end) { /* Adjust the start of the range */ + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); } else { /* Complete overlap, remove the range */ fuse_page_it_remove(range, &cache->ranges); @@ -454,9 +523,16 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, /* Check if the entire range is covered */ while (range && current_start <= end) { - /* If we're checking for a specific mode, verify it matches */ - if (lock_mode && range->mode != lock_mode) { - /* Wrong lock mode */ + /* + * The held lock must be at least as strong as the one + * requested. A WRITE lock (exclusive) satisfies a READ + * request, so only treat the range as uncovered when the + * held mode is weaker than what we ask for. This avoids + * re-requesting a READ lock for a range we already hold + * a WRITE lock on (e.g. read-after-write). + */ + if (lock_mode && range->mode < lock_mode) { + /* Held lock is weaker than requested */ up_read(&cache->lock); return false; } @@ -468,6 +544,12 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, return false; } + /* Covered through the end of the requested range? */ + if (range->end >= end) { + up_read(&cache->lock); + return true; + } + /* Move current_start past this range */ current_start = range->end + 1; @@ -487,42 +569,97 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, } /** - * request a dlm lock from the fuse server + * fuse_dlm_lock_is_held - check that a byte range is covered by a granted lock + * @fi: the fuse inode + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Re-validation helper for fuse_get_dlm_lock() callers: checks the same + * page-aligned range a fuse_get_dlm_lock() call with these arguments + * requests, against the live lock tree. */ -void fuse_get_dlm_write_lock(struct file *file, loff_t offset, - size_t length) +bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); + + /* + * An empty range needs no coverage. Reporting it held keeps the + * re-validating IO paths from re-requesting a lock the tree can + * never show (the page-aligned end would invert below). + */ + if (!length) + return true; + + return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); +} + +/** + * fuse_get_dlm_lock - request a dlm lock from the fuse server + * @file: the file being accessed + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Return: 0 when the range is covered by a recorded grant on return, + * FUSE_DLM_GRANT_UNRECORDED when the server granted the lock but + * recording it failed (covered cluster-wide, invisible to + * fuse_dlm_lock_is_held()), a negative error code otherwise. Callers + * re-validating the grant must not re-request on a nonzero return or + * they would spin. + */ +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; - uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); - - /* note that the offset and length don't have to be page aligned here - * but since we only get here on writeback caching we will send out - * page aligned requests */ - offset &= PAGE_MASK; FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; + uint64_t gen; + int tries = FUSE_DLM_RECORD_TRIES; int err; + /* An empty range needs no lock. */ + if (!length) + return 0; + +restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care - * of any races in lock requests */ - if (fuse_dlm_range_is_locked(fi, offset, - end, FUSE_PAGE_LOCK_WRITE)) - return; /* we already have this area locked */ + * of any races in lock requests. + * The early exit uses the same helper the callers re-validate + * with, so this check and a later fuse_dlm_lock_is_held() can + * never disagree about what counts as covered. */ + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) + return 0; /* we already have this area locked */ + + /* + * Sample the revocation generation before the request leaves. + * The reply and a NOTIFY revoke are serviced on different + * threads, so a revoke aimed at the grant this request returns + * can be processed before the grant is recorded below -- + * recording it anyway would resurrect a dead grant that no later + * NOTIFY will ever remove. + */ + gen = fuse_dlm_revoke_gen(fi); memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - inarg.start = offset; - inarg.end = end; - inarg.type = FUSE_DLM_LOCK_WRITE; + /* note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests */ + inarg.start = offset & PAGE_MASK; + inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); + inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? + FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; args.opcode = FUSE_DLM_WB_LOCK; args.nodeid = get_node_id(inode); @@ -536,23 +673,49 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, if (err == -ENOSYS) { /* fuse server does not support dlm, save the info */ fc->dlm = 0; - return; + return err; } if (err) - return; - else - if (inarg.start < outarg.start || - inarg.end > outarg.end) { - /* fuse server is seriously broken */ - pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", - inarg.start, inarg.end, outarg.start, outarg.end); - fuse_abort_conn(fc); - return; - } else { - /* ignore any errors here, there is no way we can react appropriately */ - fuse_dlm_lock_range(fi, outarg.start, - outarg.end, - FUSE_PAGE_LOCK_WRITE); - } + return err; + + if (inarg.start < outarg.start || inarg.end > outarg.end) { + /* fuse server is seriously broken */ + pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", + inarg.start, inarg.end, outarg.start, outarg.end); + fuse_abort_conn(fc); + return -EIO; + } + + /* + * The server granted the lock; record it so + * fuse_dlm_lock_is_held() sees it. + */ + err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + if (err == -EAGAIN) { + /* + * A revoke was processed while the request was in flight; + * the grant may already be dead, so re-request instead of + * recording it. Bounded: a revoke storm must not pin the + * IO here -- past the bound the failure is reported like + * any other request failure (the write path fails the + * write, the read path serves unlocked). + */ + if (--tries) + goto restart; + return -EAGAIN; + } + + /* + * A failure to record (small-allocation -ENOMEM) does not undo + * the grant: coverage exists cluster-wide, only the local + * bookkeeping is missing. Report that as + * FUSE_DLM_GRANT_UNRECORDED so callers neither fail an IO that + * is actually covered nor keep re-requesting a grant that will + * not become visible. + */ + if (err) + return FUSE_DLM_GRANT_UNRECORDED; + + return 0; } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 438d31d28b666e..647a8c37c36095 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -17,12 +17,28 @@ struct fuse_inode; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * fuse_get_dlm_lock() result: the server granted the lock but recording + * it locally failed, leaving the grant invisible to + * fuse_dlm_lock_is_held(). The IO is covered cluster-wide; the caller + * must proceed without re-validating (a re-request would spin) instead + * of failing the IO. + */ +#define FUSE_DLM_GRANT_UNRECORDED 1 + /* Page cache lock manager */ struct fuse_dlm_cache { /* Lock protecting the tree */ struct rw_semaphore lock; /* Interval tree of locked ranges */ struct rb_root_cached ranges; + /* + * Bumped under @lock by every revocation + * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); + * lets fuse_get_dlm_lock() order recording a reply's grant + * against revokes processed while the reply was in flight. + */ + uint64_t revoke_gen; }; /* Initialize a page cache lock manager */ @@ -35,6 +51,14 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); +/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen); + +/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); + /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); @@ -43,8 +67,12 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); -/* this is the interface to the filesystem */ -void fuse_get_dlm_write_lock(struct file *file, loff_t offset, - size_t length); +/* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ +bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); + +/* This is the interface to the filesystem */ +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); #endif /* _FS_FUSE_DLM_CACHE_H */ diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c8c0b5a1477968..7f9611079d3ee0 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -96,6 +97,20 @@ struct dlm_locked_area size_t size; }; +/* + * Force-DIO switch trigger: an exponentially weighted moving average of the + * interval (in jiffies) between FUSE_NOTIFY_INVAL_INODE data invalidations for + * a file. When the average spacing falls below FUSE_NOTIFY_DIO_INTERVAL -- a + * remote writer streaming invalidations -- and the file is open for writing + * here, it is latched into direct IO. These are the source-level (not + * externally tunable) parameters of the heuristic: EWMA weight 1/2^SHIFT, + * seeded and capped at SEED so it takes a short burst rather than a single + * notify to trip. + */ +#define FUSE_NOTIFY_DIO_INTERVAL max_t(unsigned long, HZ / 10, 1) +#define FUSE_NOTIFY_EWMA_SHIFT 2 +#define FUSE_NOTIFY_EWMA_SEED (2 * FUSE_NOTIFY_DIO_INTERVAL) + /** FUSE inode */ struct fuse_inode { /** Inode data */ @@ -163,6 +178,44 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; + + /* + * Per-inode read/write coherency gate for the + * forced-direct-IO feature. Cache-serving buffered reads + * and buffered writes hold it for read; being a + * percpu_rw_semaphore the read side is per-CPU cheap and + * scales on a shared file. The NOTIFY invalidate + * (fuse_reverse_inval_inode()) holds it for write, which + * BLOCKS so the coherency notify has priority: it fences + * cache-serving reads (and buffered writes) out for the + * whole invalidate, so no folio a remote modify has + * superseded is ever handed back. + * + * The write side may run on the server thread delivering + * the notify, so a blocking writer is safe only under a + * server that services request replies on threads other + * than the one delivering the notify (see the NOTIFY site). + * + * Allocated out of line only for writeback+dlm regular + * files (it shares storage with the readdir-cache union + * arm); NULL on other mounts and on allocation failure, + * where the gate is inactive and the invalidate falls back + * to best-effort. + */ + struct percpu_rw_semaphore *wb_inval_rwsem; + + /* + * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations + * for this whole file: notify_stamp is the jiffies of + * the last one, notify_interval_ewma the EWMA of the + * inter-arrival interval (jiffies, scaled by + * 2^FUSE_NOTIFY_EWMA_SHIFT). A rapid stream (short + * average interval) with a local writer latches the + * inode into direct IO. Protected by fi->lock; regular + * files only (shares the readdir-cache union arm). + */ + unsigned long notify_stamp; + unsigned int notify_interval_ewma; }; /* readdir cache (directory only) */ @@ -224,6 +277,14 @@ enum { FUSE_I_BTIME, /* Wants or already has page cache IO */ FUSE_I_CACHE_IO_MODE, + /* + * Latched into direct IO: a NOTIFY_INVAL_INODE arrived while the file + * was open for writing here, so another (remote) entity is modifying it + * concurrently. Reads and writes are routed direct (shared-lock + * parallel dio) until the last writer closes or the inode is mmapped. + * See fuse_reverse_inval_inode()/fuse_file_io_open(). + */ + FUSE_I_FORCE_DIO, }; struct fuse_conn; @@ -1488,6 +1549,12 @@ void fuse_file_uncached_io_end(struct inode *inode, struct fuse_file *ff); int fuse_file_io_open(struct file *file, struct inode *inode); void fuse_file_io_release(struct fuse_file *ff, struct inode *inode); +/* Inode latched into forced direct IO after a remote-modify notification */ +static inline bool fuse_inode_force_dio(struct inode *inode) +{ + return test_bit(FUSE_I_FORCE_DIO, &get_fuse_inode(inode)->state); +} + /* file.c */ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, struct inode *inode, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 82c84a81b6e2d3..f1eb5cdaae4972 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -191,6 +191,23 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } + + /* + * Free the coherency gate here rather than in ->free_inode: that runs + * from an RCU callback, where percpu_free_rwsem() may sleep in + * rcu_sync_dtor() if the write side has not fully quiesced. No user + * can remain by eviction time: gate readers hold a file reference and + * a concurrent notify holds an inode reference. wb_inval_rwsem lives + * in the regular-file union arm and is only ever allocated for regular + * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked + * regular files still own a gate); a directory's overlapping + * readdir-cache fields must not be misread. + */ + if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { + percpu_free_rwsem(fi->wb_inval_rwsem); + kfree(fi->wb_inval_rwsem); + fi->wb_inval_rwsem = NULL; + } } static int fuse_reconfigure(struct fs_context *fsc) @@ -746,9 +763,58 @@ static void fuse_invalidate_inode_entry(struct inode *inode) } } +/* + * Fold one FUSE_NOTIFY_INVAL_INODE data invalidation into the per-inode + * moving average of the notification inter-arrival interval and report whether + * the file is now "hot" -- notifications are arriving fast enough (short + * average interval) that a remote writer is repeatedly invalidating it. The + * average is an EWMA (weight 1/2^FUSE_NOTIFY_EWMA_SHIFT); the sample is clamped + * to FUSE_NOTIFY_EWMA_SEED so a notify after a long idle only cools the average + * and cannot overflow the accumulator. Must be called under fi->lock; called + * for every data invalidation so the average stays current even while no local + * writer is open. + */ +static bool fuse_notify_inval_hot(struct fuse_inode *fi) +{ + unsigned long now = jiffies; + unsigned long sample; + unsigned int avg; + + sample = min_t(unsigned long, now - fi->notify_stamp, + FUSE_NOTIFY_EWMA_SEED); + fi->notify_stamp = now; + + /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ + fi->notify_interval_ewma += sample - + (fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT); + avg = fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT; + + return avg < FUSE_NOTIFY_DIO_INTERVAL; +} + +/* + * Revoke the DLM grants backing an invalidated byte range. Grants are + * recorded page-aligned, so widen the revoke to page boundaries: dropping + * more than the server invalidated only costs a re-request, dropping less + * would leave a stale grant that fuse_dlm_lock_is_held() keeps trusting. + * len <= 0 means "invalidate to EOF" (see fuse_notify_inval_inode()) and + * revokes through U64_MAX -- it must not become an inverted range, which + * fuse_dlm_unlock_range() rejects without removing anything. + */ +static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, + loff_t len) +{ + uint64_t start = (uint64_t)offset & PAGE_MASK; + uint64_t end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + + fuse_dlm_unlock_range(fi, start, end); +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { + struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; pgoff_t pg_start; @@ -793,18 +859,111 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; - if (fc->dlm && fc->writeback_cache) - /* Invalidate the range exactly as the fuse server requested - * except for the case where it sends -1. - * Note that this can lead to some inconsistencies if - * the fuse server sends unaligned data */ - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); - - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); + /* + * A data invalidation means another (remote) entity is modifying + * the file. Two things happen here: + * + * 1. Coherency. Drop the affected page-cache range so no local + * read returns a folio the remote modify has superseded. This + * runs under the write side of the per-inode coherency gate + * (wb_inval_rwsem), which fences cache-serving buffered reads + * and buffered writes out for the whole invalidate. Unlike the + * old best-effort trylock this BLOCKS -- the notify has + * priority: percpu_down_write() parks new gate readers, drains + * in-flight ones, then invalidates. A blocking writer here is + * safe only under a server that services request replies on + * threads other than the one delivering this notify: the write + * side waits for gate readers to drain, and a cache-miss read + * holds the read side across its FUSE_READ round-trip. redfs' + * dlm server provides that contract; a server that cannot must + * not enable writeback+dlm. + * + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under + * fi->lock, updated for every data invalidation) of how fast + * these arrive; when they come in a rapid stream -- a remote + * writer repeatedly invalidating -- and the inode is also open + * for writing here, latch it into direct IO until the last + * writer closes or it is mmapped. When latched, drop the whole + * mapping rather than just the notified range, or dirty folios + * outside it would be invisible to the forced direct reads + * (stale read / lost write). + * + * The gate (and the average) exist only for writeback+dlm regular + * files; elsewhere wb_sem is NULL and the invalidate runs + * unserialized (best-effort), as before. An mmapped inode + * keeps the gate -- fuse_cache_read_iter() and + * fuse_cache_write_iter() enter it unconditionally and rely + * on the revoke staying fenced -- but is never latched: + * a mapping needs the page cache, and fuse_file_mmap() + * reverts any latch it races with. + */ + if (S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode)) + wb_sem = fi->wb_inval_rwsem; + + if (wb_sem) { + bool hot, has_writer, latched = false; + + spin_lock(&fi->lock); + hot = fuse_notify_inval_hot(fi); + has_writer = !list_empty(&fi->write_files); + spin_unlock(&fi->lock); + + /* + * Priority write side: park new gate readers, + * drain in-flight ones, then invalidate. Blocks + * (unlike the old trylock) -- see the contract in + * the comment above. + */ + percpu_down_write(wb_sem); + + /* + * Revoke the DLM lock range under the gate write + * side, atomically with the page drop: gate readers + * re-validate their grant right after entering, and + * a grant that passed that check must stay visible + * for their whole gate hold. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_revoke_inval_range(fi, offset, len); + + if (hot && has_writer && + !mapping_mapped(inode->i_mapping) && + !fuse_inode_force_dio(inode)) { + spin_lock(&fi->lock); + if (!list_empty(&fi->write_files)) { + set_bit(FUSE_I_FORCE_DIO, &fi->state); + latched = true; + } + spin_unlock(&fi->lock); + } + + /* + * Latched: drop the whole mapping (dirty folios + * outside the notified range would be invisible to + * the forced direct reads). Otherwise just the + * notified range. + */ + if (fuse_inode_force_dio(inode)) + invalidate_inode_pages2(inode->i_mapping); + else + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + + percpu_up_write(wb_sem); + + if (latched) + pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", + nodeid); + } else { + /* No gate on this inode (DAX, non-regular, or the + * gate allocation failed): drop the lock range + * unserialized (best-effort), as before. */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_revoke_inval_range(fi, offset, len); + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } } iput(inode); return 0; diff --git a/fs/fuse/iomode.c b/fs/fuse/iomode.c index c0496b971da0cb..a42c7ba01307c0 100644 --- a/fs/fuse/iomode.c +++ b/fs/fuse/iomode.c @@ -163,6 +163,15 @@ int fuse_file_io_open(struct file *file, struct inode *inode) if (ff->open_flags & FOPEN_DIRECT_IO) return 0; + /* + * The inode was latched into direct IO after a remote-modify + * notification arrived while it was open for writing here. Open this + * file uncached as well so its IO is routed direct and it does not + * re-enter caching mode. + */ + if (fuse_inode_force_dio(inode)) + return 0; + err = fuse_file_cached_io_start(inode, ff); if (err) goto fail;