diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index fe4470e39fc095..78736a1636809b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -800,15 +800,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, get_fuse_inode(inode)); truncate_pagecache(inode, 0); } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { - /* - * Only when the drop really emptied the mapping, as - * in fuse_open(): a folio that survived still needs - * its record. - */ - if (fuse_open_drop_cache(inode) && - fm->fc->dlm && fm->fc->writeback_cache) - fuse_dlm_ranges_dropped(get_fuse_inode(inode), - 0, U64_MAX); + fuse_open_drop_cache(inode); } } return err; @@ -2213,6 +2205,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, i_size_write(inode, outarg.attr.size); if (is_truncate) { + fuse_writeback_crop_truncated(inode, outarg.attr.size); /* NOTE: this may release/reacquire fi->lock */ __fuse_release_nowrite(inode); } @@ -2235,32 +2228,6 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); - - /* - * The cache above the new size is gone, so the ranges - * describing it have nothing left to say. From the first - * whole page above it: the page holding the new end of the - * file survives the truncate, and so does its record. - */ - if (fc->dlm && fc->writeback_cache) - fuse_dlm_ranges_dropped(fi, PAGE_ALIGN(outarg.attr.size), - U64_MAX); - - /* - * invalidate_inode_pages2() emptied the mapping below the new - * size too (laundering anything dirty first, so those bytes - * are on the server). A revoked range is kept only to make - * writeback take the grant again before sending the folios - * under it, so with those folios gone it describes nothing - * and would sit in the tree unfreed, keeping its neighbours - * from merging. Only when the drop really emptied it: a busy - * folio that survived still needs its record, and a fault - * populating after the check keeps its page visible to it. - */ - if (fc->dlm && fc->writeback_cache && outarg.attr.size && - !filemap_range_has_page(mapping, 0, outarg.attr.size - 1)) - fuse_dlm_ranges_dropped(fi, 0, - PAGE_ALIGN(outarg.attr.size) - 1); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index d269f88628651b..48a09dd42efa3a 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -247,7 +248,20 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, if (inode && fc->compound_open_getattr) { struct fuse_attr_out attr_outarg; + u64 attr_version; + /* + * Sampled before the request, the way every other + * caller does it. fuse_change_attributes_i() drops + * a reply whose version is older than the inode's, + * and a version read once the reply is already back + * is never older than anything that moved while it + * was on the wire -- the reply would be applied + * however stale it had become, and under DLM that + * means a size from before the open shrinking + * i_size back under the writers. + */ + attr_version = fuse_get_attr_version(fc); err = fuse_compound_open_getattr(fm, nodeid, open_flags, opcode, ff, &attr_outarg, outargp); @@ -257,7 +271,7 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, fuse_change_attributes(inode, &attr_outarg.attr, NULL, ATTR_TIMEOUT(&attr_outarg), - fuse_get_attr_version(fc)); + attr_version); } if (err == -ENOSYS) { err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); @@ -342,8 +356,7 @@ int fuse_finish_open(struct inode *inode, struct file *file) } /* - * Drop the page cache an open that did not get FOPEN_KEEP_CACHE must not keep, - * and report whether the mapping came out empty. + * Drop the page cache an open that did not get FOPEN_KEEP_CACHE must not keep. * * Serialised on the mapping's invalidate lock. Every opener runs this same * full-mapping walk and invalidate_inode_pages2_range() takes each folio's @@ -354,13 +367,10 @@ int fuse_finish_open(struct inode *inode, struct file *file) * mapping and the rest fall straight back out of mapping_empty(). * * Holding it exclusive also fences faults for the duration, which is what - * truncate already does across this same walk, and it keeps the emptiness - * test from reading a mapping another opener is halfway through. + * truncate already does across this same walk. */ -bool fuse_open_drop_cache(struct inode *inode) +void fuse_open_drop_cache(struct inode *inode) { - bool emptied; - filemap_invalidate_lock(inode->i_mapping); /* * Write back first: the drop launders whatever it finds dirty a @@ -369,10 +379,7 @@ bool fuse_open_drop_cache(struct inode *inode) */ filemap_write_and_wait(inode->i_mapping); invalidate_inode_pages2(inode->i_mapping); - emptied = !filemap_range_has_page(inode->i_mapping, 0, LLONG_MAX); filemap_invalidate_unlock(inode->i_mapping); - - return emptied; } static void fuse_truncate_update_attr(struct inode *inode, struct file *file) @@ -383,6 +390,7 @@ static void fuse_truncate_update_attr(struct inode *inode, struct file *file) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); i_size_write(inode, 0); + fuse_writeback_crop_truncated(inode, 0); spin_unlock(&fi->lock); file_update_time(file); fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); @@ -444,18 +452,7 @@ static int fuse_open(struct inode *inode, struct file *file) fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { - /* - * Only when the drop really emptied the mapping; a - * folio that survived still needs its record. This - * open holds no lock against buffered IO, which can - * repopulate the mapping once the invalidate lock is - * dropped -- but such a folio is populated after the - * drop and keeps its page, so a later reader finds - * both the page and the record it needs. - */ - if (fuse_open_drop_cache(inode) && - fc->dlm && fc->writeback_cache) - fuse_dlm_ranges_dropped(fi, 0, U64_MAX); + fuse_open_drop_cache(inode); } } if (dax_truncate) @@ -1081,20 +1078,172 @@ static int fuse_do_readpage(struct file *file, struct page *page) return 0; } +/* + * The unit the readahead window is built in: one request's worth of pages, + * never more than the readahead this fd was allowed. @ra is NULL where + * there is no per-fd state to bound it by. + * + * fuse_readahead() rounds its window up to this and fuse_readahead_lookahead() + * puts one more of them past it, so fuse_read_grant() sizes the grant from it + * as well and the two stay in step. + */ +static unsigned int fuse_readahead_unit(struct fuse_conn *fc, + struct file_ra_state *ra) +{ + unsigned int max_pages = min_t(unsigned int, fc->max_pages, + fc->max_read / PAGE_SIZE); + + if (!ra) + return max_pages; + + return min_t(unsigned int, max_pages, ra->ra_pages); +} + +/** + * fuse_read_grant - take the grant a page cache fill runs under + * @file: file to read through + * @pos: byte offset the read starts at + * @count: bytes the read asks for + * + * ->read_folio and ->readahead are entered with the pages they fill + * already locked, and no grant may be asked for under a page lock + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * A read asks here, before anything is locked, and the fill paths only + * confirm what this took. + * + * Readahead fills past the end of the read, so ask for a window beyond + * it as well, bounded by the file since readahead stops there. A run of + * pages no grant covers is given back unfilled and fetched one page at + * a time, so what is asked for here is what readahead is worth. + * + * Return: what fuse_get_dlm_lock() returned, 0 when there is nothing to + * ask for. + */ +static int fuse_read_grant(struct file *file, loff_t pos, size_t count) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + struct file_ra_state *ra = &file->f_ra; + loff_t size = i_size_read(inode); + loff_t end = pos + count; + unsigned int unit; + loff_t ahead; + + if (!fc->writeback_cache || !fc->dlm) + return 0; + + /* + * What readahead may fill past the read: the window mm builds, at + * most ra_pages wide, plus what this filesystem adds to it -- the + * round up to a whole request, under one unit, and the one request + * fuse_readahead_lookahead() puts past that. + */ + unit = fuse_readahead_unit(fc, ra); + ahead = (loff_t)(ra->ra_pages + 2 * unit) << PAGE_SHIFT; + + if (end < size) + end += min(ahead, size - end); + + if (end <= pos) + return 0; + + return fuse_get_dlm_lock(file, pos, end - pos, FUSE_PAGE_LOCK_READ); +} + +/** + * fuse_read_folio_retry - back off a fill with no grant to run under + * @file: file to read through + * @folio: the folio handed over locked, unlocked here + * @pos: byte offset of @folio + * @len: its size in bytes + * + * Neither reason a fill is refused can be dealt with while the folio is + * held: waiting a revoke out would hold the page cache that revoke is + * about to drop, and no grant may be asked for under a page lock at all + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * Unlock, do both, and send the caller round again to find the range + * covered. + * + * Return: AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_retry(struct file *file, struct folio *folio, + loff_t pos, size_t len) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + struct fuse_dlm_span pin; + int err; + + folio_unlock(folio); + + /* Wait the revoke out; what it leaves behind is asked for below */ + fuse_dlm_pin(fi, &pin, pos, len); + fuse_dlm_unpin(fi); + + err = fuse_read_grant(file, pos, len); + if (err == -ENOSYS) + return AOP_TRUNCATED_PAGE; + if (err < 0) + return err; + /* + * Granted but unrecorded, so the retry finds the range uncovered + * and comes straight back here. Report it rather than spin. + */ + if (err > 0) + return -ENOMEM; + + return AOP_TRUNCATED_PAGE; +} + static int fuse_read_folio(struct file *file, struct folio *folio) { struct page *page = &folio->page; struct inode *inode = page->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + struct fuse_dlm_span pin; + bool pinned = false; int err; err = -EIO; if (fuse_is_bad(inode)) goto out; + /* + * The grant the folio is filled under, held from the confirmation + * until the bytes are in the page cache. What lands here is served + * to every later reader of the file, so it must neither be fetched + * under a grant a revoke has taken away nor be dropped into a range + * a revoke has just swept: such a folio is uptodate and covered by + * nothing, and no further notify comes for a lock this client no + * longer holds. + * + * The pin closes the second, since a revoke over the folio waits + * for the fill and drops the folio after it; the confirmation + * closes the first. Both fail into fuse_read_folio_retry(). + */ + if (fc->dlm && fc->writeback_cache) { + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin(fi); + pinned = false; + } + if (!pinned) + return fuse_read_folio_retry(file, folio, pos, len); + } + err = fuse_do_readpage(file, page); fuse_invalidate_atime(inode); out: unlock_page(page); + /* + * After the unlock, so a revoke draining this pin finds the folio + * it has to drop unlocked and takes it out. + */ + if (pinned) + fuse_dlm_unpin(fi); return err; } @@ -1129,19 +1278,39 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, folio_end_read(folio, !err); folio_put(folio); } + + /* + * Dropped after the folios, which are filled, uptodate and unlocked + * by now: a revoke draining this pin finds them and takes them out. + * Held until here so it cannot have swept before they were there. + */ + if (ia->read.dlm_fi) + fuse_dlm_unpin_span(ia->read.dlm_fi, &ia->read.dlm_pin); + if (ia->ff) fuse_file_put(ia->ff, false); fuse_io_free(ia); } -static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) +/** + * fuse_send_readpages - read a run of pages of a readahead window + * @ia: the request, owning the pages and the pin over them + * @file: file to read through + * + * Return: 0 once the request is on its way or has been completed, + * -EAGAIN when a revoke of the range refused the grant and nothing was + * sent. The pages are given back either way. + */ +static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file) { struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); struct fuse_args_pages *ap = &ia->ap; loff_t pos = page_offset(ap->pages[0]); size_t count = ap->num_pages << PAGE_SHIFT; + unsigned int i; ssize_t res; int err; @@ -1156,6 +1325,29 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) } WARN_ON((loff_t) (pos + count) < 0); + /* + * The grant the read took in fuse_read_grant(), confirmed under a + * pin and held until the reply has filled the pages. A revoke of the range + * waits for that, so the reply cannot be fetched under a grant the + * server has since handed on, and cannot land behind a sweep that + * would leave the pages uptodate and covered by nothing. + * + * Refused, or gone since it was asked for: give the pages back + * unfilled rather than serve what no lock covers. The read that + * wanted them comes back through fuse_read_folio(), which asks + * again with no page held. + */ + if (fm->fc->dlm && fm->fc->writeback_cache) { + if (!fuse_dlm_trypin_span(fi, &ia->read.dlm_pin, pos, count)) + goto uncovered; + if (!fuse_dlm_lock_is_held(fi, pos, count, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin_span(fi, &ia->read.dlm_pin); + goto uncovered; + } + ia->read.dlm_fi = fi; + } + fuse_read_args_fill(ia, file, pos, count, FUSE_READ); ia->read.attr_ver = fuse_get_attr_version(fm->fc); if (fm->fc->async_read) { @@ -1163,12 +1355,23 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) ap->args.end = fuse_readpages_end; err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (!err) - return; + return 0; } else { res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } fuse_readpages_end(fm, &ap->args, err); + return 0; + +uncovered: + for (i = 0; i < ap->num_pages; i++) { + struct folio *folio = page_folio(ap->pages[i]); + + folio_end_read(folio, false); + folio_put(folio); + } + fuse_io_free(ia); + return -EAGAIN; } /* @@ -1236,19 +1439,11 @@ static void fuse_readahead_lookahead(struct file *file, struct inode *inode, return; /* - * Same grant the window itself takes: folios the server handed out - * no lock for are folios it will not revoke when a remote node - * writes them. + * No grant is asked for here either: this runs from inside + * ->readahead with the window it just sent still locked. What the + * read's grant does not reach fuse_send_readpages() declines, and + * the pages go back unfilled. */ - if (fc->writeback_cache && fc->dlm) { - int err = fuse_get_dlm_lock(file, (loff_t)start << PAGE_SHIFT, - (size_t)nr << PAGE_SHIFT, - FUSE_PAGE_LOCK_READ); - - if (err < 0 && err != -ENOSYS) - return; - } - ia = fuse_io_alloc(NULL, nr); if (!ia) return; @@ -1328,52 +1523,31 @@ static void fuse_readahead(struct readahead_control *rac) * negotiated large requests does not get to prefetch beyond the * window the admin allowed. */ - unit = max_pages; + unit = fuse_readahead_unit(fc, rac->ra); target_end = readahead_index(rac) + readahead_count(rac); if (rac->ra) { unsigned int nr = readahead_count(rac); - unit = min_t(unsigned int, max_pages, rac->ra->ra_pages); if (unit > 1 && nr % unit) target_end = readahead_index(rac) + roundup(nr, unit); } /* - * Readahead fills the page cache past the range the reader locked, - * so take a DLM read grant over the whole window here too. Folios + * Readahead fills the page cache past the range the reader asked + * for, and what lands there has to be covered by a grant: folios * the server handed out no lock for are folios it will not revoke * when a remote node writes them, and a later read would be served - * from stale cache. Take the grant before any folio is pulled off - * @rac, so the window that gets populated is the window that is - * covered. + * from stale cache. * - * The grant covers the window this call intends rather than the one - * it ends up with, since the folios past what mm built are not - * allocated yet. readahead_expand() stops at the first folio already - * cached, so a short realisation leaves the tail covered but not - * populated. That is the harmless direction: coverage without cached - * data serves nothing stale, and a folio already cached is one an - * earlier grant already covers. + * No grant is asked for here. ->readahead is entered with every + * page of the window already locked, and none may be asked for + * under a page lock; the read this window belongs to took one over + * it in fuse_read_grant(), before the page cache was entered. * - * Speculative pages are not worth serving uncovered: on a failed - * request drop the window and let read_pages() clean up the folios - * left in @rac. A server without DLM support answers -ENOSYS and - * clears fc->dlm, which is not a failure. - * - * The round trip is taken before any folio of the window is locked - * and with nothing fenced out, so it holds up this reader and - * nothing else. + * What that left uncovered fuse_send_readpages() declines, one run + * of pages at a time, and those pages go back unfilled for + * fuse_read_folio() to fetch with no page held. */ - if (fc->writeback_cache && fc->dlm) { - size_t len = (size_t)(target_end - readahead_index(rac)) - << PAGE_SHIFT; - int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), len, - FUSE_PAGE_LOCK_READ); - - if (err < 0 && err != -ENOSYS) - return; - } - for (;;) { struct fuse_io_args *ia; struct fuse_args_pages *ap; @@ -1443,7 +1617,8 @@ static void fuse_readahead(struct readahead_control *rac) ap->descs[i].length = PAGE_SIZE; } ap->num_pages = nr_pages; - fuse_send_readpages(ia, rac->file); + if (fuse_send_readpages(ia, rac->file)) + break; } /* @@ -1477,18 +1652,22 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } - /* if we have dlm support acquire a read lock for the area - * we are reading from. */ - if (fc->writeback_cache && fc->dlm) - fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + /* + * The grant this read and the readahead behind it fill under. Not + * for O_DIRECT, which takes no DLM lock at all: it fills no page + * cache, and what it reads is the server's to order. + */ + if (!(iocb->ki_flags & IOCB_DIRECT)) + fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); /* * A NOTIFY invalidate racing this read drops the folios it * supersedes, so the read either misses and refetches or returns - * data that was current when it was copied. There is nothing to - * fence: unlike a write, a read leaves nothing behind that could - * reach the server under a grant it no longer holds. + * data that was current when it was copied. What a read does leave + * behind is the page cache it fills, which must not outlast the + * grant it was fetched under; that is fenced where the filling + * happens, in fuse_read_folio() and fuse_send_readpages(), and the + * grant taken here is what they confirm. */ if (fuse_inode_force_dio(inode)) { size_t count = iov_iter_count(to); @@ -1920,6 +2099,62 @@ static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) return (err < 0 && err != -ENOSYS) ? err : 0; } +/* + * How many times a writer confirms its grant again before giving up on + * the range. A pass costs a round trip only when the grant has gone, + * which is a revoke landing between the request and the confirmation. + */ +#define FUSE_DLM_PIN_RETRIES 16 + +/* + * Pin [@pos, @pos + @len) with the grant over it confirmed, so the bytes + * can be dirtied under a lock that cannot be taken away meanwhile; see + * fuse_dlm_pin(). @pin is the caller's storage for the pin, which it + * drops with fuse_dlm_unpin() once the bytes are dirty. + * + * The grant is asked for again when it has gone, and the pin must not be + * held across that request: it is answered by the server the revoke + * waiting for the pin came from. So confirm and request alternate, and + * no folio may be held here. + */ +static int fuse_dlm_pin_write(struct file *file, struct fuse_dlm_span *pin, + loff_t pos, size_t len) +{ + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + unsigned int tries = FUSE_DLM_PIN_RETRIES; + int err; + + for (;;) { + fuse_dlm_pin(fi, pin, pos, len); + /* + * A server that turned out to have no DLM leaves nothing to + * confirm, and the pin still pairs with the caller's unpin. + */ + if (!fc->dlm || + fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) + return 0; + fuse_dlm_unpin(fi); + + if (!tries--) + return -EIO; + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + if (err < 0 && err != -ENOSYS) + return err; + if (err > 0) { + /* + * Granted but unrecorded, so there is nothing for the + * confirmation above to find. The range is covered + * cluster-wide; pin and proceed. + */ + fuse_dlm_pin(fi, pin, pos, len); + return 0; + } + } +} + /* * Write @len bytes of @from at the current iocb position, either straight * through to the server (@through, for an unaligned edge) or into the page @@ -1928,16 +2163,32 @@ static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) * temporarily capped to @len so the unconsumed tail stays available for the * next chunk. Returns bytes written (< @len means a short write, the caller * stops) or a negative error. + * + * The grant over the chunk is confirmed and pinned across both, so the + * bytes cannot become the server's under a revoke that has already been + * answered. For the cached interior that is one pin for the whole chunk: + * ->write_begin and ->write_end have nowhere to keep a node that spans the + * pair, and generic_perform_write() runs between them. For the edges the + * bytes never enter the page cache at all, so a revoke cannot find them by + * flushing it and the pin has to cover the FUSE_WRITE itself, which is the + * reply the revoke already waits for when the same bytes go through + * writeback. */ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, size_t len, bool through) { + struct file *file = iocb->ki_filp; + struct fuse_dlm_span pin; size_t hidden; ssize_t res; if (!len) return 0; + res = fuse_dlm_pin_write(file, &pin, iocb->ki_pos, len); + if (res) + return res; + /* Cap the iterator to this chunk, keeping the tail for later chunks. */ hidden = iov_iter_count(from) - len; iov_iter_truncate(from, len); @@ -1947,6 +2198,8 @@ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, * (which leave it partly advanced) reexpand to the exact remainder. */ iov_iter_reexpand(from, iov_iter_count(from) + hidden); + fuse_dlm_unpin(get_fuse_inode(file_inode(file))); + return res; } @@ -1975,7 +2228,8 @@ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, /* No whole folio inside the write: nothing cacheable, all through. */ if (mid_end <= mid_start) - return fuse_perform_write(iocb, from, true); + return fuse_dlm_write_chunk(iocb, from, + iov_iter_count(from), true); /* Unaligned head [pos, mid_start): through. */ res = fuse_dlm_write_chunk(iocb, from, mid_start - pos, true); @@ -1985,13 +2239,25 @@ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, if (res < mid_start - pos) return total; - /* Aligned interior [mid_start, mid_end): cached whole folios. */ - res = fuse_dlm_write_chunk(iocb, from, mid_end - mid_start, false); - if (res < 0) - return total ? total : res; - total += res; - if (res < mid_end - mid_start) - return total; + /* + * Aligned interior [mid_start, mid_end): cached whole folios, taken + * one shard at a time. fuse_dlm_write_chunk() pins the grant for + * the length of a chunk and a revoke of the range waits for that, + * so the chunk is what bounds how long a large write holds a notify + * up. + */ + while (mid_start < mid_end) { + size_t chunk = min_t(loff_t, mid_end - mid_start, + FUSE_DLM_SHARD_SIZE); + + res = fuse_dlm_write_chunk(iocb, from, chunk, false); + if (res < 0) + return total ? total : res; + total += res; + if (res < chunk) + return total; + mid_start += chunk; + } /* Unaligned tail [mid_end, end): through. */ res = fuse_dlm_write_chunk(iocb, from, end - mid_end, true); @@ -2070,7 +2336,10 @@ static bool fuse_write_stream_update(struct fuse_inode *fi, size_t len) * A hint only: nothing waits for it, and a chunk that is partly dirty or * already written back sends what it has. The run is read and written * without the inode lock, which the DLM path holds shared, so writers - * landing on it together cost a kick, not correctness. + * landing on it together cost a kick, not correctness. Each mark is read + * once into a local for that to hold: reading write_stream_start twice, + * to round down and again to compare, lets a writer moving it in between + * invert the range the kick is given. */ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, size_t len, bool stream) @@ -2078,25 +2347,27 @@ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct fuse_inode *fi = get_fuse_inode(inode); - loff_t chunk, start, end; + loff_t chunk, run, start, end; bool sequential; - sequential = pos == fi->write_stream_next; - fi->write_stream_next = pos + (loff_t)len; + sequential = pos == READ_ONCE(fi->write_stream_next); + WRITE_ONCE(fi->write_stream_next, pos + (loff_t)len); if (!sequential || !stream) { /* Nothing behind this write is known to be finished */ - fi->write_stream_start = pos; + WRITE_ONCE(fi->write_stream_start, pos); return; } + run = READ_ONCE(fi->write_stream_start); chunk = fuse_write_chunk_size(get_fuse_conn(inode)); - start = round_down(fi->write_stream_start, chunk); - end = round_down(fi->write_stream_next, chunk); + start = round_down(run, chunk); + /* This write's own end, not the mark another writer may have moved */ + end = round_down(pos + (loff_t)len, chunk); /* No bound crossed, so nothing has been left complete */ - if (end <= fi->write_stream_start) + if (end <= run) return; - fi->write_stream_start = end; + WRITE_ONCE(fi->write_stream_start, end); /* * What filemap_fdatawrite_range_kick() does upstream: a range write * with no integrity, which this kernel has no helper for. @@ -2217,8 +2488,14 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * Only the append case must wait for the lock: its range * depends on i_size, which is stable only under the exclusive * inode lock. + * + * O_DIRECT takes no DLM lock at all, here or anywhere: it + * dirties no page cache, so there is nothing for a grant to + * cover, and its bytes are the server's to order against the + * rest of the cluster. */ - if (fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + if (fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + !(iocb->ki_flags & IOCB_APPEND)) { dlm_pos = iocb->ki_pos; dlm_len = iov_iter_count(from); @@ -2254,7 +2531,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* note that this small code dup will save us a lot of headache later * when appends are done concurrently without using parallel direct writes */ - if (fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + if (fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + (iocb->ki_flags & IOCB_APPEND)) { /* * An append write lands at the current EOF no matter * what ki_pos holds: generic_write_checks() rewrites @@ -2279,7 +2557,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * generic_write_checks() may have put ki_pos past the granted * range. Re-lock where the write really lands. */ - if (fc->dlm && (iocb->ki_flags & IOCB_APPEND) && + if (fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + (iocb->ki_flags & IOCB_APPEND) && iocb->ki_pos != dlm_pos) { dlm_pos = iocb->ki_pos; dlm_len = iov_iter_count(from); @@ -2352,6 +2631,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) generic_perform_write(iocb, from)); } else if (through) { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + struct fuse_dlm_span pin; loff_t pos = iocb->ki_pos; count = iov_iter_count(from); @@ -2372,8 +2652,23 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } } + /* + * These bytes never enter the page cache, so a revoke + * cannot find them by flushing it, and the grant they + * were taken under can be handed on while they are + * still on the wire. Hold it across the FUSE_WRITE, + * as the writethrough edges do; a revoke of the range + * waits for the reply. + */ + err = fuse_dlm_pin_write(file, &pin, pos, count); + if (err) { + written = err; + goto wb_out; + } + written = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); + fuse_dlm_unpin(fi); if (written < 0) { err = written; goto wb_out; @@ -2787,8 +3082,10 @@ static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); - else - return filemap_splice_read(in, ppos, pipe, len, flags); + + fuse_read_grant(in, *ppos, len); + + return filemap_splice_read(in, ppos, pipe, len, flags); } static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, @@ -2817,6 +3114,11 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) kfree(wpa); } +/* Defined below, with the writeback paths that are its other callers. */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct writeback_control *wbc, + struct folio *folio); + static void fuse_writepage_finish(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2888,10 +3190,41 @@ __releases(fi->lock) __acquires(fi->lock) { struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); loff_t crop = i_size_read(inode); struct fuse_writepage_args *wpa; + /* + * fuse_send_writepage() crops a queued request to the size passed + * here and ends the writeback on everything past it without ever + * sending it -- the folios come back clean with their bytes gone + * and nothing reports an error, so a later fsync() succeeds over + * the hole. That is right after a truncate, where those bytes are + * meant to disappear, and it is silent data loss for any other + * shrink. + * + * Upstream there is no other shrink on a writeback mount: + * fuse_get_cache_mask() keeps STATX_SIZE, so i_size never takes + * the server's answer and only a truncate lowers it, under + * fuse_set_nowrite(). With DLM the server's size is applied + * (fuse_attr_cache_mask()), and a reply that is merely behind the + * local writers -- the normal state of a shared file being written + * -- would crop gigabytes of queued data away. + * + * So crop against the high water mark, which a truncate moves back + * down (fuse_writeback_crop_truncated(), called under the freeze) + * and which follows i_size again as soon as no request is left to + * protect (fuse_writepage_end()). In the steady state it is + * i_size, including the partial folio at EOF that the crop exists + * to clip. + */ + if (fc->writeback_cache) { + if (crop > fi->wb_crop) + fi->wb_crop = crop; + crop = fi->wb_crop; + } + while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) { wpa = list_entry(fi->queued_writes.next, struct fuse_writepage_args, queue_entry); @@ -2908,6 +3241,29 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, struct inode *inode = wpa->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_args_pages *ap = &wpa->ia.ap; + + /* + * A server that acknowledges fewer bytes than it was sent has not + * taken the rest. The write path acts on that (fuse_send_write() + * returns the short count and fuse_perform_write() stops on it); + * writeback had no caller to report it to and ended every folio + * clean, so the tail was dropped with nothing to say so and a later + * fsync() succeeded over the hole. + * + * Put the folios the reply does not reach back on the dirty list so + * the bytes survive for another pass, and record the error: the + * redirty lands after the wait an fsync() in progress already did, + * which would otherwise return success over them again. + */ + if (!error && wpa->ia.write.out.size < wpa->ia.write.in.size) { + unsigned int i = wpa->ia.write.out.size >> PAGE_SHIFT; + + for (; i < ap->num_pages; i++) + fuse_writeback_redirty(fc, NULL, + page_folio(ap->pages[i])); + error = -EIO; + } mapping_set_error(inode->i_mapping, error); /* @@ -2921,6 +3277,17 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, spin_lock(&fi->lock); fi->writectr--; fuse_writepage_finish(wpa); + /* + * Nothing sent and nothing queued: the crop has no request left to + * protect, so let it follow i_size again rather than stay at a + * high water mark this inode may never reach a second time -- a + * file shrunk by another node and written again would otherwise + * have its last folio sent whole instead of clipped at EOF. See + * fuse_flush_writepages(). + */ + if (fc->writeback_cache && fi->writectr == 0 && + list_empty(&fi->queued_writes)) + fi->wb_crop = i_size_read(inode); spin_unlock(&fi->lock); fuse_writepage_free(wpa); } @@ -3039,6 +3406,30 @@ static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio return wpa; } +/* + * Put a folio writeback could not send back on the dirty list. + * + * write_cache_pages() takes the dirty flag off a folio before it calls + * here, and fuse_launder_folio() does the same for the submit it makes + * itself, so a folio that comes back unsent has thrown its bytes away + * unless they are put back. @wbc is NULL for the launder. + * + * Only while there is a connection left to take the bytes. After an abort + * every send fails, and a folio redirtied for a retry that can no longer + * happen would keep sync() going forever. + */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct writeback_control *wbc, + struct folio *folio) +{ + if (!READ_ONCE(fc->connected)) + return; + + folio_mark_dirty(folio); + if (wbc) + wbc->pages_skipped += folio_nr_pages(folio); +} + static int fuse_writepage_locked(struct folio *folio) { struct address_space *mapping = folio->mapping; @@ -3048,6 +3439,8 @@ static int fuse_writepage_locked(struct folio *folio) struct fuse_writepage_args *wpa; struct fuse_args_pages *ap; struct fuse_file *ff; + struct fuse_dlm_span pin; + bool pinned = false; int error = -EIO; ff = fuse_write_file_get(fi); @@ -3056,14 +3449,50 @@ static int fuse_writepage_locked(struct folio *folio) /* * Hold the range again before sending it; see fuse_writepages_fill(). + * + * folio_unmap_invalidate() holds the folio locked here, so a grant + * this folio does not already hold must not be asked for and there is + * no later pass of this submit to defer it to. Leave the folio dirty + * and report no error: the invalidate that laundered it then finds it + * busy, and an ordinary writeback sends it with a grant of its own. + * + * The grant is pinned from the moment it is found until the folio is + * under writeback, where the revoke waits for it again. */ if (fc->dlm && fc->writeback_cache) { - error = fuse_dlm_regrant_range(ff, inode, folio_pos(folio), - folio_pos(folio) + - folio_size(folio) - 1); + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + + /* + * The revoke handler laundering the range it is taking away. + * That lock is still this client's until the handler returns, + * so send without asking: the record has gone already and + * asking would be a round trip for the very range being + * revoked. Only for that range, since the latched path + * launders the whole mapping and the rest of it may be + * covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_dlm_unpin(fi); + pinned = false; + } + + if (!pinned) { + fuse_file_put(ff, false); + fuse_writeback_redirty(fc, NULL, folio); + return 0; + } + + error = fuse_dlm_regrant_range(ff, inode, pos, pos + len - 1); if (error < 0 && error != -ENOSYS) goto err_writepage_args; } +queue: wpa = fuse_writepage_args_setup(folio, ff); error = -ENOMEM; @@ -3074,6 +3503,15 @@ static int fuse_writepage_locked(struct folio *folio) ap->num_pages = 1; folio_start_writeback(folio); + + /* + * Under writeback now, so the flush a revoke runs before it takes the + * grant away waits for these bytes. Nothing further is needed to + * keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); + fuse_writepage_args_page_fill(wpa, folio, 0); spin_lock(&fi->lock); @@ -3084,6 +3522,8 @@ static int fuse_writepage_locked(struct folio *folio) return 0; err_writepage_args: + if (pinned) + fuse_dlm_unpin(fi); fuse_file_put(ff, false); err: /* @@ -3093,16 +3533,30 @@ static int fuse_writepage_locked(struct folio *folio) * that laundered it then reports the folio busy, as it does for any * folio it cannot free. */ - filemap_dirty_folio(mapping, folio); + fuse_writeback_redirty(fc, NULL, folio); mapping_set_error(mapping, error); return error; } +/* + * How many times a data integrity writeback goes round for folios it had to + * skip. Each pass takes the grants the one before it deferred, so one more + * is normally enough; the cap is there because a revoke can take them again. + */ +#define FUSE_WB_DEFER_PASSES 4 + struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; struct inode *inode; unsigned int max_pages; + /* + * The folios this pass could not send because their grant had gone. + * Taken back in fuse_writepages(), where no folio is held, for the + * pass that follows; see fuse_writepages_fill(). + */ + u64 regrant_start; + u64 regrant_end; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data) @@ -3169,9 +3623,18 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, /* Reached alignment */ if (fc->alignment_pages && !(page->index % fc->alignment_pages)) { - /* we are at a point where we would write aligned - * check if we potentially could reach the next alignment */ - if (page->index + fc->alignment_pages > wbc->range_end) + /* + * We are at a point where we would write aligned, so check + * whether the next alignment is still inside the range this + * pass covers. Both sides in page indices: wbc->range_end is + * a byte offset, and it carries nothing at all for a cyclic + * writeback, where write_cache_pages() runs to the end of the + * mapping instead. + */ + pgoff_t last = wbc->range_cyclic ? (pgoff_t) -1 : + (pgoff_t) (wbc->range_end >> PAGE_SHIFT); + + if (page->index + fc->alignment_pages > last) return true; if (ap->num_pages + fc->alignment_pages > fc->max_pages) @@ -3190,6 +3653,8 @@ static int fuse_writepages_fill(struct folio *folio, struct inode *inode = data->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_dlm_span pin; + bool pinned = false; int err; if (!data->ff) { @@ -3204,7 +3669,7 @@ static int fuse_writepages_fill(struct folio *folio, * flag off before calling here, so put it back * rather than drop a write fsync reported done. */ - folio_redirty_for_writepage(wbc, folio); + fuse_writeback_redirty(fc, wbc, folio); goto out_unlock; } } @@ -3226,37 +3691,90 @@ static int fuse_writepages_fill(struct folio *folio, * -ENOSYS, which is not a failure. */ if (fc->dlm && fc->writeback_cache) { + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + + /* + * The revoke handler flushing the range it is taking away. + * That lock is still this client's until the handler returns, + * so send without asking: the record has gone already and + * asking would be a round trip for the very range being + * revoked. Only for that range, since the latched path + * launders the whole mapping and the rest of it may be + * covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + /* - * Driven by a NOTIFY invalidate. A grant this folio does not - * already hold would have to be asked for from inside the - * handler the server is waiting on, for the range that - * handler is revoking, with the folio locked and under - * writeback. The server cannot answer that until the revoke - * completes, and the revoke cannot complete until this - * returns. + * The folio is locked here, and the folios queued before it + * in this pass are under writeback, so a grant this folio + * does not already hold must not be asked for: + * Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE- + * reason.txt states the rule the read path is built around, + * that no cluster lock may be taken while a page lock is + * held. fuse_do_readpage() has AOP_TRUNCATED_PAGE to unlock + * and retry with; ->writepage has nothing of the sort. * - * Leave the folio dirty instead. An ordinary writeback sends - * it with a grant of its own; nothing is lost and no error is - * recorded for a later fsync to report. + * Skip the folio instead. It goes back on the dirty list, + * the range is remembered for fuse_writepages() to take back + * with no folio held, and the pass that follows sends it. + * Nothing is lost and no error is recorded for a later fsync + * to report. + * + * A refused pin is a revoke of this range draining, and + * leaves the folio in the same place for the same reason. A + * revoke elsewhere in the file does not refuse it. */ - if (fuse_in_notify_ctx() && - !fuse_dlm_lock_is_held(fi, folio_pos(folio), - folio_size(folio), - FUSE_PAGE_LOCK_WRITE)) { - folio_redirty_for_writepage(wbc, folio); + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_dlm_unpin(fi); + pinned = false; + } + + if (!pinned) { + fuse_writeback_redirty(fc, wbc, folio); + if (data->regrant_end <= data->regrant_start) { + data->regrant_start = pos; + data->regrant_end = pos + len; + } else { + u64 st = min_t(u64, data->regrant_start, pos); + u64 en = max_t(u64, data->regrant_end, + pos + len); + + /* + * One shard is as far as a single request is + * worth taking back. A pass sweeping a large + * file skips folios a long way apart, and the + * span between them says nothing about what + * is wanted; the folios left out stay dirty + * and a later pass asks for them. + */ + if (en - st <= FUSE_DLM_SHARD_SIZE) { + data->regrant_start = st; + data->regrant_end = en; + } + } err = 0; goto out_unlock; } - err = fuse_dlm_regrant_range(data->ff, inode, folio_pos(folio), - folio_pos(folio) + - folio_size(folio) - 1); + /* + * Held, and pinned so it stays held: this walks the record + * and sends nothing. It stays a call rather than the check + * above so a grant that arrives between them is still used. + */ + err = fuse_dlm_regrant_range(data->ff, inode, pos, + pos + len - 1); if (err < 0 && err != -ENOSYS) { - folio_redirty_for_writepage(wbc, folio); + fuse_writeback_redirty(fc, wbc, folio); + fuse_dlm_unpin(fi); goto out_unlock; } err = 0; } +queue: if (wpa && fuse_writepage_need_send(fc, &folio->page, ap, data, wbc)) { fuse_writepages_send(data); @@ -3267,7 +3785,9 @@ static int fuse_writepages_fill(struct folio *folio, err = -ENOMEM; wpa = fuse_writepage_args_setup(folio, data->ff); if (!wpa) { - folio_redirty_for_writepage(wbc, folio); + fuse_writeback_redirty(fc, wbc, folio); + if (pinned) + fuse_dlm_unpin(fi); goto out_unlock; } fuse_file_get(wpa->ia.ff); @@ -3277,6 +3797,14 @@ static int fuse_writepages_fill(struct folio *folio, ap->num_pages = 0; } folio_start_writeback(folio); + + /* + * Under writeback now, so the flush a revoke runs before it takes + * the grant away waits for these bytes. Nothing further is needed + * to keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); ap->descs[ap->num_pages].offset = 0; ap->descs[ap->num_pages].length = PAGE_SIZE; ap->pages[ap->num_pages] = &folio->page; @@ -3297,7 +3825,7 @@ static int fuse_writepages(struct address_space *mapping, { struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_fill_wb_data data; + unsigned int tries = FUSE_WB_DEFER_PASSES; int err; err = -EIO; @@ -3308,17 +3836,49 @@ static int fuse_writepages(struct address_space *mapping, fc->num_background >= fc->congestion_threshold) return 0; - data.inode = inode; - data.wpa = NULL; - data.ff = NULL; + /* + * A folio whose grant had gone is skipped and its range taken back + * below, which leaves the folio dirty for a later pass. For a data + * integrity writeback there is no later pass: fsync() and close() + * would report the bytes written while they are still only in the + * page cache. Go round again, now that the grant is held, until + * nothing is left deferred. + */ + do { + struct fuse_fill_wb_data data = { .inode = inode }; + bool regranted = false; + + err = write_cache_pages(mapping, wbc, fuse_writepages_fill, + &data); + if (data.wpa) { + WARN_ON(!data.wpa->ia.ap.num_pages); + fuse_writepages_send(&data); + } - err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); - if (data.wpa) { - WARN_ON(!data.wpa->ia.ap.num_pages); - fuse_writepages_send(&data); - } - if (data.ff) - fuse_file_put(data.ff, false); + /* + * Take back what the folios above had to skip, so the pass + * that follows finds the grant and sends the folios they left + * dirty. With no folio held, which is the whole point. + * + * Not from a revoke handler: it would ask for the very range + * it is revoking, so the folios it skipped stay dirty for an + * ordinary writeback, and every pass here would skip them + * again. + */ + if (data.ff && data.regrant_end > data.regrant_start && + !fuse_in_notify_ctx()) { + fuse_dlm_regrant_range(data.ff, inode, + data.regrant_start, + data.regrant_end - 1); + regranted = true; + } + + if (data.ff) + fuse_file_put(data.ff, false); + + if (err || wbc->sync_mode != WB_SYNC_ALL || !regranted) + break; + } while (--tries); out: return err; @@ -3393,6 +3953,9 @@ static int fuse_write_end(struct file *file, struct address_space *mapping, struct folio *folio, void *fsdata) { struct inode *inode = folio->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + bool extending; /* Haven't copied anything? Skip zeroing, size extending, dirtying. */ if (!copied) @@ -3420,18 +3983,41 @@ static int fuse_write_end(struct file *file, struct address_space *mapping, * beyond-EOF optimisation effective: folios wholly past EOF are zeroed * locally instead of sending the server a read-modify-write READ for * data that does not exist yet. + * + * Count the extension until the folio under it is dirty: in between, + * [old size, pos) is covered by nothing fuse_attr_cache_mask() can + * see, and a reply that leaves in that window would shrink i_size + * back. With i_rwsem held shared several writers sit there at once, + * which is why they are counted rather than flagged. */ - if (pos > inode->i_size) { - struct fuse_inode *fi = get_fuse_inode(inode); + extending = pos > inode->i_size; + if (extending) { + atomic_inc(&fi->size_extenders); spin_lock(&fi->lock); - if (pos > inode->i_size) + if (pos > inode->i_size) { + /* + * Retire the attribute replies already on the wire. + * fuse_attr_cache_mask() decides whether the server's + * size wins from an i_size it read before this commit + * and before it slept in the grant query, so a GETATTR + * that left while i_size still matched the server's is + * applied afterwards and shrinks it back. Moving + * attr_version makes fuse_change_attributes_i() drop + * those replies, which is what fuse_write_update_attr() + * moves it for. + */ + fi->attr_version = atomic64_inc_return(&fc->attr_version); i_size_write(inode, pos); + } spin_unlock(&fi->lock); } folio_mark_dirty(folio); + if (extending) + atomic_dec(&fi->size_extenders); + unlock: folio_unlock(folio); folio_put(folio); @@ -3562,9 +4148,22 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) return VM_FAULT_LOCKED; } +/* + * A read fault fills the page cache through ->read_folio and + * ->readahead, which run with the pages locked. Ask for the grant they + * fill under before filemap_fault() locks any of them. + */ +static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) +{ + fuse_read_grant(vmf->vma->vm_file, (loff_t)vmf->pgoff << PAGE_SHIFT, + PAGE_SIZE); + + return filemap_fault(vmf); +} + static const struct vm_operations_struct fuse_file_vm_ops = { .close = fuse_vma_close, - .fault = filemap_fault, + .fault = fuse_filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = fuse_page_mkwrite, }; @@ -4241,29 +4840,9 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, file_update_time(file); } - if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) { + if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) truncate_pagecache_range(inode, offset, offset + length - 1); - /* - * The whole pages inside the hole are gone, so a revoked - * range over them has nothing left to make writeback take - * the grant again for, and would sit in the tree unfreed. - * The pages straddling the ends survive with their punched - * part zeroed, so their record still names real (now zero) - * bytes and stays. Only when the drop really emptied the - * span: a busy folio that survived keeps its record. - */ - if (fm->fc->dlm && fm->fc->writeback_cache) { - uint64_t first = PAGE_ALIGN(offset); - uint64_t last = (uint64_t)(offset + length) & PAGE_MASK; - - if (first < last && - !filemap_range_has_page(inode->i_mapping, first, - last - 1)) - fuse_dlm_ranges_dropped(fi, first, last - 1); - } - } - fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); out: @@ -4367,24 +4946,9 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, if (err) goto out; - { - loff_t lstart = ALIGN_DOWN(pos_out, PAGE_SIZE); - loff_t lend = ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1; - - truncate_inode_pages_range(inode_out->i_mapping, lstart, lend); - - /* - * The record over the dropped span has nothing left to - * describe, and left DIRTY it would make the next partial - * write there keep and flush folio bytes nobody wrote. - * Only when the drop really emptied it: a folio a - * concurrent fault put back keeps its record. - */ - if (fc->dlm && fc->writeback_cache && - !filemap_range_has_page(inode_out->i_mapping, lstart, - lend)) - fuse_dlm_ranges_dropped(fi_out, lstart, lend); - } + truncate_inode_pages_range(inode_out->i_mapping, + ALIGN_DOWN(pos_out, PAGE_SIZE), + ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1); file_update_time(file_out); fuse_write_update_attr(inode_out, pos_out + outarg.size, outarg.size); @@ -4428,6 +4992,21 @@ int fuse_migrate_folio(struct address_space *mapping, struct folio *dst, #endif +/* + * POSIX_FADV_WILLNEED, and readahead(2) with it, populate the page cache + * through ->readahead, which runs with the pages locked. Ask for the + * grant that fill needs while nothing is held; a window no grant covers + * is given back unfilled. + */ +static int fuse_fadvise(struct file *file, loff_t offset, loff_t len, + int advice) +{ + if (advice == POSIX_FADV_WILLNEED && offset >= 0 && len > 0) + fuse_read_grant(file, offset, len); + + return generic_fadvise(file, offset, len, advice); +} + static const struct file_operations fuse_file_operations = { .llseek = fuse_file_llseek, .read_iter = fuse_file_read_iter, @@ -4447,6 +5026,7 @@ static const struct file_operations fuse_file_operations = { .poll = fuse_file_poll, .fallocate = fuse_file_fallocate, .copy_file_range = fuse_copy_file_range, + .fadvise = fuse_fadvise, }; static const struct address_space_operations fuse_file_aops = { @@ -4483,6 +5063,8 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->write_stream_run = 0; fi->write_stream_next = 0; fi->write_stream_start = 0; + atomic_set(&fi->size_extenders, 0); + fi->wb_crop = 0; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 00e5c5f8998420..bd0afb948781f3 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -2,46 +2,33 @@ /* * FUSE page lock cache implementation * - * cache->ranges records the grants the server has given this client. A - * grant still on the wire covers nothing and must not appear there, but - * a revoke has to be able to find it: otherwise a revoke processed - * before the grant is recorded removes nothing, and the grant recorded - * afterwards is never taken back. - * - * A range therefore carries enum fuse_dlm_range_state: - * - * - REQUESTED, on cache->pending, while its FUSE_DLM_WB_LOCK is in - * flight. - * - * - REVOKED, in either place. On cache->pending it is a request a - * revoke overlapped while it was in flight, and - * fuse_dlm_request_commit() drops that grant instead of recording it. - * In cache->ranges it is a grant that was recorded and has since been - * taken away, kept because the page cache under it is still - * described. It covers nothing either way, so the IO paths ask - * again. - * - * - READ or WRITE, in cache->ranges. The only states - * fuse_dlm_range_is_locked() reports as covered; the mode is not a - * separate field, since a range is either not held or held in one - * definite mode. - * - * In-flight requests are kept off the tree so the state is consulted - * only where a request is retired, not by every tree walker. - * - * The record says nothing about the page cache under a range. What is + * The shards record the grants the server has given this client, a bit + * per page. A grant still on the wire covers nothing and must not + * appear there, but a revoke has to be able to find it: otherwise a + * revoke processed before the grant is recorded removes nothing, and the + * grant recorded afterwards is never taken back. A request in flight + * therefore waits on cache->pending, where a revoke marks it killed and + * fuse_dlm_request_commit() drops the grant instead of recording it. + * + * Keeping requests out of the shards leaves every walker looking at + * grants alone. + * + * The record says nothing about the page cache under a page. What is * cached there, and whether the server has seen it, is what the page - * cache itself answers. + * cache itself answers. A revoked grant is therefore forgotten, not + * kept: writeback holds the range again for every run it sends, and an + * absent record and a revoked one both make it ask. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include #include -#include #include #include #include -#include +#include +#include /* @@ -51,81 +38,89 @@ */ #define FUSE_DLM_GRANT_RETRIES 16 -/* Lifecycle of a range; see the file comment above */ -enum fuse_dlm_range_state { - /* FUSE_DLM_WB_LOCK in flight, on cache->pending */ - FUSE_DLM_RANGE_REQUESTED, - /* - * On cache->pending, revoked in flight and the grant must not be - * recorded. In cache->ranges, granted once and taken away, kept to - * describe the page cache under it. Covers nothing either way. - */ - FUSE_DLM_RANGE_REVOKED, - /* Granted shared, in cache->ranges */ - FUSE_DLM_RANGE_READ, - /* Granted exclusive, in cache->ranges */ - FUSE_DLM_RANGE_WRITE, -}; +/* + * How far beyond the requested range a grant is recorded. + * + * A server may grant more than was asked for, and recording the extra is + * what lets the writes that follow skip the round trip entirely. But + * coverage is kept per shard, so the number of records a grant creates + * grows with its size, and outarg is the server's to choose: an + * unbounded grant would be an unbounded amount of work here. Cap it. + * Recording less than the server gave is safe - it only costs a + * re-request - and a cap this size still covers thousands of writes. + */ +#define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) -/* A range of pages with a lock */ +/* + * A FUSE_DLM_WB_LOCK request in flight, on cache->pending. + * + * Two ranges, because the range asked for and the range that may end up + * recorded are not the same one: the server may grant more, up to + * FUSE_DLM_MAX_EXTRA_GRANT either side. A revoke has to be tested against + * both, and means something different for each. + */ struct fuse_dlm_range { - /* Interval tree node; only linked once granted */ - struct rb_node rb; - /* - * The range, as byte offsets, both inclusive. Grants arrive page - * aligned, and a range is split only at the bounds of another, so - * these are page aligned too. - */ + /* The range asked for, as byte offsets, both inclusive */ uint64_t start; uint64_t end; - /* Subtree end value for interval tree */ - uint64_t __subtree_end; - /* Lifecycle and, once granted, the mode; see the enum above */ - enum fuse_dlm_range_state state; - /* Temporary list entry for operations, and the cache->pending link */ + /* The widest [start, end] fuse_dlm_request_commit() could record */ + uint64_t wide_start; + uint64_t wide_end; + /* A revoke overlapped the range asked for: the grant is dead */ + bool killed; + /* A revoke overlapped only the excess: record the asked for range */ + bool clamp; + /* The cache->pending link */ struct list_head list; }; -/* The state a grant in @mode is recorded under */ -static inline enum fuse_dlm_range_state -fuse_dlm_granted_state(enum fuse_page_lock_mode mode) +/* + * Bit of the page at @off within its shard. Callers pass page aligned + * bounds: a grant is aligned in __fuse_get_dlm_lock(), a revoke in + * fuse_dlm_revoke_inval_range() and a query in fuse_dlm_lock_is_held(). + */ +static unsigned long fuse_dlm_bit(uint64_t off) { - return mode == FUSE_PAGE_LOCK_READ ? FUSE_DLM_RANGE_READ : - FUSE_DLM_RANGE_WRITE; + return (off & (FUSE_DLM_SHARD_SIZE - 1)) >> PAGE_SHIFT; } /** - * fuse_dlm_state_satisfies - is a range in @held usable for @want - * @held: the state of a range recorded in the tree - * @want: FUSE_DLM_RANGE_READ or FUSE_DLM_RANGE_WRITE + * fuse_dlm_shard_get - the shard covering @off, created if there is none + * @cache: the page cache + * @off: byte offset the caller is about to record a grant at * - * A WRITE grant is exclusive and so covers a READ request; nothing else - * substitutes for anything. The two pending states never appear in the - * tree and cover nothing. + * Only the recording path needs a shard to exist; readers and the revoke + * paths treat a missing one as a region holding no grant. + * + * Return: the shard, or NULL if it could not be allocated. */ -static inline bool fuse_dlm_state_satisfies(enum fuse_dlm_range_state held, - enum fuse_dlm_range_state want) +static struct fuse_dlm_shard *fuse_dlm_shard_get(struct fuse_dlm_cache *cache, + uint64_t off) { - return held == want || - (held == FUSE_DLM_RANGE_WRITE && want == FUSE_DLM_RANGE_READ); -} + unsigned long idx = off >> FUSE_DLM_SHARD_SHIFT; + struct fuse_dlm_shard *shard, *old; -/* Interval tree definitions for page ranges */ -static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) -{ - return range->start; -} + shard = xa_load(&cache->shards, idx); + if (shard) + return shard; -static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) -{ - return range->end; -} + shard = kzalloc(sizeof(*shard), GFP_NOFS); + if (!shard) + return NULL; -INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); + /* + * Two recorders can reach the same empty region at once; the loser + * drops its shard and takes the winner's. + */ + old = xa_cmpxchg(&cache->shards, idx, NULL, shard, GFP_NOFS); + if (old) { + kfree(shard); + /* xa_cmpxchg() returns an errno as an internal entry */ + return xa_is_err(old) ? NULL : old; + } -static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off); + return shard; +} /** * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] @@ -140,296 +135,449 @@ static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off); * The nodes are owned by the threads waiting on their replies: mark * only, never remove or free. * - * Caller holds @cache->lock for write. + * Takes @cache->pending_lock. A request published after this returns is + * one whose FUSE_DLM_WB_LOCK had not been sent when the revoke was + * processed, so the grant it goes on to receive answers a request made + * after the revoke and is recorded, not killed. */ static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, uint64_t start, uint64_t end) { struct fuse_dlm_range *req; - list_for_each_entry(req, &cache->pending, list) + spin_lock(&cache->pending_lock); + list_for_each_entry(req, &cache->pending, list) { if (req->start <= end && start <= req->end) - req->state = FUSE_DLM_RANGE_REVOKED; + req->killed = true; + else if (req->wide_start <= end && start <= req->wide_end) + req->clamp = true; + } + spin_unlock(&cache->pending_lock); } /** - * fuse_page_cache_init - Initialize a page cache lock manager - * @cache: The cache to initialize - * - * Initialize a page cache lock manager for a FUSE inode. - * - * Return: 0 on success, negative error code on failure + * fuse_dlm_cache_init - Initialize a page cache lock manager + * @inode: The fuse inode to initialize the cache of */ -int fuse_dlm_cache_init(struct fuse_inode *inode) +void fuse_dlm_cache_init(struct fuse_inode *inode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - if (!cache) - return -EINVAL; - init_rwsem(&cache->lock); - cache->ranges = RB_ROOT_CACHED; + xa_init(&cache->shards); + spin_lock_init(&cache->pending_lock); INIT_LIST_HEAD(&cache->pending); + spin_lock_init(&cache->pin_lock); + INIT_LIST_HEAD(&cache->pins); + INIT_LIST_HEAD(&cache->fences); + init_waitqueue_head(&cache->pin_wq); +} - return 0; +/* + * Set up @span over the page-aligned range [@offset, @offset + @length), + * the same range a fuse_dlm_lock_is_held() with these arguments asks + * about, so a fence over a page cannot miss a pin on that page. + */ +static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, + size_t length, struct task_struct *owner) +{ + span->start = (uint64_t)offset & PAGE_MASK; + span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + span->owner = owner; } -/** - * fuse_page_cache_destroy - Clean up a page cache lock manager - * @cache: The cache to clean up +/* + * Is @span inside the fence the revoke handler running on this task + * published? * - * Release all locks and free all resources associated with the cache. + * Only there is a pin both unnecessary and unsafe: the lock over that + * range is still this client's until the handler returns, and the fence + * the pin would wait on is the handler's own. Outside it the handler is + * an ordinary writer with no claim on the range, and answering as if it + * held a pin would let a revoke of that other range drain without + * waiting for bytes on their way out. The latched path launders the + * whole mapping from inside a fence over one range, so that is reached. + * + * Built the way fuse_dlm_revoke_begin() builds the fence, so this + * answers what the fence list would. */ -void fuse_dlm_cache_release_locks(struct fuse_inode *inode) +static bool fuse_dlm_in_own_fence(const struct fuse_dlm_span *span) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - struct rb_node *node; + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + uint64_t start, end; - if (!cache) - return; + if (!ctx) + return false; - /* Release all locks */ - down_write(&cache->lock); - /* - * Every grant goes, so every request in flight is revoked. Mark - * only; each node is owned by the thread waiting on its reply. - */ - fuse_dlm_kill_pending(cache, 0, U64_MAX); - while ((node = rb_first_cached(&cache->ranges)) != NULL) { - range = rb_entry(node, struct fuse_dlm_range, rb); - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } - up_write(&cache->lock); + start = (uint64_t) ctx->start & PAGE_MASK; + end = ctx->end >= LLONG_MAX ? U64_MAX : + ((uint64_t) ctx->end | (PAGE_SIZE - 1)); + + return span->start >= start && span->end <= end; } -/** - * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] - * @cache: The page cache - * @start: Start byte offset - * @end: End byte offset - * - * Return: Pointer to the first overlapping range, or NULL if none found +/* + * Does anything on @head share a byte with [@start, @end]? Caller holds + * fuse_dlm_cache.pin_lock. */ -static struct fuse_dlm_range * -fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static bool fuse_dlm_overlaps_locked(struct list_head *head, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_span *span; + + list_for_each_entry(span, head, list) + if (span->start <= end && start <= span->end) + return true; + + return false; +} + +/* fuse_dlm_overlaps_locked() taking the lock itself */ +static bool fuse_dlm_overlaps(struct fuse_dlm_cache *cache, + struct list_head *head, uint64_t start, + uint64_t end) { - return fuse_page_it_iter_first(&cache->ranges, start, end); + bool overlap; + + spin_lock(&cache->pin_lock); + overlap = fuse_dlm_overlaps_locked(head, start, end); + spin_unlock(&cache->pin_lock); + + return overlap; } /** - * fuse_page_try_merge - Try to merge ranges within a specific region - * @cache: The page cache - * @start: Start byte offset - * @end: End byte offset + * fuse_dlm_pin - hold the grants over a range until fuse_dlm_unpin() + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes * - * Attempt to merge ranges within and adjacent to the specified region - * that have the same lock mode. + * Waits out a revoke overlapping that range, so it must not be called + * with a folio held: the revoke drops that same page cache once it has + * drained. A revoke elsewhere in the file is not waited for. The caller + * confirms its grant after this returns, never before; a confirmation + * from before the pin says nothing. */ -static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) { - struct fuse_dlm_range *range, *next; - uint64_t first = start ? start - 1 : start; - uint64_t last = end < U64_MAX ? end + 1 : end; + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - if (!cache) - return; + fuse_dlm_span_set(pin, offset, length, current); /* - * Find the first range that might need merging. Directly adjacent - * ranges can merge, hence the region is widened by one unit to each - * side (saturating at the type bounds). This must stay an - * interval-tree lookup: the tree holds every cached grant of the - * inode and strided writers grow it for the lifetime of the file, - * so seeding the merge by walking from the tree minimum would make - * every new grant cost a full scan. + * The revoke handler driving this inode's page cache, over the + * range it is taking away. Anywhere else it pins like any other + * writer; see fuse_dlm_in_own_fence(). */ - range = fuse_page_it_iter_first(&cache->ranges, first, last); - - /* Try to merge ranges in and around the specified region */ - while (range && range->start <= last) { - /* Get next range before we potentially modify the tree */ - next = NULL; - if (rb_next(&range->rb)) { - next = rb_entry(rb_next(&range->rb), - struct fuse_dlm_range, rb); - } - - /* Merge neighbours the server has given us on the same terms */ - if (next && range->state == next->state && - range->end + 1 == next->start) { - /* Merge ranges: re-insert so __subtree_end is updated */ - fuse_page_it_remove(next, &cache->ranges); - fuse_page_it_remove(range, &cache->ranges); - range->end = next->end; - fuse_page_it_insert(range, &cache->ranges); - kfree(next); - - /* Continue with the same range */ - continue; - } + if (fuse_dlm_in_own_fence(pin)) + return; - /* Move to next range */ - range = next; + spin_lock(&cache->pin_lock); + while (fuse_dlm_overlaps_locked(&cache->fences, pin->start, pin->end)) { + spin_unlock(&cache->pin_lock); + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->fences, + pin->start, pin->end)); + spin_lock(&cache->pin_lock); } + /* + * At the head, so fuse_dlm_unpin() drops the innermost pin of a + * task that holds more than one. + */ + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); } /** - * fuse_dlm_lock_range_locked - Record a granted range of pages - * @inode: The fuse inode - * @start: Start byte offset - * @end: End byte offset - * @mode: Lock mode (read or write) + * fuse_dlm_trypin - fuse_dlm_pin() for a caller that cannot sleep + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes + * + * For the writeback path, which holds a folio locked and under writeback + * and has nothing to wait with. A refusal means a revoke of this range + * is draining; the caller redirties and the pass that follows sends the + * folio. * - * Add a locked range on the specified range of pages. - * If parts of the range are already locked, only add the remaining parts. - * For overlapping ranges, handle lock compatibility: - * - READ locks are compatible with existing READ locks - * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * Return: true if the range is pinned, false if it is not. + */ +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; + + fuse_dlm_span_set(pin, offset, length, current); + + /* See fuse_dlm_pin() */ + if (fuse_dlm_in_own_fence(pin)) + return true; + + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); + + return !fenced; +} + +/** + * fuse_dlm_trypin_span - fuse_dlm_trypin() for a fill that ends elsewhere + * @inode: the fuse inode + * @pin: caller-owned storage, live until fuse_dlm_unpin_span() + * @offset: byte offset the caller is about to fill + * @length: length of the region in bytes * - * Everything inserted here is READ or WRITE: this runs only after the - * server has answered. + * For a read whose reply lands in another task: the node is dropped by + * fuse_dlm_unpin_span() from wherever the fill ends, and carries no + * owner, so a fuse_dlm_unpin() by the task that took it cannot match it + * instead of its own. * - * Caller holds the cache lock for write. + * Never sleeps, and has no notify-context shortcut: a fill is not + * reached from a revoke handler, and a pin taken there would have to be + * dropped from a task that is not in one. * - * Return: 0 on success, negative error code on failure + * Return: true if the range is pinned, false if a revoke of it is + * draining. */ -static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, - enum fuse_page_lock_mode mode) +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *new_range, *next; - enum fuse_dlm_range_state want; - bool covered_to_end = false; - int ret = 0; - LIST_HEAD(to_lock); - LIST_HEAD(to_upgrade); - uint64_t current_start = start; + bool fenced; - if (!cache || start > end) - return -EINVAL; + fuse_dlm_span_set(pin, offset, length, NULL); - /* The state this grant records */ - want = fuse_dlm_granted_state(mode); + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); - /* - * Ranges are upgraded whole below, so split at the grant bounds - * first: a range extending past the grant would otherwise have its - * uncovered part upgraded with it, recording coverage the server - * never gave. A read range half-covered by a write grant would - * report the other half held for write, and a revoked range - * half-regranted would report its still-revoked bytes as held, so - * writeback would send them without taking the range again. - */ - fuse_dlm_split_at(cache, start); - if (end < U64_MAX) - fuse_dlm_split_at(cache, end + 1); + return !fenced; +} - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); +/** + * fuse_dlm_unpin_span - release the pin fuse_dlm_trypin_span() took + * @inode: the fuse inode + * @pin: the node published there + */ +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool waiters; - /* - * A revoked range is covered again by this grant, and a read - * range needs upgrading when a write is granted. - */ - if (range->state == FUSE_DLM_RANGE_REVOKED || - (want == FUSE_DLM_RANGE_WRITE && - range->state != FUSE_DLM_RANGE_WRITE)) - list_add_tail(&range->list, &to_upgrade); - /* If WRITE lock already exists - nothing to do */ - - /* If there's a gap before this range, we need to add the missing range */ - if (current_start < range->start) { - new_range = kmalloc(sizeof(*new_range), GFP_NOFS); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = range->start - 1; - new_range->state = want; - INIT_LIST_HEAD(&new_range->list); - - list_add_tail(&new_range->list, &to_lock); - } + spin_lock(&cache->pin_lock); + list_del(&pin->list); + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); - /* Move current_start past this range */ - if (range->end >= end) - covered_to_end = true; - else - current_start = max(current_start, range->end + 1); + if (waiters) + wake_up_all(&cache->pin_wq); +} - /* Move to next range */ - range = next; - } +/** + * fuse_dlm_unpin - release the pin this task last took on @inode + * @inode: the fuse inode + * + * Found by owner rather than by node: iomap hands ->put_folio the inode + * and nothing of the iteration, and a task holds one pin at a time. + * + * No shortcut of its own: a revoke handler takes a real pin outside its + * fence and has to drop it, and one that took none has nothing on the + * list for the walk below to find. + */ +void fuse_dlm_unpin(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_span *pin; + bool waiters; - /* If there's a gap after the last range to the end, extend the range */ - if (!covered_to_end && current_start <= end) { - new_range = kmalloc(sizeof(*new_range), GFP_NOFS); - if (!new_range) { - ret = -ENOMEM; - goto out_free; + spin_lock(&cache->pin_lock); + list_for_each_entry(pin, &cache->pins, list) { + if (pin->owner == current) { + list_del(&pin->list); + break; } + } + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); - new_range->start = current_start; - new_range->end = end; - new_range->state = want; - INIT_LIST_HEAD(&new_range->list); + if (waiters) + wake_up_all(&cache->pin_wq); +} - list_add_tail(&new_range->list, &to_lock); - } +/** + * fuse_dlm_revoke_begin - fence the writers over a range that have not + * dirtied yet + * @inode: the fuse inode + * @fence: caller-owned storage, live until fuse_dlm_revoke_end() + * @offset: start byte offset being revoked + * @len: length in bytes, or <= 0 for everything from @offset on + * + * Publishes the range, then waits for the pins over it taken before it. + * On return no thread is between confirming a grant on that range and + * dirtying under it, and none can start, so what the caller flushes is + * everything the grants it is about to drop can have produced. A writer + * elsewhere in the file is neither waited for nor held up. + * + * Publishing before waiting is what makes the wait converge: a pin is + * refused on the same overlap this waits on, so nothing admitted after + * this can prolong it. + * + * Revokes on one inode fence independently, each over its own range. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - /* Everything on this list is now covered in @want */ - list_for_each_entry(range, &to_upgrade, list) - range->state = want; + /* The range fuse_dlm_unlock_range() will be asked to drop */ + fence->start = (uint64_t)offset & PAGE_MASK; + fence->end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + fence->owner = NULL; - /* Add all new ranges to the tree */ - list_for_each_entry(new_range, &to_lock, list) { - /* Add to interval tree */ - fuse_page_it_insert(new_range, &cache->ranges); - } + spin_lock(&cache->pin_lock); + list_add(&fence->list, &cache->fences); + spin_unlock(&cache->pin_lock); - /* Try to merge adjacent ranges with the same mode */ - fuse_dlm_try_merge(cache, start, end); + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->pins, fence->start, + fence->end)); +} - return 0; +/** + * fuse_dlm_revoke_end - drop the fence fuse_dlm_revoke_begin() published + * @inode: the fuse inode + * @fence: the fence published there + */ +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; -out_free: - /* Free any ranges we allocated but didn't insert */ - while (!list_empty(&to_lock)) { - new_range = - list_first_entry(&to_lock, struct fuse_dlm_range, list); - list_del(&new_range->list); - kfree(new_range); - } + spin_lock(&cache->pin_lock); + list_del(&fence->list); + spin_unlock(&cache->pin_lock); - /* - * Nothing to undo on @to_upgrade: every goto here is taken before - * the loop above runs, so no state has been changed yet. - */ - return ret; + wake_up_all(&cache->pin_wq); } -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +/** + * fuse_dlm_cache_release_locks - Clean up a page cache lock manager + * @inode: The fuse inode to clean up the cache of + * + * Release all locks and free all resources associated with the cache. + */ +void fuse_dlm_cache_release_locks(struct fuse_inode *inode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - int ret; + struct fuse_dlm_shard *shard; + unsigned long idx; + /* + * Coverage goes away here, so the whole cache is taken for write: + * see the locking comment on struct fuse_dlm_cache. + */ down_write(&cache->lock); - ret = fuse_dlm_lock_range_locked(inode, start, end, mode); + /* + * Every grant goes, so every request in flight is revoked. Mark + * only; each node is owned by the thread waiting on its reply. + */ + fuse_dlm_kill_pending(cache, 0, U64_MAX); + + xa_for_each(&cache->shards, idx, shard) + kfree(shard); + /* + * Leaves the xarray empty and usable: an inode is released again + * on every O_TRUNC open, not only on eviction. + */ + xa_destroy(&cache->shards); up_write(&cache->lock); +} + +/** + * fuse_dlm_shard_record - record a grant over [@start, @end] in @shard + * @shard: the shard covering [@start, @end] + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * @mode: the mode it was granted in + * + * A write grant sets both maps, so a read query is answered by @granted + * alone, and a write grant over a page already held for read upgrades it + * by setting the bit the read grant left clear. + * + * @granted is set before @write, so a query racing this sees the page + * covered for read before it sees it covered for write. Either order is + * safe, a bit not yet seen only costing a re-request of a range already + * held, but this one never reports a write grant the read map does not + * back. + * + * [@start, @end] must be page aligned and lie wholly inside @shard. + * Caller holds fuse_dlm_cache.lock for read. + */ +static void fuse_dlm_shard_record(struct fuse_dlm_shard *shard, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + unsigned long bit = fuse_dlm_bit(start); + unsigned long last = fuse_dlm_bit(end); + + for (; bit <= last; bit++) { + set_bit(bit, shard->granted); + if (mode == FUSE_PAGE_LOCK_WRITE) + set_bit(bit, shard->write); + } +} + +/** + * fuse_dlm_record_grant - record a grant across the shards it spans + * @cache: the page cache + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode it was granted in + * + * Each shard is filled in on its own: coverage only grows here, and a + * walker under @cache->lock held for read may see part of the grant + * before the rest, which makes it ask again for a range it already has + * rather than trust one it has not got. + * + * Caller holds @cache->lock for read. + * + * Return: 0 on success, negative error code on failure. A failure part + * way through leaves the shards already done recorded, which under + * reports the grant and is safe. + */ +static int fuse_dlm_record_grant(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + unsigned long idx, last_idx; - return ret; + if (start > end) + return -EINVAL; + + last_idx = end >> FUSE_DLM_SHARD_SHIFT; + + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard; + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + + shard = fuse_dlm_shard_get(cache, lo); + if (!shard) + return -ENOMEM; + + fuse_dlm_shard_record(shard, lo, hi, mode); + } + + return 0; } /** @@ -440,8 +588,8 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, * @end: end byte offset being requested (inclusive) * * The mode is not recorded here: until the server answers the range is - * held in neither, and the mode that reaches the tree is the one passed - * to fuse_dlm_request_commit(). + * held in neither, and the mode that reaches the shards is the one + * passed to fuse_dlm_request_commit(). * * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different * threads, so a revoke can be processed before the grant the reply @@ -449,11 +597,18 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, * that revoke a node to mark; without one it removes nothing, and the * grant recorded afterwards is never taken back by any later NOTIFY. * - * The request covers nothing while in flight, so it is kept off the - * tree. @req is reachable only through cache->pending, which both - * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink under - * the cache lock before the caller returns; stack storage is therefore - * fine and nothing is allocated here. + * The request covers nothing while in flight, so it is kept out of the + * shards. @req is reachable only through cache->pending, which both + * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink before + * the caller returns; stack storage is therefore fine and nothing is + * allocated here. + * + * Publishing touches the pending list and nothing else, so it takes + * @cache->pending_lock alone. It deliberately does not take + * @cache->lock: this runs on every cached write that is not already + * covered, and taking the cache rwsem for write here made every writer + * of a file queue behind every other one before its request had even + * been sent. */ void fuse_dlm_request_begin(struct fuse_inode *inode, struct fuse_dlm_range *req, uint64_t start, @@ -461,15 +616,23 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - RB_CLEAR_NODE(&req->rb); req->start = start; req->end = end; - req->state = FUSE_DLM_RANGE_REQUESTED; - /* Nothing reads this while the request is pending; publish it set */ - - down_write(&cache->lock); + /* + * The bounds __fuse_get_dlm_lock() caps the recorded grant to. A + * revoke between here and the commit must be seen by one of the two + * tests in fuse_dlm_kill_pending(), or it would be recorded over. + */ + req->wide_start = start > FUSE_DLM_MAX_EXTRA_GRANT ? + start - FUSE_DLM_MAX_EXTRA_GRANT : 0; + req->wide_end = U64_MAX - end < FUSE_DLM_MAX_EXTRA_GRANT ? + U64_MAX : end + FUSE_DLM_MAX_EXTRA_GRANT; + req->killed = false; + req->clamp = false; + + spin_lock(&cache->pending_lock); list_add_tail(&req->list, &cache->pending); - up_write(&cache->lock); + spin_unlock(&cache->pending_lock); } /** @@ -480,13 +643,25 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, * @end: end byte offset the server granted (inclusive) * @mode: the mode that was requested * - * Unlinking @req and recording the grant are one step under the cache - * lock, so a revoke lands either before it and is seen on @req, or after - * it and finds the grant in the tree. + * Unlinking @req and recording the grant are one step under + * @cache->lock held for read, so a revoke - which takes it for write - + * lands either before it and is seen on @req, or after it and finds the + * grant in the tree. + * + * A revoke processed while @req was in flight lands in one of three + * places, and fuse_dlm_kill_pending() has already said which: + * + * - over the range asked for. The grant may predate it and there is no + * way to tell, so nothing is recorded and the caller asks again. + * - over the excess the server volunteered beyond it, and nothing else. + * The range asked for is untouched by it and is recorded; the excess + * is dropped, which only costs a re-request. + * - outside both, which says nothing about this grant. The whole of + * [start, end] is recorded. * * @req is retired in every case and may be reused. * - * Return: -EAGAIN if a revoke overlapped @req while it was in flight, + * Return: -EAGAIN if a revoke overlapped the range @req asked for, * nothing recorded; otherwise the result of recording the grant. */ int fuse_dlm_request_commit(struct fuse_inode *inode, @@ -494,15 +669,29 @@ int fuse_dlm_request_commit(struct fuse_inode *inode, uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - bool revoked; + bool revoked, clamp; int ret = 0; - down_write(&cache->lock); + /* + * Read, not write: recording adds coverage. Holding it across the + * unlink and the record is what keeps a revoke from landing + * between them, since the revoke paths take it for write. + */ + down_read(&cache->lock); + + spin_lock(&cache->pending_lock); list_del(&req->list); - revoked = req->state == FUSE_DLM_RANGE_REVOKED; + revoked = req->killed; + clamp = req->clamp; + if (clamp) { + start = req->start; + end = req->end; + } + spin_unlock(&cache->pending_lock); + if (!revoked) - ret = fuse_dlm_lock_range_locked(inode, start, end, mode); - up_write(&cache->lock); + ret = fuse_dlm_record_grant(cache, start, end, mode); + up_read(&cache->lock); return revoked ? -EAGAIN : ret; } @@ -519,99 +708,9 @@ void fuse_dlm_request_abort(struct fuse_inode *inode, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - down_write(&cache->lock); + spin_lock(&cache->pending_lock); list_del(&req->list); - up_write(&cache->lock); -} - -/** - * fuse_dlm_split_at - make @off start a range - * @cache: The page cache - * @off: byte offset to split at - * - * Splits the range containing @off in two, both halves keeping the state - * of the original, so a revoke can apply to one side only. A no-op when - * @off already starts a range or falls in a gap. - * - * Caller holds @cache->lock for write. - * - * Cannot fail: the split decides which bytes a caller goes on to name, - * and both naming more than was written and lowering more than was sent - * lose data. iomap allocates the state it keeps per folio the same way. - */ -static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off) -{ - struct fuse_dlm_range *range, *tail; - - if (!off) - return; - - range = fuse_page_it_iter_first(&cache->ranges, off, off); - if (!range || range->start == off) - return; - - tail = kmalloc(sizeof(*tail), GFP_NOFS | __GFP_NOFAIL); - - *tail = *range; - INIT_LIST_HEAD(&tail->list); - tail->start = off; - - /* - * Bounds are never edited in place: the interval tree caches a - * subtree end that only insertion recomputes. - */ - fuse_page_it_remove(range, &cache->ranges); - range->end = off - 1; - fuse_page_it_insert(range, &cache->ranges); - fuse_page_it_insert(tail, &cache->ranges); -} - -/** - * fuse_dlm_ranges_dropped - the page cache under [start, end] is gone - * @inode: the fuse inode - * @start: start byte offset (inclusive) - * @end: end byte offset (inclusive) - * - * A revoked range exists only to describe page cache dirtied before the - * grant was taken away. Once that cache is gone the range has nothing - * left to say and is freed; a range still held goes back to describing - * nothing. - * - * The caller must have established that the range really is empty, not - * merely asked for it to be dropped: a folio that survived an - * invalidate is still there, and claiming otherwise would let writeback - * send it with no record of where it came from. - */ -void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, - uint64_t end) -{ - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; - - if (start > end) - return; - - down_write(&cache->lock); - - fuse_dlm_split_at(cache, start); - if (end < U64_MAX) - fuse_dlm_split_at(cache, end + 1); - - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - next = fuse_page_it_iter_next(range, start, end); - - if (range->state == FUSE_DLM_RANGE_REVOKED) { - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } - - range = next; - } - - fuse_dlm_try_merge(cache, start, end); - - up_write(&cache->lock); + spin_unlock(&cache->pending_lock); } /** @@ -620,15 +719,11 @@ void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, * @start: Start byte offset * @end: End byte offset * - * The server has taken [start, end] back. A range that has nothing - * cached under it is removed; one that has is kept and marked - * FUSE_DLM_RANGE_REVOKED, so the page cache it covers stays described. - * Removing it instead would leave a gap, and a gap reads as "no record", - * which is what an untracked range looks like: writeback would then send - * folios dirtied under the grant that was just taken away. - * - * A revoked range covers nothing, so fuse_dlm_range_is_locked() reports - * it uncovered and the IO paths request again. + * The server has taken [start, end] back, so the grants over it are + * removed and the IO paths ask again. Page cache dirtied under a grant + * that has gone is not lost by this: writeback takes the range again for + * every run it sends, and a range it finds unrecorded is a range it asks + * for. * * An inverted range is rejected rather than silently revoking nothing: * the callers revoke coverage, and a revoke that quietly keeps the grant @@ -638,62 +733,86 @@ void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_unlock_range(struct fuse_inode *inode, - uint64_t start, uint64_t end) +int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; + struct fuse_dlm_shard *shard; + unsigned long idx; - if (!cache || start > end) + if (start > end) return -EINVAL; + /* + * Write, not read: this is a path that takes coverage away, and + * the shard walkers rely on that never happening under them. See + * the locking comment on struct fuse_dlm_cache. + */ down_write(&cache->lock); /* - * Before touching the tree, and even when nothing in the tree - * overlaps: a revoke racing an in-flight grant finds no overlap - * because that grant is not recorded yet. + * Before touching any map, and even when nothing is covered: a + * revoke racing an in-flight grant finds nothing set, because that + * grant is not recorded yet. */ fuse_dlm_kill_pending(cache, start, end); - /* Split so the revoked region has its own ranges */ - fuse_dlm_split_at(cache, start); - if (end < U64_MAX) - fuse_dlm_split_at(cache, end + 1); + /* + * Only the regions that hold something, not every index in the + * range: a revoke to EOF runs to U64_MAX and walking that index + * by index would never finish. + */ + xa_for_each_range(&cache->shards, idx, shard, + start >> FUSE_DLM_SHARD_SHIFT, + end >> FUSE_DLM_SHARD_SHIFT) { + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + unsigned long first = fuse_dlm_bit(lo); + unsigned long nbits = fuse_dlm_bit(hi) - first + 1; - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we modify the tree */ - next = fuse_page_it_iter_next(range, start, end); + /* + * Plain, not atomic: @cache->lock is held for write, so no + * reader and no recorder can be looking at these words. + */ + bitmap_clear(shard->granted, first, nbits); + bitmap_clear(shard->write, first, nbits); /* - * A revoked range is kept only to say that the page cache - * under it was dirtied under a grant that has gone, so that - * writeback takes the range again before sending it. With - * nothing cached there it has nothing to say. - * - * A grant with no end is recorded to U64_MAX; the page cache - * is indexed by a signed offset, so ask it about as much of - * that as it can name. + * Keep the shard table to the regions that hold something: + * a file revoked a region at a time would otherwise leave an + * empty shard behind for every one of them. */ - if (filemap_range_has_page(inode->inode.i_mapping, range->start, - min_t(uint64_t, range->end, - LLONG_MAX))) { - range->state = FUSE_DLM_RANGE_REVOKED; - } else { - fuse_page_it_remove(range, &cache->ranges); - kfree(range); + if (bitmap_empty(shard->granted, FUSE_DLM_SHARD_PAGES)) { + xa_erase(&cache->shards, idx); + kfree(shard); } - - range = next; } - fuse_dlm_try_merge(cache, start, end); - up_write(&cache->lock); return 0; } +/* + * Is every page of [@from, @to] covered in @mode? Both bounds are page + * aligned and lie inside @shard. A write grant sets both maps, so a + * read request is answered by @granted alone. + * + * No lock of its own: bits are set atomically and cleared only under + * fuse_dlm_cache.lock held for write, which the caller holds for read. + * A bit set concurrently may be missed, which costs a re-request of a + * range already held. + */ +static bool fuse_dlm_shard_covers(struct fuse_dlm_shard *shard, uint64_t from, + uint64_t to, enum fuse_page_lock_mode mode) +{ + const unsigned long *map = mode == FUSE_PAGE_LOCK_WRITE ? + shard->write : shard->granted; + unsigned long first = fuse_dlm_bit(from); + unsigned long last = fuse_dlm_bit(to); + + return find_next_zero_bit(map, last + 1, first) > last; +} + /** * fuse_dlm_range_is_locked - Check if a byte range is already locked * @inode: The fuse inode @@ -703,106 +822,44 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, * * Return: true if the entire range is locked, false otherwise */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +static bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, + enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - enum fuse_dlm_range_state want; - uint64_t current_start = start; + unsigned long idx, last_idx; + bool covered = true; - if (!cache || start > end) + if (start > end) return false; - /* The state a range has to be in to cover this request */ - want = fuse_dlm_granted_state(mode); - + /* + * Read: coverage is only ever removed under @cache->lock held for + * write, so the set of grants can grow under this walk but never + * shrink. That is what lets the shards be visited one at a time, + * and their maps read without any further lock. The worst a + * concurrent recorder can do is make this report a range uncovered + * that has just become covered, and the caller then asks for a + * grant it already holds. + */ down_read(&cache->lock); - /* Find the first range that overlaps with [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - - /* Check if the entire range is covered */ - while (range && current_start <= end) { - /* - * The held lock must be at least as strong as the one - * requested. A WRITE lock (exclusive) satisfies a READ - * request, so only treat the range as uncovered when the - * held mode is weaker than what we ask for. This avoids - * re-requesting a READ lock for a range we already hold - * a WRITE lock on (e.g. read-after-write). - */ - if (!fuse_dlm_state_satisfies(range->state, want)) { - /* Held lock is weaker than requested */ - up_read(&cache->lock); - return false; - } - - /* Check if there's a gap before this range */ - if (current_start < range->start) { - /* Found a gap */ - up_read(&cache->lock); - return false; - } - - /* Covered through the end of the requested range? */ - if (range->end >= end) { - up_read(&cache->lock); - return true; - } - - /* Move current_start past this range */ - current_start = range->end + 1; - - /* Get next overlapping range */ - range = fuse_page_it_iter_next(range, start, end); - } - - /* Check if we covered the entire range */ - if (current_start <= end) { - /* There's a gap at the end */ - up_read(&cache->lock); - return false; - } + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - up_read(&cache->lock); - return true; -} + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard = xa_load(&cache->shards, idx); + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); -/** - * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere - * @fi: the fuse inode - * - * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully - * covered, this asks whether any part of the file is held exclusively. A - * client that holds a write grant may be sitting on dirty page cache the - * server has not seen, so its mtime and ctime run ahead of anything the - * server can report. - * - * A revoked range is not counted. The state does not keep the mode the - * range was granted in, so a revoked one cannot be told from a read that - * was taken away, and the dirty page cache such a range describes is - * what the caller checks the mapping for instead. - * - * Return: true if at least one recorded range is held for write - */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) -{ - struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; - struct fuse_dlm_range *range; - bool held = false; - - down_read(&cache->lock); - for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; - range = fuse_page_it_iter_next(range, 0, U64_MAX)) { - if (range->state == FUSE_DLM_RANGE_WRITE) { - held = true; + if (!shard || !fuse_dlm_shard_covers(shard, lo, hi, mode)) { + covered = false; break; } } + up_read(&cache->lock); - return held; + return covered; } /** @@ -859,6 +916,7 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, struct fuse_dlm_lock_out outarg; struct fuse_dlm_range req; uint64_t pg_start, pg_end; + uint64_t grant_start, grant_end; int tries = FUSE_DLM_GRANT_RETRIES; int err; @@ -891,6 +949,7 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, } memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); inarg.fh = ff->fh; inarg.start = pg_start; @@ -920,10 +979,10 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, } /* * The range is contended, the same answer a READ gets and - * fuse_do_readfolio() turns into AOP_TRUNCATED_PAGE for its + * fuse_do_readpage() turns into AOP_TRUNCATED_PAGE for its * caller to retry. There is no such convention here, and - * the writeback caller can only redirty the folio it is - * holding, so ask again instead of reporting it. + * the writeback caller loses the folio it is holding on an + * error, so ask again instead of reporting it. */ if (err == -EDEADLK || err == -EAGAIN) goto retry; @@ -940,16 +999,37 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, } /* - * Retire the request and record the grant. A server may grant more - * than was asked for, and recording all of it is what lets later IO - * over the same neighbourhood skip the round trip. - * - * fuse_dlm_kill_pending() matches a request in flight on the bounds - * published to it, so a revoke aimed only at the part beyond them - * marks nothing here. It would have to be a revoke of a sub-range - * the server is granting in the same breath. + * Keep the recorded grant to a bounded distance either side of + * what was asked for; see FUSE_DLM_MAX_EXTRA_GRANT. Both bounds + * stay outside [pg_start, pg_end], so the range this call has to + * cover is still covered. + */ + grant_start = outarg.start; + grant_end = outarg.end; + /* + * Both differences are safe: the check above established + * outarg.start <= pg_start <= pg_end <= outarg.end, and a branch + * is only taken when there is more than the cap to give back, so + * neither adjusted bound can wrap. */ - err = fuse_dlm_request_commit(fi, &req, outarg.start, outarg.end, mode); + if (pg_start - grant_start > FUSE_DLM_MAX_EXTRA_GRANT) + grant_start = pg_start - FUSE_DLM_MAX_EXTRA_GRANT; + if (grant_end - pg_end > FUSE_DLM_MAX_EXTRA_GRANT) + grant_end = pg_end + FUSE_DLM_MAX_EXTRA_GRANT; + + /* + * Align inward. A page is covered only when it is covered whole, + * and the bit helpers take that as given; the bounds themselves are + * the server's to choose, only their superset property having been + * checked. Rounding inward cannot uncover [pg_start, pg_end], + * which is page aligned already. + */ + grant_start = ALIGN(grant_start, PAGE_SIZE); + /* A last-byte offset, so it is the successor that aligns */ + grant_end -= (grant_end + 1) & (PAGE_SIZE - 1); + + /* Retire the request and record the grant */ + err = fuse_dlm_request_commit(fi, &req, grant_start, grant_end, mode); if (err == -EAGAIN) { /* * A revoke overlapping this range was processed while the diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index d1078cd84546d5..ae6aa2bcbdae93 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -7,9 +7,13 @@ #define _FS_FUSE_DLM_CACHE_H #include -#include +#include #include +#include #include +#include +#include +#include struct fuse_inode; @@ -19,6 +23,25 @@ struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * A range held on one of the two lists in struct fuse_dlm_cache: an IO + * between confirming a grant and publishing the page cache it covers, + * or a revoke taking grants away. Caller-owned storage, live until the + * matching unpin or revoke end. + * + * @owner is the pinning task where the pin is dropped by owner, and NULL + * on every fence and on a pin dropped by node because the fill it covers + * ends in another task. + */ +struct fuse_dlm_span { + /* Page-aligned byte offsets, both inclusive */ + uint64_t start; + uint64_t end; + /* The pinning task, NULL for a revoke */ + struct task_struct *owner; + struct list_head list; +}; + /* * fuse_get_dlm_lock() result: the server granted the lock but recording * it locally failed, leaving the grant invisible to @@ -28,38 +51,127 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; */ #define FUSE_DLM_GRANT_UNRECORDED 1 +/* + * Coverage is kept per aligned region of the file rather than in one + * structure for the whole inode, because one structure needs one lock + * and every thread writing the file then serialises on it however far + * apart their ranges are. The region wants to be small enough that + * concurrent writers land in different ones and large enough that the + * per-region overhead stays amortised. + */ +#define FUSE_DLM_SHARD_SHIFT 24 +#define FUSE_DLM_SHARD_SIZE (1ULL << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_PAGES (FUSE_DLM_SHARD_SIZE / PAGE_SIZE) + +/* First and last byte offset (both inclusive) covered by shard @idx */ +#define FUSE_DLM_SHARD_FIRST(idx) ((uint64_t)(idx) << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_LAST(idx) (FUSE_DLM_SHARD_FIRST(idx) + \ + FUSE_DLM_SHARD_SIZE - 1) + +/* + * The grants over one region, a bit per page. + * + * @granted says the page is covered, @write that it is covered for + * write. A write grant sets both, so a read request is answered by + * @granted alone and nothing has to compare modes; @write is a subset of + * @granted. + * + * A region is a fixed span of pages, so the maps are a fixed size and + * recording a grant neither allocates nor rearranges anything: adjacent + * grants coalesce because they set neighbouring bits, and a region + * fragmented to the last page costs no more than the two maps it already + * has. + * + * Bits are set with the atomic helpers, since recorders run concurrently + * under fuse_dlm_cache.lock held for read. They are cleared only under + * that lock held for write, which excludes every reader and every + * recorder, so the revoke path uses the plain bulk helpers and a query + * needs no lock of its own. + */ +struct fuse_dlm_shard { + unsigned long granted[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; + unsigned long write[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; +}; + /* * Page cache lock manager. * - * @ranges holds the grants the client has been given, and the ones it - * has had taken back that still describe page cache - * (FUSE_DLM_RANGE_REVOKED). A request still on the wire covers nothing - * and lives on @pending instead, so tree walkers never filter on state. - * See enum fuse_dlm_range_state in fuse_dlm_cache.c. + * The shards hold the grants the client has been given. A request still + * on the wire covers nothing and lives on @pending instead, so the + * shards answer for grants only. See struct fuse_dlm_range in + * fuse_dlm_cache.c. + * + * Locking, outermost first: + * + * @lock rw_semaphore over the whole cache. Taken for read by + * everything that adds or reads coverage, and for write + * only by the paths that take coverage away + * (fuse_dlm_unlock_range, fuse_dlm_cache_release_locks). + * That is the invariant the shard walks rely on: + * **coverage is only ever removed under @lock held for + * write**, so a walker holding it for read sees a set of + * grants that can grow under it but never shrink, and may + * therefore visit shards one at a time and read their + * maps without any further lock. Shards are freed only + * there too, so a shard pointer stays good for as long as + * the read side is held. + * @pending_lock the pending list. Innermost, and the only lock + * fuse_dlm_request_begin() and fuse_dlm_request_abort() + * take at all. + * @pin_lock the pin and fence lists. Innermost, taken alone, and + * never held across a sleep. + * + * @lock says what is covered now, which is not enough for a writer: it + * confirms a grant, then copies and dirties, and a revoke landing in + * between sends those bytes out after the server has handed the lock on. + * The pin closes that: a revoke waits for the pins over the range it is + * taking away before it removes anything, so a grant confirmed under a + * pin is still held when the bytes become visible to writeback. A read + * is the same the other way round, a fill landing in a range the revoke + * has already swept staying cached under no grant at all. + * + * Both sides are ranges rather than a count, so a revoke fences only the + * writers it overlaps and a write outside it runs on. Refusal and wait + * test the same overlap, which is what makes the wait converge: once a + * fence is published no pin that would prolong it is admitted. The + * nodes are caller storage, so nothing is allocated to take a pin and + * the writeback path can take one with a folio held. */ struct fuse_dlm_cache { - /* Lock protecting the tree and the pending list */ + /* See the locking comment above */ struct rw_semaphore lock; - /* Interval tree of recorded ranges, granted or revoked */ - struct rb_root_cached ranges; /* - * FUSE_DLM_WB_LOCK requests in flight (REQUESTED, or REVOKED once - * a revoke has overlapped one). Owned by the queueing thread; the - * revoke paths mark them only. + * struct fuse_dlm_shard by offset >> FUSE_DLM_SHARD_SHIFT, + * allocated when a region first holds a grant. + */ + struct xarray shards; + /* Protects @pending and the killed flag of everything on it */ + spinlock_t pending_lock; + /* + * FUSE_DLM_WB_LOCK requests in flight. Owned by the queueing + * thread; the revoke paths only mark them killed. */ struct list_head pending; + /* Protects @pins and @fences */ + spinlock_t pin_lock; + /* + * IO between confirming a grant and publishing under it: a writer + * over what it is about to dirty, a read over what it is about to + * fill. See the pin comment above. + */ + struct list_head pins; + /* Revokes in progress, each over the range it takes away */ + struct list_head fences; + /* Both directions: pins draining, and the fences they wait on */ + wait_queue_head_t pin_wq; }; /* Initialize a page cache lock manager */ -int fuse_dlm_cache_init(struct fuse_inode *inode); +void fuse_dlm_cache_init(struct fuse_inode *inode); /* Clean up a page cache lock manager */ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); -/* Lock a range of pages */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); - /* * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a * revoke processed while the reply is on the wire can mark it. @req is @@ -88,28 +200,53 @@ void fuse_dlm_request_abort(struct fuse_inode *inode, int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); -/* Check if a page range is already locked */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); - -/* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ -bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, - size_t length, enum fuse_page_lock_mode mode); +/* + * Hold the grants over [@offset, @offset + @length) against revocation + * until fuse_dlm_unpin(), which drops the pin this task last took. @pin + * is caller-owned storage, live until then. fuse_dlm_pin() waits out a + * revoke overlapping that range and must not be called with a folio + * held; fuse_dlm_trypin() never sleeps and fails instead. Neither may + * be held across a DLM request: that request is answered by the server + * the revoke came from. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin(struct fuse_inode *inode); -/* Is any part of the file held for write? */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); +/* + * fuse_dlm_trypin() for a fill whose reply lands in another task: @pin + * is dropped by node rather than by owner, and is live from the request + * until fuse_dlm_unpin_span(). + */ +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin); /* - * The page cache under [start, end] is gone: free the revoked ranges over - * it. The caller must have established the range really is empty. + * Fence the writers that hold a grant over [@offset, @offset + @len) but + * have not dirtied under it yet, for the duration of a revoke. @len <= 0 + * means to EOF, as in fuse_notify_inval_inode(). @fence is caller-owned + * storage, live until the matching end. Between these the caller may + * drop coverage over that range knowing nothing will be dirtied under + * what it drops, and a write outside it is left alone. */ -void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, - uint64_t end); +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len); +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence); + +/* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ +bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); /* Hold [start, end] again so writeback can send what it found revoked */ int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, uint64_t start, uint64_t end); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, size_t length, enum fuse_page_lock_mode mode); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 87ea35f48377e3..d5f5f58ed61078 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -46,11 +46,29 @@ */ extern const char fuse_notify_ctx_key[]; -static inline void *fuse_notify_ctx_enter(void) +/* + * What a task driving page cache work for a NOTIFY invalidate carries. + * + * @key tells it apart from anything else parked in journal_info. The range + * is the one being revoked: the lock over it is still this client's until + * the handler returns, so writeback of it need not ask for a grant, while + * anything outside it must. + */ +struct fuse_notify_ctx { + const char *key; + loff_t start; + loff_t end; /* inclusive; LLONG_MAX to EOF */ +}; + +static inline void *fuse_notify_ctx_enter(struct fuse_notify_ctx *ctx, + loff_t start, loff_t end) { void *old = current->journal_info; - current->journal_info = (void *)fuse_notify_ctx_key; + ctx->key = fuse_notify_ctx_key; + ctx->start = start; + ctx->end = end; + current->journal_info = ctx; return old; } @@ -59,9 +77,24 @@ static inline void fuse_notify_ctx_leave(void *old) current->journal_info = old; } +static inline struct fuse_notify_ctx *fuse_notify_ctx(void) +{ + struct fuse_notify_ctx *ctx = current->journal_info; + + return (ctx && ctx->key == fuse_notify_ctx_key) ? ctx : NULL; +} + static inline bool fuse_in_notify_ctx(void) { - return current->journal_info == (void *)fuse_notify_ctx_key; + return fuse_notify_ctx(); +} + +/* Is [@pos, @pos + @len) the range the revoke in progress is taking away? */ +static inline bool fuse_in_notify_range(loff_t pos, unsigned int len) +{ + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + + return ctx && pos >= ctx->start && pos + len - 1 <= ctx->end; } /** Default max number of pages that can be used in a single read request */ @@ -274,6 +307,38 @@ struct fuse_inode { unsigned int write_stream_run; loff_t write_stream_next; loff_t write_stream_start; + + /* + * Buffered writes that have committed an i_size + * extension and not yet dirtied the folio under + * it. + * + * The DLM path holds i_rwsem shared, so several + * writers extend i_size at once and each one is + * ahead of the server until its bytes are sent. + * While this is non zero the local size wins over + * the server's; see fuse_attr_cache_mask(). + * FUSE_I_SIZE_UNSTABLE cannot serve: it is one bit + * and every writer clears it. + */ + atomic_t size_extenders; + + /* + * High water mark of i_size, and the size the + * queued writepage requests are cropped against. + * A truncate lowers it + * (fuse_writeback_crop_truncated()); otherwise it + * only follows i_size back down once no request is + * left to protect (fuse_writepage_end()). + * + * i_size itself cannot serve: with DLM the server's + * size is applied to it (fuse_attr_cache_mask()), + * and a reply that is only behind the local writers + * would have fuse_send_writepage() crop away bytes + * on their way out. Protected by fi->lock; + * writeback-cache regular files only. + */ + loff_t wb_crop; }; /* readdir cache (directory only) */ @@ -1282,6 +1347,15 @@ struct fuse_io_args { struct { struct fuse_read_in in; u64 attr_ver; + /* + * The grant the folios are filled under, held + * from the request until the reply has filled + * them; see fuse_send_readpages(). @dlm_fi is + * the inode to drop it on, and NULL when there + * is no pin to drop. + */ + struct fuse_dlm_span dlm_pin; + struct fuse_inode *dlm_fi; } read; struct { struct fuse_write_in in; @@ -1311,7 +1385,7 @@ void fuse_file_free(struct fuse_file *ff); int fuse_finish_open(struct inode *inode, struct file *file); /* Drop the page cache an open must not keep; true if it came out empty */ -bool fuse_open_drop_cache(struct inode *inode); +void fuse_open_drop_cache(struct inode *inode); void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, unsigned int flags); @@ -1547,6 +1621,23 @@ void fuse_flush_writepages(struct inode *inode); void fuse_set_nowrite(struct inode *inode); void fuse_release_nowrite(struct inode *inode); +/* + * A truncate has lowered i_size under fuse_set_nowrite(). That shrink is + * the authoritative one: the bytes above it are gone, and the writepage + * requests still standing over them are meant to be cropped away rather + * than kept. Let the crop follow i_size back down. + * + * Every other shrink -- under DLM, any attribute reply that is merely + * behind the local writers -- must not, which is what fi->wb_crop is for. + * Called under fi->lock; see fuse_flush_writepages(). + */ +static inline void fuse_writeback_crop_truncated(struct inode *inode, + loff_t size) +{ + if (get_fuse_conn(inode)->writeback_cache && S_ISREG(inode->i_mode)) + get_fuse_inode(inode)->wb_crop = size; +} + /** * Scan all fuse_mounts belonging to fc to find the first where * ilookup5() returns a result. Return that result and the diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 09671882ac97a6..1c86892e891d2b 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -498,20 +498,26 @@ u32 fuse_get_cache_mask(struct inode *inode) * for exactly what the grant covers: * * - size, when the server reports less than i_size and the tail it does not - * know about, [srv_size, i_size), is entirely under a write grant. Taking - * the server's answer would shrink i_size and have truncate_pagecache() - * throw the unwritten tail away. - * - mtime and ctime, while a write grant covers unwritten data: our writes - * have stamped them locally and the server's stamps predate them. Only - * while the cache is actually dirty, not for as long as the grant lives: - * a grant is held until it is revoked or the inode is evicted, and past - * the writeback the server's stamps are the newer ones. Keeping ours + * know about, [attr->size, i_size), is under a write grant or still + * cached. Taking the server's answer would shrink i_size, and a shrink + * is destructive here in a way it is not upstream, where the writeback + * cache always keeps STATX_SIZE and this never runs: truncate_pagecache() + * throws the tail away dirty or not, and fuse_send_writepage() crops the + * requests already queued over it, ending their writeback as if they had + * been sent. Neither reports an error, so fsync() succeeds over the + * hole. Upstream only lowers i_size under fuse_set_nowrite(), which is + * what makes both safe there. + * - mtime and ctime, while the page cache is dirty or under writeback: our + * writes have stamped them locally and the server's stamps predate them. + * Only while the cache is actually dirty, not for as long as a grant + * lives: a grant is held until it is revoked or the inode is evicted, and + * past the writeback the server's stamps are the newer ones. Keeping ours * beyond that would hide a remote chown or chmod indefinitely. * * A remote truncate cannot slip through. It has to revoke the grant first, - * and the revoke launders the tail and drops the grant, so by the time the - * smaller size is reported neither check holds and the server's answer is - * applied as usual. A grant the server made but that could not be recorded + * and the revoke launders the tail, drops the grant and drops the folios, so + * by the time the smaller size is reported none of the checks hold and the + * server's answer is applied as usual. A grant the server made but that could not be recorded * (FUSE_DLM_GRANT_UNRECORDED) is invisible to the lock tree and falls back to * trusting the server, as before. * @@ -536,34 +542,52 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, * the tail grants itself while cached writes above the new size * are still waiting for writeback. */ - if (!fuse_dlm_write_grant_exists(fi) && - !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) && - !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) - return cache_mask; - if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) cache_mask |= STATX_MTIME | STATX_CTIME; /* * The local size stays authoritative while the extension is - * covered by a write grant, and also while anything in - * [attr->size, size) is dirty or under writeback: those bytes - * exist only here, and taking the server's smaller size would - * truncate them away before they are ever sent. The grant check - * alone misses them, because a page-mkwrite grant is never + * covered by a write grant, and also while [attr->size, size) is + * cached at all: taking the server's smaller size would truncate + * those folios away, and lowering i_size crops the writeback + * requests already queued over them (fuse_send_writepage()), both + * of which throw bytes away without reporting an error. The grant + * check alone misses them, because a page-mkwrite grant is never * recorded and a local truncate revokes its own tail grants. + * + * Any folio, not just a dirty one. A reply that is merely behind + * -- the server has not seen the tail yet -- always has those + * folios here, but not always dirty: writeback can have cleaned + * one while the reply that would move the server's size was still + * on the wire, and the beyond-EOF shortcut in fuse_write_begin() + * turns a short i_size into a folio zeroed over data the server + * does hold. A remote truncate is not caught by mistake: it has + * to revoke first, and the revoke drops the range, so by the time + * the smaller size is reported there is nothing cached there and + * the shrink applies. + * + * All three miss a write that has committed its extension and not + * yet dirtied the folio under it: nothing is cached there, and a + * NOTIFY can revoke the grant in between. With i_rwsem held + * shared several writers sit in that window at once, which is why + * they are counted rather than flagged. Cheapest test first: the + * grant query sleeps. */ if (have_size && size > (loff_t) attr->size && - (fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, - FUSE_PAGE_LOCK_WRITE) || - filemap_range_needs_writeback(inode->i_mapping, attr->size, - size - 1))) + (atomic_read(&fi->size_extenders) || + filemap_range_has_page(inode->i_mapping, attr->size, size - 1) || + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE))) cache_mask |= STATX_SIZE; return cache_mask; } +/* Defined below, next to the NOTIFY paths that are its other caller. */ +static void fuse_invalidate_mapping_range(struct inode *inode, pgoff_t start, + pgoff_t end, bool may_be_dirty); + static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, u64 attr_valid, u64 attr_version, u64 evict_ctr) @@ -602,6 +626,30 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr evict_ctr); oldsize = inode->i_size; + /* + * fuse_attr_cache_mask() answered before the grant query slept, and a + * write below EOF bumps neither fi->attr_version nor + * fi->size_extenders: it extends nothing, so the version check cannot + * drop the reply and the count cannot hold the size. Folios can have + * been dirtied in the doomed range since the answer, and + * truncate_pagecache() below throws them away with no error to report + * it. + * + * Re-test here instead, where nothing sleeps between the answer and + * the truncate acting on it, and keep the local size when the range + * still holds bytes the server has not seen. A remote truncate is + * unaffected: it revokes first, and the revoke launders and drops the + * range, so there is nothing here to find. + */ + if (have_size && !(cache_mask & STATX_SIZE) && fc->dlm && + fc->writeback_cache && S_ISREG(inode->i_mode) && + (loff_t) attr->size < oldsize && + filemap_range_needs_writeback(inode->i_mapping, attr->size, + oldsize - 1)) { + cache_mask |= STATX_SIZE; + attr->size = oldsize; + } + /* * In case of writeback_cache enabled, the cached writes beyond EOF * extend local i_size without keeping userspace server in sync. So, @@ -627,8 +675,51 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr bool have_mtime = !sx || (sx->mask & STATX_MTIME); if (have_size && oldsize != attr->size) { - truncate_pagecache(inode, attr->size); - if (!fc->explicit_inval_data) + /* + * Not under DLM. This runs after fi->lock is + * dropped and takes nothing a writer holds, so a + * folio dirtied between the decision and the walk, + * or during it, is discarded with no error to report + * it. A real truncate may do this because + * fuse_set_nowrite() and i_rwsem hold the writers + * off; an attribute reply holds off nothing. + * + * Keeping the folios costs nothing either way. If + * the size was wrong they are written back and the + * size recovers; if it was right, the revoke that had + * to precede it already dropped the range and there + * is nothing here to discard. + */ + if (!(fc->dlm && fc->writeback_cache)) + truncate_pagecache(inode, attr->size); + /* + * A size that differs from the cached one is + * upstream's evidence that another client wrote + * the file, and the data cache is dropped on the + * strength of it. Under DLM that evidence is + * redundant: a remote write cannot land without + * revoking the grant covering it, and the revoke + * invalidates the range on its way through + * (fuse_reverse_inval_inode()). Coherency is the + * grant's job here, not the size delta's. + * + * It is also not affordable. A shared file reaches + * this on every write: another node extends it, the + * GETATTR that fuse_cache_write_iter() takes for + * STATX_MODE returns the larger size, and + * fuse_attr_cache_mask() holds STATX_SIZE only for + * the opposite direction -- a local size ahead of + * the server's. So a write that wanted the mode + * bits invalidates the whole mapping, and on a + * writeback mapping that is a laundering pass, not + * a drop: every dirty folio in the file goes out + * one FUSE_WRITE at a time, with the writer parked + * behind all of it. + * + * The size itself is still taken below. Only the + * data cache is kept. + */ + if (!fc->explicit_inval_data && !fc->dlm) inval = true; } else if (have_mtime && fc->auto_inval_data) { struct timespec64 new_mtime = { @@ -644,29 +735,16 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr inval = true; } - if (inval) - invalidate_inode_pages2(inode->i_mapping); - /* - * The DLM record has to follow the cache out, as on every - * other path that drops it. A revoked range exists only to - * make writeback take the grant again before sending the - * folios under it, so once they are gone it describes - * nothing and would sit in the tree unfreed. The pages above - * the new size are gone unconditionally; the rest only when - * the invalidate really emptied the mapping, so a folio that - * survived (or was faulted back) keeps its record. + * Through the helper rather than invalidate_inode_pages2(): + * it writes the range back in batched writeback first, where + * the bare invalidate would launder it a folio per round + * trip, and it substitutes a plain drop while writepages are + * frozen, where laundering would wait for a reply that + * fuse_flush_writepages() is holding back. */ - if (fc->dlm && fc->writeback_cache) { - if (have_size && oldsize != attr->size) - fuse_dlm_ranges_dropped(fi, - PAGE_ALIGN(attr->size), - U64_MAX); - if (inval && - !filemap_range_has_page(inode->i_mapping, 0, - LLONG_MAX)) - fuse_dlm_ranges_dropped(fi, 0, U64_MAX); - } + if (inval) + fuse_invalidate_mapping_range(inode, 0, -1, true); } if (IS_ENABLED(CONFIG_FUSE_DAX)) @@ -934,8 +1012,8 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, * fuse_do_setattr() invalidates the mapping after releasing the freeze, the * O_TRUNC open path calls truncate_pagecache(). */ -static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, - pgoff_t end, bool may_be_dirty) +static void fuse_invalidate_mapping_range(struct inode *inode, pgoff_t start, + pgoff_t end, bool may_be_dirty) { struct fuse_inode *fi = get_fuse_inode(inode); loff_t last; @@ -968,8 +1046,6 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, { struct fuse_inode *fi; struct inode *inode; - uint64_t pg_first; - uint64_t pg_last; loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; @@ -1006,7 +1082,10 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * here must not send a DLM request. See * fuse_writepages_fill(). */ - void *notify_ctx = fuse_notify_ctx_enter(); + struct fuse_notify_ctx ctx; + struct fuse_dlm_span fence; + void *notify_ctx; + bool fenced; pg_start = offset >> PAGE_SHIFT; if (len <= 0) @@ -1014,17 +1093,38 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; + /* Byte bounds of the same region */ + end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + + notify_ctx = fuse_notify_ctx_enter(&ctx, offset, end_byte); + /* - * Byte bounds of the same region, and the page aligned form - * the DLM record is told about. A grant is recorded page - * aligned, so the range handed to it has to cover whole - * pages or the record would keep a range the page cache no - * longer backs. + * Fence the writers that hold a grant over this range but + * have not dirtied under it yet. Their bytes are in no page + * cache and on no wire, so nothing below can find them, and + * once the grant is gone they would go out behind the + * handover. Published before the flush so what it drains is + * flushed with everything else. + * + * Only the writers over this range: a write elsewhere in the + * file holds a grant this handler does not touch, and neither + * waits for the other. + * + * Entered after fuse_notify_ctx_enter(): the page cache work + * below is this handler's own and must not fence itself. + * + * A fenced writer may be waiting for a FUSE_WRITE reply, so + * this waits on the server the same way the flush below does. + * + * Regular files only: the record shares the readdir cache + * union arm and exists nowhere else. Latched into a local, + * since fc->dlm can be cleared while this runs and the fence + * has to come off the list either way. */ - end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; - pg_first = (uint64_t)offset & PAGE_MASK; - pg_last = len <= 0 ? U64_MAX : - (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + fenced = S_ISREG(inode->i_mode) && fc->dlm && + fc->writeback_cache; + if (fenced) + fuse_dlm_revoke_begin(fi, &fence, offset, len); /* * A data invalidation means another (remote) entity is modifying @@ -1095,6 +1195,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * waiting on. do_writepages() runs in this context, * so the grant is asked for before the revoke. * + * One pass is enough: the fence above has drained the + * writers that held a grant without having dirtied + * under it, and refuses new ones, so nothing can turn + * up dirty behind this. + * * Waited out here rather than left to the drop, which * launders when the range may be dirty and so waits * for these same replies. One explicit wait, before @@ -1102,7 +1207,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * writeback to block on. Either way a server that * revokes from a thread it also needs to answer * FUSE_WRITE on deadlocks here, the same contract - * fuse_notify_invalidate_range() states for a frozen + * fuse_invalidate_mapping_range() states for a frozen * inode. The error is left to the mapping, where * fsync collects it. */ @@ -1132,23 +1237,12 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * cached there. */ if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1, true); + fuse_invalidate_mapping_range(inode, 0, -1, + true); else if (has_pages) - fuse_notify_invalidate_range(inode, pg_start, - pg_end, - may_be_dirty); - - /* - * A revoked range exists to describe page cache - * dirtied before the grant went; with that cache gone - * it has nothing left to say. Only when it really - * went: an invalidate can leave a busy folio behind, - * and that folio still needs its record. - */ - if (has_pages && - !filemap_range_has_page(inode->i_mapping, offset, - end_byte)) - fuse_dlm_ranges_dropped(fi, pg_first, pg_last); + fuse_invalidate_mapping_range(inode, pg_start, + pg_end, + may_be_dirty); if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", @@ -1161,9 +1255,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - fuse_notify_invalidate_range(inode, pg_start, pg_end, - true); + fuse_invalidate_mapping_range(inode, pg_start, pg_end, + true); } + if (fenced) + fuse_dlm_revoke_end(fi, &fence); fuse_notify_ctx_leave(notify_ctx); } iput(inode);