From 2c35958d9383b26d6ce2816ac2e41a378683e353 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 4 Sep 2026 13:43:24 +0200 Subject: [PATCH 01/16] fuse: fill a folio under a pinned read grant A folio made uptodate from the server is served to every later reader, so the grant it was fetched under has to be held from the confirmation until the bytes are in the page cache. Without that a revoke sweeping the range leaves the fill behind it: the folio stays cached, uncovered, and the server sends no further notify for a lock this client no longer holds. Confirm the grant under a pin, as the write path does. Refused, or not covered, unlock the folio and back off with AOP_TRUNCATED_PAGE: neither the wait nor the grant request may be taken with a page lock held. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 31d78bf542b69f..fae6e03e697c0b 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1134,10 +1134,59 @@ static int fuse_read_folio_merge(struct file *file, struct folio *folio) return 0; } +/** + * fuse_read_folio_retry - back off a fill with no grant to run under + * @file: file to read through + * @folio: the folio handed over locked, unlocked here + * @pos: byte offset of @folio + * @len: its size in bytes + * + * Neither reason a fill is refused can be dealt with while the folio is + * held: waiting a revoke out would hold the page cache that revoke is + * about to drop, and no grant may be asked for under a page lock at all + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * Unlock, do both, and send the caller round again to find the range + * covered. + * + * Return: AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_retry(struct file *file, struct folio *folio, + loff_t pos, size_t len) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + struct fuse_dlm_span pin; + int err; + + folio_unlock(folio); + + /* Wait the revoke out; what it leaves behind is asked for below */ + fuse_dlm_pin(fi, &pin, pos, len); + fuse_dlm_unpin(fi); + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_READ); + if (err == -ENOSYS) + return AOP_TRUNCATED_PAGE; + if (err < 0) + return err; + /* + * Granted but unrecorded, so the retry finds the range uncovered + * and comes straight back here. Report it rather than spin. + */ + if (err > 0) + return -ENOMEM; + + return AOP_TRUNCATED_PAGE; +} + static int fuse_read_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + struct fuse_dlm_span pin; + bool pinned = false; int err; err = -EIO; @@ -1155,6 +1204,30 @@ static int fuse_read_folio(struct file *file, struct folio *folio) */ folio_wait_writeback(folio); + /* + * The grant the folio is filled under, held from the confirmation + * until the bytes are in the page cache. What lands here is served + * to every later reader of the file, so it must neither be fetched + * under a grant a revoke has taken away nor be dropped into a range + * a revoke has just swept: such a folio is uptodate and covered by + * nothing, and no further notify comes for a lock this client no + * longer holds. + * + * The pin closes the second, since a revoke over the folio waits + * for the fill and drops the folio after it; the confirmation + * closes the first. Both fail into fuse_read_folio_retry(). + */ + if (fc->dlm && fc->writeback_cache) { + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin(fi); + pinned = false; + } + if (!pinned) + return fuse_read_folio_retry(file, folio, pos, len); + } + /* * Only a folio still holding what a write put in it has anything to * keep. One that was merely being written back is clean by now, @@ -1170,6 +1243,12 @@ static int fuse_read_folio(struct file *file, struct folio *folio) fuse_invalidate_atime(inode); out: folio_unlock(folio); + /* + * After the unlock, so a revoke draining this pin finds the folio + * it has to drop unlocked and takes it out. + */ + if (pinned) + fuse_dlm_unpin(fi); return err; } From 200f4b528183c8134c614ff2f40ca131602612f6 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 4 Sep 2026 13:44:10 +0200 Subject: [PATCH 02/16] fuse: fill a readahead window under a pinned read grant A readahead reply lands in the page cache from the task that processes it, so the pin over the folios has to span the request: taken before it is sent and dropped once the folios are filled and unlocked. A revoke of the range waits for that and drops the folios after; one already draining refuses the pin and the window goes back unfilled. The node therefore outlives the pinning task, which fuse_dlm_unpin() cannot express. Add a span-owned pin, dropped by node and carrying no owner for the by-owner search to match. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 75 ++++++++++++++++++++++++++++++++++++---- fs/fuse/fuse_dlm_cache.c | 64 +++++++++++++++++++++++++++++++--- fs/fuse/fuse_dlm_cache.h | 30 ++++++++++++---- fs/fuse/fuse_i.h | 9 +++++ 4 files changed, 160 insertions(+), 18 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index fae6e03e697c0b..792a92b9f2fe51 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1338,19 +1338,40 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, folio_end_read(ap->folios[i], !err); folio_put(ap->folios[i]); } + + /* + * Dropped after the folios, which are filled, uptodate and unlocked + * by now: a revoke draining this pin finds them and takes them out. + * Held until here so it cannot have swept before they were there. + */ + if (ia->read.dlm_fi) + fuse_dlm_unpin_span(ia->read.dlm_fi, &ia->read.dlm_pin); + if (ia->ff) fuse_file_put(ia->ff, false); fuse_io_free(ia); } -static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, - unsigned int count) +/** + * fuse_send_readpages - read a run of folios of a readahead window + * @ia: the request, owning the folios and the pin over them + * @file: file to read through + * @count: bytes to read, starting at the first folio + * + * Return: 0 once the request is on its way or has been completed, + * -EAGAIN when a revoke of the range refused the grant and nothing was + * sent. The folios are given back either way. + */ +static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file, + unsigned int count) { struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); struct fuse_args_pages *ap = &ia->ap; loff_t pos = folio_pos(ap->folios[0]); + unsigned int i; ssize_t res; int err; @@ -1365,6 +1386,29 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, } WARN_ON((loff_t) (pos + count) < 0); + /* + * The grant fuse_readahead() took, confirmed under a pin and held + * until the reply has filled the folios. A revoke of the range + * waits for that, so the reply cannot be fetched under a grant the + * server has since handed on, and cannot land behind a sweep that + * would leave the folios uptodate and covered by nothing. + * + * Refused, or gone since it was asked for: give the folios back + * unfilled rather than serve what no lock covers. The read that + * wanted them comes back through fuse_read_folio(), which asks + * again with no folio held. + */ + if (fm->fc->dlm && fm->fc->writeback_cache) { + if (!fuse_dlm_trypin_span(fi, &ia->read.dlm_pin, pos, count)) + goto uncovered; + if (!fuse_dlm_lock_is_held(fi, pos, count, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin_span(fi, &ia->read.dlm_pin); + goto uncovered; + } + ia->read.dlm_fi = fi; + } + fuse_read_args_fill(ia, file, pos, count, FUSE_READ); ia->read.attr_ver = fuse_get_attr_version(fm->fc); if (fm->fc->async_read) { @@ -1372,12 +1416,21 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ap->args.end = fuse_readpages_end; err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (!err) - return; + return 0; } else { res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } fuse_readpages_end(fm, &ap->args, err); + return 0; + +uncovered: + for (i = 0; i < ap->num_folios; i++) { + folio_end_read(ap->folios[i], false); + folio_put(ap->folios[i]); + } + fuse_io_free(ia); + return -EAGAIN; } static void fuse_readahead(struct readahead_control *rac) @@ -1403,6 +1456,11 @@ static void fuse_readahead(struct readahead_control *rac) * left in @rac. A server without DLM support answers -ENOSYS and * clears fc->dlm, which is not a failure. * + * The grant is only asked for here. Confirming it and holding it + * against a revoke is fuse_send_readpages(), one run of folios at a + * time, since that is where the request the reply fills them from + * goes out; a run it declines ends the window. + * * ->readahead is entered with every folio of the window already * locked, and pulling one off @rac is what unlocks it, so the round * trip is taken under those locks. See the readahead line of @@ -1492,7 +1550,8 @@ static void fuse_readahead(struct readahead_control *rac) pages += folio_pages; folio = NULL; } - fuse_send_readpages(ia, rac->file, pages << PAGE_SHIFT); + if (fuse_send_readpages(ia, rac->file, pages << PAGE_SHIFT)) + break; nr_pages -= pages; } if (folio) { @@ -1532,9 +1591,11 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* * A NOTIFY invalidate racing this read drops the folios it * supersedes, so the read either misses and refetches or returns - * data that was current when it was copied. There is nothing to - * fence: unlike a write, a read leaves nothing behind that could - * reach the server under a grant it no longer holds. + * data that was current when it was copied. What a read does leave + * behind is the page cache it fills, which must not outlast the + * grant it was fetched under; that is fenced where the filling + * happens, in fuse_read_folio() and fuse_send_readpages(), and the + * grant taken here is what they confirm. */ if (fuse_inode_force_dio(inode)) { size_t count = iov_iter_count(to); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index d3ef8548d3b380..8b24c563ee03ec 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -179,11 +179,11 @@ void fuse_dlm_cache_init(struct fuse_inode *inode) * about, so a fence over a page cannot miss a pin on that page. */ static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, - size_t length) + size_t length, struct task_struct *owner) { span->start = (uint64_t)offset & PAGE_MASK; span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); - span->owner = current; + span->owner = owner; } /* @@ -243,7 +243,7 @@ void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, if (fuse_in_notify_ctx()) return; - fuse_dlm_span_set(pin, offset, length); + fuse_dlm_span_set(pin, offset, length, current); spin_lock(&cache->pin_lock); while (fuse_dlm_overlaps_locked(&cache->fences, pin->start, pin->end)) { @@ -285,7 +285,7 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, if (fuse_in_notify_ctx()) return true; - fuse_dlm_span_set(pin, offset, length); + fuse_dlm_span_set(pin, offset, length, current); spin_lock(&cache->pin_lock); fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, @@ -297,6 +297,62 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, return !fenced; } +/** + * fuse_dlm_trypin_span - fuse_dlm_trypin() for a fill that ends elsewhere + * @inode: the fuse inode + * @pin: caller-owned storage, live until fuse_dlm_unpin_span() + * @offset: byte offset the caller is about to fill + * @length: length of the region in bytes + * + * For a read whose reply lands in another task: the node is dropped by + * fuse_dlm_unpin_span() from wherever the fill ends, and carries no + * owner, so a fuse_dlm_unpin() by the task that took it cannot match it + * instead of its own. + * + * Never sleeps, and has no notify-context shortcut: a fill is not + * reached from a revoke handler, and a pin taken there would have to be + * dropped from a task that is not in one. + * + * Return: true if the range is pinned, false if a revoke of it is + * draining. + */ +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; + + fuse_dlm_span_set(pin, offset, length, NULL); + + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); + + return !fenced; +} + +/** + * fuse_dlm_unpin_span - release the pin fuse_dlm_trypin_span() took + * @inode: the fuse inode + * @pin: the node published there + */ +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool waiters; + + spin_lock(&cache->pin_lock); + list_del(&pin->list); + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); + + if (waiters) + wake_up_all(&cache->pin_wq); +} + /** * fuse_dlm_unpin - release the pin this task last took on @inode * @inode: the fuse inode diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index ec2e82c406a1cb..ae6aa2bcbdae93 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -24,10 +24,14 @@ struct fuse_file; enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; /* - * A range held on one of the two lists in struct fuse_dlm_cache: a - * writer between confirming a grant and dirtying under it (@owner set), - * or a revoke taking grants away (@owner NULL). Caller-owned storage, - * live until the matching unpin or revoke end. + * A range held on one of the two lists in struct fuse_dlm_cache: an IO + * between confirming a grant and publishing the page cache it covers, + * or a revoke taking grants away. Caller-owned storage, live until the + * matching unpin or revoke end. + * + * @owner is the pinning task where the pin is dropped by owner, and NULL + * on every fence and on a pin dropped by node because the fill it covers + * ends in another task. */ struct fuse_dlm_span { /* Page-aligned byte offsets, both inclusive */ @@ -122,7 +126,9 @@ struct fuse_dlm_shard { * between sends those bytes out after the server has handed the lock on. * The pin closes that: a revoke waits for the pins over the range it is * taking away before it removes anything, so a grant confirmed under a - * pin is still held when the bytes become visible to writeback. + * pin is still held when the bytes become visible to writeback. A read + * is the same the other way round, a fill landing in a range the revoke + * has already swept staying cached under no grant at all. * * Both sides are ranges rather than a count, so a revoke fences only the * writers it overlaps and a write outside it runs on. Refusal and wait @@ -149,8 +155,9 @@ struct fuse_dlm_cache { /* Protects @pins and @fences */ spinlock_t pin_lock; /* - * Writers between confirming a grant and dirtying under it, each - * over the range it is about to write. See the pin comment above. + * IO between confirming a grant and publishing under it: a writer + * over what it is about to dirty, a read over what it is about to + * fill. See the pin comment above. */ struct list_head pins; /* Revokes in progress, each over the range it takes away */ @@ -208,6 +215,15 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, loff_t offset, size_t length); void fuse_dlm_unpin(struct fuse_inode *inode); +/* + * fuse_dlm_trypin() for a fill whose reply lands in another task: @pin + * is dropped by node rather than by owner, and is live from the request + * until fuse_dlm_unpin_span(). + */ +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin); + /* * Fence the writers that hold a grant over [@offset, @offset + @len) but * have not dirtied under it yet, for the duration of a revoke. @len <= 0 diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 59dd5ca9378c7c..738264b96d61a5 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1347,6 +1347,15 @@ struct fuse_io_args { struct { struct fuse_read_in in; u64 attr_ver; + /* + * The grant the folios are filled under, held + * from the request until the reply has filled + * them; see fuse_send_readpages(). @dlm_fi is + * the inode to drop it on, and NULL when there + * is no pin to drop. + */ + struct fuse_dlm_span dlm_pin; + struct fuse_inode *dlm_fi; } read; struct { struct fuse_write_in in; From 7a62915095011bfb7a7c9e03a1b84ca5eb034c50 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 4 Sep 2026 23:49:40 +0200 Subject: [PATCH 03/16] fuse: ask for the read grant before the folios are locked A read grant may not be requested under a page lock, so every buffered read path asks for one before it enters the page cache: buffered read, splice read and read fault. The window covers what readahead may add beyond the read, bounded by the file. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 70 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 792a92b9f2fe51..73ffe304b28861 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1134,6 +1134,46 @@ static int fuse_read_folio_merge(struct file *file, struct folio *folio) return 0; } +/** + * fuse_read_grant - take the grant a page cache fill runs under + * @file: file to read through + * @pos: byte offset the read starts at + * @count: bytes the read asks for + * + * ->read_folio and ->readahead are entered with the folios they fill + * already locked, and no grant may be asked for under a page lock + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * A read asks here, before anything is locked, and the fill paths only + * confirm what this took. + * + * Readahead fills past the end of the read, so ask for a window beyond + * it as well, bounded by the file since readahead stops there. A run of + * folios no grant covers is given back unfilled and fetched one folio at + * a time, so what is asked for here is what readahead is worth. + * + * Return: what fuse_get_dlm_lock() returned, 0 when there is nothing to + * ask for. + */ +static int fuse_read_grant(struct file *file, loff_t pos, size_t count) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + loff_t size = i_size_read(inode); + loff_t ahead = (loff_t)file->f_ra.ra_pages << PAGE_SHIFT; + loff_t end = pos + count; + + if (!fc->writeback_cache || !fc->dlm) + return 0; + + if (end < size) + end += min(ahead, size - end); + + if (end <= pos) + return 0; + + return fuse_get_dlm_lock(file, pos, end - pos, FUSE_PAGE_LOCK_READ); +} + /** * fuse_read_folio_retry - back off a fill with no grant to run under * @file: file to read through @@ -1163,7 +1203,7 @@ static int fuse_read_folio_retry(struct file *file, struct folio *folio, fuse_dlm_pin(fi, &pin, pos, len); fuse_dlm_unpin(fi); - err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_READ); + err = fuse_read_grant(file, pos, len); if (err == -ENOSYS) return AOP_TRUNCATED_PAGE; if (err < 0) @@ -1582,11 +1622,8 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } - /* if we have dlm support acquire a read lock for the area - * we are reading from. */ - if (fc->writeback_cache && fc->dlm) - fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + /* The grant this read and the readahead behind it fill under */ + fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); /* * A NOTIFY invalidate racing this read drops the folios it @@ -3092,8 +3129,10 @@ static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); - else - return filemap_splice_read(in, ppos, pipe, len, flags); + + fuse_read_grant(in, *ppos, len); + + return filemap_splice_read(in, ppos, pipe, len, flags); } static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, @@ -4000,9 +4039,22 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) return VM_FAULT_LOCKED; } +/* + * A read fault fills the page cache through ->read_folio and + * ->readahead, which run with the folios locked. Ask for the grant they + * fill under before filemap_fault() locks any of them. + */ +static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) +{ + fuse_read_grant(vmf->vma->vm_file, (loff_t)vmf->pgoff << PAGE_SHIFT, + PAGE_SIZE); + + return filemap_fault(vmf); +} + static const struct vm_operations_struct fuse_file_vm_ops = { .close = fuse_vma_close, - .fault = filemap_fault, + .fault = fuse_filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = fuse_page_mkwrite, }; From 3cde3f386d2167755343daea7c695e1269ad048a Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 4 Sep 2026 23:50:13 +0200 Subject: [PATCH 04/16] fuse: stop asking for the readahead grant under the folio locks ->readahead runs with every folio of the window locked, so the grant request it sent went out under those locks and a revoke of the window had to be given up on. The read now takes the grant before the page cache is entered; fill only what it covers. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 52 ++++++++++++-------------------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 73ffe304b28861..8d70fa5b7df519 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1427,8 +1427,8 @@ static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file, WARN_ON((loff_t) (pos + count) < 0); /* - * The grant fuse_readahead() took, confirmed under a pin and held - * until the reply has filled the folios. A revoke of the range + * The grant the read took in fuse_read_grant(), confirmed under a + * pin and held until the reply has filled the folios. A revoke of the range * waits for that, so the reply cannot be fetched under a grant the * server has since handed on, and cannot land behind a sweep that * would leave the folios uptodate and covered by nothing. @@ -1484,49 +1484,21 @@ static void fuse_readahead(struct readahead_control *rac) return; /* - * Readahead fills the page cache past the range the reader locked, - * so take a DLM read grant over the whole window here too. Folios + * Readahead fills the page cache past the range the reader asked + * for, and what lands there has to be covered by a grant: folios * the server handed out no lock for are folios it will not revoke * when a remote node writes them, and a later read would be served - * from stale cache. Take the grant before any folio is pulled off - * @rac, so the window is either fully covered or not populated. + * from stale cache. * - * Speculative pages are not worth serving uncovered: on a failed - * request drop the window and let read_pages() clean up the folios - * left in @rac. A server without DLM support answers -ENOSYS and - * clears fc->dlm, which is not a failure. + * No grant is asked for here. ->readahead is entered with every + * folio of the window already locked, and none may be asked for + * under a page lock; the read this window belongs to took one over + * it in fuse_read_grant(), before the page cache was entered. * - * The grant is only asked for here. Confirming it and holding it - * against a revoke is fuse_send_readpages(), one run of folios at a - * time, since that is where the request the reply fills them from - * goes out; a run it declines ends the window. - * - * ->readahead is entered with every folio of the window already - * locked, and pulling one off @rac is what unlocks it, so the round - * trip is taken under those locks. See the readahead line of - * Documentation/filesystems/locking.rst. - * - * What keeps that from closing a cycle is the direction a revoke - * travels. This asks for a read grant on a range it holds nothing - * on, so the lock in the way is another node's, and the revoke that - * frees it is sent there. Nothing here has to run for this request - * to be answered, so the folios stay locked only for as long as the - * round trip. - * - * A window this already holds part of is the open edge: the query - * fails for the whole of it, so the request goes out while that part - * is still held, and a revoke for that part is sent here. Whether - * the server can answer while a revoke of its own is outstanding is - * not something this side can know. + * What that left uncovered fuse_send_readpages() declines, one run + * of folios at a time, and those folios go back unfilled for + * fuse_read_folio() to fetch with no folio held. */ - if (fc->writeback_cache && fc->dlm) { - int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), - readahead_length(rac), - FUSE_PAGE_LOCK_READ); - - if (err < 0 && err != -ENOSYS) - return; - } max_pages = min_t(unsigned int, fc->max_pages, fc->max_read / PAGE_SIZE); From 3bae266ef944284904270677232ae9030a74aa88 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 4 Sep 2026 23:52:17 +0200 Subject: [PATCH 05/16] fuse: ask for the read grant on a readahead advice POSIX_FADV_WILLNEED and readahead(2) fill through ->readahead, which now fills only what a grant already covers. Take the grant for the advised range first, with no folio held. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 8d70fa5b7df519..5a56bd53af5e81 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -4847,6 +4848,21 @@ static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off, return ret; } +/* + * POSIX_FADV_WILLNEED, and readahead(2) with it, populate the page cache + * through ->readahead, which runs with the folios locked. Ask for the + * grant that fill needs while nothing is held; a window no grant covers + * is given back unfilled. + */ +static int fuse_fadvise(struct file *file, loff_t offset, loff_t len, + int advice) +{ + if (advice == POSIX_FADV_WILLNEED && offset >= 0 && len > 0) + fuse_read_grant(file, offset, len); + + return generic_fadvise(file, offset, len, advice); +} + static const struct file_operations fuse_file_operations = { .llseek = fuse_file_llseek, .read_iter = fuse_file_read_iter, @@ -4866,6 +4882,7 @@ static const struct file_operations fuse_file_operations = { .poll = fuse_file_poll, .fallocate = fuse_file_fallocate, .copy_file_range = fuse_copy_file_range, + .fadvise = fuse_fadvise, }; static const struct address_space_operations fuse_file_aops = { From bb319ea7701559f34dd76a4d098337ce9e4e42d7 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 00:00:18 +0200 Subject: [PATCH 06/16] fuse: pin a streamed write across the request A streamed write goes to the server out of the caller's pages, so its bytes are in no page cache and a revoke of the range finds nothing to flush. Hold the grant across the FUSE_WRITE, as the writethrough edges do. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5a56bd53af5e81..651f93197477ce 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2614,6 +2614,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) false)); } else if (through) { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + struct fuse_dlm_span pin; loff_t pos = iocb->ki_pos; /* @@ -2630,8 +2631,20 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } + /* + * These bytes never enter the page cache, so a revoke + * cannot find them by flushing it, and the grant they were + * taken under can be handed on while they are still on the + * wire. Hold it across the FUSE_WRITE, as the writethrough + * edges do; a revoke of the range waits for the reply. + */ + err = fuse_dlm_pin_write(file, &pin, pos, count); + if (err) + goto out; + written = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); + fuse_dlm_unpin(fi); if (written < 0) { err = written; goto out; From 6f2148632e5f23b9121bb32e548d7e27735ba8c7 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 00:04:09 +0200 Subject: [PATCH 07/16] fuse: take no dlm lock for O_DIRECT An O_DIRECT read or write neither fills nor dirties the page cache, so a grant over its range covers nothing and only conflicts with the rest of the cluster. Skip it on both sides. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 651f93197477ce..91e7e98ff0d634 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1595,8 +1595,13 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } - /* The grant this read and the readahead behind it fill under */ - fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); + /* + * The grant this read and the readahead behind it fill under. Not + * for O_DIRECT, which takes no DLM lock at all: it fills no page + * cache, and what it reads is the server's to order. + */ + if (!(iocb->ki_flags & IOCB_DIRECT)) + fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); /* * A NOTIFY invalidate racing this read drops the folios it @@ -2528,8 +2533,13 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * too early, an expanding write would fall through to a READ of a * range that cannot hold data -- which fails outright on a handle * the client opened write-only. + * + * O_DIRECT takes no DLM lock at all, here or anywhere: it dirties + * no page cache, so there is nothing for a grant to cover, and its + * bytes are the server's to order against the rest of the cluster. */ - if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + !(iocb->ki_flags & IOCB_APPEND)) { err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, iov_iter_count(from)); if (err) @@ -2553,7 +2563,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * lock does not pin i_size either: attribute replies move it under * fi->lock alone. Take the grant here, where the range is settled. */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + (iocb->ki_flags & IOCB_APPEND)) { err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, count); if (err) goto out; From 8d6ace4d01f4bc9ea0f2fc9445256e7aa8fd0799 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 5 Sep 2026 23:25:25 +0200 Subject: [PATCH 08/16] fuse: keep the cache when an attribute reply shrinks the size fuse_attr_cache_mask() decides whether the server's smaller size may be taken, then sleeps in the grant query before its answer is used. A write below EOF extends nothing, so it bumps neither fi->attr_version, which would drop the reply, nor fi->size_extenders, which would hold the size. Folios dirtied in the doomed range during that window are invisible to all three legs. Re-testing closer to the truncate is not enough on its own. truncate_pagecache() runs after fi->lock is dropped and holds nothing a writer holds, so a folio dirtied between the decision and the walk, or during the walk, is discarded anyway, with no error to report it: the bytes are gone and a later fsync() succeeds over the hole. A real truncate may discard the cache because fuse_set_nowrite() and i_rwsem hold the writers off; an attribute reply holds off nothing. So keep the folios, and re-test the size under fi->lock where nothing sleeps between the answer and its use. If the size was wrong the folios are written back and the size recovers, and fuse_flush_writepages() crops against a high water mark so nothing is clipped meanwhile. If it was right, the revoke that had to precede it already laundered and dropped the range, so there is nothing left to discard. Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index a58c1524617899..91642c2aad3a64 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -631,6 +631,30 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr evict_ctr); oldsize = inode->i_size; + /* + * fuse_attr_cache_mask() answered before the grant query slept, and a + * write below EOF bumps neither fi->attr_version nor + * fi->size_extenders: it extends nothing, so the version check cannot + * drop the reply and the count cannot hold the size. Folios can have + * been dirtied in the doomed range since the answer, and + * truncate_pagecache() below throws them away with no error to report + * it. + * + * Re-test here instead, where nothing sleeps between the answer and + * the truncate acting on it, and keep the local size when the range + * still holds bytes the server has not seen. A remote truncate is + * unaffected: it revokes first, and the revoke launders and drops the + * range, so there is nothing here to find. + */ + if (have_size && !(cache_mask & STATX_SIZE) && fc->dlm && + fc->writeback_cache && S_ISREG(inode->i_mode) && + (loff_t) attr->size < oldsize && + filemap_range_needs_writeback(inode->i_mapping, attr->size, + oldsize - 1)) { + cache_mask |= STATX_SIZE; + attr->size = oldsize; + } + /* * In case of writeback_cache enabled, the cached writes beyond EOF * extend local i_size without keeping userspace server in sync. So, @@ -656,7 +680,23 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr bool have_mtime = !sx || (sx->mask & STATX_MTIME); if (have_size && oldsize != attr->size) { - truncate_pagecache(inode, attr->size); + /* + * Not under DLM. This runs after fi->lock is + * dropped and takes nothing a writer holds, so a + * folio dirtied between the decision and the walk, + * or during it, is discarded with no error to report + * it. A real truncate may do this because + * fuse_set_nowrite() and i_rwsem hold the writers + * off; an attribute reply holds off nothing. + * + * Keeping the folios costs nothing either way. If + * the size was wrong they are written back and the + * size recovers; if it was right, the revoke that had + * to precede it already dropped the range and there + * is nothing here to discard. + */ + if (!(fc->dlm && fc->writeback_cache)) + truncate_pagecache(inode, attr->size); if (!fc->explicit_inval_data) inval = true; } else if (have_mtime && fc->auto_inval_data) { From ccfe8c4e3f06e8f8630200fcc21ca1ceae9c37c3 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sun, 6 Sep 2026 00:01:39 +0200 Subject: [PATCH 09/16] fuse: bound the alignment check by the mapping when cyclic fuse_writepage_need_send() takes its alignment bound from wbc->range_end, which a cyclic writeback does not set: write_cache_pages() runs to the end of the mapping there and leaves the field at zero. The bound is then zero as well, the test is true for every aligned index, and each folio is sent as a request of its own. Run to the end of the mapping when the pass is cyclic. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 91e7e98ff0d634..831bbec7c2aadd 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3535,7 +3535,15 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, pgoff_t page_index = pos >> PAGE_SHIFT; if (wbc && !(page_index % fc->alignment_pages)) { - pgoff_t end_page_index = (wbc->range_end + PAGE_SIZE - 1) >> PAGE_SHIFT; + /* + * A cyclic pass leaves range_end at zero and runs to + * the end of the mapping instead, so reading it there + * makes every aligned index its own request. + */ + pgoff_t end_page_index = wbc->range_cyclic ? + (pgoff_t) -1 : + (pgoff_t) ((wbc->range_end + PAGE_SIZE - 1) >> + PAGE_SHIFT); /* we are at a point where we would write aligned * check if we potentially could reach the next alignment */ From 0e2b8ec264ba938f12241fe3db3de65e31684548 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sun, 6 Sep 2026 00:02:18 +0200 Subject: [PATCH 10/16] fuse: read the write stream marks once fuse_writeback_kick_stream() reads fi->write_stream_start twice in the same decision, once to round down into the start of the range and again to compare against its end. A writer moving the mark in between inverts the range the kick is given. Racing writers are meant to cost a kick rather than correctness, which needs each mark read once into a local. Take the end from this write's own position rather than reading back the mark just stored, and annotate the accesses. Still no lock: the run is a hint, and nothing waits for it. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 831bbec7c2aadd..f1d55ec783176f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2404,7 +2404,10 @@ static bool fuse_write_stream_update(struct fuse_inode *fi, size_t len) * A hint only: nothing waits for it, and a chunk that is partly dirty or * already written back sends what it has. The run is read and written * without the inode lock, which the DLM path holds shared, so writers - * landing on it together cost a kick, not correctness. + * landing on it together cost a kick, not correctness. Each mark is read + * once into a local for that to hold: reading write_stream_start twice, + * to round down and again to compare, lets a writer moving it in between + * invert the range the kick is given. */ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, size_t len, bool stream) @@ -2412,25 +2415,27 @@ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct fuse_inode *fi = get_fuse_inode(inode); - loff_t chunk, start, end; + loff_t chunk, run, start, end; bool sequential; - sequential = pos == fi->write_stream_next; - fi->write_stream_next = pos + (loff_t)len; + sequential = pos == READ_ONCE(fi->write_stream_next); + WRITE_ONCE(fi->write_stream_next, pos + (loff_t)len); if (!sequential || !stream) { /* Nothing behind this write is known to be finished */ - fi->write_stream_start = pos; + WRITE_ONCE(fi->write_stream_start, pos); return; } + run = READ_ONCE(fi->write_stream_start); chunk = fuse_write_chunk_size(get_fuse_conn(inode)); - start = round_down(fi->write_stream_start, chunk); - end = round_down(fi->write_stream_next, chunk); + start = round_down(run, chunk); + /* This write's own end, not the mark another writer may have moved */ + end = round_down(pos + (loff_t)len, chunk); /* No bound crossed, so nothing has been left complete */ - if (end <= fi->write_stream_start) + if (end <= run) return; - fi->write_stream_start = end; + WRITE_ONCE(fi->write_stream_start, end); filemap_fdatawrite_range_kick(file->f_mapping, start, end - 1); } From 8230fe55eb3c2d6cc459278580fe229e40c37500 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 7 Sep 2026 18:06:36 +0200 Subject: [PATCH 11/16] fuse: latch readers into direct IO on an invalidation storm The notify-driven latch required a local writer. Under the same storm a reader-only inode refills the page cache between two invalidations and has it dropped again before it can be read twice, so latch on any local opener and let the whole-mapping drop free the folios. The average behind the latch only folds on arrival, so an inode that stops being notified would stay uncached. Clear the latch at the top of the IO paths once the last invalidation is FUSE_NOTIFY_DIO_COLD old, and on release only for a writer. Keep the mapping empty while latched: splice reads copy through ->read_iter, readahead declines. The whole-mapping drop launders only what the mapping says may be dirty. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 72 +++++++++++++++++++++++++++++++++++++++++++++--- fs/fuse/fuse_i.h | 21 ++++++++------ fs/fuse/inode.c | 52 ++++++++++++++++++++++------------ 3 files changed, 116 insertions(+), 29 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f1d55ec783176f..adcbc8ed4584d4 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -426,15 +426,20 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, /* Inode is NULL on error path of fuse_create_open() */ if (likely(fi)) { + bool writer; + spin_lock(&fi->lock); + writer = !list_empty(&ff->write_entry); list_del(&ff->write_entry); /* * Leave forced direct IO mode once the last writer is gone: with * no local writer left there is no cached-write contention with * the remote modifier that triggered the switch. Restore - * FUSE_I_CACHE_IO_MODE for any frozen cached opens. + * FUSE_I_CACHE_IO_MODE for any frozen cached opens. A reader + * closing ends nothing: an inode latched with no writer at all + * leaves on the cold check in fuse_force_dio_active(). */ - if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + if (writer && test_bit(FUSE_I_FORCE_DIO, &fi->state) && list_empty(&fi->write_files)) { clear_bit(FUSE_I_FORCE_DIO, &fi->state); if (fi->iocachectr > 0) @@ -1484,6 +1489,14 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; + /* + * A latched inode keeps no page cache, and this window was not asked + * for by a read that has to be served (madvise(), fadvise()). Decline + * it: read_pages() drops the folios of a window left unfilled. + */ + if (fuse_inode_force_dio(inode)) + return; + /* * Readahead fills the page cache past the range the reader asked * for, and what lands there has to be covered by a grant: folios @@ -3080,6 +3093,50 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) return res; } +/* + * Whether the inode is still latched into direct IO. The latch is set on a + * stream of invalidation notifies and has to come off when the stream stops, + * or an inode that was hot once stays uncached for as long as it is open. The + * average behind the latch is folded on arrival and cannot age on its own, so + * the age of the last notify is the clock. + * + * Cleared here the way the mmap revert clears it: no inode lock, none is held + * at the top of the IO paths, and the drop is server-free because everything + * cached under the latch is clean. + */ +static bool fuse_force_dio_active(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + bool cleared = false; + + if (!fuse_inode_force_dio(inode)) + return false; + + if (!time_after(jiffies, + READ_ONCE(fi->notify_stamp) + FUSE_NOTIFY_DIO_COLD)) + return true; + + spin_lock(&fi->lock); + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + time_after(jiffies, fi->notify_stamp + FUSE_NOTIFY_DIO_COLD)) { + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + cleared = true; + } + spin_unlock(&fi->lock); + + /* + * Whatever a read racing the latch left behind, so it cannot be served + * once caching resumes. Another caller may have cleared the latch + * first, or set it again since; the bit decides, not this one's work. + */ + if (cleared) + invalidate_inode_pages2(inode->i_mapping); + + return fuse_inode_force_dio(inode); +} + static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; @@ -3093,7 +3150,7 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return fuse_dax_read_iter(iocb, to); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) return fuse_direct_read_iter(iocb, to); else if (fuse_file_passthrough(ff)) return fuse_passthrough_read_iter(iocb, to); @@ -3114,7 +3171,7 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_dax_write_iter(iocb, from); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) return fuse_direct_write_iter(iocb, from); else if (fuse_file_passthrough(ff)) return fuse_passthrough_write_iter(iocb, from); @@ -3132,6 +3189,13 @@ static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); + /* + * Latched: copy through ->read_iter, which reads direct, rather than + * fill a page cache this inode is not allowed to keep. + */ + if (fuse_force_dio_active(file_inode(in))) + return copy_splice_read(in, ppos, pipe, len, flags); + fuse_read_grant(in, *ppos, len); return filemap_splice_read(in, ppos, pipe, len, flags); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 738264b96d61a5..27e9f1808059c4 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -187,15 +187,18 @@ struct dlm_locked_area * Force-DIO switch trigger: an exponentially weighted moving average of the * interval (in jiffies) between FUSE_NOTIFY_INVAL_INODE data invalidations for * a file. When the average spacing falls below FUSE_NOTIFY_DIO_INTERVAL -- a - * remote writer streaming invalidations -- and the file is open for writing - * here, it is latched into direct IO. These are the source-level (not - * externally tunable) parameters of the heuristic: EWMA weight 1/2^SHIFT, - * seeded and capped at SEED so it takes a short burst rather than a single - * notify to trip. + * remote writer streaming invalidations -- and the file is open here, it is + * latched into direct IO. These are the source-level (not externally tunable) + * parameters of the heuristic: EWMA weight 1/2^SHIFT, seeded and capped at SEED + * so it takes a short burst rather than a single notify to trip. + * + * The average is folded on arrival and cannot age on its own, so what takes the + * latch off again is the last invalidation reaching FUSE_NOTIFY_DIO_COLD old. */ #define FUSE_NOTIFY_DIO_INTERVAL max_t(unsigned long, HZ / 10, 1) #define FUSE_NOTIFY_EWMA_SHIFT 2 #define FUSE_NOTIFY_EWMA_SEED (2 * FUSE_NOTIFY_DIO_INTERVAL) +#define FUSE_NOTIFY_DIO_COLD (8 * FUSE_NOTIFY_DIO_INTERVAL) /* * Streamed file trigger: the same buffer size arriving over and over is a @@ -287,9 +290,11 @@ struct fuse_inode { * the last one, notify_interval_ewma the EWMA of the * inter-arrival interval (jiffies, scaled by * 2^FUSE_NOTIFY_EWMA_SHIFT). A rapid stream (short - * average interval) with a local writer latches the - * inode into direct IO. Protected by fi->lock; regular - * files only (shares the readdir-cache union arm). + * average interval) with the file open here latches + * the inode into direct IO, and notify_stamp going + * stale takes it out again. Protected by fi->lock; + * regular files only (shares the readdir-cache union + * arm). */ unsigned long notify_stamp; unsigned int notify_interval_ewma; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 91642c2aad3a64..341687211723ee 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -910,6 +910,15 @@ static void fuse_invalidate_inode_entry(struct inode *inode) } } +/* + * Someone here has the file open and would use its page cache: a writeback + * writer, or any open in caching mode. Must be called under fi->lock. + */ +static bool fuse_inode_has_opener(struct fuse_inode *fi) +{ + return !list_empty(&fi->write_files) || fi->iocachectr > 0; +} + /* * Fold one FUSE_NOTIFY_INVAL_INODE data invalidation into the per-inode * moving average of the notification inter-arrival interval and report whether @@ -918,8 +927,8 @@ static void fuse_invalidate_inode_entry(struct inode *inode) * average is an EWMA (weight 1/2^FUSE_NOTIFY_EWMA_SHIFT); the sample is clamped * to FUSE_NOTIFY_EWMA_SEED so a notify after a long idle only cools the average * and cannot overflow the accumulator. Must be called under fi->lock; called - * for every data invalidation so the average stays current even while no local - * writer is open. + * for every data invalidation so the average stays current even while the file + * is not open here. */ static bool fuse_notify_inval_hot(struct fuse_inode *fi) { @@ -1118,8 +1127,12 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * under fi->lock, updated for every data invalidation) of * how fast these arrive; when they come in a rapid stream * -- a remote writer repeatedly invalidating -- and the - * inode is also open for writing here, latch it into - * direct IO until the last writer closes or it is mmapped. + * inode is open here, latch it into direct IO until the + * stream stops, the last writer closes, or it is mmapped. + * A reader-only inode is latched too: what it caches + * between two invalidations is dropped again before it can + * be read twice, so the cache costs the folios and the read + * grants behind them and returns nothing. * When latched, drop the whole mapping rather than just * the notified range, or dirty folios outside it would be * invisible to the forced direct reads (stale read / lost @@ -1128,8 +1141,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * up to date either way, so enabling it at runtime takes * effect on the next storm rather than after a warm-up. * Clearing it at runtime stops new latches but lets - * already-latched inodes run out on the usual exits (last - * writer closes, or mmap). + * already-latched inodes run out on the usual exits. * * The average and the latch exist only for writeback+dlm * regular files; elsewhere there is no record to consult and @@ -1142,12 +1154,12 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, !fuse_inode_backing(fi); if (tracked) { - bool hot, has_writer, latched = false; + bool hot, has_opener, latched = false; bool may_be_dirty, has_pages; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); - has_writer = !list_empty(&fi->write_files); + has_opener = fuse_inode_has_opener(fi); spin_unlock(&fi->lock); /* @@ -1193,11 +1205,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_dlm_revoke_inval_range(fi, offset, len); - if (enable_notify_dio && hot && has_writer && + if (enable_notify_dio && hot && has_opener && !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); - if (!list_empty(&fi->write_files)) { + if (fuse_inode_has_opener(fi)) { set_bit(FUSE_I_FORCE_DIO, &fi->state); latched = true; } @@ -1207,17 +1219,23 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* * Latched: drop the whole mapping (dirty folios * outside the notified range would be invisible to - * the forced direct reads), and the record says - * nothing about the rest of the file, so launder. - * Otherwise just the notified range, and only if - * anything is cached there. + * the forced direct reads), laundering only what the + * mapping says may be dirty, which for an inode + * latched with no writer is nothing. Otherwise just + * the notified range, and only if anything is cached + * there. */ - if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1, true); - else if (has_pages) + if (fuse_inode_force_dio(inode)) { + bool dirty; + + dirty = filemap_range_needs_writeback( + inode->i_mapping, 0, LLONG_MAX); + fuse_notify_invalidate_range(inode, 0, -1, dirty); + } else if (has_pages) { fuse_notify_invalidate_range(inode, pg_start, pg_end, may_be_dirty); + } if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", From ad450ce6e4ec9bde704549f226994b9b132a6056 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 7 Sep 2026 22:19:16 +0200 Subject: [PATCH 12/16] fuse: read a streamed file from the server A read of the same buffer size arriving over and over is a reader working through the file a record at a time, and the folios it fills are dropped unread. Serve a streamed read of FUSE_READ_STREAM_MIN or more into the caller's own pages: one copy instead of two, no folios, and no read grant. What that gives up is the readahead of the next record, not the wait for this one. The detector is the one the write side uses, taking the field pair now so reads keep their own average, and the size is folded for every read that could be cached. Dirty folios over the range are sent first: a direct read does not look in the page cache, and fuse_direct_io() flushes only for a file opened FOPEN_DIRECT_IO. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 135 ++++++++++++++++++++++++++++++++--------------- fs/fuse/fuse_i.h | 25 +++++---- 2 files changed, 107 insertions(+), 53 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index adcbc8ed4584d4..2b3e1452ab20c4 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1588,11 +1588,50 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); +/* + * Fold one request size into a moving average of this inode's sizes and + * report whether the file is being streamed: the same buffer size arriving + * FUSE_STREAM_RUN times over, which is what a task working through a file a + * record at a time looks like from here. A size outside the tolerance around + * the average starts the run again from that size, so a task changing its + * record is followed rather than averaged with what it did before. + * + * The average is per inode rather than per handle, so a stream stays one + * stream across reopens and across the handles of a shared file, whose users + * are streaming it together without any one of them being sequential. + * + * A hint only, read and written without the inode lock, which the DLM path + * holds shared: callers landing on it together cost a misread run, not + * correctness. + */ +static bool fuse_stream_update(unsigned int *ewma, unsigned int *run, + size_t len) +{ + unsigned int sample = min_t(size_t, len, FUSE_STREAM_EWMA_MAX); + unsigned int avg = *ewma >> FUSE_STREAM_EWMA_SHIFT; + + if (*run && abs_diff(sample, avg) <= avg >> FUSE_STREAM_TOL_SHIFT) { + /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ + *ewma += sample - avg; + if (*run < FUSE_STREAM_RUN) + (*run)++; + } else { + *ewma = sample << FUSE_STREAM_EWMA_SHIFT; + *run = 1; + } + + return *run >= FUSE_STREAM_RUN; +} + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; - struct inode *inode = file->f_mapping->host; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + size_t count = iov_iter_count(to); + bool stream = false; ssize_t res; /* @@ -1601,20 +1640,60 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * i_size is up to date). */ if (fc->auto_inval_data || - (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { + (iocb->ki_pos + count > i_size_read(inode))) { int err; err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE); if (err) return err; } + /* + * Every read that could be cached feeds the size average, streamed or + * not: a reader changing its record has to be seen as well. O_DIRECT + * is not one of them, and an empty read says nothing about a record. + */ + if (count && !(iocb->ki_flags & IOCB_DIRECT)) + stream = fuse_stream_update(&fi->read_size_ewma, + &fi->read_stream_run, count); + + /* + * A streamed read of FUSE_READ_STREAM_MIN or more is served into the + * caller's own pages instead. Cached, the bytes are copied twice on + * their way out of the server, into the folios and out of them into + * the caller, and the folios are dropped unread; served from here they + * are copied once, and the grant the fill would have run under is not + * asked for at all. What that gives up is the readahead of the next + * record, not the wait for this one. + * + * Not on a mapped file, which keeps its page cache either way, and not + * for IOCB_NOWAIT, which the direct read has no way to honour. A + * caller that gets EAGAIN out of the cached path here comes back + * without the flag and takes this branch on the retry. + * + * Dirty folios over the range have to reach the server first: a direct + * read does not look in the page cache, and fuse_direct_io() flushes + * only for a file opened FOPEN_DIRECT_IO. + */ + if (stream && count >= FUSE_READ_STREAM_MIN && + !(iocb->ki_flags & IOCB_NOWAIT) && !mapping_mapped(mapping)) { + loff_t end = iocb->ki_pos + count - 1; + + if (filemap_range_needs_writeback(mapping, iocb->ki_pos, end)) { + res = filemap_write_and_wait_range(mapping, iocb->ki_pos, + end); + if (res) + return res; + } + return fuse_direct_read_iter(iocb, to); + } + /* * The grant this read and the readahead behind it fill under. Not * for O_DIRECT, which takes no DLM lock at all: it fills no page * cache, and what it reads is the server's to order. */ if (!(iocb->ki_flags & IOCB_DIRECT)) - fuse_read_grant(file, iocb->ki_pos, iov_iter_count(to)); + fuse_read_grant(file, iocb->ki_pos, count); /* * A NOTIFY invalidate racing this read drops the folios it @@ -1626,16 +1705,16 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * grant taken here is what they confirm. */ if (fuse_inode_force_dio(inode)) { - size_t count = iov_iter_count(to); - /* * A write that passed this same check just before the latch * took hold dirtied the page cache after the notify dropped * it, and a direct read does not look there. Send it first. */ if (count) { - res = filemap_write_and_wait_range(inode->i_mapping, - iocb->ki_pos, iocb->ki_pos + count - 1); + loff_t end = iocb->ki_pos + count - 1; + + res = filemap_write_and_wait_range(mapping, + iocb->ki_pos, end); if (res) return res; } @@ -2363,42 +2442,6 @@ static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) return (err < 0 && err != -ENOSYS) ? err : 0; } -/* - * Fold one buffered write size into the moving average of this inode's write - * sizes and report whether the file is being streamed: the same buffer size - * arriving FUSE_WRITE_STREAM_RUN times over, which is what a writer working - * through a file a record at a time looks like from here. A size outside the - * tolerance around the average starts the run again from that size, so a - * writer changing its record is followed rather than averaged with what it - * did before. - * - * The average is per inode rather than per handle, so a stream stays one - * stream across reopens and across the handles of a shared file, whose - * writers are streaming it together without any one of them being sequential. - * - * A hint only, read and written without the inode lock, which the DLM path - * holds shared: writers landing on it together cost a misread run, not - * correctness. - */ -static bool fuse_write_stream_update(struct fuse_inode *fi, size_t len) -{ - unsigned int sample = min_t(size_t, len, FUSE_WRITE_EWMA_MAX); - unsigned int avg = fi->write_size_ewma >> FUSE_WRITE_EWMA_SHIFT; - - if (fi->write_stream_run && - abs_diff(sample, avg) <= avg >> FUSE_WRITE_TOL_SHIFT) { - /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ - fi->write_size_ewma += sample - avg; - if (fi->write_stream_run < FUSE_WRITE_STREAM_RUN) - fi->write_stream_run++; - } else { - fi->write_size_ewma = sample << FUSE_WRITE_EWMA_SHIFT; - fi->write_stream_run = 1; - } - - return fi->write_stream_run >= FUSE_WRITE_STREAM_RUN; -} - /* * Start non-integrity writeback on the aligned chunks a streamed file has * left behind. @@ -2517,7 +2560,9 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * with. */ if (writeback && !(iocb->ki_flags & IOCB_DIRECT)) - stream = fuse_write_stream_update(fi, iov_iter_count(from)); + stream = fuse_stream_update(&fi->write_size_ewma, + &fi->write_stream_run, + iov_iter_count(from)); /* * A streamed write of FUSE_WRITE_STREAM_MIN or more is sent from here @@ -5024,6 +5069,8 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->write_stream_run = 0; fi->write_stream_next = 0; fi->write_stream_start = 0; + fi->read_size_ewma = 0; + fi->read_stream_run = 0; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 27e9f1808059c4..3860125560b861 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -202,19 +202,20 @@ struct dlm_locked_area /* * Streamed file trigger: the same buffer size arriving over and over is a - * writer working through a file a record at a time. The sizes are folded + * task working through a file a record at a time. The sizes are folded * into an exponentially weighted moving average (weight 1/2^SHIFT, kept - * shifted), and a run of FUSE_WRITE_STREAM_RUN writes within - * 1/2^FUSE_WRITE_TOL_SHIFT of it says the writer is still on it. The sample + * shifted), and a run of FUSE_STREAM_RUN requests within + * 1/2^FUSE_STREAM_TOL_SHIFT of it says the task is still on it. The sample * is capped to keep the shifted accumulator inside an unsigned int. */ -#define FUSE_WRITE_EWMA_SHIFT 2 -#define FUSE_WRITE_TOL_SHIFT 3 -#define FUSE_WRITE_STREAM_RUN 4 -#define FUSE_WRITE_EWMA_MAX (UINT_MAX >> FUSE_WRITE_EWMA_SHIFT) +#define FUSE_STREAM_EWMA_SHIFT 2 +#define FUSE_STREAM_TOL_SHIFT 3 +#define FUSE_STREAM_RUN 4 +#define FUSE_STREAM_EWMA_MAX (UINT_MAX >> FUSE_STREAM_EWMA_SHIFT) -/* Under this size the copy into the page cache is not worth avoiding */ +/* Under this size the copy through the page cache is not worth avoiding */ #define FUSE_WRITE_STREAM_MIN (64 * 1024) +#define FUSE_READ_STREAM_MIN (10 * PAGE_SIZE) /** FUSE inode */ struct fuse_inode { @@ -323,13 +324,19 @@ struct fuse_inode { * next write has to land to carry the run of * positions on, and write_stream_start the first * byte of that run no writeback kick has covered. + * read_size_ewma and read_stream_run are the same + * average over the reads that could be cached, kept + * apart so a write phase and a read phase over one + * file do not fold into each other. * Hints only, read and written without a lock; see - * fuse_write_stream_update(). + * fuse_stream_update(). */ unsigned int write_size_ewma; unsigned int write_stream_run; loff_t write_stream_next; loff_t write_stream_start; + unsigned int read_size_ewma; + unsigned int read_stream_run; }; /* readdir cache (directory only) */ From 63a52859d0b245108f3e125d2e7c5fc733b74707 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 8 Sep 2026 09:23:00 +0200 Subject: [PATCH 13/16] fuse: do not freeze writepages under a shared inode lock fuse_set_nowrite() biases an inode-wide counter and asserts BUG_ON(fi->writectr < 0) under fi->lock, which holds only with i_rwsem taken exclusive. Parallel direct writes hold it shared, so two of them reaching fuse_sync_writes() in fuse_direct_io() kill the second inside the spinlock, and every later user of the inode spins on it. Wait on the folios of the caller's own range instead when the lock is shared, which is the range the writeback test above already asked about. Reachable from the streamed write, and from any parallel write over a page cache that has not drained once the direct-IO latch is set. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 37 +++++++++++++++++++++++++++++++------ fs/fuse/fuse_i.h | 3 +++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 2b3e1452ab20c4..8dd2b6062fa01c 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2717,7 +2717,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; written = fuse_direct_io(&io, from, &iocb->ki_pos, - FUSE_DIO_WRITE); + FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); fuse_dlm_unpin(fi); if (written < 0) { err = written; @@ -2968,6 +2969,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, { int write = flags & FUSE_DIO_WRITE; int cuse = flags & FUSE_DIO_CUSE; + int shared = flags & FUSE_DIO_SHARED; struct file *file = io->iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; @@ -2996,12 +2998,33 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, return res; } } + + /* + * Wait out the writeback this read or write would otherwise race: + * a reply landing after it would put the superseded bytes on the + * server on top of it. + * + * How to wait depends on the lock the caller holds. fuse_set_nowrite() + * asserts BUG_ON(fi->writectr < 0) and biases a counter the whole inode + * shares, which only an exclusive i_rwsem makes safe; two callers + * holding it shared reach that assertion together and the second one + * dies inside spin_lock(&fi->lock). A parallel direct write holds it + * shared (fuse_dio_lock(), and the streamed write in + * fuse_cache_write_iter()), so it waits on the folios of its own range + * instead, which is the range this test asked about anyway. Errors are + * left on the mapping for fsync to collect. + */ if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) { - if (!write) + if (!write) { inode_lock(inode); - fuse_sync_writes(inode); - if (!write) + fuse_sync_writes(inode); inode_unlock(inode); + } else if (shared) { + filemap_fdatawait_range_keep_errors(mapping, pos, + pos + count - 1); + } else { + fuse_sync_writes(inode); + } } if (fopen_direct_io && write) { @@ -3119,7 +3142,8 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); res = fuse_direct_io(&io, from, &iocb->ki_pos, - FUSE_DIO_WRITE); + FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); fuse_write_update_attr(inode, iocb->ki_pos, res); } if (res > 0 && mapping->nrpages) { @@ -4720,7 +4744,8 @@ __fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bool exclusive) } if (iov_iter_rw(iter) == WRITE) { - ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); + ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); } else { ret = __fuse_direct_read(io, iter, &pos); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 3860125560b861..0f44bf0229d1a3 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1669,6 +1669,9 @@ int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, /** CUSE pass fuse_direct_io() a file which f_mapping->host is not from FUSE */ #define FUSE_DIO_CUSE (1 << 1) +/** Caller holds i_rwsem shared, so fuse_set_nowrite() must not be used */ +#define FUSE_DIO_SHARED (1 << 2) + ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, loff_t *ppos, int flags); long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, From 57efa540a1245cb1402c0904ca4688636c5f71cc Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 8 Sep 2026 09:43:00 +0200 Subject: [PATCH 14/16] fuse: drain the page cache once an inode is latched into direct IO The notify that sets the latch drops the mapping without an inode lock, so a cached write already past both latch checks keeps dirtying behind it. Writeback then puts those folios on the server on top of the direct writes that replace them, and direct reads miss them entirely. The per-range flush in the reroute paths covers only the range it is about to write, and only until the next dirty. Take i_rwsem exclusive once per latch instead. fuse_cache_write_iter() dirties under it, so holding it means every such writer has finished, and one that takes the lock afterwards rechecks the latch and reroutes before touching the cache. Flush and drop the mapping there and record it in FUSE_I_FORCE_DIO_DRAINED, which is cleared with the latch. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 101 +++++++++++++++++++++++++++++++++++------------ fs/fuse/fuse_i.h | 10 ++++- 2 files changed, 84 insertions(+), 27 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 8dd2b6062fa01c..75c37b8e7acecc 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -442,6 +442,7 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, if (writer && test_bit(FUSE_I_FORCE_DIO, &fi->state) && list_empty(&fi->write_files)) { clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); } @@ -1588,6 +1589,54 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); +/* + * Empty the page cache of an inode just latched into direct IO. + * + * The notify that set the latch dropped the mapping, but it holds no inode + * lock, so a cached write that had already passed both latch checks went on + * dirtying folios behind it. Left there, writeback would put them on the + * server on top of the direct writes that replace them, and direct reads, + * which do not look in the page cache, would miss them entirely. + * + * i_rwsem taken exclusive is what settles it: fuse_cache_write_iter() dirties + * under it, so by the time it is held every such writer has finished, and none + * can start behind this one. A write that takes the lock afterwards rechecks + * the latch and reroutes before it touches the cache. So the mapping stays + * empty from here on and the bit records that, leaving the latched IO paths + * with nothing to flush. + * + * Called from the top of the IO paths, with no inode lock held. A no-op for + * a file the server itself opened direct, which is never latched. + */ +static int fuse_force_dio_drain(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + int err = 0; + + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state) || + test_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state)) + return 0; + + inode_lock(inode); + /* + * The latch may have gone while this waited for the lock, and the drain + * is only owed to one that is still on. On a flush error the bit stays + * clear so the next IO tries again, and the error goes to this caller + * rather than being left for a later fsync to find. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + !test_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state)) { + err = filemap_write_and_wait(inode->i_mapping); + if (!err) { + invalidate_inode_pages2(inode->i_mapping); + set_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); + } + } + inode_unlock(inode); + + return err; +} + /* * Fold one request size into a moving average of this inode's sizes and * report whether the file is being streamed: the same buffer size arriving @@ -1706,18 +1755,12 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) */ if (fuse_inode_force_dio(inode)) { /* - * A write that passed this same check just before the latch - * took hold dirtied the page cache after the notify dropped - * it, and a direct read does not look there. Send it first. + * Latched between fuse_file_read_iter()'s check and this one, + * so the drain it would have done falls here. */ - if (count) { - loff_t end = iocb->ki_pos + count - 1; - - res = filemap_write_and_wait_range(mapping, - iocb->ki_pos, end); - if (res) - return res; - } + res = fuse_force_dio_drain(inode); + if (res) + return res; return fuse_direct_read_iter(iocb, to); } @@ -2653,16 +2696,12 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (fuse_inode_force_dio(inode)) { /* - * As on the read side, only worse: the direct write would - * land under whatever a write racing the latch left dirty, - * and the invalidate fuse_direct_write_iter() does after it - * launders rather than drops, putting that folio on the - * server on top. Send it first and the order is ordinary. + * Latched while this write waited for the lock, so the drain + * fuse_file_write_iter() would have done falls here. Drop the + * lock first: the drain takes it exclusive. */ - if (count) - err = filemap_write_and_wait_range(inode->i_mapping, - iocb->ki_pos, iocb->ki_pos + count - 1); fuse_cache_wr_unlock(inode, exclusive); + err = fuse_force_dio_drain(inode); if (err) return err; return fuse_direct_write_iter(iocb, from); @@ -3189,6 +3228,7 @@ static bool fuse_force_dio_active(struct inode *inode) if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && time_after(jiffies, fi->notify_stamp + FUSE_NOTIFY_DIO_COLD)) { clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); cleared = true; @@ -3219,12 +3259,17 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return fuse_dax_read_iter(iocb, to); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) { + ssize_t err = fuse_force_dio_drain(inode); + + if (err) + return err; return fuse_direct_read_iter(iocb, to); - else if (fuse_file_passthrough(ff)) + } else if (fuse_file_passthrough(ff)) { return fuse_passthrough_read_iter(iocb, to); - else + } else { return fuse_cache_read_iter(iocb, to); + } } static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) @@ -3240,12 +3285,17 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_dax_write_iter(iocb, from); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) { + ssize_t err = fuse_force_dio_drain(inode); + + if (err) + return err; return fuse_direct_write_iter(iocb, from); - else if (fuse_file_passthrough(ff)) + } else if (fuse_file_passthrough(ff)) { return fuse_passthrough_write_iter(iocb, from); - else + } else { return fuse_cache_write_iter(iocb, from); + } } static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, @@ -4240,6 +4290,7 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) spin_lock(&fi->lock); clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); spin_unlock(&fi->lock); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 0f44bf0229d1a3..25d239a93c8ad1 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -412,10 +412,16 @@ enum { * Latched into direct IO: a NOTIFY_INVAL_INODE arrived while the file * was open for writing here, so another (remote) entity is modifying it * concurrently. Reads and writes are routed direct (shared-lock - * parallel dio) until the last writer closes or the inode is mmapped. - * See fuse_reverse_inval_inode()/fuse_file_io_open(). + * parallel dio) until the notifies stop, the last writer closes, or the + * inode is mmapped. See fuse_reverse_inval_inode()/fuse_file_io_open(). */ FUSE_I_FORCE_DIO, + /* + * The page cache has been emptied under that latch, so no cached write + * from before it can still be in flight. Set once per latch by + * fuse_force_dio_drain(), cleared wherever FUSE_I_FORCE_DIO is. + */ + FUSE_I_FORCE_DIO_DRAINED, }; struct fuse_conn; From 81942d6263a723897349cbc3478640f8ee1243fd Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 8 Sep 2026 14:53:16 +0200 Subject: [PATCH 15/16] fuse: keep the direct IO latch while a writer is open The cold check cleared the latch on a quiet inode even with a writer still on it, and nothing tracks the forced direct writes in flight, so a cached write could start beside one. Require write_files to be empty as well, which is what the last writer's release already waits for. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 75c37b8e7acecc..b80dab83f01e3c 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3220,12 +3220,20 @@ static bool fuse_force_dio_active(struct inode *inode) if (!fuse_inode_force_dio(inode)) return false; + /* + * Unlocked, and both may change under this: a writer that appears after + * the check leaves the latch cleared, one that goes leaves it on until + * the next IO looks again. Neither is wrong, and the list is re-read + * under fi->lock before anything is cleared. + */ if (!time_after(jiffies, - READ_ONCE(fi->notify_stamp) + FUSE_NOTIFY_DIO_COLD)) + READ_ONCE(fi->notify_stamp) + FUSE_NOTIFY_DIO_COLD) || + !list_empty_careful(&fi->write_files)) return true; spin_lock(&fi->lock); if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + list_empty(&fi->write_files) && time_after(jiffies, fi->notify_stamp + FUSE_NOTIFY_DIO_COLD)) { clear_bit(FUSE_I_FORCE_DIO, &fi->state); clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); From bb8b371c2e304edfde49372c1e5832dea1ca4a26 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 8 Sep 2026 17:58:07 +0200 Subject: [PATCH 16/16] fuse: write back before dropping the cache of an unlatched inode The three paths that leave the forced direct IO latch dropped the mapping without writing it back first, so a folio a write racing the latch left dirty reached the server through invalidate_inode_pages2(), which launders one FUSE_WRITE per page. Send those with writeback, which batches them into max_write requests, and keep the error on the mapping for fsync. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index b80dab83f01e3c..d96c81c652d6f7 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -478,6 +478,27 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, ra->inode = sync ? NULL : igrab(&fi->inode); } +/* + * Drop the page cache of an inode leaving the forced direct IO latch. + * + * Anything still dirty here is what a write racing the latch left behind, since + * the drain empties the mapping and nothing dirties it while it is on. Send + * that with writeback rather than leave it to the drop: invalidate_inode_pages2() + * launders a folio at a time, a FUSE_WRITE per page for bytes writeback batches + * into max_write requests. Errors stay on the mapping for fsync to collect. + * + * The usual case is an empty or clean mapping, where both calls are a load and + * the drop finds nothing to write. + */ +static void fuse_force_dio_drop(struct address_space *mapping) +{ + if (filemap_range_needs_writeback(mapping, 0, LLONG_MAX)) { + filemap_fdatawrite(mapping); + filemap_fdatawait_keep_errors(mapping); + } + invalidate_inode_pages2(mapping); +} + void fuse_file_release(struct inode *inode, struct fuse_file *ff, unsigned int open_flags, fl_owner_t id, bool isdir) { @@ -495,12 +516,11 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * served stale once caching mode resumes. No inode lock: release may * run on the fuse server thread (async fput * from aio completion), where blocking on a contended inode lock could - * stall the connection. Writes were routed direct while latched, so - * only clean folios exist and this invalidate is server-free; the last - * writer is gone, so no forced-dio writer can race the drop. + * stall the connection. The last writer is gone, so no forced-dio writer + * can race the drop. */ if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state)) - invalidate_inode_pages2(inode->i_mapping); + fuse_force_dio_drop(inode->i_mapping); if (ra && ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; @@ -3249,7 +3269,7 @@ static bool fuse_force_dio_active(struct inode *inode) * first, or set it again since; the bit decides, not this one's work. */ if (cleared) - invalidate_inode_pages2(inode->i_mapping); + fuse_force_dio_drop(inode->i_mapping); return fuse_inode_force_dio(inode); } @@ -4302,7 +4322,7 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); spin_unlock(&fi->lock); - invalidate_inode_pages2(file->f_mapping); + fuse_force_dio_drop(file->f_mapping); } /*