diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 31d78bf542b69f..d96c81c652d6f7 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -425,17 +426,23 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, /* Inode is NULL on error path of fuse_create_open() */ if (likely(fi)) { + bool writer; + spin_lock(&fi->lock); + writer = !list_empty(&ff->write_entry); list_del(&ff->write_entry); /* * Leave forced direct IO mode once the last writer is gone: with * no local writer left there is no cached-write contention with * the remote modifier that triggered the switch. Restore - * FUSE_I_CACHE_IO_MODE for any frozen cached opens. + * FUSE_I_CACHE_IO_MODE for any frozen cached opens. A reader + * closing ends nothing: an inode latched with no writer at all + * leaves on the cold check in fuse_force_dio_active(). */ - if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + if (writer && test_bit(FUSE_I_FORCE_DIO, &fi->state) && list_empty(&fi->write_files)) { clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); } @@ -471,6 +478,27 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, ra->inode = sync ? NULL : igrab(&fi->inode); } +/* + * Drop the page cache of an inode leaving the forced direct IO latch. + * + * Anything still dirty here is what a write racing the latch left behind, since + * the drain empties the mapping and nothing dirties it while it is on. Send + * that with writeback rather than leave it to the drop: invalidate_inode_pages2() + * launders a folio at a time, a FUSE_WRITE per page for bytes writeback batches + * into max_write requests. Errors stay on the mapping for fsync to collect. + * + * The usual case is an empty or clean mapping, where both calls are a load and + * the drop finds nothing to write. + */ +static void fuse_force_dio_drop(struct address_space *mapping) +{ + if (filemap_range_needs_writeback(mapping, 0, LLONG_MAX)) { + filemap_fdatawrite(mapping); + filemap_fdatawait_keep_errors(mapping); + } + invalidate_inode_pages2(mapping); +} + void fuse_file_release(struct inode *inode, struct fuse_file *ff, unsigned int open_flags, fl_owner_t id, bool isdir) { @@ -488,12 +516,11 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * served stale once caching mode resumes. No inode lock: release may * run on the fuse server thread (async fput * from aio completion), where blocking on a contended inode lock could - * stall the connection. Writes were routed direct while latched, so - * only clean folios exist and this invalidate is server-free; the last - * writer is gone, so no forced-dio writer can race the drop. + * stall the connection. The last writer is gone, so no forced-dio writer + * can race the drop. */ if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state)) - invalidate_inode_pages2(inode->i_mapping); + fuse_force_dio_drop(inode->i_mapping); if (ra && ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; @@ -1134,10 +1161,99 @@ static int fuse_read_folio_merge(struct file *file, struct folio *folio) return 0; } +/** + * fuse_read_grant - take the grant a page cache fill runs under + * @file: file to read through + * @pos: byte offset the read starts at + * @count: bytes the read asks for + * + * ->read_folio and ->readahead are entered with the folios they fill + * already locked, and no grant may be asked for under a page lock + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * A read asks here, before anything is locked, and the fill paths only + * confirm what this took. + * + * Readahead fills past the end of the read, so ask for a window beyond + * it as well, bounded by the file since readahead stops there. A run of + * folios no grant covers is given back unfilled and fetched one folio at + * a time, so what is asked for here is what readahead is worth. + * + * Return: what fuse_get_dlm_lock() returned, 0 when there is nothing to + * ask for. + */ +static int fuse_read_grant(struct file *file, loff_t pos, size_t count) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + loff_t size = i_size_read(inode); + loff_t ahead = (loff_t)file->f_ra.ra_pages << PAGE_SHIFT; + loff_t end = pos + count; + + if (!fc->writeback_cache || !fc->dlm) + return 0; + + if (end < size) + end += min(ahead, size - end); + + if (end <= pos) + return 0; + + return fuse_get_dlm_lock(file, pos, end - pos, FUSE_PAGE_LOCK_READ); +} + +/** + * fuse_read_folio_retry - back off a fill with no grant to run under + * @file: file to read through + * @folio: the folio handed over locked, unlocked here + * @pos: byte offset of @folio + * @len: its size in bytes + * + * Neither reason a fill is refused can be dealt with while the folio is + * held: waiting a revoke out would hold the page cache that revoke is + * about to drop, and no grant may be asked for under a page lock at all + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * Unlock, do both, and send the caller round again to find the range + * covered. + * + * Return: AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_retry(struct file *file, struct folio *folio, + loff_t pos, size_t len) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + struct fuse_dlm_span pin; + int err; + + folio_unlock(folio); + + /* Wait the revoke out; what it leaves behind is asked for below */ + fuse_dlm_pin(fi, &pin, pos, len); + fuse_dlm_unpin(fi); + + err = fuse_read_grant(file, pos, len); + if (err == -ENOSYS) + return AOP_TRUNCATED_PAGE; + if (err < 0) + return err; + /* + * Granted but unrecorded, so the retry finds the range uncovered + * and comes straight back here. Report it rather than spin. + */ + if (err > 0) + return -ENOMEM; + + return AOP_TRUNCATED_PAGE; +} + static int fuse_read_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + struct fuse_dlm_span pin; + bool pinned = false; int err; err = -EIO; @@ -1155,6 +1271,30 @@ static int fuse_read_folio(struct file *file, struct folio *folio) */ folio_wait_writeback(folio); + /* + * The grant the folio is filled under, held from the confirmation + * until the bytes are in the page cache. What lands here is served + * to every later reader of the file, so it must neither be fetched + * under a grant a revoke has taken away nor be dropped into a range + * a revoke has just swept: such a folio is uptodate and covered by + * nothing, and no further notify comes for a lock this client no + * longer holds. + * + * The pin closes the second, since a revoke over the folio waits + * for the fill and drops the folio after it; the confirmation + * closes the first. Both fail into fuse_read_folio_retry(). + */ + if (fc->dlm && fc->writeback_cache) { + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin(fi); + pinned = false; + } + if (!pinned) + return fuse_read_folio_retry(file, folio, pos, len); + } + /* * Only a folio still holding what a write put in it has anything to * keep. One that was merely being written back is clean by now, @@ -1170,6 +1310,12 @@ static int fuse_read_folio(struct file *file, struct folio *folio) fuse_invalidate_atime(inode); out: folio_unlock(folio); + /* + * After the unlock, so a revoke draining this pin finds the folio + * it has to drop unlocked and takes it out. + */ + if (pinned) + fuse_dlm_unpin(fi); return err; } @@ -1259,19 +1405,40 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, folio_end_read(ap->folios[i], !err); folio_put(ap->folios[i]); } + + /* + * Dropped after the folios, which are filled, uptodate and unlocked + * by now: a revoke draining this pin finds them and takes them out. + * Held until here so it cannot have swept before they were there. + */ + if (ia->read.dlm_fi) + fuse_dlm_unpin_span(ia->read.dlm_fi, &ia->read.dlm_pin); + if (ia->ff) fuse_file_put(ia->ff, false); fuse_io_free(ia); } -static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, - unsigned int count) +/** + * fuse_send_readpages - read a run of folios of a readahead window + * @ia: the request, owning the folios and the pin over them + * @file: file to read through + * @count: bytes to read, starting at the first folio + * + * Return: 0 once the request is on its way or has been completed, + * -EAGAIN when a revoke of the range refused the grant and nothing was + * sent. The folios are given back either way. + */ +static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file, + unsigned int count) { struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); struct fuse_args_pages *ap = &ia->ap; loff_t pos = folio_pos(ap->folios[0]); + unsigned int i; ssize_t res; int err; @@ -1286,6 +1453,29 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, } WARN_ON((loff_t) (pos + count) < 0); + /* + * The grant the read took in fuse_read_grant(), confirmed under a + * pin and held until the reply has filled the folios. A revoke of the range + * waits for that, so the reply cannot be fetched under a grant the + * server has since handed on, and cannot land behind a sweep that + * would leave the folios uptodate and covered by nothing. + * + * Refused, or gone since it was asked for: give the folios back + * unfilled rather than serve what no lock covers. The read that + * wanted them comes back through fuse_read_folio(), which asks + * again with no folio held. + */ + if (fm->fc->dlm && fm->fc->writeback_cache) { + if (!fuse_dlm_trypin_span(fi, &ia->read.dlm_pin, pos, count)) + goto uncovered; + if (!fuse_dlm_lock_is_held(fi, pos, count, + FUSE_PAGE_LOCK_READ)) { + fuse_dlm_unpin_span(fi, &ia->read.dlm_pin); + goto uncovered; + } + ia->read.dlm_fi = fi; + } + fuse_read_args_fill(ia, file, pos, count, FUSE_READ); ia->read.attr_ver = fuse_get_attr_version(fm->fc); if (fm->fc->async_read) { @@ -1293,12 +1483,21 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ap->args.end = fuse_readpages_end; err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (!err) - return; + return 0; } else { res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } fuse_readpages_end(fm, &ap->args, err); + return 0; + +uncovered: + for (i = 0; i < ap->num_folios; i++) { + folio_end_read(ap->folios[i], false); + folio_put(ap->folios[i]); + } + fuse_io_free(ia); + return -EAGAIN; } static void fuse_readahead(struct readahead_control *rac) @@ -1312,44 +1511,29 @@ static void fuse_readahead(struct readahead_control *rac) return; /* - * Readahead fills the page cache past the range the reader locked, - * so take a DLM read grant over the whole window here too. Folios + * A latched inode keeps no page cache, and this window was not asked + * for by a read that has to be served (madvise(), fadvise()). Decline + * it: read_pages() drops the folios of a window left unfilled. + */ + if (fuse_inode_force_dio(inode)) + return; + + /* + * Readahead fills the page cache past the range the reader asked + * for, and what lands there has to be covered by a grant: folios * the server handed out no lock for are folios it will not revoke * when a remote node writes them, and a later read would be served - * from stale cache. Take the grant before any folio is pulled off - * @rac, so the window is either fully covered or not populated. - * - * Speculative pages are not worth serving uncovered: on a failed - * request drop the window and let read_pages() clean up the folios - * left in @rac. A server without DLM support answers -ENOSYS and - * clears fc->dlm, which is not a failure. - * - * ->readahead is entered with every folio of the window already - * locked, and pulling one off @rac is what unlocks it, so the round - * trip is taken under those locks. See the readahead line of - * Documentation/filesystems/locking.rst. + * from stale cache. * - * What keeps that from closing a cycle is the direction a revoke - * travels. This asks for a read grant on a range it holds nothing - * on, so the lock in the way is another node's, and the revoke that - * frees it is sent there. Nothing here has to run for this request - * to be answered, so the folios stay locked only for as long as the - * round trip. + * No grant is asked for here. ->readahead is entered with every + * folio of the window already locked, and none may be asked for + * under a page lock; the read this window belongs to took one over + * it in fuse_read_grant(), before the page cache was entered. * - * A window this already holds part of is the open edge: the query - * fails for the whole of it, so the request goes out while that part - * is still held, and a revoke for that part is sent here. Whether - * the server can answer while a revoke of its own is outstanding is - * not something this side can know. + * What that left uncovered fuse_send_readpages() declines, one run + * of folios at a time, and those folios go back unfilled for + * fuse_read_folio() to fetch with no folio held. */ - if (fc->writeback_cache && fc->dlm) { - int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), - readahead_length(rac), - FUSE_PAGE_LOCK_READ); - - if (err < 0 && err != -ENOSYS) - return; - } max_pages = min_t(unsigned int, fc->max_pages, fc->max_read / PAGE_SIZE); @@ -1413,7 +1597,8 @@ static void fuse_readahead(struct readahead_control *rac) pages += folio_pages; folio = NULL; } - fuse_send_readpages(ia, rac->file, pages << PAGE_SHIFT); + if (fuse_send_readpages(ia, rac->file, pages << PAGE_SHIFT)) + break; nr_pages -= pages; } if (folio) { @@ -1424,11 +1609,98 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); +/* + * Empty the page cache of an inode just latched into direct IO. + * + * The notify that set the latch dropped the mapping, but it holds no inode + * lock, so a cached write that had already passed both latch checks went on + * dirtying folios behind it. Left there, writeback would put them on the + * server on top of the direct writes that replace them, and direct reads, + * which do not look in the page cache, would miss them entirely. + * + * i_rwsem taken exclusive is what settles it: fuse_cache_write_iter() dirties + * under it, so by the time it is held every such writer has finished, and none + * can start behind this one. A write that takes the lock afterwards rechecks + * the latch and reroutes before it touches the cache. So the mapping stays + * empty from here on and the bit records that, leaving the latched IO paths + * with nothing to flush. + * + * Called from the top of the IO paths, with no inode lock held. A no-op for + * a file the server itself opened direct, which is never latched. + */ +static int fuse_force_dio_drain(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + int err = 0; + + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state) || + test_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state)) + return 0; + + inode_lock(inode); + /* + * The latch may have gone while this waited for the lock, and the drain + * is only owed to one that is still on. On a flush error the bit stays + * clear so the next IO tries again, and the error goes to this caller + * rather than being left for a later fsync to find. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + !test_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state)) { + err = filemap_write_and_wait(inode->i_mapping); + if (!err) { + invalidate_inode_pages2(inode->i_mapping); + set_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); + } + } + inode_unlock(inode); + + return err; +} + +/* + * Fold one request size into a moving average of this inode's sizes and + * report whether the file is being streamed: the same buffer size arriving + * FUSE_STREAM_RUN times over, which is what a task working through a file a + * record at a time looks like from here. A size outside the tolerance around + * the average starts the run again from that size, so a task changing its + * record is followed rather than averaged with what it did before. + * + * The average is per inode rather than per handle, so a stream stays one + * stream across reopens and across the handles of a shared file, whose users + * are streaming it together without any one of them being sequential. + * + * A hint only, read and written without the inode lock, which the DLM path + * holds shared: callers landing on it together cost a misread run, not + * correctness. + */ +static bool fuse_stream_update(unsigned int *ewma, unsigned int *run, + size_t len) +{ + unsigned int sample = min_t(size_t, len, FUSE_STREAM_EWMA_MAX); + unsigned int avg = *ewma >> FUSE_STREAM_EWMA_SHIFT; + + if (*run && abs_diff(sample, avg) <= avg >> FUSE_STREAM_TOL_SHIFT) { + /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ + *ewma += sample - avg; + if (*run < FUSE_STREAM_RUN) + (*run)++; + } else { + *ewma = sample << FUSE_STREAM_EWMA_SHIFT; + *run = 1; + } + + return *run >= FUSE_STREAM_RUN; +} + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; - struct inode *inode = file->f_mapping->host; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + size_t count = iov_iter_count(to); + bool stream = false; ssize_t res; /* @@ -1437,40 +1709,78 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * i_size is up to date). */ if (fc->auto_inval_data || - (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { + (iocb->ki_pos + count > i_size_read(inode))) { int err; err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE); if (err) return err; } - /* if we have dlm support acquire a read lock for the area - * we are reading from. */ - if (fc->writeback_cache && fc->dlm) - fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + /* + * Every read that could be cached feeds the size average, streamed or + * not: a reader changing its record has to be seen as well. O_DIRECT + * is not one of them, and an empty read says nothing about a record. + */ + if (count && !(iocb->ki_flags & IOCB_DIRECT)) + stream = fuse_stream_update(&fi->read_size_ewma, + &fi->read_stream_run, count); + + /* + * A streamed read of FUSE_READ_STREAM_MIN or more is served into the + * caller's own pages instead. Cached, the bytes are copied twice on + * their way out of the server, into the folios and out of them into + * the caller, and the folios are dropped unread; served from here they + * are copied once, and the grant the fill would have run under is not + * asked for at all. What that gives up is the readahead of the next + * record, not the wait for this one. + * + * Not on a mapped file, which keeps its page cache either way, and not + * for IOCB_NOWAIT, which the direct read has no way to honour. A + * caller that gets EAGAIN out of the cached path here comes back + * without the flag and takes this branch on the retry. + * + * Dirty folios over the range have to reach the server first: a direct + * read does not look in the page cache, and fuse_direct_io() flushes + * only for a file opened FOPEN_DIRECT_IO. + */ + if (stream && count >= FUSE_READ_STREAM_MIN && + !(iocb->ki_flags & IOCB_NOWAIT) && !mapping_mapped(mapping)) { + loff_t end = iocb->ki_pos + count - 1; + + if (filemap_range_needs_writeback(mapping, iocb->ki_pos, end)) { + res = filemap_write_and_wait_range(mapping, iocb->ki_pos, + end); + if (res) + return res; + } + return fuse_direct_read_iter(iocb, to); + } + + /* + * The grant this read and the readahead behind it fill under. Not + * for O_DIRECT, which takes no DLM lock at all: it fills no page + * cache, and what it reads is the server's to order. + */ + if (!(iocb->ki_flags & IOCB_DIRECT)) + fuse_read_grant(file, iocb->ki_pos, count); /* * A NOTIFY invalidate racing this read drops the folios it * supersedes, so the read either misses and refetches or returns - * data that was current when it was copied. There is nothing to - * fence: unlike a write, a read leaves nothing behind that could - * reach the server under a grant it no longer holds. + * data that was current when it was copied. What a read does leave + * behind is the page cache it fills, which must not outlast the + * grant it was fetched under; that is fenced where the filling + * happens, in fuse_read_folio() and fuse_send_readpages(), and the + * grant taken here is what they confirm. */ if (fuse_inode_force_dio(inode)) { - size_t count = iov_iter_count(to); - /* - * A write that passed this same check just before the latch - * took hold dirtied the page cache after the notify dropped - * it, and a direct read does not look there. Send it first. + * Latched between fuse_file_read_iter()'s check and this one, + * so the drain it would have done falls here. */ - if (count) { - res = filemap_write_and_wait_range(inode->i_mapping, - iocb->ki_pos, iocb->ki_pos + count - 1); - if (res) - return res; - } + res = fuse_force_dio_drain(inode); + if (res) + return res; return fuse_direct_read_iter(iocb, to); } @@ -2195,42 +2505,6 @@ static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) return (err < 0 && err != -ENOSYS) ? err : 0; } -/* - * Fold one buffered write size into the moving average of this inode's write - * sizes and report whether the file is being streamed: the same buffer size - * arriving FUSE_WRITE_STREAM_RUN times over, which is what a writer working - * through a file a record at a time looks like from here. A size outside the - * tolerance around the average starts the run again from that size, so a - * writer changing its record is followed rather than averaged with what it - * did before. - * - * The average is per inode rather than per handle, so a stream stays one - * stream across reopens and across the handles of a shared file, whose - * writers are streaming it together without any one of them being sequential. - * - * A hint only, read and written without the inode lock, which the DLM path - * holds shared: writers landing on it together cost a misread run, not - * correctness. - */ -static bool fuse_write_stream_update(struct fuse_inode *fi, size_t len) -{ - unsigned int sample = min_t(size_t, len, FUSE_WRITE_EWMA_MAX); - unsigned int avg = fi->write_size_ewma >> FUSE_WRITE_EWMA_SHIFT; - - if (fi->write_stream_run && - abs_diff(sample, avg) <= avg >> FUSE_WRITE_TOL_SHIFT) { - /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ - fi->write_size_ewma += sample - avg; - if (fi->write_stream_run < FUSE_WRITE_STREAM_RUN) - fi->write_stream_run++; - } else { - fi->write_size_ewma = sample << FUSE_WRITE_EWMA_SHIFT; - fi->write_stream_run = 1; - } - - return fi->write_stream_run >= FUSE_WRITE_STREAM_RUN; -} - /* * Start non-integrity writeback on the aligned chunks a streamed file has * left behind. @@ -2249,7 +2523,10 @@ static bool fuse_write_stream_update(struct fuse_inode *fi, size_t len) * A hint only: nothing waits for it, and a chunk that is partly dirty or * already written back sends what it has. The run is read and written * without the inode lock, which the DLM path holds shared, so writers - * landing on it together cost a kick, not correctness. + * landing on it together cost a kick, not correctness. Each mark is read + * once into a local for that to hold: reading write_stream_start twice, + * to round down and again to compare, lets a writer moving it in between + * invert the range the kick is given. */ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, size_t len, bool stream) @@ -2257,25 +2534,27 @@ static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct fuse_inode *fi = get_fuse_inode(inode); - loff_t chunk, start, end; + loff_t chunk, run, start, end; bool sequential; - sequential = pos == fi->write_stream_next; - fi->write_stream_next = pos + (loff_t)len; + sequential = pos == READ_ONCE(fi->write_stream_next); + WRITE_ONCE(fi->write_stream_next, pos + (loff_t)len); if (!sequential || !stream) { /* Nothing behind this write is known to be finished */ - fi->write_stream_start = pos; + WRITE_ONCE(fi->write_stream_start, pos); return; } + run = READ_ONCE(fi->write_stream_start); chunk = fuse_write_chunk_size(get_fuse_conn(inode)); - start = round_down(fi->write_stream_start, chunk); - end = round_down(fi->write_stream_next, chunk); + start = round_down(run, chunk); + /* This write's own end, not the mark another writer may have moved */ + end = round_down(pos + (loff_t)len, chunk); /* No bound crossed, so nothing has been left complete */ - if (end <= fi->write_stream_start) + if (end <= run) return; - fi->write_stream_start = end; + WRITE_ONCE(fi->write_stream_start, end); filemap_fdatawrite_range_kick(file->f_mapping, start, end - 1); } @@ -2344,7 +2623,9 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * with. */ if (writeback && !(iocb->ki_flags & IOCB_DIRECT)) - stream = fuse_write_stream_update(fi, iov_iter_count(from)); + stream = fuse_stream_update(&fi->write_size_ewma, + &fi->write_stream_run, + iov_iter_count(from)); /* * A streamed write of FUSE_WRITE_STREAM_MIN or more is sent from here @@ -2378,8 +2659,13 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * too early, an expanding write would fall through to a READ of a * range that cannot hold data -- which fails outright on a handle * the client opened write-only. + * + * O_DIRECT takes no DLM lock at all, here or anywhere: it dirties + * no page cache, so there is nothing for a grant to cover, and its + * bytes are the server's to order against the rest of the cluster. */ - if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + !(iocb->ki_flags & IOCB_APPEND)) { err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, iov_iter_count(from)); if (err) @@ -2403,7 +2689,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * lock does not pin i_size either: attribute replies move it under * fi->lock alone. Take the grant here, where the range is settled. */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + (iocb->ki_flags & IOCB_APPEND)) { err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, count); if (err) goto out; @@ -2429,16 +2716,12 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (fuse_inode_force_dio(inode)) { /* - * As on the read side, only worse: the direct write would - * land under whatever a write racing the latch left dirty, - * and the invalidate fuse_direct_write_iter() does after it - * launders rather than drops, putting that folio on the - * server on top. Send it first and the order is ordinary. + * Latched while this write waited for the lock, so the drain + * fuse_file_write_iter() would have done falls here. Drop the + * lock first: the drain takes it exclusive. */ - if (count) - err = filemap_write_and_wait_range(inode->i_mapping, - iocb->ki_pos, iocb->ki_pos + count - 1); fuse_cache_wr_unlock(inode, exclusive); + err = fuse_force_dio_drain(inode); if (err) return err; return fuse_direct_write_iter(iocb, from); @@ -2464,6 +2747,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) false)); } else if (through) { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + struct fuse_dlm_span pin; loff_t pos = iocb->ki_pos; /* @@ -2480,8 +2764,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } + /* + * These bytes never enter the page cache, so a revoke + * cannot find them by flushing it, and the grant they were + * taken under can be handed on while they are still on the + * wire. Hold it across the FUSE_WRITE, as the writethrough + * edges do; a revoke of the range waits for the reply. + */ + err = fuse_dlm_pin_write(file, &pin, pos, count); + if (err) + goto out; + written = fuse_direct_io(&io, from, &iocb->ki_pos, - FUSE_DIO_WRITE); + FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); + fuse_dlm_unpin(fi); if (written < 0) { err = written; goto out; @@ -2731,6 +3028,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, { int write = flags & FUSE_DIO_WRITE; int cuse = flags & FUSE_DIO_CUSE; + int shared = flags & FUSE_DIO_SHARED; struct file *file = io->iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; @@ -2759,12 +3057,33 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, return res; } } + + /* + * Wait out the writeback this read or write would otherwise race: + * a reply landing after it would put the superseded bytes on the + * server on top of it. + * + * How to wait depends on the lock the caller holds. fuse_set_nowrite() + * asserts BUG_ON(fi->writectr < 0) and biases a counter the whole inode + * shares, which only an exclusive i_rwsem makes safe; two callers + * holding it shared reach that assertion together and the second one + * dies inside spin_lock(&fi->lock). A parallel direct write holds it + * shared (fuse_dio_lock(), and the streamed write in + * fuse_cache_write_iter()), so it waits on the folios of its own range + * instead, which is the range this test asked about anyway. Errors are + * left on the mapping for fsync to collect. + */ if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) { - if (!write) + if (!write) { inode_lock(inode); - fuse_sync_writes(inode); - if (!write) + fuse_sync_writes(inode); inode_unlock(inode); + } else if (shared) { + filemap_fdatawait_range_keep_errors(mapping, pos, + pos + count - 1); + } else { + fuse_sync_writes(inode); + } } if (fopen_direct_io && write) { @@ -2882,7 +3201,8 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); res = fuse_direct_io(&io, from, &iocb->ki_pos, - FUSE_DIO_WRITE); + FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); fuse_write_update_attr(inode, iocb->ki_pos, res); } if (res > 0 && mapping->nrpages) { @@ -2901,6 +3221,59 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) return res; } +/* + * Whether the inode is still latched into direct IO. The latch is set on a + * stream of invalidation notifies and has to come off when the stream stops, + * or an inode that was hot once stays uncached for as long as it is open. The + * average behind the latch is folded on arrival and cannot age on its own, so + * the age of the last notify is the clock. + * + * Cleared here the way the mmap revert clears it: no inode lock, none is held + * at the top of the IO paths, and the drop is server-free because everything + * cached under the latch is clean. + */ +static bool fuse_force_dio_active(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + bool cleared = false; + + if (!fuse_inode_force_dio(inode)) + return false; + + /* + * Unlocked, and both may change under this: a writer that appears after + * the check leaves the latch cleared, one that goes leaves it on until + * the next IO looks again. Neither is wrong, and the list is re-read + * under fi->lock before anything is cleared. + */ + if (!time_after(jiffies, + READ_ONCE(fi->notify_stamp) + FUSE_NOTIFY_DIO_COLD) || + !list_empty_careful(&fi->write_files)) + return true; + + spin_lock(&fi->lock); + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + list_empty(&fi->write_files) && + time_after(jiffies, fi->notify_stamp + FUSE_NOTIFY_DIO_COLD)) { + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + cleared = true; + } + spin_unlock(&fi->lock); + + /* + * Whatever a read racing the latch left behind, so it cannot be served + * once caching resumes. Another caller may have cleared the latch + * first, or set it again since; the bit decides, not this one's work. + */ + if (cleared) + fuse_force_dio_drop(inode->i_mapping); + + return fuse_inode_force_dio(inode); +} + static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; @@ -2914,12 +3287,17 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return fuse_dax_read_iter(iocb, to); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) { + ssize_t err = fuse_force_dio_drain(inode); + + if (err) + return err; return fuse_direct_read_iter(iocb, to); - else if (fuse_file_passthrough(ff)) + } else if (fuse_file_passthrough(ff)) { return fuse_passthrough_read_iter(iocb, to); - else + } else { return fuse_cache_read_iter(iocb, to); + } } static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) @@ -2935,12 +3313,17 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_dax_write_iter(iocb, from); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) { + ssize_t err = fuse_force_dio_drain(inode); + + if (err) + return err; return fuse_direct_write_iter(iocb, from); - else if (fuse_file_passthrough(ff)) + } else if (fuse_file_passthrough(ff)) { return fuse_passthrough_write_iter(iocb, from); - else + } else { return fuse_cache_write_iter(iocb, from); + } } static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, @@ -2952,8 +3335,17 @@ static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); - else - return filemap_splice_read(in, ppos, pipe, len, flags); + + /* + * Latched: copy through ->read_iter, which reads direct, rather than + * fill a page cache this inode is not allowed to keep. + */ + if (fuse_force_dio_active(file_inode(in))) + return copy_splice_read(in, ppos, pipe, len, flags); + + fuse_read_grant(in, *ppos, len); + + return filemap_splice_read(in, ppos, pipe, len, flags); } static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, @@ -3359,7 +3751,15 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, pgoff_t page_index = pos >> PAGE_SHIFT; if (wbc && !(page_index % fc->alignment_pages)) { - pgoff_t end_page_index = (wbc->range_end + PAGE_SIZE - 1) >> PAGE_SHIFT; + /* + * A cyclic pass leaves range_end at zero and runs to + * the end of the mapping instead, so reading it there + * makes every aligned index its own request. + */ + pgoff_t end_page_index = wbc->range_cyclic ? + (pgoff_t) -1 : + (pgoff_t) ((wbc->range_end + PAGE_SIZE - 1) >> + PAGE_SHIFT); /* we are at a point where we would write aligned * check if we potentially could reach the next alignment */ @@ -3860,9 +4260,22 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) return VM_FAULT_LOCKED; } +/* + * A read fault fills the page cache through ->read_folio and + * ->readahead, which run with the folios locked. Ask for the grant they + * fill under before filemap_fault() locks any of them. + */ +static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) +{ + fuse_read_grant(vmf->vma->vm_file, (loff_t)vmf->pgoff << PAGE_SHIFT, + PAGE_SIZE); + + return filemap_fault(vmf); +} + static const struct vm_operations_struct fuse_file_vm_ops = { .close = fuse_vma_close, - .fault = filemap_fault, + .fault = fuse_filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = fuse_page_mkwrite, }; @@ -3905,10 +4318,11 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) spin_lock(&fi->lock); clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); spin_unlock(&fi->lock); - invalidate_inode_pages2(file->f_mapping); + fuse_force_dio_drop(file->f_mapping); } /* @@ -4409,7 +4823,8 @@ __fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bool exclusive) } if (iov_iter_rw(iter) == WRITE) { - ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); + ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); } else { ret = __fuse_direct_read(io, iter, &pos); @@ -4683,6 +5098,21 @@ static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off, return ret; } +/* + * POSIX_FADV_WILLNEED, and readahead(2) with it, populate the page cache + * through ->readahead, which runs with the folios locked. Ask for the + * grant that fill needs while nothing is held; a window no grant covers + * is given back unfilled. + */ +static int fuse_fadvise(struct file *file, loff_t offset, loff_t len, + int advice) +{ + if (advice == POSIX_FADV_WILLNEED && offset >= 0 && len > 0) + fuse_read_grant(file, offset, len); + + return generic_fadvise(file, offset, len, advice); +} + static const struct file_operations fuse_file_operations = { .llseek = fuse_file_llseek, .read_iter = fuse_file_read_iter, @@ -4702,6 +5132,7 @@ static const struct file_operations fuse_file_operations = { .poll = fuse_file_poll, .fallocate = fuse_file_fallocate, .copy_file_range = fuse_copy_file_range, + .fadvise = fuse_fadvise, }; static const struct address_space_operations fuse_file_aops = { @@ -4742,6 +5173,8 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->write_stream_run = 0; fi->write_stream_next = 0; fi->write_stream_start = 0; + fi->read_size_ewma = 0; + fi->read_stream_run = 0; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index d3ef8548d3b380..8b24c563ee03ec 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -179,11 +179,11 @@ void fuse_dlm_cache_init(struct fuse_inode *inode) * about, so a fence over a page cannot miss a pin on that page. */ static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, - size_t length) + size_t length, struct task_struct *owner) { span->start = (uint64_t)offset & PAGE_MASK; span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); - span->owner = current; + span->owner = owner; } /* @@ -243,7 +243,7 @@ void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, if (fuse_in_notify_ctx()) return; - fuse_dlm_span_set(pin, offset, length); + fuse_dlm_span_set(pin, offset, length, current); spin_lock(&cache->pin_lock); while (fuse_dlm_overlaps_locked(&cache->fences, pin->start, pin->end)) { @@ -285,7 +285,7 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, if (fuse_in_notify_ctx()) return true; - fuse_dlm_span_set(pin, offset, length); + fuse_dlm_span_set(pin, offset, length, current); spin_lock(&cache->pin_lock); fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, @@ -297,6 +297,62 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, return !fenced; } +/** + * fuse_dlm_trypin_span - fuse_dlm_trypin() for a fill that ends elsewhere + * @inode: the fuse inode + * @pin: caller-owned storage, live until fuse_dlm_unpin_span() + * @offset: byte offset the caller is about to fill + * @length: length of the region in bytes + * + * For a read whose reply lands in another task: the node is dropped by + * fuse_dlm_unpin_span() from wherever the fill ends, and carries no + * owner, so a fuse_dlm_unpin() by the task that took it cannot match it + * instead of its own. + * + * Never sleeps, and has no notify-context shortcut: a fill is not + * reached from a revoke handler, and a pin taken there would have to be + * dropped from a task that is not in one. + * + * Return: true if the range is pinned, false if a revoke of it is + * draining. + */ +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; + + fuse_dlm_span_set(pin, offset, length, NULL); + + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); + + return !fenced; +} + +/** + * fuse_dlm_unpin_span - release the pin fuse_dlm_trypin_span() took + * @inode: the fuse inode + * @pin: the node published there + */ +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool waiters; + + spin_lock(&cache->pin_lock); + list_del(&pin->list); + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); + + if (waiters) + wake_up_all(&cache->pin_wq); +} + /** * fuse_dlm_unpin - release the pin this task last took on @inode * @inode: the fuse inode diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index ec2e82c406a1cb..ae6aa2bcbdae93 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -24,10 +24,14 @@ struct fuse_file; enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; /* - * A range held on one of the two lists in struct fuse_dlm_cache: a - * writer between confirming a grant and dirtying under it (@owner set), - * or a revoke taking grants away (@owner NULL). Caller-owned storage, - * live until the matching unpin or revoke end. + * A range held on one of the two lists in struct fuse_dlm_cache: an IO + * between confirming a grant and publishing the page cache it covers, + * or a revoke taking grants away. Caller-owned storage, live until the + * matching unpin or revoke end. + * + * @owner is the pinning task where the pin is dropped by owner, and NULL + * on every fence and on a pin dropped by node because the fill it covers + * ends in another task. */ struct fuse_dlm_span { /* Page-aligned byte offsets, both inclusive */ @@ -122,7 +126,9 @@ struct fuse_dlm_shard { * between sends those bytes out after the server has handed the lock on. * The pin closes that: a revoke waits for the pins over the range it is * taking away before it removes anything, so a grant confirmed under a - * pin is still held when the bytes become visible to writeback. + * pin is still held when the bytes become visible to writeback. A read + * is the same the other way round, a fill landing in a range the revoke + * has already swept staying cached under no grant at all. * * Both sides are ranges rather than a count, so a revoke fences only the * writers it overlaps and a write outside it runs on. Refusal and wait @@ -149,8 +155,9 @@ struct fuse_dlm_cache { /* Protects @pins and @fences */ spinlock_t pin_lock; /* - * Writers between confirming a grant and dirtying under it, each - * over the range it is about to write. See the pin comment above. + * IO between confirming a grant and publishing under it: a writer + * over what it is about to dirty, a read over what it is about to + * fill. See the pin comment above. */ struct list_head pins; /* Revokes in progress, each over the range it takes away */ @@ -208,6 +215,15 @@ bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, loff_t offset, size_t length); void fuse_dlm_unpin(struct fuse_inode *inode); +/* + * fuse_dlm_trypin() for a fill whose reply lands in another task: @pin + * is dropped by node rather than by owner, and is live from the request + * until fuse_dlm_unpin_span(). + */ +bool fuse_dlm_trypin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin); + /* * Fence the writers that hold a grant over [@offset, @offset + @len) but * have not dirtied under it yet, for the duration of a revoke. @len <= 0 diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 59dd5ca9378c7c..25d239a93c8ad1 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -187,31 +187,35 @@ struct dlm_locked_area * Force-DIO switch trigger: an exponentially weighted moving average of the * interval (in jiffies) between FUSE_NOTIFY_INVAL_INODE data invalidations for * a file. When the average spacing falls below FUSE_NOTIFY_DIO_INTERVAL -- a - * remote writer streaming invalidations -- and the file is open for writing - * here, it is latched into direct IO. These are the source-level (not - * externally tunable) parameters of the heuristic: EWMA weight 1/2^SHIFT, - * seeded and capped at SEED so it takes a short burst rather than a single - * notify to trip. + * remote writer streaming invalidations -- and the file is open here, it is + * latched into direct IO. These are the source-level (not externally tunable) + * parameters of the heuristic: EWMA weight 1/2^SHIFT, seeded and capped at SEED + * so it takes a short burst rather than a single notify to trip. + * + * The average is folded on arrival and cannot age on its own, so what takes the + * latch off again is the last invalidation reaching FUSE_NOTIFY_DIO_COLD old. */ #define FUSE_NOTIFY_DIO_INTERVAL max_t(unsigned long, HZ / 10, 1) #define FUSE_NOTIFY_EWMA_SHIFT 2 #define FUSE_NOTIFY_EWMA_SEED (2 * FUSE_NOTIFY_DIO_INTERVAL) +#define FUSE_NOTIFY_DIO_COLD (8 * FUSE_NOTIFY_DIO_INTERVAL) /* * Streamed file trigger: the same buffer size arriving over and over is a - * writer working through a file a record at a time. The sizes are folded + * task working through a file a record at a time. The sizes are folded * into an exponentially weighted moving average (weight 1/2^SHIFT, kept - * shifted), and a run of FUSE_WRITE_STREAM_RUN writes within - * 1/2^FUSE_WRITE_TOL_SHIFT of it says the writer is still on it. The sample + * shifted), and a run of FUSE_STREAM_RUN requests within + * 1/2^FUSE_STREAM_TOL_SHIFT of it says the task is still on it. The sample * is capped to keep the shifted accumulator inside an unsigned int. */ -#define FUSE_WRITE_EWMA_SHIFT 2 -#define FUSE_WRITE_TOL_SHIFT 3 -#define FUSE_WRITE_STREAM_RUN 4 -#define FUSE_WRITE_EWMA_MAX (UINT_MAX >> FUSE_WRITE_EWMA_SHIFT) +#define FUSE_STREAM_EWMA_SHIFT 2 +#define FUSE_STREAM_TOL_SHIFT 3 +#define FUSE_STREAM_RUN 4 +#define FUSE_STREAM_EWMA_MAX (UINT_MAX >> FUSE_STREAM_EWMA_SHIFT) -/* Under this size the copy into the page cache is not worth avoiding */ +/* Under this size the copy through the page cache is not worth avoiding */ #define FUSE_WRITE_STREAM_MIN (64 * 1024) +#define FUSE_READ_STREAM_MIN (10 * PAGE_SIZE) /** FUSE inode */ struct fuse_inode { @@ -287,9 +291,11 @@ struct fuse_inode { * the last one, notify_interval_ewma the EWMA of the * inter-arrival interval (jiffies, scaled by * 2^FUSE_NOTIFY_EWMA_SHIFT). A rapid stream (short - * average interval) with a local writer latches the - * inode into direct IO. Protected by fi->lock; regular - * files only (shares the readdir-cache union arm). + * average interval) with the file open here latches + * the inode into direct IO, and notify_stamp going + * stale takes it out again. Protected by fi->lock; + * regular files only (shares the readdir-cache union + * arm). */ unsigned long notify_stamp; unsigned int notify_interval_ewma; @@ -318,13 +324,19 @@ struct fuse_inode { * next write has to land to carry the run of * positions on, and write_stream_start the first * byte of that run no writeback kick has covered. + * read_size_ewma and read_stream_run are the same + * average over the reads that could be cached, kept + * apart so a write phase and a read phase over one + * file do not fold into each other. * Hints only, read and written without a lock; see - * fuse_write_stream_update(). + * fuse_stream_update(). */ unsigned int write_size_ewma; unsigned int write_stream_run; loff_t write_stream_next; loff_t write_stream_start; + unsigned int read_size_ewma; + unsigned int read_stream_run; }; /* readdir cache (directory only) */ @@ -400,10 +412,16 @@ enum { * Latched into direct IO: a NOTIFY_INVAL_INODE arrived while the file * was open for writing here, so another (remote) entity is modifying it * concurrently. Reads and writes are routed direct (shared-lock - * parallel dio) until the last writer closes or the inode is mmapped. - * See fuse_reverse_inval_inode()/fuse_file_io_open(). + * parallel dio) until the notifies stop, the last writer closes, or the + * inode is mmapped. See fuse_reverse_inval_inode()/fuse_file_io_open(). */ FUSE_I_FORCE_DIO, + /* + * The page cache has been emptied under that latch, so no cached write + * from before it can still be in flight. Set once per latch by + * fuse_force_dio_drain(), cleared wherever FUSE_I_FORCE_DIO is. + */ + FUSE_I_FORCE_DIO_DRAINED, }; struct fuse_conn; @@ -1347,6 +1365,15 @@ struct fuse_io_args { struct { struct fuse_read_in in; u64 attr_ver; + /* + * The grant the folios are filled under, held + * from the request until the reply has filled + * them; see fuse_send_readpages(). @dlm_fi is + * the inode to drop it on, and NULL when there + * is no pin to drop. + */ + struct fuse_dlm_span dlm_pin; + struct fuse_inode *dlm_fi; } read; struct { struct fuse_write_in in; @@ -1648,6 +1675,9 @@ int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, /** CUSE pass fuse_direct_io() a file which f_mapping->host is not from FUSE */ #define FUSE_DIO_CUSE (1 << 1) +/** Caller holds i_rwsem shared, so fuse_set_nowrite() must not be used */ +#define FUSE_DIO_SHARED (1 << 2) + ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, loff_t *ppos, int flags); long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index a58c1524617899..341687211723ee 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -631,6 +631,30 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr evict_ctr); oldsize = inode->i_size; + /* + * fuse_attr_cache_mask() answered before the grant query slept, and a + * write below EOF bumps neither fi->attr_version nor + * fi->size_extenders: it extends nothing, so the version check cannot + * drop the reply and the count cannot hold the size. Folios can have + * been dirtied in the doomed range since the answer, and + * truncate_pagecache() below throws them away with no error to report + * it. + * + * Re-test here instead, where nothing sleeps between the answer and + * the truncate acting on it, and keep the local size when the range + * still holds bytes the server has not seen. A remote truncate is + * unaffected: it revokes first, and the revoke launders and drops the + * range, so there is nothing here to find. + */ + if (have_size && !(cache_mask & STATX_SIZE) && fc->dlm && + fc->writeback_cache && S_ISREG(inode->i_mode) && + (loff_t) attr->size < oldsize && + filemap_range_needs_writeback(inode->i_mapping, attr->size, + oldsize - 1)) { + cache_mask |= STATX_SIZE; + attr->size = oldsize; + } + /* * In case of writeback_cache enabled, the cached writes beyond EOF * extend local i_size without keeping userspace server in sync. So, @@ -656,7 +680,23 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr bool have_mtime = !sx || (sx->mask & STATX_MTIME); if (have_size && oldsize != attr->size) { - truncate_pagecache(inode, attr->size); + /* + * Not under DLM. This runs after fi->lock is + * dropped and takes nothing a writer holds, so a + * folio dirtied between the decision and the walk, + * or during it, is discarded with no error to report + * it. A real truncate may do this because + * fuse_set_nowrite() and i_rwsem hold the writers + * off; an attribute reply holds off nothing. + * + * Keeping the folios costs nothing either way. If + * the size was wrong they are written back and the + * size recovers; if it was right, the revoke that had + * to precede it already dropped the range and there + * is nothing here to discard. + */ + if (!(fc->dlm && fc->writeback_cache)) + truncate_pagecache(inode, attr->size); if (!fc->explicit_inval_data) inval = true; } else if (have_mtime && fc->auto_inval_data) { @@ -870,6 +910,15 @@ static void fuse_invalidate_inode_entry(struct inode *inode) } } +/* + * Someone here has the file open and would use its page cache: a writeback + * writer, or any open in caching mode. Must be called under fi->lock. + */ +static bool fuse_inode_has_opener(struct fuse_inode *fi) +{ + return !list_empty(&fi->write_files) || fi->iocachectr > 0; +} + /* * Fold one FUSE_NOTIFY_INVAL_INODE data invalidation into the per-inode * moving average of the notification inter-arrival interval and report whether @@ -878,8 +927,8 @@ static void fuse_invalidate_inode_entry(struct inode *inode) * average is an EWMA (weight 1/2^FUSE_NOTIFY_EWMA_SHIFT); the sample is clamped * to FUSE_NOTIFY_EWMA_SEED so a notify after a long idle only cools the average * and cannot overflow the accumulator. Must be called under fi->lock; called - * for every data invalidation so the average stays current even while no local - * writer is open. + * for every data invalidation so the average stays current even while the file + * is not open here. */ static bool fuse_notify_inval_hot(struct fuse_inode *fi) { @@ -1078,8 +1127,12 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * under fi->lock, updated for every data invalidation) of * how fast these arrive; when they come in a rapid stream * -- a remote writer repeatedly invalidating -- and the - * inode is also open for writing here, latch it into - * direct IO until the last writer closes or it is mmapped. + * inode is open here, latch it into direct IO until the + * stream stops, the last writer closes, or it is mmapped. + * A reader-only inode is latched too: what it caches + * between two invalidations is dropped again before it can + * be read twice, so the cache costs the folios and the read + * grants behind them and returns nothing. * When latched, drop the whole mapping rather than just * the notified range, or dirty folios outside it would be * invisible to the forced direct reads (stale read / lost @@ -1088,8 +1141,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * up to date either way, so enabling it at runtime takes * effect on the next storm rather than after a warm-up. * Clearing it at runtime stops new latches but lets - * already-latched inodes run out on the usual exits (last - * writer closes, or mmap). + * already-latched inodes run out on the usual exits. * * The average and the latch exist only for writeback+dlm * regular files; elsewhere there is no record to consult and @@ -1102,12 +1154,12 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, !fuse_inode_backing(fi); if (tracked) { - bool hot, has_writer, latched = false; + bool hot, has_opener, latched = false; bool may_be_dirty, has_pages; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); - has_writer = !list_empty(&fi->write_files); + has_opener = fuse_inode_has_opener(fi); spin_unlock(&fi->lock); /* @@ -1153,11 +1205,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_dlm_revoke_inval_range(fi, offset, len); - if (enable_notify_dio && hot && has_writer && + if (enable_notify_dio && hot && has_opener && !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); - if (!list_empty(&fi->write_files)) { + if (fuse_inode_has_opener(fi)) { set_bit(FUSE_I_FORCE_DIO, &fi->state); latched = true; } @@ -1167,17 +1219,23 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* * Latched: drop the whole mapping (dirty folios * outside the notified range would be invisible to - * the forced direct reads), and the record says - * nothing about the rest of the file, so launder. - * Otherwise just the notified range, and only if - * anything is cached there. + * the forced direct reads), laundering only what the + * mapping says may be dirty, which for an inode + * latched with no writer is nothing. Otherwise just + * the notified range, and only if anything is cached + * there. */ - if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1, true); - else if (has_pages) + if (fuse_inode_force_dio(inode)) { + bool dirty; + + dirty = filemap_range_needs_writeback( + inode->i_mapping, 0, LLONG_MAX); + fuse_notify_invalidate_range(inode, 0, -1, dirty); + } else if (has_pages) { fuse_notify_invalidate_range(inode, pg_start, pg_end, may_be_dirty); + } if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n",