diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 90117fb13ec8e0..a5127324709bab 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -370,24 +370,27 @@ void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) } } +/* + * fuse_args_to_req() assigns the unique already, so that the early tracepoints + * see it. Assign here only for requests that did not pass through it. + * + * The send is not traced here: this runs where a request is queued, which + * trace_fuse_request_enqueue() already marks. trace_fuse_request_send() + * belongs where the request reaches the server, in fuse_dev_do_read() and + * fuse_uring_send(). + */ static inline void fuse_request_assign_unique_locked(struct fuse_iqueue *fiq, struct fuse_req *req) { - if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + if (!req->in.h.unique && req->in.h.opcode != FUSE_NOTIFY_REPLY) req->in.h.unique = fuse_get_unique_locked(fiq); - - /* tracepoint captures in.h.unique and in.h.len */ - trace_fuse_request_send(req); } inline void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req) { - if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + if (!req->in.h.unique && req->in.h.opcode != FUSE_NOTIFY_REPLY) req->in.h.unique = fuse_get_unique(fiq); - - /* tracepoint captures in.h.unique and in.h.len */ - trace_fuse_request_send(req); } EXPORT_SYMBOL_GPL(fuse_request_assign_unique); @@ -662,6 +665,9 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; if (args->end) __set_bit(FR_ASYNC, &req->flags); + + if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + req->in.h.unique = fuse_get_unique(&req->fm->fc->iq); } ssize_t fuse_compound_request(struct fuse_mount *fm, struct fuse_args *args) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 7fa79e68afd5c9..88662d909fe8bd 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -12,6 +12,7 @@ #include #include #include +#include static bool __read_mostly enable_uring; module_param(enable_uring, bool, 0644); @@ -28,7 +29,6 @@ MODULE_PARM_DESC(enable_uring, /* Number of (re)tries to find a better queue */ #define FUSE_URING_Q_TRIES 3 - bool fuse_uring_enabled(void) { return enable_uring; @@ -85,8 +85,8 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) } } -static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, - int error) +static void __fuse_uring_req_end(struct fuse_ring_ent *ent, + struct fuse_req *req, int error) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; @@ -109,6 +109,43 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, req->out.h.error = error; clear_bit(FR_SENT, &req->flags); +} + +static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, + int error) +{ + __fuse_uring_req_end(ent, req, error); + fuse_request_end(req); +} + +static void fuse_uring_req_end_work(struct callback_head *work) +{ + struct fuse_req *req = container_of(work, struct fuse_req, + ring_end_work); + + fuse_request_end(req); +} + +/* + * On the commit path ->uring_cmd() runs with ctx->uring_lock held by + * io_uring_enter(). fuse_request_end() wakes the request submitter, which + * typically preempts the ring task right away (same CPU) - while the mutex + * is still held. Defer the completion to task work, which runs once the + * submission path has released the lock (in io_cqring_wait() or on return + * to userspace), so the ring task can finish its critical section first. + */ +static void fuse_uring_req_end_deferred(struct fuse_ring_ent *ent, + struct fuse_req *req, int error, + unsigned int issue_flags) +{ + __fuse_uring_req_end(ent, req, error); + + if (!(issue_flags & IO_URING_F_UNLOCKED)) { + init_task_work(&req->ring_end_work, fuse_uring_req_end_work); + if (!task_work_add(current, &req->ring_end_work, TWA_RESUME)) + return; + } + fuse_request_end(req); } @@ -153,41 +190,6 @@ void fuse_uring_flush_bg(struct fuse_conn *fc) } } -/* - * Copy from memmap.c, should be exported - */ -static void io_pages_free(struct page ***pages, int npages) -{ - struct page **page_array = *pages; - - if (!page_array) - return; - - unpin_user_pages(page_array, npages); - kvfree(page_array); - *pages = NULL; -} - - -static void fuse_ring_destruct_q_map(struct fuse_queue_map *q_map) -{ - free_cpumask_var(q_map->registered_q_mask); - kfree(q_map->cpu_to_qid); -} - -static void fuse_uring_destruct_q_masks(struct fuse_ring *ring) -{ - int node; - - fuse_ring_destruct_q_map(&ring->q_map); - - if (ring->numa_q_map) { - for (node = 0; node < ring->nr_numa_nodes; node++) - fuse_ring_destruct_q_map(&ring->numa_q_map[node]); - kfree(ring->numa_q_map); - } -} - static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list) { struct fuse_ring_ent *ent; @@ -231,6 +233,40 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) return false; } +/* + * Copy from memmap.c, should be exported + */ +static void io_pages_free(struct page ***pages, int npages) +{ + struct page **page_array = *pages; + + if (!page_array) + return; + + unpin_user_pages(page_array, npages); + kvfree(page_array); + *pages = NULL; +} + +static void fuse_ring_destruct_q_map(struct fuse_queue_map *q_map) +{ + free_cpumask_var(q_map->registered_q_mask); + kfree(q_map->cpu_to_qid); +} + +static void fuse_uring_destruct_q_masks(struct fuse_ring *ring) +{ + int node; + + fuse_ring_destruct_q_map(&ring->q_map); + + if (ring->numa_q_map) { + for (node = 0; node < ring->nr_numa_nodes; node++) + fuse_ring_destruct_q_map(&ring->numa_q_map[node]); + kfree(ring->numa_q_map); + } +} + void fuse_uring_destruct(struct fuse_conn *fc) { struct fuse_ring *ring = fc->ring; @@ -323,8 +359,8 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) ring->nr_numa_nodes = num_online_nodes(); - ring->queues = kcalloc(nr_queues, sizeof(struct fuse_ring_queue *), - GFP_KERNEL_ACCOUNT); + ring->queues = kzalloc_objs(struct fuse_ring_queue *, nr_queues, + GFP_KERNEL_ACCOUNT); if (!ring->queues) goto out_err; @@ -849,7 +885,6 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, /* copy the payload */ err = fuse_copy_args(&cs, num_args, args->in_pages, (struct fuse_arg *)in_args, 0); - fuse_copy_finish(&cs); if (err) { pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); goto copy_finish; @@ -921,6 +956,21 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, return err; } +static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, + ssize_t ret, unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + ent->state = FRRS_USERSPACE; + list_move_tail(&ent->list, &queue->ent_in_userspace); + ent->cmd = NULL; + spin_unlock(&queue->lock); + + trace_fuse_request_send(ent->fuse_req); + io_uring_cmd_done(cmd, ret, issue_flags); +} + /* * Write data to the ring buffer and send the request to userspace, * userspace will read it @@ -930,22 +980,13 @@ static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { - struct fuse_ring_queue *queue = ent->queue; int err; - struct io_uring_cmd *cmd; err = fuse_uring_prepare_send(ent, req); if (err) return err; - spin_lock(&queue->lock); - cmd = ent->cmd; - ent->cmd = NULL; - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - spin_unlock(&queue->lock); - - io_uring_cmd_done(cmd, 0, issue_flags); + fuse_uring_send(ent, ent->cmd, 0, issue_flags); return 0; } @@ -1041,7 +1082,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, err = fuse_uring_copy_from_ring(ring, req, ent); out: - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end_deferred(ent, req, err, issue_flags); } /* @@ -1459,21 +1500,6 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) return -EIOCBQUEUED; } -static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, - ssize_t ret, unsigned int issue_flags) -{ - struct fuse_ring_queue *queue = ent->queue; - - spin_lock(&queue->lock); - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - ent->cmd = NULL; - spin_unlock(&queue->lock); - - trace_fuse_request_send(ent->fuse_req); - io_uring_cmd_done(cmd, ret, issue_flags); -} - /* * This prepares and sends the ring request in fuse-uring task context. * User buffers are not mapped yet - the application does not have permission @@ -1500,6 +1526,28 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) fuse_uring_send(ent, cmd, err, issue_flags); } +/* + * The request was already copied to the ring buffer in the submitter's + * context, only the io_uring cmd completion is left to do. + * io_uring_cmd_done() must not run in the submitter's context as it would + * have to take ctx->uring_lock (io_uring_cmd_del_cancelable()) - a mutex + * the ring task holds across its whole submission path and frequently gets + * preempted under while the just-woken submitter runs. + */ +static void fuse_uring_send_prepared_in_task(struct io_tw_req tw_req, + io_tw_token_t tw) +{ + unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + int err = 0; + + if (unlikely(tw.cancel)) + err = -ECANCELED; + + fuse_uring_send(ent, cmd, err, issue_flags); +} + static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, bool background) { @@ -1599,7 +1647,9 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg) IO_URING_F_UNLOCKED); return; } - fuse_uring_send(ent, cmd, 0, IO_URING_F_UNLOCKED); + uring_cmd_set_ring_ent(cmd, ent); + io_uring_cmd_complete_in_task(cmd, + fuse_uring_send_prepared_in_task); } } diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 4518990e98bdd5..667f668ef3277f 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -231,10 +231,6 @@ static inline bool fuse_uring_request_expired(struct fuse_conn *fc) return false; } -static inline bool fuse_uring_request_expired(struct fuse_conn *fc) -{ -} - static inline void fuse_uring_flush_bg(struct fuse_conn *fc) { } diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 05c081ca0ca59d..b19ec2669f9d03 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -28,6 +28,8 @@ module_param(allow_sys_admin_access, bool, 0644); MODULE_PARM_DESC(allow_sys_admin_access, "Allow users with CAP_SYS_ADMIN in initial userns to bypass allow_other access check"); +static void fuse_attr_to_statx(struct fuse_attr *attr, struct fuse_statx *sx, uint32_t mask); + struct dentry_bucket { struct rb_root tree; spinlock_t lock; @@ -373,6 +375,73 @@ static void fuse_lookup_init(struct fuse_conn *fc, struct fuse_args *args, args->out_args[0].value = outarg; } +/** + * fuse_do_lookupx - Perform a FUSE_LOOKUPX operation + * + * @ext_out: extended output argument structure + * @lookup_flags: lookup flags (e.g., FUSE_LOOKUPX_FOR_REVALIDATE) + */ +static int fuse_do_lookupx(struct fuse_mount *fm, u64 nodeid, + const struct qstr *name, + struct fuse_lookupx_out *ext_out, + uint32_t lookup_flags) +{ + struct fuse_conn *fc = fm->fc; + FUSE_ARGS(args); + struct fuse_lookupx_in inarg = { .lookup_flags = lookup_flags }; + int err; + + memset(ext_out, 0, sizeof(*ext_out)); + args.nodeid = nodeid; + + if (!fc->lookupx) + goto fallback; + + args.opcode = FUSE_LOOKUPX; + args.in_numargs = 4; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = 0; + args.in_args[1].value = NULL; + args.in_args[2].size = name->len; + args.in_args[2].value = name->name; + args.in_args[3].size = 1; + args.in_args[3].value = ""; + args.out_numargs = 1; + args.out_args[0].size = sizeof(struct fuse_lookupx_out); + args.out_args[0].value = ext_out; + + err = fuse_simple_request(fm, &args); + if (err) { + if (err == -ENOSYS) { + fc->lookupx = 0; + goto fallback; + } + return err; + } + + return 0; + +fallback: + args.opcode = FUSE_LOOKUP; + args.in_numargs = 3; + fuse_set_zero_arg0(&args); + args.in_args[1].size = name->len; + args.in_args[1].value = name->name; + args.in_args[2].size = 1; + args.in_args[2].value = ""; + args.out_numargs = 1; + args.out_args[0].size = sizeof(struct fuse_entry_out); + args.out_args[0].value = &ext_out->entry; + + err = fuse_simple_request(fm, &args); + if (err) + return err; + + ext_out->mask = STATX_BASIC_STATS; + return 0; +} + /* * Check whether the dentry is still valid * @@ -400,10 +469,11 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, goto invalid; else if (time_before64(fuse_dentry_time(entry), get_jiffies_64()) || (flags & (LOOKUP_EXCL | LOOKUP_REVAL | LOOKUP_RENAME_TARGET))) { - struct fuse_entry_out outarg; - FUSE_ARGS(args); + struct fuse_lookupx_out ext_out; + struct fuse_statx sx; struct fuse_forget_link *forget; u64 attr_version; + uint32_t lookupx_flags = FUSE_LOOKUPX_FOR_REVALIDATE; /* For negative dentries, always do a fresh lookup */ if (!inode) @@ -421,19 +491,21 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, goto out; attr_version = fuse_get_attr_version(fm->fc); + if (S_ISDIR(inode->i_mode)) + lookupx_flags |= FUSE_LOOKUPX_TARGET_WAS_DIR; - fuse_lookup_init(fm->fc, &args, get_node_id(dir), - name, &outarg); - ret = fuse_simple_request(fm, &args); + ret = fuse_do_lookupx(fm, get_node_id(dir), + name, &ext_out, + lookupx_flags); /* Zero nodeid is same as -ENOENT */ - if (!ret && !outarg.nodeid) + if (!ret && !ext_out.entry.nodeid) ret = -ENOENT; if (!ret) { fi = get_fuse_inode(inode); - if (outarg.nodeid != get_node_id(inode) || - (bool) IS_AUTOMOUNT(inode) != (bool) (outarg.attr.flags & FUSE_ATTR_SUBMOUNT)) { + if (ext_out.entry.nodeid != get_node_id(inode) || + (bool) IS_AUTOMOUNT(inode) != (bool) (ext_out.entry.attr.flags & FUSE_ATTR_SUBMOUNT)) { fuse_queue_forget(fm->fc, forget, - outarg.nodeid, 1); + ext_out.entry.nodeid, 1); goto invalid; } spin_lock(&fi->lock); @@ -443,15 +515,16 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, kfree(forget); if (ret == -ENOMEM || ret == -EINTR) goto out; - if (ret || fuse_invalid_attr(&outarg.attr) || - fuse_stale_inode(inode, outarg.generation, &outarg.attr)) + if (ret || fuse_invalid_attr(&ext_out.entry.attr) || + fuse_stale_inode(inode, ext_out.entry.generation, &ext_out.entry.attr)) goto invalid; forget_all_cached_acls(inode); - fuse_change_attributes(inode, &outarg.attr, NULL, - ATTR_TIMEOUT(&outarg), + fuse_attr_to_statx(&ext_out.entry.attr, &sx, ext_out.mask); + fuse_change_attributes(inode, &ext_out.entry.attr, &sx, + ATTR_TIMEOUT(&ext_out.entry), attr_version); - fuse_change_entry_timeout(entry, &outarg); + fuse_change_entry_timeout(entry, &ext_out.entry); } else if (inode) { fi = get_fuse_inode(inode); if (flags & LOOKUP_RCU) { @@ -846,6 +919,10 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, memset(&inarg, 0, sizeof(inarg)); memset(&outentry, 0, sizeof(outentry)); inarg.flags = flags; + + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; inarg.mode = mode; inarg.umask = current_umask(); @@ -1411,8 +1488,35 @@ static void fuse_statx_to_attr(struct fuse_statx *sx, struct fuse_attr *attr) attr->blksize = sx->blksize; } +static void fuse_attr_to_statx(struct fuse_attr *attr, struct fuse_statx *sx, uint32_t mask) +{ + memset(sx, 0, sizeof(*sx)); + sx->mask = mask; + sx->ino = attr->ino; + sx->size = attr->size; + sx->blocks = attr->blocks; + sx->atime.tv_sec = attr->atime; + sx->mtime.tv_sec = attr->mtime; + sx->ctime.tv_sec = attr->ctime; + sx->atime.tv_nsec = attr->atimensec; + sx->mtime.tv_nsec = attr->mtimensec; + sx->ctime.tv_nsec = attr->ctimensec; + sx->mode = attr->mode; + sx->nlink = attr->nlink; + sx->uid = attr->uid; + sx->gid = attr->gid; + sx->rdev_major = MAJOR(attr->rdev); + sx->rdev_minor = MINOR(attr->rdev); + sx->blksize = attr->blksize; +} + +/* + * @param sx_mask request mask send to to fuse-server + * @param mandatory_sx_mask subset of (or complete) sx_mask that the server + * has to fulfill +*/ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, - struct file *file, struct kstat *stat) + struct file *file, struct kstat *stat, u32 sx_mask, u32 mandatory_sx_mask) { int err; struct fuse_attr attr; @@ -1423,6 +1527,12 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, u64 attr_version = fuse_get_attr_version(fm->fc); FUSE_ARGS(args); + /* + * mandatory_sx_mask should be a subset of sx_mask. + * If it's not, we have a logic error somewhere in the call chain. + */ + WARN_ON_ONCE((mandatory_sx_mask & sx_mask) != mandatory_sx_mask); + memset(&inarg, 0, sizeof(inarg)); memset(&outarg, 0, sizeof(outarg)); /* Directories have separate file-handle space */ @@ -1432,9 +1542,12 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, inarg.getattr_flags |= FUSE_GETATTR_FH; inarg.fh = ff->fh; } - /* For now leave sync hints as the default, request all stats. */ + /* + * For permission checks, we only need mode, uid, gid. + * This is an optimization to avoid fetching all stats when not needed. + */ inarg.sx_flags = 0; - inarg.sx_mask = STATX_BASIC_STATS | STATX_BTIME; + inarg.sx_mask = sx_mask; args.opcode = FUSE_STATX; args.nodeid = get_node_id(inode); args.in_numargs = 1; @@ -1448,6 +1561,17 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, return err; sx = &outarg.stat; + + /* + * Verify the server returned at least what we requested. + * The server may return more attributes than requested (which is fine), + * but must not return fewer. + */ + if ((sx->mask & mandatory_sx_mask) != mandatory_sx_mask) { + fuse_make_bad(inode); + return -EIO; + } + if (((sx->mask & STATX_SIZE) && !fuse_valid_size(sx->size)) || ((sx->mask & STATX_TYPE) && (!fuse_valid_type(sx->mode) || inode_wrong_type(inode, sx->mode)))) { @@ -1456,7 +1580,7 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, } fuse_statx_to_attr(&outarg.stat, &attr); - if ((sx->mask & STATX_BASIC_STATS) == STATX_BASIC_STATS) { + if (sx->mask & STATX_BASIC_STATS) { fuse_change_attributes(inode, &attr, &outarg.stat, ATTR_TIMEOUT(&outarg), attr_version); } @@ -1521,30 +1645,31 @@ static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode, bool sync; u32 inval_mask = READ_ONCE(fi->inval_mask); u32 cache_mask = fuse_get_cache_mask(inode); - + u32 mandatory_sx_mask = request_mask & STATX_BASIC_STATS; + u32 sx_mask = request_mask; /* FUSE only supports basic stats and possibly btime */ - request_mask &= STATX_BASIC_STATS | STATX_BTIME; + sx_mask &= STATX_BASIC_STATS | STATX_BTIME; retry: if (fc->no_statx) - request_mask &= STATX_BASIC_STATS; + sx_mask &= STATX_BASIC_STATS; - if (!request_mask) + if (!sx_mask) sync = false; else if (flags & AT_STATX_FORCE_SYNC) sync = true; else if (flags & AT_STATX_DONT_SYNC) sync = false; - else if (request_mask & inval_mask & ~cache_mask) + else if (sx_mask & inval_mask & ~cache_mask) sync = true; else sync = time_before64(fi->i_time, get_jiffies_64()); if (sync) { forget_all_cached_acls(inode); - /* Try statx if BTIME is requested */ - if (!fc->no_statx && (request_mask & ~STATX_BASIC_STATS)) { - err = fuse_do_statx(idmap, inode, file, stat); + if (!fc->no_statx) { + err = fuse_do_statx(idmap, inode, file, stat, sx_mask, + mandatory_sx_mask); if (err == -ENOSYS) { fc->no_statx = 1; err = 0; @@ -1554,7 +1679,7 @@ static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode, err = fuse_do_getattr(idmap, inode, stat, file); } } else if (stat) { - generic_fillattr(idmap, request_mask, inode, stat); + generic_fillattr(idmap, sx_mask, inode, stat); stat->mode = fi->orig_i_mode; stat->ino = fi->orig_ino; stat->blksize = 1 << fi->cached_i_blkbits; @@ -1721,13 +1846,14 @@ static int fuse_access(struct inode *inode, int mask) return err; } -static int fuse_perm_getattr(struct inode *inode, int mask) +static int fuse_perm_getattr(struct inode *inode, int mask, int perm_mask) { if (mask & MAY_NOT_BLOCK) return -ECHILD; forget_all_cached_acls(inode); - return fuse_do_getattr(&nop_mnt_idmap, inode, NULL, NULL); + return fuse_update_get_attr(&nop_mnt_idmap, inode, NULL, NULL, perm_mask, + AT_STATX_FORCE_SYNC); } /* @@ -1749,6 +1875,7 @@ static int fuse_permission(struct mnt_idmap *idmap, struct fuse_conn *fc = get_fuse_conn(inode); bool refreshed = false; int err = 0; + int perm_mask = STATX_MODE | STATX_UID | STATX_GID; if (fuse_is_bad(inode)) return -EIO; @@ -1762,13 +1889,12 @@ static int fuse_permission(struct mnt_idmap *idmap, if (fc->default_permissions || ((mask & MAY_EXEC) && S_ISREG(inode->i_mode))) { struct fuse_inode *fi = get_fuse_inode(inode); - u32 perm_mask = STATX_MODE | STATX_UID | STATX_GID; if (perm_mask & READ_ONCE(fi->inval_mask) || - time_before64(fi->i_time, get_jiffies_64())) { + time_before64(fi->i_perm_time, get_jiffies_64())) { refreshed = true; - err = fuse_perm_getattr(inode, mask); + err = fuse_perm_getattr(inode, mask, perm_mask); if (err) return err; } @@ -1781,7 +1907,7 @@ static int fuse_permission(struct mnt_idmap *idmap, attributes. This is also needed, because the root node will at first have no permissions */ if (err == -EACCES && !refreshed) { - err = fuse_perm_getattr(inode, mask); + err = fuse_perm_getattr(inode, mask, perm_mask); if (!err) err = generic_permission(idmap, inode, mask); @@ -1798,7 +1924,7 @@ static int fuse_permission(struct mnt_idmap *idmap, if (refreshed) return -EACCES; - err = fuse_perm_getattr(inode, mask); + err = fuse_perm_getattr(inode, mask, perm_mask); if (!err && !(inode->i_mode & S_IXUGO)) return -EACCES; } @@ -2176,35 +2302,24 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. * - * Revoke and drop under the coherency gate write side, - * like the NOTIFY invalidate path: a gate reader that - * already re-validated its grant must not have the - * lock tree and the cache yanked mid-hold, or it - * would repopulate the truncated range trusting a - * grant that no longer exists. Waiting for gate - * readers here is safe: we hold i_rwsem exclusive, so - * no gate holder can be waiting on it (the write path - * takes i_rwsem before the gate, the read path never - * takes it). + * Dropping every grant here does not need a reader or + * writer fenced out: truncate_pagecache() discards the + * folios rather than writing them, and a write racing + * this is a write racing an O_TRUNC open, which has no + * order to preserve. i_rwsem is held exclusive + * anyway, so no cached write is in progress. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); spin_lock(&fi->lock); - fi->server_size = 0; i_size_write(inode, 0); spin_unlock(&fi->lock); truncate_pagecache(inode, 0); - if (wb_sem) - percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2295,13 +2410,6 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, /* see the comment in fuse_change_attributes() */ if (!is_wb || is_truncate) i_size_write(inode, outarg.attr.size); - /* - * A truncate settles the size on the server; only shrink the - * server-materialized bound: growing just exposes zeros, which the - * bound need not cover (see fuse_iomap_read_folio_range()). - */ - if (is_truncate && (loff_t) outarg.attr.size < fi->server_size) - fi->server_size = outarg.attr.size; if (is_truncate) { /* NOTE: this may release/reacquire fi->lock */ @@ -2315,23 +2423,17 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* - * Revoke and drop under the coherency gate write side; see - * the atomic-O_TRUNC branch above. i_rwsem is held - * exclusive here as well (setattr), so waiting out gate - * readers cannot deadlock. + * Revoke past the new size and drop what is beyond it; see + * the atomic-O_TRUNC branch above for why this needs nothing + * fenced out. i_rwsem is held exclusive here as well. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); + fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, + U64_MAX); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); - if (wb_sem) - percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f019dccaeff6d2..0f616c962cce81 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,17 @@ static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) inarg.flags &= ~O_TRUNC; + /* + * With the writeback cache the kernel owns append positioning: + * writeback sends FUSE_WRITE with explicit offsets, and a server + * that opens its backing file O_APPEND has pwrite(2) ignore them + * (Linux appends regardless of offset). Any re-sent or reordered + * run is then placed at EOF: duplicated data and a growing file. + * Do not hand the flag to the server at all. + */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -176,6 +188,10 @@ static int fuse_compound_open_getattr(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) open_in.flags &= ~O_TRUNC; + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + open_in.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (open_in.flags & O_TRUNC) && !capable(CAP_FSETID)) open_in.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -251,19 +267,19 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, &attr_outarg, outargp); if (err == -ENOSYS) fc->compound_open_getattr = 0; - if (!err) + if (!err) fuse_change_attributes(inode, &attr_outarg.attr, - NULL, - ATTR_TIMEOUT(&attr_outarg), - fuse_get_attr_version(fc)); - } - if (err == -ENOSYS) { + NULL, + ATTR_TIMEOUT(&attr_outarg), + fuse_get_attr_version(fc)); + } + if (err == -ENOSYS) { err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); - if (!err) { + if (!err) { ff->fh = outargp->fh; ff->open_flags = outargp->open_flags; - } - } + } + } if (err) { if (err != -ENOSYS) { @@ -346,7 +362,6 @@ static void fuse_truncate_update_attr(struct inode *inode, struct file *file) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); - fi->server_size = 0; i_size_write(inode, 0); spin_unlock(&fi->lock); file_update_time(file); @@ -397,10 +412,26 @@ static int fuse_open(struct inode *inode, struct file *file) if (is_wb_truncate || dax_truncate) fuse_release_nowrite(inode); if (!err) { - if (is_truncate) + if (is_truncate) { + /* + * Every grant goes with the cache, as on the + * fuse_do_setattr() O_TRUNC path: a record left + * behind would keep naming bytes the folios no + * longer hold. i_rwsem is held exclusive + * (is_wb_truncate), so no cached write is mid-record. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); - else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { + /* + * Write back first: the drop launders whatever it + * finds dirty a folio at a time, a FUSE_WRITE per + * page, where writeback batches the same bytes. + */ + filemap_write_and_wait(inode->i_mapping); invalidate_inode_pages2(inode->i_mapping); + } } if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); @@ -422,17 +453,23 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, /* Inode is NULL on error path of fuse_create_open() */ if (likely(fi)) { + bool writer; + spin_lock(&fi->lock); + writer = !list_empty(&ff->write_entry); list_del(&ff->write_entry); /* * Leave forced direct IO mode once the last writer is gone: with * no local writer left there is no cached-write contention with * the remote modifier that triggered the switch. Restore - * FUSE_I_CACHE_IO_MODE for any frozen cached opens. + * FUSE_I_CACHE_IO_MODE for any frozen cached opens. A reader + * closing ends nothing: an inode latched with no writer at all + * leaves on the cold check in fuse_force_dio_active(). */ - if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + if (writer && test_bit(FUSE_I_FORCE_DIO, &fi->state) && list_empty(&fi->write_files)) { clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); } @@ -468,6 +505,27 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, ra->inode = sync ? NULL : igrab(&fi->inode); } +/* + * Drop the page cache of an inode leaving the forced direct IO latch. + * + * Anything still dirty here is what a write racing the latch left behind, since + * the drain empties the mapping and nothing dirties it while it is on. Send + * that with writeback rather than leave it to the drop: invalidate_inode_pages2() + * launders a folio at a time, a FUSE_WRITE per page for bytes writeback batches + * into max_write requests. Errors stay on the mapping for fsync to collect. + * + * The usual case is an empty or clean mapping, where both calls are a load and + * the drop finds nothing to write. + */ +static void fuse_force_dio_drop(struct address_space *mapping) +{ + if (filemap_range_needs_writeback(mapping, 0, LLONG_MAX)) { + filemap_fdatawrite(mapping); + filemap_fdatawait_keep_errors(mapping); + } + invalidate_inode_pages2(mapping); +} + void fuse_file_release(struct inode *inode, struct fuse_file *ff, unsigned int open_flags, fl_owner_t id, bool isdir) { @@ -482,15 +540,14 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * If this release dropped the last writer, fuse_prepare_release() * cleared the forced-direct-IO latch (under fi->lock). Drop any clean * folios a read racing the latch may have repopulated so they cannot be - * served stale once caching mode resumes. No inode lock or - * wb_inval_rwsem: release may run on the fuse server thread (async fput + * served stale once caching mode resumes. No inode lock: release may + * run on the fuse server thread (async fput * from aio completion), where blocking on a contended inode lock could - * stall the connection. Writes were routed direct while latched, so - * only clean folios exist and this invalidate is server-free; the last - * writer is gone, so no forced-dio writer can race the drop. + * stall the connection. The last writer is gone, so no forced-dio writer + * can race the drop. */ if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state)) - invalidate_inode_pages2(inode->i_mapping); + fuse_force_dio_drop(inode->i_mapping); if (ra && ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; @@ -674,6 +731,16 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end, if (fuse_is_bad(inode)) return -EIO; + /* + * Get the sending out of the way before the lock. It is the long + * part: a pass takes back every grant it finds gone, a cluster round + * trip each, and cached writers hold i_rwsem shared + * (fuse_cache_wr_exclusive_lock()) and would all wait behind it. + * Errors are left to the pass below, which collects them from the + * mapping. + */ + filemap_fdatawrite_range(file->f_mapping, start, end); + inode_lock(inode); /* @@ -1020,6 +1087,107 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, return 0; } +/** + * fuse_read_folio_range - read part of a folio from the server + * @file: file to read through + * @folio: the folio to fill + * @off: offset within @folio to start at + * @len: bytes to read + * + * fuse_do_readfolio() cannot serve a partial folio: it asks for + * page_zeroing, and fuse_copy_folio() answers that by zeroing the whole + * folio whenever the request covers less than all of it. Ask without it + * and zero exactly what the reply left short, which is the server saying + * the file ends there. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_range(struct file *file, struct folio *folio, + size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = folio_pos(folio) + off; + struct fuse_folio_desc desc = { + .offset = off, + .length = len, + }; + struct fuse_io_args ia = { + .ap.args.out_pages = true, + .ap.num_folios = 1, + .ap.folios = &folio, + .ap.descs = &desc, + }; + ssize_t res; + + /* Don't overflow end offset */ + if (pos + (desc.length - 1) == LLONG_MAX) + desc.length--; + + fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); + res = fuse_simple_request(fm, &ia.ap.args); + if (res < 0) { + /* See fuse_do_readfolio() for why READ can return -EDEADLK */ + if ((res == -EDEADLK || res == -EAGAIN) && fm->fc->dlm) + res = AOP_TRUNCATED_PAGE; + return res; + } + + if (res < desc.length) + folio_zero_range(folio, off + res, desc.length - res); + + return 0; +} + +/** + * fuse_read_folio_merge - fill @folio without disturbing what it holds + * @file: file to read through + * @folio: the folio to fill + * + * iomap tracks a folio a block at a time, and a write that covered some + * of its blocks and not others leaves it valid in the ones it covered + * and not in the rest. The valid ones hold what that write put there, + * which writeback may not have sent yet; reading over them would lose + * it. Fetch the rest, in as few requests as the gaps allow. + * + * A folio the page cache tracks in one piece has no per block state to + * ask, and none to have: it is dirty only if a write covered it whole, + * and then it is valid and never reaches here. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_merge(struct file *file, struct folio *folio) +{ + size_t bsize = i_blocksize(folio->mapping->host); + size_t size = folio_size(folio); + size_t off = 0; + + while (off < size) { + size_t run = 0; + int err; + + /* Skip what the folio already holds */ + while (off < size && + iomap_is_partially_uptodate(folio, off, bsize)) + off += bsize; + + /* Take the gap behind it in one request */ + while (off + run < size && + !iomap_is_partially_uptodate(folio, off + run, bsize)) + run += bsize; + + if (!run) + break; + + err = fuse_read_folio_range(file, folio, off, run); + if (err) + return err; + off += run; + } + + return 0; +} + static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) @@ -1047,8 +1215,8 @@ struct fuse_fill_read_data { static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, unsigned len, struct fuse_args_pages *ap, unsigned cur_bytes, bool write); -static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, - unsigned int count, bool async); +static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file, + unsigned int count, bool async); static int fuse_handle_readahead(struct folio *folio, struct readahead_control *rac, @@ -1135,9 +1303,129 @@ static const struct iomap_read_ops fuse_iomap_read_ops = { .submit_read = fuse_iomap_read_submit, }; +/** + * fuse_read_grant - take the grant a page cache fill runs under + * @file: file to read through + * @pos: byte offset the read starts at + * @count: bytes the read asks for + * + * ->read_folio and ->readahead are entered with the folios they fill + * already locked, and no grant may be asked for under a page lock + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * A read asks here, before anything is locked, and the fill paths only + * confirm what this took. + * + * Readahead fills past the end of the read, so ask for what it will fill + * as well, bounded by the file since readahead stops there. A run of + * folios no grant covers is given back unfilled and fetched one folio at + * a time, so what is asked for here is what readahead is worth. + * + * That is two windows, not one. A read landing on the marker of the + * current window [s, s + W) has the next one, [s + W, s + 2W), submitted + * on its behalf (page_cache_async_ra()), and with a grant of one window + * past the read that next window ends up to W - count beyond the grant. + * With a server that grants exactly what is asked, nearly all of it was + * declined and refetched a folio at a time: 29024 of 32768 pages of a + * 32k-record stream, at 85 MiB/s against 2 GB/s. + * + * And what is needed is not what is asked for. The need moves with + * every read, so a grant sized to it is never wide enough for the next + * read and a server that grants exactly is asked once per read(2): 4096 + * round trips for 128 MiB of 32k records, at 500 MiB/s. Ask only when + * the need is not already held, and then for two windows more than it, + * so the grant carries the reads across them before the next request. + * A server that grants wider is asked no more often than before. + * + * Return: what fuse_get_dlm_lock() returned, 0 when there is nothing to + * ask for or the need is already held. A caller filling through the page + * cache can drop it: the fill confirms the grant itself and declines what + * this did not take, so the failure arrives there with a folio to report + * it on. The two that cannot fall back on that act on it instead -- + * fuse_read_folio_retry(), which would come straight back here, and + * fuse_fadvise(), which has nothing left to populate. + */ +static int fuse_read_grant(struct file *file, loff_t pos, size_t count) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + loff_t size = i_size_read(inode); + loff_t ahead = 2 * ((loff_t)file->f_ra.ra_pages << PAGE_SHIFT); + loff_t need = pos + count; + loff_t want; + + if (!fc->writeback_cache || !fc->dlm) + return 0; + + if (need < size) + need += min(ahead, size - need); + + if (need <= pos) + return 0; + + if (fuse_dlm_lock_is_held(fi, pos, need - pos, FUSE_PAGE_LOCK_READ)) + return 0; + + want = need; + if (want < size) + want += min(ahead, size - want); + + return fuse_get_dlm_lock(file, pos, want - pos, FUSE_PAGE_LOCK_READ); +} + +/** + * fuse_read_folio_retry - back off a fill with no grant to run under + * @file: file to read through + * @folio: the folio handed over locked, unlocked here + * @pos: byte offset of @folio + * @len: its size in bytes + * + * Neither reason a fill is refused can be dealt with while the folio is + * held: waiting a revoke out would hold the page cache that revoke is + * about to drop, and no grant may be asked for under a page lock at all + * (Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt). + * Unlock, do both, and send the caller round again to find the range + * covered. + * + * Return: AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_retry(struct file *file, struct folio *folio, + loff_t pos, size_t len) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + struct fuse_dlm_span pin; + int err; + + folio_unlock(folio); + + /* Wait the revoke out; what it leaves behind is asked for below */ + fuse_dlm_pin(fi, &pin, pos, len); + fuse_dlm_unpin(fi); + + err = fuse_read_grant(file, pos, len); + if (err == -ENOSYS) + return AOP_TRUNCATED_PAGE; + if (err < 0) + return err; + /* + * Granted but unrecorded, so the retry finds the range uncovered + * and comes straight back here. Report it rather than spin. + */ + if (err > 0) + return -ENOMEM; + + return AOP_TRUNCATED_PAGE; +} + static int fuse_read_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + struct fuse_dlm_span pin; + bool pinned = false; struct fuse_fill_read_data data = { .file = file, }; @@ -1145,7 +1433,6 @@ static int fuse_read_folio(struct file *file, struct folio *folio) .cur_folio = folio, .ops = &fuse_iomap_read_ops, .read_ctx = &data, - }; if (fuse_is_bad(inode)) { @@ -1153,54 +1440,96 @@ static int fuse_read_folio(struct file *file, struct folio *folio) return -EIO; } + /* + * Writeback unlocks a folio as soon as it has handed it over, with + * the writeback flag still on it, so this can be reached while a + * FUSE_WRITE is still reading out of it. Filling it now would + * rewrite what is being sent, and past the end of the file the reply + * comes back short and zeroes it. Nothing reached here before a + * partial write started leaving folios invalid, because a dirty + * folio was always valid and never came this way. + */ + folio_wait_writeback(folio); + + /* + * The grant the folio is filled under, held from the confirmation + * until the bytes are in the page cache. What lands here is served + * to every later reader of the file, so it must neither be fetched + * under a grant a revoke has taken away nor be dropped into a range + * a revoke has just swept: such a folio is uptodate and covered by + * nothing, and no further notify comes for a lock this client no + * longer holds. + * + * The pin closes the second, since a revoke over the folio waits + * for the fill and drops the folio after it; the confirmation + * closes the first. Both fail into fuse_read_folio_retry(). + */ + if (fc->dlm && fc->writeback_cache) { + pinned = fuse_dlm_trypin_held(fi, &pin, pos, len, + FUSE_PAGE_LOCK_READ); + if (!pinned) + return fuse_read_folio_retry(file, folio, pos, len); + } + + /* + * Only a folio still holding what a write put in it has anything to + * keep. One that was merely being written back is clean by now, + * waited out just above, and reads whole. + * + * The gaps are read one run at a time rather than through iomap, + * which owns the whole folio and would fill what is kept here. + */ + if (fc->writeback_cache && folio_test_dirty(folio)) { + int err = fuse_read_folio_merge(file, folio); + + if (!err) + folio_mark_uptodate(folio); + + fuse_invalidate_atime(inode); + folio_unlock(folio); + /* + * After the unlock, so a revoke draining this pin finds the + * folio it has to drop unlocked and takes it out. + */ + if (pinned) + fuse_dlm_unpin(fi); + return err; + } + iomap_read_folio(&fuse_iomap_ops, &ctx, NULL); fuse_invalidate_atime(inode); + if (pinned) + fuse_dlm_unpin(fi); return 0; } +/* + * What iomap_file_buffered_write() carries for fuse, reached from the + * read-back callback as iter->private. @file is what that callback needs + * anyway, so the retry flag rides along and lives exactly as long as the + * call, with nothing to allocate or free. + */ +struct fuse_iomap_write_ctx { + struct file *file; + /* fuse_iomap_read_folio_range() hit AOP_TRUNCATED_PAGE */ + bool retry_needed; + /* + * The grant ->get_folio pinned, live until ->put_folio drops it. + * Here rather than on their stacks, which do not span the pair. + */ + struct fuse_dlm_span pin; +}; + static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, struct folio *folio, loff_t pos, size_t len) { - struct file *file = iter->private; - struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_iomap_write_ctx *ctx = iter->private; + struct file *file = ctx->file; size_t off = offset_in_folio(folio, pos); - bool hole; int ret; - /* - * Expanding writes claim their new i_size up front (see - * fuse_cache_write_iter()), which keeps iomap's own beyond-EOF - * zeroing in iomap_block_needs_zeroing() from ever firing for the - * write's own range: every block of a file expansion would be read - * from the server although it cannot contain data. Zero-fill - * locally instead when the server is known to hold no data in the - * range and we hold the DLM write lock covering it: - * - * - fi->server_size bounds the data materialized on the server - * (writeback and direct write acknowledgements, server - * attributes), - * - local data not yet acknowledged sits in uptodate blocks, which - * iomap never passes to this callback, - * - the page-granular DLM write lock excludes data written by - * other nodes, re-checked against the live lock tree so a - * revoked lock falls back to reading. - */ - if (fc->dlm) { - spin_lock(&fi->lock); - hole = pos >= fi->server_size; - spin_unlock(&fi->lock); - - if (hole && fuse_dlm_range_is_locked(fi, pos, pos + len - 1, - FUSE_PAGE_LOCK_WRITE)) { - folio_zero_range(folio, off, len); - return 0; - } - } - - ret = fuse_do_readfolio(file, folio, off, len); + ret = fuse_read_folio_range(file, folio, off, len); /* * TEMPORARY WORKAROUND for iomap write deadlock: @@ -1211,7 +1540,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * * However, iomap doesn't understand AOP_TRUNCATED_PAGE. * We need to: - * 1. Mark the retry flag (caller stored it in xarray) + * 1. Mark the retry flag on the caller's write context * 2. Convert to -EAGAIN so iomap sees an error * 3. Let fuse_cache_write_iter() detect and retry * @@ -1222,13 +1551,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE support. */ if (ret == AOP_TRUNCATED_PAGE) { - struct fuse_dlm_retry *retry; - unsigned long task_key = (unsigned long)current; - - retry = xa_load(&fc->dlm_retry_tasks, task_key); - if (retry) { - retry->retry_needed = true; - } + ctx->retry_needed = true; /* Convert to -EAGAIN for iomap */ ret = -EAGAIN; @@ -1265,19 +1588,41 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, ap->descs[i].length, err); folio_put(ap->folios[i]); } + + /* + * Dropped after the folios, which are filled, uptodate and unlocked + * by now: a revoke draining this pin finds them and takes them out. + * Held until here so it cannot have swept before they were there. + */ + if (ia->read.dlm_fi) + fuse_dlm_unpin_span(ia->read.dlm_fi, &ia->read.dlm_pin); + if (ia->ff) fuse_file_put(ia->ff, false); fuse_io_free(ia); } -static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, - unsigned int count, bool async) +/** + * fuse_send_readpages - read a run of folios of a read or readahead window + * @ia: the request, owning the folios and the pin over them + * @file: file to read through + * @count: bytes to read, starting at the first folio + * @async: send in the background instead of waiting for the reply + * + * Return: 0 once the request is on its way or has been completed, + * -EAGAIN when a revoke of the range refused the grant and nothing was + * sent. The folios are given back either way. + */ +static int fuse_send_readpages(struct fuse_io_args *ia, struct file *file, + unsigned int count, bool async) { struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); struct fuse_args_pages *ap = &ia->ap; loff_t pos = folio_pos(ap->folios[0]); + unsigned int i; ssize_t res; int err; @@ -1292,6 +1637,25 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, } WARN_ON((loff_t) (pos + count) < 0); + /* + * The grant the read took in fuse_read_grant(), confirmed under a + * pin and held until the reply has filled the folios. A revoke of the range + * waits for that, so the reply cannot be fetched under a grant the + * server has since handed on, and cannot land behind a sweep that + * would leave the folios uptodate and covered by nothing. + * + * Refused, or gone since it was asked for: give the folios back + * unfilled rather than serve what no lock covers. The read that + * wanted them comes back through fuse_read_folio(), which asks + * again with no folio held. + */ + if (fm->fc->dlm && fm->fc->writeback_cache) { + if (!fuse_dlm_trypin_held_span(fi, &ia->read.dlm_pin, pos, + count, FUSE_PAGE_LOCK_READ)) + goto uncovered; + ia->read.dlm_fi = fi; + } + fuse_read_args_fill(ia, file, pos, count, FUSE_READ); ia->read.attr_ver = fuse_get_attr_version(fm->fc); if (async) { @@ -1299,12 +1663,21 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ap->args.end = fuse_readpages_end; err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (!err) - return; + return 0; } else { res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } fuse_readpages_end(fm, &ap->args, err); + return 0; + +uncovered: + for (i = 0; i < ap->num_folios; i++) { + folio_end_read(ap->folios[i], false); + folio_put(ap->folios[i]); + } + fuse_io_free(ia); + return -EAGAIN; } static void fuse_readahead(struct readahead_control *rac) @@ -1324,26 +1697,129 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; + /* + * A latched inode keeps no page cache, and this window was not asked + * for by a read that has to be served (madvise(), fadvise()). Decline + * it: read_pages() drops the folios of a window left unfilled. + */ + if (fuse_inode_force_dio(inode)) + return; + + /* + * Readahead fills the page cache past the range the reader asked + * for, and what lands there has to be covered by a grant: folios + * the server handed out no lock for are folios it will not revoke + * when a remote node writes them, and a later read would be served + * from stale cache. + * + * No grant is asked for here. ->readahead is entered with every + * folio of the window already locked, and none may be asked for + * under a page lock; the read this window belongs to took one over + * it in fuse_read_grant(), before the page cache was entered. + * + * What that left uncovered fuse_send_readpages() declines, one run + * of folios at a time, and those folios go back unfilled for + * fuse_read_folio() to fetch with no folio held. + */ + iomap_readahead(&fuse_iomap_ops, &ctx, NULL); } static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); /* - * Bound on re-requesting a revoked DLM grant before a cached read is - * served unlocked; see fuse_cache_read_iter(). + * Empty the page cache of an inode just latched into direct IO. + * + * The notify that set the latch dropped the mapping, but it holds no inode + * lock, so a cached write that had already passed both latch checks went on + * dirtying folios behind it. Left there, writeback would put them on the + * server on top of the direct writes that replace them, and direct reads, + * which do not look in the page cache, would miss them entirely. + * + * i_rwsem taken exclusive is what settles it: fuse_cache_write_iter() dirties + * under it, so by the time it is held every such writer has finished, and none + * can start behind this one. A write that takes the lock afterwards rechecks + * the latch and reroutes before it touches the cache. So the mapping stays + * empty from here on and the bit records that, leaving the latched IO paths + * with nothing to flush. + * + * Called from the top of the IO paths, with no inode lock held. A no-op for + * a file the server itself opened direct, which is never latched. + */ +static int fuse_force_dio_drain(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + int err = 0; + + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state) || + test_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state)) + return 0; + + inode_lock(inode); + /* + * The latch may have gone while this waited for the lock, and the drain + * is only owed to one that is still on. On a flush error the bit stays + * clear so the next IO tries again, and the error goes to this caller + * rather than being left for a later fsync to find. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + !test_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state)) { + err = filemap_write_and_wait(inode->i_mapping); + if (!err) { + invalidate_inode_pages2(inode->i_mapping); + set_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); + } + } + inode_unlock(inode); + + return err; +} + +/* + * Fold one request size into a moving average of this inode's sizes and + * report whether the file is being streamed: the same buffer size arriving + * FUSE_STREAM_RUN times over, which is what a task working through a file a + * record at a time looks like from here. A size outside the tolerance around + * the average starts the run again from that size, so a task changing its + * record is followed rather than averaged with what it did before. + * + * The average is per inode rather than per handle, so a stream stays one + * stream across reopens and across the handles of a shared file, whose users + * are streaming it together without any one of them being sequential. + * + * A hint only, read and written without the inode lock, which the DLM path + * holds shared: callers landing on it together cost a misread run, not + * correctness. */ -#define FUSE_DLM_READ_RETRIES 3 +static bool fuse_stream_update(unsigned int *ewma, unsigned int *run, + size_t len) +{ + unsigned int sample = min_t(size_t, len, FUSE_STREAM_EWMA_MAX); + unsigned int avg = *ewma >> FUSE_STREAM_EWMA_SHIFT; + + if (*run && abs_diff(sample, avg) <= avg >> FUSE_STREAM_TOL_SHIFT) { + /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ + *ewma += sample - avg; + if (*run < FUSE_STREAM_RUN) + (*run)++; + } else { + *ewma = sample << FUSE_STREAM_EWMA_SHIFT; + *run = 1; + } + + return *run >= FUSE_STREAM_RUN; +} static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; - struct inode *inode = file->f_mapping->host; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + size_t count = iov_iter_count(to); + bool stream = false; ssize_t res; - int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1351,75 +1827,83 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * i_size is up to date). */ if (fc->auto_inval_data || - (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { + (iocb->ki_pos + count > i_size_read(inode))) { int err; err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE); if (err) return err; } - /* if we have dlm support acquire a read lock for the area - * we are reading from. */ - if (fc->writeback_cache && fc->dlm) - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); - /* - * Fence the cache-serving read against a NOTIFY invalidate so we never - * hand back a folio the server has just superseded. The gate read side - * is per-CPU cheap; the NOTIFY holds the write side with priority. - * Re-check the forced-DIO latch under it: if a storm latched us while we - * waited on a pending writer, reroute to direct like the buffered write - * path, so we do not repopulate the cache the latch just dropped. - * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + * Every read that could be cached feeds the size average, streamed or + * not: a reader changing its record has to be seen as well. O_DIRECT + * is not one of them, and an empty read says nothing about a record. */ - if (wb_sem) { - int tries = FUSE_DLM_READ_RETRIES; + if (count && !(iocb->ki_flags & IOCB_DIRECT)) + stream = fuse_stream_update(&fi->read_size_ewma, + &fi->read_stream_run, count); -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - return fuse_direct_read_iter(iocb, to); + /* + * A streamed read of FUSE_READ_STREAM_MIN or more is served into the + * caller's own pages instead. Cached, the bytes are copied twice on + * their way out of the server, into the folios and out of them into + * the caller, and the folios are dropped unread; served from here they + * are copied once, and the grant the fill would have run under is not + * asked for at all. What that gives up is the readahead of the next + * record, not the wait for this one. + * + * Not on a mapped file, which keeps its page cache either way, and not + * for IOCB_NOWAIT, which the direct read has no way to honour. A + * caller that gets EAGAIN out of the cached path here comes back + * without the flag and takes this branch on the retry. + * + * Dirty folios over the range have to reach the server first: a direct + * read does not look in the page cache, and fuse_direct_io() flushes + * only for a file opened FOPEN_DIRECT_IO. + */ + if (stream && count >= FUSE_READ_STREAM_MIN && + !(iocb->ki_flags & IOCB_NOWAIT) && !mapping_mapped(mapping)) { + loff_t end = iocb->ki_pos + count - 1; + + if (filemap_range_needs_writeback(mapping, iocb->ki_pos, end)) { + res = filemap_write_and_wait_range(mapping, iocb->ki_pos, + end); + if (res) + return res; } + return fuse_direct_read_iter(iocb, to); + } + + /* + * The grant this read and the readahead behind it fill under. Not + * for O_DIRECT, which takes no DLM lock at all: it fills no page + * cache, and what it reads is the server's to order. + */ + if (!(iocb->ki_flags & IOCB_DIRECT)) + fuse_read_grant(file, iocb->ki_pos, count); + + /* + * A NOTIFY invalidate racing this read drops the folios it + * supersedes, so the read either misses and refetches or returns + * data that was current when it was copied. What a read does leave + * behind is the page cache it fills, which must not outlast the + * grant it was fetched under; that is fenced where the filling + * happens, in fuse_read_folio() and fuse_send_readpages(), and the + * grant taken here is what they confirm. + */ + if (fuse_inode_force_dio(inode)) { /* - * The DLM lock was requested before entering the gate, and - * the NOTIFY invalidate we may just have waited on revokes - * locks under the gate write side. Re-check the grant here - * and re-request with the gate dropped, so a - * FUSE_DLM_WB_LOCK round trip never parks a pending - * invalidate behind our own gate hold. Once the check - * passes the lock cannot go away for the rest of the gate - * hold. A failed or unrecorded request falls through - * unlocked, as before: the retry is taken even then (the - * latch must be re-checked under the re-entered gate), so - * lock_err has to stay sticky across it -- seeded by the - * pre-gate request above -- or a grant that failed would - * be re-requested forever. The retry is also bounded: a - * remote writer can revoke each successful grant before - * the gate is re-entered, and a reader-only inode has no - * force-DIO latch to end such a storm, so after - * FUSE_DLM_READ_RETRIES re-requests the read is served - * unlocked rather than looping without bound. + * Latched between fuse_file_read_iter()'s check and this one, + * so the drain it would have done falls here. */ - if (!lock_err && fc->dlm && tries-- > 0 && - !fuse_dlm_lock_is_held(fi, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ)) { - percpu_up_read(wb_sem); - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); - goto retry; - } + res = fuse_force_dio_drain(inode); + if (res) + return res; + return fuse_direct_read_iter(iocb, to); } res = generic_file_read_iter(iocb, to); - if (wb_sem) - percpu_up_read(wb_sem); - return res; } @@ -1492,15 +1976,6 @@ bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); - if (written > 0 && S_ISREG(inode->i_mode)) { - /* - * The server acknowledged data up to @pos, keep the - * server-materialized bound in sync for the expansion - * zero-fill in fuse_iomap_read_folio_range(). - */ - if (pos > fi->server_size) - fi->server_size = pos; - } if (written > 0 && pos > inode->i_size) { i_size_write(inode, pos); ret = true; @@ -1569,7 +2044,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, { struct fuse_args_pages *ap = &ia->ap; struct fuse_conn *fc = get_fuse_conn(mapping->host); - unsigned offset = pos & (PAGE_SIZE - 1); size_t count = 0; unsigned int num; int err = 0; @@ -1596,7 +2070,13 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, if (mapping_writably_mapped(mapping)) flush_dcache_folio(folio); - folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; + /* + * From @pos, not carried across iterations: a write landing + * inside a folio it does not start covers the rest of it, and + * a residue kept from that lands the next folio's copy past + * its end. + */ + folio_offset = offset_in_folio(folio, pos); bytes = min(folio_size(folio) - folio_offset, num); tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii); @@ -1626,9 +2106,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, count += tmp; pos += tmp; num -= tmp; - offset += tmp; - if (offset == folio_size(folio)) - offset = 0; /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) @@ -1640,7 +2117,12 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, ia->write.folio_locked = true; break; } - if (!fc->big_writes || offset != 0) + /* + * Carry on only from a folio boundary: a copy that stopped + * short leaves the next one starting inside a folio, which is + * one request's worth on its own. + */ + if (!fc->big_writes || folio_offset + tmp != folio_size(folio)) break; } @@ -1787,38 +2269,119 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) } } +/* + * Pin [@pos, @pos + @len) with the grant over it confirmed, so the bytes + * can be dirtied under a lock that cannot be taken away meanwhile; see + * fuse_dlm_pin(). @pin is the caller's storage for the pin, which it + * drops with fuse_dlm_unpin() once the bytes are dirty. + * + * The grant is asked for again when it has gone, and the pin must not be + * held across that request: it is answered by the server the revoke + * waiting for the pin came from. So confirm and request alternate, and + * no folio may be held here. + * + * Not capped. A pass costs a round trip only when the grant has gone, + * which is a revoke landing between the request and the confirmation, so + * a busy range paces the loop rather than ending it. Giving up would + * fail a write the caller has no reason to expect to fail and cannot + * distinguish from a real one, over a range that is contended and + * nothing worse. A fatal signal ends it instead, so a killed task and + * close() get out. + */ +static int fuse_dlm_pin_write(struct file *file, struct fuse_dlm_span *pin, + loff_t pos, size_t len) +{ + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + int err; + + for (;;) { + fuse_dlm_pin(fi, pin, pos, len); + /* + * A server that turned out to have no DLM leaves nothing to + * confirm, and the pin still pairs with the caller's unpin. + */ + if (!fc->dlm || + fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) + return 0; + fuse_dlm_unpin(fi); + + if (fatal_signal_pending(current)) + return -EINTR; + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + if (err < 0 && err != -ENOSYS) + return err; + if (err > 0) { + /* + * Granted but unrecorded, so there is nothing for the + * confirmation above to find. The range is covered + * cluster-wide; pin and proceed. + */ + fuse_dlm_pin(fi, pin, pos, len); + return 0; + } + } +} + +/* + * The grant over the bytes about to be copied, held until ->put_folio + * hands the folio back dirty. Nothing is locked yet, which is what lets + * this wait out a revoke and ask for the grant again; ->writeback_range + * can do neither. + */ +static struct folio *fuse_iomap_get_folio(struct iomap_iter *iter, loff_t pos, + unsigned int len) +{ + struct fuse_iomap_write_ctx *ctx = iter->private; + struct folio *folio; + int err; + + err = fuse_dlm_pin_write(ctx->file, &ctx->pin, pos, len); + if (err) + return ERR_PTR(err); + + folio = iomap_get_folio(iter, pos, len); + if (IS_ERR(folio)) + fuse_dlm_unpin(get_fuse_inode(iter->inode)); + + return folio; +} + +/* iomap_write_end() has dirtied whatever was copied by now */ +static void fuse_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned int copied, struct folio *folio) +{ + folio_unlock(folio); + folio_put(folio); + fuse_dlm_unpin(get_fuse_inode(inode)); +} + static const struct iomap_write_ops fuse_iomap_write_ops = { .read_folio_range = fuse_iomap_read_folio_range, }; +static const struct iomap_write_ops fuse_iomap_dlm_write_ops = { + .get_folio = fuse_iomap_get_folio, + .put_folio = fuse_iomap_put_folio, + .read_folio_range = fuse_iomap_read_folio_range, +}; + static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, struct iov_iter *from, struct file *file) { - struct fuse_conn *fc = get_fuse_conn(file_inode(file)); - ssize_t written, total_written = 0; - /* * TEMPORARY WORKAROUND for iomap write deadlock: * - * Stack-allocate retry state and register it before calling - * iomap. If fuse_iomap_read_folio_range() encounters - * AOP_TRUNCATED_PAGE, it will mark retry_needed. - * - * Stack allocation ensures no memory leaks - the state is - * valid for the duration of this function call and is - * automatically cleaned up. + * The context fuse_iomap_read_folio_range() marks when it hits + * AOP_TRUNCATED_PAGE. iomap hands it back through its private + * pointer, which fuse needs for @file either way. */ - struct fuse_dlm_retry retry_state = { - .retry_needed = false, - }; - unsigned long task_key = (unsigned long)current; - int xa_ret; - - xa_ret = xa_err(xa_store(&fc->dlm_retry_tasks, task_key, - &retry_state, GFP_KERNEL)); - if (xa_ret) - return xa_ret; + struct fuse_iomap_write_ctx ctx = { .file = file }; + struct fuse_conn *fc = get_fuse_conn(file_inode(file)); + ssize_t written, total_written = 0; retry: /* @@ -1829,14 +2392,16 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * the next iteration with iov_iter already drained, and iomap * would re-enter with len==0 and livelock on a 0-length mapping. */ - retry_state.retry_needed = false; + ctx.retry_needed = false; /* * Use iomap so that we can do granular uptodate reads * and granular dirty tracking for large folios. */ written = iomap_file_buffered_write(iocb, from, &fuse_iomap_ops, - &fuse_iomap_write_ops, file); + fc->dlm ? &fuse_iomap_dlm_write_ops : + &fuse_iomap_write_ops, + &ctx); if (written > 0) total_written += written; @@ -1848,39 +2413,136 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * The folio has been unlocked by fuse_do_readfolio(), * breaking the ABBA deadlock with page invalidation. * - * Keep the entry in xarray and reuse it for the retry. - * * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE * retry support. */ - if (retry_state.retry_needed && iov_iter_count(from)) + if (ctx.retry_needed && iov_iter_count(from)) goto retry; - /* Remove from xarray now that we're done */ - xa_erase(&fc->dlm_retry_tasks, task_key); - return written < 0 ? written : total_written; } +/* + * The size a writeback request ends on: the alignment the server asked for, + * or without one the largest power of two write it takes. fc->max_write is + * at least 4096 once INIT has been answered, and only then is there a + * writeback cache to dirty. + */ +static loff_t fuse_write_chunk_size(struct fuse_conn *fc) +{ + if (fc->alignment_pages) + return (loff_t)fc->alignment_pages << PAGE_SHIFT; + + return rounddown_pow_of_two(fc->max_write); +} + static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); +/* + * Does a write published before @r still cover a byte of it? Only the + * entries ahead of it on the list: the oldest of an overlapping set waits + * for nobody, so the list always drains. Caller holds fuse_inode.wr_lock. + */ +static bool fuse_write_range_blocked_locked(struct fuse_inode *fi, + struct fuse_write_range *r) +{ + struct fuse_write_range *o; + + list_for_each_entry(o, &fi->wr_ranges, list) { + if (o == r) + return false; + if (o->start <= r->end && r->start <= o->end) + return true; + } + + return false; +} + +/* fuse_write_range_blocked_locked() taking the lock itself */ +static bool fuse_write_range_blocked(struct fuse_inode *fi, + struct fuse_write_range *r) +{ + bool blocked; + + spin_lock(&fi->wr_lock); + blocked = fuse_write_range_blocked_locked(fi, r); + spin_unlock(&fi->wr_lock); + + return blocked; +} + +/** + * fuse_write_range_lock - hold a write's bytes against the other writers + * @fi: the fuse inode + * @r: caller-owned storage, live until fuse_write_range_unlock() + * @pos: first byte the write covers + * @count: how many bytes it covers + * + * Writers sharing the inode rwsem are serialised by the folio lock alone, + * which lets two of them over the same bytes interleave a folio at a time. + * write(2) is atomic against another write(2), so hold the bytes for as + * long as the copy takes. Writers on disjoint bytes, which is what the + * shared lock is for, never wait here. + * + * Not a DLM object: the grant over these bytes is the node's and orders + * this client against the cluster, not the tasks against each other. No + * revoke path takes this, so it may be held across a folio lock, a + * read-modify-write and a grant request alike. + */ +static void fuse_write_range_lock(struct fuse_inode *fi, + struct fuse_write_range *r, loff_t pos, + size_t count) +{ + r->start = pos; + r->end = pos + count - 1; + + spin_lock(&fi->wr_lock); + /* + * Published before the wait and at the tail, so a write arriving + * later waits for this one and an overlapping set runs in the order + * it arrived. + */ + list_add_tail(&r->list, &fi->wr_ranges); + while (fuse_write_range_blocked_locked(fi, r)) { + spin_unlock(&fi->wr_lock); + wait_event(fi->wr_wq, !fuse_write_range_blocked(fi, r)); + spin_lock(&fi->wr_lock); + } + spin_unlock(&fi->wr_lock); +} + +/** + * fuse_write_range_unlock - release the bytes fuse_write_range_lock() held + * @fi: the fuse inode + * @r: the entry published there + */ +static void fuse_write_range_unlock(struct fuse_inode *fi, + struct fuse_write_range *r) +{ + spin_lock(&fi->wr_lock); + list_del(&r->list); + spin_unlock(&fi->wr_lock); + + wake_up_all(&fi->wr_wq); +} + /* * @return true if an exclusive inode lock is needed for a cached (buffered) * write. * * Buffered writes normally hold the inode rwsem exclusively, serialising all - * writers even on disjoint ranges. The DLM-serialised iomap writeback path is - * the exception: the DLM already excludes cluster-wide, and i_size is committed - * under fi->lock rather than the inode rwsem (see fuse_cache_write_iter()), so - * disjoint writers (MPI-IO / IOR) may share the lock. Mirrors - * fuse_dio_wr_exclusive_lock() for the direct path. + * writers even on disjoint ranges. The iomap writeback path under DLM is the + * exception: i_size is committed under fi->lock rather than the inode rwsem + * (see fuse_cache_write_iter()), and the bytes of two writers are held apart + * by fuse_write_range_lock() instead, so disjoint writers (MPI-IO / IOR) may + * share the lock. Mirrors fuse_dio_wr_exclusive_lock() for the direct path. */ static bool fuse_cache_wr_exclusive_lock(struct kiocb *iocb, bool writeback) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_conn *fc = get_fuse_conn(inode); - /* Only the DLM-serialised iomap writeback path relaxes the lock. */ + /* Only the iomap writeback path under DLM relaxes the lock. */ if (!fc->dlm || !writeback) return true; @@ -1908,19 +2570,67 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) * fc->dlm: the server has no DLM, proceed as a plain cached write. Any * other failure means the cache would be dirtied without DLM coverage - * the caller must fail the write instead. A granted-but-unrecorded - * lock (positive return) is covered cluster-wide; proceed, but flag it - * so the in-gate re-validation skips a check an invisible grant could - * never pass. + * lock (positive return) is covered cluster-wide; proceed. */ -static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, - bool *unrecorded) +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) { int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); - if (err < 0 && err != -ENOSYS) - return err; - *unrecorded = err > 0; - return 0; + return (err < 0 && err != -ENOSYS) ? err : 0; +} + +/* + * Start non-integrity writeback on the aligned chunks a streamed file has + * left behind. + * + * fuse_writeback_reached_alignment() ends a request on the server's + * alignment and fuse_folios_need_send() on the largest write it takes, but + * where one starts is the flusher's choice, and nothing sends the range at + * all until a dirty limit or the closing flush asks for it. Kicking a + * chunk as the writer leaves it puts both bounds on that same size and + * keeps the tail off the flush. + * + * @stream is the size average saying the file is streamed; the run of + * positions is kept here, because a chunk is complete only once the writes + * have carried on past it. A write that does not continue the previous one + * leaves nothing behind it and sends nothing. + * + * A hint only: nothing waits for it, and a chunk that is partly dirty or + * already written back sends what it has. The run is read and written + * without the inode lock, which the DLM path holds shared, so writers + * landing on it together cost a kick, not correctness. Each mark is read + * once into a local for that to hold: reading write_stream_start twice, + * to round down and again to compare, lets a writer moving it in between + * invert the range the kick is given. + */ +static void fuse_writeback_kick_stream(struct kiocb *iocb, loff_t pos, + size_t len, bool stream) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + loff_t chunk, run, start, end; + bool sequential; + + sequential = pos == READ_ONCE(fi->write_stream_next); + WRITE_ONCE(fi->write_stream_next, pos + (loff_t)len); + if (!sequential || !stream) { + /* Nothing behind this write is known to be finished */ + WRITE_ONCE(fi->write_stream_start, pos); + return; + } + + run = READ_ONCE(fi->write_stream_start); + chunk = fuse_write_chunk_size(get_fuse_conn(inode)); + start = round_down(run, chunk); + /* This write's own end, not the mark another writer may have moved */ + end = round_down(pos + (loff_t)len, chunk); + /* No bound crossed, so nothing has been left complete */ + if (end <= run) + return; + + WRITE_ONCE(fi->write_stream_start, end); + filemap_flush_range(file->f_mapping, start, end - 1); } static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) @@ -1933,13 +2643,12 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + struct fuse_write_range wr; bool writeback = false; - bool wb_guard = false; - bool exclusive = true; - bool dlm_unrecorded = false; - loff_t dlm_pos = 0; - size_t dlm_len = 0; + bool stream = false; + bool through = false; + bool claimed = false; + bool exclusive; if (fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); @@ -1983,82 +2692,94 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) writeback = true; } - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + /* + * Every write that can be cached feeds the size average, streamed or + * not: a writer changing its record has to be seen as well. The size + * is the one the caller asked for, before generic_write_checks() has + * had a chance to clamp it, which is the record the writer is working + * with. + */ + if (writeback && !(iocb->ki_flags & IOCB_DIRECT)) + stream = fuse_stream_update(&fi->write_size_ewma, + &fi->write_stream_run, + iov_iter_count(from)); + + /* + * A streamed write of FUSE_WRITE_STREAM_MIN or more is sent from here + * instead, out of the caller's own pages. Cached, the bytes are + * copied twice on their way to the server, into the folios and out of + * them into the ring the request is read from, and the folios are + * dropped unread; sent from here they are copied once. Size is the + * whole of the test: what a record is worth saving the copy on, not + * where it lands or how it fits the alignment the server asked for. + * + * Not on a mapped file: the folios these bytes replace have to be + * dropped afterwards, and a mapped one is not. + */ + through = stream && !mapping_mapped(mapping) && + iov_iter_count(from) >= FUSE_WRITE_STREAM_MIN; /* * Request the DLM write lock before taking i_rwsem: the request is * an unbounded cluster round trip, and holding the writer-priority * rwsem across it would park a truncate -- and behind it every - * later writer -- for the duration. The grant-to-use window this - * leaves open is closed by the in-gate re-validation below. Only - * the append case must wait for the lock: its range depends on - * i_size, which is stable only under the exclusive inode lock. + * later writer -- for the duration. Only the append case must wait + * for the lock: its range depends on i_size, which is settled by + * generic_write_checks() under the exclusive inode lock. + * + * The request may find that the server has no DLM at all and clear + * fc->dlm, so pick the lock mode after it rather than before. The + * grant says nothing about the writers on this node: it is the + * node's, and every task here writes under the same one. What keeps + * two of them off each other's bytes is fuse_write_range_lock() + * below. + * + * O_DIRECT takes no DLM lock at all, here or anywhere: it dirties + * no page cache, so there is nothing for a grant to cover, and its + * bytes are the server's to order against the rest of the cluster. */ - if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { - dlm_pos = iocb->ki_pos; - dlm_len = iov_iter_count(from); - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + !(iocb->ki_flags & IOCB_APPEND)) { + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, + iov_iter_count(from)); if (err) return err; - - /* - * The request above may have found that the server has no DLM - * at all, in which case it cleared fc->dlm. The relaxed shared - * lock was chosen just before, while fc->dlm still read 1, and - * it is only sound under DLM: the shared path claims the i_size - * extension up front, which stops iomap from zeroing beyond - * EOF, and the zero-fill that replaces it in - * fuse_iomap_read_folio_range() is itself gated on fc->dlm. - * Left as chosen, an expanding write would fall through to a - * READ of a range that cannot hold data -- which fails outright - * on a handle the client opened write-only. Re-decide now, - * while no lock is held yet. - */ - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); } + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + if (exclusive) inode_lock(inode); else inode_lock_shared(inode); - /* note that this small code dup will save us a lot of headache later - * when appends are done concurrently without using parallel direct writes */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { - /* - * An append write lands at the current EOF no matter what - * ki_pos holds: generic_write_checks() rewrites ki_pos to - * i_size for IOCB_APPEND, and i_size is stable here because - * append writes hold the inode lock exclusive. Lock where - * the data will land. - */ - dlm_pos = i_size_read(inode); - dlm_len = iov_iter_count(from); + err = count = generic_write_checks(iocb, from); + if (err <= 0) + goto out; - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + /* + * An append lands at the EOF generic_write_checks() has just written + * into ki_pos, not where the caller pointed, and the exclusive inode + * lock does not pin i_size either: attribute replies move it under + * fi->lock alone. Take the grant here, where the range is settled. + */ + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_DIRECT) && + (iocb->ki_flags & IOCB_APPEND)) { + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, count); if (err) goto out; } - err = count = generic_write_checks(iocb, from); - if (err <= 0) - goto out; - /* - * Kill suid/sgid and stamp the timestamps here, before the gate, - * instead of leaving them next to the write itself. kiocb_modified() - * -> file_remove_privs() is the one that reaches the server: without - * handle_killpriv[_v2] fuse_setattr() kills the bits by asking it (a - * FUSE_GETATTR to refresh the mode, then a FUSE_SETATTR, which for a - * writeback inode first flushes and freezes writepages), and - * security_inode_killpriv() can drop the capability xattr with another - * round trip. A server may have to invalidate this inode from inside - * such a handler; its NOTIFY_INVAL_INODE then blocks in - * percpu_down_write() draining a gate reader that is itself waiting for - * the reply. Nothing held under the gate may wait for the server. + * Kill suid/sgid and stamp the timestamps here, ahead of the write + * itself. kiocb_modified() -> file_remove_privs() is the one that + * reaches the server: without handle_killpriv[_v2] fuse_setattr() + * kills the bits by asking it (a FUSE_GETATTR to refresh the mode, + * then a FUSE_SETATTR, which for a writeback inode first flushes and + * freezes writepages), and security_inode_killpriv() can drop the + * capability xattr with another round trip. A server may have to + * invalidate this inode from inside such a handler, and it must not + * find this write holding anything it needs. * * This also runs before the forced-DIO re-route below, so a re-routed * write repeats it; there is nothing left to do the second time. @@ -2067,31 +2788,39 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err) goto out; - wb_guard = !!wb_sem; - if (wb_guard) { -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - fuse_cache_wr_unlock(inode, exclusive); - return fuse_direct_write_iter(iocb, from); - } - if (writeback && fc->dlm && !dlm_unrecorded && - !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, - FUSE_PAGE_LOCK_WRITE)) { - percpu_up_read(wb_sem); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); - if (err) { - /* The gate is already dropped; funnel the - * failure through the one audited exit. */ - wb_guard = false; - goto out; - } - goto retry; - } + if (fuse_inode_force_dio(inode)) { + /* + * Latched while this write waited for the lock, so the drain + * fuse_file_write_iter() would have done falls here. Drop the + * lock first: the drain takes it exclusive. + */ + fuse_cache_wr_unlock(inode, exclusive); + err = fuse_force_dio_drain(inode); + if (err) + return err; + return fuse_direct_write_iter(iocb, from); + } + + /* + * Hold these bytes against the writers sharing the inode lock. + * Taken after the re-route above, which returns without passing + * the release below, and after generic_write_checks() has settled + * what the write covers. + */ + if (!exclusive) { + fuse_write_range_lock(fi, &wr, iocb->ki_pos, count); + claimed = true; } + /* + * A NOTIFY invalidate can revoke the grant requested above between + * here and the dirtying below, and nothing stops it: the bytes are + * caught on the way out instead. fuse_dlm_unlock_range() keeps a + * revoked range for as long as there is page cache under it, and + * writeback holds the range again before sending anything. So a + * write racing a revoke costs a round trip, not coverage. + */ + task_io_account_write(count); if (iocb->ki_flags & IOCB_DIRECT) { @@ -2099,7 +2828,58 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (written < 0 || !iov_iter_count(from)) goto out; written = direct_write_fallback(iocb, from, written, - fuse_perform_write(iocb, from)); + fuse_perform_write(iocb, from)); + } else if (through) { + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + struct fuse_dlm_span pin; + loff_t pos = iocb->ki_pos; + + /* + * What is cached under the write goes to the server before + * it and is dropped after: the folios hold the bytes these + * replace, and invalidate_inode_pages2_range() launders + * rather than drops, so a dirty one left here would reach + * the server on top of them. + */ + if (mapping->nrpages) { + err = filemap_write_and_wait_range(mapping, pos, + pos + count - 1); + if (err) + goto out; + } + + /* + * These bytes never enter the page cache, so a revoke + * cannot find them by flushing it, and the grant they were + * taken under can be handed on while they are still on the + * wire. Hold it across the FUSE_WRITE, as the writethrough + * edges do; a revoke of the range waits for the reply. + */ + err = fuse_dlm_pin_write(file, &pin, pos, count); + if (err) + goto out; + + written = fuse_direct_io(&io, from, &iocb->ki_pos, + FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); + fuse_dlm_unpin(fi); + if (written < 0) { + err = written; + goto out; + } + + /* + * i_size is the server's again as soon as the bytes are + * there, so no extension is claimed for them: this commits + * it the way the direct path does, retiring the attribute + * replies that left before the write. + */ + fuse_write_update_attr(inode, iocb->ki_pos, written); + + if (written > 0 && mapping->nrpages) + invalidate_inode_pages2_range(mapping, + pos >> PAGE_SHIFT, + (iocb->ki_pos - 1) >> PAGE_SHIFT); } else if (writeback) { loff_t pos = iocb->ki_pos; loff_t end = pos + count; @@ -2130,12 +2910,39 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) spin_lock(&fi->lock); orig_size = i_size_read(inode); if (end > orig_size) { + /* + * Retire the attribute replies already on the + * wire. fuse_attr_cache_mask() decides whether + * the server's size wins from an i_size it read + * before this claim and before it slept in the + * lock tree query, so a GETATTR that left while + * i_size still matched the server's is applied + * afterwards and shrinks it back. Moving + * attr_version makes fuse_change_attributes_i() + * drop those replies, which is what + * fuse_write_update_attr() moves it for. + */ + fi->attr_version = + atomic64_inc_return(&fc->attr_version); + /* + * And count the claim, for a reply that leaves + * after it: until the bytes are dirtied, + * [orig_size, end) is covered by nothing else + * fuse_attr_cache_mask() can see. + */ + atomic_inc(&fi->size_extenders); i_size_write(inode, end); extended = true; } spin_unlock(&fi->lock); - /* Zero the tail of the folio straddling the old EOF. */ + /* + * Zero the tail of the folio straddling the old EOF. + * Inert while the fuse block size is PAGE_SIZE, which + * it always is, and nothing is recorded for it either + * way: claiming the whole gap as written would hand + * writeback bytes no one wrote. + */ if (extended && orig_size < pos) pagecache_isize_extended(inode, orig_size, pos); } @@ -2159,6 +2966,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) i_size_write(inode, reached); spin_unlock(&fi->lock); } + atomic_dec(&fi->size_extenders); } if (written < 0) { @@ -2169,11 +2977,16 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from); } out: - if (wb_guard) - percpu_up_read(wb_sem); + if (claimed) + fuse_write_range_unlock(fi, &wr); fuse_cache_wr_unlock(inode, exclusive); - if (written > 0) + if (written > 0) { + /* The buffered branch above, the only one leaving folios dirty */ + if (writeback && !through && !(iocb->ki_flags & IOCB_DIRECT)) + fuse_writeback_kick_stream(iocb, iocb->ki_pos - written, + written, stream); written = generic_write_sync(iocb, written); + } return written ? written : err; } @@ -2292,6 +3105,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, { int write = flags & FUSE_DIO_WRITE; int cuse = flags & FUSE_DIO_CUSE; + int shared = flags & FUSE_DIO_SHARED; struct file *file = io->iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; @@ -2320,12 +3134,33 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, return res; } } + + /* + * Wait out the writeback this read or write would otherwise race: + * a reply landing after it would put the superseded bytes on the + * server on top of it. + * + * How to wait depends on the lock the caller holds. fuse_set_nowrite() + * asserts BUG_ON(fi->writectr < 0) and biases a counter the whole inode + * shares, which only an exclusive i_rwsem makes safe; two callers + * holding it shared reach that assertion together and the second one + * dies inside spin_lock(&fi->lock). A parallel direct write holds it + * shared (fuse_dio_lock(), and the streamed write in + * fuse_cache_write_iter()), so it waits on the folios of its own range + * instead, which is the range this test asked about anyway. Errors are + * left on the mapping for fsync to collect. + */ if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) { - if (!write) + if (!write) { inode_lock(inode); - fuse_sync_writes(inode); - if (!write) + fuse_sync_writes(inode); inode_unlock(inode); + } else if (shared) { + filemap_fdatawait_range_keep_errors(mapping, pos, + pos + count - 1); + } else { + fuse_sync_writes(inode); + } } if (fopen_direct_io && write) { @@ -2387,6 +3222,15 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (res > 0) *ppos = pos; + if (res > 0 && write && fopen_direct_io) { + /* + * As in generic_file_direct_write(), invalidate after the + * write, to invalidate read-ahead cache that may have competed + * with the write. + */ + invalidate_inode_pages2_range(mapping, idx_from, idx_to); + } + return res > 0 ? res : err; } EXPORT_SYMBOL_GPL(fuse_direct_io); @@ -2443,7 +3287,8 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); res = fuse_direct_io(&io, from, &iocb->ki_pos, - FUSE_DIO_WRITE); + FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); fuse_write_update_attr(inode, iocb->ki_pos, res); } if (res > 0 && mapping->nrpages) { @@ -2462,6 +3307,59 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) return res; } +/* + * Whether the inode is still latched into direct IO. The latch is set on a + * stream of invalidation notifies and has to come off when the stream stops, + * or an inode that was hot once stays uncached for as long as it is open. The + * average behind the latch is folded on arrival and cannot age on its own, so + * the age of the last notify is the clock. + * + * Cleared here the way the mmap revert clears it: no inode lock, none is held + * at the top of the IO paths, and the drop is server-free because everything + * cached under the latch is clean. + */ +static bool fuse_force_dio_active(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + bool cleared = false; + + if (!fuse_inode_force_dio(inode)) + return false; + + /* + * Unlocked, and both may change under this: a writer that appears after + * the check leaves the latch cleared, one that goes leaves it on until + * the next IO looks again. Neither is wrong, and the list is re-read + * under fi->lock before anything is cleared. + */ + if (!time_after(jiffies, + READ_ONCE(fi->notify_stamp) + FUSE_NOTIFY_DIO_COLD) || + !list_empty_careful(&fi->write_files)) + return true; + + spin_lock(&fi->lock); + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + list_empty(&fi->write_files) && + time_after(jiffies, fi->notify_stamp + FUSE_NOTIFY_DIO_COLD)) { + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + cleared = true; + } + spin_unlock(&fi->lock); + + /* + * Whatever a read racing the latch left behind, so it cannot be served + * once caching resumes. Another caller may have cleared the latch + * first, or set it again since; the bit decides, not this one's work. + */ + if (cleared) + fuse_force_dio_drop(inode->i_mapping); + + return fuse_inode_force_dio(inode); +} + static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; @@ -2475,12 +3373,17 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return fuse_dax_read_iter(iocb, to); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) { + ssize_t err = fuse_force_dio_drain(inode); + + if (err) + return err; return fuse_direct_read_iter(iocb, to); - else if (fuse_file_passthrough(ff)) + } else if (fuse_file_passthrough(ff)) { return fuse_passthrough_read_iter(iocb, to); - else + } else { return fuse_cache_read_iter(iocb, to); + } } static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) @@ -2496,12 +3399,17 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_dax_write_iter(iocb, from); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_force_dio_active(inode)) { + ssize_t err = fuse_force_dio_drain(inode); + + if (err) + return err; return fuse_direct_write_iter(iocb, from); - else if (fuse_file_passthrough(ff)) + } else if (fuse_file_passthrough(ff)) { return fuse_passthrough_write_iter(iocb, from); - else + } else { return fuse_cache_write_iter(iocb, from); + } } static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, @@ -2513,8 +3421,17 @@ static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); - else - return filemap_splice_read(in, ppos, pipe, len, flags); + + /* + * Latched: copy through ->read_iter, which reads direct, rather than + * fill a page cache this inode is not allowed to keep. + */ + if (fuse_force_dio_active(file_inode(in))) + return copy_splice_read(in, ppos, pipe, len, flags); + + fuse_read_grant(in, *ppos, len); + + return filemap_splice_read(in, ppos, pipe, len, flags); } static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, @@ -2529,6 +3446,25 @@ static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, return iter_file_splice_write(pipe, out, ppos, len, flags); } +/* + * The folios and descs of a writeback request come from one allocation, + * which kfree(ap->folios) releases. + */ +static struct folio **fuse_wb_folios_alloc(unsigned int nfolios, gfp_t flags, + struct fuse_folio_desc **descs) +{ + struct folio **folios; + + folios = kzalloc(nfolios * (sizeof(struct folio *) + + sizeof(struct fuse_folio_desc)), flags); + if (!folios) + return NULL; + + *descs = (void *) (folios + nfolios); + + return folios; +} + static void fuse_writepage_free(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2542,6 +3478,25 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) kfree(wpa); } +/* + * Ending a folio's writeback is iomap's own accounting, and fuse reports + * each queued run's bytes as its request completes. + * + * For a folio of more than one block, iomap_writeback_init() charges the + * whole folio to ifs->write_bytes_pending before any run is queued and + * iomap_writeback_folio() takes back what it did not submit once the last + * run has been offered, so the count cannot reach zero while a run is + * still to come and the completions sum to exactly the folio. + * + * For a folio of a single block there is no iomap_folio_state and + * iomap_finish_folio_write() ends the writeback on every call, but such a + * folio is offered as exactly one run: iomap_find_dirty_range() has no + * per-block state to walk and returns the whole range at once, and + * ->writeback_range never returns short, so iomap_writeback_range() does + * not go round again. One run is one entry in one request, and one call + * ends it. fuse_iomap_writeback_range() asserts that. + */ + static void fuse_writepage_finish(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2654,20 +3609,6 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, if (!fc->writeback_cache) fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY); spin_lock(&fi->lock); - if (!error) { - struct fuse_write_in *inarg = &wpa->ia.write.in; - - /* - * The server acknowledged this writeback, so data up to the - * end of the request is materialized on the server. Advance - * the bound before the folios end writeback below, i.e. - * before they can go clean and be reclaimed, so that - * fuse_iomap_read_folio_range() can never zero-fill a - * reclaimed range the server holds data in. - */ - if ((loff_t) (inarg->offset + inarg->size) > fi->server_size) - fi->server_size = inarg->offset + inarg->size; - } fi->writectr--; fuse_writepage_finish(wpa); spin_unlock(&fi->lock); @@ -2688,13 +3629,6 @@ static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi) return ff; } -static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi) -{ - struct fuse_file *ff = __fuse_write_file_get(fi); - WARN_ON(!ff); - return ff; -} - int fuse_write_inode(struct inode *inode, struct writeback_control *wbc) { struct fuse_inode *fi = get_fuse_inode(inode); @@ -2718,7 +3652,7 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) if (wpa) { ap = &wpa->ia.ap; ap->num_folios = 0; - ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); + ap->folios = fuse_wb_folios_alloc(1, GFP_NOFS, &ap->descs); if (!ap->folios) { kfree(wpa); wpa = NULL; @@ -2743,7 +3677,7 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, } static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, - uint32_t folio_index, loff_t offset, unsigned len) + uint32_t folio_index, loff_t offset, unsigned int len) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2782,12 +3716,42 @@ struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; unsigned int max_folios; + /* + * The folio currently being split into runs, and whether any run of + * it has already been queued. Only a folio iomap tracks per block + * can be offered as more than one run; see fuse_writepage_finish(). + */ + struct folio *wb_folio; + bool wb_queued; /* * nr_bytes won't overflow since fuse_folios_need_send() caps * wb requests to never exceed fc->max_pages (which has an upper bound * of U16_MAX). */ unsigned int nr_bytes; + /* + * The runs this pass could not send because their grant had gone. + * Taken back in fuse_iomap_writeback_submit(), where no folio is + * held, for the pass that follows; see fuse_iomap_writeback_range(). + */ + u64 regrant_start; + u64 regrant_end; + /* fuse_iomap_writeback_submit() took that range back */ + bool regranted; + /* + * Reference to a folio whose run was skipped and kept under writeback + * for redirty_len bytes until it can be put back on the dirty list; + * see fuse_writeback_redirty(). + */ + struct folio *redirty; + unsigned int redirty_len; + /* + * An error a run hit that was deferred rather than returned, so its + * bytes stayed put: no writer handle, a failed regrant (in the range + * callback or in the submit), or no memory. Reported from + * fuse_iomap_writeback_submit(). + */ + int defer_err; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, @@ -2802,7 +3766,7 @@ static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, max_pages); WARN_ON(nfolios <= data->max_folios); - folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); + folios = fuse_wb_folios_alloc(nfolios, GFP_NOFS, &descs); if (!folios) return false; @@ -2857,6 +3821,122 @@ static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, return false; } +/* + * A server that asked for an alignment wants its writes to start on it. Close + * the run at an aligned position when the next aligned run cannot be reached, + * either because writeback ends before it or because it would not fit. + */ +static bool fuse_writeback_reached_alignment(struct fuse_conn *fc, loff_t pos, + unsigned int bytes, + struct writeback_control *wbc) +{ + unsigned int total_pages = (bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; + pgoff_t page_index = pos >> PAGE_SHIFT; + pgoff_t end_page_index; + + if (!fc->alignment_pages) + return false; + + if (page_index % fc->alignment_pages) + return false; + + /* + * fuse_launder_folio() has no writeback_control, and a folio of more + * than one run reaches here from it. + */ + if (!wbc) + return false; + + /* + * A cyclic pass leaves range_end at zero and runs to the end of the + * mapping instead, so reading it there makes every aligned index its + * own request. + */ + end_page_index = wbc->range_cyclic ? + (pgoff_t) -1 : + (pgoff_t) ((wbc->range_end + PAGE_SIZE - 1) >> PAGE_SHIFT); + if (page_index + fc->alignment_pages > end_page_index) + return true; + + return total_pages + fc->alignment_pages > fc->max_pages; +} + +/* + * Put a folio writeback could not send back on the dirty list. + * + * iomap takes the dirty flag off a folio before it offers it to + * ->writeback_range, and a run reporting an error, or reporting a hole + * because the grant has gone, has thrown its bytes away unless they are put + * back. + * + * Not from inside the callback, though. iomap_writeback_folio() runs + * iomap_clear_range_dirty() over the whole folio once that has returned, so + * a range put back there is wiped again. For a folio one block wide that + * call does nothing and it would not matter, but a large folio carries an + * iomap_folio_state, and the folio would then be left with the dirty flag + * and no dirty block under it: the next pass finds nothing to write and the + * folio goes clean with its bytes never sent. + * + * So hold the folio and dirty it once iomap has let go of it, which on the + * ->writepages path is the next call or the submit, and in + * fuse_launder_folio() is the submit it makes itself. + * + * And keep it under writeback until then. Once iomap_writeback_folio() + * has cleared the folio and iomap_writepages() has unlocked it, a folio + * that is not dirty and not under writeback is anyone's: reclaim cannot + * take it while the reference is held, but invalidate_inode_pages2_range() + * does not look at references, and a revoke's invalidation landing in that + * window removed the folio with its bytes never sent. The run is reported + * as handled rather than as a hole, so iomap counts its bytes submitted + * and leaves them charged on the folio, and they are given back only after + * the folio is dirty again. A folio under writeback is neither reclaimed nor invalidated, so + * there is no window; the revoke waits for it, as it would for a send. + * + * Only while there is a connection left to take the bytes. After an abort + * every send fails, and a folio redirtied for a retry that can no longer + * happen would keep sync() going forever. + */ +static void fuse_writeback_redirty_done(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, + struct writeback_control *wbc) +{ + struct folio *folio = data->redirty; + unsigned int len = data->redirty_len; + + if (!folio) + return; + data->redirty = NULL; + data->redirty_len = 0; + + if (READ_ONCE(fc->connected)) { + folio_mark_dirty(folio); + if (wbc) + wbc->pages_skipped += folio_nr_pages(folio); + } + /* Dirty again (or abandoned), so the writeback can end now */ + iomap_finish_folio_write(folio->mapping->host, folio, len); + folio_put(folio); +} + +/* + * Remember @folio for fuse_writeback_redirty_done(), with the run + * [@pos, @pos + @len) of it counted as pending writeback. The caller + * reports the run to iomap as handled (@len, not a hole), so iomap leaves + * the folio under writeback for it. + */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, + struct writeback_control *wbc, + struct folio *folio, unsigned int len) +{ + if (data->redirty != folio) { + fuse_writeback_redirty_done(fc, data, wbc); + folio_get(folio); + data->redirty = folio; + } + data->redirty_len += len; +} + static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct folio *folio, u64 pos, unsigned len, u64 end_pos) @@ -2867,15 +3947,127 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct inode *inode = wpc->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); - loff_t offset = offset_in_folio(folio, pos); + struct fuse_dlm_span pin; + bool pinned = false; + loff_t offset; WARN_ON_ONCE(!data); + /* + * A folio iomap has not asked about before: the one before it has all + * of its runs queued. + */ + if (data->wb_folio != folio) { + data->wb_folio = folio; + data->wb_queued = false; + /* + * iomap has unlocked whatever it offered before this, so a + * folio held from then can go back on the dirty list now. + */ + fuse_writeback_redirty_done(fc, data, wpc->wbc); + } + + /* + * Every run that cannot be sent is deferred the same way, whatever + * the reason: the folio stays under writeback for it and is dirtied + * again once iomap has let go of it (fuse_writeback_redirty()), and + * an error is kept for the submit to report rather than returned + * here, where iomap would end the folio's writeback with the bytes + * still to send and nothing keeping them. + */ + wpc->iomap.type = IOMAP_MAPPED; + if (!data->ff) { - data->ff = fuse_write_file_get(fi); - if (!data->ff) - return -EIO; + data->ff = __fuse_write_file_get(fi); + if (!data->ff) { + /* + * No file left open for writing, which + * fuse_open()'s invalidate reaches through + * fuse_launder_folio() once the last writer has + * closed. The bytes are still the newest there + * are, so keep them: dropping them here loses a + * write that fsync and close both reported done. + */ + fuse_writeback_redirty(fc, data, wpc->wbc, folio, len); + if (!data->defer_err) + data->defer_err = -EIO; + return len; + } + } + + if (fc->dlm && fc->writeback_cache) { + /* + * The revoke handler flushing the range it is taking + * away. That lock is still this client's until the + * handler returns, so send without asking: the record has + * gone already and asking would be a round trip for the + * very range being revoked. Only for that range, since + * the latched path launders the whole mapping and the + * rest of it may be covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + + /* + * The folio is locked and under writeback here, so a grant + * this run does not already hold must not be asked for: + * Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE- + * reason.txt states the rule the read path is built around, + * that no cluster lock may be taken while a page lock is + * held. read_folio() has AOP_TRUNCATED_PAGE to unlock and + * retry with; ->writeback_range has nothing of the sort. + * + * Defer the run instead. The folio stays under writeback + * and goes back on the dirty list once iomap has let go of + * it, the range is remembered for + * fuse_iomap_writeback_submit() to take back with no folio + * held, and the pass that follows sends it. Nothing is lost + * and no error is recorded for a later fsync to report. + * + * A refused pin is a revoke of this range draining, and + * leaves the run in the same place for the same reason. A + * revoke elsewhere in the file does not refuse it. + */ + pinned = fuse_dlm_trypin_held(fi, &pin, pos, len, + FUSE_PAGE_LOCK_WRITE); + if (!pinned) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio, len); + if (data->regrant_end <= data->regrant_start) { + data->regrant_start = pos; + data->regrant_end = pos + len; + } else { + u64 s = min(data->regrant_start, pos); + u64 e = max(data->regrant_end, pos + len); + + /* + * One shard is as far as a single request is + * worth taking back. A pass sweeping a large + * file skips runs a long way apart, and the + * span between them says nothing about what + * is wanted; the runs left out stay dirty and + * a later pass asks for them. + */ + if (e - s <= FUSE_DLM_SHARD_SIZE) { + data->regrant_start = s; + data->regrant_end = e; + } + } + return len; + } + + /* + * Confirmed and pinned, so the run goes out under a grant + * that is held now and cannot be taken away before the + * bytes are under writeback: a revoke of the range drains + * the pins before fuse_dlm_unlock_range() removes anything, + * and one already draining would have refused the pin. + * Asking the record a second time here would answer the + * same and cost a round of the cache lock per folio. + */ } +queue: + + offset = offset_in_folio(folio, pos); if (wpa) { bool send = fuse_folios_need_send(fc, pos, len, ap, @@ -2890,6 +4082,10 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, !fuse_pages_realloc(data, fc->max_pages); } + if (!send) + send = fuse_writeback_reached_alignment(fc, pos, + data->nr_bytes + len, wpc->wbc); + if (send) { fuse_writepages_send(inode, data); data->wpa = NULL; @@ -2899,15 +4095,38 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (data->wpa == NULL) { wpa = fuse_writepage_args_setup(folio, offset, data->ff); - if (!wpa) - return -ENOMEM; + if (!wpa) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio, len); + if (pinned) + fuse_dlm_unpin(fi); + if (!data->defer_err) + data->defer_err = -ENOMEM; + return len; + } fuse_file_get(wpa->ia.ff); data->max_folios = 1; ap = &wpa->ia.ap; } - fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, - offset, len); + /* + * fuse_writepage_finish() reports this run's bytes when the request + * completes. A folio of a single block carries no iomap_folio_state + * and that report ends its writeback outright, which is only correct + * because iomap offers such a folio as exactly one run. Nothing + * above can make it offer a second, but the accounting depends on it. + */ + WARN_ON_ONCE(data->wb_queued && i_blocks_per_folio(inode, folio) == 1); + data->wb_queued = true; + + /* + * Under writeback now, so the flush a revoke runs before it takes + * the grant away waits for these bytes. Nothing further is needed + * to keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); + + fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, offset, len); data->nr_bytes += len; ap->num_folios++; @@ -2924,14 +4143,57 @@ static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, WARN_ON_ONCE(!data); + /* No more runs are coming for the folio last seen */ + data->wb_folio = NULL; + data->wb_queued = false; + fuse_writeback_redirty_done(get_fuse_conn(wpc->inode), data, wpc->wbc); + if (data->wpa) { WARN_ON(!data->wpa->ia.ap.num_folios); fuse_writepages_send(wpc->inode, data); } + /* + * Take back what the runs above had to skip, so the pass that + * follows finds the grant and sends the folios they left dirty. + * + * Only on the ->writepages path, which is what @wbc marks. + * fuse_launder_folio() reaches here with the folio still locked by + * folio_unmap_invalidate(), which is the ordering this is avoiding, + * and a revoke handler would ask for the very range it is revoking. + * In both the skip simply stands and writeback picks it up. + */ + if (wpc->wbc && data->ff && data->regrant_end > data->regrant_start && + !fuse_in_notify_ctx()) { + int err = fuse_dlm_regrant_range(data->ff, wpc->inode, + data->regrant_start, + data->regrant_end - 1); + + /* + * Capture a deferred err so that a sync writeback knows that a + * grant is not coming rather than continuing to loop waiting for + * it. Otherwise the runs should stay dirty for a later pass. + */ + if (err < 0 && err != -ENOSYS) { + if (!data->defer_err) + data->defer_err = err; + } else { + data->regranted = true; + } + } + if (data->ff) fuse_file_put(data->ff, false); + /* + * A run deferred with an error keeps its bytes, so report the err the + * same way a failed send would, so that fsync and close see it. + */ + if (data->defer_err && !error) { + error = data->defer_err; + mapping_set_error(wpc->inode->i_mapping, error); + } + return error; } @@ -2945,14 +4207,7 @@ static int fuse_writepages(struct address_space *mapping, { struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_fill_wb_data data = {}; - struct iomap_writepage_ctx wpc = { - .inode = inode, - .iomap.type = IOMAP_MAPPED, - .wbc = wbc, - .ops = &fuse_writeback_ops, - .wb_ctx = &data, - }; + int err; if (fuse_is_bad(inode)) return -EIO; @@ -2961,7 +4216,51 @@ static int fuse_writepages(struct address_space *mapping, fc->num_background >= fc->congestion_threshold) return 0; - return iomap_writepages(&wpc); + /* + * A run whose grant had gone is skipped and its range taken back in + * fuse_iomap_writeback_submit(), which leaves the folio dirty for a + * later pass. For a data integrity writeback there is no later + * pass: fsync() and close() would report the bytes written while + * they are still only in the page cache. Go round again, now that + * the grant is held, until nothing is left deferred. + * + * Without a cap on the passes: each one is paced by the regrant, + * which the server answers once the revoke that refused the runs is + * done, and a pass whose regrant fails reports that instead of + * going round again. A cap returned success with runs still dirty, + * and a close that succeeded on that left the folios behind with no + * handle to send them from. + * + * Bounded by the regrant rather than by a count, so a fatal signal + * ends it as well: a grant taken away as often as it is given keeps + * the loop going, and close() has to stay killable through it. + */ + for (;;) { + struct fuse_fill_wb_data data = {}; + struct iomap_writepage_ctx wpc = { + .inode = inode, + .iomap.type = IOMAP_MAPPED, + .wbc = wbc, + .ops = &fuse_writeback_ops, + .wb_ctx = &data, + }; + + err = iomap_writepages(&wpc); + /* + * Only where the submit took the grant back. A revoke + * handler driving this is not allowed to, so the runs it + * skipped would be skipped again by every pass. + */ + if (err || wbc->sync_mode != WB_SYNC_ALL || !data.regranted) + break; + if (fatal_signal_pending(current)) { + err = -EINTR; + break; + } + cond_resched(); + } + + return err; } static int fuse_launder_folio(struct folio *folio) @@ -3048,6 +4347,7 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l fuse_abort_conn(fc); err = -EINVAL; } + return err; } /* @@ -3073,9 +4373,13 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct fuse_mount *fm = get_fuse_mount(inode); if (fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; - size_t length = PAGE_SIZE; - int err = fuse_get_page_mkwrite_lock(file, pos, length); + /* + * The whole folio is dirtied on the way out of this fault + * (fault_dirty_shared_page()), so the lock has to cover the + * folio, not the page that faulted. + */ + int err = fuse_get_page_mkwrite_lock(file, folio_pos(folio), + folio_size(folio)); if (err < 0) { return vmf_error(err); } @@ -3092,9 +4396,22 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) return VM_FAULT_LOCKED; } +/* + * A read fault fills the page cache through ->read_folio and + * ->readahead, which run with the folios locked. Ask for the grant they + * fill under before filemap_fault() locks any of them. + */ +static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) +{ + fuse_read_grant(vmf->vma->vm_file, (loff_t)vmf->pgoff << PAGE_SHIFT, + PAGE_SIZE); + + return filemap_fault(vmf); +} + static const struct vm_operations_struct fuse_file_vm_ops = { .close = fuse_vma_close, - .fault = filemap_fault, + .fault = fuse_filemap_fault, .map_pages = filemap_map_pages, .page_mkwrite = fuse_page_mkwrite, }; @@ -3123,7 +4440,7 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) /* * If the inode was latched into forced direct IO after a remote-modify * notification, a mapping needs the page cache, so revert to caching - * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * mode. Revert without the inode lock: ->mmap runs * under mmap_lock and the buffered write path holds both across a fault * on the user buffer (which takes mmap_lock), so taking either here * would invert lock order (ABBA). Clearing the latch and dropping the @@ -3137,10 +4454,11 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) spin_lock(&fi->lock); clear_bit(FUSE_I_FORCE_DIO, &fi->state); + clear_bit(FUSE_I_FORCE_DIO_DRAINED, &fi->state); if (fi->iocachectr > 0) set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); spin_unlock(&fi->lock); - invalidate_inode_pages2(file->f_mapping); + fuse_force_dio_drop(file->f_mapping); } /* @@ -3641,7 +4959,8 @@ __fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bool exclusive) } if (iov_iter_rw(iter) == WRITE) { - ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); + ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE | + (exclusive ? 0 : FUSE_DIO_SHARED)); fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); } else { ret = __fuse_direct_read(io, iter, &pos); @@ -3935,6 +5254,31 @@ static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off, return ret; } +/* + * POSIX_FADV_WILLNEED, and readahead(2) with it, populate the page cache + * through ->readahead, which runs with the folios locked. Ask for the + * grant that fill needs while nothing is held; a window no grant covers + * is given back unfilled. + * + * Report a grant that could not be taken rather than populate anyway, + * which would fill nothing: every folio of the window is declined and + * dropped again. A server without DLM answers -ENOSYS and has cleared + * fc->dlm, which is not a failure, and neither is a grant the server gave + * and the client could not record. + */ +static int fuse_fadvise(struct file *file, loff_t offset, loff_t len, + int advice) +{ + if (advice == POSIX_FADV_WILLNEED && offset >= 0 && len > 0) { + int err = fuse_read_grant(file, offset, len); + + if (err < 0 && err != -ENOSYS) + return err; + } + + return generic_fadvise(file, offset, len, advice); +} + static const struct file_operations fuse_file_operations = { .llseek = fuse_file_llseek, .read_iter = fuse_file_read_iter, @@ -3954,6 +5298,7 @@ static const struct file_operations fuse_file_operations = { .poll = fuse_file_poll, .fallocate = fuse_file_fallocate, .copy_file_range = fuse_copy_file_range, + .fadvise = fuse_fadvise, .setlease = generic_setlease, }; @@ -3986,28 +5331,20 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fuse_dlm_cache_init(fi); fi->writectr = 0; fi->iocachectr = 0; - fi->server_size = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); - /* - * Coherency gate for the forced-direct-IO feature; only writeback+dlm - * regular files need it. A percpu_rw_semaphore embeds per-CPU state, - * so allocate it out of line and only when the mount can use it rather - * than paying it on every inode. On failure leave it NULL: the gate - * stays inactive (best-effort invalidate) and the inode is still usable. - */ - fi->wb_inval_rwsem = NULL; - if (fc->writeback_cache && fc->dlm) { - struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); - - if (sem && percpu_init_rwsem(sem)) { - kfree(sem); - sem = NULL; - } - fi->wb_inval_rwsem = sem; - } fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; + atomic_set(&fi->size_extenders, 0); + spin_lock_init(&fi->wr_lock); + INIT_LIST_HEAD(&fi->wr_ranges); + init_waitqueue_head(&fi->wr_wq); + fi->write_size_ewma = 0; + fi->write_stream_run = 0; + fi->write_stream_next = 0; + fi->write_stream_start = 0; + fi->read_size_ewma = 0; + fi->read_stream_run = 0; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 6531186d63b54b..8f52f3c3dd65b4 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -1,599 +1,892 @@ // SPDX-License-Identifier: GPL-2.0-only /* * FUSE page lock cache implementation + * + * The shards record the grants the server has given this client, a bit + * per page. A grant still on the wire covers nothing and must not + * appear there, but a revoke has to be able to find it: otherwise a + * revoke processed before the grant is recorded removes nothing, and the + * grant recorded afterwards is never taken back. A request in flight + * therefore waits on cache->pending, where a revoke marks it killed and + * fuse_dlm_request_commit() drops the grant instead of recording it. + * + * Keeping requests out of the shards leaves every walker looking at + * grants alone. + * + * The record says nothing about the page cache under a page. What is + * cached there, and whether the server has seen it, is what the page + * cache itself answers. A revoked grant is therefore forgotten, not + * kept: writeback holds the range again for every run it sends, and an + * absent record and a revoked one both make it ask. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include #include +#include #include #include -#include +#include +#include -/* A range of pages with a lock */ +/* + * How far beyond the requested range a grant is recorded. + * + * A server may grant more than was asked for, and recording the extra is + * what lets the writes that follow skip the round trip entirely. But + * coverage is kept per shard, so the number of records a grant creates + * grows with its size, and outarg is the server's to choose: an + * unbounded grant would be an unbounded amount of work here. Cap it. + * Recording less than the server gave is safe - it only costs a + * re-request - and a cap this size still covers thousands of writes. + */ +#define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) + +/* + * A FUSE_DLM_WB_LOCK request in flight, on cache->pending. + * + * Two ranges, because the range asked for and the range that may end up + * recorded are not the same one: the server may grant more, up to + * FUSE_DLM_MAX_EXTRA_GRANT either side. A revoke has to be tested against + * both, and means something different for each. + */ struct fuse_dlm_range { - /* Interval tree node */ - struct rb_node rb; - /* Start page offset (inclusive) */ + /* The range asked for, as byte offsets, both inclusive */ uint64_t start; - /* End page offset (inclusive) */ uint64_t end; - /* Subtree end value for interval tree */ - uint64_t __subtree_end; - /* Lock mode */ - enum fuse_page_lock_mode mode; - /* Temporary list entry for operations */ + /* The widest [start, end] fuse_dlm_request_commit() could record */ + uint64_t wide_start; + uint64_t wide_end; + /* A revoke overlapped the range asked for: the grant is dead */ + bool killed; + /* A revoke overlapped only the excess: record the asked for range */ + bool clamp; + /* The cache->pending link */ struct list_head list; }; -/* Lock modes for FUSE page cache */ -#define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ -#define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ - -/* Interval tree definitions for page ranges */ -static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) +/* + * Bit of the page at @off within its shard. Callers pass page aligned + * bounds: a grant is aligned in __fuse_get_dlm_lock(), a revoke in + * fuse_dlm_revoke_inval_range() and a query in fuse_dlm_lock_is_held(). + */ +static unsigned long fuse_dlm_bit(uint64_t off) { - return range->start; + return (off & (FUSE_DLM_SHARD_SIZE - 1)) >> PAGE_SHIFT; } -static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) +/** + * fuse_dlm_shard_get - the shard covering @off, created if there is none + * @cache: the page cache + * @off: byte offset the caller is about to record a grant at + * + * Only the recording path needs a shard to exist; readers and the revoke + * paths treat a missing one as a region holding no grant. + * + * Return: the shard, or NULL if it could not be allocated. + */ +static struct fuse_dlm_shard *fuse_dlm_shard_get(struct fuse_dlm_cache *cache, + uint64_t off) { - return range->end; -} + unsigned long idx = off >> FUSE_DLM_SHARD_SHIFT; + struct fuse_dlm_shard *shard, *old; -INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); + shard = xa_load(&cache->shards, idx); + if (shard) + return shard; + + shard = kzalloc(sizeof(*shard), GFP_NOFS); + if (!shard) + return NULL; + + /* + * Two recorders can reach the same empty region at once; the loser + * drops its shard and takes the winner's. + */ + old = xa_cmpxchg(&cache->shards, idx, NULL, shard, GFP_NOFS); + if (old) { + kfree(shard); + /* xa_cmpxchg() returns an errno as an internal entry */ + return xa_is_err(old) ? NULL : old; + } + + return shard; +} /** - * fuse_page_cache_init - Initialize a page cache lock manager - * @cache: The cache to initialize + * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] + * @cache: The page cache + * @start: Start byte offset of the revoked region + * @end: End byte offset of the revoked region * - * Initialize a page cache lock manager for a FUSE inode. + * A revoke overlapping a request still on the wire has nothing to remove + * from the tree, since that grant is not recorded yet. Marking it makes + * fuse_dlm_request_commit() drop the grant instead of recording it. * - * Return: 0 on success, negative error code on failure + * The nodes are owned by the threads waiting on their replies: mark + * only, never remove or free. + * + * Takes @cache->pending_lock. A request published after this returns is + * one whose FUSE_DLM_WB_LOCK had not been sent when the revoke was + * processed, so the grant it goes on to receive answers a request made + * after the revoke and is recorded, not killed. */ -int fuse_dlm_cache_init(struct fuse_inode *inode) +static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, + uint64_t start, uint64_t end) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_range *req; + + spin_lock(&cache->pending_lock); + list_for_each_entry(req, &cache->pending, list) { + if (req->start <= end && start <= req->end) + req->killed = true; + else if (req->wide_start <= end && start <= req->wide_end) + req->clamp = true; + } + spin_unlock(&cache->pending_lock); +} - if (!cache) - return -EINVAL; +/** + * fuse_dlm_cache_init - Initialize a page cache lock manager + * @inode: The fuse inode to initialize the cache of + */ +void fuse_dlm_cache_init(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; init_rwsem(&cache->lock); - cache->ranges = RB_ROOT_CACHED; - cache->revoke_gen = 0; + xa_init(&cache->shards); + spin_lock_init(&cache->pending_lock); + INIT_LIST_HEAD(&cache->pending); + spin_lock_init(&cache->pin_lock); + INIT_LIST_HEAD(&cache->pins); + INIT_LIST_HEAD(&cache->fences); + init_waitqueue_head(&cache->pin_wq); +} - return 0; +/* + * Set up @span over the page-aligned range [@offset, @offset + @length), + * the same range a fuse_dlm_lock_is_held() with these arguments asks + * about, so a fence over a page cannot miss a pin on that page. + */ +static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, + size_t length, struct task_struct *owner) +{ + span->start = (uint64_t)offset & PAGE_MASK; + span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + span->owner = owner; +} + +/* + * Does anything on @head share a byte with [@start, @end]? Caller holds + * fuse_dlm_cache.pin_lock. + */ +static bool fuse_dlm_overlaps_locked(struct list_head *head, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_span *span; + + list_for_each_entry(span, head, list) + if (span->start <= end && start <= span->end) + return true; + + return false; +} + +/* fuse_dlm_overlaps_locked() taking the lock itself */ +static bool fuse_dlm_overlaps(struct fuse_dlm_cache *cache, + struct list_head *head, uint64_t start, + uint64_t end) +{ + bool overlap; + + spin_lock(&cache->pin_lock); + overlap = fuse_dlm_overlaps_locked(head, start, end); + spin_unlock(&cache->pin_lock); + + return overlap; } /** - * fuse_page_cache_destroy - Clean up a page cache lock manager - * @cache: The cache to clean up + * fuse_dlm_trypin - hold the grants over a range without sleeping + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes * - * Release all locks and free all resources associated with the cache. + * For a caller with nothing to wait with, the writeback path holding a + * folio locked and under writeback. A refusal means a revoke of this + * range is draining; the caller redirties and the pass that follows + * sends the folio. + * + * The caller confirms its grant after this returns, never before; a + * confirmation from before the pin says nothing. + * + * Return: true if the range is pinned, false if it is not. */ -void fuse_dlm_cache_release_locks(struct fuse_inode *inode) +static bool fuse_dlm_trypin(struct fuse_inode *inode, + struct fuse_dlm_span *pin, loff_t offset, + size_t length) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - struct rb_node *node; + bool fenced; - if (!cache) - return; + /* + * A revoke handler driving this inode's page cache: the lock over + * the range is still this client's until the handler returns, and + * the fence this would wait on is the handler's own. Both ends + * test the same task, so nothing is left on the list. + */ + if (fuse_in_notify_ctx()) + return true; - /* Release all locks */ - down_write(&cache->lock); - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - while ((node = rb_first_cached(&cache->ranges)) != NULL) { - range = rb_entry(node, struct fuse_dlm_range, rb); - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } - up_write(&cache->lock); + fuse_dlm_span_set(pin, offset, length, current); + + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + /* + * At the head, so fuse_dlm_unpin() drops the innermost pin of a + * task that holds more than one. + */ + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); + + return !fenced; } /** - * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_pin - fuse_dlm_trypin() that waits the revoke out + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes * - * Return: Pointer to the first overlapping range, or NULL if none found + * Must not be called with a folio held: the revoke waited for here drops + * that same page cache once it has drained. A revoke elsewhere in the + * file is not waited for. */ -static struct fuse_dlm_range * -fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) { - return fuse_page_it_iter_first(&cache->ranges, start, end); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + /* A refusal leaves @pin holding the range it was refused over */ + while (!fuse_dlm_trypin(inode, pin, offset, length)) + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->fences, + pin->start, pin->end)); } /** - * fuse_page_try_merge - Try to merge ranges within a specific region - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_trypin_span - fuse_dlm_trypin() for a fill that ends elsewhere + * @inode: the fuse inode + * @pin: caller-owned storage, live until fuse_dlm_unpin_span() + * @offset: byte offset the caller is about to fill + * @length: length of the region in bytes + * + * For a read whose reply lands in another task: the node is dropped by + * fuse_dlm_unpin_span() from wherever the fill ends, and carries no + * owner, so a fuse_dlm_unpin() by the task that took it cannot match it + * instead of its own. + * + * Never sleeps, and has no notify-context shortcut: a fill is not + * reached from a revoke handler, and a pin taken there would have to be + * dropped from a task that is not in one. * - * Attempt to merge ranges within and adjacent to the specified region - * that have the same lock mode. + * Return: true if the range is pinned, false if a revoke of it is + * draining. */ -static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static bool fuse_dlm_trypin_span(struct fuse_inode *inode, + struct fuse_dlm_span *pin, loff_t offset, + size_t length) { - struct fuse_dlm_range *range, *next; - uint64_t first = start ? start - 1 : start; - uint64_t last = end < U64_MAX ? end + 1 : end; + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; - if (!cache) - return; + fuse_dlm_span_set(pin, offset, length, NULL); - /* - * Find the first range that might need merging. Directly adjacent - * ranges can merge, hence the region is widened by one unit to each - * side (saturating at the type bounds). This must stay an - * interval-tree lookup: the tree holds every cached grant of the - * inode and strided writers grow it for the lifetime of the file, - * so seeding the merge by walking from the tree minimum would make - * every new grant cost a full scan. - */ - range = fuse_page_it_iter_first(&cache->ranges, first, last); - - /* Try to merge ranges in and around the specified region */ - while (range && range->start <= last) { - /* Get next range before we potentially modify the tree */ - next = NULL; - if (rb_next(&range->rb)) { - next = rb_entry(rb_next(&range->rb), - struct fuse_dlm_range, rb); - } + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); - /* Try to merge with next range if adjacent and same mode */ - if (next && range->mode == next->mode && - range->end + 1 == next->start) { - /* Merge ranges: re-insert so __subtree_end is updated */ - fuse_page_it_remove(next, &cache->ranges); - fuse_page_it_remove(range, &cache->ranges); - range->end = next->end; - fuse_page_it_insert(range, &cache->ranges); - kfree(next); - - /* Continue with the same range */ - continue; - } - - /* Move to next range */ - range = next; - } + return !fenced; } /** - * __fuse_dlm_lock_range - Lock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode (read or write) - * @genp: If non-NULL, the revocation generation sampled before the grant - * was requested; recording fails with -EAGAIN if it has moved - * - * Add a locked range on the specified range of pages. - * If parts of the range are already locked, only add the remaining parts. - * For overlapping ranges, handle lock compatibility: - * - READ locks are compatible with existing READ locks - * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks - * - * Return: 0 on success, negative error code on failure + * fuse_dlm_unpin_span - release the pin fuse_dlm_trypin_span() took + * @inode: the fuse inode + * @pin: the node published there */ -static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - const uint64_t *genp) +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *new_range, *next; - int lock_mode; - bool covered_to_end = false; - int ret = 0; - LIST_HEAD(to_lock); - LIST_HEAD(to_upgrade); - uint64_t current_start = start; + bool waiters; - if (!cache || start > end) - return -EINVAL; + spin_lock(&cache->pin_lock); + list_del(&pin->list); + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); - /* Convert to lock mode */ - lock_mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : - FUSE_PCACHE_LK_WRITE; + if (waiters) + wake_up_all(&cache->pin_wq); +} - down_write(&cache->lock); +/** + * fuse_dlm_unpin - release the pin this task last took on @inode + * @inode: the fuse inode + * + * Found by owner rather than by node: iomap hands ->put_folio the inode + * and nothing of the iteration, and a task holds one pin at a time. + */ +void fuse_dlm_unpin(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_span *pin; + bool waiters; - /* - * A revoke was processed after @genp was sampled; the grant this - * record carries may be the very one it targeted (a revoke of a - * not-yet-recorded grant removes nothing and would never be - * retried). Refuse, the caller re-requests. - */ - if (genp && cache->revoke_gen != *genp) { - up_write(&cache->lock); - return -EAGAIN; - } + /* See fuse_dlm_pin() */ + if (fuse_in_notify_ctx()) + return; - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check lock compatibility */ - if (lock_mode == FUSE_PCACHE_LK_WRITE && - lock_mode != range->mode) { - /* we own the lock but have to update it. */ - list_add_tail(&range->list, &to_upgrade); - } - /* If WRITE lock already exists - nothing to do */ - - /* If there's a gap before this range, we need to add the missing range */ - if (current_start < range->start) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = range->start - 1; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); - - list_add_tail(&new_range->list, &to_lock); + spin_lock(&cache->pin_lock); + list_for_each_entry(pin, &cache->pins, list) { + if (pin->owner == current) { + list_del(&pin->list); + break; } - - /* Move current_start past this range */ - if (range->end >= end) - covered_to_end = true; - else - current_start = max(current_start, range->end + 1); - - /* Move to next range */ - range = next; } + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); - /* If there's a gap after the last range to the end, extend the range */ - if (!covered_to_end && current_start <= end) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } + if (waiters) + wake_up_all(&cache->pin_wq); +} - new_range->start = current_start; - new_range->end = end; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); +/* + * Pin [@offset, @offset + @length) and confirm the grant over it, which + * is the only order that says anything: a confirmation from before the + * pin can be revoked before the pin is published, and a pin over a range + * that turns out uncovered is a pin holding a revoke up for nothing. + * Nothing is left published when either step fails. + * + * @owner is the pinning task, or NULL for a pin dropped by node from + * wherever the IO ends; see fuse_dlm_trypin_span(). + */ +static bool fuse_dlm_trypin_confirm(struct fuse_inode *inode, + struct fuse_dlm_span *pin, loff_t offset, + size_t length, + enum fuse_page_lock_mode mode, + struct task_struct *owner) +{ + bool pinned = owner ? fuse_dlm_trypin(inode, pin, offset, length) : + fuse_dlm_trypin_span(inode, pin, offset, length); - list_add_tail(&new_range->list, &to_lock); - } + if (!pinned) + return false; + if (fuse_dlm_lock_is_held(inode, offset, length, mode)) + return true; - /* update locks, if any lock is in this list it has the wrong mode */ - list_for_each_entry(range, &to_upgrade, list) { - /* Update the lock mode */ - range->mode = lock_mode; - } + if (owner) + fuse_dlm_unpin(inode); + else + fuse_dlm_unpin_span(inode, pin); - /* Add all new ranges to the tree */ - list_for_each_entry(new_range, &to_lock, list) { - /* Add to interval tree */ - fuse_page_it_insert(new_range, &cache->ranges); - } + return false; +} - /* Try to merge adjacent ranges with the same mode */ - fuse_dlm_try_merge(cache, start, end); +/** + * fuse_dlm_trypin_held - pin a range the grant over it is held on + * @inode: the fuse inode + * @pin: caller-owned storage, live until fuse_dlm_unpin() + * @offset: byte offset the caller is about to read or write + * @length: length of the region in bytes + * @mode: the grant the IO needs + * + * Return: true with the range pinned and covered, false with nothing + * published, either because a revoke of the range is draining or + * because no grant covers it. The caller cannot tell the two apart and + * has no reason to: both mean it may not touch the page cache here yet. + */ +bool fuse_dlm_trypin_held(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length, + enum fuse_page_lock_mode mode) +{ + return fuse_dlm_trypin_confirm(inode, pin, offset, length, mode, + current); +} - up_write(&cache->lock); - return 0; +/** + * fuse_dlm_trypin_held_span - fuse_dlm_trypin_held() for a fill that ends + * elsewhere + * @inode: the fuse inode + * @pin: caller-owned storage, live until fuse_dlm_unpin_span() + * @offset: byte offset the caller is about to fill + * @length: length of the region in bytes + * @mode: the grant the fill needs + * + * Return: as fuse_dlm_trypin_held(). + */ +bool fuse_dlm_trypin_held_span(struct fuse_inode *inode, + struct fuse_dlm_span *pin, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + return fuse_dlm_trypin_confirm(inode, pin, offset, length, mode, NULL); +} -out_free: - /* Free any ranges we allocated but didn't insert */ - while (!list_empty(&to_lock)) { - new_range = - list_first_entry(&to_lock, struct fuse_dlm_range, list); - list_del(&new_range->list); - kfree(new_range); - } +/** + * fuse_dlm_revoke_begin - fence the writers over a range that have not + * dirtied yet + * @inode: the fuse inode + * @fence: caller-owned storage, live until fuse_dlm_revoke_end() + * @offset: start byte offset being revoked + * @len: length in bytes, or <= 0 for everything from @offset on + * + * Publishes the range, then waits for the pins over it taken before it. + * On return no thread is between confirming a grant on that range and + * dirtying under it, and none can start, so what the caller flushes is + * everything the grants it is about to drop can have produced. A writer + * elsewhere in the file is neither waited for nor held up. + * + * Publishing before waiting is what makes the wait converge: a pin is + * refused on the same overlap this waits on, so nothing admitted after + * this can prolong it. + * + * Revokes on one inode fence independently, each over its own range. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - /* Restore original lock modes for any partially upgraded locks */ - list_for_each_entry(range, &to_upgrade, list) { - if (lock_mode == FUSE_PCACHE_LK_WRITE) { - /* We upgraded this lock but failed later, downgrade it back */ - range->mode = FUSE_PCACHE_LK_READ; - } - } + /* The range fuse_dlm_unlock_range() will be asked to drop */ + fence->start = (uint64_t)offset & PAGE_MASK; + fence->end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + fence->owner = NULL; - up_write(&cache->lock); - return ret; + spin_lock(&cache->pin_lock); + list_add(&fence->list, &cache->fences); + spin_unlock(&cache->pin_lock); + + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->pins, fence->start, + fence->end)); } -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +/** + * fuse_dlm_revoke_end - drop the fence fuse_dlm_revoke_begin() published + * @inode: the fuse inode + * @fence: the fence published there + */ +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence) { - return __fuse_dlm_lock_range(inode, start, end, mode, NULL); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pin_lock); + list_del(&fence->list); + spin_unlock(&cache->pin_lock); + + wake_up_all(&cache->pin_wq); } -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen) +/** + * fuse_dlm_cache_release_locks - Clean up a page cache lock manager + * @inode: The fuse inode to clean up the cache of + * + * Release all locks and free all resources associated with the cache. + */ +void fuse_dlm_cache_release_locks(struct fuse_inode *inode) { - return __fuse_dlm_lock_range(inode, start, end, mode, &gen); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_shard *shard; + unsigned long idx; + + /* + * Coverage goes away here, so the whole cache is taken for write: + * see the locking comment on struct fuse_dlm_cache. + */ + down_write(&cache->lock); + /* + * Every grant goes, so every request in flight is revoked. Mark + * only; each node is owned by the thread waiting on its reply. + */ + fuse_dlm_kill_pending(cache, 0, U64_MAX); + + xa_for_each(&cache->shards, idx, shard) + kfree(shard); + /* + * Leaves the xarray empty and usable: an inode is released again + * on every O_TRUNC open, not only on eviction. + */ + xa_destroy(&cache->shards); + up_write(&cache->lock); } /** - * fuse_dlm_revoke_gen - sample the revocation generation - * @inode: the fuse inode + * fuse_dlm_shard_record - record a grant over [@start, @end] in @shard + * @shard: the shard covering [@start, @end] + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * @mode: the mode it was granted in + * + * A write grant sets both maps, so a read query is answered by @granted + * alone, and a write grant over a page already held for read upgrades it + * by setting the bit the read grant left clear. + * + * @granted is set before @write, so a query racing this sees the page + * covered for read before it sees it covered for write. Either order is + * safe, a bit not yet seen only costing a re-request of a range already + * held, but this one never reports a write grant the read map does not + * back. * - * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The - * reply and a NOTIFY revoke can be serviced on different threads, so a - * revoke may be processed between the reply arriving and its grant - * being recorded. fuse_dlm_lock_range_gen() re-checks the generation - * under the cache lock and refuses to record a grant such a revoke may - * have already killed. + * [@start, @end] must be page aligned and lie wholly inside @shard. + * Caller holds fuse_dlm_cache.lock for read. */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +static void fuse_dlm_shard_record(struct fuse_dlm_shard *shard, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - return READ_ONCE(inode->dlm_locked_areas.revoke_gen); + unsigned long bit = fuse_dlm_bit(start); + unsigned long last = fuse_dlm_bit(end); + + for (; bit <= last; bit++) { + set_bit(bit, shard->granted); + if (mode == FUSE_PAGE_LOCK_WRITE) + set_bit(bit, shard->write); + } } /** - * fuse_dlm_punch_hole - Punch a hole in a locked range - * @cache: The page cache - * @start: Start page offset of the hole - * @end: End page offset of the hole + * fuse_dlm_record_grant - record a grant across the shards it spans + * @cache: the page cache + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode it was granted in * - * Create a hole in a locked range by splitting it into two ranges. + * Each shard is filled in on its own: coverage only grows here, and a + * walker under @cache->lock held for read may see part of the grant + * before the rest, which makes it ask again for a range it already has + * rather than trust one it has not got. * - * Return: 0 on success, negative error code on failure + * Caller holds @cache->lock for read. + * + * Return: 0 on success, negative error code on failure. A failure part + * way through leaves the shards already done recorded, which under + * reports the grant and is safe. */ -static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static int fuse_dlm_record_grant(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - struct fuse_dlm_range *range, *new_range; - int ret = 0; + unsigned long idx, last_idx; - if (!cache || start > end) + if (start > end) return -EINVAL; - /* Find a range that contains [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - if (!range) { - ret = -EINVAL; - goto out; - } + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - /* If the hole is at the beginning of the range */ - if (start == range->start) { - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard; + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); - /* If the hole is at the end of the range */ - if (end == range->end) { - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + shard = fuse_dlm_shard_get(cache, lo); + if (!shard) + return -ENOMEM; - /* The hole is in the middle, need to split */ - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out; + fuse_dlm_shard_record(shard, lo, hi, mode); } - /* Copy properties from original range, keeping the original end */ - *new_range = *range; - INIT_LIST_HEAD(&new_range->list); - new_range->start = end + 1; + return 0; +} +/** + * fuse_dlm_request_begin - publish a lock request before it is sent + * @inode: the fuse inode + * @req: caller-owned storage for the request, live until commit or abort + * @start: start byte offset being requested (inclusive) + * @end: end byte offset being requested (inclusive) + * + * The mode is not recorded here: until the server answers the range is + * held in neither, and the mode that reaches the shards is the one + * passed to fuse_dlm_request_commit(). + * + * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different + * threads, so a revoke can be processed before the grant the reply + * carries is recorded. Publishing the request before it leaves gives + * that revoke a node to mark; without one it removes nothing, and the + * grant recorded afterwards is never taken back by any later NOTIFY. + * + * The request covers nothing while in flight, so it is kept out of the + * shards. @req is reachable only through cache->pending, which both + * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink before + * the caller returns; stack storage is therefore fine and nothing is + * allocated here. + * + * Publishing touches the pending list and nothing else, so it takes + * @cache->pending_lock alone. It deliberately does not take + * @cache->lock: this runs on every cached write that is not already + * covered, and taking the cache rwsem for write here made every writer + * of a file queue behind every other one before its request had even + * been sent. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + req->start = start; + req->end = end; /* - * Shorten the original end only while it is unlinked. range->end is - * used in calulating the interval tree's __subtree_end, so changes - * made while the node is still in the tree leaves every ancestor - * stale. Update range->end after fuse_page_it_remove() + * The bounds __fuse_get_dlm_lock() caps the recorded grant to. A + * revoke between here and the commit must be seen by one of the two + * tests in fuse_dlm_kill_pending(), or it would be recorded over. */ - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - fuse_page_it_insert(new_range, &cache->ranges); - -out: - return ret; + req->wide_start = start > FUSE_DLM_MAX_EXTRA_GRANT ? + start - FUSE_DLM_MAX_EXTRA_GRANT : 0; + req->wide_end = U64_MAX - end < FUSE_DLM_MAX_EXTRA_GRANT ? + U64_MAX : end + FUSE_DLM_MAX_EXTRA_GRANT; + req->killed = false; + req->clamp = false; + + spin_lock(&cache->pending_lock); + list_add_tail(&req->list, &cache->pending); + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_unlock_range - Unlock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_request_commit - retire a request and record its grant + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode that was requested * - * Release locks on the specified range of pages. An inverted range is - * rejected rather than silently removing nothing: the callers revoke - * coverage, and a revoke that quietly keeps the grant alive would let - * the re-validating IO paths trust a lock the server has taken away. - * To drop every grant use fuse_dlm_cache_release_locks() (there is no - * in-band sentinel range for it). + * Unlinking @req and recording the grant are one step under + * @cache->lock held for read, so a revoke - which takes it for write - + * lands either before it and is seen on @req, or after it and finds the + * grant in the tree. * - * Return: 0 on success, negative error code on failure + * A revoke processed while @req was in flight lands in one of three + * places, and fuse_dlm_kill_pending() has already said which: + * + * - over the range asked for. The grant may predate it and there is no + * way to tell, so nothing is recorded and the caller asks again. + * - over the excess the server volunteered beyond it, and nothing else. + * The range asked for is untouched by it and is recorded; the excess + * is dropped, which only costs a re-request. + * - outside both, which says nothing about this grant. The whole of + * [start, end] is recorded. + * + * @req is retired in every case and may be reused. + * + * Return: -EAGAIN if a revoke overlapped the range @req asked for, + * nothing recorded; otherwise the result of recording the grant. */ -int fuse_dlm_unlock_range(struct fuse_inode *inode, - uint64_t start, uint64_t end) +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; + bool revoked, clamp; int ret = 0; - if (!cache || start > end) - return -EINVAL; - - down_write(&cache->lock); - /* - * Unconditional, even when nothing overlaps: the revoke racing - * with an in-flight grant finds an empty tree precisely because - * the grant is not recorded yet, and the bump is what makes the - * recording side notice (see fuse_dlm_lock_range_gen()). + * Read, not write: recording adds coverage. Holding it across the + * unlink and the record is what keeps a revoke from landing + * between them, since the revoke paths take it for write. */ - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check if we need to punch a hole */ - if (start > range->start && end < range->end) { - /* Punch a hole in the middle */ - ret = fuse_dlm_punch_hole(cache, start, end); - if (ret) - goto out; - /* After punching a hole, we're done */ - break; - } else if (start > range->start) { - /* Adjust the end of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - } else if (end < range->end) { - /* Adjust the start of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - } else { - /* Complete overlap, remove the range */ - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } + down_read(&cache->lock); - range = next; + spin_lock(&cache->pending_lock); + list_del(&req->list); + revoked = req->killed; + clamp = req->clamp; + if (clamp) { + start = req->start; + end = req->end; } + spin_unlock(&cache->pending_lock); -out: - up_write(&cache->lock); - return ret; + if (!revoked) + ret = fuse_dlm_record_grant(cache, start, end, mode); + up_read(&cache->lock); + + return revoked ? -EAGAIN : ret; } /** - * fuse_dlm_range_is_locked - Check if a page range is already locked - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode to check for (or NULL to check for any lock) + * fuse_dlm_request_abort - retire a request that got no usable reply + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() * - * Check if the specified range of pages is already locked. - * The entire range must be locked for this to return true. + * Nothing is recorded, so a mark left by a revoke does not matter. + */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pending_lock); + list_del(&req->list); + spin_unlock(&cache->pending_lock); +} + +/** + * fuse_dlm_unlock_range - Revoke the grants over a range of pages + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset * - * Return: true if the entire range is locked, false otherwise + * The server has taken [start, end] back, so the grants over it are + * removed and the IO paths ask again. Page cache dirtied under a grant + * that has gone is not lost by this: writeback takes the range again for + * every run it sends, and a range it finds unrecorded is a range it asks + * for. + * + * An inverted range is rejected rather than silently revoking nothing: + * the callers revoke coverage, and a revoke that quietly keeps the grant + * alive would let the re-validating IO paths trust a lock the server has + * taken away. To drop every grant use fuse_dlm_cache_release_locks() + * (there is no in-band sentinel range for it). + * + * Return: 0 on success, negative error code on failure */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - int lock_mode = 0; - uint64_t current_start = start; + struct fuse_dlm_shard *shard; + unsigned long idx; - if (!cache || start > end) - return false; + if (start > end) + return -EINVAL; - /* Convert to lock mode if specified */ - if (mode == FUSE_PAGE_LOCK_READ) - lock_mode = FUSE_PCACHE_LK_READ; - else if (mode == FUSE_PAGE_LOCK_WRITE) - lock_mode = FUSE_PCACHE_LK_WRITE; + /* + * Write, not read: this is a path that takes coverage away, and + * the shard walkers rely on that never happening under them. See + * the locking comment on struct fuse_dlm_cache. + */ + down_write(&cache->lock); - down_read(&cache->lock); + /* + * Before touching any map, and even when nothing is covered: a + * revoke racing an in-flight grant finds nothing set, because that + * grant is not recorded yet. + */ + fuse_dlm_kill_pending(cache, start, end); - /* Find the first range that overlaps with [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); + /* + * Only the regions that hold something, not every index in the + * range: a revoke to EOF runs to U64_MAX and walking that index + * by index would never finish. + */ + xa_for_each_range(&cache->shards, idx, shard, + start >> FUSE_DLM_SHARD_SHIFT, + end >> FUSE_DLM_SHARD_SHIFT) { + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + unsigned long first = fuse_dlm_bit(lo); + unsigned long nbits = fuse_dlm_bit(hi) - first + 1; - /* Check if the entire range is covered */ - while (range && current_start <= end) { /* - * The held lock must be at least as strong as the one - * requested. A WRITE lock (exclusive) satisfies a READ - * request, so only treat the range as uncovered when the - * held mode is weaker than what we ask for. This avoids - * re-requesting a READ lock for a range we already hold - * a WRITE lock on (e.g. read-after-write). + * Plain, not atomic: @cache->lock is held for write, so no + * reader and no recorder can be looking at these words. */ - if (lock_mode && range->mode < lock_mode) { - /* Held lock is weaker than requested */ - up_read(&cache->lock); - return false; - } - - /* Check if there's a gap before this range */ - if (current_start < range->start) { - /* Found a gap */ - up_read(&cache->lock); - return false; - } + bitmap_clear(shard->granted, first, nbits); + bitmap_clear(shard->write, first, nbits); - /* Covered through the end of the requested range? */ - if (range->end >= end) { - up_read(&cache->lock); - return true; + /* + * Keep the shard table to the regions that hold something: + * a file revoked a region at a time would otherwise leave an + * empty shard behind for every one of them. + */ + if (bitmap_empty(shard->granted, FUSE_DLM_SHARD_PAGES)) { + xa_erase(&cache->shards, idx); + kfree(shard); } - - /* Move current_start past this range */ - current_start = range->end + 1; - - /* Get next overlapping range */ - range = fuse_page_it_iter_next(range, start, end); } - /* Check if we covered the entire range */ - if (current_start <= end) { - /* There's a gap at the end */ - up_read(&cache->lock); - return false; - } + up_write(&cache->lock); + return 0; +} - up_read(&cache->lock); - return true; +/* + * Is every page of [@from, @to] covered in @mode? Both bounds are page + * aligned and lie inside @shard. A write grant sets both maps, so a + * read request is answered by @granted alone. + * + * No lock of its own: bits are set atomically and cleared only under + * fuse_dlm_cache.lock held for write, which the caller holds for read. + * A bit set concurrently may be missed, which costs a re-request of a + * range already held. + */ +static bool fuse_dlm_shard_covers(struct fuse_dlm_shard *shard, uint64_t from, + uint64_t to, enum fuse_page_lock_mode mode) +{ + const unsigned long *map = mode == FUSE_PAGE_LOCK_WRITE ? + shard->write : shard->granted; + unsigned long first = fuse_dlm_bit(from); + unsigned long last = fuse_dlm_bit(to); + + return find_next_zero_bit(map, last + 1, first) > last; } /** - * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere - * @fi: the fuse inode - * - * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully - * covered, this asks whether any part of the file is held exclusively. A - * client that holds a write grant may be sitting on dirty page cache the - * server has not seen, so its mtime and ctime run ahead of anything the - * server can report. + * fuse_dlm_range_is_locked - Check if a byte range is already locked + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset + * @mode: Lock mode to check for * - * Return: true if at least one recorded range is held for write + * Return: true if the entire range is locked, false otherwise */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) +static bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, + enum fuse_page_lock_mode mode) { - struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; - struct fuse_dlm_range *range; - bool held = false; + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + unsigned long idx, last_idx; + bool covered = true; + if (start > end) + return false; + + /* + * Read: coverage is only ever removed under @cache->lock held for + * write, so the set of grants can grow under this walk but never + * shrink. That is what lets the shards be visited one at a time, + * and their maps read without any further lock. The worst a + * concurrent recorder can do is make this report a range uncovered + * that has just become covered, and the caller then asks for a + * grant it already holds. + */ down_read(&cache->lock); - for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; - range = fuse_page_it_iter_next(range, 0, U64_MAX)) { - if (range->mode == FUSE_PCACHE_LK_WRITE) { - held = true; + + last_idx = end >> FUSE_DLM_SHARD_SHIFT; + + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard = xa_load(&cache->shards, idx); + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + + if (!shard || !fuse_dlm_shard_covers(shard, lo, hi, mode)) { + covered = false; break; } } + up_read(&cache->lock); - return held; + return covered; } /** @@ -623,25 +916,22 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); } -/** - * fuse_get_dlm_lock - request a dlm lock from the fuse server - * @file: the file being accessed - * @offset: byte offset into the file (need not be page-aligned) - * @length: length of the region in bytes (need not be page-aligned) - * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE +/* + * Send one FUSE_DLM_WB_LOCK for [@pg_start, @pg_end] and record what it + * grants. Split out of __fuse_get_dlm_lock() and kept out of line so + * the caller's fast path, which finds the range already covered and + * sends nothing, does not carry this frame: struct fuse_args alone is + * over a hundred bytes and would be zeroed on every lookup. * - * Return: 0 when the range is covered by a recorded grant on return, - * FUSE_DLM_GRANT_UNRECORDED when the server granted the lock but - * recording it failed (covered cluster-wide, invisible to - * fuse_dlm_lock_is_held()), a negative error code otherwise. Callers - * re-validating the grant must not re-request on a nonzero return or - * they would spin. + * Return: 0 when the grant is recorded, -EAGAIN to ask again, + * FUSE_DLM_GRANT_UNRECORDED when the server granted but recording + * failed, a negative error otherwise. */ -int fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode) +static noinline int fuse_dlm_send_lock(struct fuse_file *ff, + struct inode *inode, uint64_t pg_start, + uint64_t pg_end, + enum fuse_page_lock_mode mode) { - struct fuse_file *ff = file->private_data; - struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; @@ -649,42 +939,16 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; - uint64_t gen; + struct fuse_dlm_range req; + uint64_t grant_start, grant_end; int err; - /* An empty range needs no lock. */ - if (!length) - return 0; - -restart: - /* note that this can be run from different processes - * at the same time. It is intentionally not protected - * since a DLM implementation in the FUSE server should take care - * of any races in lock requests. - * The early exit uses the same helper the callers re-validate - * with, so this check and a later fuse_dlm_lock_is_held() can - * never disagree about what counts as covered. */ - if (fuse_dlm_lock_is_held(fi, offset, length, mode)) - return 0; /* we already have this area locked */ - - /* - * Sample the revocation generation before the request leaves. - * The reply and a NOTIFY revoke are serviced on different - * threads, so a revoke aimed at the grant this request returns - * can be processed before the grant is recorded below -- - * recording it anyway would resurrect a dead grant that no later - * NOTIFY will ever remove. - */ - gen = fuse_dlm_revoke_gen(fi); - memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned - * here but since we only get here on writeback caching we will - * send out page aligned requests */ - inarg.start = offset & PAGE_MASK; - inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); + inarg.start = pg_start; + inarg.end = pg_end; inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; @@ -696,42 +960,78 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; + + /* Publish before sending; see fuse_dlm_request_begin() */ + fuse_dlm_request_begin(fi, &req, inarg.start, inarg.end); + err = fuse_simple_request(fm, &args); - if (err == -ENOSYS) { - /* fuse server does not support dlm, save the info */ - fc->dlm = 0; + if (err) { + fuse_dlm_request_abort(fi, &req); + if (err == -ENOSYS) { + /* fuse server does not support dlm, save the info */ + fc->dlm = 0; + return err; + } + /* + * The range is contended, the same answer a READ gets and + * fuse_do_readfolio() turns into AOP_TRUNCATED_PAGE for its + * caller to retry. There is no such convention here, and + * the writeback caller loses the folio it is holding on an + * error, so ask again instead of reporting it. + */ + if (err == -EDEADLK || err == -EAGAIN) + return -EAGAIN; return err; } - if (err) - return err; - - if (inarg.start < outarg.start || inarg.end > outarg.end) { + if (pg_start < outarg.start || pg_end > outarg.end) { /* fuse server is seriously broken */ + fuse_dlm_request_abort(fi, &req); pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", - inarg.start, inarg.end, outarg.start, outarg.end); + pg_start, pg_end, outarg.start, outarg.end); fuse_abort_conn(fc); return -EIO; } /* - * The server granted the lock; record it so - * fuse_dlm_lock_is_held() sees it. + * Keep the recorded grant to a bounded distance either side of + * what was asked for; see FUSE_DLM_MAX_EXTRA_GRANT. Both bounds + * stay outside [pg_start, pg_end], so the range this call has to + * cover is still covered. */ - err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); - if (err == -EAGAIN) { - /* - * A revoke was processed while the request was in flight; - * the grant may already be dead, so re-request instead of - * recording it. Retry until a grant survives long enough to - * be recorded: giving up here would hand the caller an error - * for a range no one else holds, and the write path turns - * that into a failed write. Each pass makes a fresh server - * round trip, so a revoke storm throttles this loop rather - * than spinning it. - */ - goto restart; - } + grant_start = outarg.start; + grant_end = outarg.end; + /* + * Both differences are safe: the check above established + * outarg.start <= pg_start <= pg_end <= outarg.end, and a branch + * is only taken when there is more than the cap to give back, so + * neither adjusted bound can wrap. + */ + if (pg_start - grant_start > FUSE_DLM_MAX_EXTRA_GRANT) + grant_start = pg_start - FUSE_DLM_MAX_EXTRA_GRANT; + if (grant_end - pg_end > FUSE_DLM_MAX_EXTRA_GRANT) + grant_end = pg_end + FUSE_DLM_MAX_EXTRA_GRANT; + + /* + * Align inward. A page is covered only when it is covered whole, + * and the bit helpers take that as given; the bounds themselves are + * the server's to choose, only their superset property having been + * checked. Rounding inward cannot uncover [pg_start, pg_end], + * which is page aligned already. + */ + grant_start = ALIGN(grant_start, PAGE_SIZE); + /* A last-byte offset, so it is the successor that aligns */ + grant_end -= (grant_end + 1) & (PAGE_SIZE - 1); + + /* + * Retire the request and record the grant. -EAGAIN here is a + * revoke overlapping the range while it was in flight, so the + * grant is dead and the caller asks again: no one else holds the + * range, and the write path turns an error into a failed write. + */ + err = fuse_dlm_request_commit(fi, &req, grant_start, grant_end, mode); + if (err == -EAGAIN) + return -EAGAIN; /* * A failure to record (small-allocation -ENOMEM) does not undo @@ -746,3 +1046,110 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, return 0; } + +/** + * fuse_get_dlm_lock - request a dlm lock from the fuse server + * @file: the file being accessed + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Return: 0 when the range is covered by a recorded grant on return, + * FUSE_DLM_GRANT_UNRECORDED when the server granted the lock but + * recording it failed (covered cluster-wide, invisible to + * fuse_dlm_lock_is_held()), a negative error code otherwise. Callers + * re-validating the grant must not re-request on a nonzero return or + * they would spin. + * + * The common case sends nothing: the range is already covered and this + * is the lookup and nothing else. + */ +static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, + loff_t offset, size_t length, + enum fuse_page_lock_mode mode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + uint64_t pg_start, pg_end; + int err; + + /* An empty range needs no lock. */ + if (!length) + return 0; + + /* + * note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests + */ + pg_start = (uint64_t)offset & PAGE_MASK; + pg_end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + + for (;;) { + /* + * note that this can be run from different processes + * at the same time. It is intentionally not protected + * since a DLM implementation in the FUSE server should take + * care of any races in lock requests. + * The early exit uses the same helper the callers + * re-validate with, so this check and a later + * fuse_dlm_lock_is_held() can never disagree about what + * counts as covered. + */ + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) + return 0; + + err = fuse_dlm_send_lock(ff, inode, pg_start, pg_end, mode); + if (err != -EAGAIN) + return err; + + /* + * Ask again, for as long as it takes. Every pass is a whole + * round trip, so a range being taken away as fast as it is + * given paces this loop rather than spinning it, and no + * caller holds a folio while a request is out: writeback + * confirms its grant under a pin and so never reaches the + * request above, and ->readahead asks for nothing. + * + * A count would turn a contended range into an IO error, + * which the callers cannot tell from a real one and which a + * write reports to a caller that has no reason to expect it. + * Waiting the notifications out is the better answer. A + * fatal signal still ends it, so a killed task and close() + * get out, and so does the inode going bad, which is the + * exit a writeback kworker has. + */ + if (fatal_signal_pending(current)) + return -EINTR; + if (fuse_is_bad(inode)) + return -EIO; + } +} + +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + return __fuse_get_dlm_lock(file->private_data, file_inode(file), + offset, length, mode); +} + +/** + * fuse_dlm_regrant_range - hold [start, end] again for writeback + * @ff: a fuse file open for writing on @inode + * @inode: the inode + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * + * Writeback holds the range again before sending a folio, since a revoke + * may have arrived between the write and the send. Whatever the other + * holder wrote in between is overwritten, which for two writers that + * never synchronised is a legitimate order. + * + * A range still held is the ordinary case: the grant is found recorded + * and nothing is sent to the server. + */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end) +{ + return __fuse_get_dlm_lock(ff, inode, start, end - start + 1, + FUSE_PAGE_LOCK_WRITE); +} diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 30fdbb26bd3daf..3b01cd6448c243 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -7,16 +7,41 @@ #define _FS_FUSE_DLM_CACHE_H #include -#include +#include #include +#include #include +#include +#include +#include struct fuse_inode; +struct fuse_dlm_range; +struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * A range held on one of the two lists in struct fuse_dlm_cache: an IO + * between confirming a grant and publishing the page cache it covers, + * or a revoke taking grants away. Caller-owned storage, live until the + * matching unpin or revoke end. + * + * @owner is the pinning task where the pin is dropped by owner, and NULL + * on every fence and on a pin dropped by node because the fill it covers + * ends in another task. + */ +struct fuse_dlm_span { + /* Page-aligned byte offsets, both inclusive */ + uint64_t start; + uint64_t end; + /* The pinning task, NULL for a revoke */ + struct task_struct *owner; + struct list_head list; +}; + /* * fuse_get_dlm_lock() result: the server granted the lock but recording * it locally failed, leaving the grant invisible to @@ -26,53 +51,210 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; */ #define FUSE_DLM_GRANT_UNRECORDED 1 -/* Page cache lock manager */ +/* + * Coverage is kept per aligned region of the file rather than in one + * structure for the whole inode, because one structure needs one lock + * and every thread writing the file then serialises on it however far + * apart their ranges are. The region wants to be small enough that + * concurrent writers land in different ones and large enough that the + * per-region overhead stays amortised. + */ +#define FUSE_DLM_SHARD_SHIFT 24 +#define FUSE_DLM_SHARD_SIZE (1ULL << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_PAGES (FUSE_DLM_SHARD_SIZE / PAGE_SIZE) + +/* First and last byte offset (both inclusive) covered by shard @idx */ +#define FUSE_DLM_SHARD_FIRST(idx) ((uint64_t)(idx) << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_LAST(idx) (FUSE_DLM_SHARD_FIRST(idx) + \ + FUSE_DLM_SHARD_SIZE - 1) + +/* + * The grants over one region, a bit per page. + * + * @granted says the page is covered, @write that it is covered for + * write. A write grant sets both, so a read request is answered by + * @granted alone and nothing has to compare modes; @write is a subset of + * @granted. + * + * A region is a fixed span of pages, so the maps are a fixed size and + * recording a grant neither allocates nor rearranges anything: adjacent + * grants coalesce because they set neighbouring bits, and a region + * fragmented to the last page costs no more than the two maps it already + * has. + * + * Bits are set with the atomic helpers, since recorders run concurrently + * under fuse_dlm_cache.lock held for read. They are cleared only under + * that lock held for write, which excludes every reader and every + * recorder, so the revoke path uses the plain bulk helpers and a query + * needs no lock of its own. + */ +struct fuse_dlm_shard { + unsigned long granted[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; + unsigned long write[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; +}; + +/* + * Page cache lock manager. + * + * The shards hold the grants the client has been given. A request still + * on the wire covers nothing and lives on @pending instead, so the + * shards answer for grants only. See struct fuse_dlm_range in + * fuse_dlm_cache.c. + * + * Locking, outermost first: + * + * @lock rw_semaphore over the whole cache. Taken for read by + * everything that adds or reads coverage, and for write + * only by the paths that take coverage away + * (fuse_dlm_unlock_range, fuse_dlm_cache_release_locks). + * That is the invariant the shard walks rely on: + * **coverage is only ever removed under @lock held for + * write**, so a walker holding it for read sees a set of + * grants that can grow under it but never shrink, and may + * therefore visit shards one at a time and read their + * maps without any further lock. Shards are freed only + * there too, so a shard pointer stays good for as long as + * the read side is held. + * @pending_lock the pending list. Innermost, and the only lock + * fuse_dlm_request_begin() and fuse_dlm_request_abort() + * take at all. + * @pin_lock the pin and fence lists. Innermost, taken alone, and + * never held across a sleep. + * + * @lock says what is covered now, which is not enough for a writer: it + * confirms a grant, then copies and dirties, and a revoke landing in + * between sends those bytes out after the server has handed the lock on. + * The pin closes that: a revoke waits for the pins over the range it is + * taking away before it removes anything, so a grant confirmed under a + * pin is still held when the bytes become visible to writeback. A read + * is the same the other way round, a fill landing in a range the revoke + * has already swept staying cached under no grant at all. + * + * Both sides are ranges rather than a count, so a revoke fences only the + * writers it overlaps and a write outside it runs on. Refusal and wait + * test the same overlap, which is what makes the wait converge: once a + * fence is published no pin that would prolong it is admitted. The + * nodes are caller storage, so nothing is allocated to take a pin and + * the writeback path can take one with a folio held. + */ struct fuse_dlm_cache { - /* Lock protecting the tree */ + /* See the locking comment above */ struct rw_semaphore lock; - /* Interval tree of locked ranges */ - struct rb_root_cached ranges; /* - * Bumped under @lock by every revocation - * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); - * lets fuse_get_dlm_lock() order recording a reply's grant - * against revokes processed while the reply was in flight. + * struct fuse_dlm_shard by offset >> FUSE_DLM_SHARD_SHIFT, + * allocated when a region first holds a grant. + */ + struct xarray shards; + /* Protects @pending and the killed flag of everything on it */ + spinlock_t pending_lock; + /* + * FUSE_DLM_WB_LOCK requests in flight. Owned by the queueing + * thread; the revoke paths only mark them killed. + */ + struct list_head pending; + /* Protects @pins and @fences */ + spinlock_t pin_lock; + /* + * IO between confirming a grant and publishing under it: a writer + * over what it is about to dirty, a read over what it is about to + * fill. See the pin comment above. */ - uint64_t revoke_gen; + struct list_head pins; + /* Revokes in progress, each over the range it takes away */ + struct list_head fences; + /* Both directions: pins draining, and the fences they wait on */ + wait_queue_head_t pin_wq; }; /* Initialize a page cache lock manager */ -int fuse_dlm_cache_init(struct fuse_inode *inode); +void fuse_dlm_cache_init(struct fuse_inode *inode); /* Clean up a page cache lock manager */ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); -/* Lock a range of pages */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); +/* + * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a + * revoke processed while the reply is on the wire can mark it. @req is + * caller-owned storage, live until the matching commit or abort. The + * mode is not recorded until the grant is, so only the commit takes it. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end); -/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen); +/* + * Retire @req and record the grant [start, end] as one step under the + * cache lock. -EAGAIN means a revoke overlapped @req in flight and + * nothing was recorded; the caller must request again. @req is retired + * either way. + */ +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); -/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); +/* Retire @req without recording anything */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req); /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); -/* Check if a page range is already locked */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); +/* + * Hold the grants over [@offset, @offset + @length) against revocation + * until fuse_dlm_unpin(), which drops the pin this task last took. @pin + * is caller-owned storage, live until then. fuse_dlm_pin() waits out a + * revoke overlapping that range and must not be called with a folio + * held; fuse_dlm_trypin() never sleeps and fails instead. Neither may + * be held across a DLM request: that request is answered by the server + * the revoke came from. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin(struct fuse_inode *inode); + +/* + * The unpin of a fill whose reply lands in another task: @pin is dropped + * by node rather than by owner, and is live from the request until here. + */ +void fuse_dlm_unpin_span(struct fuse_inode *inode, struct fuse_dlm_span *pin); + +/* + * Pin a range and confirm the grant over it as one step, which is the + * order every IO site needs: pin first, confirm second, publish nothing + * if either fails. False means the caller may not touch the page cache + * over that range yet, whether because a revoke is draining or because + * no grant covers it. + */ +bool fuse_dlm_trypin_held(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length, + enum fuse_page_lock_mode mode); +bool fuse_dlm_trypin_held_span(struct fuse_inode *inode, + struct fuse_dlm_span *pin, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); + +/* + * Fence the writers that hold a grant over [@offset, @offset + @len) but + * have not dirtied under it yet, for the duration of a revoke. @len <= 0 + * means to EOF, as in fuse_notify_inval_inode(). @fence is caller-owned + * storage, live until the matching end. Between these the caller may + * drop coverage over that range knowing nothing will be dirtied under + * what it drops, and a write outside it is left alone. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len); +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence); /* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); -/* Is any part of the file held for write? */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); +/* Hold [start, end] again so writeback can send what it found revoked */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c18ded6cd394a6..6e5a7323169434 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -32,8 +32,71 @@ #include #include #include +#include #include "fuse_dlm_cache.h" +/* + * Page cache work driven by a NOTIFY invalidate is marked on the task. + * + * Writeback reached from there must not ask the server for a grant: the + * range is the one the server is revoking, and the request would go out + * from inside the handler the server is waiting on, with the folio locked + * and under writeback. See fuse_reverse_inval_inode() and + * fuse_iomap_writeback_range(). + */ +extern const char fuse_notify_ctx_key[]; + +/* + * What a task driving page cache work for a NOTIFY invalidate carries. + * + * @key tells it apart from anything else parked in journal_info. The range + * is the one being revoked: the lock over it is still this client's until + * the handler returns, so writeback of it need not ask for a grant, while + * anything outside it must. + */ +struct fuse_notify_ctx { + const char *key; + loff_t start; + loff_t end; /* inclusive; LLONG_MAX to EOF */ +}; + +static inline void *fuse_notify_ctx_enter(struct fuse_notify_ctx *ctx, + loff_t start, loff_t end) +{ + void *old = current->journal_info; + + ctx->key = fuse_notify_ctx_key; + ctx->start = start; + ctx->end = end; + current->journal_info = ctx; + return old; +} + +static inline void fuse_notify_ctx_leave(void *old) +{ + current->journal_info = old; +} + +static inline struct fuse_notify_ctx *fuse_notify_ctx(void) +{ + struct fuse_notify_ctx *ctx = current->journal_info; + + return (ctx && ctx->key == fuse_notify_ctx_key) ? ctx : NULL; +} + +static inline bool fuse_in_notify_ctx(void) +{ + return fuse_notify_ctx(); +} + +/* Is [@pos, @pos + @len) the range the revoke in progress is taking away? */ +static inline bool fuse_in_notify_range(loff_t pos, unsigned int len) +{ + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + + return ctx && pos >= ctx->start && pos + len - 1 <= ctx->end; +} + /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 @@ -131,15 +194,47 @@ struct dlm_locked_area * Force-DIO switch trigger: an exponentially weighted moving average of the * interval (in jiffies) between FUSE_NOTIFY_INVAL_INODE data invalidations for * a file. When the average spacing falls below FUSE_NOTIFY_DIO_INTERVAL -- a - * remote writer streaming invalidations -- and the file is open for writing - * here, it is latched into direct IO. These are the source-level (not - * externally tunable) parameters of the heuristic: EWMA weight 1/2^SHIFT, - * seeded and capped at SEED so it takes a short burst rather than a single - * notify to trip. + * remote writer streaming invalidations -- and the file is open here, it is + * latched into direct IO. These are the source-level (not externally tunable) + * parameters of the heuristic: EWMA weight 1/2^SHIFT, seeded and capped at SEED + * so it takes a short burst rather than a single notify to trip. + * + * The average is folded on arrival and cannot age on its own, so what takes the + * latch off again is the last invalidation reaching FUSE_NOTIFY_DIO_COLD old. */ #define FUSE_NOTIFY_DIO_INTERVAL max_t(unsigned long, HZ / 10, 1) #define FUSE_NOTIFY_EWMA_SHIFT 2 #define FUSE_NOTIFY_EWMA_SEED (2 * FUSE_NOTIFY_DIO_INTERVAL) +#define FUSE_NOTIFY_DIO_COLD (8 * FUSE_NOTIFY_DIO_INTERVAL) + +/* + * Streamed file trigger: the same buffer size arriving over and over is a + * task working through a file a record at a time. The sizes are folded + * into an exponentially weighted moving average (weight 1/2^SHIFT, kept + * shifted), and a run of FUSE_STREAM_RUN requests within + * 1/2^FUSE_STREAM_TOL_SHIFT of it says the task is still on it. The sample + * is capped to keep the shifted accumulator inside an unsigned int. + */ +#define FUSE_STREAM_EWMA_SHIFT 2 +#define FUSE_STREAM_TOL_SHIFT 3 +#define FUSE_STREAM_RUN 4 +#define FUSE_STREAM_EWMA_MAX (UINT_MAX >> FUSE_STREAM_EWMA_SHIFT) + +/* Under this size the copy through the page cache is not worth avoiding */ +#define FUSE_WRITE_STREAM_MIN (64 * 1024) +#define FUSE_READ_STREAM_MIN (10 * PAGE_SIZE) + +/* + * A write(2) extent held against the other writers of the same inode. + * Caller storage, live from fuse_write_range_lock() until the matching + * unlock. + */ +struct fuse_write_range { + /* Byte offsets, both inclusive */ + loff_t start; + loff_t end; + struct list_head list; +}; /** FUSE inode */ struct fuse_inode { @@ -159,6 +254,15 @@ struct fuse_inode { /** Time in jiffies until the file attributes are valid */ u64 i_time; + /* + * Time in jiffies until mode/uid/gid (the permission-check subset of + * STATX_BASIC_STATS) are valid. Tracked separately from i_time so that + * a partial statx refresh covering only the perm bits can extend the + * permission-check cache without falsely advancing i_time for the + * other (un-refreshed) attributes. + */ + u64 i_perm_time; + /* Which attributes are invalid */ u32 inval_mask; @@ -200,47 +304,76 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; - /* - * Server-materialized size: an upper bound for how far - * the server holds file data. Seeded from - * server-reported attributes, advanced when the server - * acknowledges data (writeback completion, - * fuse_write_update_attr()), lowered again on - * truncate. A read-modify-write of a block starting - * at or past this bound needs no READ request under a - * held DLM write lock: the server has no data there - * (see fuse_iomap_read_folio_range()). Protected by - * fi->lock. - */ - loff_t server_size; - - /* - * Serializes buffered-write page-cache dirtying against - * the forced-direct-IO latch transition driven by - * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which - * may be delivered by the same server thread that still - * owes a reply to an in-flight write holding the inode - * lock. The buffered writer holds this for read around - * the dirtying and re-checks the latch under it; the - * NOTIFY latch site takes it for write (trylock, never - * blocking) around its page-cache invalidate + latch set. - * Only regular files initialise it -- it shares storage - * with the readdir-cache union arm. - */ - struct percpu_rw_semaphore *wb_inval_rwsem; - /* * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations * for this whole file: notify_stamp is the jiffies of * the last one, notify_interval_ewma the EWMA of the * inter-arrival interval (jiffies, scaled by * 2^FUSE_NOTIFY_EWMA_SHIFT). A rapid stream (short - * average interval) with a local writer latches the - * inode into direct IO. Protected by fi->lock; regular - * files only (shares the readdir-cache union arm). + * average interval) with the file open here latches + * the inode into direct IO, and notify_stamp going + * stale takes it out again. Protected by fi->lock; + * regular files only (shares the readdir-cache union + * arm). */ unsigned long notify_stamp; unsigned int notify_interval_ewma; + + /* + * Buffered writes that have claimed an i_size + * extension and not yet dirtied it. + * + * The DLM path holds i_rwsem shared, so several + * writers extend i_size at once and each one is + * ahead of the server until its bytes are sent. + * While this is non zero the local size wins over + * the server's; see fuse_attr_cache_mask(). + * FUSE_I_SIZE_UNSTABLE cannot serve: it is one bit + * and every writer clears it. + */ + atomic_t size_extenders; + + /* + * The buffered writes in flight over this inode, + * one entry per write(2) over the bytes it covers. + * A write waits for the overlapping entries + * published before its own, so two writers on the + * same bytes do not interleave a folio at a time. + * + * Node local. The DLM grant over those bytes is + * held by the node rather than by a task: it orders + * this client against the rest of the cluster and + * says nothing about the writers on it. Nothing in + * the revoke path takes this, so a revoke never + * waits behind a write. + */ + spinlock_t wr_lock; + struct list_head wr_ranges; + wait_queue_head_t wr_wq; + + /* + * The buffered writes of this inode, whatever + * handle they come through: write_size_ewma is the + * moving average of their sizes and + * write_stream_run how many of the last ones came + * in at that size, which together say the file is + * being streamed; write_stream_next is where the + * next write has to land to carry the run of + * positions on, and write_stream_start the first + * byte of that run no writeback kick has covered. + * read_size_ewma and read_stream_run are the same + * average over the reads that could be cached, kept + * apart so a write phase and a read phase over one + * file do not fold into each other. + * Hints only, read and written without a lock; see + * fuse_stream_update(). + */ + unsigned int write_size_ewma; + unsigned int write_stream_run; + loff_t write_stream_next; + loff_t write_stream_start; + unsigned int read_size_ewma; + unsigned int read_stream_run; }; /* readdir cache (directory only) */ @@ -321,10 +454,16 @@ enum { * Latched into direct IO: a NOTIFY_INVAL_INODE arrived while the file * was open for writing here, so another (remote) entity is modifying it * concurrently. Reads and writes are routed direct (shared-lock - * parallel dio) until the last writer closes or the inode is mmapped. - * See fuse_reverse_inval_inode()/fuse_file_io_open(). + * parallel dio) until the notifies stop, the last writer closes, or the + * inode is mmapped. See fuse_reverse_inval_inode()/fuse_file_io_open(). */ FUSE_I_FORCE_DIO, + /* + * The page cache has been emptied under that latch, so no cached write + * from before it can still be in flight. Set once per latch by + * fuse_force_dio_drain(), cleared wherever FUSE_I_FORCE_DIO is. + */ + FUSE_I_FORCE_DIO_DRAINED, }; struct fuse_conn; @@ -561,6 +700,8 @@ struct fuse_req { #ifdef CONFIG_FUSE_IO_URING void *ring_entry; void *ring_queue; + /** Defers fuse_request_end() to the ring task's task work */ + struct callback_head ring_end_work; #endif /** When (in jiffies) the request was created */ unsigned long create_time; @@ -716,17 +857,6 @@ struct fuse_sync_bucket { struct rcu_head rcu; }; -/** - * DLM retry tracking for iomap write deadlock workaround. - * - * Temporary workaround until mainline iomap gains AOP_TRUNCATED_PAGE - * retry support. Tracks tasks that need to retry write operations due - * to DLM lock contention (-EAGAIN from FUSE server). - */ -struct fuse_dlm_retry { - bool retry_needed; -}; - /** * A Fuse connection. * @@ -1014,6 +1144,9 @@ struct fuse_conn { /* do we have support for dlm in the fs? */ unsigned int dlm:1; + /* Is extended lookup implemented by fs? */ + unsigned int lookupx:1; + /** Passthrough support for read/write IO */ unsigned int passthrough:1; @@ -1029,7 +1162,6 @@ struct fuse_conn { /* Use io_uring for communication */ unsigned int io_uring; - /* Does the filesystem support compound operations? */ unsigned int compound_open_getattr:1; /** Maximum stack depth for passthrough backing files */ @@ -1107,12 +1239,8 @@ struct fuse_conn { unsigned int req_timeout; } timeout; - /** - * XArray tracking tasks that need DLM retry. - * Maps task pointer -> struct fuse_dlm_retry. - * Temporary workaround for iomap write deadlock. - */ - struct xarray dlm_retry_tasks; + /* The foffset alignment in PAGE */ + unsigned int alignment_pages; }; /* @@ -1282,6 +1410,15 @@ struct fuse_io_args { struct { struct fuse_read_in in; u64 attr_ver; + /* + * The grant the folios are filled under, held + * from the request until the reply has filled + * them; see fuse_send_readpages(). @dlm_fi is + * the inode to drop it on, and NULL when there + * is no pin to drop. + */ + struct fuse_dlm_span dlm_pin; + struct fuse_inode *dlm_fi; } read; struct { struct fuse_write_in in; @@ -1597,6 +1734,9 @@ int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, /** CUSE pass fuse_direct_io() a file which f_mapping->host is not from FUSE */ #define FUSE_DIO_CUSE (1 << 1) +/** Caller holds i_rwsem shared, so fuse_set_nowrite() must not be used */ +#define FUSE_DIO_SHARED (1 << 2) + ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, loff_t *ppos, int flags); long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 431ae386872646..de79ffc5097db9 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -37,7 +37,14 @@ static bool __read_mostly enable_compound; module_param(enable_compound, bool, 0644); MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); -bool __read_mostly enable_large_folios = true; +/* + * A folio wider than a block carries per-block dirty state, which + * iomap_writeback_folio() clears whole after ->writeback_range. A run + * fuse defers and puts back with folio_mark_dirty() comes back dirty over + * the whole folio, so the next pass sends blocks this client never wrote. + * Off until there is a way to restore only the blocks that were dirty. + */ +bool __read_mostly enable_large_folios; module_param(enable_large_folios, bool, 0644); MODULE_PARM_DESC(enable_large_folios, "Enable large folios support"); @@ -219,23 +226,6 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } - - /* - * Free the coherency gate here rather than in ->free_inode: that runs - * from an RCU callback, where percpu_free_rwsem() may sleep in - * rcu_sync_dtor() if the write side has not fully quiesced. No user - * can remain by eviction time: gate readers hold a file reference and - * a concurrent notify holds an inode reference. wb_inval_rwsem lives - * in the regular-file union arm and is only ever allocated for regular - * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked - * regular files still own a gate); a directory's overlapping - * readdir-cache fields must not be misread. - */ - if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { - percpu_free_rwsem(fi->wb_inval_rwsem); - kfree(fi->wb_inval_rwsem); - fi->wb_inval_rwsem = NULL; - } } static int fuse_reconfigure(struct fs_context *fsc) @@ -261,6 +251,150 @@ static ino_t fuse_squash_ino(u64 ino64) return ino; } +/* + * Handle statx-specific attribute updates with partial attribute support. + */ +static void fuse_change_attributes_common_sx(struct inode *inode, + struct fuse_attr *attr, + struct fuse_statx *sx, + u64 attr_valid, u32 cache_mask, + u64 evict_ctr) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + u32 returned_attrs = sx->mask & STATX_BASIC_STATS; + + lockdep_assert_held(&fi->lock); + + /* + * Clear returned basic stats from invalid mask. + * + * Don't do this if this is coming from a fuse_iget() call and there + * might have been a racing evict which would've invalidated the result + * if the attr_version would've been preserved. + * + * !evict_ctr -> this is create + * fi->attr_version != 0 -> this is not a new inode + * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request + */ + if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) + set_mask_bits(&fi->inval_mask, returned_attrs, 0); + + fi->attr_version = atomic64_inc_return(&fc->attr_version); + + /* + * Only update i_time if we got all the attributes we care about. + * + * With writeback_cache (cache_mask set): cache_mask attributes are + * managed locally and their values from the server are ignored. + * So we only need all the OTHER attributes (non-cache_mask). + */ + if (cache_mask) { + /* writeback_cache: ignore cache_mask attrs, check everything else */ + if ((returned_attrs | cache_mask) == STATX_BASIC_STATS) + fi->i_time = attr_valid; + } else { + /* no writeback_cache: need all basic stats */ + if (returned_attrs == STATX_BASIC_STATS) + fi->i_time = attr_valid; + } + + /* + * Permission-check cache: independent of i_time so that a partial + * refresh which covers only mode/uid/gid (e.g. fuse_perm_getattr()) + * still extends the window during which fuse_permission() can hit + * the cache. Requires all three perm bits because generic_permission() + * needs the full triple. + */ + if ((returned_attrs & (STATX_MODE | STATX_UID | STATX_GID)) == + (STATX_MODE | STATX_UID | STATX_GID)) + fi->i_perm_time = attr_valid; + + /* + * Only update inode fields for attributes that were actually returned. + * TYPE is part of i_mode but already set during inode creation. + */ + if (returned_attrs & STATX_INO) + inode->i_ino = fuse_squash_ino(attr->ino); + if (returned_attrs & STATX_MODE) + inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); + if (returned_attrs & STATX_NLINK) + set_nlink(inode, attr->nlink); + if (returned_attrs & STATX_UID) + inode->i_uid = make_kuid(fc->user_ns, attr->uid); + if (returned_attrs & STATX_GID) + inode->i_gid = make_kgid(fc->user_ns, attr->gid); + if (returned_attrs & STATX_BLOCKS) + inode->i_blocks = attr->blocks; + + if (returned_attrs & STATX_ATIME) { + attr->atimensec = min_t(u32, attr->atimensec, NSEC_PER_SEC - 1); + inode_set_atime(inode, attr->atime, attr->atimensec); + } + /* mtime from server may be stale due to local buffered write */ + if ((returned_attrs & STATX_MTIME) && !(cache_mask & STATX_MTIME)) { + attr->mtimensec = min_t(u32, attr->mtimensec, NSEC_PER_SEC - 1); + inode_set_mtime(inode, attr->mtime, attr->mtimensec); + } + if ((returned_attrs & STATX_CTIME) && !(cache_mask & STATX_CTIME)) { + attr->ctimensec = min_t(u32, attr->ctimensec, NSEC_PER_SEC - 1); + inode_set_ctime(inode, attr->ctime, attr->ctimensec); + } + if (sx) { + /* Sanitize nsecs */ + sx->btime.tv_nsec = + min_t(u32, sx->btime.tv_nsec, NSEC_PER_SEC - 1); + + /* + * Btime has been queried, cache is valid (whether or not btime + * is available or not) so clear STATX_BTIME from inval_mask. + * + * Availability of the btime attribute is indicated in + * FUSE_I_BTIME + */ + set_mask_bits(&fi->inval_mask, STATX_BTIME, 0); + if (sx->mask & STATX_BTIME) { + set_bit(FUSE_I_BTIME, &fi->state); + fi->i_btime.tv_sec = sx->btime.tv_sec; + fi->i_btime.tv_nsec = sx->btime.tv_nsec; + } + } + + /* + * Common fields for both statx and getattr. + * + * inode->i_blkbits, which is what the page cache is tracked in, + * stays at the superblock block size; FUSE_INIT refuses a writeback + * connection whose block is not a page. What the server named per + * inode is reported as st_blksize out of fi->cached_i_blkbits. + */ + if (attr->blksize != 0) + fi->cached_i_blkbits = ilog2(attr->blksize); + else + fi->cached_i_blkbits = inode->i_sb->s_blocksize_bits; + + /* + * Don't set the sticky bit in i_mode, unless we want the VFS + * to check permissions. This prevents failures due to the + * check in may_delete(). + */ + fi->orig_i_mode = inode->i_mode; + if (!fc->default_permissions) + inode->i_mode &= ~S_ISVTX; + + fi->orig_ino = attr->ino; + + /* + * We are refreshing inode data and it is possible that another + * client set suid/sgid or security.capability xattr. So clear + * S_NOSEC. Ideally, we could have cleared it only if suid/sgid + * was set or if security.capability xattr was set. But we don't + * know if security.capability has been set or not. So clear it + * anyway. Its less efficient but should be safe. + */ + inode->i_flags &= ~S_NOSEC; +} + void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, u64 attr_valid, u32 cache_mask, @@ -271,6 +405,12 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, lockdep_assert_held(&fi->lock); + if (sx) { + return fuse_change_attributes_common_sx(inode, attr, sx, + attr_valid, cache_mask, + evict_ctr); + } + /* * Clear basic stats from invalid mask. * @@ -288,6 +428,7 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, fi->attr_version = atomic64_inc_return(&fc->attr_version); wake_up_all(&fc->attr_version_waitq); fi->i_time = attr_valid; + fi->i_perm_time = attr_valid; inode->i_ino = fuse_squash_ino(attr->ino); inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); @@ -382,14 +523,14 @@ u32 fuse_get_cache_mask(struct inode *inode) * for exactly what the grant covers: * * - size, when the server reports less than i_size and the tail it does not - * know about, [srv_size, i_size), is entirely under a write grant. Taking - * the server's answer would shrink i_size and have truncate_pagecache() - * throw the unwritten tail away. - * - mtime and ctime, while a write grant covers unwritten data: our writes - * have stamped them locally and the server's stamps predate them. Only - * while the cache is actually dirty, not for as long as the grant lives: - * a grant is held until it is revoked or the inode is evicted, and past - * the writeback the server's stamps are the newer ones. Keeping ours + * know about, [attr->size, i_size), is entirely under a write grant. + * Taking the server's answer would shrink i_size and have + * truncate_pagecache() throw the unwritten tail away. + * - mtime and ctime, while the page cache is dirty or under writeback: our + * writes have stamped them locally and the server's stamps predate them. + * Only while the cache is actually dirty, not for as long as a grant + * lives: a grant is held until it is revoked or the inode is evicted, and + * past the writeback the server's stamps are the newer ones. Keeping ours * beyond that would hide a remote chown or chmod indefinitely. * * A remote truncate cannot slip through. It has to revoke the grant first, @@ -413,16 +554,38 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, !S_ISREG(inode->i_mode)) return cache_mask; - if (!fuse_dlm_write_grant_exists(fi)) - return cache_mask; - + /* + * A dirty mapping keeps the local attributes authoritative even + * when no grant is recorded: a fault dirties pages under a + * page-mkwrite lock that is never recorded, and a truncate revokes + * the tail grants itself while cached writes above the new size + * are still waiting for writeback. + */ if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) cache_mask |= STATX_MTIME | STATX_CTIME; + /* + * The local size stays authoritative while the extension is + * covered by a write grant, and also while anything in + * [attr->size, size) is dirty or under writeback: those bytes + * exist only here, and taking the server's smaller size would + * truncate them away before they are ever sent. The grant check + * alone misses them, because a page-mkwrite grant is never + * recorded and a local truncate revokes its own tail grants. + * + * Both miss a write that has claimed its extension and not yet + * dirtied it: nothing is dirty there, and a NOTIFY can revoke the + * grant in between. With i_rwsem held shared several writers sit + * in that window at once, which is why they are counted rather + * than flagged. + */ if (have_size && size > (loff_t) attr->size && - fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, - FUSE_PAGE_LOCK_WRITE)) + (atomic_read(&fi->size_extenders) || + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE) || + filemap_range_needs_writeback(inode->i_mapping, attr->size, + size - 1))) cache_mask |= STATX_SIZE; return cache_mask; @@ -436,15 +599,42 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr struct fuse_inode *fi = get_fuse_inode(inode); u32 cache_mask; loff_t oldsize; + loff_t unsent_size = 0; struct timespec64 old_mtime; bool have_size = !sx || (sx->mask & STATX_SIZE); - u64 srv_size; + bool unsent = false; cache_mask = fuse_attr_cache_mask(inode, attr, have_size); - spin_lock(&fi->lock); - srv_size = attr->size; + /* + * fuse_attr_cache_mask() answered before the grant query slept, and a + * write below EOF bumps neither fi->attr_version nor + * fi->size_extenders: it extends nothing, so the version check cannot + * drop the reply and the count cannot hold the size. Folios can have + * been dirtied in the doomed range since the answer, and + * truncate_pagecache() below throws them away with no error to report + * it. + * + * Ask again here, after everything that sleeps, and keep the local + * size when the range still holds bytes the server has not seen. A + * remote truncate is unaffected: it revokes first, and the revoke + * launders and drops the range, so there is nothing here to find. + * + * Outside fi->lock, which excludes nothing this asks about: a folio is + * dirtied without it, so holding it would not make the answer any more + * current, only the walk longer. What the answer is tied to is the + * i_size it was bounded by, which the decision below insists on. + */ + if (have_size && !(cache_mask & STATX_SIZE) && fc->dlm && + fc->writeback_cache && S_ISREG(inode->i_mode)) { + unsent_size = i_size_read(inode); + unsent = (loff_t) attr->size < unsent_size && + filemap_range_needs_writeback(inode->i_mapping, + attr->size, + unsent_size - 1); + } + spin_lock(&fi->lock); if (cache_mask & STATX_SIZE) attr->size = i_size_read(inode); @@ -463,30 +653,24 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr return; } - /* - * srv_size is the size the server reported before the writeback - * cache_mask above replaced attr->size with the local value. It - * bounds how far the server can hold data, letting the iomap write - * path zero-fill expansion read-modify-writes instead of sending - * READ requests, see fuse_iomap_read_folio_range(). Only ever grow - * it here: stale attributes were rejected above and truncation - * lowers it directly. - */ - if (have_size && S_ISREG(inode->i_mode) && - (loff_t) srv_size > fi->server_size) - fi->server_size = srv_size; - old_mtime = inode_get_mtime(inode); fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, evict_ctr); oldsize = inode->i_size; + /* Only the range the walk above covered, or it answered for another */ + if (unsent && oldsize == unsent_size) { + cache_mask |= STATX_SIZE; + attr->size = oldsize; + } + /* * In case of writeback_cache enabled, the cached writes beyond EOF * extend local i_size without keeping userspace server in sync. So, * attr->size coming from server can be stale. We cannot trust it. + * Only update i_size if SIZE was actually returned by the server. */ - if (!(cache_mask & STATX_SIZE)) + if (have_size && !(cache_mask & STATX_SIZE)) i_size_write(inode, attr->size); spin_unlock(&fi->lock); @@ -502,12 +686,29 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr */ if (!(cache_mask & STATX_SIZE) && S_ISREG(inode->i_mode)) { bool inval = false; + bool have_mtime = !sx || (sx->mask & STATX_MTIME); - if (oldsize != attr->size) { - truncate_pagecache(inode, attr->size); + if (have_size && oldsize != attr->size) { + /* + * Not under DLM. This runs after fi->lock is + * dropped and takes nothing a writer holds, so a + * folio dirtied between the decision and the walk, + * or during it, is discarded with no error to report + * it. A real truncate may do this because + * fuse_set_nowrite() and i_rwsem hold the writers + * off; an attribute reply holds off nothing. + * + * Keeping the folios costs nothing either way. If + * the size was wrong they are written back and the + * size recovers; if it was right, the revoke that had + * to precede it already dropped the range and there + * is nothing here to discard. + */ + if (!(fc->dlm && fc->writeback_cache)) + truncate_pagecache(inode, attr->size); if (!fc->explicit_inval_data) inval = true; - } else if (fc->auto_inval_data) { + } else if (have_mtime && fc->auto_inval_data) { struct timespec64 new_mtime = { .tv_sec = attr->mtime, .tv_nsec = attr->mtimensec, @@ -718,6 +919,15 @@ static void fuse_invalidate_inode_entry(struct inode *inode) } } +/* + * Someone here has the file open and would use its page cache: a writeback + * writer, or any open in caching mode. Must be called under fi->lock. + */ +static bool fuse_inode_has_opener(struct fuse_inode *fi) +{ + return !list_empty(&fi->write_files) || fi->iocachectr > 0; +} + /* * Fold one FUSE_NOTIFY_INVAL_INODE data invalidation into the per-inode * moving average of the notification inter-arrival interval and report whether @@ -726,8 +936,8 @@ static void fuse_invalidate_inode_entry(struct inode *inode) * average is an EWMA (weight 1/2^FUSE_NOTIFY_EWMA_SHIFT); the sample is clamped * to FUSE_NOTIFY_EWMA_SEED so a notify after a long idle only cools the average * and cannot overflow the accumulator. Must be called under fi->lock; called - * for every data invalidation so the average stays current even while no local - * writer is open. + * for every data invalidation so the average stays current even while the file + * is not open here. */ static bool fuse_notify_inval_hot(struct fuse_inode *fi) { @@ -747,6 +957,9 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) return avg < FUSE_NOTIFY_DIO_INTERVAL; } +/* Address only; see fuse_notify_ctx_enter() */ +const char fuse_notify_ctx_key[1]; + /* * Revoke the DLM grants backing an invalidated byte range. Grants are * recorded page-aligned, so widen the revoke to page boundaries: dropping @@ -770,44 +983,63 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, * Drop a page-cache range on behalf of a NOTIFY invalidate. * * invalidate_inode_pages2_range() waits out folios under writeback and - * launders dirty ones, both of which need a FUSE_WRITE reply. While - * writepages are frozen (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, - * pre-SETATTR flush) no reply can arrive, because fuse_flush_writepages() - * parks the request on fi->queued_writes until fuse_release_nowrite(). A - * server that revokes from inside the handler it is revoking for then - * deadlocks against its own reply. fuse_do_setattr() states the same rule - * for its own invalidate. + * launders dirty ones, both of which need a FUSE_WRITE reply. It is only + * needed when the range can hold data the server has not seen. + * + * @may_be_dirty false says it cannot, on the strength of the DLM range + * record: every way a folio gets dirtied under a grant raises that record + * before the data lands, so a range it reports clean has no dirty folio to + * launder. invalidate_mapping_pages() then drops the same folios without + * ever waiting for the server. * - * So while frozen use invalidate_mapping_pages(), which skips dirty and - * under-writeback folios and never blocks. The stale clean folios still - * go, and the freezes that span a request drop the cache themselves once - * they complete: fuse_do_setattr() invalidates the mapping after releasing - * the freeze, the O_TRUNC open path calls truncate_pagecache(). + * The same substitution is forced while writepages are frozen + * (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, pre-SETATTR flush), + * where no reply can arrive because fuse_flush_writepages() parks the + * request on fi->queued_writes until fuse_release_nowrite(). A server that + * revokes from inside the handler it is revoking for would otherwise + * deadlock against its own reply. fuse_do_setattr() states the same rule + * for its own invalidate. There the dirty folios are left behind, and the + * freezes that span a request drop the cache themselves once they complete: + * fuse_do_setattr() invalidates the mapping after releasing the freeze, the + * O_TRUNC open path calls truncate_pagecache(). + * + * A drop that can launder writes the range back itself first. + * fuse_launder_folio() is handed one folio at a time and sends a FUSE_WRITE + * for each, where a writeback pass batches the same bytes up to + * fc->max_write; what it sends the drop then only waits for. */ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, bool may_be_dirty) { struct fuse_inode *fi = get_fuse_inode(inode); + loff_t last; bool frozen; spin_lock(&fi->lock); frozen = fi->writectr < 0; spin_unlock(&fi->lock); - if (frozen) + if (frozen || !may_be_dirty) { invalidate_mapping_pages(inode->i_mapping, start, end); - else - invalidate_inode_pages2_range(inode->i_mapping, start, end); + return; + } + + last = end == (pgoff_t)-1 ? LLONG_MAX : + (((loff_t)end + 1) << PAGE_SHIFT) - 1; + filemap_write_and_wait_range(inode->i_mapping, + (loff_t)start << PAGE_SHIFT, last); + invalidate_inode_pages2_range(inode->i_mapping, start, end); } int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { - struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; + loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; + bool tracked; inode = fuse_ilookup(fc, nodeid, NULL); if (!inode) @@ -834,92 +1066,159 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, forget_all_cached_acls(inode); security_inode_invalidate_secctx(inode); if (offset >= 0) { + /* + * Everything below drives this inode's page cache on behalf + * of the revoke, so mark the task: writeback reached from + * here must not send a DLM request. See + * fuse_iomap_writeback_range(). + */ + struct fuse_notify_ctx ctx; + struct fuse_dlm_span fence; + void *notify_ctx; + bool fenced; + pg_start = offset >> PAGE_SHIFT; if (len <= 0) pg_end = -1; else pg_end = (offset + len - 1) >> PAGE_SHIFT; + /* Byte bounds of the same region */ + end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + + notify_ctx = fuse_notify_ctx_enter(&ctx, offset, end_byte); + + /* + * Fence the writers that hold a grant over this range but + * have not dirtied under it yet. Their bytes are in no page + * cache and on no wire, so nothing below can find them, and + * once the grant is gone they would go out behind the + * handover. Published before the flush so what it drains is + * flushed with everything else. + * + * Only the writers over this range: a write elsewhere in the + * file holds a grant this handler does not touch, and neither + * waits for the other. + * + * Entered after fuse_notify_ctx_enter(): the page cache work + * below is this handler's own and must not fence itself. + * + * A fenced writer may be waiting for a FUSE_READ or a + * FUSE_WRITE reply, so this waits on the server the same way + * the flush below does. + * + * Regular files only: the record shares the readdir cache + * union arm and exists nowhere else. Latched into a local, + * since fc->dlm can be cleared while this runs and the fence + * has to come off the list either way. + */ + fenced = S_ISREG(inode->i_mode) && fc->dlm && + fc->writeback_cache; + if (fenced) + fuse_dlm_revoke_begin(fi, &fence, offset, len); + /* - * A data invalidation means another (remote) entity is modifying - * the file. Two things happen here: + * A data invalidation means another (remote) entity is + * modifying the file. Two things happen here: * - * 1. Coherency. Drop the affected page-cache range so no local - * read returns a folio the remote modify has superseded. This - * runs under the write side of the per-inode coherency gate - * (wb_inval_rwsem), which fences cache-serving buffered reads - * and buffered writes out for the whole invalidate. Unlike the - * old best-effort trylock this BLOCKS -- the notify has - * priority: percpu_down_write() parks new gate readers, drains - * in-flight ones, then invalidates. A blocking writer here is - * safe only under a server that services request replies on - * threads other than the one delivering this notify: the write - * side waits for gate readers to drain, and a cache-miss read - * holds the read side across its FUSE_READ round-trip. redfs' - * dlm server provides that contract; a server that cannot must - * not enable writeback+dlm. + * 1. Coherency. Drop the affected page-cache range so no + * local read returns a folio the remote modify has + * superseded. Nothing is fenced out for it. A read + * racing the drop either misses and refetches or returns + * data that was current when it was copied. A write + * racing it is caught on the way out instead: its bytes + * were recorded before they were dirtied, this revoke + * marks the range rather than forgetting it, and + * writeback holds the range again before sending + * anything it finds marked that way. * - * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under - * fi->lock, updated for every data invalidation) of how fast - * these arrive; when they come in a rapid stream -- a remote - * writer repeatedly invalidating -- and the inode is also open - * for writing here, latch it into direct IO until the last - * writer closes or it is mmapped. When latched, drop the whole - * mapping rather than just the notified range, or dirty folios - * outside it would be invisible to the forced direct reads - * (stale read / lost write). Latching is opt-in via the - * enable_notify_dio module parameter and off by default; the - * average is kept up to date either way, so enabling it at - * runtime takes effect on the next storm rather than after a - * warm-up. Clearing it at runtime stops new latches but lets - * already-latched inodes run out on the usual exits (last - * writer closes, or mmap). + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), + * under fi->lock, updated for every data invalidation) of + * how fast these arrive; when they come in a rapid stream + * -- a remote writer repeatedly invalidating -- and the + * inode is open here, latch it into direct IO until the + * stream stops, the last writer closes, or it is mmapped. + * A reader-only inode is latched too: what it caches + * between two invalidations is dropped again before it can + * be read twice, so the cache costs the folios and the read + * grants behind them and returns nothing. + * When latched, drop the whole mapping rather than just + * the notified range, or dirty folios outside it would be + * invisible to the forced direct reads (stale read / lost + * write). Latching is opt-in via the enable_notify_dio + * module parameter and off by default; the average is kept + * up to date either way, so enabling it at runtime takes + * effect on the next storm rather than after a warm-up. + * Clearing it at runtime stops new latches but lets + * already-latched inodes run out on the usual exits. * - * The gate (and the average) exist only for writeback+dlm regular - * files; elsewhere wb_sem is NULL and the invalidate runs - * unserialized (best-effort), as before. An mmapped inode - * keeps the gate -- fuse_cache_read_iter() and - * fuse_cache_write_iter() enter it unconditionally and rely - * on the revoke staying fenced -- but is never latched: - * a mapping needs the page cache, and fuse_file_mmap() - * reverts any latch it races with. + * The average and the latch exist only for writeback+dlm + * regular files; elsewhere there is no record to consult and + * the range is dropped as it always was. An mmapped inode is + * never latched: a mapping needs the page cache, and + * fuse_file_mmap() reverts any latch it races with. */ - if (S_ISREG(inode->i_mode) && fc->writeback_cache && - fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi)) - wb_sem = fi->wb_inval_rwsem; + tracked = S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi); - if (wb_sem) { - bool hot, has_writer, latched = false; + if (tracked) { + bool hot, has_opener, latched = false; + bool may_be_dirty, has_pages; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); - has_writer = !list_empty(&fi->write_files); + has_opener = fuse_inode_has_opener(fi); spin_unlock(&fi->lock); /* - * Priority write side: park new gate readers, - * drain in-flight ones, then invalidate. Blocks - * (unlike the old trylock) -- see the contract in - * the comment above. + * What this notify has to do. Nothing cached in the + * range means the drop is a no-op and the revoke is + * the whole job. Otherwise the page cache says + * whether the drop has to launder, which is what + * makes it wait for a FUSE_WRITE reply. */ - percpu_down_write(wb_sem); + has_pages = filemap_range_has_page(inode->i_mapping, + offset, end_byte); + may_be_dirty = filemap_range_needs_writeback( + inode->i_mapping, offset, end_byte); /* - * Revoke the DLM lock range under the gate write - * side, atomically with the page drop: gate readers - * re-validate their grant right after entering, and - * a grant that passed that check must stay visible - * for their whole gate hold. + * Put unwritten data on the server while the grant + * still covers it, rather than leaving it to the drop + * below. After the revoke writeback would have to + * take the range again to send those bytes: a DLM + * round trip from inside the handler the server is + * waiting on. do_writepages() runs in this context, + * so the grant is asked for before the revoke. + * + * One pass is enough: the fence above has drained the + * writers that held a grant without having dirtied + * under it, and refuses new ones, so nothing can turn + * up dirty behind this. + * + * Waited out here rather than left to the drop, which + * launders when the record says the range may be dirty + * and so waits for these same replies. One explicit + * wait, before the revoke, and the drop then finds + * nothing under writeback to block on. Either way a + * server that revokes from a thread it also needs to + * answer FUSE_WRITE on deadlocks here, the same + * contract fuse_notify_invalidate_range() states for a + * frozen inode. The error is left to the mapping, + * where fsync collects it. */ - if (fc->dlm && fc->writeback_cache) - fuse_dlm_revoke_inval_range(fi, offset, len); + if (has_pages && may_be_dirty) + filemap_write_and_wait_range(inode->i_mapping, + offset, end_byte); + + fuse_dlm_revoke_inval_range(fi, offset, len); - if (enable_notify_dio && hot && has_writer && + if (enable_notify_dio && hot && has_opener && !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); - if (!list_empty(&fi->write_files)) { + if (fuse_inode_has_opener(fi)) { set_bit(FUSE_I_FORCE_DIO, &fi->state); latched = true; } @@ -929,28 +1228,41 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* * Latched: drop the whole mapping (dirty folios * outside the notified range would be invisible to - * the forced direct reads). Otherwise just the - * notified range. + * the forced direct reads), laundering only what the + * mapping says may be dirty, which for an inode + * latched with no writer is nothing. Otherwise just + * the notified range, and only if anything is cached + * there. */ - if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1); - else - fuse_notify_invalidate_range(inode, pg_start, - pg_end); + if (fuse_inode_force_dio(inode)) { + bool dirty; - percpu_up_write(wb_sem); + dirty = filemap_range_needs_writeback( + inode->i_mapping, 0, LLONG_MAX); + fuse_notify_invalidate_range(inode, 0, -1, dirty); + } else if (has_pages) { + fuse_notify_invalidate_range(inode, pg_start, + pg_end, + may_be_dirty); + } if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { - /* No gate on this inode (DAX, backing, non-regular, - * or the gate allocation failed): drop the lock - * range unserialized (best-effort), as before. */ + /* + * No record on this inode (DAX, backing, non-regular, + * or no DLM), so assume the range can hold unwritten + * data and drop it as before. + */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - fuse_notify_invalidate_range(inode, pg_start, pg_end); + fuse_notify_invalidate_range(inode, pg_start, pg_end, + true); } + if (fenced) + fuse_dlm_revoke_end(fi, &fence); + fuse_notify_ctx_leave(notify_ctx); } iput(inode); return 0; @@ -1364,11 +1676,11 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->initialized = 0; fc->connected = 1; fc->dlm = 1; + fc->lookupx = 1; /* module option for now */ fc->compound_open_getattr = enable_compound; - xa_init(&fc->dlm_retry_tasks); atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); @@ -1421,7 +1733,6 @@ void fuse_conn_put(struct fuse_conn *fc) } if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_free(fc); - xa_destroy(&fc->dlm_retry_tasks); call_rcu(&fc->rcu, delayed_release); } EXPORT_SYMBOL_GPL(fuse_conn_put); @@ -1751,8 +2062,26 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, } if (flags & FUSE_ASYNC_DIO) fc->async_dio = 1; - if (flags & FUSE_WRITEBACK_CACHE) + if (flags & FUSE_WRITEBACK_CACHE) { + /* + * A buffered write goes through iomap, which + * tracks a folio a block at a time and fills + * any block the write covers only part of. + * Writeback then sends whole dirty blocks. + * Everything the DLM path grants, pins and + * revokes is a whole page, and the two have + * only ever been the same size here. + * + * Refuse the connection rather than run at a + * granularity nothing has been tried at. + */ + if (fm->sb->s_blocksize_bits != PAGE_SHIFT) { + pr_err("fuse: writeback cache needs a page sized block, got %lu\n", + fm->sb->s_blocksize); + ok = false; + } fc->writeback_cache = 1; + } if (flags & FUSE_PARALLEL_DIROPS) fc->parallel_dirops = 1; if (flags & FUSE_HANDLE_KILLPRIV) @@ -1822,6 +2151,14 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, fc->max_stack_depth = arg->max_stack_depth; fm->sb->s_stack_depth = arg->max_stack_depth; } + + if (flags & FUSE_ALIGN_PG_ORDER) { + if (arg->align_page_order > 0) { + fc->alignment_pages = + (1UL << arg->align_page_order) + >> PAGE_SHIFT; + } + } if (flags & FUSE_NO_EXPORT_SUPPORT) fm->sb->s_export_op = &fuse_export_fid_operations; if (flags & FUSE_ALLOW_IDMAP) { diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index ebe2735c186e08..4c4ecf6cb4ee89 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -461,6 +461,8 @@ struct fuse_file_lock { * init_out.request_timeout contains the timeout (in secs) * FUSE_INVAL_INODE_ENTRY: invalidate inode aliases when doing inode invalidation * FUSE_EXPIRE_INODE_ENTRY: expire inode aliases when doing inode invalidation + * FUSE_ALIGN_PG_ORDER: page order (power of 2 exponent for number of pages) for + * optimal io-size alignment * FUSE_URING_REDUCED_Q: Client (kernel) supports less queues - Server is free * to register between 1 and nr-core io-uring queues * FUSE_SETATTR_WRITEBACK: kernel marks writeback-initiated SETATTR requests @@ -610,6 +612,14 @@ struct fuse_file_lock { */ #define FUSE_OPEN_KILL_SUIDGID (1 << 0) +/** + * Lookup flags + * FUSE_LOOKUPX_FOR_REVALIDATE: lookup called from revalidate + * FUSE_LOOKUPX_TARGET_WASDIR: (hint) the lookup target was a directory + */ +#define FUSE_LOOKUPX_FOR_REVALIDATE (1 << 0) +#define FUSE_LOOKUPX_TARGET_WAS_DIR (1 << 1) + /** * setxattr flags * FUSE_SETXATTR_ACL_KILL_SGID: Clear SGID when system.posix_acl_access is set @@ -696,6 +706,9 @@ enum fuse_opcode { */ FUSE_COMPOUND = 101, + /* Extented lookup operation */ + FUSE_LOOKUPX = 102, + /* CUSE specific operations */ CUSE_INIT = 4096, @@ -732,6 +745,15 @@ struct fuse_entry_out { struct fuse_attr attr; }; +struct fuse_lookupx_in { + uint32_t lookup_flags; +}; + +struct fuse_lookupx_out { + struct fuse_entry_out entry; + uint32_t mask; /* Mask of valid attributes in statx format */ +}; + struct fuse_forget_in { uint64_t nlookup; }; @@ -943,6 +965,9 @@ struct fuse_init_in { #define FUSE_COMPAT_INIT_OUT_SIZE 8 #define FUSE_COMPAT_22_INIT_OUT_SIZE 24 +/* + * align_page_order: Number of pages for optimal IO, or a multiple of that + */ struct fuse_init_out { uint32_t major; uint32_t minor; @@ -957,7 +982,9 @@ struct fuse_init_out { uint32_t flags2; uint32_t max_stack_depth; uint16_t request_timeout; - uint16_t unused[11]; + uint8_t align_page_order; + uint8_t padding; + uint16_t unused[10]; }; #define CUSE_INIT_INFO_MAX 4096