fuse: add all improvements for io500

Documentation/admin-guide/sysctl/fs.rst

@@ -367,6 +367,12 @@ greater than max_request_timeout, the system will use max_request_timeout as the
 timeout. 0 indicates no max request timeout. The maximum value that can be set
 is 65535.
 
+If the server did not specify a timeout at mount and both of the above are set
+to 0, the timeout is derived from ``/proc/sys/kernel/hung_task_timeout_secs``
+instead, so that a server that stops answering aborts the connection rather than
+leaving the waiters around for the hung task detector to report on. Setting
+hung_task_timeout_secs to 0 disables this fallback as well.
+
 For timeouts, if the server does not respond to the request by the time
 the set timeout elapses, then the connection to the fuse server will be aborted.
 Please note that the timeouts are not 100% precise (eg you may set 60 seconds but

fs/fuse/dev_uring.c

@@ -12,6 +12,7 @@
 #include <linux/fs.h>
 #include <linux/io_uring/cmd.h>
 #include <linux/page-flags.h>
+#include <linux/task_work.h>
 
 static bool __read_mostly enable_uring;
 module_param(enable_uring, bool, 0644);
@@ -99,8 +100,8 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue)
 	}
 }
 
-static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
-			       int error)
+static void __fuse_uring_req_end(struct fuse_ring_ent *ent,
+				 struct fuse_req *req, int error)
 {
 	struct fuse_ring_queue *queue = ent->queue;
 	struct fuse_ring *ring = queue->ring;
@@ -123,6 +124,43 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
 		req->out.h.error = error;
 
 	clear_bit(FR_SENT, &req->flags);
+}
+
+static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
+			       int error)
+{
+	__fuse_uring_req_end(ent, req, error);
+	fuse_request_end(req);
+}
+
+static void fuse_uring_req_end_work(struct callback_head *work)
+{
+	struct fuse_req *req = container_of(work, struct fuse_req,
+					    ring_end_work);
+
+	fuse_request_end(req);
+}
+
+/*
+ * On the commit path ->uring_cmd() runs with ctx->uring_lock held by
+ * io_uring_enter(). fuse_request_end() wakes the request submitter, which
+ * typically preempts the ring task right away (same CPU) - while the mutex
+ * is still held. Defer the completion to task work, which runs once the
+ * submission path has released the lock (in io_cqring_wait() or on return
+ * to userspace), so the ring task can finish its critical section first.
+ */
+static void fuse_uring_req_end_deferred(struct fuse_ring_ent *ent,
+					struct fuse_req *req, int error,
+					unsigned int issue_flags)
+{
+	__fuse_uring_req_end(ent, req, error);
+
+	if (!(issue_flags & IO_URING_F_UNLOCKED)) {
+		init_task_work(&req->ring_end_work, fuse_uring_req_end_work);
+		if (!task_work_add(current, &req->ring_end_work, TWA_RESUME))
+			return;
+	}
+
 	fuse_request_end(req);
 }
 
@@ -1059,7 +1097,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
 
 	err = fuse_uring_copy_from_ring(ring, req, ent);
 out:
-	fuse_uring_req_end(ent, req, err);
+	fuse_uring_req_end_deferred(ent, req, err, issue_flags);
 }
 
 /*
@@ -1500,6 +1538,26 @@ static void fuse_uring_send_in_task(struct io_uring_cmd *cmd,
 	fuse_uring_send(ent, cmd, err, issue_flags);
 }
 
+/*
+ * The request was already copied to the ring buffer in the submitter's
+ * context, only the io_uring cmd completion is left to do.
+ * io_uring_cmd_done() must not run in the submitter's context as it would
+ * have to take ctx->uring_lock (io_uring_cmd_del_cancelable()) - a mutex
+ * the ring task holds across its whole submission path and frequently gets
+ * preempted under while the just-woken submitter runs.
+ */
+static void fuse_uring_send_prepared_in_task(struct io_uring_cmd *cmd,
+					     unsigned int issue_flags)
+{
+	struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);
+	int err = 0;
+
+	if (unlikely(issue_flags & IO_URING_F_TASK_DEAD))
+		err = -ECANCELED;
+
+	fuse_uring_send(ent, cmd, err, issue_flags);
+}
+
 static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring,
 						       bool background)
 {
@@ -1599,7 +1657,9 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg)
 						 IO_URING_F_UNLOCKED);
 			return;
 		}
-		fuse_uring_send(ent, cmd, 0, IO_URING_F_UNLOCKED);
+		uring_cmd_set_ring_ent(cmd, ent);
+		io_uring_cmd_complete_in_task(cmd,
+					      fuse_uring_send_prepared_in_task);
 	}
 }
 

fs/fuse/dir.c

@@ -2100,18 +2100,35 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		WARN_ON(!(attr->ia_valid & ATTR_SIZE));
 		WARN_ON(attr->ia_size != 0);
 		if (fc->atomic_o_trunc) {
+			struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+
 			/*
 			 * No need to send request to userspace, since actual
 			 * truncation has already been done by OPEN.  But still
 			 * need to truncate page cache.
+			 *
+			 * Revoke and drop under the coherency gate write side,
+			 * like the NOTIFY invalidate path: a gate reader that
+			 * already re-validated its grant must not have the
+			 * lock tree and the cache yanked mid-hold, or it
+			 * would repopulate the truncated range trusting a
+			 * grant that no longer exists.  Waiting for gate
+			 * readers here is safe: we hold i_rwsem exclusive, so
+			 * no gate holder can be waiting on it (the write path
+			 * takes i_rwsem before the gate, the read path never
+			 * takes it).
 			 */
+			if (wb_sem)
+				percpu_down_write(wb_sem);
 			if (fc->dlm && fc->writeback_cache)
 				fuse_dlm_cache_release_locks(fi);
 			spin_lock(&fi->lock);
 			fi->server_size = 0;
 			i_size_write(inode, 0);
 			spin_unlock(&fi->lock);
 			truncate_pagecache(inode, 0);
+			if (wb_sem)
+				percpu_up_write(wb_sem);
 			goto out;
 		}
 		file = NULL;
@@ -2222,11 +2239,23 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	 */
 	if ((is_truncate || !is_wb) &&
 	    S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) {
+		struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+
+		/*
+		 * Revoke and drop under the coherency gate write side; see
+		 * the atomic-O_TRUNC branch above.  i_rwsem is held
+		 * exclusive here as well (setattr), so waiting out gate
+		 * readers cannot deadlock.
+		 */
+		if (wb_sem)
+			percpu_down_write(wb_sem);
 		if (fc->dlm && fc->writeback_cache)
 			fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1);
 
 		truncate_pagecache(inode, outarg.attr.size);
 		invalidate_inode_pages2(mapping);
+		if (wb_sem)
+			percpu_up_write(wb_sem);
 	}
 
 	clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);