vfs: use fast_list for superblock's inode listfast_list

Use the new fast_list for super_block.s_inodes.

This gives similar performance to Dave's dlock list approach [1]; lock
contention is now moved to the lru_list locks.

Iteration is now fully lockless - instead we iterate using
rcu_read_lock(), which means we must take care for racing with removal.

Generally this is already handled - code that iterates over s_inodes
takes i_lock and checks i_state, skipping inodes that are
I_WILL_FREE|I_FREEING. However, code may also check for nonzero
i_sb_list_idx if it wishes to iterate over precisely the inodes that are
on the s_inodes list.

[1]: https://lore.kernel.org/linux-fsdevel/20231206060629.2827226-4-david@fromorbit.com/

Cc: Christian Brauner <brauner@kernel.org>
Cc: Dave Chinner <dchinner@redhat.com>
Cc: Waiman Long <longman@redhat.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

1 files changed, 32 insertions, 28 deletions

diff --git a/fs/inode.c b/fs/inode.c
index 471ae4a31549..1b6a2f5cede4 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -230,8 +230,15 @@ int inode_init_always(struct super_block *sb, struct inode *inode)
 #endif
 	inode->i_flctx = NULL;
 
-	if (unlikely(security_inode_alloc(inode)))
+	int idx = fast_list_get_idx(&sb->s_inodes);
+	if (idx < 0)
 		return -ENOMEM;
+	inode->i_sb_list_idx = idx;
+
+	if (unlikely(security_inode_alloc(inode))) {
+		fast_list_remove(&sb->s_inodes, idx);
+		return -ENOMEM;
+	}
 
 	this_cpu_inc(nr_inodes);
 
@@ -425,7 +432,6 @@ void inode_init_once(struct inode *inode)
 	INIT_LIST_HEAD(&inode->i_io_list);
 	INIT_LIST_HEAD(&inode->i_wb_list);
 	INIT_LIST_HEAD(&inode->i_lru);
-	INIT_LIST_HEAD(&inode->i_sb_list);
 	__address_space_init_once(&inode->i_data);
 	i_size_ordered_init(inode);
 }
@@ -540,19 +546,14 @@ static void inode_wait_for_lru_isolating(struct inode *inode)
  */
 void inode_sb_list_add(struct inode *inode)
 {
-	spin_lock(&inode->i_sb->s_inode_list_lock);
-	list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
-	spin_unlock(&inode->i_sb->s_inode_list_lock);
+	*genradix_ptr_inlined(&inode->i_sb->s_inodes.items, inode->i_sb_list_idx) = inode;
 }
 EXPORT_SYMBOL_GPL(inode_sb_list_add);
 
 static inline void inode_sb_list_del(struct inode *inode)
 {
-	if (!list_empty(&inode->i_sb_list)) {
-		spin_lock(&inode->i_sb->s_inode_list_lock);
-		list_del_init(&inode->i_sb_list);
-		spin_unlock(&inode->i_sb->s_inode_list_lock);
-	}
+	*genradix_ptr(&inode->i_sb->s_inodes.items, inode->i_sb_list_idx) = NULL;
+	inode->i_sb_list_idx = 0;
 }
 
 static unsigned long hash(struct super_block *sb, unsigned long hashval)
@@ -785,12 +786,16 @@ static void dispose_list(struct list_head *head)
  */
 void evict_inodes(struct super_block *sb)
 {
-	struct inode *inode, *next;
+	struct genradix_iter iter;
+	void **i;
 	LIST_HEAD(dispose);
-
 again:
-	spin_lock(&sb->s_inode_list_lock);
-	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
+	rcu_read_lock();
+	genradix_for_each(&sb->s_inodes.items, iter, i) {
+		struct inode *inode = *((struct inode **) i);
+		if (!inode)
+			continue;
+
 		if (atomic_read(&inode->i_count))
 			continue;
 
@@ -815,13 +820,13 @@ again:
 		 * bit so we don't livelock.
 		 */
 		if (need_resched()) {
-			spin_unlock(&sb->s_inode_list_lock);
+			rcu_read_unlock();
 			cond_resched();
 			dispose_list(&dispose);
 			goto again;
 		}
 	}
-	spin_unlock(&sb->s_inode_list_lock);
+	rcu_read_unlock();
 
 	dispose_list(&dispose);
 }
@@ -835,12 +840,16 @@ EXPORT_SYMBOL_GPL(evict_inodes);
  */
 void invalidate_inodes(struct super_block *sb)
 {
-	struct inode *inode, *next;
+	struct genradix_iter iter;
+	void **i;
 	LIST_HEAD(dispose);
-
 again:
-	spin_lock(&sb->s_inode_list_lock);
-	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
+	rcu_read_lock();
+	genradix_for_each(&sb->s_inodes.items, iter, i) {
+		struct inode *inode = *((struct inode **) i);
+		if (!inode)
+			continue;
+
 		spin_lock(&inode->i_lock);
 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
 			spin_unlock(&inode->i_lock);
@@ -856,13 +865,13 @@ again:
 		spin_unlock(&inode->i_lock);
 		list_add(&inode->i_lru, &dispose);
 		if (need_resched()) {
-			spin_unlock(&sb->s_inode_list_lock);
+			rcu_read_unlock();
 			cond_resched();
 			dispose_list(&dispose);
 			goto again;
 		}
 	}
-	spin_unlock(&sb->s_inode_list_lock);
+	rcu_read_unlock();
 
 	dispose_list(&dispose);
 }
@@ -1290,12 +1299,7 @@ again:
 	hlist_add_head_rcu(&inode->i_hash, head);
 	spin_unlock(&inode->i_lock);
 
-	/*
-	 * Add inode to the sb list if it's not already. It has I_NEW at this
-	 * point, so it should be safe to test i_sb_list locklessly.
-	 */
-	if (list_empty(&inode->i_sb_list))
-		inode_sb_list_add(inode);
+	inode_sb_list_add(inode);
 unlock:
 	spin_unlock(&inode_hash_lock);