summaryrefslogtreecommitdiff
path: root/fs
diff options
context:
space:
mode:
Diffstat (limited to 'fs')
-rw-r--r--fs/drop_caches.c21
-rw-r--r--fs/gfs2/ops_fstype.c22
-rw-r--r--fs/inode.c60
-rw-r--r--fs/notify/fsnotify.c25
-rw-r--r--fs/quota/dquot.c53
-rw-r--r--fs/super.c9
6 files changed, 104 insertions, 86 deletions
diff --git a/fs/drop_caches.c b/fs/drop_caches.c
index d45ef541d848..72c59fb22c81 100644
--- a/fs/drop_caches.c
+++ b/fs/drop_caches.c
@@ -18,10 +18,15 @@ int sysctl_drop_caches;
static void drop_pagecache_sb(struct super_block *sb, void *unused)
{
- struct inode *inode, *toput_inode = NULL;
+ struct genradix_iter iter;
+ void **i;
+
+ rcu_read_lock();
+ genradix_for_each(&sb->s_inodes.items, iter, i) {
+ struct inode *inode = *((struct inode **) i);
+ if (!inode)
+ continue;
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
spin_lock(&inode->i_lock);
/*
* We must skip inodes in unusual state. We may also skip
@@ -35,17 +40,15 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)
}
__iget(inode);
spin_unlock(&inode->i_lock);
- spin_unlock(&sb->s_inode_list_lock);
+ rcu_read_unlock();
invalidate_mapping_pages(inode->i_mapping, 0, -1);
- iput(toput_inode);
- toput_inode = inode;
+ iput(inode);
cond_resched();
- spin_lock(&sb->s_inode_list_lock);
+ rcu_read_lock();
}
- spin_unlock(&sb->s_inode_list_lock);
- iput(toput_inode);
+ rcu_read_unlock();
}
int drop_caches_sysctl_handler(const struct ctl_table *table, int write,
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index e83d293c3614..9cb222e68154 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -1736,13 +1736,18 @@ static int gfs2_meta_init_fs_context(struct fs_context *fc)
*/
static void gfs2_evict_inodes(struct super_block *sb)
{
- struct inode *inode, *toput_inode = NULL;
+ struct genradix_iter iter;
+ void **i;
struct gfs2_sbd *sdp = sb->s_fs_info;
set_bit(SDF_EVICTING, &sdp->sd_flags);
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
+ rcu_read_lock();
+ genradix_for_each(&sb->s_inodes.items, iter, i) {
+ struct inode *inode = *((struct inode **) i);
+ if (!inode)
+ continue;
+
spin_lock(&inode->i_lock);
if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) &&
!need_resched()) {
@@ -1751,16 +1756,13 @@ static void gfs2_evict_inodes(struct super_block *sb)
}
atomic_inc(&inode->i_count);
spin_unlock(&inode->i_lock);
- spin_unlock(&sb->s_inode_list_lock);
-
- iput(toput_inode);
- toput_inode = inode;
+ rcu_read_unlock();
+ iput(inode);
cond_resched();
- spin_lock(&sb->s_inode_list_lock);
+ rcu_read_lock();
}
- spin_unlock(&sb->s_inode_list_lock);
- iput(toput_inode);
+ rcu_read_unlock();
}
static void gfs2_kill_sb(struct super_block *sb)
diff --git a/fs/inode.c b/fs/inode.c
index 471ae4a31549..1b6a2f5cede4 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -230,8 +230,15 @@ int inode_init_always(struct super_block *sb, struct inode *inode)
#endif
inode->i_flctx = NULL;
- if (unlikely(security_inode_alloc(inode)))
+ int idx = fast_list_get_idx(&sb->s_inodes);
+ if (idx < 0)
return -ENOMEM;
+ inode->i_sb_list_idx = idx;
+
+ if (unlikely(security_inode_alloc(inode))) {
+ fast_list_remove(&sb->s_inodes, idx);
+ return -ENOMEM;
+ }
this_cpu_inc(nr_inodes);
@@ -425,7 +432,6 @@ void inode_init_once(struct inode *inode)
INIT_LIST_HEAD(&inode->i_io_list);
INIT_LIST_HEAD(&inode->i_wb_list);
INIT_LIST_HEAD(&inode->i_lru);
- INIT_LIST_HEAD(&inode->i_sb_list);
__address_space_init_once(&inode->i_data);
i_size_ordered_init(inode);
}
@@ -540,19 +546,14 @@ static void inode_wait_for_lru_isolating(struct inode *inode)
*/
void inode_sb_list_add(struct inode *inode)
{
- spin_lock(&inode->i_sb->s_inode_list_lock);
- list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
- spin_unlock(&inode->i_sb->s_inode_list_lock);
+ *genradix_ptr_inlined(&inode->i_sb->s_inodes.items, inode->i_sb_list_idx) = inode;
}
EXPORT_SYMBOL_GPL(inode_sb_list_add);
static inline void inode_sb_list_del(struct inode *inode)
{
- if (!list_empty(&inode->i_sb_list)) {
- spin_lock(&inode->i_sb->s_inode_list_lock);
- list_del_init(&inode->i_sb_list);
- spin_unlock(&inode->i_sb->s_inode_list_lock);
- }
+ *genradix_ptr(&inode->i_sb->s_inodes.items, inode->i_sb_list_idx) = NULL;
+ inode->i_sb_list_idx = 0;
}
static unsigned long hash(struct super_block *sb, unsigned long hashval)
@@ -785,12 +786,16 @@ static void dispose_list(struct list_head *head)
*/
void evict_inodes(struct super_block *sb)
{
- struct inode *inode, *next;
+ struct genradix_iter iter;
+ void **i;
LIST_HEAD(dispose);
-
again:
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
+ rcu_read_lock();
+ genradix_for_each(&sb->s_inodes.items, iter, i) {
+ struct inode *inode = *((struct inode **) i);
+ if (!inode)
+ continue;
+
if (atomic_read(&inode->i_count))
continue;
@@ -815,13 +820,13 @@ again:
* bit so we don't livelock.
*/
if (need_resched()) {
- spin_unlock(&sb->s_inode_list_lock);
+ rcu_read_unlock();
cond_resched();
dispose_list(&dispose);
goto again;
}
}
- spin_unlock(&sb->s_inode_list_lock);
+ rcu_read_unlock();
dispose_list(&dispose);
}
@@ -835,12 +840,16 @@ EXPORT_SYMBOL_GPL(evict_inodes);
*/
void invalidate_inodes(struct super_block *sb)
{
- struct inode *inode, *next;
+ struct genradix_iter iter;
+ void **i;
LIST_HEAD(dispose);
-
again:
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
+ rcu_read_lock();
+ genradix_for_each(&sb->s_inodes.items, iter, i) {
+ struct inode *inode = *((struct inode **) i);
+ if (!inode)
+ continue;
+
spin_lock(&inode->i_lock);
if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
spin_unlock(&inode->i_lock);
@@ -856,13 +865,13 @@ again:
spin_unlock(&inode->i_lock);
list_add(&inode->i_lru, &dispose);
if (need_resched()) {
- spin_unlock(&sb->s_inode_list_lock);
+ rcu_read_unlock();
cond_resched();
dispose_list(&dispose);
goto again;
}
}
- spin_unlock(&sb->s_inode_list_lock);
+ rcu_read_unlock();
dispose_list(&dispose);
}
@@ -1290,12 +1299,7 @@ again:
hlist_add_head_rcu(&inode->i_hash, head);
spin_unlock(&inode->i_lock);
- /*
- * Add inode to the sb list if it's not already. It has I_NEW at this
- * point, so it should be safe to test i_sb_list locklessly.
- */
- if (list_empty(&inode->i_sb_list))
- inode_sb_list_add(inode);
+ inode_sb_list_add(inode);
unlock:
spin_unlock(&inode_hash_lock);
diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c
index 272c8a1dab3c..9a8da7a02a6f 100644
--- a/fs/notify/fsnotify.c
+++ b/fs/notify/fsnotify.c
@@ -33,14 +33,19 @@ void __fsnotify_vfsmount_delete(struct vfsmount *mnt)
* @sb: superblock being unmounted.
*
* Called during unmount with no locks held, so needs to be safe against
- * concurrent modifiers. We temporarily drop sb->s_inode_list_lock and CAN block.
+ * concurrent modifiers. Can block.
*/
static void fsnotify_unmount_inodes(struct super_block *sb)
{
- struct inode *inode, *iput_inode = NULL;
+ struct genradix_iter iter;
+ void **i;
+
+ rcu_read_lock();
+ genradix_for_each(&sb->s_inodes.items, iter, i) {
+ struct inode *inode = *((struct inode **) i);
+ if (!inode)
+ continue;
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
/*
* We cannot __iget() an inode in state I_FREEING,
* I_WILL_FREE, or I_NEW which is fine because by that point
@@ -68,23 +73,19 @@ static void fsnotify_unmount_inodes(struct super_block *sb)
__iget(inode);
spin_unlock(&inode->i_lock);
- spin_unlock(&sb->s_inode_list_lock);
-
- iput(iput_inode);
+ rcu_read_unlock();
/* for each watch, send FS_UNMOUNT and then remove it */
fsnotify_inode(inode, FS_UNMOUNT);
fsnotify_inode_delete(inode);
- iput_inode = inode;
+ iput(inode);
cond_resched();
- spin_lock(&sb->s_inode_list_lock);
+ rcu_read_lock();
}
- spin_unlock(&sb->s_inode_list_lock);
-
- iput(iput_inode);
+ rcu_read_unlock();
}
void fsnotify_sb_delete(struct super_block *sb)
diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index b40410cd39af..1d5d08787cce 100644
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -1020,14 +1020,19 @@ static int dqinit_needed(struct inode *inode, int type)
/* This routine is guarded by s_umount semaphore */
static int add_dquot_ref(struct super_block *sb, int type)
{
- struct inode *inode, *old_inode = NULL;
+ void **i;
+ struct genradix_iter iter;
#ifdef CONFIG_QUOTA_DEBUG
int reserved = 0;
#endif
int err = 0;
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
+ rcu_read_lock();
+ genradix_for_each(&sb->s_inodes.items, iter, i) {
+ struct inode *inode = *((struct inode **) i);
+ if (!inode)
+ continue;
+
spin_lock(&inode->i_lock);
if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
!atomic_read(&inode->i_writecount) ||
@@ -1037,33 +1042,21 @@ static int add_dquot_ref(struct super_block *sb, int type)
}
__iget(inode);
spin_unlock(&inode->i_lock);
- spin_unlock(&sb->s_inode_list_lock);
+ rcu_read_unlock();
#ifdef CONFIG_QUOTA_DEBUG
if (unlikely(inode_get_rsv_space(inode) > 0))
reserved = 1;
#endif
- iput(old_inode);
err = __dquot_initialize(inode, type);
- if (err) {
- iput(inode);
+ iput(inode);
+ if (err)
goto out;
- }
- /*
- * We hold a reference to 'inode' so it couldn't have been
- * removed from s_inodes list while we dropped the
- * s_inode_list_lock. We cannot iput the inode now as we can be
- * holding the last reference and we cannot iput it under
- * s_inode_list_lock. So we keep the reference and iput it
- * later.
- */
- old_inode = inode;
cond_resched();
- spin_lock(&sb->s_inode_list_lock);
+ rcu_read_lock();
}
- spin_unlock(&sb->s_inode_list_lock);
- iput(old_inode);
+ rcu_read_unlock();
out:
#ifdef CONFIG_QUOTA_DEBUG
if (reserved) {
@@ -1077,13 +1070,25 @@ out:
static void remove_dquot_ref(struct super_block *sb, int type)
{
- struct inode *inode;
+ struct genradix_iter iter;
+ void **i;
#ifdef CONFIG_QUOTA_DEBUG
int reserved = 0;
#endif
- spin_lock(&sb->s_inode_list_lock);
- list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
+ rcu_read_lock();
+ genradix_for_each(&sb->s_inodes.items, iter, i) {
+ struct inode *inode = *((struct inode **) i);
+ if (!inode)
+ continue;
+
+ spin_lock(&inode->i_lock);
+ bool on_list = inode->i_sb_list_idx != 0;
+ spin_unlock(&inode->i_lock);
+
+ if (!on_list)
+ continue;
+
/*
* We have to scan also I_NEW inodes because they can already
* have quota pointer initialized. Luckily, we need to touch
@@ -1107,7 +1112,7 @@ static void remove_dquot_ref(struct super_block *sb, int type)
}
spin_unlock(&dq_data_lock);
}
- spin_unlock(&sb->s_inode_list_lock);
+ rcu_read_unlock();
#ifdef CONFIG_QUOTA_DEBUG
if (reserved) {
printk(KERN_WARNING "VFS (%s): Writes happened after quota"
diff --git a/fs/super.c b/fs/super.c
index 1db230432960..132af76cf5c3 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -274,6 +274,7 @@ static void destroy_super_work(struct work_struct *work)
{
struct super_block *s = container_of(work, struct super_block,
destroy_work);
+ fast_list_exit(&s->s_inodes);
fsnotify_sb_free(s);
security_sb_free(s);
put_user_ns(s->s_user_ns);
@@ -359,8 +360,6 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags,
INIT_HLIST_NODE(&s->s_instances);
INIT_HLIST_BL_HEAD(&s->s_roots);
mutex_init(&s->s_sync_lock);
- INIT_LIST_HEAD(&s->s_inodes);
- spin_lock_init(&s->s_inode_list_lock);
INIT_LIST_HEAD(&s->s_inodes_wb);
spin_lock_init(&s->s_inode_wblist_lock);
@@ -375,6 +374,9 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags,
s->s_time_min = TIME64_MIN;
s->s_time_max = TIME64_MAX;
+ if (fast_list_init(&s->s_inodes))
+ goto fail;
+
s->s_shrink = shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE,
"sb-%s", type->name);
if (!s->s_shrink)
@@ -646,7 +648,7 @@ void generic_shutdown_super(struct super_block *sb)
* the fscrypt keyring can be destroyed.
*/
fscrypt_destroy_keyring(sb);
-
+#if 0
if (CHECK_DATA_CORRUPTION(!list_empty(&sb->s_inodes),
"VFS: Busy inodes after unmount of %s (%s)",
sb->s_id, sb->s_type->name)) {
@@ -665,6 +667,7 @@ void generic_shutdown_super(struct super_block *sb)
}
spin_unlock(&sb->s_inode_list_lock);
}
+#endif
}
/*
* Broadcast to everyone that grabbed a temporary reference to this
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-23 11:50:25 +0000