diff options
| -rw-r--r-- | fs/bcachefs/bcachefs.h | 8 | ||||
| -rw-r--r-- | fs/bcachefs/btree_cache.c | 12 | ||||
| -rw-r--r-- | fs/bcachefs/btree_iter.c | 44 | ||||
| -rw-r--r-- | fs/bcachefs/btree_iter.h | 2 | ||||
| -rw-r--r-- | fs/bcachefs/btree_key_cache.c | 8 | ||||
| -rw-r--r-- | fs/bcachefs/btree_types.h | 3 | ||||
| -rw-r--r-- | fs/bcachefs/btree_update_interior.c | 2 |
7 files changed, 75 insertions, 4 deletions
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 35311dbb189c..019ea92f5de8 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -193,6 +193,7 @@ #include <linux/semaphore.h> #include <linux/seqlock.h> #include <linux/shrinker.h> +#include <linux/srcu.h> #include <linux/types.h> #include <linux/workqueue.h> #include <linux/zstd.h> @@ -642,6 +643,13 @@ struct bch_fs { mempool_t btree_iters_pool; struct btree_iter_buf __percpu *btree_iters_bufs; + struct srcu_struct btree_trans_barrier; + struct rcu_head btree_trans_barrier_update; + spinlock_t btree_trans_barrier_lock; + u32 btree_trans_barrier_seq; + unsigned btree_trans_barrier_running:1; + unsigned btree_trans_barrier_exiting:1; + struct btree_key_cache btree_key_cache; struct workqueue_struct *wq; diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index f8f6079c0199..6048affc5c3e 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -253,10 +253,13 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, unsigned long touched = 0; unsigned long freed = 0; unsigned i, flags; + u32 seq; if (bch2_btree_shrinker_disabled) return SHRINK_STOP; + seq = bch2_btree_trans_barrier_seq(c); + /* Return -1 if we can't do anything right now */ if (sc->gfp_mask & __GFP_FS) mutex_lock(&bc->lock); @@ -276,6 +279,13 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, can_free = btree_cache_can_free(bc); nr = min_t(unsigned long, nr, can_free); + list_for_each_entry_safe(b, t, &bc->freed, list) { + if (seq != b->btree_trans_barrier_seq) { + list_del(&b->list); + kfree(b); + } + } + i = 0; list_for_each_entry_safe(b, t, &bc->freeable, list) { touched++; @@ -447,7 +457,7 @@ int bch2_fs_btree_cache_init(struct bch_fs *c) bc->shrink.scan_objects = bch2_btree_cache_scan; bc->shrink.seeks = 4; bc->shrink.batch = btree_pages(c) * 2; - register_shrinker(&bc->shrink); + ret = register_shrinker(&bc->shrink); out: pr_verbose_init(c->opts, "ret %i", ret); return ret; diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 58f1a3dd97d3..02a82091d950 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -2372,6 +2372,8 @@ void bch2_trans_init(struct btree_trans *trans, struct bch_fs *c, if (expected_mem_bytes) bch2_trans_preload_mem(trans, expected_mem_bytes); + trans->srcu_idx = srcu_read_lock(&c->btree_trans_barrier); + #ifdef CONFIG_BCACHEFS_DEBUG trans->pid = current->pid; mutex_lock(&c->btree_trans_lock); @@ -2392,6 +2394,8 @@ int bch2_trans_exit(struct btree_trans *trans) mutex_unlock(&trans->c->btree_trans_lock); #endif + srcu_read_unlock(&c->btree_trans_barrier, trans->srcu_idx); + bch2_journal_preres_put(&trans->c->journal, &trans->journal_preres); kfree(trans->fs_usage_deltas); @@ -2407,6 +2411,35 @@ int bch2_trans_exit(struct btree_trans *trans) return trans->error ? -EIO : 0; } +static void btree_trans_barrier_update_seq(struct rcu_head *head) +{ + struct bch_fs *c = container_of(head, struct bch_fs, btree_trans_barrier_update); + + spin_lock(&c->btree_trans_barrier_lock); + c->btree_trans_barrier_running = false; + c->btree_trans_barrier_seq++; + spin_unlock(&c->btree_trans_barrier_lock); +} + +u32 bch2_btree_trans_barrier_seq(struct bch_fs *c) +{ + u32 ret; + + spin_lock(&c->btree_trans_barrier_lock); + ret = c->btree_trans_barrier_seq; + + if (!c->btree_trans_barrier_running && + !c->btree_trans_barrier_exiting) { + c->btree_trans_barrier_running = true; + call_srcu(&c->btree_trans_barrier, + &c->btree_trans_barrier_update, + btree_trans_barrier_update_seq); + } + spin_unlock(&c->btree_trans_barrier_lock); + + return ret; +} + static void __maybe_unused bch2_btree_iter_node_to_text(struct printbuf *out, struct btree_bkey_cached_common *_b, @@ -2473,7 +2506,14 @@ void bch2_btree_trans_to_text(struct printbuf *out, struct bch_fs *c) void bch2_fs_btree_iter_exit(struct bch_fs *c) { + spin_lock(&c->btree_trans_barrier_lock); + c->btree_trans_barrier_exiting = true; + spin_unlock(&c->btree_trans_barrier_lock); + + srcu_barrier(&c->btree_trans_barrier); + mempool_exit(&c->btree_iters_pool); + cleanup_srcu_struct(&c->btree_trans_barrier); } int bch2_fs_btree_iter_init(struct bch_fs *c) @@ -2482,8 +2522,10 @@ int bch2_fs_btree_iter_init(struct bch_fs *c) INIT_LIST_HEAD(&c->btree_trans_list); mutex_init(&c->btree_trans_lock); + spin_lock_init(&c->btree_trans_barrier_lock); - return mempool_init_kmalloc_pool(&c->btree_iters_pool, 1, + return init_srcu_struct(&c->btree_trans_barrier) ?: + mempool_init_kmalloc_pool(&c->btree_iters_pool, 1, sizeof(struct btree_iter) * nr + sizeof(struct btree_insert_entry) * nr + sizeof(struct btree_insert_entry) * nr); diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index f7a73619c85b..9318c1c950db 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -307,6 +307,8 @@ void *bch2_trans_kmalloc(struct btree_trans *, size_t); void bch2_trans_init(struct btree_trans *, struct bch_fs *, unsigned, size_t); int bch2_trans_exit(struct btree_trans *); +u32 bch2_btree_trans_barrier_seq(struct bch_fs *); + void bch2_btree_trans_to_text(struct printbuf *, struct bch_fs *); void bch2_fs_btree_iter_exit(struct bch_fs *); diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 6a3d909c6d6e..22163163ed4f 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -70,10 +70,14 @@ static void bkey_cached_evict(struct btree_key_cache *c, c->nr_keys--; } -static void bkey_cached_free(struct btree_key_cache *c, +static void bkey_cached_free(struct btree_key_cache *bc, struct bkey_cached *ck) { - list_move(&ck->list, &c->freed); + struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); + + ck->btree_trans_barrier_seq = bch2_btree_trans_barrier_seq(c); + + list_move(&ck->list, &bc->freed); kfree(ck->k); ck->k = NULL; diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 0ec782a69cb9..2211951f6fec 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -76,6 +76,7 @@ struct btree { u16 written; u8 nsets; u8 nr_key_bits; + u32 btree_trans_barrier_seq; struct bkey_format format; @@ -314,6 +315,7 @@ struct bkey_cached { unsigned long flags; u8 u64s; bool valid; + u32 btree_trans_barrier_seq; struct bkey_cached_key key; struct rhash_head hash; @@ -350,6 +352,7 @@ struct btree_trans { pid_t pid; #endif unsigned long ip; + int srcu_idx; u64 iters_linked; u64 iters_live; diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index c0ae76411c2e..2b64171afd9a 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -138,6 +138,8 @@ static void __btree_node_free(struct bch_fs *c, struct btree *b) bch2_btree_node_hash_remove(&c->btree_cache, b); + b->btree_trans_barrier_seq = bch2_btree_trans_barrier_seq(c); + mutex_lock(&c->btree_cache.lock); list_move(&b->list, &c->btree_cache.freeable); mutex_unlock(&c->btree_cache.lock); |