bcachefs: Add a single slot percpu buf for btree iters

Allocating our array of btree iters is a big enough allocation that it
hits the buddy allocator, and we're seeing lots of lock contention.
Sticking a single element buffer in front of it should help.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

3 files changed, 59 insertions, 27 deletions

diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 10c15d125038..35311dbb189c 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -541,6 +541,10 @@ struct journal_keys {
 	u64			journal_seq_base;
 };
 
+struct btree_iter_buf {
+	struct btree_iter	*iter;
+};
+
 struct bch_fs {
 	struct closure		cl;
 
@@ -636,6 +640,7 @@ struct bch_fs {
 	struct mutex		btree_trans_lock;
 	struct list_head	btree_trans_list;
 	mempool_t		btree_iters_pool;
+	struct btree_iter_buf  __percpu	*btree_iters_bufs;
 
 	struct btree_key_cache	btree_key_cache;
 
diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index 9e1971e6f484..f8c0b68c77c7 100644
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -1991,6 +1991,7 @@ int bch2_trans_iter_free(struct btree_trans *trans,
 	return bch2_trans_iter_put(trans, iter);
 }
 
+#if 0
 static int bch2_trans_realloc_iters(struct btree_trans *trans,
 				    unsigned new_size)
 {
@@ -2053,6 +2054,7 @@ success:
 
 	return 0;
 }
+#endif
 
 static struct btree_iter *btree_trans_iter_alloc(struct btree_trans *trans)
 {
@@ -2062,28 +2064,27 @@ static struct btree_iter *btree_trans_iter_alloc(struct btree_trans *trans)
 		goto got_slot;
 
 	if (trans->nr_iters == trans->size) {
-		int ret;
-
-		if (trans->nr_iters >= BTREE_ITER_MAX) {
-			struct btree_iter *iter;
-
-			trans_for_each_iter(trans, iter) {
-				pr_err("iter: btree %s pos %llu:%llu%s%s%s %ps",
-				       bch2_btree_ids[iter->btree_id],
-				       iter->pos.inode,
-				       iter->pos.offset,
-				       (trans->iters_live & (1ULL << iter->idx)) ? " live" : "",
-				       (trans->iters_touched & (1ULL << iter->idx)) ? " touched" : "",
-				       iter->flags & BTREE_ITER_KEEP_UNTIL_COMMIT ? " keep" : "",
-				       (void *) iter->ip_allocated);
-			}
+		struct btree_iter *iter;
 
-			panic("trans iter oveflow\n");
+		BUG_ON(trans->size < BTREE_ITER_MAX);
+
+		trans_for_each_iter(trans, iter) {
+			pr_err("iter: btree %s pos %llu:%llu%s%s%s %ps",
+			       bch2_btree_ids[iter->btree_id],
+			       iter->pos.inode,
+			       iter->pos.offset,
+			       (trans->iters_live & (1ULL << iter->idx)) ? " live" : "",
+			       (trans->iters_touched & (1ULL << iter->idx)) ? " touched" : "",
+			       iter->flags & BTREE_ITER_KEEP_UNTIL_COMMIT ? " keep" : "",
+			       (void *) iter->ip_allocated);
 		}
 
+		panic("trans iter oveflow\n");
+#if 0
 		ret = bch2_trans_realloc_iters(trans, trans->size * 2);
 		if (ret)
 			return ERR_PTR(ret);
+#endif
 	}
 
 	idx = trans->nr_iters++;
@@ -2325,22 +2326,37 @@ void bch2_trans_reset(struct btree_trans *trans, unsigned flags)
 		bch2_btree_iter_traverse_all(trans);
 }
 
+static void bch2_trans_alloc_iters(struct btree_trans *trans, struct bch_fs *c)
+{
+	unsigned new_size = BTREE_ITER_MAX;
+	size_t iters_bytes	= sizeof(struct btree_iter) * new_size;
+	size_t updates_bytes	= sizeof(struct btree_insert_entry) * new_size;
+	void *p;
+
+	BUG_ON(trans->used_mempool);
+
+	p =     this_cpu_xchg(c->btree_iters_bufs->iter, NULL) ?:
+		mempool_alloc(&trans->c->btree_iters_pool, GFP_NOFS);
+
+	trans->iters		= p; p += iters_bytes;
+	trans->updates		= p; p += updates_bytes;
+	trans->updates2		= p; p += updates_bytes;
+	trans->size		= new_size;
+}
+
 void bch2_trans_init(struct btree_trans *trans, struct bch_fs *c,
 		     unsigned expected_nr_iters,
 		     size_t expected_mem_bytes)
 {
-	/*
-	 * reallocating iterators currently completely breaks
-	 * bch2_trans_iter_put():
-	 */
-	expected_nr_iters = BTREE_ITER_MAX;
-
 	memset(trans, 0, sizeof(*trans));
 	trans->c		= c;
 	trans->ip		= _RET_IP_;
 
-	if (expected_nr_iters > trans->size)
-		bch2_trans_realloc_iters(trans, expected_nr_iters);
+	/*
+	 * reallocating iterators currently completely breaks
+	 * bch2_trans_iter_put(), we always allocate the max:
+	 */
+	bch2_trans_alloc_iters(trans, c);
 
 	if (expected_mem_bytes)
 		bch2_trans_preload_mem(trans, expected_mem_bytes);
@@ -2355,6 +2371,8 @@ void bch2_trans_init(struct btree_trans *trans, struct bch_fs *c,
 
 int bch2_trans_exit(struct btree_trans *trans)
 {
+	struct bch_fs *c = trans->c;
+
 	bch2_trans_unlock(trans);
 
 #ifdef CONFIG_BCACHEFS_DEBUG
@@ -2367,10 +2385,11 @@ int bch2_trans_exit(struct btree_trans *trans)
 
 	kfree(trans->fs_usage_deltas);
 	kfree(trans->mem);
-	if (trans->used_mempool)
+
+	trans->iters = this_cpu_xchg(c->btree_iters_bufs->iter, trans->iters);
+	if (trans->iters)
 		mempool_free(trans->iters, &trans->c->btree_iters_pool);
-	else
-		kfree(trans->iters);
+
 	trans->mem	= (void *) 0x1;
 	trans->iters	= (void *) 0x1;
 
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index ad1967d5a47d..b9cc6fe536ba 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -451,6 +451,7 @@ int bch2_fs_read_write_early(struct bch_fs *c)
 static void __bch2_fs_free(struct bch_fs *c)
 {
 	unsigned i;
+	int cpu;
 
 	for (i = 0; i < BCH_TIME_STAT_NR; i++)
 		bch2_time_stats_exit(&c->times[i]);
@@ -475,6 +476,12 @@ static void __bch2_fs_free(struct bch_fs *c)
 	free_percpu(c->usage[1]);
 	free_percpu(c->usage[0]);
 	kfree(c->usage_base);
+
+	if (c->btree_iters_bufs)
+		for_each_possible_cpu(cpu)
+			kfree(per_cpu_ptr(c->btree_iters_bufs, cpu)->iter);
+
+	free_percpu(c->btree_iters_bufs);
 	free_percpu(c->pcpu);
 	mempool_exit(&c->large_bkey_pool);
 	mempool_exit(&c->btree_bounce_pool);
@@ -753,6 +760,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 			    offsetof(struct btree_write_bio, wbio.bio)),
 			BIOSET_NEED_BVECS) ||
 	    !(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) ||
+	    !(c->btree_iters_bufs = alloc_percpu(struct btree_iter_buf)) ||
 	    mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1,
 					btree_bytes(c)) ||
 	    mempool_init_kmalloc_pool(&c->large_bkey_pool, 1, 2048) ||