1 files changed, 68 insertions, 122 deletions

diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index 58f1a3dd97d3..21253be5aab6 100644
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -346,7 +346,7 @@ void bch2_btree_trans_verify_locks(struct btree_trans *trans)
 {
 	struct btree_iter *iter;
 
-	trans_for_each_iter_all(trans, iter)
+	trans_for_each_iter(trans, iter)
 		bch2_btree_iter_verify_locks(iter);
 }
 #else
@@ -875,9 +875,19 @@ static void btree_iter_verify_new_node(struct btree_iter *iter, struct btree *b)
 		char buf[100];
 		struct bkey uk = bkey_unpack_key(b, k);
 
+		bch2_dump_btree_node(iter->trans->c, l->b);
 		bch2_bkey_to_text(&PBUF(buf), &uk);
-		panic("parent iter doesn't point to new node:\n%s\n%llu:%llu\n",
-		      buf, b->key.k.p.inode, b->key.k.p.offset);
+		panic("parent iter doesn't point to new node:\n"
+		      "iter pos %s %llu:%llu\n"
+		      "iter key %s\n"
+		      "new node %llu:%llu-%llu:%llu\n",
+		      bch2_btree_ids[iter->btree_id],
+		      iter->pos.inode,
+		      iter->pos.offset,
+		      buf,
+		      b->data->min_key.inode,
+		      b->data->min_key.offset,
+		      b->key.k.p.inode, b->key.k.p.offset);
 	}
 
 	if (!parent_locked)
@@ -2002,110 +2012,46 @@ int bch2_trans_iter_free(struct btree_trans *trans,
 	return bch2_trans_iter_put(trans, iter);
 }
 
-#if 0
-static int bch2_trans_realloc_iters(struct btree_trans *trans,
-				    unsigned new_size)
+noinline __cold
+static void btree_trans_iter_alloc_fail(struct btree_trans *trans)
 {
-	void *p, *new_iters, *new_updates, *new_updates2;
-	size_t iters_bytes;
-	size_t updates_bytes;
-
-	new_size = roundup_pow_of_two(new_size);
-
-	BUG_ON(new_size > BTREE_ITER_MAX);
 
-	if (new_size <= trans->size)
-		return 0;
-
-	BUG_ON(trans->used_mempool);
-
-	bch2_trans_unlock(trans);
-
-	iters_bytes	= sizeof(struct btree_iter) * new_size;
-	updates_bytes	= sizeof(struct btree_insert_entry) * new_size;
-
-	p = kmalloc(iters_bytes +
-		    updates_bytes +
-		    updates_bytes, GFP_NOFS);
-	if (p)
-		goto success;
-
-	p = mempool_alloc(&trans->c->btree_iters_pool, GFP_NOFS);
-	new_size = BTREE_ITER_MAX;
+	struct btree_iter *iter;
+	struct btree_insert_entry *i;
 
-	trans->used_mempool = true;
-success:
-	new_iters	= p; p += iters_bytes;
-	new_updates	= p; p += updates_bytes;
-	new_updates2	= p; p += updates_bytes;
-
-	memcpy(new_iters, trans->iters,
-	       sizeof(struct btree_iter) * trans->nr_iters);
-	memcpy(new_updates, trans->updates,
-	       sizeof(struct btree_insert_entry) * trans->nr_updates);
-	memcpy(new_updates2, trans->updates2,
-	       sizeof(struct btree_insert_entry) * trans->nr_updates2);
-
-	if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG))
-		memset(trans->iters, POISON_FREE,
-		       sizeof(struct btree_iter) * trans->nr_iters +
-		       sizeof(struct btree_insert_entry) * trans->nr_iters);
-
-	kfree(trans->iters);
-
-	trans->iters		= new_iters;
-	trans->updates		= new_updates;
-	trans->updates2		= new_updates2;
-	trans->size		= new_size;
-
-	if (trans->iters_live) {
-		trace_trans_restart_iters_realloced(trans->ip, trans->size);
-		return -EINTR;
+	trans_for_each_iter(trans, iter)
+		printk(KERN_ERR "iter: btree %s pos %llu:%llu%s%s%s %ps\n",
+		       bch2_btree_ids[iter->btree_id],
+		       iter->pos.inode,
+		       iter->pos.offset,
+		       (trans->iters_live & (1ULL << iter->idx)) ? " live" : "",
+		       (trans->iters_touched & (1ULL << iter->idx)) ? " touched" : "",
+		       iter->flags & BTREE_ITER_KEEP_UNTIL_COMMIT ? " keep" : "",
+		       (void *) iter->ip_allocated);
+
+	trans_for_each_update(trans, i) {
+		char buf[300];
+
+		bch2_bkey_val_to_text(&PBUF(buf), trans->c, bkey_i_to_s_c(i->k));
+		printk(KERN_ERR "update: btree %s %s\n",
+		       bch2_btree_ids[i->iter->btree_id], buf);
 	}
-
-	return 0;
+	panic("trans iter oveflow\n");
 }
-#endif
 
 static struct btree_iter *btree_trans_iter_alloc(struct btree_trans *trans)
 {
-	unsigned idx = __ffs64(~trans->iters_linked);
-
-	if (idx < trans->nr_iters)
-		goto got_slot;
+	unsigned idx;
 
-	if (trans->nr_iters == trans->size) {
-		struct btree_iter *iter;
+	if (unlikely(trans->iters_linked ==
+		     ~((~0ULL << 1) << (BTREE_ITER_MAX - 1))))
+		btree_trans_iter_alloc_fail(trans);
 
-		BUG_ON(trans->size < BTREE_ITER_MAX);
+	idx = __ffs64(~trans->iters_linked);
 
-		trans_for_each_iter(trans, iter) {
-			pr_err("iter: btree %s pos %llu:%llu%s%s%s %ps",
-			       bch2_btree_ids[iter->btree_id],
-			       iter->pos.inode,
-			       iter->pos.offset,
-			       (trans->iters_live & (1ULL << iter->idx)) ? " live" : "",
-			       (trans->iters_touched & (1ULL << iter->idx)) ? " touched" : "",
-			       iter->flags & BTREE_ITER_KEEP_UNTIL_COMMIT ? " keep" : "",
-			       (void *) iter->ip_allocated);
-		}
-
-		panic("trans iter oveflow\n");
-#if 0
-		ret = bch2_trans_realloc_iters(trans, trans->size * 2);
-		if (ret)
-			return ERR_PTR(ret);
-#endif
-	}
-
-	idx = trans->nr_iters++;
-	BUG_ON(trans->nr_iters > trans->size);
-
-	trans->iters[idx].idx = idx;
-got_slot:
-	BUG_ON(trans->iters_linked & (1ULL << idx));
-	trans->iters_linked |= 1ULL << idx;
-	trans->iters[idx].flags = 0;
+	trans->iters_linked	|= 1ULL << idx;
+	trans->iters[idx].idx	 = idx;
+	trans->iters[idx].flags	 = 0;
 	return &trans->iters[idx];
 }
 
@@ -2141,8 +2087,6 @@ static struct btree_iter *__btree_trans_get_iter(struct btree_trans *trans,
 {
 	struct btree_iter *iter, *best = NULL;
 
-	BUG_ON(trans->nr_iters > BTREE_ITER_MAX);
-
 	trans_for_each_iter(trans, iter) {
 		if (btree_iter_type(iter) != (flags & BTREE_ITER_TYPE))
 			continue;
@@ -2160,16 +2104,10 @@ static struct btree_iter *__btree_trans_get_iter(struct btree_trans *trans,
 
 	if (!best) {
 		iter = btree_trans_iter_alloc(trans);
-		if (IS_ERR(iter))
-			return iter;
-
 		bch2_btree_iter_init(trans, iter, btree_id, pos, flags);
 	} else if ((trans->iters_live & (1ULL << best->idx)) ||
 		   (best->flags & BTREE_ITER_KEEP_UNTIL_COMMIT)) {
 		iter = btree_trans_iter_alloc(trans);
-		if (IS_ERR(iter))
-			return iter;
-
 		btree_iter_copy(iter, best);
 	} else {
 		iter = best;
@@ -2203,9 +2141,8 @@ struct btree_iter *__bch2_trans_get_iter(struct btree_trans *trans,
 	struct btree_iter *iter =
 		__btree_trans_get_iter(trans, btree_id, pos, flags);
 
-	if (!IS_ERR(iter))
-		__bch2_btree_iter_set_pos(iter, pos,
-			btree_node_type_is_extents(btree_id));
+	__bch2_btree_iter_set_pos(iter, pos,
+		btree_node_type_is_extents(btree_id));
 	return iter;
 }
 
@@ -2221,7 +2158,6 @@ struct btree_iter *bch2_trans_get_node_iter(struct btree_trans *trans,
 				       flags|BTREE_ITER_NODES);
 	unsigned i;
 
-	BUG_ON(IS_ERR(iter));
 	BUG_ON(bkey_cmp(iter->pos, pos));
 
 	iter->locks_want = locks_want;
@@ -2241,9 +2177,6 @@ struct btree_iter *__bch2_trans_copy_iter(struct btree_trans *trans,
 	struct btree_iter *iter;
 
 	iter = btree_trans_iter_alloc(trans);
-	if (IS_ERR(iter))
-		return iter;
-
 	btree_iter_copy(iter, src);
 
 	trans->iters_live |= 1ULL << iter->idx;
@@ -2318,7 +2251,6 @@ void bch2_trans_reset(struct btree_trans *trans, unsigned flags)
 
 	trans->iters_touched &= trans->iters_live;
 
-	trans->need_reset		= 0;
 	trans->nr_updates		= 0;
 	trans->nr_updates2		= 0;
 	trans->mem_top			= 0;
@@ -2339,20 +2271,21 @@ void bch2_trans_reset(struct btree_trans *trans, unsigned flags)
 
 static void bch2_trans_alloc_iters(struct btree_trans *trans, struct bch_fs *c)
 {
-	unsigned new_size = BTREE_ITER_MAX;
-	size_t iters_bytes	= sizeof(struct btree_iter) * new_size;
-	size_t updates_bytes	= sizeof(struct btree_insert_entry) * new_size;
-	void *p;
+	size_t iters_bytes	= sizeof(struct btree_iter) * BTREE_ITER_MAX;
+	size_t updates_bytes	= sizeof(struct btree_insert_entry) * BTREE_ITER_MAX;
+	void *p = NULL;
 
 	BUG_ON(trans->used_mempool);
 
-	p =     this_cpu_xchg(c->btree_iters_bufs->iter, NULL) ?:
-		mempool_alloc(&trans->c->btree_iters_pool, GFP_NOFS);
+#ifdef __KERNEL__
+	p = this_cpu_xchg(c->btree_iters_bufs->iter, NULL);
+#endif
+	if (!p)
+		p = mempool_alloc(&trans->c->btree_iters_pool, GFP_NOFS);
 
 	trans->iters		= p; p += iters_bytes;
 	trans->updates		= p; p += updates_bytes;
 	trans->updates2		= p; p += updates_bytes;
-	trans->size		= new_size;
 }
 
 void bch2_trans_init(struct btree_trans *trans, struct bch_fs *c,
@@ -2369,8 +2302,12 @@ void bch2_trans_init(struct btree_trans *trans, struct bch_fs *c,
 	 */
 	bch2_trans_alloc_iters(trans, c);
 
-	if (expected_mem_bytes)
-		bch2_trans_preload_mem(trans, expected_mem_bytes);
+	if (expected_mem_bytes) {
+		trans->mem_bytes = roundup_pow_of_two(expected_mem_bytes);
+		trans->mem = kmalloc(trans->mem_bytes, GFP_KERNEL|__GFP_NOFAIL);
+	}
+
+	trans->srcu_idx = srcu_read_lock(&c->btree_trans_barrier);
 
 #ifdef CONFIG_BCACHEFS_DEBUG
 	trans->pid = current->pid;
@@ -2392,12 +2329,19 @@ int bch2_trans_exit(struct btree_trans *trans)
 	mutex_unlock(&trans->c->btree_trans_lock);
 #endif
 
+	srcu_read_unlock(&c->btree_trans_barrier, trans->srcu_idx);
+
 	bch2_journal_preres_put(&trans->c->journal, &trans->journal_preres);
 
 	kfree(trans->fs_usage_deltas);
 	kfree(trans->mem);
 
+#ifdef __KERNEL__
+	/*
+	 * Userspace doesn't have a real percpu implementation:
+	 */
 	trans->iters = this_cpu_xchg(c->btree_iters_bufs->iter, trans->iters);
+#endif
 	if (trans->iters)
 		mempool_free(trans->iters, &trans->c->btree_iters_pool);
 
@@ -2474,6 +2418,7 @@ void bch2_btree_trans_to_text(struct printbuf *out, struct bch_fs *c)
 void bch2_fs_btree_iter_exit(struct bch_fs *c)
 {
 	mempool_exit(&c->btree_iters_pool);
+	cleanup_srcu_struct(&c->btree_trans_barrier);
 }
 
 int bch2_fs_btree_iter_init(struct bch_fs *c)
@@ -2483,7 +2428,8 @@ int bch2_fs_btree_iter_init(struct bch_fs *c)
 	INIT_LIST_HEAD(&c->btree_trans_list);
 	mutex_init(&c->btree_trans_lock);
 
-	return mempool_init_kmalloc_pool(&c->btree_iters_pool, 1,
+	return  init_srcu_struct(&c->btree_trans_barrier) ?:
+		mempool_init_kmalloc_pool(&c->btree_iters_pool, 1,
 			sizeof(struct btree_iter) * nr +
 			sizeof(struct btree_insert_entry) * nr +
 			sizeof(struct btree_insert_entry) * nr);