1 files changed, 83 insertions, 69 deletions

diff --git a/libbcachefs/btree_gc.c b/libbcachefs/btree_gc.c
index 73b947a..5c54a0c 100644
--- a/libbcachefs/btree_gc.c
+++ b/libbcachefs/btree_gc.c
@@ -571,37 +571,37 @@ static int bch2_check_fix_ptrs(struct bch_fs *c, enum btree_id btree_id,
 				(printbuf_reset(&buf),
 				 bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) {
 			if (!p.ptr.cached) {
-				g->_mark.gen		= p.ptr.gen;
 				g->gen_valid		= true;
+				g->gen			= p.ptr.gen;
 			} else {
 				do_update = true;
 			}
 		}
 
-		if (fsck_err_on(gen_cmp(p.ptr.gen, g->mark.gen) > 0, c,
+		if (fsck_err_on(gen_cmp(p.ptr.gen, g->gen) > 0, c,
 				"bucket %u:%zu data type %s ptr gen in the future: %u > %u\n"
 				"while marking %s",
 				p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 				bch2_data_types[ptr_data_type(k->k, &p.ptr)],
-				p.ptr.gen, g->mark.gen,
+				p.ptr.gen, g->gen,
 				(printbuf_reset(&buf),
 				 bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) {
 			if (!p.ptr.cached) {
-				g->_mark.gen		= p.ptr.gen;
 				g->gen_valid		= true;
-				g->_mark.data_type	= 0;
-				g->_mark.dirty_sectors	= 0;
-				g->_mark.cached_sectors	= 0;
+				g->gen			= p.ptr.gen;
+				g->data_type		= 0;
+				g->dirty_sectors	= 0;
+				g->cached_sectors	= 0;
 				set_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
 			} else {
 				do_update = true;
 			}
 		}
 
-		if (fsck_err_on(gen_cmp(g->mark.gen, p.ptr.gen) > BUCKET_GC_GEN_MAX, c,
+		if (fsck_err_on(gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX, c,
 				"bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n"
 				"while marking %s",
-				p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->mark.gen,
+				p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen,
 				bch2_data_types[ptr_data_type(k->k, &p.ptr)],
 				p.ptr.gen,
 				(printbuf_reset(&buf),
@@ -609,30 +609,30 @@ static int bch2_check_fix_ptrs(struct bch_fs *c, enum btree_id btree_id,
 			do_update = true;
 
 		if (fsck_err_on(!p.ptr.cached &&
-				gen_cmp(p.ptr.gen, g->mark.gen) < 0, c,
+				gen_cmp(p.ptr.gen, g->gen) < 0, c,
 				"bucket %u:%zu data type %s stale dirty ptr: %u < %u\n"
 				"while marking %s",
 				p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 				bch2_data_types[ptr_data_type(k->k, &p.ptr)],
-				p.ptr.gen, g->mark.gen,
+				p.ptr.gen, g->gen,
 				(printbuf_reset(&buf),
 				 bch2_bkey_val_to_text(&buf, c, *k), buf.buf)))
 			do_update = true;
 
-		if (data_type != BCH_DATA_btree && p.ptr.gen != g->mark.gen)
+		if (data_type != BCH_DATA_btree && p.ptr.gen != g->gen)
 			continue;
 
-		if (fsck_err_on(g->mark.data_type &&
-				g->mark.data_type != data_type, c,
+		if (fsck_err_on(g->data_type &&
+				g->data_type != data_type, c,
 				"bucket %u:%zu different types of data in same bucket: %s, %s\n"
 				"while marking %s",
 				p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
-				bch2_data_types[g->mark.data_type],
+				bch2_data_types[g->data_type],
 				bch2_data_types[data_type],
 				(printbuf_reset(&buf),
 				 bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) {
 			if (data_type == BCH_DATA_btree) {
-				g->_mark.data_type	= data_type;
+				g->data_type	= data_type;
 				set_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
 			} else {
 				do_update = true;
@@ -692,7 +692,7 @@ static int bch2_check_fix_ptrs(struct bch_fs *c, enum btree_id btree_id,
 				struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
 				struct bucket *g = PTR_GC_BUCKET(ca, ptr);
 
-				ptr->gen = g->mark.gen;
+				ptr->gen = g->gen;
 			}
 		} else {
 			bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, ({
@@ -701,12 +701,12 @@ static int bch2_check_fix_ptrs(struct bch_fs *c, enum btree_id btree_id,
 				enum bch_data_type data_type = bch2_bkey_ptr_data_type(*k, ptr);
 
 				(ptr->cached &&
-				 (!g->gen_valid || gen_cmp(ptr->gen, g->mark.gen) > 0)) ||
+				 (!g->gen_valid || gen_cmp(ptr->gen, g->gen) > 0)) ||
 				(!ptr->cached &&
-				 gen_cmp(ptr->gen, g->mark.gen) < 0) ||
-				gen_cmp(g->mark.gen, ptr->gen) > BUCKET_GC_GEN_MAX ||
-				(g->mark.data_type &&
-				 g->mark.data_type != data_type);
+				 gen_cmp(ptr->gen, g->gen) < 0) ||
+				gen_cmp(g->gen, ptr->gen) > BUCKET_GC_GEN_MAX ||
+				(g->data_type &&
+				 g->data_type != data_type);
 			}));
 again:
 			ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
@@ -1163,10 +1163,10 @@ static void bch2_gc_free(struct bch_fs *c)
 	genradix_free(&c->gc_stripes);
 
 	for_each_member_device(ca, c, i) {
-		kvpfree(rcu_dereference_protected(ca->buckets[1], 1),
+		kvpfree(rcu_dereference_protected(ca->buckets_gc, 1),
 			sizeof(struct bucket_array) +
 			ca->mi.nbuckets * sizeof(struct bucket));
-		ca->buckets[1] = NULL;
+		ca->buckets_gc = NULL;
 
 		free_percpu(ca->usage_gc);
 		ca->usage_gc = NULL;
@@ -1295,7 +1295,7 @@ static int bch2_gc_start(struct bch_fs *c,
 	}
 
 	for_each_member_device(ca, c, i) {
-		BUG_ON(ca->buckets[1]);
+		BUG_ON(ca->buckets_gc);
 		BUG_ON(ca->usage_gc);
 
 		ca->usage_gc = alloc_percpu(struct bch_dev_usage);
@@ -1315,9 +1315,9 @@ static int bch2_alloc_write_key(struct btree_trans *trans,
 {
 	struct bch_fs *c = trans->c;
 	struct bch_dev *ca = bch_dev_bkey_exists(c, iter->pos.inode);
-	struct bucket *g;
+	struct bucket gc;
 	struct bkey_s_c k;
-	struct bkey_alloc_unpacked old_u, new_u, gc_u;
+	struct bkey_alloc_unpacked old_u, new_u;
 	struct bkey_alloc_buf *a;
 	int ret;
 
@@ -1329,39 +1329,27 @@ static int bch2_alloc_write_key(struct btree_trans *trans,
 	old_u = new_u = bch2_alloc_unpack(k);
 
 	percpu_down_read(&c->mark_lock);
-	g	= gc_bucket(ca, iter->pos.offset);
-	gc_u = (struct bkey_alloc_unpacked) {
-		.dev		= iter->pos.inode,
-		.bucket		= iter->pos.offset,
-		.gen		= g->mark.gen,
-		.data_type	= g->mark.data_type,
-		.dirty_sectors	= g->mark.dirty_sectors,
-		.cached_sectors	= g->mark.cached_sectors,
-		.read_time	= g->io_time[READ],
-		.write_time	= g->io_time[WRITE],
-		.stripe		= g->stripe,
-		.stripe_redundancy = g->stripe_redundancy,
-	};
+	gc = *gc_bucket(ca, iter->pos.offset);
 	percpu_up_read(&c->mark_lock);
 
 	if (metadata_only &&
-	    gc_u.data_type != BCH_DATA_sb &&
-	    gc_u.data_type != BCH_DATA_journal &&
-	    gc_u.data_type != BCH_DATA_btree)
+	    gc.data_type != BCH_DATA_sb &&
+	    gc.data_type != BCH_DATA_journal &&
+	    gc.data_type != BCH_DATA_btree)
 		return 0;
 
-	if (gen_after(old_u.gen, gc_u.gen))
+	if (gen_after(old_u.gen, gc.gen))
 		return 0;
 
 #define copy_bucket_field(_f)						\
-	if (fsck_err_on(new_u._f != gc_u._f, c,				\
+	if (fsck_err_on(new_u._f != gc._f, c,				\
 			"bucket %llu:%llu gen %u data type %s has wrong " #_f	\
 			": got %u, should be %u",			\
 			iter->pos.inode, iter->pos.offset,		\
-			new_u.gen,					\
-			bch2_data_types[new_u.data_type],		\
-			new_u._f, gc_u._f))				\
-		new_u._f = gc_u._f;					\
+			gc.gen,						\
+			bch2_data_types[gc.data_type],			\
+			new_u._f, gc._f))				\
+		new_u._f = gc._f;					\
 
 	copy_bucket_field(gen);
 	copy_bucket_field(data_type);
@@ -1379,7 +1367,7 @@ static int bch2_alloc_write_key(struct btree_trans *trans,
 	if (IS_ERR(a))
 		return PTR_ERR(a);
 
-	ret = bch2_trans_update(trans, iter, &a->k, BTREE_TRIGGER_NORUN);
+	ret = bch2_trans_update(trans, iter, &a->k, 0);
 fsck_err:
 	return ret;
 }
@@ -1426,7 +1414,13 @@ static int bch2_gc_alloc_done(struct bch_fs *c, bool metadata_only)
 static int bch2_gc_alloc_start(struct bch_fs *c, bool metadata_only)
 {
 	struct bch_dev *ca;
+	struct btree_trans trans;
+	struct btree_iter iter;
+	struct bkey_s_c k;
+	struct bucket *g;
+	struct bkey_alloc_unpacked u;
 	unsigned i;
+	int ret;
 
 	for_each_member_device(ca, c, i) {
 		struct bucket_array *buckets = kvpmalloc(sizeof(struct bucket_array) +
@@ -1434,17 +1428,45 @@ static int bch2_gc_alloc_start(struct bch_fs *c, bool metadata_only)
 				GFP_KERNEL|__GFP_ZERO);
 		if (!buckets) {
 			percpu_ref_put(&ca->ref);
-			percpu_up_write(&c->mark_lock);
 			bch_err(c, "error allocating ca->buckets[gc]");
 			return -ENOMEM;
 		}
 
 		buckets->first_bucket	= ca->mi.first_bucket;
 		buckets->nbuckets	= ca->mi.nbuckets;
-		rcu_assign_pointer(ca->buckets[1], buckets);
+		rcu_assign_pointer(ca->buckets_gc, buckets);
 	};
 
-	return bch2_alloc_read(c, true, metadata_only);
+	bch2_trans_init(&trans, c, 0, 0);
+
+	for_each_btree_key(&trans, iter, BTREE_ID_alloc, POS_MIN,
+			   BTREE_ITER_PREFETCH, k, ret) {
+		ca = bch_dev_bkey_exists(c, k.k->p.inode);
+		g = gc_bucket(ca, k.k->p.offset);
+		u = bch2_alloc_unpack(k);
+
+		g->gen_valid	= 1;
+		g->gen		= u.gen;
+
+		if (metadata_only &&
+		    (u.data_type == BCH_DATA_user ||
+		     u.data_type == BCH_DATA_cached ||
+		     u.data_type == BCH_DATA_parity)) {
+			g->data_type		= u.data_type;
+			g->dirty_sectors	= u.dirty_sectors;
+			g->cached_sectors	= u.cached_sectors;
+			g->stripe		= u.stripe;
+			g->stripe_redundancy	= u.stripe_redundancy;
+		}
+	}
+	bch2_trans_iter_exit(&trans, &iter);
+
+	bch2_trans_exit(&trans);
+
+	if (ret)
+		bch_err(c, "error reading alloc info at gc start: %i", ret);
+
+	return ret;
 }
 
 static void bch2_gc_alloc_reset(struct bch_fs *c, bool metadata_only)
@@ -1453,17 +1475,17 @@ static void bch2_gc_alloc_reset(struct bch_fs *c, bool metadata_only)
 	unsigned i;
 
 	for_each_member_device(ca, c, i) {
-		struct bucket_array *buckets = __bucket_array(ca, true);
+		struct bucket_array *buckets = gc_bucket_array(ca);
 		struct bucket *g;
 
 		for_each_bucket(g, buckets) {
 			if (metadata_only &&
-			    (g->mark.data_type == BCH_DATA_user ||
-			     g->mark.data_type == BCH_DATA_cached ||
-			     g->mark.data_type == BCH_DATA_parity))
+			    (g->data_type == BCH_DATA_user ||
+			     g->data_type == BCH_DATA_cached ||
+			     g->data_type == BCH_DATA_parity))
 				continue;
-			g->_mark.dirty_sectors = 0;
-			g->_mark.cached_sectors = 0;
+			g->dirty_sectors = 0;
+			g->cached_sectors = 0;
 		}
 	};
 }
@@ -1673,9 +1695,8 @@ static void bch2_gc_stripes_reset(struct bch_fs *c, bool metadata_only)
  */
 int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only)
 {
-	struct bch_dev *ca;
 	u64 start_time = local_clock();
-	unsigned i, iter = 0;
+	unsigned iter = 0;
 	int ret;
 
 	lockdep_assert_held(&c->state_lock);
@@ -1777,13 +1798,6 @@ out:
 	bch2_time_stats_update(&c->times[BCH_TIME_btree_gc], start_time);
 
 	/*
-	 * Wake up allocator in case it was waiting for buckets
-	 * because of not being able to inc gens
-	 */
-	for_each_member_device(ca, c, i)
-		bch2_wake_allocator(ca);
-
-	/*
 	 * At startup, allocations can happen directly instead of via the
 	 * allocator thread - issue wakeup in case they blocked on gc_lock:
 	 */
@@ -1891,7 +1905,7 @@ static int bch2_alloc_write_oldest_gen(struct btree_trans *trans, struct btree_i
 
 	u.oldest_gen = ca->oldest_gen[iter->pos.offset];
 
-	return bch2_alloc_write(trans, iter, &u, BTREE_TRIGGER_NORUN);
+	return bch2_alloc_write(trans, iter, &u, 0);
 }
 
 int bch2_gc_gens(struct bch_fs *c)