bcachefs: move dirty into bucket_mark

6 files changed, 49 insertions, 23 deletions

diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c
index 3179eb26aac3..ff8623e4c2c7 100644
--- a/fs/bcachefs/alloc_background.c
+++ b/fs/bcachefs/alloc_background.c
@@ -184,9 +184,9 @@ static void __alloc_read_key(struct bucket *g, const struct bch_alloc *a)
 	g->_mark.cached_sectors	= get_alloc_field(a, &d, idx++);
 }
 
-static void __alloc_write_key(struct bkey_i_alloc *a, struct bucket *g)
+static void __alloc_write_key(struct bkey_i_alloc *a, struct bucket *g,
+			      struct bucket_mark m)
 {
-	struct bucket_mark m = READ_ONCE(g->mark);
 	unsigned idx = 0;
 	void *d = a->v.data;
 
@@ -274,12 +274,17 @@ static int __bch2_alloc_write_key(struct bch_fs *c, struct bch_dev *ca,
 {
 	__BKEY_PADDED(k, BKEY_ALLOC_VAL_U64s_MAX) alloc_key;
 	struct bkey_i_alloc *a = bkey_alloc_init(&alloc_key.k);
+	struct bucket *g;
+	struct bucket_mark m;
 	int ret;
 
 	a->k.p = POS(ca->dev_idx, b);
 
 	percpu_down_read_preempt_disable(&c->usage_lock);
-	__alloc_write_key(a, bucket(ca, b));
+	g = bucket(ca, b);
+	m = bucket_cmpxchg(g, m, m.dirty = false);
+
+	__alloc_write_key(a, g, m);
 	percpu_up_read_preempt_enable(&c->usage_lock);
 
 	bch2_btree_iter_cond_resched(iter);
@@ -342,19 +347,24 @@ int bch2_alloc_write(struct bch_fs *c)
 
 	for_each_rw_member(ca, c, i) {
 		struct btree_iter iter;
-		unsigned long bucket;
+		struct bucket_array *buckets;
+		size_t b;
 
 		bch2_btree_iter_init(&iter, c, BTREE_ID_ALLOC, POS_MIN,
 				     BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
 
 		down_read(&ca->bucket_lock);
-		for_each_set_bit(bucket, ca->buckets_dirty, ca->mi.nbuckets) {
-			ret = __bch2_alloc_write_key(c, ca, bucket,
-						     &iter, NULL, 0);
+		buckets = bucket_array(ca);
+
+		for (b = buckets->first_bucket;
+		     b < buckets->nbuckets;
+		     b++) {
+			if (!buckets->b[b].mark.dirty)
+				continue;
+
+			ret = __bch2_alloc_write_key(c, ca, b, &iter, NULL, 0);
 			if (ret)
 				break;
-
-			clear_bit(bucket, ca->buckets_dirty);
 		}
 		up_read(&ca->bucket_lock);
 		bch2_btree_iter_unlock(&iter);
@@ -533,6 +543,10 @@ static bool bch2_can_invalidate_bucket(struct bch_dev *ca,
 	if (!is_available_bucket(mark))
 		return false;
 
+	if (ca->buckets_nouse &&
+	    test_bit(bucket, ca->buckets_nouse))
+		return false;
+
 	gc_gen = bucket_gc_gen(ca, bucket);
 
 	if (gc_gen >= BUCKET_GC_GEN_MAX / 2)
@@ -1332,6 +1346,7 @@ static int __bch2_fs_allocator_start(struct bch_fs *c)
 			m = READ_ONCE(buckets->b[bu].mark);
 
 			if (!buckets->b[bu].gen_valid ||
+			    !test_bit(bu, ca->buckets_nouse) ||
 			    !is_available_bucket(m) ||
 			    m.cached_sectors)
 				continue;
@@ -1370,7 +1385,7 @@ not_enough:
 				bch2_invalidate_one_bucket(c, ca, bu, &journal_seq);
 
 			fifo_push(&ca->free[RESERVE_BTREE], bu);
-			set_bit(bu, ca->buckets_dirty);
+			bucket_set_dirty(ca, bu);
 		}
 	}
 
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 2fad9b116556..7eb89e008331 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -392,7 +392,7 @@ struct bch_dev {
 	 * Or rcu_read_lock(), but only for ptr_stale():
 	 */
 	struct bucket_array __rcu *buckets[2];
-	unsigned long		*buckets_dirty;
+	unsigned long		*buckets_nouse;
 	unsigned long		*buckets_written;
 	/* most out of date gen in the btree */
 	u8			*oldest_gens;
diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c
index 8f13113c5340..361e60e490f1 100644
--- a/fs/bcachefs/btree_gc.c
+++ b/fs/bcachefs/btree_gc.c
@@ -147,7 +147,7 @@ static int bch2_gc_mark_key(struct bch_fs *c, struct bkey_s_c k,
 					k.k->type, ptr->gen)) {
 				g->_mark.gen = ptr->gen;
 				g->gen_valid = 1;
-				set_bit(b, ca->buckets_dirty);
+				bucket_set_dirty(ca, b);
 			}
 
 			if (mustfix_fsck_err_on(gen_cmp(ptr->gen, g->mark.gen) > 0, c,
@@ -155,7 +155,7 @@ static int bch2_gc_mark_key(struct bch_fs *c, struct bkey_s_c k,
 					k.k->type, ptr->gen, g->mark.gen)) {
 				g->_mark.gen = ptr->gen;
 				g->gen_valid = 1;
-				set_bit(b, ca->buckets_dirty);
+				bucket_set_dirty(ca, b);
 				set_bit(BCH_FS_FIXED_GENS, &c->flags);
 			}
 		}
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index c2951482dc8a..66e122edb2e0 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -1121,7 +1121,7 @@ static void buckets_free_rcu(struct rcu_head *rcu)
 int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
 {
 	struct bucket_array *buckets = NULL, *old_buckets = NULL;
-	unsigned long *buckets_dirty = NULL;
+	unsigned long *buckets_nouse = NULL;
 	unsigned long *buckets_written = NULL;
 	u8 *oldest_gens = NULL;
 	alloc_fifo	free[RESERVE_NR];
@@ -1151,7 +1151,7 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
 					    GFP_KERNEL|__GFP_ZERO)) ||
 	    !(oldest_gens	= kvpmalloc(nbuckets * sizeof(u8),
 					    GFP_KERNEL|__GFP_ZERO)) ||
-	    !(buckets_dirty	= kvpmalloc(BITS_TO_LONGS(nbuckets) *
+	    !(buckets_nouse	= kvpmalloc(BITS_TO_LONGS(nbuckets) *
 					    sizeof(unsigned long),
 					    GFP_KERNEL|__GFP_ZERO)) ||
 	    !(buckets_written	= kvpmalloc(BITS_TO_LONGS(nbuckets) *
@@ -1188,8 +1188,8 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
 		memcpy(oldest_gens,
 		       ca->oldest_gens,
 		       n * sizeof(u8));
-		memcpy(buckets_dirty,
-		       ca->buckets_dirty,
+		memcpy(buckets_nouse,
+		       ca->buckets_nouse,
 		       BITS_TO_LONGS(n) * sizeof(unsigned long));
 		memcpy(buckets_written,
 		       ca->buckets_written,
@@ -1200,7 +1200,7 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
 	buckets = old_buckets;
 
 	swap(ca->oldest_gens, oldest_gens);
-	swap(ca->buckets_dirty, buckets_dirty);
+	swap(ca->buckets_nouse, buckets_nouse);
 	swap(ca->buckets_written, buckets_written);
 
 	if (resize)
@@ -1239,7 +1239,7 @@ err:
 	free_fifo(&free_inc);
 	for (i = 0; i < RESERVE_NR; i++)
 		free_fifo(&free[i]);
-	kvpfree(buckets_dirty,
+	kvpfree(buckets_nouse,
 		BITS_TO_LONGS(nbuckets) * sizeof(unsigned long));
 	kvpfree(buckets_written,
 		BITS_TO_LONGS(nbuckets) * sizeof(unsigned long));
@@ -1262,7 +1262,7 @@ void bch2_dev_buckets_free(struct bch_dev *ca)
 		free_fifo(&ca->free[i]);
 	kvpfree(ca->buckets_written,
 		BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long));
-	kvpfree(ca->buckets_dirty,
+	kvpfree(ca->buckets_nouse,
 		BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long));
 	kvpfree(ca->oldest_gens, ca->mi.nbuckets * sizeof(u8));
 	kvpfree(rcu_dereference_protected(ca->buckets[0], 1),
diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 17a9b44577a1..f2cd70810274 100644
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -56,6 +56,18 @@ static inline struct bucket *bucket(struct bch_dev *ca, size_t b)
 	return __bucket(ca, b, false);
 }
 
+static inline void bucket_set_dirty(struct bch_dev *ca, size_t b)
+{
+	struct bucket *g;
+	struct bucket_mark m;
+
+	rcu_read_lock();
+	g = bucket(ca, b);
+	bucket_cmpxchg(g, m, m.dirty = true);
+	rcu_read_unlock();
+
+}
+
 static inline void bucket_io_clock_reset(struct bch_fs *c, struct bch_dev *ca,
 					 size_t b, int rw)
 {
@@ -195,8 +207,7 @@ static inline bool is_available_bucket(struct bucket_mark mark)
 {
 	return (!mark.owned_by_allocator &&
 		!mark.dirty_sectors &&
-		!mark.stripe &&
-		!mark.nouse);
+		!mark.stripe);
 }
 
 static inline bool bucket_needs_journal_commit(struct bucket_mark m,
diff --git a/fs/bcachefs/buckets_types.h b/fs/bcachefs/buckets_types.h
index 3262583016e2..56289559d256 100644
--- a/fs/bcachefs/buckets_types.h
+++ b/fs/bcachefs/buckets_types.h
@@ -14,7 +14,7 @@ struct bucket_mark {
 	u8		gen;
 	u8		data_type:3,
 			owned_by_allocator:1,
-			nouse:1,
+			dirty:1,
 			journal_seq_valid:1,
 			stripe:1;
 	u16		dirty_sectors;