bcachefs: assorted microoptimizations

5 files changed, 56 insertions, 47 deletions

diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 3b6a13c766c8..298f26d4161d 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -520,6 +520,8 @@ struct bch_fs {
 
 	unsigned short		block_bits;	/* ilog2(block_size) */
 
+	u16			btree_foreground_merge_threshold;
+
 	struct closure		sb_write;
 	struct mutex		sb_lock;
 
diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c
index 02be5bb42e4e..718f094bdfa7 100644
--- a/fs/bcachefs/bset.c
+++ b/fs/bcachefs/bset.c
@@ -569,7 +569,7 @@ static struct bkey_packed *rw_aux_to_bkey(const struct btree *b,
 static void rw_aux_tree_set(const struct btree *b, struct bset_tree *t,
 			    unsigned j, struct bkey_packed *k)
 {
-	BUG_ON(k >= btree_bkey_last(b, t));
+	EBUG_ON(k >= btree_bkey_last(b, t));
 
 	rw_aux_tree(b, t)[j] = (struct rw_aux_tree) {
 		.offset	= __btree_node_key_to_offset(b, k),
@@ -619,7 +619,7 @@ static unsigned rw_aux_tree_bsearch(struct btree *b,
 {
 	unsigned l = 0, r = t->size;
 
-	BUG_ON(bset_aux_tree_type(t) != BSET_RW_AUX_TREE);
+	EBUG_ON(bset_aux_tree_type(t) != BSET_RW_AUX_TREE);
 
 	while (l < r) {
 		unsigned m = (l + r) >> 1;
@@ -630,13 +630,13 @@ static unsigned rw_aux_tree_bsearch(struct btree *b,
 			r = m;
 	}
 
-	BUG_ON(l < t->size &&
-	       rw_aux_tree(b, t)[l].offset < offset);
-	BUG_ON(l &&
-	       rw_aux_tree(b, t)[l - 1].offset >= offset);
+	EBUG_ON(l < t->size &&
+		rw_aux_tree(b, t)[l].offset < offset);
+	EBUG_ON(l &&
+		rw_aux_tree(b, t)[l - 1].offset >= offset);
 
-	BUG_ON(l > r);
-	BUG_ON(l > t->size);
+	EBUG_ON(l > r);
+	EBUG_ON(l > t->size);
 
 	return l;
 }
@@ -764,16 +764,16 @@ static void make_bfloat(struct btree *b, struct bset_tree *t,
 #ifdef __LITTLE_ENDIAN
 	shift = (int) (b->format.key_u64s * 64 - b->nr_key_bits) + exponent;
 
-	BUG_ON(shift + bits > b->format.key_u64s * 64);
+	EBUG_ON(shift + bits > b->format.key_u64s * 64);
 #else
 	shift = high_bit_offset +
 		b->nr_key_bits -
 		exponent -
 		bits;
 
-	BUG_ON(shift < KEY_PACKED_BITS_START);
+	EBUG_ON(shift < KEY_PACKED_BITS_START);
 #endif
-	BUG_ON(shift < 0 || shift >= BFLOAT_FAILED);
+	EBUG_ON(shift < 0 || shift >= BFLOAT_FAILED);
 
 	f->exponent = shift;
 	mantissa = bkey_mantissa(m, f, j);
@@ -890,6 +890,7 @@ retry:
 			prev = k, k = bkey_next(k);
 
 		if (k >= btree_bkey_last(b, t)) {
+			/* XXX: this path sucks */
 			t->size--;
 			goto retry;
 		}
@@ -898,8 +899,8 @@ retry:
 		bkey_float(b, t, j)->key_offset =
 			bkey_to_cacheline_offset(b, t, cacheline++, k);
 
-		BUG_ON(tree_to_prev_bkey(b, t, j) != prev);
-		BUG_ON(tree_to_bkey(b, t, j) != k);
+		EBUG_ON(tree_to_prev_bkey(b, t, j) != prev);
+		EBUG_ON(tree_to_bkey(b, t, j) != k);
 	}
 
 	while (bkey_next(k) != btree_bkey_last(b, t))
@@ -1074,7 +1075,7 @@ static void ro_aux_tree_fix_invalidated_key(struct btree *b,
 	struct bkey_packed min_key, max_key;
 	unsigned inorder, j;
 
-	BUG_ON(bset_aux_tree_type(t) != BSET_RO_AUX_TREE);
+	EBUG_ON(bset_aux_tree_type(t) != BSET_RO_AUX_TREE);
 
 	/* signal to make_bfloat() that they're uninitialized: */
 	min_key.u64s = max_key.u64s = 0;
@@ -1148,7 +1149,7 @@ static void bch2_bset_fix_lookup_table(struct btree *b,
 	int shift = new_u64s - clobber_u64s;
 	unsigned l, j, where = __btree_node_key_to_offset(b, _where);
 
-	BUG_ON(bset_has_ro_aux_tree(t));
+	EBUG_ON(bset_has_ro_aux_tree(t));
 
 	if (!bset_has_rw_aux_tree(t))
 		return;
@@ -1157,8 +1158,8 @@ static void bch2_bset_fix_lookup_table(struct btree *b,
 
 	/* l is first >= than @where */
 
-	BUG_ON(l < t->size && rw_aux_tree(b, t)[l].offset < where);
-	BUG_ON(l && rw_aux_tree(b, t)[l - 1].offset >= where);
+	EBUG_ON(l < t->size && rw_aux_tree(b, t)[l].offset < where);
+	EBUG_ON(l && rw_aux_tree(b, t)[l - 1].offset >= where);
 
 	if (!l) /* never delete first entry */
 		l++;
@@ -1189,9 +1190,9 @@ static void bch2_bset_fix_lookup_table(struct btree *b,
 	for (j = l; j < t->size; j++)
 	       rw_aux_tree(b, t)[j].offset += shift;
 
-	BUG_ON(l < t->size &&
-	       rw_aux_tree(b, t)[l].offset ==
-	       rw_aux_tree(b, t)[l - 1].offset);
+	EBUG_ON(l < t->size &&
+		rw_aux_tree(b, t)[l].offset ==
+		rw_aux_tree(b, t)[l - 1].offset);
 
 	if (t->size < bset_rw_tree_capacity(b, t) &&
 	    (l < t->size
@@ -1248,8 +1249,8 @@ void bch2_bset_insert(struct btree *b,
 		u64 *src_p = where->_data + clobber_u64s;
 		u64 *dst_p = where->_data + src->u64s;
 
-		BUG_ON((int) le16_to_cpu(bset(b, t)->u64s) <
-		       (int) clobber_u64s - src->u64s);
+		EBUG_ON((int) le16_to_cpu(bset(b, t)->u64s) <
+			(int) clobber_u64s - src->u64s);
 
 		memmove_u64s(dst_p, src_p, btree_bkey_last(b, t)->_data - src_p);
 		le16_add_cpu(&bset(b, t)->u64s, src->u64s - clobber_u64s);
@@ -1277,7 +1278,7 @@ void bch2_bset_delete(struct btree *b,
 
 	bch2_bset_verify_rw_aux_tree(b, t);
 
-	BUG_ON(le16_to_cpu(bset(b, t)->u64s) < clobber_u64s);
+	EBUG_ON(le16_to_cpu(bset(b, t)->u64s) < clobber_u64s);
 
 	memmove_u64s_down(dst_p, src_p, btree_bkey_last(b, t)->_data - src_p);
 	le16_add_cpu(&bset(b, t)->u64s, -clobber_u64s);
@@ -1594,7 +1595,7 @@ struct bkey_packed *bch2_btree_node_iter_bset_pos(struct btree_node_iter *iter,
 {
 	struct btree_node_iter_set *set;
 
-	BUG_ON(iter->used > MAX_BSETS);
+	EBUG_ON(iter->used > MAX_BSETS);
 
 	btree_node_iter_for_each(iter, set)
 		if (set->end == t->end_offset)
@@ -1657,10 +1658,10 @@ void bch2_btree_node_iter_advance(struct btree_node_iter *iter,
 
 	iter->data->k += __bch2_btree_node_iter_peek_all(iter, b)->u64s;
 
-	BUG_ON(iter->data->k > iter->data->end);
+	EBUG_ON(iter->data->k > iter->data->end);
 
 	if (iter->data->k == iter->data->end) {
-		BUG_ON(iter->used == 0);
+		EBUG_ON(iter->used == 0);
 		iter->data[0] = iter->data[--iter->used];
 	}
 
diff --git a/fs/bcachefs/btree_update_leaf.c b/fs/bcachefs/btree_update_leaf.c
index dbf705696973..ad318d130967 100644
--- a/fs/bcachefs/btree_update_leaf.c
+++ b/fs/bcachefs/btree_update_leaf.c
@@ -138,21 +138,14 @@ void bch2_btree_journal_key(struct btree_insert *trans,
 	EBUG_ON(trans->journal_res.ref !=
 		!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY));
 
-	if (!journal_pin_active(&w->journal))
-		bch2_journal_pin_add(j, &trans->journal_res,
-				     &w->journal,
-				     btree_node_write_idx(b) == 0
-				     ? btree_node_flush0
-				     : btree_node_flush1);
-
-	if (trans->journal_res.ref) {
+	if (likely(trans->journal_res.ref)) {
 		u64 seq = trans->journal_res.seq;
 		bool needs_whiteout = insert->k.needs_whiteout;
 
 		/* ick */
 		insert->k.needs_whiteout = false;
 		bch2_journal_add_keys(j, &trans->journal_res,
-				      b->btree_id, insert);
+				      iter->btree_id, insert);
 		insert->k.needs_whiteout = needs_whiteout;
 
 		bch2_journal_set_has_inode(j, &trans->journal_res,
@@ -163,7 +156,14 @@ void bch2_btree_journal_key(struct btree_insert *trans,
 		btree_bset_last(b)->journal_seq = cpu_to_le64(seq);
 	}
 
-	if (!btree_node_dirty(b))
+	if (unlikely(!journal_pin_active(&w->journal)))
+		bch2_journal_pin_add(j, &trans->journal_res,
+				     &w->journal,
+				     btree_node_write_idx(b) == 0
+				     ? btree_node_flush0
+				     : btree_node_flush1);
+
+	if (unlikely(!btree_node_dirty(b)))
 		set_btree_node_dirty(b);
 }
 
@@ -173,8 +173,8 @@ bch2_insert_fixup_key(struct btree_insert *trans,
 {
 	struct btree_iter *iter = insert->iter;
 
-	BUG_ON(iter->level);
-	BUG_ON(insert->k->k.u64s >
+	EBUG_ON(iter->level);
+	EBUG_ON(insert->k->k.u64s >
 	       bch_btree_keys_u64s_remaining(trans->c, iter->nodes[0]));
 
 	if (bch2_btree_bset_insert_key(iter, iter->nodes[0],
@@ -196,7 +196,7 @@ static int inline foreground_maybe_merge(struct bch_fs *c,
 		return 0;
 
 	b = iter->nodes[iter->level];
-	if (b->sib_u64s[sib] > BTREE_FOREGROUND_MERGE_THRESHOLD(c))
+	if (b->sib_u64s[sib] > c->btree_foreground_merge_threshold)
 		return 0;
 
 	return bch2_foreground_maybe_merge(c, iter, sib);
@@ -288,12 +288,15 @@ static void multi_unlock_write(struct btree_insert *trans)
 			bch2_btree_node_unlock_write(i->iter->nodes[0], i->iter);
 }
 
-static int btree_trans_entry_cmp(const void *_l, const void *_r)
+static inline void btree_trans_sort(struct btree_insert *trans)
 {
-	const struct btree_insert_entry *l = _l;
-	const struct btree_insert_entry *r = _r;
+	int i, end = trans->nr;
 
-	return btree_iter_cmp(l->iter, r->iter);
+	while (--end > 0)
+		for (i = 0; i < end; i++)
+			if (btree_iter_cmp(trans->entries[i].iter,
+					   trans->entries[i + 1].iter) > 0)
+				swap(trans->entries[i], trans->entries[i + 1]);
 }
 
 /* Normal update interface: */
@@ -326,8 +329,7 @@ int __bch2_btree_insert_at(struct btree_insert *trans)
 					 bkey_i_to_s_c(i->k)));
 	}
 
-	sort(trans->entries, trans->nr, sizeof(trans->entries[0]),
-	     btree_trans_entry_cmp, NULL);
+	btree_trans_sort(trans);
 
 	if (unlikely(!percpu_ref_tryget(&c->writes)))
 		return -EROFS;
diff --git a/fs/bcachefs/journal.h b/fs/bcachefs/journal.h
index 9461d2d8a23e..52d74eec3bec 100644
--- a/fs/bcachefs/journal.h
+++ b/fs/bcachefs/journal.h
@@ -197,8 +197,11 @@ static inline void bch2_journal_set_has_inode(struct journal *j,
 					      u64 inum)
 {
 	struct journal_buf *buf = &j->buf[res->idx];
+	unsigned long bit = hash_64(inum, ilog2(sizeof(buf->has_inode) * 8));
 
-	set_bit(hash_64(inum, ilog2(sizeof(buf->has_inode) * 8)), buf->has_inode);
+	/* avoid atomic op if possible */
+	if (unlikely(!test_bit(bit, buf->has_inode)))
+		set_bit(bit, buf->has_inode);
 }
 
 /*
@@ -236,7 +239,7 @@ static inline void bch2_journal_add_entry(struct journal *j, struct journal_res
 	unsigned actual = jset_u64s(u64s);
 
 	EBUG_ON(!res->ref);
-	BUG_ON(actual > res->u64s);
+	EBUG_ON(actual > res->u64s);
 
 	bch2_journal_add_entry_at(buf, res->offset, type,
 				  id, level, data, u64s);
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index b0773d4bd84a..4c8569b02e1f 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -569,6 +569,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 	bch2_opts_apply(&c->opts, opts);
 
 	c->block_bits		= ilog2(c->opts.block_size);
+	c->btree_foreground_merge_threshold = BTREE_FOREGROUND_MERGE_THRESHOLD(c);
 
 	c->opts.nochanges	|= c->opts.noreplay;
 	c->opts.read_only	|= c->opts.nochanges;