bcache: inline some fastpaths

4 files changed, 50 insertions, 22 deletions

diff --git a/drivers/md/bcache/btree_cache.c b/drivers/md/bcache/btree_cache.c
index 0fd92253d376..bc6f6a7de8a2 100644
--- a/drivers/md/bcache/btree_cache.c
+++ b/drivers/md/bcache/btree_cache.c
@@ -129,7 +129,7 @@ int mca_hash_insert(struct cache_set *c, struct btree *b,
 }
 
 noinline __flatten
-static struct btree *mca_find(struct cache_set *c,
+static inline struct btree *mca_find(struct cache_set *c,
 				     const struct bkey_i *k)
 {
 	return rhashtable_lookup_fast(&c->btree_cache_table, &PTR_HASH(k),
diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c
index 59245e75499e..4c132f452774 100644
--- a/drivers/md/bcache/btree_io.c
+++ b/drivers/md/bcache/btree_io.c
@@ -262,12 +262,6 @@ static unsigned sort_extent_whiteouts(struct bkey_packed *dst,
 	return (u64 *) out - (u64 *) dst;
 }
 
-enum compact_mode {
-	COMPACT_LAZY,
-	COMPACT_WRITTEN,
-	COMPACT_WRITTEN_NO_WRITE_LOCK,
-};
-
 static unsigned should_compact_bset(struct btree *b, struct bset_tree *t,
 				    bool compacting,
 				    enum compact_mode mode)
@@ -290,8 +284,8 @@ static unsigned should_compact_bset(struct btree *b, struct bset_tree *t,
 	return 0;
 }
 
-static bool __compact_whiteouts(struct cache_set *c, struct btree *b,
-				enum compact_mode mode)
+bool __bch_compact_whiteouts(struct cache_set *c, struct btree *b,
+			     enum compact_mode mode)
 {
 	const struct bkey_format *f = &b->keys.format;
 	struct bset_tree *t;
@@ -424,11 +418,6 @@ static bool __compact_whiteouts(struct cache_set *c, struct btree *b,
 	return true;
 }
 
-bool bch_maybe_compact_whiteouts(struct cache_set *c, struct btree *b)
-{
-	return __compact_whiteouts(c, b, COMPACT_LAZY);
-}
-
 static bool bch_drop_whiteouts(struct btree *b)
 {
 	struct bset_tree *t;
@@ -475,9 +464,9 @@ static bool bch_drop_whiteouts(struct btree *b)
 	return ret;
 }
 
-static int sort_keys_cmp(struct btree_keys *b,
-			 struct bkey_packed *l,
-			 struct bkey_packed *r)
+static inline int sort_keys_cmp(struct btree_keys *b,
+				struct bkey_packed *l,
+				struct bkey_packed *r)
 {
 	return bkey_cmp_packed(b, l, r) ?:
 		(int) bkey_whiteout(r) - (int) bkey_whiteout(l) ?:
@@ -1348,10 +1337,10 @@ void __bch_btree_node_write(struct cache_set *c, struct btree *b,
 
 	if (lock_type_held == SIX_LOCK_intent) {
 		six_lock_write(&b->lock);
-		__compact_whiteouts(c, b, COMPACT_WRITTEN);
+		__bch_compact_whiteouts(c, b, COMPACT_WRITTEN);
 		six_unlock_write(&b->lock);
 	} else {
-		__compact_whiteouts(c, b, COMPACT_WRITTEN_NO_WRITE_LOCK);
+		__bch_compact_whiteouts(c, b, COMPACT_WRITTEN_NO_WRITE_LOCK);
 	}
 
 	BUG_ON(b->uncompacted_whiteout_u64s);
diff --git a/drivers/md/bcache/btree_io.h b/drivers/md/bcache/btree_io.h
index 9b66e707d4b0..20faadce3ffa 100644
--- a/drivers/md/bcache/btree_io.h
+++ b/drivers/md/bcache/btree_io.h
@@ -19,7 +19,30 @@ static inline void btree_node_io_lock(struct btree *b)
 			    TASK_UNINTERRUPTIBLE);
 }
 
-bool bch_maybe_compact_whiteouts(struct cache_set *, struct btree *);
+enum compact_mode {
+	COMPACT_LAZY,
+	COMPACT_WRITTEN,
+	COMPACT_WRITTEN_NO_WRITE_LOCK,
+};
+
+bool __bch_compact_whiteouts(struct cache_set *, struct btree *, enum compact_mode);
+
+static inline bool bch_maybe_compact_whiteouts(struct cache_set *c, struct btree *b)
+{
+	struct bset_tree *t;
+
+	for_each_bset(&b->keys, t) {
+		unsigned live_u64s = b->keys.nr.bset_u64s[t - b->keys.set];
+		unsigned bset_u64s = le16_to_cpu(t->data->u64s);
+
+		if (live_u64s * 4 < bset_u64s * 3)
+			goto compact;
+	}
+
+	return false;
+compact:
+	return __bch_compact_whiteouts(c, b, COMPACT_LAZY);
+}
 
 void bch_btree_sort_into(struct cache_set *, struct btree *, struct btree *);
 
diff --git a/drivers/md/bcache/btree_update.c b/drivers/md/bcache/btree_update.c
index 6097a421c35d..6a116051f678 100644
--- a/drivers/md/bcache/btree_update.c
+++ b/drivers/md/bcache/btree_update.c
@@ -1633,8 +1633,8 @@ static struct btree *btree_node_get_sibling(struct btree_iter *iter,
 	return ret;
 }
 
-static int foreground_maybe_merge(struct btree_iter *iter,
-				  enum btree_node_sibling sib)
+static int __foreground_maybe_merge(struct btree_iter *iter,
+				    enum btree_node_sibling sib)
 {
 	struct cache_set *c = iter->c;
 	struct btree_reserve *reserve;
@@ -1787,6 +1787,22 @@ out:
 	return ret;
 }
 
+static int inline foreground_maybe_merge(struct btree_iter *iter,
+					 enum btree_node_sibling sib)
+{
+	struct cache_set *c = iter->c;
+	struct btree *b;
+
+	if (!btree_node_locked(iter, iter->level))
+		return 0;
+
+	b = iter->nodes[iter->level];
+	if (b->sib_u64s[sib] > BTREE_FOREGROUND_MERGE_THRESHOLD(c))
+		return 0;
+
+	return __foreground_maybe_merge(iter, sib);
+}
+
 /**
  * btree_insert_key - insert a key one key into a leaf node
  */