xfs: use separate btree cursor slab for each btree typebtree-cursor-zones_2021-10-22

Now that we have the infrastructure to track the max possible height of
each btree type, we can create a separate slab zone for cursors of each
type of btree.  For smaller indices like the free space btrees, this
means that we can pack more cursors into a slab page, improving slab
utilization.

Signed-off-by: Darrick J. Wong <djwong@kernel.org>

3 files changed, 34 insertions, 21 deletions

diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c
index 31826a4c98f0..7dfa2b83aa99 100644
--- a/fs/xfs/libxfs/xfs_btree.c
+++ b/fs/xfs/libxfs/xfs_btree.c
@@ -27,10 +27,9 @@
 /*
  * Cursor allocation zone.
  */
-kmem_zone_t	*xfs_btree_cur_zone;
-
 struct xfs_btree_cur_cache {
 	const char		*name;
+	kmem_zone_t		*cache;
 	unsigned short		maxlevels;
 	bool			alias;
 };
@@ -374,6 +373,7 @@ xfs_btree_del_cursor(
 	struct xfs_btree_cur	*cur,		/* btree cursor */
 	int			error)		/* del because of error */
 {
+	struct xfs_btree_cur_cache *cc = &xfs_btree_cur_caches[cur->bc_btnum];
 	int			i;		/* btree level */
 
 	/*
@@ -396,7 +396,7 @@ xfs_btree_del_cursor(
 		kmem_free(cur->bc_ops);
 	if (!(cur->bc_flags & XFS_BTREE_LONG_PTRS) && cur->bc_ag.pag)
 		xfs_perag_put(cur->bc_ag.pag);
-	kmem_cache_free(xfs_btree_cur_zone, cur);
+	kmem_cache_free(cc->cache, cur);
 }
 
 /*
@@ -5329,10 +5329,11 @@ xfs_btree_alloc_cursor(
 {
 	struct xfs_btree_cur	*cur;
 	unsigned int		maxlevels = xfs_btree_maxlevels(mp, btnum);
+	struct xfs_btree_cur_cache *cc = &xfs_btree_cur_caches[btnum];
 
-	ASSERT(maxlevels <= XFS_BTREE_CUR_ZONE_MAXLEVELS);
+	ASSERT(maxlevels <= cc->maxlevels);
 
-	cur = kmem_cache_zalloc(xfs_btree_cur_zone, GFP_NOFS | __GFP_NOFAIL);
+	cur = kmem_cache_zalloc(cc->cache, GFP_NOFS | __GFP_NOFAIL);
 	cur->bc_tp = tp;
 	cur->bc_mp = mp;
 	cur->bc_btnum = btnum;
@@ -5390,6 +5391,12 @@ xfs_btree_create_cursor_cache(
 	unsigned int				maxlevels)
 {
 	struct xfs_btree_cur_cache		*cc;
+	kmem_zone_t				*cache;
+
+	cache = kmem_cache_create(name, xfs_btree_cur_sizeof(maxlevels), 0, 0,
+			NULL);
+	if (!cache)
+		return -ENOMEM;
 
 	cc = &xfs_btree_cur_caches[btnum];
 
@@ -5398,6 +5405,7 @@ xfs_btree_create_cursor_cache(
 	cc->name = name;
 	cc->maxlevels = maxlevels;
 	cc->alias = false;
+	cc->cache = cache;
 
 	return 0;
 }
@@ -5421,10 +5429,28 @@ xfs_btree_alias_cursor_cache(
 	cd->name = cs->name;
 	cd->maxlevels = cs->maxlevels;
 	cd->alias = true;
+	cd->cache = cs->cache;
 
 	return 0;
 }
 
+/* Destroy all btree cursor caches. */
+void
+xfs_btree_destroy_cursor_caches(void)
+{
+	struct xfs_btree_cur_cache	*cc = xfs_btree_cur_caches;
+	unsigned int			i = 0;
+
+	for (; i < ARRAY_SIZE(xfs_btree_cur_caches); i--, cc++) {
+		cc->name = NULL;
+		cc->maxlevels = 0;
+		if (!cc->alias && cc->cache)
+			kmem_cache_destroy(cc->cache);
+		cc->alias = NULL;
+		cc->cache = NULL;
+	}
+}
+
 /* Return the maximum possible height for a given type of btree. */
 unsigned int
 xfs_btree_absolute_maxlevels(
diff --git a/fs/xfs/libxfs/xfs_btree.h b/fs/xfs/libxfs/xfs_btree.h
index f84d36b4b6c2..276da72e8597 100644
--- a/fs/xfs/libxfs/xfs_btree.h
+++ b/fs/xfs/libxfs/xfs_btree.h
@@ -13,8 +13,6 @@ struct xfs_trans;
 struct xfs_ifork;
 struct xfs_perag;
 
-extern kmem_zone_t	*xfs_btree_cur_zone;
-
 /*
  * Generic key, ptr and record wrapper structures.
  *
@@ -92,12 +90,6 @@ uint32_t xfs_btree_magic(int crc, xfs_btnum_t btnum);
 #define XFS_BTREE_STATS_ADD(cur, stat, val)	\
 	XFS_STATS_ADD_OFF((cur)->bc_mp, (cur)->bc_statoff + __XBTS_ ## stat, val)
 
-/*
- * The btree cursor zone hands out cursors that can handle up to this many
- * levels.  This is the known maximum for all btree types.
- */
-#define XFS_BTREE_CUR_ZONE_MAXLEVELS	(9)
-
 struct xfs_btree_ops {
 	/* size of the key and record structures */
 	size_t	key_len;
@@ -599,5 +591,6 @@ int __init xfs_btree_create_cursor_cache(xfs_btnum_t btnum, const char *name,
 		unsigned int maxlevels);
 int __init xfs_btree_alias_cursor_cache(xfs_btnum_t btnum, xfs_btnum_t src);
 unsigned int xfs_btree_absolute_maxlevels(xfs_btnum_t btnum);
+void xfs_btree_destroy_cursor_caches(void);
 
 #endif	/* __XFS_BTREE_H__ */
diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
index 0b1c1960e6ce..96ba050f51d5 100644
--- a/fs/xfs/xfs_super.c
+++ b/fs/xfs/xfs_super.c
@@ -2057,12 +2057,6 @@ xfs_init_zones(void)
 	if (error)
 		goto out_destroy_bmap_free_item_zone;
 
-	xfs_btree_cur_zone = kmem_cache_create("xfs_btree_cur",
-			xfs_btree_cur_sizeof(XFS_BTREE_CUR_ZONE_MAXLEVELS),
-			0, 0, NULL);
-	if (!xfs_btree_cur_zone)
-		goto out_destroy_bmap_free_item_zone;
-
 	xfs_da_state_zone = kmem_cache_create("xfs_da_state",
 					      sizeof(struct xfs_da_state),
 					      0, 0, NULL);
@@ -2214,7 +2208,7 @@ xfs_init_zones(void)
  out_destroy_da_state_zone:
 	kmem_cache_destroy(xfs_da_state_zone);
  out_destroy_btree_cur_zone:
-	kmem_cache_destroy(xfs_btree_cur_zone);
+	xfs_btree_destroy_cursor_caches();
  out_destroy_bmap_free_item_zone:
 	kmem_cache_destroy(xfs_bmap_free_item_zone);
  out_destroy_log_ticket_zone:
@@ -2246,7 +2240,7 @@ xfs_destroy_zones(void)
 	kmem_cache_destroy(xfs_trans_zone);
 	kmem_cache_destroy(xfs_ifork_zone);
 	kmem_cache_destroy(xfs_da_state_zone);
-	kmem_cache_destroy(xfs_btree_cur_zone);
+	xfs_btree_destroy_cursor_caches();
 	kmem_cache_destroy(xfs_bmap_free_item_zone);
 	kmem_cache_destroy(xfs_log_ticket_zone);
 }