bcache: make sectors_written percpu

6 files changed, 34 insertions, 17 deletions

diff --git a/drivers/md/bcache/alloc.c b/drivers/md/bcache/alloc.c
index 753c32442da0..034df7e45afe 100644
--- a/drivers/md/bcache/alloc.c
+++ b/drivers/md/bcache/alloc.c
@@ -244,8 +244,8 @@ static int bch_prio_write(struct cache *ca)
 
 	trace_bcache_prio_write_start(ca);
 
-	atomic_long_add(ca->mi.bucket_size * prio_buckets(ca),
-			&ca->meta_sectors_written);
+	atomic64_add(ca->mi.bucket_size * prio_buckets(ca),
+		     &ca->meta_sectors_written);
 
 	for (i = prio_buckets(ca) - 1; i >= 0; --i) {
 		struct bucket *g;
@@ -1446,7 +1446,7 @@ void bch_alloc_sectors_done(struct cache_set *c, struct write_point *wp,
 		BUG_ON(xchg(&wp->b, NULL) != ob);
 
 	open_bucket_for_each_online_device(c, ob, ptr, ca)
-		atomic_long_add(sectors, &ca->sectors_written);
+		this_cpu_add(*ca->sectors_written, sectors);
 	cache_member_info_put();
 
 	mutex_unlock(&ob->lock);
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 47f169a9a3c1..cc95529e26da 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -461,9 +461,9 @@ struct cache {
 	atomic_t		io_errors;
 	atomic_t		io_count;
 
-	atomic_long_t		meta_sectors_written;
-	atomic_long_t		btree_sectors_written;
-	atomic_long_t		sectors_written;
+	atomic64_t		meta_sectors_written;
+	atomic64_t		btree_sectors_written;
+	u64 __percpu		*sectors_written;
 };
 
 /*
diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c
index 3e58c91f4ea8..71479a9a2ea1 100644
--- a/drivers/md/bcache/btree_io.c
+++ b/drivers/md/bcache/btree_io.c
@@ -635,8 +635,8 @@ static void do_btree_node_write(struct closure *cl)
 
 	rcu_read_lock();
 	extent_for_each_online_device(c, e, ptr, ca)
-		atomic_long_add(blocks_to_write << c->block_bits,
-				&ca->btree_sectors_written);
+		atomic64_add(blocks_to_write << c->block_bits,
+			     &ca->btree_sectors_written);
 	rcu_read_unlock();
 
 	b->written += blocks_to_write;
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index 5c35881f0378..836afe2c7f2d 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -1679,7 +1679,7 @@ static void journal_write(struct closure *cl)
 
 		bio = &ca->journal.bio;
 
-		atomic_long_add(sectors, &ca->meta_sectors_written);
+		atomic64_add(sectors, &ca->meta_sectors_written);
 
 		bio_reset(bio);
 		bio->bi_iter.bi_sector	= ptr->offset;
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 48d90e3edb33..61394008e18f 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -1667,6 +1667,7 @@ static void bch_cache_free_work(struct work_struct *work)
 	 * However, they were zeroed when the object was allocated.
 	 */
 
+	free_percpu(ca->sectors_written);
 	bioset_exit(&ca->replica_set);
 	free_percpu(ca->bucket_stats_percpu);
 	kfree(ca->journal.bucket_seq);
@@ -1945,7 +1946,8 @@ static const char *cache_alloc(struct bcache_superblock *sb,
 					       sizeof(u64), GFP_KERNEL)) ||
 	    !(ca->bio_prio = bio_kmalloc(GFP_NOIO, bucket_pages(ca))) ||
 	    bioset_init(&ca->replica_set, 4,
-			offsetof(struct bch_write_bio, bio.bio)))
+			offsetof(struct bch_write_bio, bio.bio)) ||
+	    !(ca->sectors_written = alloc_percpu(*ca->sectors_written)))
 		goto err;
 
 	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
index 1e1380c2eb7d..02961774496d 100644
--- a/drivers/md/bcache/sysfs.c
+++ b/drivers/md/bcache/sysfs.c
@@ -1158,6 +1158,17 @@ static ssize_t show_cache_alloc_debug(struct cache *ca, char *buf)
 			 ca->reserve_buckets_count);
 }
 
+static u64 sectors_written(struct cache *ca)
+{
+	u64 ret = 0;
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		ret += *per_cpu_ptr(ca->sectors_written, cpu);
+
+	return ret;
+}
+
 SHOW(bch_cache)
 {
 	struct cache *ca = container_of(kobj, struct cache, kobj);
@@ -1173,12 +1184,12 @@ SHOW(bch_cache)
 	sysfs_print(first_bucket,	ca->mi.first_bucket);
 	sysfs_print(nbuckets,		ca->mi.nbuckets);
 	sysfs_print(discard,		ca->mi.discard);
-	sysfs_hprint(written, atomic_long_read(&ca->sectors_written) << 9);
+	sysfs_hprint(written, sectors_written(ca) << 9);
 	sysfs_hprint(btree_written,
-		     atomic_long_read(&ca->btree_sectors_written) << 9);
+		     atomic64_read(&ca->btree_sectors_written) << 9);
 	sysfs_hprint(metadata_written,
-		     (atomic_long_read(&ca->meta_sectors_written) +
-		      atomic_long_read(&ca->btree_sectors_written)) << 9);
+		     (atomic64_read(&ca->meta_sectors_written) +
+		      atomic64_read(&ca->btree_sectors_written)) << 9);
 
 	sysfs_print(io_errors,
 		    atomic_read(&ca->io_errors) >> IO_ERROR_SHIFT);
@@ -1310,9 +1321,13 @@ STORE(__bch_cache)
 	}
 
 	if (attr == &sysfs_clear_stats) {
-		atomic_long_set(&ca->sectors_written, 0);
-		atomic_long_set(&ca->btree_sectors_written, 0);
-		atomic_long_set(&ca->meta_sectors_written, 0);
+		int cpu;
+
+		for_each_possible_cpu(cpu)
+			*per_cpu_ptr(ca->sectors_written, cpu) = 0;
+
+		atomic64_set(&ca->btree_sectors_written, 0);
+		atomic64_set(&ca->meta_sectors_written, 0);
 		atomic_set(&ca->io_count, 0);
 		atomic_set(&ca->io_errors, 0);
 	}