summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKent Overstreet <kent.overstreet@gmail.com>2016-04-25 17:17:30 -0800
committerKent Overstreet <kent.overstreet@gmail.com>2017-01-18 21:38:33 -0900
commitd1b3e4a41c3ec7025873830276cbea2d75f3895f (patch)
tree8b91a11ff77be46b809d60b7d6f861c3e5744d9a
parent5e649b72f33716a57fa28dff5dd26b688f72a9c9 (diff)
bcache: Plumb disk_reservation through bch_write
-rw-r--r--drivers/md/bcache/btree_gc.c74
-rw-r--r--drivers/md/bcache/btree_gc.h3
-rw-r--r--drivers/md/bcache/btree_io.c2
-rw-r--r--drivers/md/bcache/btree_types.h5
-rw-r--r--drivers/md/bcache/btree_update.c151
-rw-r--r--drivers/md/bcache/btree_update.h12
-rw-r--r--drivers/md/bcache/buckets.c123
-rw-r--r--drivers/md/bcache/buckets.h22
-rw-r--r--drivers/md/bcache/buckets_types.h8
-rw-r--r--drivers/md/bcache/dirent.c4
-rw-r--r--drivers/md/bcache/extents.c78
-rw-r--r--drivers/md/bcache/extents.h4
-rw-r--r--drivers/md/bcache/fs-gc.c2
-rw-r--r--drivers/md/bcache/fs-io.c79
-rw-r--r--drivers/md/bcache/fs-io.h1
-rw-r--r--drivers/md/bcache/fs.c2
-rw-r--r--drivers/md/bcache/inode.c5
-rw-r--r--drivers/md/bcache/io.c16
-rw-r--r--drivers/md/bcache/io.h3
-rw-r--r--drivers/md/bcache/io_types.h3
-rw-r--r--drivers/md/bcache/journal.c46
-rw-r--r--drivers/md/bcache/journal.h3
-rw-r--r--drivers/md/bcache/migrate.c7
-rw-r--r--drivers/md/bcache/move.c2
-rw-r--r--drivers/md/bcache/move.h1
-rw-r--r--drivers/md/bcache/movinggc.c5
-rw-r--r--drivers/md/bcache/request.c13
-rw-r--r--drivers/md/bcache/super.c2
-rw-r--r--drivers/md/bcache/tier.c1
-rw-r--r--drivers/md/bcache/writeback.c2
-rw-r--r--drivers/md/bcache/xattr.c4
31 files changed, 417 insertions, 266 deletions
diff --git a/drivers/md/bcache/btree_gc.c b/drivers/md/bcache/btree_gc.c
index a62d7778f4df..724024c0c06e 100644
--- a/drivers/md/bcache/btree_gc.c
+++ b/drivers/md/bcache/btree_gc.c
@@ -122,7 +122,8 @@ u8 bch_btree_key_recalc_oldest_gen(struct cache_set *c, struct bkey_s_c k)
* For runtime mark and sweep:
*/
static u8 __bch_btree_mark_key(struct cache_set *c, enum bkey_type type,
- struct bkey_s_c k)
+ struct bkey_s_c k,
+ struct bucket_stats_cache_set *stats)
{
switch (type) {
case BKEY_TYPE_BTREE:
@@ -135,7 +136,7 @@ static u8 __bch_btree_mark_key(struct cache_set *c, enum bkey_type type,
? c->sb.btree_node_size
: e.k->size, false,
type == BKEY_TYPE_BTREE,
- true, GC_POS_MIN);
+ true, GC_POS_MIN, stats);
}
return bch_btree_key_recalc_oldest_gen(c, k);
@@ -145,25 +146,27 @@ static u8 __bch_btree_mark_key(struct cache_set *c, enum bkey_type type,
}
static u8 btree_mark_key(struct cache_set *c, struct btree *b,
- struct bkey_s_c k)
+ struct bkey_s_c k,
+ struct bucket_stats_cache_set *stats)
{
- return __bch_btree_mark_key(c, btree_node_type(b), k);
+ return __bch_btree_mark_key(c, btree_node_type(b), k, stats);
}
/*
* For initial cache set bringup:
*/
u8 __bch_btree_mark_key_initial(struct cache_set *c, enum bkey_type type,
- struct bkey_s_c k)
+ struct bkey_s_c k,
+ struct bucket_stats_cache_set *stats)
{
switch (type) {
case BKEY_TYPE_BTREE:
case BKEY_TYPE_EXTENTS:
if (k.k->type == BCH_RESERVATION)
- bch_mark_reservation(c, k.k->size);
+ stats->sectors_reserved += k.k->size;
- return __bch_btree_mark_key(c, type, k);
+ return __bch_btree_mark_key(c, type, k, stats);
default:
BUG();
}
@@ -171,12 +174,14 @@ u8 __bch_btree_mark_key_initial(struct cache_set *c, enum bkey_type type,
}
static u8 btree_mark_key_initial(struct cache_set *c, struct btree *b,
- struct bkey_s_c k)
+ struct bkey_s_c k,
+ struct bucket_stats_cache_set *stats)
{
- return __bch_btree_mark_key_initial(c, btree_node_type(b), k);
+ return __bch_btree_mark_key_initial(c, btree_node_type(b), k, stats);
}
-static bool btree_gc_mark_node(struct cache_set *c, struct btree *b)
+static bool btree_gc_mark_node(struct cache_set *c, struct btree *b,
+ struct bucket_stats_cache_set *stats)
{
if (btree_node_has_ptrs(b)) {
struct btree_node_iter iter;
@@ -187,7 +192,7 @@ static bool btree_gc_mark_node(struct cache_set *c, struct btree *b)
for_each_btree_node_key_unpack(&b->keys, k, &iter, &unpacked) {
bkey_debugcheck(c, b, k);
- stale = max(stale, btree_mark_key(c, b, k));
+ stale = max(stale, btree_mark_key(c, b, k, stats));
}
if (btree_gc_rewrite_disabled(c))
@@ -216,7 +221,8 @@ static inline void gc_pos_set(struct cache_set *c, struct gc_pos new_pos)
__gc_pos_set(c, new_pos);
}
-static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id)
+static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id,
+ struct bucket_stats_cache_set *stats)
{
struct btree_iter iter;
struct btree *b;
@@ -230,7 +236,7 @@ static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id)
bch_verify_btree_nr_keys(&b->keys);
- should_rewrite = btree_gc_mark_node(c, b);
+ should_rewrite = btree_gc_mark_node(c, b, stats);
gc_pos_set(c, gc_pos_btree_node(b));
@@ -244,7 +250,7 @@ static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id)
spin_lock(&c->btree_root_lock);
b = c->btree_roots[btree_id];
- __bch_btree_mark_key(c, BKEY_TYPE_BTREE, bkey_i_to_s_c(&b->key));
+ __bch_btree_mark_key(c, BKEY_TYPE_BTREE, bkey_i_to_s_c(&b->key), stats);
gc_pos_set(c, gc_pos_btree_root(b->btree_id));
spin_unlock(&c->btree_root_lock);
@@ -300,7 +306,7 @@ static void bch_mark_metadata(struct cache_set *c)
for (j = 0; j < bch_nr_journal_buckets(ca->disk_sb.sb); j++)
bch_mark_metadata_bucket(ca,
&ca->buckets[journal_bucket(ca, j)],
- true);
+ true);
spin_lock(&ca->prio_buckets_lock);
@@ -312,6 +318,7 @@ static void bch_mark_metadata(struct cache_set *c)
}
}
+/* Also see bch_pending_btree_node_free_insert_done() */
static void bch_mark_pending_btree_node_frees(struct cache_set *c)
{
struct pending_btree_node_free *d;
@@ -319,12 +326,20 @@ static void bch_mark_pending_btree_node_frees(struct cache_set *c)
mutex_lock(&c->btree_node_pending_free_lock);
gc_pos_set(c, gc_phase(GC_PHASE_PENDING_DELETE));
- list_for_each_entry(d, &c->btree_node_pending_free, list)
+ list_for_each_entry(d, &c->btree_node_pending_free, list) {
+ /*
+ * Already accounted for in cache_set_stats - we won't apply
+ * @stats (see bch_pending_btree_node_free_insert_done()):
+ */
+ struct bucket_stats_cache_set stats = { 0 };
+
if (d->index_update_done)
bch_mark_pointers(c, bkey_i_to_s_c_extent(&d->key),
c->sb.btree_node_size,
false, true,
- true, GC_POS_MIN);
+ true, GC_POS_MIN, &stats);
+ }
+
mutex_unlock(&c->btree_node_pending_free_lock);
}
@@ -350,6 +365,7 @@ static void bch_mark_scan_keylists(struct cache_set *c)
*/
void bch_gc(struct cache_set *c)
{
+ struct bucket_stats_cache_set stats = { 0 };
struct cache *ca;
struct bucket *g;
u64 start_time = local_clock();
@@ -411,7 +427,7 @@ void bch_gc(struct cache_set *c)
/* Walk btree: */
while (c->gc_pos.phase < (int) BTREE_ID_NR) {
int ret = c->btree_roots[c->gc_pos.phase]
- ? bch_gc_btree(c, (int) c->gc_pos.phase)
+ ? bch_gc_btree(c, (int) c->gc_pos.phase, &stats)
: 0;
if (ret) {
@@ -432,6 +448,8 @@ void bch_gc(struct cache_set *c)
for_each_cache(ca, c, i)
atomic_long_set(&ca->saturated_count, 0);
+ bch_cache_set_stats_set(c, &stats);
+
/* Indicates that gc is no longer in progress: */
gc_pos_set(c, gc_phase(GC_PHASE_DONE));
@@ -852,7 +870,8 @@ int bch_gc_thread_start(struct cache_set *c)
/* Initial GC computes bucket marks during startup */
-static void bch_initial_gc_btree(struct cache_set *c, enum btree_id id)
+static void bch_initial_gc_btree(struct cache_set *c, enum btree_id id,
+ struct bucket_stats_cache_set *stats)
{
struct btree_iter iter;
struct btree *b;
@@ -873,30 +892,35 @@ static void bch_initial_gc_btree(struct cache_set *c, enum btree_id id)
for_each_btree_node_key_unpack(&b->keys, k,
&node_iter, &unpacked)
- btree_mark_key_initial(c, b, k);
+ btree_mark_key_initial(c, b, k, stats);
}
- __bch_btree_mark_key_initial(c, BKEY_TYPE_BTREE,
- bkey_i_to_s_c(&b->key));
-
bch_btree_iter_cond_resched(&iter);
}
bch_btree_iter_unlock(&iter);
+
+ b = c->btree_roots[id];
+ __bch_btree_mark_key_initial(c, BKEY_TYPE_BTREE,
+ bkey_i_to_s_c(&b->key),
+ stats);
}
int bch_initial_gc(struct cache_set *c, struct list_head *journal)
{
+ struct bucket_stats_cache_set stats = { 0 };
enum btree_id id;
if (journal) {
for (id = 0; id < BTREE_ID_NR; id++)
- bch_initial_gc_btree(c, id);
+ bch_initial_gc_btree(c, id, &stats);
- bch_journal_mark(c, journal);
+ bch_journal_mark(c, journal, &stats);
}
bch_mark_metadata(c);
+
+ bch_cache_set_stats_set(c, &stats);
gc_pos_set(c, gc_phase(GC_PHASE_DONE));
set_bit(CACHE_SET_INITIAL_GC_DONE, &c->flags);
diff --git a/drivers/md/bcache/btree_gc.h b/drivers/md/bcache/btree_gc.h
index cba0a5be5378..97e33f46cc98 100644
--- a/drivers/md/bcache/btree_gc.h
+++ b/drivers/md/bcache/btree_gc.h
@@ -11,7 +11,8 @@ int bch_gc_thread_start(struct cache_set *);
int bch_initial_gc(struct cache_set *, struct list_head *);
u8 bch_btree_key_recalc_oldest_gen(struct cache_set *, struct bkey_s_c);
u8 __bch_btree_mark_key_initial(struct cache_set *, enum bkey_type,
- struct bkey_s_c);
+ struct bkey_s_c,
+ struct bucket_stats_cache_set *);
/*
* For concurrent mark and sweep (with other index updates), we define a total
diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c
index b997909706f1..56788144acbb 100644
--- a/drivers/md/bcache/btree_io.c
+++ b/drivers/md/bcache/btree_io.c
@@ -463,7 +463,7 @@ int bch_btree_root_read(struct cache_set *c, enum btree_id id,
return -EIO;
}
- bch_btree_set_root_initial(c, b);
+ bch_btree_set_root_initial(c, b, NULL);
six_unlock_intent(&b->lock);
return 0;
diff --git a/drivers/md/bcache/btree_types.h b/drivers/md/bcache/btree_types.h
index 4f335f56aa67..f3af91e3d23c 100644
--- a/drivers/md/bcache/btree_types.h
+++ b/drivers/md/bcache/btree_types.h
@@ -140,12 +140,15 @@ static inline bool btree_node_has_ptrs(struct btree *b)
*/
struct btree_iter;
+struct bucket_stats_cache_set;
+
struct btree_insert_hook {
void (*fn)(struct btree_insert_hook *,
struct btree_iter *iter,
struct bkey_s_c k,
struct bkey_i *insert,
- struct journal_res *res);
+ struct journal_res *res,
+ struct bucket_stats_cache_set *);
};
#endif /* _BCACHE_BTREE_TYPES_H */
diff --git a/drivers/md/bcache/btree_update.c b/drivers/md/bcache/btree_update.c
index d5da485d0cb7..3305c6eff047 100644
--- a/drivers/md/bcache/btree_update.c
+++ b/drivers/md/bcache/btree_update.c
@@ -141,12 +141,24 @@ found:
* cancel out one of mark and sweep's markings if necessary:
*/
- if (gc_pos_cmp(c->gc_pos, gc_phase(GC_PHASE_PENDING_DELETE)) < 0)
+ /*
+ * bch_mark_pointers() compares the current gc pos to the pos we're
+ * moving this reference from, hence one comparison here:
+ */
+ if (gc_pos_cmp(c->gc_pos, gc_phase(GC_PHASE_PENDING_DELETE)) < 0) {
+ /*
+ * Already accounted for in cache_set_stats - we won't apply
+ * @stats:
+ */
+ struct bucket_stats_cache_set stats = { 0 };
+
bch_mark_pointers(c, bkey_i_to_s_c_extent(&d->key),
-c->sb.btree_node_size,
false, true, false, b
? gc_pos_btree_node(b)
- : gc_pos_btree_root(id));
+ : gc_pos_btree_root(id),
+ &stats);
+ }
mutex_unlock(&c->btree_node_pending_free_lock);
}
@@ -210,6 +222,8 @@ void bch_btree_node_free(struct btree_iter *iter, struct btree *b)
static void bch_btree_node_free_ondisk(struct cache_set *c,
struct pending_btree_node_free *pending)
{
+ struct bucket_stats_cache_set stats = { 0 };
+
BUG_ON(!pending->index_update_done);
mutex_lock(&c->btree_node_pending_free_lock);
@@ -217,7 +231,10 @@ static void bch_btree_node_free_ondisk(struct cache_set *c,
bch_mark_pointers(c, bkey_i_to_s_c_extent(&pending->key),
-c->sb.btree_node_size, false, true,
- false, gc_phase(GC_PHASE_PENDING_DELETE));
+ false, gc_phase(GC_PHASE_PENDING_DELETE),
+ &stats);
+
+ /* Already accounted for in cache_set_stats - don't apply @stats: */
mutex_unlock(&c->btree_node_pending_free_lock);
}
@@ -341,7 +358,8 @@ struct btree *btree_node_alloc_replacement(struct cache_set *c,
return __btree_node_alloc_replacement(c, b, new_f, reserve);
}
-static void __bch_btree_set_root(struct cache_set *c, struct btree *b)
+static void __bch_btree_set_root(struct cache_set *c, struct btree *b,
+ struct bucket_stats_cache_set *stats)
{
bool stale;
@@ -355,7 +373,8 @@ static void __bch_btree_set_root(struct cache_set *c, struct btree *b)
stale = bch_mark_pointers(c, bkey_i_to_s_c_extent(&b->key),
c->sb.btree_node_size, true, true,
- false, gc_pos_btree_root(b->btree_id));
+ false, gc_pos_btree_root(b->btree_id),
+ stats);
BUG_ON(stale);
spin_unlock(&c->btree_root_lock);
@@ -366,11 +385,17 @@ static void __bch_btree_set_root(struct cache_set *c, struct btree *b)
* Only for cache set bringup, when first reading the btree roots or allocating
* btree roots when initializing a new cache set:
*/
-void bch_btree_set_root_initial(struct cache_set *c, struct btree *b)
+void bch_btree_set_root_initial(struct cache_set *c, struct btree *b,
+ struct btree_reserve *btree_reserve)
{
+ struct bucket_stats_cache_set stats = { 0 };
+
BUG_ON(btree_node_root(b));
- __bch_btree_set_root(c, b);
+ __bch_btree_set_root(c, b, &stats);
+
+ if (btree_reserve)
+ bch_cache_set_stats_apply(c, &stats, &btree_reserve->disk_res);
}
/**
@@ -386,8 +411,10 @@ void bch_btree_set_root_initial(struct cache_set *c, struct btree *b)
* journal write.
*/
static int bch_btree_set_root(struct btree_iter *iter, struct btree *b,
- struct journal_res *res)
+ struct journal_res *res,
+ struct btree_reserve *btree_reserve)
{
+ struct bucket_stats_cache_set stats = { 0 };
struct cache_set *c = iter->c;
struct btree *old;
u64 seq;
@@ -403,7 +430,7 @@ static int bch_btree_set_root(struct btree_iter *iter, struct btree *b,
*/
btree_node_lock_write(old, iter);
- __bch_btree_set_root(c, b);
+ __bch_btree_set_root(c, b, &stats);
/*
* Unlock old root after new root is visible:
@@ -417,6 +444,9 @@ static int bch_btree_set_root(struct btree_iter *iter, struct btree *b,
bch_pending_btree_node_free_insert_done(c, NULL, old->btree_id,
bkey_i_to_s_c(&old->key));
+ stats.sectors_meta -= c->sb.btree_node_size;
+ bch_cache_set_stats_apply(c, &stats, &btree_reserve->disk_res);
+
/*
* Ensure new btree root is persistent (reachable via the
* journal) before returning and the caller unlocking it:
@@ -499,8 +529,10 @@ static struct btree_reserve *__bch_btree_reserve_get(struct cache_set *c,
* open bucket reserve:
*/
ret = mca_cannibalize_lock(c, cl);
- if (ret)
+ if (ret) {
+ bch_disk_reservation_put(c, &disk_res);
return ERR_PTR(ret);
+ }
reserve = mempool_alloc(&c->btree_reserve_pool, GFP_NOIO);
@@ -562,27 +594,31 @@ int bch_btree_root_alloc(struct cache_set *c, enum btree_id id,
}
b = __btree_root_alloc(c, 0, id, reserve);
- bch_btree_reserve_put(c, reserve);
bch_btree_node_write(b, writes, NULL);
- bch_btree_set_root_initial(c, b);
+ bch_btree_set_root_initial(c, b, reserve);
btree_open_bucket_put(c, b);
six_unlock_intent(&b->lock);
+ bch_btree_reserve_put(c, reserve);
+
return 0;
}
static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter,
struct btree *b,
struct bkey_i *insert,
- struct btree_node_iter *node_iter)
+ struct btree_node_iter *node_iter,
+ struct disk_reservation *disk_res)
{
struct cache_set *c = iter->c;
const struct bkey_format *f = &b->keys.format;
+ struct bucket_stats_cache_set stats = { 0 };
struct bkey_packed *k;
int cmp;
+ bch_btree_node_iter_verify(node_iter, &b->keys);
EBUG_ON((k = bch_btree_node_iter_prev_all(node_iter, &b->keys)) &&
(bkey_deleted(k)
? bkey_cmp_packed(f, k, &insert->k) > 0
@@ -594,7 +630,7 @@ static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter,
stale = bch_mark_pointers(c, bkey_i_to_s_c_extent(insert),
c->sb.btree_node_size,
true, true, false,
- gc_pos_btree_node(b));
+ gc_pos_btree_node(b), &stats);
BUG_ON(stale);
}
@@ -614,6 +650,7 @@ static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter,
*/
k->type = KEY_TYPE_DELETED;
btree_keys_account_key_drop(&b->keys.nr, k);
+ stats.sectors_meta -= c->sb.btree_node_size;
}
bch_btree_node_iter_next_all(node_iter, &b->keys);
@@ -621,6 +658,8 @@ static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter,
bch_btree_bset_insert(iter, b, node_iter, insert);
set_btree_node_dirty(b);
+
+ bch_cache_set_stats_apply(c, &stats, disk_res);
return true;
}
@@ -635,6 +674,7 @@ void bch_btree_bset_insert(struct btree_iter *iter,
struct btree_iter *linked;
struct bkey_packed *where;
+ EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k));
EBUG_ON(insert->k.u64s > bch_btree_keys_u64s_remaining(iter->c, b));
EBUG_ON(bkey_cmp(bkey_start_pos(&insert->k), b->data->min_key) < 0 ||
bkey_cmp(insert->k.p, b->data->max_key) > 0);
@@ -736,7 +776,7 @@ void bch_btree_insert_and_journal(struct btree_iter *iter,
* @insert_keys: list of keys to insert
* @replace: old key for for exchange (+ stats)
* @res: journal reservation
- * @flags: BTREE_INSERT_NOFAIL_IF_STALE
+ * @flags: BTREE_INSERT_NO_MARK_KEY
*
* Inserts the first key from @insert_keys
*
@@ -746,6 +786,7 @@ void bch_btree_insert_and_journal(struct btree_iter *iter,
static void btree_insert_key(struct btree_iter *iter, struct btree *b,
struct btree_node_iter *node_iter,
struct keylist *insert_keys,
+ struct disk_reservation *disk_res,
struct btree_insert_hook *hook,
struct journal_res *res,
unsigned flags)
@@ -754,31 +795,22 @@ static void btree_insert_key(struct btree_iter *iter, struct btree *b,
BKEY_PADDED(key) temp;
s64 oldsize = bch_count_data(&b->keys);
- BUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k));
bch_btree_node_iter_verify(node_iter, &b->keys);
+ BUG_ON(b->level);
+ BUG_ON(iter->nodes[0] != b || &iter->node_iters[0] != node_iter);
- if (b->level) {
- BUG_ON(res->ref);
-
- bch_insert_fixup_btree_ptr(iter, b, insert, node_iter);
- bch_keylist_dequeue(insert_keys);
- } else if (!b->keys.ops->is_extents) {
- BUG_ON(iter->nodes[0] != b ||
- &iter->node_iters[0] != node_iter);
-
+ if (!b->keys.ops->is_extents) {
bch_insert_fixup_key(iter, insert, hook, res);
bch_keylist_dequeue(insert_keys);
} else {
- BUG_ON(iter->nodes[0] != b ||
- &iter->node_iters[0] != node_iter);
-
bkey_copy(&temp.key, insert);
insert = &temp.key;
if (bkey_cmp(insert->k.p, b->key.k.p) > 0)
bch_cut_back(b->key.k.p, &insert->k);
- bch_insert_fixup_extent(iter, insert, hook, res, flags);
+ bch_insert_fixup_extent(iter, insert, disk_res,
+ hook, res, flags);
bch_cut_front(iter->pos, orig);
if (orig->k.size == 0)
@@ -1068,12 +1100,12 @@ static enum btree_insert_status
bch_btree_insert_keys_interior(struct btree *b,
struct btree_iter *iter,
struct keylist *insert_keys,
- struct async_split *as)
+ struct async_split *as,
+ struct btree_reserve *res)
{
struct btree_node_iter *node_iter = &iter->node_iters[b->level];
const struct bkey_format *f = &b->keys.format;
struct bkey_packed *k;
- struct journal_res res = { 0, 0 };
BUG_ON(!btree_node_intent_locked(iter, btree_node_root(b)->level));
BUG_ON(!b->level);
@@ -1109,8 +1141,9 @@ bch_btree_insert_keys_interior(struct btree *b,
(bkey_cmp_packed(f, k, &insert->k) >= 0))
;
- btree_insert_key(iter, b, node_iter, insert_keys,
- NULL, &res, 0);
+ bch_insert_fixup_btree_ptr(iter, b, insert,
+ node_iter, &res->disk_res);
+ bch_keylist_dequeue(insert_keys);
}
btree_node_unlock_write(b, iter);
@@ -1146,6 +1179,7 @@ static enum btree_insert_status
bch_btree_insert_keys_leaf(struct btree *b,
struct btree_iter *iter,
struct keylist *insert_keys,
+ struct disk_reservation *disk_res,
struct btree_insert_hook *hook,
u64 *journal_seq,
unsigned flags)
@@ -1203,7 +1237,8 @@ bch_btree_insert_keys_leaf(struct btree *b,
break;
btree_insert_key(iter, b, &iter->node_iters[b->level],
- insert_keys, hook, &res, flags);
+ insert_keys, disk_res,
+ hook, &res, flags);
}
btree_node_unlock_write(b, iter);
@@ -1299,9 +1334,10 @@ static struct btree *__btree_split_node(struct btree_iter *iter, struct btree *n
}
static void btree_split_insert_keys(struct btree_iter *iter, struct btree *b,
- struct keylist *keys, bool is_last)
+ struct keylist *keys,
+ struct btree_reserve *res,
+ bool is_last)
{
- struct journal_res res = { 0, 0 };
struct btree_node_iter node_iter;
struct bkey_i *k = bch_keylist_front(keys);
@@ -1323,8 +1359,8 @@ static void btree_split_insert_keys(struct btree_iter *iter, struct btree *b,
break;
}
- btree_insert_key(iter, b, &node_iter, keys,
- NULL, &res, 0);
+ bch_insert_fixup_btree_ptr(iter, b, k, &node_iter, &res->disk_res);
+ bch_keylist_dequeue(keys);
}
six_unlock_write(&b->lock);
@@ -1384,7 +1420,7 @@ static int btree_split(struct btree *b, struct btree_iter *iter,
struct bset *i;
six_unlock_write(&n1->lock);
- btree_split_insert_keys(iter, n1, insert_keys, true);
+ btree_split_insert_keys(iter, n1, insert_keys, reserve, true);
six_lock_write(&n1->lock);
/*
@@ -1438,7 +1474,8 @@ static int btree_split(struct btree *b, struct btree_iter *iter,
iter->btree_id,
reserve);
- btree_split_insert_keys(iter, n3, &as->parent_keys, true);
+ btree_split_insert_keys(iter, n3, &as->parent_keys,
+ reserve, true);
bch_btree_node_write(n3, &as->cl, NULL);
}
} else {
@@ -1470,12 +1507,12 @@ static int btree_split(struct btree *b, struct btree_iter *iter,
goto err;
if (n3) {
- ret = bch_btree_set_root(iter, n3, &as->res);
+ ret = bch_btree_set_root(iter, n3, &as->res, reserve);
if (ret)
goto err;
} else {
/* Root filled up but didn't need to be split */
- ret = bch_btree_set_root(iter, n1, &as->res);
+ ret = bch_btree_set_root(iter, n1, &as->res, reserve);
if (ret)
goto err;
}
@@ -1533,7 +1570,7 @@ err:
* @insert_keys: list of keys to insert
* @hook: insert callback
* @persistent: if not null, @persistent will wait on journal write
- * @flags: BTREE_INSERT_NOFAIL_IF_STALE
+ * @flags: BTREE_INSERT_NO_MARK_KEY
*
* Inserts as many keys as it can into a given btree node, splitting it if full.
* If a split occurred, this function will return early. This can only happen
@@ -1548,7 +1585,8 @@ int bch_btree_insert_node(struct btree *b,
BUG_ON(!b->level);
BUG_ON(!reserve || !as);
- switch (bch_btree_insert_keys_interior(b, iter, insert_keys, as)) {
+ switch (bch_btree_insert_keys_interior(b, iter, insert_keys,
+ as, reserve)) {
case BTREE_INSERT_OK:
return 0;
case BTREE_INSERT_NEED_SPLIT:
@@ -1610,7 +1648,7 @@ out_unlock:
* @insert_keys: list of keys to insert
* @hook: insert callback
* @persistent: if not null, @persistent will wait on journal write
- * @flags: BTREE_INSERT_ATOMIC | BTREE_INSERT_NOFAIL_IF_STALE
+ * @flags: BTREE_INSERT_ATOMIC | BTREE_INSERT_NO_MARK_KEY
*
* This is top level for common btree insertion/index update code. The control
* flow goes roughly like:
@@ -1639,6 +1677,7 @@ out_unlock:
*/
int bch_btree_insert_at(struct btree_iter *iter,
struct keylist *insert_keys,
+ struct disk_reservation *disk_res,
struct btree_insert_hook *hook,
u64 *journal_seq, unsigned flags)
{
@@ -1658,8 +1697,8 @@ int bch_btree_insert_at(struct btree_iter *iter,
iter->pos));
switch (bch_btree_insert_keys_leaf(iter->nodes[0], iter,
- insert_keys, hook,
- journal_seq, flags)) {
+ insert_keys, disk_res,
+ hook, journal_seq, flags)) {
case BTREE_INSERT_OK:
ret = 0;
break;
@@ -1794,7 +1833,8 @@ retry:
for (i = m; i < m + nr; i++)
btree_insert_key(i->iter, i->iter->nodes[0],
&i->iter->node_iters[0],
- &keylist_single(i->k), NULL,
+ &keylist_single(i->k),
+ NULL, NULL,
&res, flags);
do {
@@ -1873,7 +1913,7 @@ int bch_btree_insert_check_key(struct btree_iter *iter,
bch_btree_iter_rewind(iter, bkey_start_pos(&check_key->k));
ret = bch_btree_insert_at(iter, &keylist_single(&tmp.key),
- NULL, NULL, BTREE_INSERT_ATOMIC);
+ NULL, NULL, NULL, BTREE_INSERT_ATOMIC);
bch_btree_iter_set_pos(iter, saved_pos);
@@ -1888,7 +1928,9 @@ int bch_btree_insert_check_key(struct btree_iter *iter,
* @hook: insert callback
*/
int bch_btree_insert(struct cache_set *c, enum btree_id id,
- struct keylist *keys, struct btree_insert_hook *hook,
+ struct keylist *keys,
+ struct disk_reservation *disk_res,
+ struct btree_insert_hook *hook,
u64 *journal_seq, int flags)
{
struct btree_iter iter;
@@ -1901,7 +1943,8 @@ int bch_btree_insert(struct cache_set *c, enum btree_id id,
if (unlikely(ret))
goto out;
- ret = bch_btree_insert_at(&iter, keys, hook, journal_seq, flags);
+ ret = bch_btree_insert_at(&iter, keys, disk_res,
+ hook, journal_seq, flags);
out: ret2 = bch_btree_iter_unlock(&iter);
return ret ?: ret2;
@@ -1928,7 +1971,7 @@ int bch_btree_update(struct cache_set *c, enum btree_id id,
return -ENOENT;
ret = bch_btree_insert_at(&iter, &keylist_single(k), NULL,
- journal_seq, 0);
+ NULL, journal_seq, 0);
ret2 = bch_btree_iter_unlock(&iter);
return ret ?: ret2;
@@ -1991,7 +2034,7 @@ int bch_btree_delete_range(struct cache_set *c, enum btree_id id,
}
ret = bch_btree_insert_at(&iter, &keylist_single(&delete),
- hook, journal_seq,
+ NULL, hook, journal_seq,
BTREE_INSERT_NOFAIL);
if (ret)
break;
@@ -2056,7 +2099,7 @@ int bch_btree_node_rewrite(struct btree *b, struct btree_iter *iter, bool wait)
return -EIO;
}
- bch_btree_set_root(iter, n, &as->res);
+ bch_btree_set_root(iter, n, &as->res, reserve);
continue_at_noreturn(&as->cl, async_split_writes_done,
system_wq);
diff --git a/drivers/md/bcache/btree_update.h b/drivers/md/bcache/btree_update.h
index eea17208c12f..8deb6b187b49 100644
--- a/drivers/md/bcache/btree_update.h
+++ b/drivers/md/bcache/btree_update.h
@@ -80,7 +80,8 @@ struct async_split *__bch_async_split_alloc(struct btree *[], unsigned,
struct btree_iter *);
struct async_split *bch_async_split_alloc(struct btree *, struct btree_iter *);
-void bch_btree_set_root_initial(struct cache_set *, struct btree *);
+void bch_btree_set_root_initial(struct cache_set *, struct btree *,
+ struct btree_reserve *);
void bch_btree_reserve_put(struct cache_set *, struct btree_reserve *);
struct btree_reserve *bch_btree_reserve_get(struct cache_set *c,
@@ -168,13 +169,13 @@ int bch_btree_insert_node(struct btree *, struct btree_iter *,
#define BTREE_INSERT_NOFAIL (1 << 1)
/*
- * Don't fail a btree insert if dirty stale pointers are being added
- *
- * Only for journal replay:
+ * Don't account key being insert (bch_mark_pointers) - only for journal replay,
+ * where we've already marked the new keys:
*/
-#define BTREE_INSERT_NOFAIL_IF_STALE (1 << 2)
+#define BTREE_INSERT_NO_MARK_KEY (1 << 2)
int bch_btree_insert_at(struct btree_iter *, struct keylist *,
+ struct disk_reservation *,
struct btree_insert_hook *, u64 *, unsigned);
struct btree_insert_multi {
@@ -187,6 +188,7 @@ int bch_btree_insert_at_multi(struct btree_insert_multi[], unsigned,
int bch_btree_insert_check_key(struct btree_iter *, struct bkey_i *);
int bch_btree_insert(struct cache_set *, enum btree_id, struct keylist *,
+ struct disk_reservation *,
struct btree_insert_hook *, u64 *, int flags);
int bch_btree_update(struct cache_set *, enum btree_id,
struct bkey_i *, u64 *);
diff --git a/drivers/md/bcache/buckets.c b/drivers/md/bcache/buckets.c
index 6c4ac8b91a60..742f2c3d5b62 100644
--- a/drivers/md/bcache/buckets.c
+++ b/drivers/md/bcache/buckets.c
@@ -69,20 +69,22 @@
#include <trace/events/bcache.h>
+#define bucket_stats_add(_acc, _stats) \
+do { \
+ typeof(_acc) _a = (_acc), _s = (_stats); \
+ unsigned i; \
+ \
+ for (i = 0; i < sizeof(*_a) / sizeof(u64); i++) \
+ ((u64 *) (_a))[i] += ((u64 *) (_s))[i]; \
+} while (0)
+
#define bucket_stats_read_raw(_stats) \
({ \
- typeof(*this_cpu_ptr(_stats)) _acc, *_s; \
- unsigned i; \
+ typeof(*this_cpu_ptr(_stats)) _acc = { 0 }; \
int cpu; \
\
- memset(&_acc, 0, sizeof(_acc)); \
- \
- for_each_possible_cpu(cpu) { \
- _s = per_cpu_ptr((_stats), cpu); \
- \
- for (i = 0; i < sizeof(_acc) / sizeof(u64); i++) \
- ((u64 *) &_acc)[i] += ((u64 *) _s)[i]; \
- } \
+ for_each_possible_cpu(cpu) \
+ bucket_stats_add(&_acc, per_cpu_ptr((_stats), cpu)); \
\
_acc; \
})
@@ -143,23 +145,45 @@ static inline int is_cached_bucket(struct bucket_mark m)
return !m.owned_by_allocator && !m.dirty_sectors && !!m.cached_sectors;
}
+void bch_cache_set_stats_apply(struct cache_set *c,
+ struct bucket_stats_cache_set *stats,
+ struct disk_reservation *disk_res)
+{
+ s64 added = stats->sectors_dirty +
+ stats->sectors_meta +
+ stats->sectors_reserved;
+
+ /*
+ * Not allowed to reduce sectors_available except by getting a
+ * reservation:
+ */
+ BUG_ON(added > (disk_res ? disk_res->sectors : 0));
+
+ if (disk_res && added > 0) {
+ disk_res->sectors -= added;
+ stats->sectors_reserved -= added;
+ }
+
+ lg_local_lock(&c->bucket_stats_lock);
+ bucket_stats_add(this_cpu_ptr(c->bucket_stats_percpu), stats);
+ lg_local_unlock(&c->bucket_stats_lock);
+}
+
static void bucket_stats_update(struct cache *ca,
- struct bucket_mark old,
- struct bucket_mark new,
- bool may_make_unavailable)
+ struct bucket_mark old, struct bucket_mark new,
+ bool may_make_unavailable,
+ struct bucket_stats_cache_set *cache_set_stats)
{
struct cache_set *c = ca->set;
struct bucket_stats_cache *cache_stats;
- struct bucket_stats_cache_set *cache_set_stats;
BUG_ON(!may_make_unavailable &&
is_available_bucket(old) &&
!is_available_bucket(new) &&
c->gc_pos.phase == GC_PHASE_DONE);
- lg_local_lock(&c->bucket_stats_lock);
+ preempt_disable();
cache_stats = this_cpu_ptr(ca->bucket_stats_percpu);
- cache_set_stats = this_cpu_ptr(c->bucket_stats_percpu);
cache_stats->sectors_cached +=
(int) new.cached_sectors - (int) old.cached_sectors;
@@ -188,26 +212,29 @@ static void bucket_stats_update(struct cache *ca,
cache_stats->buckets_meta += is_meta_bucket(new) - is_meta_bucket(old);
cache_stats->buckets_cached += is_cached_bucket(new) - is_cached_bucket(old);
cache_stats->buckets_dirty += is_dirty_bucket(new) - is_dirty_bucket(old);
- lg_local_unlock(&c->bucket_stats_lock);
+ preempt_enable();
if (!is_available_bucket(old) && is_available_bucket(new))
bch_wake_allocator(ca);
}
static struct bucket_mark bch_bucket_mark_set(struct cache *ca,
- struct bucket *g,
- struct bucket_mark new,
- bool may_make_unavailable)
+ struct bucket *g, struct bucket_mark new,
+ bool may_make_unavailable)
{
+ struct bucket_stats_cache_set stats = { 0 };
struct bucket_mark old;
old.counter = xchg(&g->mark.counter, new.counter);
- bucket_stats_update(ca, old, new, may_make_unavailable);
+ bucket_stats_update(ca, old, new, may_make_unavailable, &stats);
+ bch_cache_set_stats_apply(ca->set, &stats, NULL);
return old;
}
-#define bucket_cmpxchg(g, old, new, may_make_unavailable, expr) \
+#define bucket_cmpxchg(g, old, new, \
+ may_make_unavailable, \
+ cache_set_stats, expr) \
do { \
u32 _v = READ_ONCE((g)->mark.counter); \
\
@@ -217,7 +244,9 @@ do { \
} while ((_v = cmpxchg(&(g)->mark.counter, \
old.counter, \
new.counter)) != old.counter); \
- bucket_stats_update(ca, old, new, may_make_unavailable);\
+ bucket_stats_update(ca, old, new, \
+ may_make_unavailable, \
+ cache_set_stats); \
} while (0)
void bch_mark_free_bucket(struct cache *ca, struct bucket *g)
@@ -274,14 +303,16 @@ do { \
static u8 bch_mark_bucket(struct cache_set *c, struct cache *ca,
const struct bch_extent_ptr *ptr, int sectors,
bool dirty, bool metadata, bool is_gc,
- struct gc_pos gc_pos)
+ struct gc_pos gc_pos,
+ struct bucket_stats_cache_set *stats)
{
struct bucket_mark old, new;
unsigned long bucket_nr = PTR_BUCKET_NR(ca, ptr);
unsigned saturated;
u8 stale;
- bucket_cmpxchg(&ca->buckets[bucket_nr], old, new, is_gc, ({
+ bucket_cmpxchg(&ca->buckets[bucket_nr], old, new,
+ is_gc, stats, ({
saturated = 0;
/*
* cmpxchg() only implies a full barrier on success, not
@@ -360,7 +391,8 @@ static u8 bch_mark_bucket(struct cache_set *c, struct cache *ca,
*/
int bch_mark_pointers(struct cache_set *c, struct bkey_s_c_extent e,
int sectors, bool fail_if_stale, bool metadata,
- bool is_gc, struct gc_pos pos)
+ bool is_gc, struct gc_pos pos,
+ struct bucket_stats_cache_set *stats)
{
const struct bch_extent_ptr *ptr, *ptr2;
struct cache *ca;
@@ -407,7 +439,7 @@ int bch_mark_pointers(struct cache_set *c, struct bkey_s_c_extent e,
* Fuck me, I hate my life.
*/
stale = bch_mark_bucket(c, ca, ptr, sectors, dirty,
- metadata, is_gc, pos);
+ metadata, is_gc, pos, stats);
if (stale && dirty && fail_if_stale)
goto stale;
}
@@ -421,7 +453,7 @@ stale:
bch_mark_bucket(c, ca, ptr, -sectors,
bch_extent_ptr_is_dirty(c, e, ptr),
- metadata, is_gc, pos);
+ metadata, is_gc, pos, stats);
}
rcu_read_unlock();
@@ -440,11 +472,14 @@ void bch_mark_reservation(struct cache_set *c, int sectors)
void bch_unmark_open_bucket(struct cache *ca, struct bucket *g)
{
+ struct bucket_stats_cache_set stats = { 0 };
struct bucket_mark old, new;
- bucket_cmpxchg(g, old, new, false, ({
+ bucket_cmpxchg(g, old, new, false, &stats, ({
new.owned_by_allocator = 0;
}));
+
+ bch_cache_set_stats_apply(ca->set, &stats, NULL);
}
static u64 __recalc_sectors_available(struct cache_set *c)
@@ -485,16 +520,6 @@ void bch_disk_reservation_put(struct cache_set *c,
#define SECTORS_CACHE 1024
-/*
- * XXX
- *
- * For the trick we're using here to work, we have to ensure that everything
- * that decreases the amount of space available goes through here and decreases
- * sectors_available:
- *
- * Need to figure out a way of asserting that that's happening (e.g. btree node
- * allocations?)
- */
int __bch_disk_reservation_get(struct cache_set *c,
struct disk_reservation *res,
unsigned sectors,
@@ -568,3 +593,23 @@ int bch_disk_reservation_get(struct cache_set *c,
{
return __bch_disk_reservation_get(c, res, sectors, true, false);
}
+
+void bch_cache_set_stats_set(struct cache_set *c,
+ struct bucket_stats_cache_set *stats)
+{
+ struct bucket_stats_cache_set *s;
+ int cpu;
+
+ lg_global_lock(&c->bucket_stats_lock);
+ for_each_possible_cpu(cpu) {
+ s = per_cpu_ptr(c->bucket_stats_percpu, cpu);
+ memset(s, 0, sizeof(*s));
+ }
+
+ s = this_cpu_ptr(c->bucket_stats_percpu);
+ *s = *stats;
+
+ atomic64_set(&c->sectors_available, 0);
+
+ lg_global_unlock(&c->bucket_stats_lock);
+}
diff --git a/drivers/md/bcache/buckets.h b/drivers/md/bcache/buckets.h
index 561e97cb7d2b..270ab0d599ad 100644
--- a/drivers/md/bcache/buckets.h
+++ b/drivers/md/bcache/buckets.h
@@ -194,15 +194,19 @@ static inline size_t buckets_free_cache(struct cache *ca,
struct bucket_stats_cache_set __bch_bucket_stats_read_cache_set(struct cache_set *);
struct bucket_stats_cache_set bch_bucket_stats_read_cache_set(struct cache_set *);
+void bch_cache_set_stats_apply(struct cache_set *,
+ struct bucket_stats_cache_set *,
+ struct disk_reservation *);
static inline u64 cache_set_sectors_used(struct cache_set *c)
{
- struct bucket_stats_cache_set stats = bch_bucket_stats_read_cache_set(c);
+ struct bucket_stats_cache_set stats = __bch_bucket_stats_read_cache_set(c);
return min(c->capacity,
stats.sectors_meta +
stats.sectors_dirty +
- stats.sectors_reserved);
+ stats.sectors_reserved +
+ (stats.sectors_reserved >> 7));
}
/* XXX: kill? */
@@ -233,19 +237,12 @@ void bch_mark_metadata_bucket(struct cache *, struct bucket *, bool);
void bch_unmark_open_bucket(struct cache *, struct bucket *);
int bch_mark_pointers(struct cache_set *, struct bkey_s_c_extent,
- int, bool, bool, bool, struct gc_pos);
+ int, bool, bool, bool, struct gc_pos,
+ struct bucket_stats_cache_set *);
void bch_mark_reservation(struct cache_set *, int);
void bch_recalc_sectors_available(struct cache_set *);
-/*
- * A reservation for space on disk:
- */
-struct disk_reservation {
- u32 sectors;
- u32 gen;
-};
-
void bch_disk_reservation_put(struct cache_set *,
struct disk_reservation *);
int __bch_disk_reservation_get(struct cache_set *,
@@ -255,4 +252,7 @@ int bch_disk_reservation_get(struct cache_set *,
struct disk_reservation *,
unsigned);
+void bch_cache_set_stats_set(struct cache_set *,
+ struct bucket_stats_cache_set *);
+
#endif /* _BUCKETS_H */
diff --git a/drivers/md/bcache/buckets_types.h b/drivers/md/bcache/buckets_types.h
index 1b7df1662144..48ac31cc04c7 100644
--- a/drivers/md/bcache/buckets_types.h
+++ b/drivers/md/bcache/buckets_types.h
@@ -56,4 +56,12 @@ struct bucket_heap_entry {
unsigned long val;
};
+/*
+ * A reservation for space on disk:
+ */
+struct disk_reservation {
+ u32 sectors;
+ u32 gen;
+};
+
#endif /* _BUCKETS_TYPES_H */
diff --git a/drivers/md/bcache/dirent.c b/drivers/md/bcache/dirent.c
index 4c8b2cdb2d8e..8868e41c8e89 100644
--- a/drivers/md/bcache/dirent.c
+++ b/drivers/md/bcache/dirent.c
@@ -210,7 +210,7 @@ int bch_dirent_create(struct inode *dir, u8 type,
dirent->k.p = k.k->p;
ret = bch_btree_insert_at(&iter, &keylist_single(&dirent->k_i),
- NULL, &ei->journal_seq,
+ NULL, NULL, &ei->journal_seq,
BTREE_INSERT_ATOMIC);
/*
* XXX: if we ever cleanup whiteouts, we may need to rewind
@@ -366,7 +366,7 @@ int bch_dirent_delete(struct inode *dir, const struct qstr *name)
ret = bch_btree_insert_at(&iter,
&keylist_single(&delete),
- NULL, &ei->journal_seq,
+ NULL, NULL, &ei->journal_seq,
BTREE_INSERT_NOFAIL|
BTREE_INSERT_ATOMIC);
/*
diff --git a/drivers/md/bcache/extents.c b/drivers/md/bcache/extents.c
index 06127a117a16..1a0aaf63a17f 100644
--- a/drivers/md/bcache/extents.c
+++ b/drivers/md/bcache/extents.c
@@ -796,7 +796,8 @@ struct btree_nr_keys bch_extent_sort_fix_overlapping(struct btree_keys *b,
}
static int bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k,
- u64 offset, s64 sectors, bool fail_if_stale)
+ u64 offset, s64 sectors, bool fail_if_stale,
+ struct bucket_stats_cache_set *stats)
{
struct cache_set *c = iter->c;
struct btree *b = iter->nodes[0];
@@ -814,7 +815,7 @@ static int bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k,
ret = bch_mark_pointers(c, e, sectors, fail_if_stale,
false, false,
- gc_pos_btree_node(b));
+ gc_pos_btree_node(b), stats);
if (ret)
return ret;
@@ -822,40 +823,47 @@ static int bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k,
bcache_dev_sectors_dirty_add(c, e.k->p.inode,
offset, sectors);
} else if (k.k->type == BCH_RESERVATION) {
- bch_mark_reservation(c, sectors);
+ stats->sectors_reserved += sectors;
}
return 0;
}
static void bch_subtract_sectors(struct btree_iter *iter, struct bkey_s_c k,
- u64 offset, s64 sectors)
+ u64 offset, s64 sectors,
+ struct bucket_stats_cache_set *stats)
{
- bch_add_sectors(iter, k, offset, -sectors, false);
+ bch_add_sectors(iter, k, offset, -sectors, false, stats);
}
/* These wrappers subtract exactly the sectors that we're removing from @k */
static void bch_cut_subtract_back(struct btree_iter *iter,
- struct bpos where, struct bkey_s k)
+ struct bpos where, struct bkey_s k,
+ struct bucket_stats_cache_set *stats)
{
bch_subtract_sectors(iter, k.s_c, where.offset,
- k.k->p.offset - where.offset);
+ k.k->p.offset - where.offset,
+ stats);
bch_cut_back(where, k.k);
}
static void bch_cut_subtract_front(struct btree_iter *iter,
- struct bpos where, struct bkey_s k)
+ struct bpos where, struct bkey_s k,
+ struct bucket_stats_cache_set *stats)
{
bch_subtract_sectors(iter, k.s_c, bkey_start_offset(k.k),
- where.offset - bkey_start_offset(k.k));
+ where.offset - bkey_start_offset(k.k),
+ stats);
__bch_cut_front(where, k);
}
-static void bch_drop_subtract(struct btree_iter *iter, struct bkey_s k)
+static void bch_drop_subtract(struct btree_iter *iter, struct bkey_s k,
+ struct bucket_stats_cache_set *stats)
{
if (k.k->size)
bch_subtract_sectors(iter, k.s_c,
- bkey_start_offset(k.k), k.k->size);
+ bkey_start_offset(k.k), k.k->size,
+ stats);
k.k->size = 0;
__set_bkey_deleted(k.k);
}
@@ -948,7 +956,8 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook,
struct btree_iter *iter,
struct bkey_s_c k,
struct bkey_i *new,
- struct journal_res *res)
+ struct journal_res *res,
+ struct bucket_stats_cache_set *stats)
{
struct bch_replace_info *replace = container_of(hook,
struct bch_replace_info, hook);
@@ -965,7 +974,8 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook,
bkey_start_pos(&old->k)) < 0);
if (!k.k) {
- bch_cut_subtract_back(iter, iter->pos, bkey_i_to_s(new));
+ bch_cut_subtract_back(iter, iter->pos,
+ bkey_i_to_s(new), stats);
return;
}
@@ -993,7 +1003,7 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook,
}
bch_cut_subtract_front(iter, bkey_start_pos(k.k),
- bkey_i_to_s(new));
+ bkey_i_to_s(new), stats);
/* advance @iter->pos from the end of prev key to the start of @k */
bch_btree_iter_set_pos(iter, bkey_start_pos(k.k));
}
@@ -1019,9 +1029,10 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook,
/* update @new to be the part we haven't checked yet */
if (bkey_cmp(k.k->p, new->k.p) > 0)
- bch_drop_subtract(iter, bkey_i_to_s(new));
+ bch_drop_subtract(iter, bkey_i_to_s(new), stats);
else
- bch_cut_subtract_front(iter, k.k->p, bkey_i_to_s(new));
+ bch_cut_subtract_front(iter, k.k->p,
+ bkey_i_to_s(new), stats);
} else
replace->successes += 1;
}
@@ -1120,6 +1131,7 @@ static void handle_existing_key_newer(struct btree_iter *iter,
*/
void bch_insert_fixup_extent(struct btree_iter *iter,
struct bkey_i *insert,
+ struct disk_reservation *disk_res,
struct btree_insert_hook *hook,
struct journal_res *res,
unsigned flags)
@@ -1133,6 +1145,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
struct bkey_s k;
struct bpos next_pos;
BKEY_PADDED(k) split;
+ struct bucket_stats_cache_set stats = { 0 };
unsigned nr_done = 0;
u64 start_time = local_clock();
@@ -1164,14 +1177,16 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
* can also insert keys with stale pointers, but for those we still need
* to proceed with the insertion.
*/
- if (bch_add_sectors(iter, bkey_i_to_s_c(insert),
+ if (!(flags & BTREE_INSERT_NO_MARK_KEY) &&
+ bch_add_sectors(iter, bkey_i_to_s_c(insert),
bkey_start_offset(&insert->k),
insert->k.size,
- !(flags & BTREE_INSERT_NOFAIL_IF_STALE))) {
+ true,
+ &stats)) {
/* We raced - a dirty pointer was stale */
bch_btree_iter_set_pos(iter, insert->k.p);
insert->k.size = 0;
- return;
+ goto apply_stats;
}
while (insert->k.size &&
@@ -1210,9 +1225,10 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
* XXX: would be better to explicitly signal that we
* need to split
*/
- bch_cut_subtract_back(iter, iter->pos, bkey_i_to_s(insert));
+ bch_cut_subtract_back(iter, iter->pos,
+ bkey_i_to_s(insert), &stats);
if (!insert->k.size)
- return;
+ goto apply_stats;
break;
}
#if 0
@@ -1245,7 +1261,8 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
? k.k->p : insert->k.p;
if (hook)
- hook->fn(hook, iter, k.s_c, insert, res);
+ hook->fn(hook, iter, k.s_c, insert,
+ res, &stats);
/*
* Don't update iter->pos until after calling the hook,
@@ -1254,7 +1271,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
bch_btree_iter_set_pos(iter, next_pos);
if (!insert->k.size)
- return;
+ goto apply_stats;
/* insert and k might not overlap after calling hook fn: */
if (bkey_cmp(insert->k.p, bkey_start_pos(k.k)) <= 0 ||
@@ -1267,14 +1284,15 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
switch (bch_extent_overlap(&insert->k, k.k)) {
case BCH_EXTENT_OVERLAP_FRONT:
/* insert and k share the start, invalidate in k */
- bch_cut_subtract_front(iter, insert->k.p, k);
+ bch_cut_subtract_front(iter, insert->k.p, k, &stats);
extent_save(_k, k.k, f);
break;
case BCH_EXTENT_OVERLAP_BACK:
/* insert and k share the end, invalidate in k */
bch_cut_subtract_back(iter,
- bkey_start_pos(&insert->k), k);
+ bkey_start_pos(&insert->k),
+ k, &stats);
extent_save(_k, k.k, f);
/*
@@ -1291,7 +1309,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
if (!bkey_deleted(_k))
btree_keys_account_key_drop(&b->keys.nr, _k);
- bch_drop_subtract(iter, k);
+ bch_drop_subtract(iter, k, &stats);
k.k->p = bkey_start_pos(&insert->k);
extent_save(_k, k.k, f);
@@ -1318,7 +1336,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
bch_cut_back(bkey_start_pos(&insert->k), &split.k.k);
__bch_cut_front(bkey_start_pos(&insert->k), k);
- bch_cut_subtract_front(iter, insert->k.p, k);
+ bch_cut_subtract_front(iter, insert->k.p, k, &stats);
extent_save(_k, k.k, f);
bch_btree_bset_insert(iter, b, node_iter, &split.k);
@@ -1329,14 +1347,16 @@ void bch_insert_fixup_extent(struct btree_iter *iter,
next_pos = insert->k.p;
if (hook)
- hook->fn(hook, iter, bkey_s_c_null, insert, res);
+ hook->fn(hook, iter, bkey_s_c_null, insert, res, &stats);
bch_btree_iter_set_pos(iter, next_pos);
if (!insert->k.size)
- return;
+ goto apply_stats;
bch_btree_insert_and_journal(iter, insert, res);
+apply_stats:
+ bch_cache_set_stats_apply(c, &stats, disk_res);
}
static const char *bch_extent_invalid(const struct cache_set *c,
diff --git a/drivers/md/bcache/extents.h b/drivers/md/bcache/extents.h
index 90f7f8ffa849..42db3d1ca933 100644
--- a/drivers/md/bcache/extents.h
+++ b/drivers/md/bcache/extents.h
@@ -51,9 +51,11 @@ void bch_extent_cmpxchg(struct btree_insert_hook *,
struct btree_iter *,
struct bkey_s_c,
struct bkey_i *,
- struct journal_res *);
+ struct journal_res *,
+ struct bucket_stats_cache_set *);
void bch_insert_fixup_extent(struct btree_iter *, struct bkey_i *,
+ struct disk_reservation *,
struct btree_insert_hook *,
struct journal_res *, unsigned);
diff --git a/drivers/md/bcache/fs-gc.c b/drivers/md/bcache/fs-gc.c
index 772ef7bb59f4..264744850422 100644
--- a/drivers/md/bcache/fs-gc.c
+++ b/drivers/md/bcache/fs-gc.c
@@ -174,7 +174,7 @@ static int bch_gc_do_inode(struct cache_set *c, struct btree_iter *iter,
return bch_btree_insert_at(iter,
&keylist_single(&update.k_i),
- NULL, NULL,
+ NULL, NULL, NULL,
BTREE_INSERT_NOFAIL);
}
diff --git a/drivers/md/bcache/fs-io.c b/drivers/md/bcache/fs-io.c
index 127f608aabf0..33c059c253e4 100644
--- a/drivers/md/bcache/fs-io.c
+++ b/drivers/md/bcache/fs-io.c
@@ -199,7 +199,8 @@ static void i_sectors_hook_fn(struct btree_insert_hook *hook,
struct btree_iter *iter,
struct bkey_s_c k,
struct bkey_i *insert,
- struct journal_res *res)
+ struct journal_res *res,
+ struct bucket_stats_cache_set *stats)
{
struct i_sectors_hook *h = container_of(hook,
struct i_sectors_hook, hook);
@@ -440,7 +441,8 @@ static void bch_put_page_reservation(struct cache_set *c, struct page *page)
}
}
-static int bch_get_page_reservation(struct cache_set *c, struct page *page)
+static int bch_get_page_reservation(struct cache_set *c, struct page *page,
+ bool check_enospc)
{
struct bch_page_state *s = page_state(page), old, new;
struct disk_reservation res;
@@ -449,7 +451,8 @@ static int bch_get_page_reservation(struct cache_set *c, struct page *page)
if (s->alloc_state != BCH_PAGE_UNALLOCATED)
return 0;
- ret = bch_disk_reservation_get(c, &res, PAGE_SECTORS);
+ ret = __bch_disk_reservation_get(c, &res, PAGE_SECTORS,
+ check_enospc, false);
if (ret)
return ret;
@@ -801,12 +804,6 @@ static void bch_writepage_io_done(struct closure *cl)
struct bio_vec *bvec;
unsigned i;
- if (io->sectors_reserved) {
- struct disk_reservation res = { .sectors = io->sectors_reserved };
-
- bch_disk_reservation_put(c, &res);
- }
-
for (i = 0; i < ARRAY_SIZE(io->i_size_update_count); i++)
i_size_update_put(c, ei, i, io->i_size_update_count[i]);
@@ -880,7 +877,6 @@ alloc_io:
w->io->ei = ei;
memset(w->io->i_size_update_count, 0,
sizeof(w->io->i_size_update_count));
- w->io->sectors_reserved = 0;
ret = i_sectors_dirty_get(ei, &w->io->i_sectors_hook);
/*
@@ -891,7 +887,8 @@ alloc_io:
*/
BUG_ON(ret);
- bch_write_op_init(&w->io->op, w->c, &w->io->bio, NULL,
+ bch_write_op_init(&w->io->op, w->c, &w->io->bio,
+ (struct disk_reservation) { 0 }, NULL,
bkey_to_s_c(&KEY(w->inum, 0, 0)),
&w->io->i_sectors_hook.hook,
&ei->journal_seq, 0);
@@ -980,7 +977,7 @@ do_io:
BUG_ON(old.alloc_state == BCH_PAGE_UNALLOCATED);
if (old.alloc_state == BCH_PAGE_RESERVED)
- w->io->sectors_reserved += PAGE_SECTORS;
+ w->io->op.res.sectors += PAGE_SECTORS;
BUG_ON(PageWriteback(page));
set_page_writeback(page);
@@ -1115,7 +1112,7 @@ readpage:
if (ret)
goto err;
out:
- ret = bch_get_page_reservation(c, page);
+ ret = bch_get_page_reservation(c, page, true);
if (ret) {
if (!PageUptodate(page)) {
/*
@@ -1386,13 +1383,19 @@ static void bch_do_direct_IO_write(struct dio_write *dio, bool sync)
break;
}
- bch_write_op_init(&dio->iop, c, &dio->bio, NULL,
+ bch_write_op_init(&dio->iop, c, &dio->bio,
+ (struct disk_reservation) {
+ .sectors = bio_sectors(bio),
+ .gen = dio->res.gen
+ }, NULL,
bkey_to_s_c(&KEY(inode->i_ino,
bio_end_sector(bio),
bio_sectors(bio))),
&dio->i_sectors_hook.hook,
&ei->journal_seq, flags);
+ dio->res.sectors -= bio_sectors(bio);
+
task_io_account_write(bio->bi_iter.bi_size);
closure_call(&dio->iop.cl, bch_write, NULL, &dio->cl);
@@ -1709,7 +1712,7 @@ int bch_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
goto out;
}
- if (bch_get_page_reservation(c, page)) {
+ if (bch_get_page_reservation(c, page, true)) {
unlock_page(page);
ret = VM_FAULT_SIGBUS;
goto out;
@@ -1833,7 +1836,6 @@ static int __bch_truncate_page(struct address_space *mapping,
struct cache_set *c = inode->i_sb->s_fs_info;
unsigned start_offset = start & (PAGE_SIZE - 1);
unsigned end_offset = ((end - 1) & (PAGE_SIZE - 1)) + 1;
- struct bch_page_state new;
struct page *page;
int ret = 0;
@@ -1885,20 +1887,14 @@ create:
goto unlock;
}
-#if 0
/*
- * XXX: this is a hack, because we don't want truncate to fail due to
- * -ENOSPC
+ * Bit of a hack - we don't want truncate to fail due to -ENOSPC.
*
- * Note that because we aren't currently tracking whether the page has
- * actual data in it (vs. just 0s, or only partially written) this is
- * also wrong. ick.
+ * XXX: because we aren't currently tracking whether the page has actual
+ * data in it (vs. just 0s, or only partially written) this wrong. ick.
*/
-#endif
- page_state_cmpxchg(page_state(page), new, {
- if (new.alloc_state == BCH_PAGE_UNALLOCATED)
- new.alloc_state = BCH_PAGE_ALLOCATED;
- });
+ ret = bch_get_page_reservation(c, page, false);
+ BUG_ON(ret);
if (index == start >> PAGE_SHIFT &&
index == end >> PAGE_SHIFT)
@@ -2139,6 +2135,8 @@ static long bch_fcollapse(struct inode *inode, loff_t offset, loff_t len)
while (bkey_cmp(dst.pos,
POS(inode->i_ino,
round_up(new_size, PAGE_SIZE) >> 9)) < 0) {
+ struct disk_reservation disk_res;
+
bch_btree_iter_set_pos(&src,
POS(dst.pos.inode, dst.pos.offset + (len >> 9)));
@@ -2163,12 +2161,20 @@ static long bch_fcollapse(struct inode *inode, loff_t offset, loff_t len)
BUG_ON(bkey_cmp(dst.pos, bkey_start_pos(&copy.k.k)));
+ ret = __bch_disk_reservation_get(c, &disk_res,
+ copy.k.k.size,
+ false, false);
+ BUG_ON(ret);
+
ret = bch_btree_insert_at(&dst,
&keylist_single(&copy.k),
+ &disk_res,
&i_sectors_hook.hook,
&ei->journal_seq,
BTREE_INSERT_ATOMIC|
BTREE_INSERT_NOFAIL);
+ bch_disk_reservation_put(c, &disk_res);
+
if (ret < 0 && ret != -EINTR)
goto err_unwind;
@@ -2281,8 +2287,7 @@ static long bch_fallocate(struct inode *inode, int mode,
goto err;
while (bkey_cmp(iter.pos, end) < 0) {
- struct disk_reservation disk_res;
- unsigned flags = 0;
+ struct disk_reservation disk_res = { 0 };
k = bch_btree_iter_peek_with_holes(&iter);
if (!k.k) {
@@ -2301,9 +2306,6 @@ static long bch_fallocate(struct inode *inode, int mode,
bch_btree_iter_advance_pos(&iter);
continue;
}
-
- /* don't check for -ENOSPC if we're deleting data: */
- flags |= BTREE_INSERT_NOFAIL;
}
bkey_init(&reservation.k);
@@ -2316,16 +2318,19 @@ static long bch_fallocate(struct inode *inode, int mode,
sectors = reservation.k.size;
- ret = bch_disk_reservation_get(c, &disk_res, sectors);
- if (ret)
- goto err_put_sectors_dirty;
+ if (!bkey_extent_is_allocation(k.k)) {
+ ret = bch_disk_reservation_get(c, &disk_res, sectors);
+ if (ret)
+ goto err_put_sectors_dirty;
+ }
ret = bch_btree_insert_at(&iter,
&keylist_single(&reservation),
+ &disk_res,
&i_sectors_hook.hook,
&ei->journal_seq,
- BTREE_INSERT_ATOMIC|flags);
-
+ BTREE_INSERT_ATOMIC|
+ BTREE_INSERT_NOFAIL);
bch_disk_reservation_put(c, &disk_res);
if (ret < 0 && ret != -EINTR)
diff --git a/drivers/md/bcache/fs-io.h b/drivers/md/bcache/fs-io.h
index 23d443222c08..c5681dd5cde4 100644
--- a/drivers/md/bcache/fs-io.h
+++ b/drivers/md/bcache/fs-io.h
@@ -48,7 +48,6 @@ struct bch_writepage_io {
struct bch_inode_info *ei;
unsigned long i_size_update_count[I_SIZE_UPDATE_ENTRIES];
- unsigned long sectors_reserved;
struct bch_write_op op;
struct i_sectors_hook i_sectors_hook;
diff --git a/drivers/md/bcache/fs.c b/drivers/md/bcache/fs.c
index 4a6cb2b51143..b9919627245c 100644
--- a/drivers/md/bcache/fs.c
+++ b/drivers/md/bcache/fs.c
@@ -134,7 +134,7 @@ int __must_check __bch_write_inode(struct cache_set *c,
ret = bch_btree_insert_at(&iter,
&keylist_single(&new_inode.k_i),
- NULL, &ei->journal_seq,
+ NULL, NULL, &ei->journal_seq,
BTREE_INSERT_ATOMIC|
BTREE_INSERT_NOFAIL);
} while (ret == -EINTR);
diff --git a/drivers/md/bcache/inode.c b/drivers/md/bcache/inode.c
index c9469fd115c8..58e853c03360 100644
--- a/drivers/md/bcache/inode.c
+++ b/drivers/md/bcache/inode.c
@@ -135,7 +135,7 @@ again:
inode->k.p.inode, inode->k.u64s);
ret = bch_btree_insert_at(&iter, &keylist_single(inode),
- NULL, NULL,
+ NULL, NULL, NULL,
BTREE_INSERT_ATOMIC);
if (ret == -EINTR)
@@ -206,7 +206,8 @@ int bch_inode_rm(struct cache_set *c, u64 inode_nr)
return bch_btree_insert(c, BTREE_ID_INODES,
&keylist_single(&delete),
- NULL, NULL, BTREE_INSERT_NOFAIL);
+ NULL, NULL, NULL,
+ BTREE_INSERT_NOFAIL);
}
int bch_inode_update(struct cache_set *c, struct bkey_i *inode,
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index eb6df9d9f86c..6b240f4446c1 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -566,6 +566,7 @@ static void bch_write_done(struct closure *cl)
if (!op->error && (op->flags & BCH_WRITE_FLUSH))
op->error = bch_journal_error(&op->c->journal);
+ bch_disk_reservation_put(op->c, &op->res);
percpu_ref_put(&op->c->writes);
bch_keylist_free(&op->insert_keys);
closure_return(cl);
@@ -592,9 +593,12 @@ static void bch_write_index(struct closure *cl)
u64 sectors_start = keylist_sectors(&op->insert_keys);
int ret;
- ret = bch_btree_insert(op->c, BTREE_ID_EXTENTS, &op->insert_keys,
+ ret = bch_btree_insert(op->c, BTREE_ID_EXTENTS,
+ &op->insert_keys,
+ &op->res,
op->insert_hook,
- op_journal_seq(op), BTREE_INSERT_NOFAIL);
+ op_journal_seq(op),
+ BTREE_INSERT_NOFAIL);
op->written += sectors_start - keylist_sectors(&op->insert_keys);
@@ -1132,12 +1136,14 @@ void bch_write(struct closure *cl)
if (!bio_sectors(bio)) {
WARN_ONCE(1, "bch_write() called with empty bio");
+ bch_disk_reservation_put(op->c, &op->res);
closure_return(cl);
}
if (!percpu_ref_tryget(&c->writes)) {
__bcache_io_error(c, "read only");
op->error = -EROFS;
+ bch_disk_reservation_put(op->c, &op->res);
closure_return(cl);
}
@@ -1206,8 +1212,8 @@ void bch_write(struct closure *cl)
}
void bch_write_op_init(struct bch_write_op *op, struct cache_set *c,
- struct bch_write_bio *bio, struct write_point *wp,
- struct bkey_s_c insert_key,
+ struct bch_write_bio *bio, struct disk_reservation res,
+ struct write_point *wp, struct bkey_s_c insert_key,
struct btree_insert_hook *insert_hook,
u64 *journal_seq, unsigned flags)
{
@@ -1227,6 +1233,7 @@ void bch_write_op_init(struct bch_write_op *op, struct cache_set *c,
op->flags = flags;
op->compression_type = c->opts.compression;
op->nr_replicas = c->opts.data_replicas;
+ op->res = res;
op->wp = wp;
if (journal_seq) {
@@ -1721,6 +1728,7 @@ void bch_read_extent_iter(struct cache_set *c, struct bio *orig,
bch_write_op_init(&promote_op->iop, c,
&promote_op->bio,
+ (struct disk_reservation) { 0 },
&c->promote_write_point,
k, NULL, NULL,
BCH_WRITE_ALLOC_NOWAIT);
diff --git a/drivers/md/bcache/io.h b/drivers/md/bcache/io.h
index 8fb4c2b36acd..147ccd969586 100644
--- a/drivers/md/bcache/io.h
+++ b/drivers/md/bcache/io.h
@@ -29,7 +29,8 @@ enum bch_write_flags {
};
void bch_write_op_init(struct bch_write_op *, struct cache_set *,
- struct bch_write_bio *, struct write_point *,
+ struct bch_write_bio *,
+ struct disk_reservation, struct write_point *,
struct bkey_s_c,
struct btree_insert_hook *, u64 *, unsigned);
void bch_write(struct closure *);
diff --git a/drivers/md/bcache/io_types.h b/drivers/md/bcache/io_types.h
index 064e2c7d8cdc..f0a58401295a 100644
--- a/drivers/md/bcache/io_types.h
+++ b/drivers/md/bcache/io_types.h
@@ -2,6 +2,7 @@
#define _BCACHE_IO_TYPES_H
#include "btree_types.h"
+#include "buckets_types.h"
#include "keylist_types.h"
#include <linux/llist.h>
@@ -82,6 +83,8 @@ struct bch_write_op {
unsigned compression_type:4;
unsigned nr_replicas:4;
+ struct disk_reservation res;
+
struct write_point *wp;
union {
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index a877bb1e169c..742ccf7ad3f3 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -777,7 +777,8 @@ const char *bch_journal_read(struct cache_set *c, struct list_head *list)
return NULL;
}
-void bch_journal_mark(struct cache_set *c, struct list_head *list)
+void bch_journal_mark(struct cache_set *c, struct list_head *list,
+ struct bucket_stats_cache_set *stats)
{
struct bkey_i *k, *n;
struct jset_entry *j;
@@ -790,7 +791,8 @@ void bch_journal_mark(struct cache_set *c, struct list_head *list)
if (btree_type_has_ptrs(type) &&
!bkey_invalid(c, type, bkey_i_to_s_c(k)))
__bch_btree_mark_key_initial(c, type,
- bkey_i_to_s_c(k));
+ bkey_i_to_s_c(k),
+ stats);
}
}
@@ -955,36 +957,6 @@ void bch_journal_start(struct cache_set *c)
queue_work(system_long_wq, &j->reclaim_work);
}
-static int bch_journal_replay_key(struct cache_set *c, enum btree_id id,
- struct bkey_i *k)
-{
- int ret;
- BKEY_PADDED(key) temp;
- bool do_subtract = id == BTREE_ID_EXTENTS && bkey_extent_is_data(&k->k);
-
- trace_bcache_journal_replay_key(&k->k);
-
- if (do_subtract)
- bkey_copy(&temp.key, k);
-
- ret = bch_btree_insert(c, id, &keylist_single(k), NULL, NULL,
- BTREE_INSERT_NOFAIL|
- BTREE_INSERT_NOFAIL_IF_STALE);
- if (ret)
- return ret;
-
- /*
- * Subtract sectors after replay since bch_btree_insert() added
- * them again
- */
- if (do_subtract)
- bch_mark_pointers(c, bkey_i_to_s_c_extent(&temp.key),
- -temp.key.k.size, false, false,
- true, GC_POS_MIN);
-
- return 0;
-}
-
int bch_journal_replay(struct cache_set *c, struct list_head *list)
{
int ret = 0, keys = 0, entries = 0;
@@ -1025,11 +997,17 @@ int bch_journal_replay(struct cache_set *c, struct list_head *list)
BUG_ON(atomic_read(&j->cur_pin_list->count) != 1);
for_each_jset_key(k, _n, jkeys, &i->j) {
- cond_resched();
- ret = bch_journal_replay_key(c, jkeys->btree_id, k);
+ trace_bcache_journal_replay_key(&k->k);
+
+ ret = bch_btree_insert(c, jkeys->btree_id,
+ &keylist_single(k),
+ NULL, NULL, NULL,
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_NO_MARK_KEY);
if (ret)
goto err;
+ cond_resched();
keys++;
}
diff --git a/drivers/md/bcache/journal.h b/drivers/md/bcache/journal.h
index 180019a571e0..ec9acbc8f8cc 100644
--- a/drivers/md/bcache/journal.h
+++ b/drivers/md/bcache/journal.h
@@ -231,7 +231,8 @@ static inline bool journal_res_full(struct journal_res *res,
}
void bch_journal_start(struct cache_set *);
-void bch_journal_mark(struct cache_set *, struct list_head *);
+void bch_journal_mark(struct cache_set *, struct list_head *,
+ struct bucket_stats_cache_set *);
const char *bch_journal_read(struct cache_set *, struct list_head *);
int bch_journal_replay(struct cache_set *, struct list_head *);
diff --git a/drivers/md/bcache/migrate.c b/drivers/md/bcache/migrate.c
index 31dfb5f4dcde..39e212a1c3b5 100644
--- a/drivers/md/bcache/migrate.c
+++ b/drivers/md/bcache/migrate.c
@@ -50,13 +50,11 @@ static int issue_migration_move(struct cache *ca,
return -ENOMEM;
}
- io->disk_res = res;
-
/* This also copies k into the write op's replace_key and insert_key */
bch_replace_init(&io->replace, k);
- bch_write_op_init(&io->op, c, &io->bio,
+ bch_write_op_init(&io->op, c, &io->bio, res,
&c->migration_write_point,
k, &io->replace.hook, NULL,
0);
@@ -385,7 +383,8 @@ static int bch_flag_key_bad(struct btree_iter *iter,
bch_extent_normalize(c, e.s);
return bch_btree_insert_at(iter, &keylist_single(&tmp.key),
- NULL, NULL, BTREE_INSERT_ATOMIC);
+ NULL, NULL, NULL,
+ BTREE_INSERT_ATOMIC);
}
/*
diff --git a/drivers/md/bcache/move.c b/drivers/md/bcache/move.c
index f7dfcbd5935b..3a20a03892e8 100644
--- a/drivers/md/bcache/move.c
+++ b/drivers/md/bcache/move.c
@@ -115,8 +115,6 @@ void moving_io_free(struct moving_io *io)
struct bio_vec *bv;
int i;
- bch_disk_reservation_put(io->op.c, &io->disk_res);
-
bio_for_each_segment_all(bv, &io->bio.bio.bio, i)
if (bv->bv_page)
__free_page(bv->bv_page);
diff --git a/drivers/md/bcache/move.h b/drivers/md/bcache/move.h
index fe545a6aec9b..8b311a8b3676 100644
--- a/drivers/md/bcache/move.h
+++ b/drivers/md/bcache/move.h
@@ -92,7 +92,6 @@ struct moving_io {
unsigned read_issued:1;
unsigned read_completed:1;
unsigned write_issued:1;
- struct disk_reservation disk_res;
/* Must be last since it is variable size */
struct bch_write_bio bio;
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index 475cfa392a70..f1a1aa3e49ea 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -60,8 +60,9 @@ static int issue_moving_gc_move(struct moving_queue *q,
bch_replace_init(&io->replace, bkey_i_to_s_c(k));
- bch_write_op_init(&io->op, c, &io->bio, NULL,
- bkey_i_to_s_c(k),
+ bch_write_op_init(&io->op, c, &io->bio,
+ (struct disk_reservation) { 0 },
+ NULL, bkey_i_to_s_c(k),
&io->replace.hook, NULL,
bkey_extent_is_cached(&k->k)
? BCH_WRITE_CACHED : 0);
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 6f366e29f910..10732c4a8f52 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -556,7 +556,8 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
if (bypass)
flags |= BCH_WRITE_DISCARD;
- bch_write_op_init(&s->iop, dc->disk.c, &s->bio, NULL,
+ bch_write_op_init(&s->iop, dc->disk.c, &s->bio,
+ (struct disk_reservation) { 0 }, NULL,
bkey_to_s_c(&insert_key),
NULL, NULL, flags);
@@ -681,8 +682,16 @@ static void __blockdev_volume_make_request(struct request_queue *q,
continue_at(&s->cl, search_free, NULL);
} else if (rw) {
+ struct disk_reservation res = { 0 };
unsigned flags = 0;
+ if (bio_op(bio) != REQ_OP_DISCARD &&
+ bch_disk_reservation_get(d->c, &res, bio_sectors(bio))) {
+ s->iop.error = -ENOSPC;
+ continue_at(&s->cl, search_free, NULL);
+ return;
+ }
+
if (bio->bi_opf & (REQ_PREFLUSH|REQ_FUA))
flags |= BCH_WRITE_FLUSH;
if (bio_op(bio) == REQ_OP_DISCARD)
@@ -690,7 +699,7 @@ static void __blockdev_volume_make_request(struct request_queue *q,
s = search_alloc(bio, d);
- bch_write_op_init(&s->iop, d->c, &s->bio, NULL,
+ bch_write_op_init(&s->iop, d->c, &s->bio, res, NULL,
bkey_to_s_c(&KEY(s->inode, 0, 0)),
NULL, NULL, flags);
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 724ca30c9d0a..c44f44c44712 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -1376,7 +1376,7 @@ static const char *run_cache_set(struct cache_set *c)
err = "error creating root directory";
if (bch_btree_insert(c, BTREE_ID_INODES,
&keylist_single(&inode.k_i),
- NULL, NULL, 0))
+ NULL, NULL, NULL, 0))
goto err;
err = "error writing first journal entry";
diff --git a/drivers/md/bcache/tier.c b/drivers/md/bcache/tier.c
index 29b7d4db320a..20cbbf9295ed 100644
--- a/drivers/md/bcache/tier.c
+++ b/drivers/md/bcache/tier.c
@@ -223,6 +223,7 @@ static int issue_tiering_move(struct moving_queue *q,
bch_replace_init(&io->replace, bkey_i_to_s_c(&io->key));
bch_write_op_init(&io->op, c, &io->bio,
+ (struct disk_reservation) { 0 },
&ca->tiering_write_point,
bkey_i_to_s_c(&io->key),
&io->replace.hook, NULL, 0);
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index 1a0424961a40..53ceb56a6a38 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -120,7 +120,7 @@ static void write_dirty_finish(struct closure *cl)
ret = bch_btree_insert(dc->disk.c, BTREE_ID_EXTENTS,
&keylist_single(&tmp.k),
- &io->replace.hook, NULL, 0);
+ NULL, &io->replace.hook, NULL, 0);
if (io->replace.successes == 0)
trace_bcache_writeback_collision(&io->replace.key.k);
diff --git a/drivers/md/bcache/xattr.c b/drivers/md/bcache/xattr.c
index dca982563c93..297ef230c4cc 100644
--- a/drivers/md/bcache/xattr.c
+++ b/drivers/md/bcache/xattr.c
@@ -225,7 +225,7 @@ int bch_xattr_set(struct inode *inode, const char *name,
bch_keylist_enqueue(&keys);
- ret = bch_btree_insert_at(&iter, &keys, NULL,
+ ret = bch_btree_insert_at(&iter, &keys, NULL, NULL,
&ei->journal_seq,
insert_flags);
bch_keylist_free(&keys);
@@ -238,7 +238,7 @@ int bch_xattr_set(struct inode *inode, const char *name,
ret = bch_btree_insert_at(&iter,
&keylist_single(&whiteout),
- NULL, &ei->journal_seq,
+ NULL, NULL, &ei->journal_seq,
insert_flags);
}