diff options
31 files changed, 417 insertions, 265 deletions
diff --git a/drivers/md/bcache/btree_gc.c b/drivers/md/bcache/btree_gc.c index a62d7778f4df..724024c0c06e 100644 --- a/drivers/md/bcache/btree_gc.c +++ b/drivers/md/bcache/btree_gc.c @@ -122,7 +122,8 @@ u8 bch_btree_key_recalc_oldest_gen(struct cache_set *c, struct bkey_s_c k) * For runtime mark and sweep: */ static u8 __bch_btree_mark_key(struct cache_set *c, enum bkey_type type, - struct bkey_s_c k) + struct bkey_s_c k, + struct bucket_stats_cache_set *stats) { switch (type) { case BKEY_TYPE_BTREE: @@ -135,7 +136,7 @@ static u8 __bch_btree_mark_key(struct cache_set *c, enum bkey_type type, ? c->sb.btree_node_size : e.k->size, false, type == BKEY_TYPE_BTREE, - true, GC_POS_MIN); + true, GC_POS_MIN, stats); } return bch_btree_key_recalc_oldest_gen(c, k); @@ -145,25 +146,27 @@ static u8 __bch_btree_mark_key(struct cache_set *c, enum bkey_type type, } static u8 btree_mark_key(struct cache_set *c, struct btree *b, - struct bkey_s_c k) + struct bkey_s_c k, + struct bucket_stats_cache_set *stats) { - return __bch_btree_mark_key(c, btree_node_type(b), k); + return __bch_btree_mark_key(c, btree_node_type(b), k, stats); } /* * For initial cache set bringup: */ u8 __bch_btree_mark_key_initial(struct cache_set *c, enum bkey_type type, - struct bkey_s_c k) + struct bkey_s_c k, + struct bucket_stats_cache_set *stats) { switch (type) { case BKEY_TYPE_BTREE: case BKEY_TYPE_EXTENTS: if (k.k->type == BCH_RESERVATION) - bch_mark_reservation(c, k.k->size); + stats->sectors_reserved += k.k->size; - return __bch_btree_mark_key(c, type, k); + return __bch_btree_mark_key(c, type, k, stats); default: BUG(); } @@ -171,12 +174,14 @@ u8 __bch_btree_mark_key_initial(struct cache_set *c, enum bkey_type type, } static u8 btree_mark_key_initial(struct cache_set *c, struct btree *b, - struct bkey_s_c k) + struct bkey_s_c k, + struct bucket_stats_cache_set *stats) { - return __bch_btree_mark_key_initial(c, btree_node_type(b), k); + return __bch_btree_mark_key_initial(c, btree_node_type(b), k, stats); } -static bool btree_gc_mark_node(struct cache_set *c, struct btree *b) +static bool btree_gc_mark_node(struct cache_set *c, struct btree *b, + struct bucket_stats_cache_set *stats) { if (btree_node_has_ptrs(b)) { struct btree_node_iter iter; @@ -187,7 +192,7 @@ static bool btree_gc_mark_node(struct cache_set *c, struct btree *b) for_each_btree_node_key_unpack(&b->keys, k, &iter, &unpacked) { bkey_debugcheck(c, b, k); - stale = max(stale, btree_mark_key(c, b, k)); + stale = max(stale, btree_mark_key(c, b, k, stats)); } if (btree_gc_rewrite_disabled(c)) @@ -216,7 +221,8 @@ static inline void gc_pos_set(struct cache_set *c, struct gc_pos new_pos) __gc_pos_set(c, new_pos); } -static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id) +static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id, + struct bucket_stats_cache_set *stats) { struct btree_iter iter; struct btree *b; @@ -230,7 +236,7 @@ static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id) bch_verify_btree_nr_keys(&b->keys); - should_rewrite = btree_gc_mark_node(c, b); + should_rewrite = btree_gc_mark_node(c, b, stats); gc_pos_set(c, gc_pos_btree_node(b)); @@ -244,7 +250,7 @@ static int bch_gc_btree(struct cache_set *c, enum btree_id btree_id) spin_lock(&c->btree_root_lock); b = c->btree_roots[btree_id]; - __bch_btree_mark_key(c, BKEY_TYPE_BTREE, bkey_i_to_s_c(&b->key)); + __bch_btree_mark_key(c, BKEY_TYPE_BTREE, bkey_i_to_s_c(&b->key), stats); gc_pos_set(c, gc_pos_btree_root(b->btree_id)); spin_unlock(&c->btree_root_lock); @@ -300,7 +306,7 @@ static void bch_mark_metadata(struct cache_set *c) for (j = 0; j < bch_nr_journal_buckets(ca->disk_sb.sb); j++) bch_mark_metadata_bucket(ca, &ca->buckets[journal_bucket(ca, j)], - true); + true); spin_lock(&ca->prio_buckets_lock); @@ -312,6 +318,7 @@ static void bch_mark_metadata(struct cache_set *c) } } +/* Also see bch_pending_btree_node_free_insert_done() */ static void bch_mark_pending_btree_node_frees(struct cache_set *c) { struct pending_btree_node_free *d; @@ -319,12 +326,20 @@ static void bch_mark_pending_btree_node_frees(struct cache_set *c) mutex_lock(&c->btree_node_pending_free_lock); gc_pos_set(c, gc_phase(GC_PHASE_PENDING_DELETE)); - list_for_each_entry(d, &c->btree_node_pending_free, list) + list_for_each_entry(d, &c->btree_node_pending_free, list) { + /* + * Already accounted for in cache_set_stats - we won't apply + * @stats (see bch_pending_btree_node_free_insert_done()): + */ + struct bucket_stats_cache_set stats = { 0 }; + if (d->index_update_done) bch_mark_pointers(c, bkey_i_to_s_c_extent(&d->key), c->sb.btree_node_size, false, true, - true, GC_POS_MIN); + true, GC_POS_MIN, &stats); + } + mutex_unlock(&c->btree_node_pending_free_lock); } @@ -350,6 +365,7 @@ static void bch_mark_scan_keylists(struct cache_set *c) */ void bch_gc(struct cache_set *c) { + struct bucket_stats_cache_set stats = { 0 }; struct cache *ca; struct bucket *g; u64 start_time = local_clock(); @@ -411,7 +427,7 @@ void bch_gc(struct cache_set *c) /* Walk btree: */ while (c->gc_pos.phase < (int) BTREE_ID_NR) { int ret = c->btree_roots[c->gc_pos.phase] - ? bch_gc_btree(c, (int) c->gc_pos.phase) + ? bch_gc_btree(c, (int) c->gc_pos.phase, &stats) : 0; if (ret) { @@ -432,6 +448,8 @@ void bch_gc(struct cache_set *c) for_each_cache(ca, c, i) atomic_long_set(&ca->saturated_count, 0); + bch_cache_set_stats_set(c, &stats); + /* Indicates that gc is no longer in progress: */ gc_pos_set(c, gc_phase(GC_PHASE_DONE)); @@ -852,7 +870,8 @@ int bch_gc_thread_start(struct cache_set *c) /* Initial GC computes bucket marks during startup */ -static void bch_initial_gc_btree(struct cache_set *c, enum btree_id id) +static void bch_initial_gc_btree(struct cache_set *c, enum btree_id id, + struct bucket_stats_cache_set *stats) { struct btree_iter iter; struct btree *b; @@ -873,30 +892,35 @@ static void bch_initial_gc_btree(struct cache_set *c, enum btree_id id) for_each_btree_node_key_unpack(&b->keys, k, &node_iter, &unpacked) - btree_mark_key_initial(c, b, k); + btree_mark_key_initial(c, b, k, stats); } - __bch_btree_mark_key_initial(c, BKEY_TYPE_BTREE, - bkey_i_to_s_c(&b->key)); - bch_btree_iter_cond_resched(&iter); } bch_btree_iter_unlock(&iter); + + b = c->btree_roots[id]; + __bch_btree_mark_key_initial(c, BKEY_TYPE_BTREE, + bkey_i_to_s_c(&b->key), + stats); } int bch_initial_gc(struct cache_set *c, struct list_head *journal) { + struct bucket_stats_cache_set stats = { 0 }; enum btree_id id; if (journal) { for (id = 0; id < BTREE_ID_NR; id++) - bch_initial_gc_btree(c, id); + bch_initial_gc_btree(c, id, &stats); - bch_journal_mark(c, journal); + bch_journal_mark(c, journal, &stats); } bch_mark_metadata(c); + + bch_cache_set_stats_set(c, &stats); gc_pos_set(c, gc_phase(GC_PHASE_DONE)); set_bit(CACHE_SET_INITIAL_GC_DONE, &c->flags); diff --git a/drivers/md/bcache/btree_gc.h b/drivers/md/bcache/btree_gc.h index cba0a5be5378..97e33f46cc98 100644 --- a/drivers/md/bcache/btree_gc.h +++ b/drivers/md/bcache/btree_gc.h @@ -11,7 +11,8 @@ int bch_gc_thread_start(struct cache_set *); int bch_initial_gc(struct cache_set *, struct list_head *); u8 bch_btree_key_recalc_oldest_gen(struct cache_set *, struct bkey_s_c); u8 __bch_btree_mark_key_initial(struct cache_set *, enum bkey_type, - struct bkey_s_c); + struct bkey_s_c, + struct bucket_stats_cache_set *); /* * For concurrent mark and sweep (with other index updates), we define a total diff --git a/drivers/md/bcache/btree_io.c b/drivers/md/bcache/btree_io.c index 4a03c22a0815..26ca269155ee 100644 --- a/drivers/md/bcache/btree_io.c +++ b/drivers/md/bcache/btree_io.c @@ -463,7 +463,7 @@ int bch_btree_root_read(struct cache_set *c, enum btree_id id, return -EIO; } - bch_btree_set_root_initial(c, b); + bch_btree_set_root_initial(c, b, NULL); six_unlock_intent(&b->lock); return 0; diff --git a/drivers/md/bcache/btree_types.h b/drivers/md/bcache/btree_types.h index 4f335f56aa67..f3af91e3d23c 100644 --- a/drivers/md/bcache/btree_types.h +++ b/drivers/md/bcache/btree_types.h @@ -140,12 +140,15 @@ static inline bool btree_node_has_ptrs(struct btree *b) */ struct btree_iter; +struct bucket_stats_cache_set; + struct btree_insert_hook { void (*fn)(struct btree_insert_hook *, struct btree_iter *iter, struct bkey_s_c k, struct bkey_i *insert, - struct journal_res *res); + struct journal_res *res, + struct bucket_stats_cache_set *); }; #endif /* _BCACHE_BTREE_TYPES_H */ diff --git a/drivers/md/bcache/btree_update.c b/drivers/md/bcache/btree_update.c index d5da485d0cb7..3305c6eff047 100644 --- a/drivers/md/bcache/btree_update.c +++ b/drivers/md/bcache/btree_update.c @@ -141,12 +141,24 @@ found: * cancel out one of mark and sweep's markings if necessary: */ - if (gc_pos_cmp(c->gc_pos, gc_phase(GC_PHASE_PENDING_DELETE)) < 0) + /* + * bch_mark_pointers() compares the current gc pos to the pos we're + * moving this reference from, hence one comparison here: + */ + if (gc_pos_cmp(c->gc_pos, gc_phase(GC_PHASE_PENDING_DELETE)) < 0) { + /* + * Already accounted for in cache_set_stats - we won't apply + * @stats: + */ + struct bucket_stats_cache_set stats = { 0 }; + bch_mark_pointers(c, bkey_i_to_s_c_extent(&d->key), -c->sb.btree_node_size, false, true, false, b ? gc_pos_btree_node(b) - : gc_pos_btree_root(id)); + : gc_pos_btree_root(id), + &stats); + } mutex_unlock(&c->btree_node_pending_free_lock); } @@ -210,6 +222,8 @@ void bch_btree_node_free(struct btree_iter *iter, struct btree *b) static void bch_btree_node_free_ondisk(struct cache_set *c, struct pending_btree_node_free *pending) { + struct bucket_stats_cache_set stats = { 0 }; + BUG_ON(!pending->index_update_done); mutex_lock(&c->btree_node_pending_free_lock); @@ -217,7 +231,10 @@ static void bch_btree_node_free_ondisk(struct cache_set *c, bch_mark_pointers(c, bkey_i_to_s_c_extent(&pending->key), -c->sb.btree_node_size, false, true, - false, gc_phase(GC_PHASE_PENDING_DELETE)); + false, gc_phase(GC_PHASE_PENDING_DELETE), + &stats); + + /* Already accounted for in cache_set_stats - don't apply @stats: */ mutex_unlock(&c->btree_node_pending_free_lock); } @@ -341,7 +358,8 @@ struct btree *btree_node_alloc_replacement(struct cache_set *c, return __btree_node_alloc_replacement(c, b, new_f, reserve); } -static void __bch_btree_set_root(struct cache_set *c, struct btree *b) +static void __bch_btree_set_root(struct cache_set *c, struct btree *b, + struct bucket_stats_cache_set *stats) { bool stale; @@ -355,7 +373,8 @@ static void __bch_btree_set_root(struct cache_set *c, struct btree *b) stale = bch_mark_pointers(c, bkey_i_to_s_c_extent(&b->key), c->sb.btree_node_size, true, true, - false, gc_pos_btree_root(b->btree_id)); + false, gc_pos_btree_root(b->btree_id), + stats); BUG_ON(stale); spin_unlock(&c->btree_root_lock); @@ -366,11 +385,17 @@ static void __bch_btree_set_root(struct cache_set *c, struct btree *b) * Only for cache set bringup, when first reading the btree roots or allocating * btree roots when initializing a new cache set: */ -void bch_btree_set_root_initial(struct cache_set *c, struct btree *b) +void bch_btree_set_root_initial(struct cache_set *c, struct btree *b, + struct btree_reserve *btree_reserve) { + struct bucket_stats_cache_set stats = { 0 }; + BUG_ON(btree_node_root(b)); - __bch_btree_set_root(c, b); + __bch_btree_set_root(c, b, &stats); + + if (btree_reserve) + bch_cache_set_stats_apply(c, &stats, &btree_reserve->disk_res); } /** @@ -386,8 +411,10 @@ void bch_btree_set_root_initial(struct cache_set *c, struct btree *b) * journal write. */ static int bch_btree_set_root(struct btree_iter *iter, struct btree *b, - struct journal_res *res) + struct journal_res *res, + struct btree_reserve *btree_reserve) { + struct bucket_stats_cache_set stats = { 0 }; struct cache_set *c = iter->c; struct btree *old; u64 seq; @@ -403,7 +430,7 @@ static int bch_btree_set_root(struct btree_iter *iter, struct btree *b, */ btree_node_lock_write(old, iter); - __bch_btree_set_root(c, b); + __bch_btree_set_root(c, b, &stats); /* * Unlock old root after new root is visible: @@ -417,6 +444,9 @@ static int bch_btree_set_root(struct btree_iter *iter, struct btree *b, bch_pending_btree_node_free_insert_done(c, NULL, old->btree_id, bkey_i_to_s_c(&old->key)); + stats.sectors_meta -= c->sb.btree_node_size; + bch_cache_set_stats_apply(c, &stats, &btree_reserve->disk_res); + /* * Ensure new btree root is persistent (reachable via the * journal) before returning and the caller unlocking it: @@ -499,8 +529,10 @@ static struct btree_reserve *__bch_btree_reserve_get(struct cache_set *c, * open bucket reserve: */ ret = mca_cannibalize_lock(c, cl); - if (ret) + if (ret) { + bch_disk_reservation_put(c, &disk_res); return ERR_PTR(ret); + } reserve = mempool_alloc(&c->btree_reserve_pool, GFP_NOIO); @@ -562,27 +594,31 @@ int bch_btree_root_alloc(struct cache_set *c, enum btree_id id, } b = __btree_root_alloc(c, 0, id, reserve); - bch_btree_reserve_put(c, reserve); bch_btree_node_write(b, writes, NULL); - bch_btree_set_root_initial(c, b); + bch_btree_set_root_initial(c, b, reserve); btree_open_bucket_put(c, b); six_unlock_intent(&b->lock); + bch_btree_reserve_put(c, reserve); + return 0; } static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter, struct btree *b, struct bkey_i *insert, - struct btree_node_iter *node_iter) + struct btree_node_iter *node_iter, + struct disk_reservation *disk_res) { struct cache_set *c = iter->c; const struct bkey_format *f = &b->keys.format; + struct bucket_stats_cache_set stats = { 0 }; struct bkey_packed *k; int cmp; + bch_btree_node_iter_verify(node_iter, &b->keys); EBUG_ON((k = bch_btree_node_iter_prev_all(node_iter, &b->keys)) && (bkey_deleted(k) ? bkey_cmp_packed(f, k, &insert->k) > 0 @@ -594,7 +630,7 @@ static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter, stale = bch_mark_pointers(c, bkey_i_to_s_c_extent(insert), c->sb.btree_node_size, true, true, false, - gc_pos_btree_node(b)); + gc_pos_btree_node(b), &stats); BUG_ON(stale); } @@ -614,6 +650,7 @@ static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter, */ k->type = KEY_TYPE_DELETED; btree_keys_account_key_drop(&b->keys.nr, k); + stats.sectors_meta -= c->sb.btree_node_size; } bch_btree_node_iter_next_all(node_iter, &b->keys); @@ -621,6 +658,8 @@ static bool bch_insert_fixup_btree_ptr(struct btree_iter *iter, bch_btree_bset_insert(iter, b, node_iter, insert); set_btree_node_dirty(b); + + bch_cache_set_stats_apply(c, &stats, disk_res); return true; } @@ -635,6 +674,7 @@ void bch_btree_bset_insert(struct btree_iter *iter, struct btree_iter *linked; struct bkey_packed *where; + EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k)); EBUG_ON(insert->k.u64s > bch_btree_keys_u64s_remaining(iter->c, b)); EBUG_ON(bkey_cmp(bkey_start_pos(&insert->k), b->data->min_key) < 0 || bkey_cmp(insert->k.p, b->data->max_key) > 0); @@ -736,7 +776,7 @@ void bch_btree_insert_and_journal(struct btree_iter *iter, * @insert_keys: list of keys to insert * @replace: old key for for exchange (+ stats) * @res: journal reservation - * @flags: BTREE_INSERT_NOFAIL_IF_STALE + * @flags: BTREE_INSERT_NO_MARK_KEY * * Inserts the first key from @insert_keys * @@ -746,6 +786,7 @@ void bch_btree_insert_and_journal(struct btree_iter *iter, static void btree_insert_key(struct btree_iter *iter, struct btree *b, struct btree_node_iter *node_iter, struct keylist *insert_keys, + struct disk_reservation *disk_res, struct btree_insert_hook *hook, struct journal_res *res, unsigned flags) @@ -754,31 +795,22 @@ static void btree_insert_key(struct btree_iter *iter, struct btree *b, BKEY_PADDED(key) temp; s64 oldsize = bch_count_data(&b->keys); - BUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k)); bch_btree_node_iter_verify(node_iter, &b->keys); + BUG_ON(b->level); + BUG_ON(iter->nodes[0] != b || &iter->node_iters[0] != node_iter); - if (b->level) { - BUG_ON(res->ref); - - bch_insert_fixup_btree_ptr(iter, b, insert, node_iter); - bch_keylist_dequeue(insert_keys); - } else if (!b->keys.ops->is_extents) { - BUG_ON(iter->nodes[0] != b || - &iter->node_iters[0] != node_iter); - + if (!b->keys.ops->is_extents) { bch_insert_fixup_key(iter, insert, hook, res); bch_keylist_dequeue(insert_keys); } else { - BUG_ON(iter->nodes[0] != b || - &iter->node_iters[0] != node_iter); - bkey_copy(&temp.key, insert); insert = &temp.key; if (bkey_cmp(insert->k.p, b->key.k.p) > 0) bch_cut_back(b->key.k.p, &insert->k); - bch_insert_fixup_extent(iter, insert, hook, res, flags); + bch_insert_fixup_extent(iter, insert, disk_res, + hook, res, flags); bch_cut_front(iter->pos, orig); if (orig->k.size == 0) @@ -1068,12 +1100,12 @@ static enum btree_insert_status bch_btree_insert_keys_interior(struct btree *b, struct btree_iter *iter, struct keylist *insert_keys, - struct async_split *as) + struct async_split *as, + struct btree_reserve *res) { struct btree_node_iter *node_iter = &iter->node_iters[b->level]; const struct bkey_format *f = &b->keys.format; struct bkey_packed *k; - struct journal_res res = { 0, 0 }; BUG_ON(!btree_node_intent_locked(iter, btree_node_root(b)->level)); BUG_ON(!b->level); @@ -1109,8 +1141,9 @@ bch_btree_insert_keys_interior(struct btree *b, (bkey_cmp_packed(f, k, &insert->k) >= 0)) ; - btree_insert_key(iter, b, node_iter, insert_keys, - NULL, &res, 0); + bch_insert_fixup_btree_ptr(iter, b, insert, + node_iter, &res->disk_res); + bch_keylist_dequeue(insert_keys); } btree_node_unlock_write(b, iter); @@ -1146,6 +1179,7 @@ static enum btree_insert_status bch_btree_insert_keys_leaf(struct btree *b, struct btree_iter *iter, struct keylist *insert_keys, + struct disk_reservation *disk_res, struct btree_insert_hook *hook, u64 *journal_seq, unsigned flags) @@ -1203,7 +1237,8 @@ bch_btree_insert_keys_leaf(struct btree *b, break; btree_insert_key(iter, b, &iter->node_iters[b->level], - insert_keys, hook, &res, flags); + insert_keys, disk_res, + hook, &res, flags); } btree_node_unlock_write(b, iter); @@ -1299,9 +1334,10 @@ static struct btree *__btree_split_node(struct btree_iter *iter, struct btree *n } static void btree_split_insert_keys(struct btree_iter *iter, struct btree *b, - struct keylist *keys, bool is_last) + struct keylist *keys, + struct btree_reserve *res, + bool is_last) { - struct journal_res res = { 0, 0 }; struct btree_node_iter node_iter; struct bkey_i *k = bch_keylist_front(keys); @@ -1323,8 +1359,8 @@ static void btree_split_insert_keys(struct btree_iter *iter, struct btree *b, break; } - btree_insert_key(iter, b, &node_iter, keys, - NULL, &res, 0); + bch_insert_fixup_btree_ptr(iter, b, k, &node_iter, &res->disk_res); + bch_keylist_dequeue(keys); } six_unlock_write(&b->lock); @@ -1384,7 +1420,7 @@ static int btree_split(struct btree *b, struct btree_iter *iter, struct bset *i; six_unlock_write(&n1->lock); - btree_split_insert_keys(iter, n1, insert_keys, true); + btree_split_insert_keys(iter, n1, insert_keys, reserve, true); six_lock_write(&n1->lock); /* @@ -1438,7 +1474,8 @@ static int btree_split(struct btree *b, struct btree_iter *iter, iter->btree_id, reserve); - btree_split_insert_keys(iter, n3, &as->parent_keys, true); + btree_split_insert_keys(iter, n3, &as->parent_keys, + reserve, true); bch_btree_node_write(n3, &as->cl, NULL); } } else { @@ -1470,12 +1507,12 @@ static int btree_split(struct btree *b, struct btree_iter *iter, goto err; if (n3) { - ret = bch_btree_set_root(iter, n3, &as->res); + ret = bch_btree_set_root(iter, n3, &as->res, reserve); if (ret) goto err; } else { /* Root filled up but didn't need to be split */ - ret = bch_btree_set_root(iter, n1, &as->res); + ret = bch_btree_set_root(iter, n1, &as->res, reserve); if (ret) goto err; } @@ -1533,7 +1570,7 @@ err: * @insert_keys: list of keys to insert * @hook: insert callback * @persistent: if not null, @persistent will wait on journal write - * @flags: BTREE_INSERT_NOFAIL_IF_STALE + * @flags: BTREE_INSERT_NO_MARK_KEY * * Inserts as many keys as it can into a given btree node, splitting it if full. * If a split occurred, this function will return early. This can only happen @@ -1548,7 +1585,8 @@ int bch_btree_insert_node(struct btree *b, BUG_ON(!b->level); BUG_ON(!reserve || !as); - switch (bch_btree_insert_keys_interior(b, iter, insert_keys, as)) { + switch (bch_btree_insert_keys_interior(b, iter, insert_keys, + as, reserve)) { case BTREE_INSERT_OK: return 0; case BTREE_INSERT_NEED_SPLIT: @@ -1610,7 +1648,7 @@ out_unlock: * @insert_keys: list of keys to insert * @hook: insert callback * @persistent: if not null, @persistent will wait on journal write - * @flags: BTREE_INSERT_ATOMIC | BTREE_INSERT_NOFAIL_IF_STALE + * @flags: BTREE_INSERT_ATOMIC | BTREE_INSERT_NO_MARK_KEY * * This is top level for common btree insertion/index update code. The control * flow goes roughly like: @@ -1639,6 +1677,7 @@ out_unlock: */ int bch_btree_insert_at(struct btree_iter *iter, struct keylist *insert_keys, + struct disk_reservation *disk_res, struct btree_insert_hook *hook, u64 *journal_seq, unsigned flags) { @@ -1658,8 +1697,8 @@ int bch_btree_insert_at(struct btree_iter *iter, iter->pos)); switch (bch_btree_insert_keys_leaf(iter->nodes[0], iter, - insert_keys, hook, - journal_seq, flags)) { + insert_keys, disk_res, + hook, journal_seq, flags)) { case BTREE_INSERT_OK: ret = 0; break; @@ -1794,7 +1833,8 @@ retry: for (i = m; i < m + nr; i++) btree_insert_key(i->iter, i->iter->nodes[0], &i->iter->node_iters[0], - &keylist_single(i->k), NULL, + &keylist_single(i->k), + NULL, NULL, &res, flags); do { @@ -1873,7 +1913,7 @@ int bch_btree_insert_check_key(struct btree_iter *iter, bch_btree_iter_rewind(iter, bkey_start_pos(&check_key->k)); ret = bch_btree_insert_at(iter, &keylist_single(&tmp.key), - NULL, NULL, BTREE_INSERT_ATOMIC); + NULL, NULL, NULL, BTREE_INSERT_ATOMIC); bch_btree_iter_set_pos(iter, saved_pos); @@ -1888,7 +1928,9 @@ int bch_btree_insert_check_key(struct btree_iter *iter, * @hook: insert callback */ int bch_btree_insert(struct cache_set *c, enum btree_id id, - struct keylist *keys, struct btree_insert_hook *hook, + struct keylist *keys, + struct disk_reservation *disk_res, + struct btree_insert_hook *hook, u64 *journal_seq, int flags) { struct btree_iter iter; @@ -1901,7 +1943,8 @@ int bch_btree_insert(struct cache_set *c, enum btree_id id, if (unlikely(ret)) goto out; - ret = bch_btree_insert_at(&iter, keys, hook, journal_seq, flags); + ret = bch_btree_insert_at(&iter, keys, disk_res, + hook, journal_seq, flags); out: ret2 = bch_btree_iter_unlock(&iter); return ret ?: ret2; @@ -1928,7 +1971,7 @@ int bch_btree_update(struct cache_set *c, enum btree_id id, return -ENOENT; ret = bch_btree_insert_at(&iter, &keylist_single(k), NULL, - journal_seq, 0); + NULL, journal_seq, 0); ret2 = bch_btree_iter_unlock(&iter); return ret ?: ret2; @@ -1991,7 +2034,7 @@ int bch_btree_delete_range(struct cache_set *c, enum btree_id id, } ret = bch_btree_insert_at(&iter, &keylist_single(&delete), - hook, journal_seq, + NULL, hook, journal_seq, BTREE_INSERT_NOFAIL); if (ret) break; @@ -2056,7 +2099,7 @@ int bch_btree_node_rewrite(struct btree *b, struct btree_iter *iter, bool wait) return -EIO; } - bch_btree_set_root(iter, n, &as->res); + bch_btree_set_root(iter, n, &as->res, reserve); continue_at_noreturn(&as->cl, async_split_writes_done, system_wq); diff --git a/drivers/md/bcache/btree_update.h b/drivers/md/bcache/btree_update.h index eea17208c12f..8deb6b187b49 100644 --- a/drivers/md/bcache/btree_update.h +++ b/drivers/md/bcache/btree_update.h @@ -80,7 +80,8 @@ struct async_split *__bch_async_split_alloc(struct btree *[], unsigned, struct btree_iter *); struct async_split *bch_async_split_alloc(struct btree *, struct btree_iter *); -void bch_btree_set_root_initial(struct cache_set *, struct btree *); +void bch_btree_set_root_initial(struct cache_set *, struct btree *, + struct btree_reserve *); void bch_btree_reserve_put(struct cache_set *, struct btree_reserve *); struct btree_reserve *bch_btree_reserve_get(struct cache_set *c, @@ -168,13 +169,13 @@ int bch_btree_insert_node(struct btree *, struct btree_iter *, #define BTREE_INSERT_NOFAIL (1 << 1) /* - * Don't fail a btree insert if dirty stale pointers are being added - * - * Only for journal replay: + * Don't account key being insert (bch_mark_pointers) - only for journal replay, + * where we've already marked the new keys: */ -#define BTREE_INSERT_NOFAIL_IF_STALE (1 << 2) +#define BTREE_INSERT_NO_MARK_KEY (1 << 2) int bch_btree_insert_at(struct btree_iter *, struct keylist *, + struct disk_reservation *, struct btree_insert_hook *, u64 *, unsigned); struct btree_insert_multi { @@ -187,6 +188,7 @@ int bch_btree_insert_at_multi(struct btree_insert_multi[], unsigned, int bch_btree_insert_check_key(struct btree_iter *, struct bkey_i *); int bch_btree_insert(struct cache_set *, enum btree_id, struct keylist *, + struct disk_reservation *, struct btree_insert_hook *, u64 *, int flags); int bch_btree_update(struct cache_set *, enum btree_id, struct bkey_i *, u64 *); diff --git a/drivers/md/bcache/buckets.c b/drivers/md/bcache/buckets.c index 6c4ac8b91a60..742f2c3d5b62 100644 --- a/drivers/md/bcache/buckets.c +++ b/drivers/md/bcache/buckets.c @@ -69,20 +69,22 @@ #include <trace/events/bcache.h> +#define bucket_stats_add(_acc, _stats) \ +do { \ + typeof(_acc) _a = (_acc), _s = (_stats); \ + unsigned i; \ + \ + for (i = 0; i < sizeof(*_a) / sizeof(u64); i++) \ + ((u64 *) (_a))[i] += ((u64 *) (_s))[i]; \ +} while (0) + #define bucket_stats_read_raw(_stats) \ ({ \ - typeof(*this_cpu_ptr(_stats)) _acc, *_s; \ - unsigned i; \ + typeof(*this_cpu_ptr(_stats)) _acc = { 0 }; \ int cpu; \ \ - memset(&_acc, 0, sizeof(_acc)); \ - \ - for_each_possible_cpu(cpu) { \ - _s = per_cpu_ptr((_stats), cpu); \ - \ - for (i = 0; i < sizeof(_acc) / sizeof(u64); i++) \ - ((u64 *) &_acc)[i] += ((u64 *) _s)[i]; \ - } \ + for_each_possible_cpu(cpu) \ + bucket_stats_add(&_acc, per_cpu_ptr((_stats), cpu)); \ \ _acc; \ }) @@ -143,23 +145,45 @@ static inline int is_cached_bucket(struct bucket_mark m) return !m.owned_by_allocator && !m.dirty_sectors && !!m.cached_sectors; } +void bch_cache_set_stats_apply(struct cache_set *c, + struct bucket_stats_cache_set *stats, + struct disk_reservation *disk_res) +{ + s64 added = stats->sectors_dirty + + stats->sectors_meta + + stats->sectors_reserved; + + /* + * Not allowed to reduce sectors_available except by getting a + * reservation: + */ + BUG_ON(added > (disk_res ? disk_res->sectors : 0)); + + if (disk_res && added > 0) { + disk_res->sectors -= added; + stats->sectors_reserved -= added; + } + + lg_local_lock(&c->bucket_stats_lock); + bucket_stats_add(this_cpu_ptr(c->bucket_stats_percpu), stats); + lg_local_unlock(&c->bucket_stats_lock); +} + static void bucket_stats_update(struct cache *ca, - struct bucket_mark old, - struct bucket_mark new, - bool may_make_unavailable) + struct bucket_mark old, struct bucket_mark new, + bool may_make_unavailable, + struct bucket_stats_cache_set *cache_set_stats) { struct cache_set *c = ca->set; struct bucket_stats_cache *cache_stats; - struct bucket_stats_cache_set *cache_set_stats; BUG_ON(!may_make_unavailable && is_available_bucket(old) && !is_available_bucket(new) && c->gc_pos.phase == GC_PHASE_DONE); - lg_local_lock(&c->bucket_stats_lock); + preempt_disable(); cache_stats = this_cpu_ptr(ca->bucket_stats_percpu); - cache_set_stats = this_cpu_ptr(c->bucket_stats_percpu); cache_stats->sectors_cached += (int) new.cached_sectors - (int) old.cached_sectors; @@ -188,26 +212,29 @@ static void bucket_stats_update(struct cache *ca, cache_stats->buckets_meta += is_meta_bucket(new) - is_meta_bucket(old); cache_stats->buckets_cached += is_cached_bucket(new) - is_cached_bucket(old); cache_stats->buckets_dirty += is_dirty_bucket(new) - is_dirty_bucket(old); - lg_local_unlock(&c->bucket_stats_lock); + preempt_enable(); if (!is_available_bucket(old) && is_available_bucket(new)) bch_wake_allocator(ca); } static struct bucket_mark bch_bucket_mark_set(struct cache *ca, - struct bucket *g, - struct bucket_mark new, - bool may_make_unavailable) + struct bucket *g, struct bucket_mark new, + bool may_make_unavailable) { + struct bucket_stats_cache_set stats = { 0 }; struct bucket_mark old; old.counter = xchg(&g->mark.counter, new.counter); - bucket_stats_update(ca, old, new, may_make_unavailable); + bucket_stats_update(ca, old, new, may_make_unavailable, &stats); + bch_cache_set_stats_apply(ca->set, &stats, NULL); return old; } -#define bucket_cmpxchg(g, old, new, may_make_unavailable, expr) \ +#define bucket_cmpxchg(g, old, new, \ + may_make_unavailable, \ + cache_set_stats, expr) \ do { \ u32 _v = READ_ONCE((g)->mark.counter); \ \ @@ -217,7 +244,9 @@ do { \ } while ((_v = cmpxchg(&(g)->mark.counter, \ old.counter, \ new.counter)) != old.counter); \ - bucket_stats_update(ca, old, new, may_make_unavailable);\ + bucket_stats_update(ca, old, new, \ + may_make_unavailable, \ + cache_set_stats); \ } while (0) void bch_mark_free_bucket(struct cache *ca, struct bucket *g) @@ -274,14 +303,16 @@ do { \ static u8 bch_mark_bucket(struct cache_set *c, struct cache *ca, const struct bch_extent_ptr *ptr, int sectors, bool dirty, bool metadata, bool is_gc, - struct gc_pos gc_pos) + struct gc_pos gc_pos, + struct bucket_stats_cache_set *stats) { struct bucket_mark old, new; unsigned long bucket_nr = PTR_BUCKET_NR(ca, ptr); unsigned saturated; u8 stale; - bucket_cmpxchg(&ca->buckets[bucket_nr], old, new, is_gc, ({ + bucket_cmpxchg(&ca->buckets[bucket_nr], old, new, + is_gc, stats, ({ saturated = 0; /* * cmpxchg() only implies a full barrier on success, not @@ -360,7 +391,8 @@ static u8 bch_mark_bucket(struct cache_set *c, struct cache *ca, */ int bch_mark_pointers(struct cache_set *c, struct bkey_s_c_extent e, int sectors, bool fail_if_stale, bool metadata, - bool is_gc, struct gc_pos pos) + bool is_gc, struct gc_pos pos, + struct bucket_stats_cache_set *stats) { const struct bch_extent_ptr *ptr, *ptr2; struct cache *ca; @@ -407,7 +439,7 @@ int bch_mark_pointers(struct cache_set *c, struct bkey_s_c_extent e, * Fuck me, I hate my life. */ stale = bch_mark_bucket(c, ca, ptr, sectors, dirty, - metadata, is_gc, pos); + metadata, is_gc, pos, stats); if (stale && dirty && fail_if_stale) goto stale; } @@ -421,7 +453,7 @@ stale: bch_mark_bucket(c, ca, ptr, -sectors, bch_extent_ptr_is_dirty(c, e, ptr), - metadata, is_gc, pos); + metadata, is_gc, pos, stats); } rcu_read_unlock(); @@ -440,11 +472,14 @@ void bch_mark_reservation(struct cache_set *c, int sectors) void bch_unmark_open_bucket(struct cache *ca, struct bucket *g) { + struct bucket_stats_cache_set stats = { 0 }; struct bucket_mark old, new; - bucket_cmpxchg(g, old, new, false, ({ + bucket_cmpxchg(g, old, new, false, &stats, ({ new.owned_by_allocator = 0; })); + + bch_cache_set_stats_apply(ca->set, &stats, NULL); } static u64 __recalc_sectors_available(struct cache_set *c) @@ -485,16 +520,6 @@ void bch_disk_reservation_put(struct cache_set *c, #define SECTORS_CACHE 1024 -/* - * XXX - * - * For the trick we're using here to work, we have to ensure that everything - * that decreases the amount of space available goes through here and decreases - * sectors_available: - * - * Need to figure out a way of asserting that that's happening (e.g. btree node - * allocations?) - */ int __bch_disk_reservation_get(struct cache_set *c, struct disk_reservation *res, unsigned sectors, @@ -568,3 +593,23 @@ int bch_disk_reservation_get(struct cache_set *c, { return __bch_disk_reservation_get(c, res, sectors, true, false); } + +void bch_cache_set_stats_set(struct cache_set *c, + struct bucket_stats_cache_set *stats) +{ + struct bucket_stats_cache_set *s; + int cpu; + + lg_global_lock(&c->bucket_stats_lock); + for_each_possible_cpu(cpu) { + s = per_cpu_ptr(c->bucket_stats_percpu, cpu); + memset(s, 0, sizeof(*s)); + } + + s = this_cpu_ptr(c->bucket_stats_percpu); + *s = *stats; + + atomic64_set(&c->sectors_available, 0); + + lg_global_unlock(&c->bucket_stats_lock); +} diff --git a/drivers/md/bcache/buckets.h b/drivers/md/bcache/buckets.h index 561e97cb7d2b..270ab0d599ad 100644 --- a/drivers/md/bcache/buckets.h +++ b/drivers/md/bcache/buckets.h @@ -194,15 +194,19 @@ static inline size_t buckets_free_cache(struct cache *ca, struct bucket_stats_cache_set __bch_bucket_stats_read_cache_set(struct cache_set *); struct bucket_stats_cache_set bch_bucket_stats_read_cache_set(struct cache_set *); +void bch_cache_set_stats_apply(struct cache_set *, + struct bucket_stats_cache_set *, + struct disk_reservation *); static inline u64 cache_set_sectors_used(struct cache_set *c) { - struct bucket_stats_cache_set stats = bch_bucket_stats_read_cache_set(c); + struct bucket_stats_cache_set stats = __bch_bucket_stats_read_cache_set(c); return min(c->capacity, stats.sectors_meta + stats.sectors_dirty + - stats.sectors_reserved); + stats.sectors_reserved + + (stats.sectors_reserved >> 7)); } /* XXX: kill? */ @@ -233,19 +237,12 @@ void bch_mark_metadata_bucket(struct cache *, struct bucket *, bool); void bch_unmark_open_bucket(struct cache *, struct bucket *); int bch_mark_pointers(struct cache_set *, struct bkey_s_c_extent, - int, bool, bool, bool, struct gc_pos); + int, bool, bool, bool, struct gc_pos, + struct bucket_stats_cache_set *); void bch_mark_reservation(struct cache_set *, int); void bch_recalc_sectors_available(struct cache_set *); -/* - * A reservation for space on disk: - */ -struct disk_reservation { - u32 sectors; - u32 gen; -}; - void bch_disk_reservation_put(struct cache_set *, struct disk_reservation *); int __bch_disk_reservation_get(struct cache_set *, @@ -255,4 +252,7 @@ int bch_disk_reservation_get(struct cache_set *, struct disk_reservation *, unsigned); +void bch_cache_set_stats_set(struct cache_set *, + struct bucket_stats_cache_set *); + #endif /* _BUCKETS_H */ diff --git a/drivers/md/bcache/buckets_types.h b/drivers/md/bcache/buckets_types.h index 1b7df1662144..48ac31cc04c7 100644 --- a/drivers/md/bcache/buckets_types.h +++ b/drivers/md/bcache/buckets_types.h @@ -56,4 +56,12 @@ struct bucket_heap_entry { unsigned long val; }; +/* + * A reservation for space on disk: + */ +struct disk_reservation { + u32 sectors; + u32 gen; +}; + #endif /* _BUCKETS_TYPES_H */ diff --git a/drivers/md/bcache/dirent.c b/drivers/md/bcache/dirent.c index 4c8b2cdb2d8e..8868e41c8e89 100644 --- a/drivers/md/bcache/dirent.c +++ b/drivers/md/bcache/dirent.c @@ -210,7 +210,7 @@ int bch_dirent_create(struct inode *dir, u8 type, dirent->k.p = k.k->p; ret = bch_btree_insert_at(&iter, &keylist_single(&dirent->k_i), - NULL, &ei->journal_seq, + NULL, NULL, &ei->journal_seq, BTREE_INSERT_ATOMIC); /* * XXX: if we ever cleanup whiteouts, we may need to rewind @@ -366,7 +366,7 @@ int bch_dirent_delete(struct inode *dir, const struct qstr *name) ret = bch_btree_insert_at(&iter, &keylist_single(&delete), - NULL, &ei->journal_seq, + NULL, NULL, &ei->journal_seq, BTREE_INSERT_NOFAIL| BTREE_INSERT_ATOMIC); /* diff --git a/drivers/md/bcache/extents.c b/drivers/md/bcache/extents.c index 06127a117a16..1a0aaf63a17f 100644 --- a/drivers/md/bcache/extents.c +++ b/drivers/md/bcache/extents.c @@ -796,7 +796,8 @@ struct btree_nr_keys bch_extent_sort_fix_overlapping(struct btree_keys *b, } static int bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k, - u64 offset, s64 sectors, bool fail_if_stale) + u64 offset, s64 sectors, bool fail_if_stale, + struct bucket_stats_cache_set *stats) { struct cache_set *c = iter->c; struct btree *b = iter->nodes[0]; @@ -814,7 +815,7 @@ static int bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k, ret = bch_mark_pointers(c, e, sectors, fail_if_stale, false, false, - gc_pos_btree_node(b)); + gc_pos_btree_node(b), stats); if (ret) return ret; @@ -822,40 +823,47 @@ static int bch_add_sectors(struct btree_iter *iter, struct bkey_s_c k, bcache_dev_sectors_dirty_add(c, e.k->p.inode, offset, sectors); } else if (k.k->type == BCH_RESERVATION) { - bch_mark_reservation(c, sectors); + stats->sectors_reserved += sectors; } return 0; } static void bch_subtract_sectors(struct btree_iter *iter, struct bkey_s_c k, - u64 offset, s64 sectors) + u64 offset, s64 sectors, + struct bucket_stats_cache_set *stats) { - bch_add_sectors(iter, k, offset, -sectors, false); + bch_add_sectors(iter, k, offset, -sectors, false, stats); } /* These wrappers subtract exactly the sectors that we're removing from @k */ static void bch_cut_subtract_back(struct btree_iter *iter, - struct bpos where, struct bkey_s k) + struct bpos where, struct bkey_s k, + struct bucket_stats_cache_set *stats) { bch_subtract_sectors(iter, k.s_c, where.offset, - k.k->p.offset - where.offset); + k.k->p.offset - where.offset, + stats); bch_cut_back(where, k.k); } static void bch_cut_subtract_front(struct btree_iter *iter, - struct bpos where, struct bkey_s k) + struct bpos where, struct bkey_s k, + struct bucket_stats_cache_set *stats) { bch_subtract_sectors(iter, k.s_c, bkey_start_offset(k.k), - where.offset - bkey_start_offset(k.k)); + where.offset - bkey_start_offset(k.k), + stats); __bch_cut_front(where, k); } -static void bch_drop_subtract(struct btree_iter *iter, struct bkey_s k) +static void bch_drop_subtract(struct btree_iter *iter, struct bkey_s k, + struct bucket_stats_cache_set *stats) { if (k.k->size) bch_subtract_sectors(iter, k.s_c, - bkey_start_offset(k.k), k.k->size); + bkey_start_offset(k.k), k.k->size, + stats); k.k->size = 0; __set_bkey_deleted(k.k); } @@ -948,7 +956,8 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook, struct btree_iter *iter, struct bkey_s_c k, struct bkey_i *new, - struct journal_res *res) + struct journal_res *res, + struct bucket_stats_cache_set *stats) { struct bch_replace_info *replace = container_of(hook, struct bch_replace_info, hook); @@ -965,7 +974,8 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook, bkey_start_pos(&old->k)) < 0); if (!k.k) { - bch_cut_subtract_back(iter, iter->pos, bkey_i_to_s(new)); + bch_cut_subtract_back(iter, iter->pos, + bkey_i_to_s(new), stats); return; } @@ -993,7 +1003,7 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook, } bch_cut_subtract_front(iter, bkey_start_pos(k.k), - bkey_i_to_s(new)); + bkey_i_to_s(new), stats); /* advance @iter->pos from the end of prev key to the start of @k */ bch_btree_iter_set_pos(iter, bkey_start_pos(k.k)); } @@ -1019,9 +1029,10 @@ void bch_extent_cmpxchg(struct btree_insert_hook *hook, /* update @new to be the part we haven't checked yet */ if (bkey_cmp(k.k->p, new->k.p) > 0) - bch_drop_subtract(iter, bkey_i_to_s(new)); + bch_drop_subtract(iter, bkey_i_to_s(new), stats); else - bch_cut_subtract_front(iter, k.k->p, bkey_i_to_s(new)); + bch_cut_subtract_front(iter, k.k->p, + bkey_i_to_s(new), stats); } else replace->successes += 1; } @@ -1120,6 +1131,7 @@ static void handle_existing_key_newer(struct btree_iter *iter, */ void bch_insert_fixup_extent(struct btree_iter *iter, struct bkey_i *insert, + struct disk_reservation *disk_res, struct btree_insert_hook *hook, struct journal_res *res, unsigned flags) @@ -1133,6 +1145,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter, struct bkey_s k; struct bpos next_pos; BKEY_PADDED(k) split; + struct bucket_stats_cache_set stats = { 0 }; unsigned nr_done = 0; u64 start_time = local_clock(); @@ -1164,14 +1177,16 @@ void bch_insert_fixup_extent(struct btree_iter *iter, * can also insert keys with stale pointers, but for those we still need * to proceed with the insertion. */ - if (bch_add_sectors(iter, bkey_i_to_s_c(insert), + if (!(flags & BTREE_INSERT_NO_MARK_KEY) && + bch_add_sectors(iter, bkey_i_to_s_c(insert), bkey_start_offset(&insert->k), insert->k.size, - !(flags & BTREE_INSERT_NOFAIL_IF_STALE))) { + true, + &stats)) { /* We raced - a dirty pointer was stale */ bch_btree_iter_set_pos(iter, insert->k.p); insert->k.size = 0; - return; + goto apply_stats; } while (insert->k.size && @@ -1210,9 +1225,10 @@ void bch_insert_fixup_extent(struct btree_iter *iter, * XXX: would be better to explicitly signal that we * need to split */ - bch_cut_subtract_back(iter, iter->pos, bkey_i_to_s(insert)); + bch_cut_subtract_back(iter, iter->pos, + bkey_i_to_s(insert), &stats); if (!insert->k.size) - return; + goto apply_stats; break; } #if 0 @@ -1245,7 +1261,8 @@ void bch_insert_fixup_extent(struct btree_iter *iter, ? k.k->p : insert->k.p; if (hook) - hook->fn(hook, iter, k.s_c, insert, res); + hook->fn(hook, iter, k.s_c, insert, + res, &stats); /* * Don't update iter->pos until after calling the hook, @@ -1254,7 +1271,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter, bch_btree_iter_set_pos(iter, next_pos); if (!insert->k.size) - return; + goto apply_stats; /* insert and k might not overlap after calling hook fn: */ if (bkey_cmp(insert->k.p, bkey_start_pos(k.k)) <= 0 || @@ -1267,14 +1284,15 @@ void bch_insert_fixup_extent(struct btree_iter *iter, switch (bch_extent_overlap(&insert->k, k.k)) { case BCH_EXTENT_OVERLAP_FRONT: /* insert and k share the start, invalidate in k */ - bch_cut_subtract_front(iter, insert->k.p, k); + bch_cut_subtract_front(iter, insert->k.p, k, &stats); extent_save(_k, k.k, f); break; case BCH_EXTENT_OVERLAP_BACK: /* insert and k share the end, invalidate in k */ bch_cut_subtract_back(iter, - bkey_start_pos(&insert->k), k); + bkey_start_pos(&insert->k), + k, &stats); extent_save(_k, k.k, f); /* @@ -1291,7 +1309,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter, if (!bkey_deleted(_k)) btree_keys_account_key_drop(&b->keys.nr, _k); - bch_drop_subtract(iter, k); + bch_drop_subtract(iter, k, &stats); k.k->p = bkey_start_pos(&insert->k); extent_save(_k, k.k, f); @@ -1318,7 +1336,7 @@ void bch_insert_fixup_extent(struct btree_iter *iter, bch_cut_back(bkey_start_pos(&insert->k), &split.k.k); __bch_cut_front(bkey_start_pos(&insert->k), k); - bch_cut_subtract_front(iter, insert->k.p, k); + bch_cut_subtract_front(iter, insert->k.p, k, &stats); extent_save(_k, k.k, f); bch_btree_bset_insert(iter, b, node_iter, &split.k); @@ -1329,14 +1347,16 @@ void bch_insert_fixup_extent(struct btree_iter *iter, next_pos = insert->k.p; if (hook) - hook->fn(hook, iter, bkey_s_c_null, insert, res); + hook->fn(hook, iter, bkey_s_c_null, insert, res, &stats); bch_btree_iter_set_pos(iter, next_pos); if (!insert->k.size) - return; + goto apply_stats; bch_btree_insert_and_journal(iter, insert, res); +apply_stats: + bch_cache_set_stats_apply(c, &stats, disk_res); } static const char *bch_extent_invalid(const struct cache_set *c, diff --git a/drivers/md/bcache/extents.h b/drivers/md/bcache/extents.h index 90f7f8ffa849..42db3d1ca933 100644 --- a/drivers/md/bcache/extents.h +++ b/drivers/md/bcache/extents.h @@ -51,9 +51,11 @@ void bch_extent_cmpxchg(struct btree_insert_hook *, struct btree_iter *, struct bkey_s_c, struct bkey_i *, - struct journal_res *); + struct journal_res *, + struct bucket_stats_cache_set *); void bch_insert_fixup_extent(struct btree_iter *, struct bkey_i *, + struct disk_reservation *, struct btree_insert_hook *, struct journal_res *, unsigned); diff --git a/drivers/md/bcache/fs-gc.c b/drivers/md/bcache/fs-gc.c index 772ef7bb59f4..264744850422 100644 --- a/drivers/md/bcache/fs-gc.c +++ b/drivers/md/bcache/fs-gc.c @@ -174,7 +174,7 @@ static int bch_gc_do_inode(struct cache_set *c, struct btree_iter *iter, return bch_btree_insert_at(iter, &keylist_single(&update.k_i), - NULL, NULL, + NULL, NULL, NULL, BTREE_INSERT_NOFAIL); } diff --git a/drivers/md/bcache/fs-io.c b/drivers/md/bcache/fs-io.c index 127f608aabf0..33c059c253e4 100644 --- a/drivers/md/bcache/fs-io.c +++ b/drivers/md/bcache/fs-io.c @@ -199,7 +199,8 @@ static void i_sectors_hook_fn(struct btree_insert_hook *hook, struct btree_iter *iter, struct bkey_s_c k, struct bkey_i *insert, - struct journal_res *res) + struct journal_res *res, + struct bucket_stats_cache_set *stats) { struct i_sectors_hook *h = container_of(hook, struct i_sectors_hook, hook); @@ -440,7 +441,8 @@ static void bch_put_page_reservation(struct cache_set *c, struct page *page) } } -static int bch_get_page_reservation(struct cache_set *c, struct page *page) +static int bch_get_page_reservation(struct cache_set *c, struct page *page, + bool check_enospc) { struct bch_page_state *s = page_state(page), old, new; struct disk_reservation res; @@ -449,7 +451,8 @@ static int bch_get_page_reservation(struct cache_set *c, struct page *page) if (s->alloc_state != BCH_PAGE_UNALLOCATED) return 0; - ret = bch_disk_reservation_get(c, &res, PAGE_SECTORS); + ret = __bch_disk_reservation_get(c, &res, PAGE_SECTORS, + check_enospc, false); if (ret) return ret; @@ -801,12 +804,6 @@ static void bch_writepage_io_done(struct closure *cl) struct bio_vec *bvec; unsigned i; - if (io->sectors_reserved) { - struct disk_reservation res = { .sectors = io->sectors_reserved }; - - bch_disk_reservation_put(c, &res); - } - for (i = 0; i < ARRAY_SIZE(io->i_size_update_count); i++) i_size_update_put(c, ei, i, io->i_size_update_count[i]); @@ -880,7 +877,6 @@ alloc_io: w->io->ei = ei; memset(w->io->i_size_update_count, 0, sizeof(w->io->i_size_update_count)); - w->io->sectors_reserved = 0; ret = i_sectors_dirty_get(ei, &w->io->i_sectors_hook); /* @@ -891,7 +887,8 @@ alloc_io: */ BUG_ON(ret); - bch_write_op_init(&w->io->op, w->c, &w->io->bio, NULL, + bch_write_op_init(&w->io->op, w->c, &w->io->bio, + (struct disk_reservation) { 0 }, NULL, bkey_to_s_c(&KEY(w->inum, 0, 0)), &w->io->i_sectors_hook.hook, &ei->journal_seq, 0); @@ -980,7 +977,7 @@ do_io: BUG_ON(old.alloc_state == BCH_PAGE_UNALLOCATED); if (old.alloc_state == BCH_PAGE_RESERVED) - w->io->sectors_reserved += PAGE_SECTORS; + w->io->op.res.sectors += PAGE_SECTORS; BUG_ON(PageWriteback(page)); set_page_writeback(page); @@ -1115,7 +1112,7 @@ readpage: if (ret) goto err; out: - ret = bch_get_page_reservation(c, page); + ret = bch_get_page_reservation(c, page, true); if (ret) { if (!PageUptodate(page)) { /* @@ -1386,13 +1383,19 @@ static void bch_do_direct_IO_write(struct dio_write *dio, bool sync) break; } - bch_write_op_init(&dio->iop, c, &dio->bio, NULL, + bch_write_op_init(&dio->iop, c, &dio->bio, + (struct disk_reservation) { + .sectors = bio_sectors(bio), + .gen = dio->res.gen + }, NULL, bkey_to_s_c(&KEY(inode->i_ino, bio_end_sector(bio), bio_sectors(bio))), &dio->i_sectors_hook.hook, &ei->journal_seq, flags); + dio->res.sectors -= bio_sectors(bio); + task_io_account_write(bio->bi_iter.bi_size); closure_call(&dio->iop.cl, bch_write, NULL, &dio->cl); @@ -1709,7 +1712,7 @@ int bch_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) goto out; } - if (bch_get_page_reservation(c, page)) { + if (bch_get_page_reservation(c, page, true)) { unlock_page(page); ret = VM_FAULT_SIGBUS; goto out; @@ -1833,7 +1836,6 @@ static int __bch_truncate_page(struct address_space *mapping, struct cache_set *c = inode->i_sb->s_fs_info; unsigned start_offset = start & (PAGE_SIZE - 1); unsigned end_offset = ((end - 1) & (PAGE_SIZE - 1)) + 1; - struct bch_page_state new; struct page *page; int ret = 0; @@ -1885,20 +1887,14 @@ create: goto unlock; } -#if 0 /* - * XXX: this is a hack, because we don't want truncate to fail due to - * -ENOSPC + * Bit of a hack - we don't want truncate to fail due to -ENOSPC. * - * Note that because we aren't currently tracking whether the page has - * actual data in it (vs. just 0s, or only partially written) this is - * also wrong. ick. + * XXX: because we aren't currently tracking whether the page has actual + * data in it (vs. just 0s, or only partially written) this wrong. ick. */ -#endif - page_state_cmpxchg(page_state(page), new, { - if (new.alloc_state == BCH_PAGE_UNALLOCATED) - new.alloc_state = BCH_PAGE_ALLOCATED; - }); + ret = bch_get_page_reservation(c, page, false); + BUG_ON(ret); if (index == start >> PAGE_SHIFT && index == end >> PAGE_SHIFT) @@ -2139,6 +2135,8 @@ static long bch_fcollapse(struct inode *inode, loff_t offset, loff_t len) while (bkey_cmp(dst.pos, POS(inode->i_ino, round_up(new_size, PAGE_SIZE) >> 9)) < 0) { + struct disk_reservation disk_res; + bch_btree_iter_set_pos(&src, POS(dst.pos.inode, dst.pos.offset + (len >> 9))); @@ -2163,12 +2161,20 @@ static long bch_fcollapse(struct inode *inode, loff_t offset, loff_t len) BUG_ON(bkey_cmp(dst.pos, bkey_start_pos(©.k.k))); + ret = __bch_disk_reservation_get(c, &disk_res, + copy.k.k.size, + false, false); + BUG_ON(ret); + ret = bch_btree_insert_at(&dst, &keylist_single(©.k), + &disk_res, &i_sectors_hook.hook, &ei->journal_seq, BTREE_INSERT_ATOMIC| BTREE_INSERT_NOFAIL); + bch_disk_reservation_put(c, &disk_res); + if (ret < 0 && ret != -EINTR) goto err_unwind; @@ -2281,8 +2287,7 @@ static long bch_fallocate(struct inode *inode, int mode, goto err; while (bkey_cmp(iter.pos, end) < 0) { - struct disk_reservation disk_res; - unsigned flags = 0; + struct disk_reservation disk_res = { 0 }; k = bch_btree_iter_peek_with_holes(&iter); if (!k.k) { @@ -2301,9 +2306,6 @@ static long bch_fallocate(struct inode *inode, int mode, bch_btree_iter_advance_pos(&iter); continue; } - - /* don't check for -ENOSPC if we're deleting data: */ - flags |= BTREE_INSERT_NOFAIL; } bkey_init(&reservation.k); @@ -2316,16 +2318,19 @@ static long bch_fallocate(struct inode *inode, int mode, sectors = reservation.k.size; - ret = bch_disk_reservation_get(c, &disk_res, sectors); - if (ret) - goto err_put_sectors_dirty; + if (!bkey_extent_is_allocation(k.k)) { + ret = bch_disk_reservation_get(c, &disk_res, sectors); + if (ret) + goto err_put_sectors_dirty; + } ret = bch_btree_insert_at(&iter, &keylist_single(&reservation), + &disk_res, &i_sectors_hook.hook, &ei->journal_seq, - BTREE_INSERT_ATOMIC|flags); - + BTREE_INSERT_ATOMIC| + BTREE_INSERT_NOFAIL); bch_disk_reservation_put(c, &disk_res); if (ret < 0 && ret != -EINTR) diff --git a/drivers/md/bcache/fs-io.h b/drivers/md/bcache/fs-io.h index 23d443222c08..c5681dd5cde4 100644 --- a/drivers/md/bcache/fs-io.h +++ b/drivers/md/bcache/fs-io.h @@ -48,7 +48,6 @@ struct bch_writepage_io { struct bch_inode_info *ei; unsigned long i_size_update_count[I_SIZE_UPDATE_ENTRIES]; - unsigned long sectors_reserved; struct bch_write_op op; struct i_sectors_hook i_sectors_hook; diff --git a/drivers/md/bcache/fs.c b/drivers/md/bcache/fs.c index d34384c929c4..b1b84f2280f6 100644 --- a/drivers/md/bcache/fs.c +++ b/drivers/md/bcache/fs.c @@ -134,7 +134,7 @@ int __must_check __bch_write_inode(struct cache_set *c, ret = bch_btree_insert_at(&iter, &keylist_single(&new_inode.k_i), - NULL, &ei->journal_seq, + NULL, NULL, &ei->journal_seq, BTREE_INSERT_ATOMIC| BTREE_INSERT_NOFAIL); } while (ret == -EINTR); diff --git a/drivers/md/bcache/inode.c b/drivers/md/bcache/inode.c index c9469fd115c8..58e853c03360 100644 --- a/drivers/md/bcache/inode.c +++ b/drivers/md/bcache/inode.c @@ -135,7 +135,7 @@ again: inode->k.p.inode, inode->k.u64s); ret = bch_btree_insert_at(&iter, &keylist_single(inode), - NULL, NULL, + NULL, NULL, NULL, BTREE_INSERT_ATOMIC); if (ret == -EINTR) @@ -206,7 +206,8 @@ int bch_inode_rm(struct cache_set *c, u64 inode_nr) return bch_btree_insert(c, BTREE_ID_INODES, &keylist_single(&delete), - NULL, NULL, BTREE_INSERT_NOFAIL); + NULL, NULL, NULL, + BTREE_INSERT_NOFAIL); } int bch_inode_update(struct cache_set *c, struct bkey_i *inode, diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c index eb6df9d9f86c..6b240f4446c1 100644 --- a/drivers/md/bcache/io.c +++ b/drivers/md/bcache/io.c @@ -566,6 +566,7 @@ static void bch_write_done(struct closure *cl) if (!op->error && (op->flags & BCH_WRITE_FLUSH)) op->error = bch_journal_error(&op->c->journal); + bch_disk_reservation_put(op->c, &op->res); percpu_ref_put(&op->c->writes); bch_keylist_free(&op->insert_keys); closure_return(cl); @@ -592,9 +593,12 @@ static void bch_write_index(struct closure *cl) u64 sectors_start = keylist_sectors(&op->insert_keys); int ret; - ret = bch_btree_insert(op->c, BTREE_ID_EXTENTS, &op->insert_keys, + ret = bch_btree_insert(op->c, BTREE_ID_EXTENTS, + &op->insert_keys, + &op->res, op->insert_hook, - op_journal_seq(op), BTREE_INSERT_NOFAIL); + op_journal_seq(op), + BTREE_INSERT_NOFAIL); op->written += sectors_start - keylist_sectors(&op->insert_keys); @@ -1132,12 +1136,14 @@ void bch_write(struct closure *cl) if (!bio_sectors(bio)) { WARN_ONCE(1, "bch_write() called with empty bio"); + bch_disk_reservation_put(op->c, &op->res); closure_return(cl); } if (!percpu_ref_tryget(&c->writes)) { __bcache_io_error(c, "read only"); op->error = -EROFS; + bch_disk_reservation_put(op->c, &op->res); closure_return(cl); } @@ -1206,8 +1212,8 @@ void bch_write(struct closure *cl) } void bch_write_op_init(struct bch_write_op *op, struct cache_set *c, - struct bch_write_bio *bio, struct write_point *wp, - struct bkey_s_c insert_key, + struct bch_write_bio *bio, struct disk_reservation res, + struct write_point *wp, struct bkey_s_c insert_key, struct btree_insert_hook *insert_hook, u64 *journal_seq, unsigned flags) { @@ -1227,6 +1233,7 @@ void bch_write_op_init(struct bch_write_op *op, struct cache_set *c, op->flags = flags; op->compression_type = c->opts.compression; op->nr_replicas = c->opts.data_replicas; + op->res = res; op->wp = wp; if (journal_seq) { @@ -1721,6 +1728,7 @@ void bch_read_extent_iter(struct cache_set *c, struct bio *orig, bch_write_op_init(&promote_op->iop, c, &promote_op->bio, + (struct disk_reservation) { 0 }, &c->promote_write_point, k, NULL, NULL, BCH_WRITE_ALLOC_NOWAIT); diff --git a/drivers/md/bcache/io.h b/drivers/md/bcache/io.h index 8fb4c2b36acd..147ccd969586 100644 --- a/drivers/md/bcache/io.h +++ b/drivers/md/bcache/io.h @@ -29,7 +29,8 @@ enum bch_write_flags { }; void bch_write_op_init(struct bch_write_op *, struct cache_set *, - struct bch_write_bio *, struct write_point *, + struct bch_write_bio *, + struct disk_reservation, struct write_point *, struct bkey_s_c, struct btree_insert_hook *, u64 *, unsigned); void bch_write(struct closure *); diff --git a/drivers/md/bcache/io_types.h b/drivers/md/bcache/io_types.h index 064e2c7d8cdc..f0a58401295a 100644 --- a/drivers/md/bcache/io_types.h +++ b/drivers/md/bcache/io_types.h @@ -2,6 +2,7 @@ #define _BCACHE_IO_TYPES_H #include "btree_types.h" +#include "buckets_types.h" #include "keylist_types.h" #include <linux/llist.h> @@ -82,6 +83,8 @@ struct bch_write_op { unsigned compression_type:4; unsigned nr_replicas:4; + struct disk_reservation res; + struct write_point *wp; union { diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c index a877bb1e169c..742ccf7ad3f3 100644 --- a/drivers/md/bcache/journal.c +++ b/drivers/md/bcache/journal.c @@ -777,7 +777,8 @@ const char *bch_journal_read(struct cache_set *c, struct list_head *list) return NULL; } -void bch_journal_mark(struct cache_set *c, struct list_head *list) +void bch_journal_mark(struct cache_set *c, struct list_head *list, + struct bucket_stats_cache_set *stats) { struct bkey_i *k, *n; struct jset_entry *j; @@ -790,7 +791,8 @@ void bch_journal_mark(struct cache_set *c, struct list_head *list) if (btree_type_has_ptrs(type) && !bkey_invalid(c, type, bkey_i_to_s_c(k))) __bch_btree_mark_key_initial(c, type, - bkey_i_to_s_c(k)); + bkey_i_to_s_c(k), + stats); } } @@ -955,36 +957,6 @@ void bch_journal_start(struct cache_set *c) queue_work(system_long_wq, &j->reclaim_work); } -static int bch_journal_replay_key(struct cache_set *c, enum btree_id id, - struct bkey_i *k) -{ - int ret; - BKEY_PADDED(key) temp; - bool do_subtract = id == BTREE_ID_EXTENTS && bkey_extent_is_data(&k->k); - - trace_bcache_journal_replay_key(&k->k); - - if (do_subtract) - bkey_copy(&temp.key, k); - - ret = bch_btree_insert(c, id, &keylist_single(k), NULL, NULL, - BTREE_INSERT_NOFAIL| - BTREE_INSERT_NOFAIL_IF_STALE); - if (ret) - return ret; - - /* - * Subtract sectors after replay since bch_btree_insert() added - * them again - */ - if (do_subtract) - bch_mark_pointers(c, bkey_i_to_s_c_extent(&temp.key), - -temp.key.k.size, false, false, - true, GC_POS_MIN); - - return 0; -} - int bch_journal_replay(struct cache_set *c, struct list_head *list) { int ret = 0, keys = 0, entries = 0; @@ -1025,11 +997,17 @@ int bch_journal_replay(struct cache_set *c, struct list_head *list) BUG_ON(atomic_read(&j->cur_pin_list->count) != 1); for_each_jset_key(k, _n, jkeys, &i->j) { - cond_resched(); - ret = bch_journal_replay_key(c, jkeys->btree_id, k); + trace_bcache_journal_replay_key(&k->k); + + ret = bch_btree_insert(c, jkeys->btree_id, + &keylist_single(k), + NULL, NULL, NULL, + BTREE_INSERT_NOFAIL| + BTREE_INSERT_NO_MARK_KEY); if (ret) goto err; + cond_resched(); keys++; } diff --git a/drivers/md/bcache/journal.h b/drivers/md/bcache/journal.h index 180019a571e0..ec9acbc8f8cc 100644 --- a/drivers/md/bcache/journal.h +++ b/drivers/md/bcache/journal.h @@ -231,7 +231,8 @@ static inline bool journal_res_full(struct journal_res *res, } void bch_journal_start(struct cache_set *); -void bch_journal_mark(struct cache_set *, struct list_head *); +void bch_journal_mark(struct cache_set *, struct list_head *, + struct bucket_stats_cache_set *); const char *bch_journal_read(struct cache_set *, struct list_head *); int bch_journal_replay(struct cache_set *, struct list_head *); diff --git a/drivers/md/bcache/migrate.c b/drivers/md/bcache/migrate.c index 31dfb5f4dcde..39e212a1c3b5 100644 --- a/drivers/md/bcache/migrate.c +++ b/drivers/md/bcache/migrate.c @@ -50,13 +50,11 @@ static int issue_migration_move(struct cache *ca, return -ENOMEM; } - io->disk_res = res; - /* This also copies k into the write op's replace_key and insert_key */ bch_replace_init(&io->replace, k); - bch_write_op_init(&io->op, c, &io->bio, + bch_write_op_init(&io->op, c, &io->bio, res, &c->migration_write_point, k, &io->replace.hook, NULL, 0); @@ -385,7 +383,8 @@ static int bch_flag_key_bad(struct btree_iter *iter, bch_extent_normalize(c, e.s); return bch_btree_insert_at(iter, &keylist_single(&tmp.key), - NULL, NULL, BTREE_INSERT_ATOMIC); + NULL, NULL, NULL, + BTREE_INSERT_ATOMIC); } /* diff --git a/drivers/md/bcache/move.c b/drivers/md/bcache/move.c index 32cd255b7fc3..4cf573cceea3 100644 --- a/drivers/md/bcache/move.c +++ b/drivers/md/bcache/move.c @@ -112,7 +112,6 @@ struct moving_io *moving_io_alloc(struct bkey_s_c k) void moving_io_free(struct moving_io *io) { - bch_disk_reservation_put(io->op.c, &io->disk_res); bch_bio_free_pages(&io->bio.bio.bio); kfree(io); } diff --git a/drivers/md/bcache/move.h b/drivers/md/bcache/move.h index fe545a6aec9b..8b311a8b3676 100644 --- a/drivers/md/bcache/move.h +++ b/drivers/md/bcache/move.h @@ -92,7 +92,6 @@ struct moving_io { unsigned read_issued:1; unsigned read_completed:1; unsigned write_issued:1; - struct disk_reservation disk_res; /* Must be last since it is variable size */ struct bch_write_bio bio; diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c index 475cfa392a70..f1a1aa3e49ea 100644 --- a/drivers/md/bcache/movinggc.c +++ b/drivers/md/bcache/movinggc.c @@ -60,8 +60,9 @@ static int issue_moving_gc_move(struct moving_queue *q, bch_replace_init(&io->replace, bkey_i_to_s_c(k)); - bch_write_op_init(&io->op, c, &io->bio, NULL, - bkey_i_to_s_c(k), + bch_write_op_init(&io->op, c, &io->bio, + (struct disk_reservation) { 0 }, + NULL, bkey_i_to_s_c(k), &io->replace.hook, NULL, bkey_extent_is_cached(&k->k) ? BCH_WRITE_CACHED : 0); diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 6f366e29f910..10732c4a8f52 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -556,7 +556,8 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s) if (bypass) flags |= BCH_WRITE_DISCARD; - bch_write_op_init(&s->iop, dc->disk.c, &s->bio, NULL, + bch_write_op_init(&s->iop, dc->disk.c, &s->bio, + (struct disk_reservation) { 0 }, NULL, bkey_to_s_c(&insert_key), NULL, NULL, flags); @@ -681,8 +682,16 @@ static void __blockdev_volume_make_request(struct request_queue *q, continue_at(&s->cl, search_free, NULL); } else if (rw) { + struct disk_reservation res = { 0 }; unsigned flags = 0; + if (bio_op(bio) != REQ_OP_DISCARD && + bch_disk_reservation_get(d->c, &res, bio_sectors(bio))) { + s->iop.error = -ENOSPC; + continue_at(&s->cl, search_free, NULL); + return; + } + if (bio->bi_opf & (REQ_PREFLUSH|REQ_FUA)) flags |= BCH_WRITE_FLUSH; if (bio_op(bio) == REQ_OP_DISCARD) @@ -690,7 +699,7 @@ static void __blockdev_volume_make_request(struct request_queue *q, s = search_alloc(bio, d); - bch_write_op_init(&s->iop, d->c, &s->bio, NULL, + bch_write_op_init(&s->iop, d->c, &s->bio, res, NULL, bkey_to_s_c(&KEY(s->inode, 0, 0)), NULL, NULL, flags); diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 724ca30c9d0a..c44f44c44712 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1376,7 +1376,7 @@ static const char *run_cache_set(struct cache_set *c) err = "error creating root directory"; if (bch_btree_insert(c, BTREE_ID_INODES, &keylist_single(&inode.k_i), - NULL, NULL, 0)) + NULL, NULL, NULL, 0)) goto err; err = "error writing first journal entry"; diff --git a/drivers/md/bcache/tier.c b/drivers/md/bcache/tier.c index 29b7d4db320a..20cbbf9295ed 100644 --- a/drivers/md/bcache/tier.c +++ b/drivers/md/bcache/tier.c @@ -223,6 +223,7 @@ static int issue_tiering_move(struct moving_queue *q, bch_replace_init(&io->replace, bkey_i_to_s_c(&io->key)); bch_write_op_init(&io->op, c, &io->bio, + (struct disk_reservation) { 0 }, &ca->tiering_write_point, bkey_i_to_s_c(&io->key), &io->replace.hook, NULL, 0); diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 1a0424961a40..53ceb56a6a38 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -120,7 +120,7 @@ static void write_dirty_finish(struct closure *cl) ret = bch_btree_insert(dc->disk.c, BTREE_ID_EXTENTS, &keylist_single(&tmp.k), - &io->replace.hook, NULL, 0); + NULL, &io->replace.hook, NULL, 0); if (io->replace.successes == 0) trace_bcache_writeback_collision(&io->replace.key.k); diff --git a/drivers/md/bcache/xattr.c b/drivers/md/bcache/xattr.c index dca982563c93..297ef230c4cc 100644 --- a/drivers/md/bcache/xattr.c +++ b/drivers/md/bcache/xattr.c @@ -225,7 +225,7 @@ int bch_xattr_set(struct inode *inode, const char *name, bch_keylist_enqueue(&keys); - ret = bch_btree_insert_at(&iter, &keys, NULL, + ret = bch_btree_insert_at(&iter, &keys, NULL, NULL, &ei->journal_seq, insert_flags); bch_keylist_free(&keys); @@ -238,7 +238,7 @@ int bch_xattr_set(struct inode *inode, const char *name, ret = bch_btree_insert_at(&iter, &keylist_single(&whiteout), - NULL, &ei->journal_seq, + NULL, NULL, &ei->journal_seq, insert_flags); } |