summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--fs/bcachefs/Makefile1
-rw-r--r--fs/bcachefs/bcachefs.h4
-rw-r--r--fs/bcachefs/bcachefs_format.h1
-rw-r--r--fs/bcachefs/btree_iter.c5
-rw-r--r--fs/bcachefs/btree_types.h3
-rw-r--r--fs/bcachefs/btree_update.h9
-rw-r--r--fs/bcachefs/btree_update_leaf.c93
-rw-r--r--fs/bcachefs/btree_write_buffer.c233
-rw-r--r--fs/bcachefs/btree_write_buffer.h13
-rw-r--r--fs/bcachefs/btree_write_buffer_types.h26
-rw-r--r--fs/bcachefs/buckets.c41
-rw-r--r--fs/bcachefs/buckets.h1
-rw-r--r--fs/bcachefs/errcode.h2
-rw-r--r--fs/bcachefs/opts.h5
-rw-r--r--fs/bcachefs/super.c3
15 files changed, 433 insertions, 7 deletions
diff --git a/fs/bcachefs/Makefile b/fs/bcachefs/Makefile
index d4203af14bba..ffbfc60ae6d0 100644
--- a/fs/bcachefs/Makefile
+++ b/fs/bcachefs/Makefile
@@ -16,6 +16,7 @@ bcachefs-y := \
btree_locking.o \
btree_update_interior.o \
btree_update_leaf.o \
+ btree_write_buffer.o \
buckets.o \
buckets_waiting_for_journal.o \
chardev.o \
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 5a7f68039933..77a797ca971a 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -387,6 +387,7 @@ enum bch_time_stats {
#include "alloc_types.h"
#include "btree_types.h"
+#include "btree_write_buffer_types.h"
#include "buckets_types.h"
#include "buckets_waiting_for_journal_types.h"
#include "clock_types.h"
@@ -569,6 +570,7 @@ struct btree_transaction_stats {
struct mutex lock;
struct bch2_time_stats lock_hold_times;
unsigned nr_max_paths;
+ unsigned wb_updates_size;
unsigned max_mem;
char *max_paths_text;
};
@@ -756,6 +758,8 @@ struct bch_fs {
struct btree_key_cache btree_key_cache;
unsigned btree_key_cache_btrees;
+ struct btree_write_buffer btree_write_buffer;
+
struct workqueue_struct *btree_update_wq;
struct workqueue_struct *btree_io_complete_wq;
/* copygc needs its own workqueue for index updates.. */
diff --git a/fs/bcachefs/bcachefs_format.h b/fs/bcachefs/bcachefs_format.h
index fc86c47dad72..34d6a1a9a0c1 100644
--- a/fs/bcachefs/bcachefs_format.h
+++ b/fs/bcachefs/bcachefs_format.h
@@ -1629,6 +1629,7 @@ LE64_BITMASK(BCH_SB_JOURNAL_FLUSH_DELAY,struct bch_sb, flags[3], 30, 62);
LE64_BITMASK(BCH_SB_JOURNAL_FLUSH_DISABLED,struct bch_sb, flags[3], 62, 63);
LE64_BITMASK(BCH_SB_JOURNAL_RECLAIM_DELAY,struct bch_sb, flags[4], 0, 32);
LE64_BITMASK(BCH_SB_JOURNAL_TRANSACTION_NAMES,struct bch_sb, flags[4], 32, 33);
+LE64_BITMASK(BCH_SB_WRITE_BUFFER_SIZE, struct bch_sb, flags[4], 34, 54);
/*
* Features:
diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index 5cdfd0d44000..6682acccc820 100644
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -2909,8 +2909,11 @@ void __bch2_trans_init(struct btree_trans *trans, struct bch_fs *c, unsigned fn_
trans->mem_bytes = expected_mem_bytes;
}
}
- if (s)
+
+ if (s) {
trans->nr_max_paths = s->nr_max_paths;
+ trans->wb_updates_size = s->wb_updates_size;
+ }
trans->srcu_idx = srcu_read_lock(&c->btree_trans_barrier);
trans->srcu_lock_time = jiffies;
diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h
index 5feb9eafad88..a3b2fbbd5075 100644
--- a/fs/bcachefs/btree_types.h
+++ b/fs/bcachefs/btree_types.h
@@ -419,6 +419,8 @@ struct btree_trans {
u8 fn_idx;
u8 nr_sorted;
u8 nr_updates;
+ u8 nr_wb_updates;
+ u8 wb_updates_size;
bool used_mempool:1;
bool in_traverse_all:1;
bool paths_sorted:1;
@@ -448,6 +450,7 @@ struct btree_trans {
u8 sorted[BTREE_ITER_MAX + 8];
struct btree_path *paths;
struct btree_insert_entry *updates;
+ struct btree_write_buffered_key *wb_updates;
/* update path: */
struct btree_trans_commit_hook *hooks;
diff --git a/fs/bcachefs/btree_update.h b/fs/bcachefs/btree_update.h
index 9a3c859ea572..c55458a0491d 100644
--- a/fs/bcachefs/btree_update.h
+++ b/fs/bcachefs/btree_update.h
@@ -77,6 +77,8 @@ int bch2_trans_update_extent(struct btree_trans *, struct btree_iter *,
int __must_check bch2_trans_update(struct btree_trans *, struct btree_iter *,
struct bkey_i *, enum btree_update_flags);
+int __must_check bch2_trans_update_buffered(struct btree_trans *,
+ enum btree_id, struct bkey_i *);
void bch2_trans_commit_hook(struct btree_trans *,
struct btree_trans_commit_hook *);
@@ -143,6 +145,11 @@ static inline int bch2_trans_commit(struct btree_trans *trans,
(_i) < (_trans)->updates + (_trans)->nr_updates; \
(_i)++)
+#define trans_for_each_wb_update(_trans, _i) \
+ for ((_i) = (_trans)->wb_updates; \
+ (_i) < (_trans)->wb_updates + (_trans)->nr_wb_updates; \
+ (_i)++)
+
static inline void bch2_trans_reset_updates(struct btree_trans *trans)
{
struct btree_insert_entry *i;
@@ -152,6 +159,8 @@ static inline void bch2_trans_reset_updates(struct btree_trans *trans)
trans->extra_journal_res = 0;
trans->nr_updates = 0;
+ trans->nr_wb_updates = 0;
+ trans->wb_updates = NULL;
trans->hooks = NULL;
trans->extra_journal_entries.nr = 0;
diff --git a/fs/bcachefs/btree_update_leaf.c b/fs/bcachefs/btree_update_leaf.c
index 867d92945cc9..07a18b623475 100644
--- a/fs/bcachefs/btree_update_leaf.c
+++ b/fs/bcachefs/btree_update_leaf.c
@@ -8,6 +8,7 @@
#include "btree_iter.h"
#include "btree_key_cache.h"
#include "btree_locking.h"
+#include "btree_write_buffer.h"
#include "buckets.h"
#include "debug.h"
#include "errcode.h"
@@ -595,6 +596,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
{
struct bch_fs *c = trans->c;
struct btree_insert_entry *i;
+ struct btree_write_buffered_key *wb;
struct btree_trans_commit_hook *h;
unsigned u64s = 0;
bool marking = false;
@@ -674,17 +676,23 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
bch2_trans_fs_usage_apply(trans, trans->fs_usage_deltas))
return -BCH_ERR_btree_insert_need_mark_replicas;
+ if (trans->nr_wb_updates) {
+ ret = bch2_btree_insert_keys_write_buffer(trans);
+ if (ret)
+ goto revert_fs_usage;
+ }
+
trans_for_each_update(trans, i)
if (BTREE_NODE_TYPE_HAS_MEM_TRIGGERS & (1U << i->bkey_type)) {
ret = run_one_mem_trigger(trans, i, i->flags);
if (ret)
- return ret;
+ goto fatal_err;
}
if (unlikely(c->gc_pos.phase)) {
ret = bch2_trans_commit_run_gc_triggers(trans);
if (ret)
- return ret;
+ goto fatal_err;
}
if (unlikely(trans->extra_journal_entries.nr)) {
@@ -697,10 +705,10 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
}
if (likely(!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))) {
- trans_for_each_update(trans, i) {
- struct journal *j = &c->journal;
- struct jset_entry *entry;
+ struct journal *j = &c->journal;
+ struct jset_entry *entry;
+ trans_for_each_update(trans, i) {
if (i->key_cache_already_flushed)
continue;
@@ -725,6 +733,14 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
bkey_copy(&entry->start[0], i->k);
}
+ trans_for_each_wb_update(trans, wb) {
+ entry = bch2_journal_add_entry(j, &trans->journal_res,
+ BCH_JSET_ENTRY_btree_keys,
+ wb->btree, 0,
+ wb->k.k.u64s);
+ bkey_copy(&entry->start[0], &wb->k);
+ }
+
if (trans->journal_seq)
*trans->journal_seq = trans->journal_res.seq;
}
@@ -742,6 +758,12 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
}
}
+ return 0;
+fatal_err:
+ bch2_fatal_error(c);
+revert_fs_usage:
+ if (trans->fs_usage_deltas)
+ bch2_trans_fs_usage_revert(trans, trans->fs_usage_deltas);
return ret;
}
@@ -778,9 +800,13 @@ static inline int trans_lock_write(struct btree_trans *trans)
static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans)
{
struct btree_insert_entry *i;
+ struct btree_write_buffered_key *wb;
trans_for_each_update(trans, i)
bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p);
+
+ trans_for_each_wb_update(trans, wb)
+ bch2_journal_key_overwritten(trans->c, wb->btree, 0, wb->k.k.p);
}
#ifdef CONFIG_BCACHEFS_DEBUG
@@ -833,8 +859,8 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans,
return bch2_trans_commit_bkey_invalid(trans, i, &buf);
btree_insert_entry_checks(trans, i);
}
-#endif
printbuf_exit(&buf);
+#endif
trans_for_each_update(trans, i) {
if (i->cached)
@@ -969,6 +995,11 @@ int bch2_trans_commit_error(struct btree_trans *trans,
if (ret)
trace_and_count(c, trans_restart_journal_reclaim, trans, trace_ip);
break;
+ case -BCH_ERR_btree_insert_need_flush_buffer:
+ bch2_trans_reset_updates(trans);
+ ret = __bch2_btree_write_buffer_flush(trans, trans->flags) ?:
+ btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush);
+ break;
default:
BUG_ON(ret >= 0);
break;
@@ -1030,6 +1061,7 @@ int __bch2_trans_commit(struct btree_trans *trans)
{
struct bch_fs *c = trans->c;
struct btree_insert_entry *i = NULL;
+ struct btree_write_buffered_key *wb;
unsigned u64s;
int ret = 0;
@@ -1096,6 +1128,13 @@ int __bch2_trans_commit(struct btree_trans *trans)
trans->journal_u64s += jset_u64s(i->old_k.u64s);
}
+ trans_for_each_wb_update(trans, wb) {
+ u64s = jset_u64s(wb->k.k.u64s);
+
+ trans->journal_preres_u64s += u64s;
+ trans->journal_u64s += u64s;
+ }
+
if (trans->extra_journal_res) {
ret = bch2_disk_reservation_add(c, trans->disk_res,
trans->extra_journal_res,
@@ -1613,6 +1652,48 @@ int __must_check bch2_trans_update(struct btree_trans *trans, struct btree_iter
return bch2_trans_update_by_path(trans, path, k, flags);
}
+int __must_check bch2_trans_update_buffered(struct btree_trans *trans,
+ enum btree_id btree,
+ struct bkey_i *k)
+{
+ int ret;
+
+ BUG_ON(trans->nr_wb_updates > trans->wb_updates_size);
+
+ if (!trans->wb_updates ||
+ trans->nr_wb_updates == trans->wb_updates_size) {
+ struct btree_write_buffered_key *u;
+
+ if (trans->nr_wb_updates == trans->wb_updates_size) {
+ struct btree_transaction_stats *s = btree_trans_stats(trans);
+
+ BUG_ON(trans->wb_updates_size > U8_MAX / 2);
+ trans->wb_updates_size = max(1, trans->wb_updates_size * 2);
+ if (s)
+ s->wb_updates_size = trans->wb_updates_size;
+ }
+
+ u = bch2_trans_kmalloc_nomemzero(trans,
+ trans->wb_updates_size *
+ sizeof(struct btree_write_buffered_key));
+ ret = PTR_ERR_OR_ZERO(u);
+ if (ret)
+ return ret;
+
+ if (trans->nr_wb_updates)
+ memcpy(u, trans->wb_updates, trans->nr_wb_updates *
+ sizeof(struct btree_write_buffered_key));
+ trans->wb_updates = u;
+ }
+
+ trans->wb_updates[trans->nr_wb_updates++] = (struct btree_write_buffered_key) {
+ .btree = btree,
+ .k = *k,
+ };
+
+ return 0;
+}
+
void bch2_trans_commit_hook(struct btree_trans *trans,
struct btree_trans_commit_hook *h)
{
diff --git a/fs/bcachefs/btree_write_buffer.c b/fs/bcachefs/btree_write_buffer.c
new file mode 100644
index 000000000000..b5a419e1960a
--- /dev/null
+++ b/fs/bcachefs/btree_write_buffer.c
@@ -0,0 +1,233 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "bcachefs.h"
+#include "btree_update.h"
+#include "btree_write_buffer.h"
+#include "error.h"
+#include "journal.h"
+#include "journal_reclaim.h"
+
+#include <linux/sort.h>
+
+static int btree_write_buffered_key_cmp(const void *_l, const void *_r)
+{
+ const struct btree_write_buffered_key *l = _l;
+ const struct btree_write_buffered_key *r = _r;
+
+ return cmp_int(l->btree, r->btree) ?:
+ bpos_cmp(l->k.k.p, r->k.k.p) ?:
+ cmp_int(l->journal_seq, r->journal_seq) ?:
+ cmp_int(l->journal_offset, r->journal_offset);
+}
+
+static int btree_write_buffered_journal_cmp(const void *_l, const void *_r)
+{
+ const struct btree_write_buffered_key *l = _l;
+ const struct btree_write_buffered_key *r = _r;
+
+ return cmp_int(l->journal_seq, r->journal_seq);
+}
+
+int __bch2_btree_write_buffer_flush(struct btree_trans *trans, unsigned commit_flags)
+{
+ struct bch_fs *c = trans->c;
+ struct journal *j = &c->journal;
+ struct btree_write_buffer *wb = &c->btree_write_buffer;
+ struct journal_entry_pin pin;
+ struct journal_preres res = { 0 };
+ struct btree_write_buffered_key *i, *dst;
+ size_t nr = 0;
+ int ret = 0;
+
+ memset(&pin, 0, sizeof(pin));
+
+ if (!mutex_trylock(&wb->flush_lock))
+ return 0;
+
+ mutex_lock(&wb->lock);
+ swap(wb->keys, wb->flushing);
+ swap(wb->nr, nr);
+ swap(wb->res, res);
+
+ bch2_journal_pin_copy(j, &pin, &wb->journal_pin, NULL);
+ bch2_journal_pin_drop(j, &wb->journal_pin);
+ mutex_unlock(&wb->lock);
+
+ /*
+ * We first sort so that we can detect and skip redundant updates, and
+ * then we attempt to flush in sorted btree order, as this is most
+ * efficient.
+ *
+ * However, since we're not flushing in the order they appear in the
+ * journal we won't be able to drop our journal pin until everything is
+ * flushed - which means this could deadlock the journal, if we weren't
+ * passing BTREE_INSERT_JORUNAL_RECLAIM. This causes the update to fail
+ * if it would block taking a journal reservation.
+ *
+ * If that happens, we sort them by the order they appeared in the
+ * journal - after dropping redundant entries - and then restart
+ * flushing, this time dropping journal pins as we go.
+ */
+
+ sort(wb->flushing, nr,
+ sizeof(wb->flushing[0]),
+ btree_write_buffered_key_cmp,
+ NULL);
+
+ for (i = wb->flushing;
+ i < wb->flushing + nr;
+ i++) {
+ if (i + 1 < wb->flushing + nr &&
+ i[0].btree == i[1].btree &&
+ bpos_eq(i[0].k.k.p, i[1].k.k.p)) {
+ if (bkey_deleted(&i[1].k.k))
+ i++;
+ continue;
+ }
+
+ ret = commit_do(trans, NULL, NULL,
+ commit_flags|
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_JOURNAL_RECLAIM,
+ __bch2_btree_insert(trans, i->btree, &i->k));
+ if (ret == -BCH_ERR_journal_reclaim_would_deadlock)
+ goto slowpath;
+ if (bch2_fs_fatal_err_on(ret, c, "%s: insert error %s", __func__, bch2_err_str(ret)))
+ break;
+ }
+out:
+ bch2_journal_pin_drop(j, &pin);
+ bch2_journal_preres_put(j, &res);
+ mutex_unlock(&wb->flush_lock);
+ return ret;
+slowpath:
+ dst = wb->flushing;
+ for (;
+ i < wb->flushing + nr;
+ i++) {
+ if (i + 1 < wb->flushing + nr &&
+ i[0].btree == i[1].btree &&
+ bpos_eq(i[0].k.k.p, i[1].k.k.p)) {
+ if (bkey_deleted(&i[1].k.k))
+ i++;
+ continue;
+ }
+
+
+ *dst = *i;
+ dst++;
+ }
+ nr = dst - wb->flushing;
+
+ sort(wb->flushing, nr,
+ sizeof(wb->flushing[0]),
+ btree_write_buffered_journal_cmp,
+ NULL);
+
+ for (i = wb->flushing;
+ i < wb->flushing + nr;
+ i++) {
+ if (i->journal_seq > pin.seq) {
+ struct journal_entry_pin pin2;
+
+ memset(&pin2, 0, sizeof(pin2));
+
+ bch2_journal_pin_add(j, i->journal_seq, &pin2, NULL);
+ bch2_journal_pin_drop(j, &pin);
+ bch2_journal_pin_copy(j, &pin, &pin2, NULL);
+ bch2_journal_pin_drop(j, &pin2);
+ }
+
+ ret = commit_do(trans, NULL, NULL,
+ commit_flags|
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_JOURNAL_RECLAIM|
+ JOURNAL_WATERMARK_reserved,
+ __bch2_btree_insert(trans, i->btree, &i->k));
+ if (bch2_fs_fatal_err_on(ret, c, "%s: insert error %s", __func__, bch2_err_str(ret)))
+ break;
+ }
+
+ goto out;
+}
+
+int bch2_btree_write_buffer_flush(struct btree_trans *trans)
+{
+ return __bch2_btree_write_buffer_flush(trans, 0);
+}
+
+static int bch2_btree_write_buffer_journal_flush(struct journal *j,
+ struct journal_entry_pin *_pin, u64 seq)
+{
+ struct bch_fs *c = container_of(j, struct bch_fs, journal);
+
+ return bch2_trans_run(c,
+ __bch2_btree_write_buffer_flush(&trans, BTREE_INSERT_NOCHECK_RW));
+}
+
+int bch2_btree_insert_keys_write_buffer(struct btree_trans *trans)
+{
+ struct bch_fs *c = trans->c;
+ struct btree_write_buffer *wb = &c->btree_write_buffer;
+ struct btree_write_buffered_key *i;
+ unsigned u64s = 0;
+
+ BUG_ON(trans->flags & BTREE_INSERT_JOURNAL_REPLAY);
+
+ mutex_lock(&wb->lock);
+ if (wb->nr + trans->nr_wb_updates > wb->size) {
+ mutex_unlock(&wb->lock);
+ return -BCH_ERR_btree_insert_need_flush_buffer;
+ }
+
+ trans_for_each_wb_update(trans, i) {
+ BUG_ON(i->k.k.u64s != BKEY_U64s);
+
+ i->journal_seq = trans->journal_res.seq;
+ i->journal_offset = trans->journal_res.offset;
+
+ u64s += jset_u64s(i->k.k.u64s);
+ }
+
+ memcpy(wb->keys + wb->nr,
+ trans->wb_updates,
+ sizeof(trans->wb_updates[0]) * trans->nr_wb_updates);
+ wb->nr += trans->nr_wb_updates;
+
+ if (likely(!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))) {
+ BUG_ON(u64s > trans->journal_preres.u64s);
+
+ trans->journal_preres.u64s -= u64s;
+ wb->res.u64s += u64s;
+
+ bch2_journal_pin_add(&c->journal, trans->journal_res.seq, &wb->journal_pin,
+ bch2_btree_write_buffer_journal_flush);
+ }
+ mutex_unlock(&wb->lock);
+
+ return 0;
+}
+
+void bch2_fs_btree_write_buffer_exit(struct bch_fs *c)
+{
+ struct btree_write_buffer *wb = &c->btree_write_buffer;
+
+ kvfree(wb->flushing);
+ kvfree(wb->keys);
+}
+
+int bch2_fs_btree_write_buffer_init(struct bch_fs *c)
+{
+ struct btree_write_buffer *wb = &c->btree_write_buffer;
+
+ mutex_init(&wb->lock);
+ mutex_init(&wb->flush_lock);
+ wb->size = c->opts.btree_write_buffer_size;
+
+ wb->keys = kvmalloc_array(wb->size, sizeof(wb->keys[0]), GFP_KERNEL);
+ wb->flushing = kvmalloc_array(wb->size, sizeof(wb->keys[0]), GFP_KERNEL);
+ if (!wb->keys || !wb->flushing)
+ return -ENOMEM;
+
+ return 0;
+}
diff --git a/fs/bcachefs/btree_write_buffer.h b/fs/bcachefs/btree_write_buffer.h
new file mode 100644
index 000000000000..ffac314d61a5
--- /dev/null
+++ b/fs/bcachefs/btree_write_buffer.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _BCACHEFS_BTREE_WRITE_BUFFER_H
+#define _BCACHEFS_BTREE_WRITE_BUFFER_H
+
+int __bch2_btree_write_buffer_flush(struct btree_trans *, unsigned);
+int bch2_btree_write_buffer_flush(struct btree_trans *);
+
+int bch2_btree_insert_keys_write_buffer(struct btree_trans *);
+
+void bch2_fs_btree_write_buffer_exit(struct bch_fs *);
+int bch2_fs_btree_write_buffer_init(struct bch_fs *);
+
+#endif /* _BCACHEFS_BTREE_WRITE_BUFFER_H */
diff --git a/fs/bcachefs/btree_write_buffer_types.h b/fs/bcachefs/btree_write_buffer_types.h
new file mode 100644
index 000000000000..a6a9489afce9
--- /dev/null
+++ b/fs/bcachefs/btree_write_buffer_types.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H
+#define _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H
+
+#include "journal_types.h"
+
+struct btree_write_buffered_key {
+ u64 journal_seq;
+ unsigned journal_offset;
+ enum btree_id btree;
+ struct bkey_i k;
+};
+
+struct btree_write_buffer {
+ struct mutex lock;
+ struct mutex flush_lock;
+ struct journal_entry_pin journal_pin;
+ struct journal_preres res;
+
+ struct btree_write_buffered_key *keys;
+ struct btree_write_buffered_key *flushing;
+ size_t nr;
+ size_t size;
+};
+
+#endif /* _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H */
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index f17aff9397db..f02526969ba6 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -1278,6 +1278,47 @@ int bch2_mark_reflink_p(struct btree_trans *trans,
return ret;
}
+void bch2_trans_fs_usage_revert(struct btree_trans *trans,
+ struct replicas_delta_list *deltas)
+{
+ struct bch_fs *c = trans->c;
+ struct bch_fs_usage *dst;
+ struct replicas_delta *d, *top = (void *) deltas->d + deltas->used;
+ s64 added = 0;
+ unsigned i;
+
+ percpu_down_read(&c->mark_lock);
+ preempt_disable();
+ dst = fs_usage_ptr(c, trans->journal_res.seq, false);
+
+ /* revert changes: */
+ for (d = deltas->d; d != top; d = replicas_delta_next(d)) {
+ switch (d->r.data_type) {
+ case BCH_DATA_btree:
+ case BCH_DATA_user:
+ case BCH_DATA_parity:
+ added += d->delta;
+ }
+ BUG_ON(__update_replicas(c, dst, &d->r, -d->delta));
+ }
+
+ dst->nr_inodes -= deltas->nr_inodes;
+
+ for (i = 0; i < BCH_REPLICAS_MAX; i++) {
+ added -= deltas->persistent_reserved[i];
+ dst->reserved -= deltas->persistent_reserved[i];
+ dst->persistent_reserved[i] -= deltas->persistent_reserved[i];
+ }
+
+ if (added > 0) {
+ trans->disk_res->sectors += added;
+ this_cpu_add(*c->online_reserved, added);
+ }
+
+ preempt_enable();
+ percpu_up_read(&c->mark_lock);
+}
+
int bch2_trans_fs_usage_apply(struct btree_trans *trans,
struct replicas_delta_list *deltas)
{
diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 0fc101b9aaf1..e8e3a3b09714 100644
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -229,6 +229,7 @@ int bch2_trans_mark_inode(struct btree_trans *, enum btree_id, unsigned, struct
int bch2_trans_mark_reservation(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_i *, unsigned);
int bch2_trans_mark_reflink_p(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_i *, unsigned);
+void bch2_trans_fs_usage_revert(struct btree_trans *, struct replicas_delta_list *);
int bch2_trans_fs_usage_apply(struct btree_trans *, struct replicas_delta_list *);
int bch2_trans_mark_metadata_bucket(struct btree_trans *, struct bch_dev *,
diff --git a/fs/bcachefs/errcode.h b/fs/bcachefs/errcode.h
index bb296edcf4f7..7a6448f48fca 100644
--- a/fs/bcachefs/errcode.h
+++ b/fs/bcachefs/errcode.h
@@ -42,6 +42,7 @@
x(BCH_ERR_transaction_restart, transaction_restart_key_cache_realloced)\
x(BCH_ERR_transaction_restart, transaction_restart_journal_preres_get) \
x(BCH_ERR_transaction_restart, transaction_restart_split_race) \
+ x(BCH_ERR_transaction_restart, transaction_restart_write_buffer_flush) \
x(BCH_ERR_transaction_restart, transaction_restart_nested) \
x(0, no_btree_node) \
x(BCH_ERR_no_btree_node, no_btree_node_relock) \
@@ -58,6 +59,7 @@
x(BCH_ERR_btree_insert_fail, btree_insert_need_mark_replicas) \
x(BCH_ERR_btree_insert_fail, btree_insert_need_journal_res) \
x(BCH_ERR_btree_insert_fail, btree_insert_need_journal_reclaim) \
+ x(BCH_ERR_btree_insert_fail, btree_insert_need_flush_buffer) \
x(0, lock_fail_root_changed) \
x(0, journal_reclaim_would_deadlock) \
x(0, fsck) \
diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h
index 960142b8e55f..1a89f2d4c360 100644
--- a/fs/bcachefs/opts.h
+++ b/fs/bcachefs/opts.h
@@ -206,6 +206,11 @@ enum opt_type {
OPT_BOOL(), \
BCH2_NO_SB_OPT, true, \
NULL, "Stash pointer to in memory btree node in btree ptr")\
+ x(btree_write_buffer_size, u32, \
+ OPT_FS|OPT_MOUNT, \
+ OPT_UINT(16, (1U << 20) - 1), \
+ BCH2_NO_SB_OPT, 1U << 10, \
+ NULL, "Number of btree write buffer entries") \
x(gc_reserve_percent, u8, \
OPT_FS|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME, \
OPT_UINT(5, 21), \
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index 75a5a9842d4c..48780c65824a 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -16,6 +16,7 @@
#include "btree_key_cache.h"
#include "btree_update_interior.h"
#include "btree_io.h"
+#include "btree_write_buffer.h"
#include "buckets_waiting_for_journal.h"
#include "chardev.h"
#include "checksum.h"
@@ -451,6 +452,7 @@ static void __bch2_fs_free(struct bch_fs *c)
bch2_fs_compress_exit(c);
bch2_journal_keys_free(&c->journal_keys);
bch2_journal_entries_free(c);
+ bch2_fs_btree_write_buffer_exit(c);
percpu_free_rwsem(&c->mark_lock);
free_percpu(c->online_reserved);
@@ -799,6 +801,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
bch2_fs_btree_iter_init(c) ?:
bch2_fs_btree_interior_update_init(c) ?:
bch2_fs_buckets_waiting_for_journal_init(c) ?:
+ bch2_fs_btree_write_buffer_init(c) ?:
bch2_fs_subvolumes_init(c) ?:
bch2_fs_io_init(c) ?:
bch2_fs_encryption_init(c) ?:
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-31 23:29:41 +0000