1 // SPDX-License-Identifier: GPL-2.0
2
3 #include "bcachefs.h"
4 #include "bcachefs_ioctl.h"
5 #include "btree_cache.h"
6 #include "btree_journal_iter.h"
7 #include "btree_update.h"
8 #include "btree_write_buffer.h"
9 #include "buckets.h"
10 #include "compress.h"
11 #include "disk_accounting.h"
12 #include "error.h"
13 #include "journal_io.h"
14 #include "replicas.h"
15
16 /*
17 * Notes on disk accounting:
18 *
19 * We have two parallel sets of counters to be concerned with, and both must be
20 * kept in sync.
21 *
22 * - Persistent/on disk accounting, stored in the accounting btree and updated
23 * via btree write buffer updates that treat new accounting keys as deltas to
24 * apply to existing values. But reading from a write buffer btree is
25 * expensive, so we also have
26 *
27 * - In memory accounting, where accounting is stored as an array of percpu
28 * counters, indexed by an eytzinger array of disk acounting keys/bpos (which
29 * are the same thing, excepting byte swabbing on big endian).
30 *
31 * Cheap to read, but non persistent.
32 *
33 * Disk accounting updates are generated by transactional triggers; these run as
34 * keys enter and leave the btree, and can compare old and new versions of keys;
35 * the output of these triggers are deltas to the various counters.
36 *
37 * Disk accounting updates are done as btree write buffer updates, where the
38 * counters in the disk accounting key are deltas that will be applied to the
39 * counter in the btree when the key is flushed by the write buffer (or journal
40 * replay).
41 *
42 * To do a disk accounting update:
43 * - initialize a disk_accounting_pos, to specify which counter is being update
44 * - initialize counter deltas, as an array of 1-3 s64s
45 * - call bch2_disk_accounting_mod()
46 *
47 * This queues up the accounting update to be done at transaction commit time.
48 * Underneath, it's a normal btree write buffer update.
49 *
50 * The transaction commit path is responsible for propagating updates to the in
51 * memory counters, with bch2_accounting_mem_mod().
52 *
53 * The commit path also assigns every disk accounting update a unique version
54 * number, based on the journal sequence number and offset within that journal
55 * buffer; this is used by journal replay to determine which updates have been
56 * done.
57 *
58 * The transaction commit path also ensures that replicas entry accounting
59 * updates are properly marked in the superblock (so that we know whether we can
60 * mount without data being unavailable); it will update the superblock if
61 * bch2_accounting_mem_mod() tells it to.
62 */
63
64 static const char * const disk_accounting_type_strs[] = {
65 #define x(t, n, ...) [n] = #t,
66 BCH_DISK_ACCOUNTING_TYPES()
67 #undef x
68 NULL
69 };
70
71 static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos,
72 s64 *d, unsigned nr)
73 {
74 struct bkey_i_accounting *acc = bkey_accounting_init(k);
75
76 acc->k.p = disk_accounting_pos_to_bpos(pos);
77 set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr);
78
79 memcpy_u64s_small(acc->v.d, d, nr);
80 }
81
82 int bch2_disk_accounting_mod(struct btree_trans *trans,
83 struct disk_accounting_pos *k,
84 s64 *d, unsigned nr, bool gc)
85 {
86 /* Normalize: */
87 switch (k->type) {
88 case BCH_DISK_ACCOUNTING_replicas:
89 bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp);
90 break;
91 }
92
93 BUG_ON(nr > BCH_ACCOUNTING_MAX_COUNTERS);
94
95 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;
96
97 accounting_key_init(&k_i.k, k, d, nr);
98
99 return likely(!gc)
100 ? bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k)
101 : bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true);
102 }
103
104 int bch2_mod_dev_cached_sectors(struct btree_trans *trans,
105 unsigned dev, s64 sectors,
106 bool gc)
107 {
108 struct disk_accounting_pos acc = {
109 .type = BCH_DISK_ACCOUNTING_replicas,
110 };
111
112 bch2_replicas_entry_cached(&acc.replicas, dev);
113
114 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc);
115 }
116
117 static inline bool is_zero(char *start, char *end)
118 {
119 BUG_ON(start > end);
120
121 for (; start < end; start++)
122 if (*start)
123 return false;
124 return true;
125 }
126
127 #define field_end(p, member) (((void *) (&p.member)) + sizeof(p.member))
128
129 int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k,
130 enum bch_validate_flags flags)
131 {
132 struct disk_accounting_pos acc_k;
133 bpos_to_disk_accounting_pos(&acc_k, k.k->p);
134 void *end = &acc_k + 1;
135 int ret = 0;
136
137 switch (acc_k.type) {
138 case BCH_DISK_ACCOUNTING_nr_inodes:
139 end = field_end(acc_k, nr_inodes);
140 break;
141 case BCH_DISK_ACCOUNTING_persistent_reserved:
142 end = field_end(acc_k, persistent_reserved);
143 break;
144 case BCH_DISK_ACCOUNTING_replicas:
145 bkey_fsck_err_on(!acc_k.replicas.nr_devs,
146 c, accounting_key_replicas_nr_devs_0,
147 "accounting key replicas entry with nr_devs=0");
148
149 bkey_fsck_err_on(acc_k.replicas.nr_required > acc_k.replicas.nr_devs ||
150 (acc_k.replicas.nr_required > 1 &&
151 acc_k.replicas.nr_required == acc_k.replicas.nr_devs),
152 c, accounting_key_replicas_nr_required_bad,
153 "accounting key replicas entry with bad nr_required");
154
155 for (unsigned i = 0; i + 1 < acc_k.replicas.nr_devs; i++)
156 bkey_fsck_err_on(acc_k.replicas.devs[i] >= acc_k.replicas.devs[i + 1],
157 c, accounting_key_replicas_devs_unsorted,
158 "accounting key replicas entry with unsorted devs");
159
160 end = (void *) &acc_k.replicas + replicas_entry_bytes(&acc_k.replicas);
161 break;
162 case BCH_DISK_ACCOUNTING_dev_data_type:
163 end = field_end(acc_k, dev_data_type);
164 break;
165 case BCH_DISK_ACCOUNTING_compression:
166 end = field_end(acc_k, compression);
167 break;
168 case BCH_DISK_ACCOUNTING_snapshot:
169 end = field_end(acc_k, snapshot);
170 break;
171 case BCH_DISK_ACCOUNTING_btree:
172 end = field_end(acc_k, btree);
173 break;
174 case BCH_DISK_ACCOUNTING_rebalance_work:
175 end = field_end(acc_k, rebalance_work);
176 break;
177 }
178
179 bkey_fsck_err_on(!is_zero(end, (void *) (&acc_k + 1)),
180 c, accounting_key_junk_at_end,
181 "junk at end of accounting key");
182 fsck_err:
183 return ret;
184 }
185
186 void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k)
187 {
188 if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) {
189 prt_printf(out, "unknown type %u", k->type);
190 return;
191 }
192
193 prt_str(out, disk_accounting_type_strs[k->type]);
194 prt_str(out, " ");
195
196 switch (k->type) {
197 case BCH_DISK_ACCOUNTING_nr_inodes:
198 break;
199 case BCH_DISK_ACCOUNTING_persistent_reserved:
200 prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas);
201 break;
202 case BCH_DISK_ACCOUNTING_replicas:
203 bch2_replicas_entry_to_text(out, &k->replicas);
204 break;
205 case BCH_DISK_ACCOUNTING_dev_data_type:
206 prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev);
207 bch2_prt_data_type(out, k->dev_data_type.data_type);
208 break;
209 case BCH_DISK_ACCOUNTING_compression:
210 bch2_prt_compression_type(out, k->compression.type);
211 break;
212 case BCH_DISK_ACCOUNTING_snapshot:
213 prt_printf(out, "id=%u", k->snapshot.id);
214 break;
215 case BCH_DISK_ACCOUNTING_btree:
216 prt_printf(out, "btree=%s", bch2_btree_id_str(k->btree.id));
217 break;
218 }
219 }
220
221 void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)
222 {
223 struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k);
224 struct disk_accounting_pos acc_k;
225 bpos_to_disk_accounting_pos(&acc_k, k.k->p);
226
227 bch2_accounting_key_to_text(out, &acc_k);
228
229 for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++)
230 prt_printf(out, " %lli", acc.v->d[i]);
231 }
232
233 void bch2_accounting_swab(struct bkey_s k)
234 {
235 for (u64 *p = (u64 *) k.v;
236 p < (u64 *) bkey_val_end(k);
237 p++)
238 *p = swab64(*p);
239 }
240
241 static inline bool accounting_to_replicas(struct bch_replicas_entry_v1 *r, struct bpos p)
242 {
243 struct disk_accounting_pos acc_k;
244 bpos_to_disk_accounting_pos(&acc_k, p);
245
246 switch (acc_k.type) {
247 case BCH_DISK_ACCOUNTING_replicas:
248 unsafe_memcpy(r, &acc_k.replicas,
249 replicas_entry_bytes(&acc_k.replicas),
250 "variable length struct");
251 return true;
252 default:
253 return false;
254 }
255 }
256
257 static int bch2_accounting_update_sb_one(struct bch_fs *c, struct bpos p)
258 {
259 struct bch_replicas_padded r;
260 return accounting_to_replicas(&r.e, p)
261 ? bch2_mark_replicas(c, &r.e)
262 : 0;
263 }
264
265 /*
266 * Ensure accounting keys being updated are present in the superblock, when
267 * applicable (i.e. replicas updates)
268 */
269 int bch2_accounting_update_sb(struct btree_trans *trans)
270 {
271 for (struct jset_entry *i = trans->journal_entries;
272 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
273 i = vstruct_next(i))
274 if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) {
275 int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p);
276 if (ret)
277 return ret;
278 }
279
280 return 0;
281 }
282
283 static int __bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a)
284 {
285 struct bch_accounting_mem *acc = &c->accounting;
286
287 /* raced with another insert, already present: */
288 if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
289 accounting_pos_cmp, &a.k->p) < acc->k.nr)
290 return 0;
291
292 struct accounting_mem_entry n = {
293 .pos = a.k->p,
294 .version = a.k->version,
295 .nr_counters = bch2_accounting_counters(a.k),
296 .v[0] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),
297 sizeof(u64), GFP_KERNEL),
298 };
299
300 if (!n.v[0])
301 goto err;
302
303 if (acc->gc_running) {
304 n.v[1] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),
305 sizeof(u64), GFP_KERNEL);
306 if (!n.v[1])
307 goto err;
308 }
309
310 if (darray_push(&acc->k, n))
311 goto err;
312
313 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
314 accounting_pos_cmp, NULL);
315 return 0;
316 err:
317 free_percpu(n.v[1]);
318 free_percpu(n.v[0]);
319 return -BCH_ERR_ENOMEM_disk_accounting;
320 }
321
322 int bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a, bool gc)
323 {
324 struct bch_replicas_padded r;
325
326 if (accounting_to_replicas(&r.e, a.k->p) &&
327 !bch2_replicas_marked_locked(c, &r.e))
328 return -BCH_ERR_btree_insert_need_mark_replicas;
329
330 percpu_up_read(&c->mark_lock);
331 percpu_down_write(&c->mark_lock);
332 int ret = __bch2_accounting_mem_insert(c, a);
333 percpu_up_write(&c->mark_lock);
334 percpu_down_read(&c->mark_lock);
335 return ret;
336 }
337
338 static bool accounting_mem_entry_is_zero(struct accounting_mem_entry *e)
339 {
340 for (unsigned i = 0; i < e->nr_counters; i++)
341 if (percpu_u64_get(e->v[0] + i) ||
342 (e->v[1] &&
343 percpu_u64_get(e->v[1] + i)))
344 return false;
345 return true;
346 }
347
348 void bch2_accounting_mem_gc(struct bch_fs *c)
349 {
350 struct bch_accounting_mem *acc = &c->accounting;
351
352 percpu_down_write(&c->mark_lock);
353 struct accounting_mem_entry *dst = acc->k.data;
354
355 darray_for_each(acc->k, src) {
356 if (accounting_mem_entry_is_zero(src)) {
357 free_percpu(src->v[0]);
358 free_percpu(src->v[1]);
359 } else {
360 *dst++ = *src;
361 }
362 }
363
364 acc->k.nr = dst - acc->k.data;
365 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
366 accounting_pos_cmp, NULL);
367 percpu_up_write(&c->mark_lock);
368 }
369
370 /*
371 * Read out accounting keys for replicas entries, as an array of
372 * bch_replicas_usage entries.
373 *
374 * Note: this may be deprecated/removed at smoe point in the future and replaced
375 * with something more general, it exists to support the ioctl used by the
376 * 'bcachefs fs usage' command.
377 */
378 int bch2_fs_replicas_usage_read(struct bch_fs *c, darray_char *usage)
379 {
380 struct bch_accounting_mem *acc = &c->accounting;
381 int ret = 0;
382
383 darray_init(usage);
384
385 percpu_down_read(&c->mark_lock);
386 darray_for_each(acc->k, i) {
387 struct {
388 struct bch_replicas_usage r;
389 u8 pad[BCH_BKEY_PTRS_MAX];
390 } u;
391
392 if (!accounting_to_replicas(&u.r.r, i->pos))
393 continue;
394
395 u64 sectors;
396 bch2_accounting_mem_read_counters(acc, i - acc->k.data, §ors, 1, false);
397 u.r.sectors = sectors;
398
399 ret = darray_make_room(usage, replicas_usage_bytes(&u.r));
400 if (ret)
401 break;
402
403 memcpy(&darray_top(*usage), &u.r, replicas_usage_bytes(&u.r));
404 usage->nr += replicas_usage_bytes(&u.r);
405 }
406 percpu_up_read(&c->mark_lock);
407
408 if (ret)
409 darray_exit(usage);
410 return ret;
411 }
412
413 int bch2_fs_accounting_read(struct bch_fs *c, darray_char *out_buf, unsigned accounting_types_mask)
414 {
415
416 struct bch_accounting_mem *acc = &c->accounting;
417 int ret = 0;
418
419 darray_init(out_buf);
420
421 percpu_down_read(&c->mark_lock);
422 darray_for_each(acc->k, i) {
423 struct disk_accounting_pos a_p;
424 bpos_to_disk_accounting_pos(&a_p, i->pos);
425
426 if (!(accounting_types_mask & BIT(a_p.type)))
427 continue;
428
429 ret = darray_make_room(out_buf, sizeof(struct bkey_i_accounting) +
430 sizeof(u64) * i->nr_counters);
431 if (ret)
432 break;
433
434 struct bkey_i_accounting *a_out =
435 bkey_accounting_init((void *) &darray_top(*out_buf));
436 set_bkey_val_u64s(&a_out->k, i->nr_counters);
437 a_out->k.p = i->pos;
438 bch2_accounting_mem_read_counters(acc, i - acc->k.data,
439 a_out->v.d, i->nr_counters, false);
440
441 if (!bch2_accounting_key_is_zero(accounting_i_to_s_c(a_out)))
442 out_buf->nr += bkey_bytes(&a_out->k);
443 }
444
445 percpu_up_read(&c->mark_lock);
446
447 if (ret)
448 darray_exit(out_buf);
449 return ret;
450 }
451
452 void bch2_fs_accounting_to_text(struct printbuf *out, struct bch_fs *c)
453 {
454 struct bch_accounting_mem *acc = &c->accounting;
455
456 percpu_down_read(&c->mark_lock);
457 out->atomic++;
458
459 eytzinger0_for_each(i, acc->k.nr) {
460 struct disk_accounting_pos acc_k;
461 bpos_to_disk_accounting_pos(&acc_k, acc->k.data[i].pos);
462
463 bch2_accounting_key_to_text(out, &acc_k);
464
465 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
466 bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);
467
468 prt_str(out, ":");
469 for (unsigned j = 0; j < acc->k.data[i].nr_counters; j++)
470 prt_printf(out, " %llu", v[j]);
471 prt_newline(out);
472 }
473
474 --out->atomic;
475 percpu_up_read(&c->mark_lock);
476 }
477
478 static void bch2_accounting_free_counters(struct bch_accounting_mem *acc, bool gc)
479 {
480 darray_for_each(acc->k, e) {
481 free_percpu(e->v[gc]);
482 e->v[gc] = NULL;
483 }
484 }
485
486 int bch2_gc_accounting_start(struct bch_fs *c)
487 {
488 struct bch_accounting_mem *acc = &c->accounting;
489 int ret = 0;
490
491 percpu_down_write(&c->mark_lock);
492 darray_for_each(acc->k, e) {
493 e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64),
494 sizeof(u64), GFP_KERNEL);
495 if (!e->v[1]) {
496 bch2_accounting_free_counters(acc, true);
497 ret = -BCH_ERR_ENOMEM_disk_accounting;
498 break;
499 }
500 }
501
502 acc->gc_running = !ret;
503 percpu_up_write(&c->mark_lock);
504
505 return ret;
506 }
507
508 int bch2_gc_accounting_done(struct bch_fs *c)
509 {
510 struct bch_accounting_mem *acc = &c->accounting;
511 struct btree_trans *trans = bch2_trans_get(c);
512 struct printbuf buf = PRINTBUF;
513 struct bpos pos = POS_MIN;
514 int ret = 0;
515
516 percpu_down_write(&c->mark_lock);
517 while (1) {
518 unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),
519 accounting_pos_cmp, &pos);
520
521 if (idx >= acc->k.nr)
522 break;
523
524 struct accounting_mem_entry *e = acc->k.data + idx;
525 pos = bpos_successor(e->pos);
526
527 struct disk_accounting_pos acc_k;
528 bpos_to_disk_accounting_pos(&acc_k, e->pos);
529
530 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)
531 continue;
532
533 u64 src_v[BCH_ACCOUNTING_MAX_COUNTERS];
534 u64 dst_v[BCH_ACCOUNTING_MAX_COUNTERS];
535
536 unsigned nr = e->nr_counters;
537 bch2_accounting_mem_read_counters(acc, idx, dst_v, nr, false);
538 bch2_accounting_mem_read_counters(acc, idx, src_v, nr, true);
539
540 if (memcmp(dst_v, src_v, nr * sizeof(u64))) {
541 printbuf_reset(&buf);
542 prt_str(&buf, "accounting mismatch for ");
543 bch2_accounting_key_to_text(&buf, &acc_k);
544
545 prt_str(&buf, ": got");
546 for (unsigned j = 0; j < nr; j++)
547 prt_printf(&buf, " %llu", dst_v[j]);
548
549 prt_str(&buf, " should be");
550 for (unsigned j = 0; j < nr; j++)
551 prt_printf(&buf, " %llu", src_v[j]);
552
553 for (unsigned j = 0; j < nr; j++)
554 src_v[j] -= dst_v[j];
555
556 if (fsck_err(trans, accounting_mismatch, "%s", buf.buf)) {
557 percpu_up_write(&c->mark_lock);
558 ret = commit_do(trans, NULL, NULL, 0,
559 bch2_disk_accounting_mod(trans, &acc_k, src_v, nr, false));
560 percpu_down_write(&c->mark_lock);
561 if (ret)
562 goto err;
563
564 if (!test_bit(BCH_FS_may_go_rw, &c->flags)) {
565 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));
566 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;
567
568 accounting_key_init(&k_i.k, &acc_k, src_v, nr);
569 bch2_accounting_mem_mod_locked(trans, bkey_i_to_s_c_accounting(&k_i.k), false, false);
570
571 preempt_disable();
572 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage);
573 struct bch_fs_usage_base *src = &trans->fs_usage_delta;
574 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64));
575 preempt_enable();
576 }
577 }
578 }
579 }
580 err:
581 fsck_err:
582 percpu_up_write(&c->mark_lock);
583 printbuf_exit(&buf);
584 bch2_trans_put(trans);
585 bch_err_fn(c, ret);
586 return ret;
587 }
588
589 static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k)
590 {
591 struct bch_fs *c = trans->c;
592 struct printbuf buf = PRINTBUF;
593
594 if (k.k->type != KEY_TYPE_accounting)
595 return 0;
596
597 percpu_down_read(&c->mark_lock);
598 int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k), false, true);
599 percpu_up_read(&c->mark_lock);
600
601 if (bch2_accounting_key_is_zero(bkey_s_c_to_accounting(k)) &&
602 ret == -BCH_ERR_btree_insert_need_mark_replicas)
603 ret = 0;
604
605 struct disk_accounting_pos acc;
606 bpos_to_disk_accounting_pos(&acc, k.k->p);
607
608 if (fsck_err_on(ret == -BCH_ERR_btree_insert_need_mark_replicas,
609 trans, accounting_replicas_not_marked,
610 "accounting not marked in superblock replicas\n %s",
611 (bch2_accounting_key_to_text(&buf, &acc),
612 buf.buf)))
613 ret = bch2_accounting_update_sb_one(c, k.k->p);
614 fsck_err:
615 printbuf_exit(&buf);
616 return ret;
617 }
618
619 /*
620 * At startup time, initialize the in memory accounting from the btree (and
621 * journal)
622 */
623 int bch2_accounting_read(struct bch_fs *c)
624 {
625 struct bch_accounting_mem *acc = &c->accounting;
626 struct btree_trans *trans = bch2_trans_get(c);
627
628 int ret = for_each_btree_key(trans, iter,
629 BTREE_ID_accounting, POS_MIN,
630 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({
631 struct bkey u;
632 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u);
633 accounting_read_key(trans, k);
634 }));
635 if (ret)
636 goto err;
637
638 struct journal_keys *keys = &c->journal_keys;
639 struct journal_key *dst = keys->data;
640 move_gap(keys, keys->nr);
641
642 darray_for_each(*keys, i) {
643 if (i->k->k.type == KEY_TYPE_accounting) {
644 struct bkey_s_c k = bkey_i_to_s_c(i->k);
645 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr,
646 sizeof(acc->k.data[0]),
647 accounting_pos_cmp, &k.k->p);
648
649 bool applied = idx < acc->k.nr &&
650 bversion_cmp(acc->k.data[idx].version, k.k->version) >= 0;
651
652 if (applied)
653 continue;
654
655 if (i + 1 < &darray_top(*keys) &&
656 i[1].k->k.type == KEY_TYPE_accounting &&
657 !journal_key_cmp(i, i + 1)) {
658 BUG_ON(bversion_cmp(i[0].k->k.version, i[1].k->k.version) >= 0);
659
660 i[1].journal_seq = i[0].journal_seq;
661
662 bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k),
663 bkey_s_c_to_accounting(k));
664 continue;
665 }
666
667 ret = accounting_read_key(trans, k);
668 if (ret)
669 goto err;
670 }
671
672 *dst++ = *i;
673 }
674 keys->gap = keys->nr = dst - keys->data;
675
676 percpu_down_read(&c->mark_lock);
677 preempt_disable();
678 struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage);
679
680 for (unsigned i = 0; i < acc->k.nr; i++) {
681 struct disk_accounting_pos k;
682 bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos);
683
684 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
685 bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);
686
687 switch (k.type) {
688 case BCH_DISK_ACCOUNTING_persistent_reserved:
689 usage->reserved += v[0] * k.persistent_reserved.nr_replicas;
690 break;
691 case BCH_DISK_ACCOUNTING_replicas:
692 fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]);
693 break;
694 case BCH_DISK_ACCOUNTING_dev_data_type:
695 rcu_read_lock();
696 struct bch_dev *ca = bch2_dev_rcu(c, k.dev_data_type.dev);
697 if (ca) {
698 struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type];
699 percpu_u64_set(&d->buckets, v[0]);
700 percpu_u64_set(&d->sectors, v[1]);
701 percpu_u64_set(&d->fragmented, v[2]);
702
703 if (k.dev_data_type.data_type == BCH_DATA_sb ||
704 k.dev_data_type.data_type == BCH_DATA_journal)
705 usage->hidden += v[0] * ca->mi.bucket_size;
706 }
707 rcu_read_unlock();
708 break;
709 }
710 }
711 preempt_enable();
712 percpu_up_read(&c->mark_lock);
713 err:
714 bch2_trans_put(trans);
715 bch_err_fn(c, ret);
716 return ret;
717 }
718
719 int bch2_dev_usage_remove(struct bch_fs *c, unsigned dev)
720 {
721 return bch2_trans_run(c,
722 bch2_btree_write_buffer_flush_sync(trans) ?:
723 for_each_btree_key_commit(trans, iter, BTREE_ID_accounting, POS_MIN,
724 BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({
725 struct disk_accounting_pos acc;
726 bpos_to_disk_accounting_pos(&acc, k.k->p);
727
728 acc.type == BCH_DISK_ACCOUNTING_dev_data_type &&
729 acc.dev_data_type.dev == dev
730 ? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0)
731 : 0;
732 })) ?:
733 bch2_btree_write_buffer_flush_sync(trans));
734 }
735
736 int bch2_dev_usage_init(struct bch_dev *ca, bool gc)
737 {
738 struct bch_fs *c = ca->fs;
739 struct disk_accounting_pos acc = {
740 .type = BCH_DISK_ACCOUNTING_dev_data_type,
741 .dev_data_type.dev = ca->dev_idx,
742 .dev_data_type.data_type = BCH_DATA_free,
743 };
744 u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 };
745
746 int ret = bch2_trans_do(c, NULL, NULL, 0,
747 bch2_disk_accounting_mod(trans, &acc, v, ARRAY_SIZE(v), gc));
748 bch_err_fn(c, ret);
749 return ret;
750 }
751
752 void bch2_verify_accounting_clean(struct bch_fs *c)
753 {
754 bool mismatch = false;
755 struct bch_fs_usage_base base = {}, base_inmem = {};
756
757 bch2_trans_run(c,
758 for_each_btree_key(trans, iter,
759 BTREE_ID_accounting, POS_MIN,
760 BTREE_ITER_all_snapshots, k, ({
761 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];
762 struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k);
763 unsigned nr = bch2_accounting_counters(k.k);
764
765 struct disk_accounting_pos acc_k;
766 bpos_to_disk_accounting_pos(&acc_k, k.k->p);
767
768 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)
769 continue;
770
771 if (acc_k.type == BCH_DISK_ACCOUNTING_inum)
772 continue;
773
774 bch2_accounting_mem_read(c, k.k->p, v, nr);
775
776 if (memcmp(a.v->d, v, nr * sizeof(u64))) {
777 struct printbuf buf = PRINTBUF;
778
779 bch2_bkey_val_to_text(&buf, c, k);
780 prt_str(&buf, " !=");
781 for (unsigned j = 0; j < nr; j++)
782 prt_printf(&buf, " %llu", v[j]);
783
784 pr_err("%s", buf.buf);
785 printbuf_exit(&buf);
786 mismatch = true;
787 }
788
789 switch (acc_k.type) {
790 case BCH_DISK_ACCOUNTING_persistent_reserved:
791 base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0];
792 break;
793 case BCH_DISK_ACCOUNTING_replicas:
794 fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]);
795 break;
796 case BCH_DISK_ACCOUNTING_dev_data_type: {
797 rcu_read_lock();
798 struct bch_dev *ca = bch2_dev_rcu(c, acc_k.dev_data_type.dev);
799 if (!ca) {
800 rcu_read_unlock();
801 continue;
802 }
803
804 v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets);
805 v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors);
806 v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented);
807 rcu_read_unlock();
808
809 if (memcmp(a.v->d, v, 3 * sizeof(u64))) {
810 struct printbuf buf = PRINTBUF;
811
812 bch2_bkey_val_to_text(&buf, c, k);
813 prt_str(&buf, " in mem");
814 for (unsigned j = 0; j < nr; j++)
815 prt_printf(&buf, " %llu", v[j]);
816
817 pr_err("dev accounting mismatch: %s", buf.buf);
818 printbuf_exit(&buf);
819 mismatch = true;
820 }
821 }
822 }
823
824 0;
825 })));
826
827 acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64));
828
829 #define check(x) \
830 if (base.x != base_inmem.x) { \
831 pr_err("fs_usage_base.%s mismatch: %llu != %llu", #x, base.x, base_inmem.x); \
832 mismatch = true; \
833 }
834
835 //check(hidden);
836 check(btree);
837 check(data);
838 check(cached);
839 check(reserved);
840 check(nr_inodes);
841
842 WARN_ON(mismatch);
843 }
844
845 void bch2_accounting_gc_free(struct bch_fs *c)
846 {
847 lockdep_assert_held(&c->mark_lock);
848
849 struct bch_accounting_mem *acc = &c->accounting;
850
851 bch2_accounting_free_counters(acc, true);
852 acc->gc_running = false;
853 }
854
855 void bch2_fs_accounting_exit(struct bch_fs *c)
856 {
857 struct bch_accounting_mem *acc = &c->accounting;
858
859 bch2_accounting_free_counters(acc, false);
860 darray_exit(&acc->k);
861 }
862
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.