1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2019 Oracle. All Rights Reserved.
4 * Author: Darrick J. Wong <darrick.wong@oracle.com>
5 */
6 #include "xfs.h"
7 #include "xfs_fs.h"
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_inode.h"
14 #include "xfs_btree.h"
15 #include "xfs_ialloc.h"
16 #include "xfs_ialloc_btree.h"
17 #include "xfs_iwalk.h"
18 #include "xfs_error.h"
19 #include "xfs_trace.h"
20 #include "xfs_icache.h"
21 #include "xfs_health.h"
22 #include "xfs_trans.h"
23 #include "xfs_pwork.h"
24 #include "xfs_ag.h"
25 #include "xfs_bit.h"
26
27 /*
28 * Walking Inodes in the Filesystem
29 * ================================
30 *
31 * This iterator function walks a subset of filesystem inodes in increasing
32 * order from @startino until there are no more inodes. For each allocated
33 * inode it finds, it calls a walk function with the relevant inode number and
34 * a pointer to caller-provided data. The walk function can return the usual
35 * negative error code to stop the iteration; 0 to continue the iteration; or
36 * -ECANCELED to stop the iteration. This return value is returned to the
37 * caller.
38 *
39 * Internally, we allow the walk function to do anything, which means that we
40 * cannot maintain the inobt cursor or our lock on the AGI buffer. We
41 * therefore cache the inobt records in kernel memory and only call the walk
42 * function when our memory buffer is full. @nr_recs is the number of records
43 * that we've cached, and @sz_recs is the size of our cache.
44 *
45 * It is the responsibility of the walk function to ensure it accesses
46 * allocated inodes, as the inobt records may be stale by the time they are
47 * acted upon.
48 */
49
50 struct xfs_iwalk_ag {
51 /* parallel work control data; will be null if single threaded */
52 struct xfs_pwork pwork;
53
54 struct xfs_mount *mp;
55 struct xfs_trans *tp;
56 struct xfs_perag *pag;
57
58 /* Where do we start the traversal? */
59 xfs_ino_t startino;
60
61 /* What was the last inode number we saw when iterating the inobt? */
62 xfs_ino_t lastino;
63
64 /* Array of inobt records we cache. */
65 struct xfs_inobt_rec_incore *recs;
66
67 /* Number of entries allocated for the @recs array. */
68 unsigned int sz_recs;
69
70 /* Number of entries in the @recs array that are in use. */
71 unsigned int nr_recs;
72
73 /* Inode walk function and data pointer. */
74 xfs_iwalk_fn iwalk_fn;
75 xfs_inobt_walk_fn inobt_walk_fn;
76 void *data;
77
78 /*
79 * Make it look like the inodes up to startino are free so that
80 * bulkstat can start its inode iteration at the correct place without
81 * needing to special case everywhere.
82 */
83 unsigned int trim_start:1;
84
85 /* Skip empty inobt records? */
86 unsigned int skip_empty:1;
87
88 /* Drop the (hopefully empty) transaction when calling iwalk_fn. */
89 unsigned int drop_trans:1;
90 };
91
92 /*
93 * Loop over all clusters in a chunk for a given incore inode allocation btree
94 * record. Do a readahead if there are any allocated inodes in that cluster.
95 */
96 STATIC void
97 xfs_iwalk_ichunk_ra(
98 struct xfs_mount *mp,
99 struct xfs_perag *pag,
100 struct xfs_inobt_rec_incore *irec)
101 {
102 struct xfs_ino_geometry *igeo = M_IGEO(mp);
103 xfs_agnumber_t agno = pag->pag_agno;
104 xfs_agblock_t agbno;
105 struct blk_plug plug;
106 int i; /* inode chunk index */
107
108 agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino);
109
110 blk_start_plug(&plug);
111 for (i = 0; i < XFS_INODES_PER_CHUNK; i += igeo->inodes_per_cluster) {
112 xfs_inofree_t imask;
113
114 imask = xfs_inobt_maskn(i, igeo->inodes_per_cluster);
115 if (imask & ~irec->ir_free) {
116 xfs_buf_readahead(mp->m_ddev_targp,
117 XFS_AGB_TO_DADDR(mp, agno, agbno),
118 igeo->blocks_per_cluster * mp->m_bsize,
119 &xfs_inode_buf_ops);
120 }
121 agbno += igeo->blocks_per_cluster;
122 }
123 blk_finish_plug(&plug);
124 }
125
126 /*
127 * Set the bits in @irec's free mask that correspond to the inodes before
128 * @agino so that we skip them. This is how we restart an inode walk that was
129 * interrupted in the middle of an inode record.
130 */
131 STATIC void
132 xfs_iwalk_adjust_start(
133 xfs_agino_t agino, /* starting inode of chunk */
134 struct xfs_inobt_rec_incore *irec) /* btree record */
135 {
136 int idx; /* index into inode chunk */
137
138 idx = agino - irec->ir_startino;
139
140 irec->ir_free |= xfs_inobt_maskn(0, idx);
141 irec->ir_freecount = hweight64(irec->ir_free);
142 }
143
144 /* Allocate memory for a walk. */
145 STATIC int
146 xfs_iwalk_alloc(
147 struct xfs_iwalk_ag *iwag)
148 {
149 size_t size;
150
151 ASSERT(iwag->recs == NULL);
152 iwag->nr_recs = 0;
153
154 /* Allocate a prefetch buffer for inobt records. */
155 size = iwag->sz_recs * sizeof(struct xfs_inobt_rec_incore);
156 iwag->recs = kmalloc(size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
157 if (iwag->recs == NULL)
158 return -ENOMEM;
159
160 return 0;
161 }
162
163 /* Free memory we allocated for a walk. */
164 STATIC void
165 xfs_iwalk_free(
166 struct xfs_iwalk_ag *iwag)
167 {
168 kfree(iwag->recs);
169 iwag->recs = NULL;
170 }
171
172 /* For each inuse inode in each cached inobt record, call our function. */
173 STATIC int
174 xfs_iwalk_ag_recs(
175 struct xfs_iwalk_ag *iwag)
176 {
177 struct xfs_mount *mp = iwag->mp;
178 struct xfs_trans *tp = iwag->tp;
179 struct xfs_perag *pag = iwag->pag;
180 xfs_ino_t ino;
181 unsigned int i, j;
182 int error;
183
184 for (i = 0; i < iwag->nr_recs; i++) {
185 struct xfs_inobt_rec_incore *irec = &iwag->recs[i];
186
187 trace_xfs_iwalk_ag_rec(mp, pag->pag_agno, irec);
188
189 if (xfs_pwork_want_abort(&iwag->pwork))
190 return 0;
191
192 if (iwag->inobt_walk_fn) {
193 error = iwag->inobt_walk_fn(mp, tp, pag->pag_agno, irec,
194 iwag->data);
195 if (error)
196 return error;
197 }
198
199 if (!iwag->iwalk_fn)
200 continue;
201
202 for (j = 0; j < XFS_INODES_PER_CHUNK; j++) {
203 if (xfs_pwork_want_abort(&iwag->pwork))
204 return 0;
205
206 /* Skip if this inode is free */
207 if (XFS_INOBT_MASK(j) & irec->ir_free)
208 continue;
209
210 /* Otherwise call our function. */
211 ino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
212 irec->ir_startino + j);
213 error = iwag->iwalk_fn(mp, tp, ino, iwag->data);
214 if (error)
215 return error;
216 }
217 }
218
219 return 0;
220 }
221
222 /* Delete cursor and let go of AGI. */
223 static inline void
224 xfs_iwalk_del_inobt(
225 struct xfs_trans *tp,
226 struct xfs_btree_cur **curpp,
227 struct xfs_buf **agi_bpp,
228 int error)
229 {
230 if (*curpp) {
231 xfs_btree_del_cursor(*curpp, error);
232 *curpp = NULL;
233 }
234 if (*agi_bpp) {
235 xfs_trans_brelse(tp, *agi_bpp);
236 *agi_bpp = NULL;
237 }
238 }
239
240 /*
241 * Set ourselves up for walking inobt records starting from a given point in
242 * the filesystem.
243 *
244 * If caller passed in a nonzero start inode number, load the record from the
245 * inobt and make the record look like all the inodes before agino are free so
246 * that we skip them, and then move the cursor to the next inobt record. This
247 * is how we support starting an iwalk in the middle of an inode chunk.
248 *
249 * If the caller passed in a start number of zero, move the cursor to the first
250 * inobt record.
251 *
252 * The caller is responsible for cleaning up the cursor and buffer pointer
253 * regardless of the error status.
254 */
255 STATIC int
256 xfs_iwalk_ag_start(
257 struct xfs_iwalk_ag *iwag,
258 xfs_agino_t agino,
259 struct xfs_btree_cur **curpp,
260 struct xfs_buf **agi_bpp,
261 int *has_more)
262 {
263 struct xfs_mount *mp = iwag->mp;
264 struct xfs_trans *tp = iwag->tp;
265 struct xfs_perag *pag = iwag->pag;
266 struct xfs_inobt_rec_incore *irec;
267 int error;
268
269 /* Set up a fresh cursor and empty the inobt cache. */
270 iwag->nr_recs = 0;
271 error = xfs_ialloc_read_agi(pag, tp, 0, agi_bpp);
272 if (error)
273 return error;
274 *curpp = xfs_inobt_init_cursor(pag, tp, *agi_bpp);
275
276 /* Starting at the beginning of the AG? That's easy! */
277 if (agino == 0)
278 return xfs_inobt_lookup(*curpp, 0, XFS_LOOKUP_GE, has_more);
279
280 /*
281 * Otherwise, we have to grab the inobt record where we left off, stuff
282 * the record into our cache, and then see if there are more records.
283 * We require a lookup cache of at least two elements so that the
284 * caller doesn't have to deal with tearing down the cursor to walk the
285 * records.
286 */
287 error = xfs_inobt_lookup(*curpp, agino, XFS_LOOKUP_LE, has_more);
288 if (error)
289 return error;
290
291 /*
292 * If the LE lookup at @agino yields no records, jump ahead to the
293 * inobt cursor increment to see if there are more records to process.
294 */
295 if (!*has_more)
296 goto out_advance;
297
298 /* Get the record, should always work */
299 irec = &iwag->recs[iwag->nr_recs];
300 error = xfs_inobt_get_rec(*curpp, irec, has_more);
301 if (error)
302 return error;
303 if (XFS_IS_CORRUPT(mp, *has_more != 1)) {
304 xfs_btree_mark_sick(*curpp);
305 return -EFSCORRUPTED;
306 }
307
308 iwag->lastino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
309 irec->ir_startino + XFS_INODES_PER_CHUNK - 1);
310
311 /*
312 * If the LE lookup yielded an inobt record before the cursor position,
313 * skip it and see if there's another one after it.
314 */
315 if (irec->ir_startino + XFS_INODES_PER_CHUNK <= agino)
316 goto out_advance;
317
318 /*
319 * If agino fell in the middle of the inode record, make it look like
320 * the inodes up to agino are free so that we don't return them again.
321 */
322 if (iwag->trim_start)
323 xfs_iwalk_adjust_start(agino, irec);
324
325 /*
326 * The prefetch calculation is supposed to give us a large enough inobt
327 * record cache that grab_ichunk can stage a partial first record and
328 * the loop body can cache a record without having to check for cache
329 * space until after it reads an inobt record.
330 */
331 iwag->nr_recs++;
332 ASSERT(iwag->nr_recs < iwag->sz_recs);
333
334 out_advance:
335 return xfs_btree_increment(*curpp, 0, has_more);
336 }
337
338 /*
339 * The inobt record cache is full, so preserve the inobt cursor state and
340 * run callbacks on the cached inobt records. When we're done, restore the
341 * cursor state to wherever the cursor would have been had the cache not been
342 * full (and therefore we could've just incremented the cursor) if *@has_more
343 * is true. On exit, *@has_more will indicate whether or not the caller should
344 * try for more inode records.
345 */
346 STATIC int
347 xfs_iwalk_run_callbacks(
348 struct xfs_iwalk_ag *iwag,
349 struct xfs_btree_cur **curpp,
350 struct xfs_buf **agi_bpp,
351 int *has_more)
352 {
353 struct xfs_mount *mp = iwag->mp;
354 xfs_agino_t next_agino;
355 int error;
356
357 next_agino = XFS_INO_TO_AGINO(mp, iwag->lastino) + 1;
358
359 ASSERT(iwag->nr_recs > 0);
360
361 /* Delete cursor but remember the last record we cached... */
362 xfs_iwalk_del_inobt(iwag->tp, curpp, agi_bpp, 0);
363 ASSERT(next_agino >= iwag->recs[iwag->nr_recs - 1].ir_startino +
364 XFS_INODES_PER_CHUNK);
365
366 if (iwag->drop_trans) {
367 xfs_trans_cancel(iwag->tp);
368 iwag->tp = NULL;
369 }
370
371 error = xfs_iwalk_ag_recs(iwag);
372 if (error)
373 return error;
374
375 /* ...empty the cache... */
376 iwag->nr_recs = 0;
377
378 if (!has_more)
379 return 0;
380
381 if (iwag->drop_trans) {
382 error = xfs_trans_alloc_empty(mp, &iwag->tp);
383 if (error)
384 return error;
385 }
386
387 /* ...and recreate the cursor just past where we left off. */
388 error = xfs_ialloc_read_agi(iwag->pag, iwag->tp, 0, agi_bpp);
389 if (error)
390 return error;
391 *curpp = xfs_inobt_init_cursor(iwag->pag, iwag->tp, *agi_bpp);
392 return xfs_inobt_lookup(*curpp, next_agino, XFS_LOOKUP_GE, has_more);
393 }
394
395 /* Walk all inodes in a single AG, from @iwag->startino to the end of the AG. */
396 STATIC int
397 xfs_iwalk_ag(
398 struct xfs_iwalk_ag *iwag)
399 {
400 struct xfs_mount *mp = iwag->mp;
401 struct xfs_perag *pag = iwag->pag;
402 struct xfs_buf *agi_bp = NULL;
403 struct xfs_btree_cur *cur = NULL;
404 xfs_agino_t agino;
405 int has_more;
406 int error = 0;
407
408 /* Set up our cursor at the right place in the inode btree. */
409 ASSERT(pag->pag_agno == XFS_INO_TO_AGNO(mp, iwag->startino));
410 agino = XFS_INO_TO_AGINO(mp, iwag->startino);
411 error = xfs_iwalk_ag_start(iwag, agino, &cur, &agi_bp, &has_more);
412
413 while (!error && has_more) {
414 struct xfs_inobt_rec_incore *irec;
415 xfs_ino_t rec_fsino;
416
417 cond_resched();
418 if (xfs_pwork_want_abort(&iwag->pwork))
419 goto out;
420
421 /* Fetch the inobt record. */
422 irec = &iwag->recs[iwag->nr_recs];
423 error = xfs_inobt_get_rec(cur, irec, &has_more);
424 if (error || !has_more)
425 break;
426
427 /* Make sure that we always move forward. */
428 rec_fsino = XFS_AGINO_TO_INO(mp, pag->pag_agno, irec->ir_startino);
429 if (iwag->lastino != NULLFSINO &&
430 XFS_IS_CORRUPT(mp, iwag->lastino >= rec_fsino)) {
431 xfs_btree_mark_sick(cur);
432 error = -EFSCORRUPTED;
433 goto out;
434 }
435 iwag->lastino = rec_fsino + XFS_INODES_PER_CHUNK - 1;
436
437 /* No allocated inodes in this chunk; skip it. */
438 if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) {
439 error = xfs_btree_increment(cur, 0, &has_more);
440 if (error)
441 break;
442 continue;
443 }
444
445 /*
446 * Start readahead for this inode chunk in anticipation of
447 * walking the inodes.
448 */
449 if (iwag->iwalk_fn)
450 xfs_iwalk_ichunk_ra(mp, pag, irec);
451
452 /*
453 * If there's space in the buffer for more records, increment
454 * the btree cursor and grab more.
455 */
456 if (++iwag->nr_recs < iwag->sz_recs) {
457 error = xfs_btree_increment(cur, 0, &has_more);
458 if (error || !has_more)
459 break;
460 continue;
461 }
462
463 /*
464 * Otherwise, we need to save cursor state and run the callback
465 * function on the cached records. The run_callbacks function
466 * is supposed to return a cursor pointing to the record where
467 * we would be if we had been able to increment like above.
468 */
469 ASSERT(has_more);
470 error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
471 }
472
473 if (iwag->nr_recs == 0 || error)
474 goto out;
475
476 /* Walk the unprocessed records in the cache. */
477 error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
478
479 out:
480 xfs_iwalk_del_inobt(iwag->tp, &cur, &agi_bp, error);
481 return error;
482 }
483
484 /*
485 * We experimentally determined that the reduction in ioctl call overhead
486 * diminishes when userspace asks for more than 2048 inodes, so we'll cap
487 * prefetch at this point.
488 */
489 #define IWALK_MAX_INODE_PREFETCH (2048U)
490
491 /*
492 * Given the number of inodes to prefetch, set the number of inobt records that
493 * we cache in memory, which controls the number of inodes we try to read
494 * ahead. Set the maximum if @inodes == 0.
495 */
496 static inline unsigned int
497 xfs_iwalk_prefetch(
498 unsigned int inodes)
499 {
500 unsigned int inobt_records;
501
502 /*
503 * If the caller didn't tell us the number of inodes they wanted,
504 * assume the maximum prefetch possible for best performance.
505 * Otherwise, cap prefetch at that maximum so that we don't start an
506 * absurd amount of prefetch.
507 */
508 if (inodes == 0)
509 inodes = IWALK_MAX_INODE_PREFETCH;
510 inodes = min(inodes, IWALK_MAX_INODE_PREFETCH);
511
512 /* Round the inode count up to a full chunk. */
513 inodes = round_up(inodes, XFS_INODES_PER_CHUNK);
514
515 /*
516 * In order to convert the number of inodes to prefetch into an
517 * estimate of the number of inobt records to cache, we require a
518 * conversion factor that reflects our expectations of the average
519 * loading factor of an inode chunk. Based on data gathered, most
520 * (but not all) filesystems manage to keep the inode chunks totally
521 * full, so we'll underestimate slightly so that our readahead will
522 * still deliver the performance we want on aging filesystems:
523 *
524 * inobt = inodes / (INODES_PER_CHUNK * (4 / 5));
525 *
526 * The funny math is to avoid integer division.
527 */
528 inobt_records = (inodes * 5) / (4 * XFS_INODES_PER_CHUNK);
529
530 /*
531 * Allocate enough space to prefetch at least two inobt records so that
532 * we can cache both the record where the iwalk started and the next
533 * record. This simplifies the AG inode walk loop setup code.
534 */
535 return max(inobt_records, 2U);
536 }
537
538 /*
539 * Walk all inodes in the filesystem starting from @startino. The @iwalk_fn
540 * will be called for each allocated inode, being passed the inode's number and
541 * @data. @max_prefetch controls how many inobt records' worth of inodes we
542 * try to readahead.
543 */
544 int
545 xfs_iwalk(
546 struct xfs_mount *mp,
547 struct xfs_trans *tp,
548 xfs_ino_t startino,
549 unsigned int flags,
550 xfs_iwalk_fn iwalk_fn,
551 unsigned int inode_records,
552 void *data)
553 {
554 struct xfs_iwalk_ag iwag = {
555 .mp = mp,
556 .tp = tp,
557 .iwalk_fn = iwalk_fn,
558 .data = data,
559 .startino = startino,
560 .sz_recs = xfs_iwalk_prefetch(inode_records),
561 .trim_start = 1,
562 .skip_empty = 1,
563 .pwork = XFS_PWORK_SINGLE_THREADED,
564 .lastino = NULLFSINO,
565 };
566 struct xfs_perag *pag;
567 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
568 int error;
569
570 ASSERT(agno < mp->m_sb.sb_agcount);
571 ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
572
573 error = xfs_iwalk_alloc(&iwag);
574 if (error)
575 return error;
576
577 for_each_perag_from(mp, agno, pag) {
578 iwag.pag = pag;
579 error = xfs_iwalk_ag(&iwag);
580 if (error)
581 break;
582 iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0);
583 if (flags & XFS_INOBT_WALK_SAME_AG)
584 break;
585 iwag.pag = NULL;
586 }
587
588 if (iwag.pag)
589 xfs_perag_rele(pag);
590 xfs_iwalk_free(&iwag);
591 return error;
592 }
593
594 /* Run per-thread iwalk work. */
595 static int
596 xfs_iwalk_ag_work(
597 struct xfs_mount *mp,
598 struct xfs_pwork *pwork)
599 {
600 struct xfs_iwalk_ag *iwag;
601 int error = 0;
602
603 iwag = container_of(pwork, struct xfs_iwalk_ag, pwork);
604 if (xfs_pwork_want_abort(pwork))
605 goto out;
606
607 error = xfs_iwalk_alloc(iwag);
608 if (error)
609 goto out;
610 /*
611 * Grab an empty transaction so that we can use its recursive buffer
612 * locking abilities to detect cycles in the inobt without deadlocking.
613 */
614 error = xfs_trans_alloc_empty(mp, &iwag->tp);
615 if (error)
616 goto out;
617 iwag->drop_trans = 1;
618
619 error = xfs_iwalk_ag(iwag);
620 if (iwag->tp)
621 xfs_trans_cancel(iwag->tp);
622 xfs_iwalk_free(iwag);
623 out:
624 xfs_perag_put(iwag->pag);
625 kfree(iwag);
626 return error;
627 }
628
629 /*
630 * Walk all the inodes in the filesystem using multiple threads to process each
631 * AG.
632 */
633 int
634 xfs_iwalk_threaded(
635 struct xfs_mount *mp,
636 xfs_ino_t startino,
637 unsigned int flags,
638 xfs_iwalk_fn iwalk_fn,
639 unsigned int inode_records,
640 bool polled,
641 void *data)
642 {
643 struct xfs_pwork_ctl pctl;
644 struct xfs_perag *pag;
645 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
646 int error;
647
648 ASSERT(agno < mp->m_sb.sb_agcount);
649 ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
650
651 error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk");
652 if (error)
653 return error;
654
655 for_each_perag_from(mp, agno, pag) {
656 struct xfs_iwalk_ag *iwag;
657
658 if (xfs_pwork_ctl_want_abort(&pctl))
659 break;
660
661 iwag = kzalloc(sizeof(struct xfs_iwalk_ag),
662 GFP_KERNEL | __GFP_NOFAIL);
663 iwag->mp = mp;
664
665 /*
666 * perag is being handed off to async work, so take a passive
667 * reference for the async work to release.
668 */
669 iwag->pag = xfs_perag_hold(pag);
670 iwag->iwalk_fn = iwalk_fn;
671 iwag->data = data;
672 iwag->startino = startino;
673 iwag->sz_recs = xfs_iwalk_prefetch(inode_records);
674 iwag->lastino = NULLFSINO;
675 xfs_pwork_queue(&pctl, &iwag->pwork);
676 startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
677 if (flags & XFS_INOBT_WALK_SAME_AG)
678 break;
679 }
680 if (pag)
681 xfs_perag_rele(pag);
682 if (polled)
683 xfs_pwork_poll(&pctl);
684 return xfs_pwork_destroy(&pctl);
685 }
686
687 /*
688 * Allow callers to cache up to a page's worth of inobt records. This reflects
689 * the existing inumbers prefetching behavior. Since the inobt walk does not
690 * itself do anything with the inobt records, we can set a fairly high limit
691 * here.
692 */
693 #define MAX_INOBT_WALK_PREFETCH \
694 (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore))
695
696 /*
697 * Given the number of records that the user wanted, set the number of inobt
698 * records that we buffer in memory. Set the maximum if @inobt_records == 0.
699 */
700 static inline unsigned int
701 xfs_inobt_walk_prefetch(
702 unsigned int inobt_records)
703 {
704 /*
705 * If the caller didn't tell us the number of inobt records they
706 * wanted, assume the maximum prefetch possible for best performance.
707 */
708 if (inobt_records == 0)
709 inobt_records = MAX_INOBT_WALK_PREFETCH;
710
711 /*
712 * Allocate enough space to prefetch at least two inobt records so that
713 * we can cache both the record where the iwalk started and the next
714 * record. This simplifies the AG inode walk loop setup code.
715 */
716 inobt_records = max(inobt_records, 2U);
717
718 /*
719 * Cap prefetch at that maximum so that we don't use an absurd amount
720 * of memory.
721 */
722 return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH);
723 }
724
725 /*
726 * Walk all inode btree records in the filesystem starting from @startino. The
727 * @inobt_walk_fn will be called for each btree record, being passed the incore
728 * record and @data. @max_prefetch controls how many inobt records we try to
729 * cache ahead of time.
730 */
731 int
732 xfs_inobt_walk(
733 struct xfs_mount *mp,
734 struct xfs_trans *tp,
735 xfs_ino_t startino,
736 unsigned int flags,
737 xfs_inobt_walk_fn inobt_walk_fn,
738 unsigned int inobt_records,
739 void *data)
740 {
741 struct xfs_iwalk_ag iwag = {
742 .mp = mp,
743 .tp = tp,
744 .inobt_walk_fn = inobt_walk_fn,
745 .data = data,
746 .startino = startino,
747 .sz_recs = xfs_inobt_walk_prefetch(inobt_records),
748 .pwork = XFS_PWORK_SINGLE_THREADED,
749 .lastino = NULLFSINO,
750 };
751 struct xfs_perag *pag;
752 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
753 int error;
754
755 ASSERT(agno < mp->m_sb.sb_agcount);
756 ASSERT(!(flags & ~XFS_INOBT_WALK_FLAGS_ALL));
757
758 error = xfs_iwalk_alloc(&iwag);
759 if (error)
760 return error;
761
762 for_each_perag_from(mp, agno, pag) {
763 iwag.pag = pag;
764 error = xfs_iwalk_ag(&iwag);
765 if (error)
766 break;
767 iwag.startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
768 if (flags & XFS_INOBT_WALK_SAME_AG)
769 break;
770 iwag.pag = NULL;
771 }
772
773 if (iwag.pag)
774 xfs_perag_rele(pag);
775 xfs_iwalk_free(&iwag);
776 return error;
777 }
778
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.