1 /* SPDX-License-Identifier: GPL-2.0 */ 1
2 #ifndef _BCACHEFS_BKEY_H
3 #define _BCACHEFS_BKEY_H
4
5 #include <linux/bug.h>
6 #include "bcachefs_format.h"
7 #include "bkey_types.h"
8 #include "btree_types.h"
9 #include "util.h"
10 #include "vstructs.h"
11
12 enum bch_validate_flags {
13 BCH_VALIDATE_write = BIT(
14 BCH_VALIDATE_commit = BIT(
15 BCH_VALIDATE_journal = BIT(
16 BCH_VALIDATE_silent = BIT(
17 };
18
19 #if 0
20
21 /*
22 * compiled unpack functions are disabled, pen
23 * dynamically allocating executable memory:
24 */
25
26 #ifdef CONFIG_X86_64
27 #define HAVE_BCACHEFS_COMPILED_UNPACK 1
28 #endif
29 #endif
30
31 void bch2_bkey_packed_to_binary_text(struct pr
32 const str
33 const str
34
35 enum bkey_lr_packed {
36 BKEY_PACKED_BOTH,
37 BKEY_PACKED_RIGHT,
38 BKEY_PACKED_LEFT,
39 BKEY_PACKED_NONE,
40 };
41
42 #define bkey_lr_packed(_l, _r)
43 ((_l)->format + ((_r)->format << 1))
44
45 static inline void bkey_p_copy(struct bkey_pac
46 {
47 memcpy_u64s_small(dst, src, src->u64s)
48 }
49
50 static inline void bkey_copy(struct bkey_i *ds
51 {
52 memcpy_u64s_small(dst, src, src->k.u64
53 }
54
55 struct btree;
56
57 __pure
58 unsigned bch2_bkey_greatest_differing_bit(cons
59 cons
60 cons
61 __pure
62 unsigned bch2_bkey_ffs(const struct btree *, c
63
64 __pure
65 int __bch2_bkey_cmp_packed_format_checked(cons
66 const str
67 const str
68
69 __pure
70 int __bch2_bkey_cmp_left_packed_format_checked
71 cons
72 cons
73
74 __pure
75 int bch2_bkey_cmp_packed(const struct btree *,
76 const struct bkey_pac
77 const struct bkey_pac
78
79 __pure
80 int __bch2_bkey_cmp_left_packed(const struct b
81 const struct b
82 const struct b
83
84 static inline __pure
85 int bkey_cmp_left_packed(const struct btree *b
86 const struct bkey_pac
87 {
88 return __bch2_bkey_cmp_left_packed(b,
89 }
90
91 /*
92 * The compiler generates better code when we
93 * enough terribly convenient to pass it by va
94 * ref would be nice here:
95 */
96 __pure __flatten
97 static inline int bkey_cmp_left_packed_byval(c
98 c
99 s
100 {
101 return bkey_cmp_left_packed(b, l, &r);
102 }
103
104 static __always_inline bool bpos_eq(struct bpo
105 {
106 return !((l.inode ^ r.inode) |
107 (l.offset ^ r.offset) |
108 (l.snapshot ^ r.snapshot))
109 }
110
111 static __always_inline bool bpos_lt(struct bpo
112 {
113 return l.inode != r.inode ? l.inode <
114 l.offset != r.offset ? l.offse
115 l.snapshot != r.snapshot ? l.s
116 }
117
118 static __always_inline bool bpos_le(struct bpo
119 {
120 return l.inode != r.inode ? l.inode <
121 l.offset != r.offset ? l.offse
122 l.snapshot != r.snapshot ? l.s
123 }
124
125 static __always_inline bool bpos_gt(struct bpo
126 {
127 return bpos_lt(r, l);
128 }
129
130 static __always_inline bool bpos_ge(struct bpo
131 {
132 return bpos_le(r, l);
133 }
134
135 static __always_inline int bpos_cmp(struct bpo
136 {
137 return cmp_int(l.inode, r.inode) ?
138 cmp_int(l.offset, r.offset)
139 cmp_int(l.snapshot, r.snapshot
140 }
141
142 static inline struct bpos bpos_min(struct bpos
143 {
144 return bpos_lt(l, r) ? l : r;
145 }
146
147 static inline struct bpos bpos_max(struct bpos
148 {
149 return bpos_gt(l, r) ? l : r;
150 }
151
152 static __always_inline bool bkey_eq(struct bpo
153 {
154 return !((l.inode ^ r.inode) |
155 (l.offset ^ r.offset));
156 }
157
158 static __always_inline bool bkey_lt(struct bpo
159 {
160 return l.inode != r.inode
161 ? l.inode < r.inode
162 : l.offset < r.offset;
163 }
164
165 static __always_inline bool bkey_le(struct bpo
166 {
167 return l.inode != r.inode
168 ? l.inode < r.inode
169 : l.offset <= r.offset;
170 }
171
172 static __always_inline bool bkey_gt(struct bpo
173 {
174 return bkey_lt(r, l);
175 }
176
177 static __always_inline bool bkey_ge(struct bpo
178 {
179 return bkey_le(r, l);
180 }
181
182 static __always_inline int bkey_cmp(struct bpo
183 {
184 return cmp_int(l.inode, r.inode) ?
185 cmp_int(l.offset, r.offset);
186 }
187
188 static inline struct bpos bkey_min(struct bpos
189 {
190 return bkey_lt(l, r) ? l : r;
191 }
192
193 static inline struct bpos bkey_max(struct bpos
194 {
195 return bkey_gt(l, r) ? l : r;
196 }
197
198 static inline bool bkey_and_val_eq(struct bkey
199 {
200 return bpos_eq(l.k->p, r.k->p) &&
201 bkey_bytes(l.k) == bkey_bytes(
202 !memcmp(l.v, r.v, bkey_val_byt
203 }
204
205 void bch2_bpos_swab(struct bpos *);
206 void bch2_bkey_swab_key(const struct bkey_form
207
208 static __always_inline int bversion_cmp(struct
209 {
210 return cmp_int(l.hi, r.hi) ?:
211 cmp_int(l.lo, r.lo);
212 }
213
214 #define ZERO_VERSION ((struct bversion) { .
215 #define MAX_VERSION ((struct bversion) { .
216
217 static __always_inline bool bversion_zero(stru
218 {
219 return bversion_cmp(v, ZERO_VERSION) =
220 }
221
222 #ifdef CONFIG_BCACHEFS_DEBUG
223 /* statement expressions confusing unlikely()?
224 #define bkey_packed(_k)
225 ({ EBUG_ON((_k)->format > KEY_FORMAT_C
226 (_k)->format != KEY_FORMAT_CURRENT; }
227 #else
228 #define bkey_packed(_k) ((_k)->format
229 #endif
230
231 /*
232 * It's safe to treat an unpacked bkey as a pa
233 */
234 static inline struct bkey_packed *bkey_to_pack
235 {
236 return (struct bkey_packed *) k;
237 }
238
239 static inline const struct bkey_packed *bkey_t
240 {
241 return (const struct bkey_packed *) k;
242 }
243
244 static inline struct bkey_i *packed_to_bkey(st
245 {
246 return bkey_packed(k) ? NULL : (struct
247 }
248
249 static inline const struct bkey *packed_to_bke
250 {
251 return bkey_packed(k) ? NULL : (const
252 }
253
254 static inline unsigned bkey_format_key_bits(co
255 {
256 return format->bits_per_field[BKEY_FIE
257 format->bits_per_field[BKEY_FI
258 format->bits_per_field[BKEY_FI
259 }
260
261 static inline struct bpos bpos_successor(struc
262 {
263 if (!++p.snapshot &&
264 !++p.offset &&
265 !++p.inode)
266 BUG();
267
268 return p;
269 }
270
271 static inline struct bpos bpos_predecessor(str
272 {
273 if (!p.snapshot-- &&
274 !p.offset-- &&
275 !p.inode--)
276 BUG();
277
278 return p;
279 }
280
281 static inline struct bpos bpos_nosnap_successo
282 {
283 p.snapshot = 0;
284
285 if (!++p.offset &&
286 !++p.inode)
287 BUG();
288
289 return p;
290 }
291
292 static inline struct bpos bpos_nosnap_predeces
293 {
294 p.snapshot = 0;
295
296 if (!p.offset-- &&
297 !p.inode--)
298 BUG();
299
300 return p;
301 }
302
303 static inline u64 bkey_start_offset(const stru
304 {
305 return k->p.offset - k->size;
306 }
307
308 static inline struct bpos bkey_start_pos(const
309 {
310 return (struct bpos) {
311 .inode = k->p.inode,
312 .offset = bkey_start_o
313 .snapshot = k->p.snapsho
314 };
315 }
316
317 /* Packed helpers */
318
319 static inline unsigned bkeyp_key_u64s(const st
320 const st
321 {
322 return bkey_packed(k) ? format->key_u6
323 }
324
325 static inline bool bkeyp_u64s_valid(const stru
326 const stru
327 {
328 return ((unsigned) k->u64s - bkeyp_key
329 }
330
331 static inline unsigned bkeyp_key_bytes(const s
332 const s
333 {
334 return bkeyp_key_u64s(format, k) * siz
335 }
336
337 static inline unsigned bkeyp_val_u64s(const st
338 const st
339 {
340 return k->u64s - bkeyp_key_u64s(format
341 }
342
343 static inline size_t bkeyp_val_bytes(const str
344 const str
345 {
346 return bkeyp_val_u64s(format, k) * siz
347 }
348
349 static inline void set_bkeyp_val_u64s(const st
350 struct b
351 {
352 k->u64s = bkeyp_key_u64s(format, k) +
353 }
354
355 #define bkeyp_val(_format, _k)
356 ((struct bch_val *) ((u64 *) (_k)->_d
357
358 extern const struct bkey_format bch2_bkey_form
359
360 bool bch2_bkey_transform(const struct bkey_for
361 struct bkey_packed *,
362 const struct bkey_for
363 const struct bkey_pac
364
365 struct bkey __bch2_bkey_unpack_key(const struc
366 const struc
367
368 #ifndef HAVE_BCACHEFS_COMPILED_UNPACK
369 struct bpos __bkey_unpack_pos(const struct bke
370 const struct bke
371 #endif
372
373 bool bch2_bkey_pack_key(struct bkey_packed *,
374 const struct bkey_format *)
375
376 enum bkey_pack_pos_ret {
377 BKEY_PACK_POS_EXACT,
378 BKEY_PACK_POS_SMALLER,
379 BKEY_PACK_POS_FAIL,
380 };
381
382 enum bkey_pack_pos_ret bch2_bkey_pack_pos_loss
383 con
384
385 static inline bool bkey_pack_pos(struct bkey_p
386 const struct
387 {
388 return bch2_bkey_pack_pos_lossy(out, i
389 }
390
391 void bch2_bkey_unpack(const struct btree *, st
392 const struct bkey_packed *);
393 bool bch2_bkey_pack(struct bkey_packed *, cons
394 const struct bkey_format *);
395
396 typedef void (*compiled_unpack_fn)(struct bkey
397
398 static inline void
399 __bkey_unpack_key_format_checked(const struct
400 struct bkey *ds
401 const struct bk
402 {
403 if (IS_ENABLED(HAVE_BCACHEFS_COMPILED_
404 compiled_unpack_fn unpack_fn =
405 unpack_fn(dst, src);
406
407 if (IS_ENABLED(CONFIG_BCACHEFS
408 bch2_expensive_debug_check
409 struct bkey dst2 = __b
410
411 BUG_ON(memcmp(dst, &ds
412 }
413 } else {
414 *dst = __bch2_bkey_unpack_key(
415 }
416 }
417
418 static inline struct bkey
419 bkey_unpack_key_format_checked(const struct bt
420 const struct bk
421 {
422 struct bkey dst;
423
424 __bkey_unpack_key_format_checked(b, &d
425 return dst;
426 }
427
428 static inline void __bkey_unpack_key(const str
429 struct bk
430 const str
431 {
432 if (likely(bkey_packed(src)))
433 __bkey_unpack_key_format_check
434 else
435 *dst = *packed_to_bkey_c(src);
436 }
437
438 /**
439 * bkey_unpack_key -- unpack just the key, not
440 */
441 static inline struct bkey bkey_unpack_key(cons
442 cons
443 {
444 return likely(bkey_packed(src))
445 ? bkey_unpack_key_format_check
446 : *packed_to_bkey_c(src);
447 }
448
449 static inline struct bpos
450 bkey_unpack_pos_format_checked(const struct bt
451 const struct bk
452 {
453 #ifdef HAVE_BCACHEFS_COMPILED_UNPACK
454 return bkey_unpack_key_format_checked(
455 #else
456 return __bkey_unpack_pos(&b->format, s
457 #endif
458 }
459
460 static inline struct bpos bkey_unpack_pos(cons
461 cons
462 {
463 return likely(bkey_packed(src))
464 ? bkey_unpack_pos_format_check
465 : packed_to_bkey_c(src)->p;
466 }
467
468 /* Disassembled bkeys */
469
470 static inline struct bkey_s_c bkey_disassemble
471
472
473 {
474 __bkey_unpack_key(b, u, k);
475
476 return (struct bkey_s_c) { u, bkeyp_va
477 }
478
479 /* non const version: */
480 static inline struct bkey_s __bkey_disassemble
481
482
483 {
484 __bkey_unpack_key(b, u, k);
485
486 return (struct bkey_s) { .k = u, .v =
487 }
488
489 static inline u64 bkey_field_max(const struct
490 enum bch_bkey
491 {
492 return f->bits_per_field[nr] < 64
493 ? (le64_to_cpu(f->field_offset
494 ~(~0ULL << f->bits_per_fiel
495 : U64_MAX;
496 }
497
498 #ifdef HAVE_BCACHEFS_COMPILED_UNPACK
499
500 int bch2_compile_bkey_format(const struct bkey
501
502 #else
503
504 static inline int bch2_compile_bkey_format(con
505 void
506
507 #endif
508
509 static inline void bkey_reassemble(struct bkey
510 struct bkey
511 {
512 dst->k = *src.k;
513 memcpy_u64s_small(&dst->v, src.v, bkey
514 }
515
516 /* byte order helpers */
517
518 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
519
520 static inline unsigned high_word_offset(const
521 {
522 return f->key_u64s - 1;
523 }
524
525 #define high_bit_offset 0
526 #define nth_word(p, n) ((p) - (n))
527
528 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
529
530 static inline unsigned high_word_offset(const
531 {
532 return 0;
533 }
534
535 #define high_bit_offset KEY_PACKED_BIT
536 #define nth_word(p, n) ((p) + (n))
537
538 #else
539 #error edit for your odd byteorder.
540 #endif
541
542 #define high_word(f, k) ((u64 *) (k)->
543 #define next_word(p) nth_word(p, 1)
544 #define prev_word(p) nth_word(p, -1
545
546 #ifdef CONFIG_BCACHEFS_DEBUG
547 void bch2_bkey_pack_test(void);
548 #else
549 static inline void bch2_bkey_pack_test(void) {
550 #endif
551
552 #define bkey_fields()
553 x(BKEY_FIELD_INODE, p.inod
554 x(BKEY_FIELD_OFFSET, p.offs
555 x(BKEY_FIELD_SNAPSHOT, p.snap
556 x(BKEY_FIELD_SIZE, size)
557 x(BKEY_FIELD_VERSION_HI, bversi
558 x(BKEY_FIELD_VERSION_LO, bversi
559
560 struct bkey_format_state {
561 u64 field_min[BKEY_NR_FIELDS];
562 u64 field_max[BKEY_NR_FIELDS];
563 };
564
565 void bch2_bkey_format_init(struct bkey_format_
566
567 static inline void __bkey_format_add(struct bk
568 {
569 s->field_min[field] = min(s->field_min
570 s->field_max[field] = max(s->field_max
571 }
572
573 /*
574 * Changes @format so that @k can be successfu
575 */
576 static inline void bch2_bkey_format_add_key(st
577 {
578 #define x(id, field) __bkey_format_add(s, id,
579 bkey_fields()
580 #undef x
581 }
582
583 void bch2_bkey_format_add_pos(struct bkey_form
584 struct bkey_format bch2_bkey_format_done(struc
585
586 static inline bool bch2_bkey_format_field_over
587 {
588 unsigned f_bits = f->bits_per_field[i]
589 unsigned unpacked_bits = bch2_bkey_for
590 u64 unpacked_mask = ~((~0ULL << 1) <<
591 u64 field_offset = le64_to_cpu(f->fiel
592
593 if (f_bits > unpacked_bits)
594 return true;
595
596 if ((f_bits == unpacked_bits) && field
597 return true;
598
599 u64 f_mask = f_bits
600 ? ~((~0ULL << (f_bits - 1)) <<
601 : 0;
602
603 if (((field_offset + f_mask) & unpacke
604 return true;
605 return false;
606 }
607
608 int bch2_bkey_format_invalid(struct bch_fs *,
609 enum bch_validate
610 void bch2_bkey_format_to_text(struct printbuf
611
612 #endif /* _BCACHEFS_BKEY_H */
613
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.