1 // SPDX-License-Identifier: GPL-2.0 1
2
3 // Generated by scripts/atomic/gen-atomic-inst
4 // DO NOT MODIFY THIS FILE DIRECTLY
5
6 /*
7 * This file provoides atomic operations with
8 * KASAN, KCSAN), which should be used unless
9 * instrumentation. Where it is necessary to a
10 * raw_atomic*() operations should be used.
11 */
12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
13 #define _LINUX_ATOMIC_INSTRUMENTED_H
14
15 #include <linux/build_bug.h>
16 #include <linux/compiler.h>
17 #include <linux/instrumented.h>
18
19 /**
20 * atomic_read() - atomic load with relaxed or
21 * @v: pointer to atomic_t
22 *
23 * Atomically loads the value of @v with relax
24 *
25 * Unsafe to use in noinstr code; use raw_atom
26 *
27 * Return: The value loaded from @v.
28 */
29 static __always_inline int
30 atomic_read(const atomic_t *v)
31 {
32 instrument_atomic_read(v, sizeof(*v));
33 return raw_atomic_read(v);
34 }
35
36 /**
37 * atomic_read_acquire() - atomic load with ac
38 * @v: pointer to atomic_t
39 *
40 * Atomically loads the value of @v with acqui
41 *
42 * Unsafe to use in noinstr code; use raw_atom
43 *
44 * Return: The value loaded from @v.
45 */
46 static __always_inline int
47 atomic_read_acquire(const atomic_t *v)
48 {
49 instrument_atomic_read(v, sizeof(*v));
50 return raw_atomic_read_acquire(v);
51 }
52
53 /**
54 * atomic_set() - atomic set with relaxed orde
55 * @v: pointer to atomic_t
56 * @i: int value to assign
57 *
58 * Atomically sets @v to @i with relaxed order
59 *
60 * Unsafe to use in noinstr code; use raw_atom
61 *
62 * Return: Nothing.
63 */
64 static __always_inline void
65 atomic_set(atomic_t *v, int i)
66 {
67 instrument_atomic_write(v, sizeof(*v))
68 raw_atomic_set(v, i);
69 }
70
71 /**
72 * atomic_set_release() - atomic set with rele
73 * @v: pointer to atomic_t
74 * @i: int value to assign
75 *
76 * Atomically sets @v to @i with release order
77 *
78 * Unsafe to use in noinstr code; use raw_atom
79 *
80 * Return: Nothing.
81 */
82 static __always_inline void
83 atomic_set_release(atomic_t *v, int i)
84 {
85 kcsan_release();
86 instrument_atomic_write(v, sizeof(*v))
87 raw_atomic_set_release(v, i);
88 }
89
90 /**
91 * atomic_add() - atomic add with relaxed orde
92 * @i: int value to add
93 * @v: pointer to atomic_t
94 *
95 * Atomically updates @v to (@v + @i) with rel
96 *
97 * Unsafe to use in noinstr code; use raw_atom
98 *
99 * Return: Nothing.
100 */
101 static __always_inline void
102 atomic_add(int i, atomic_t *v)
103 {
104 instrument_atomic_read_write(v, sizeof
105 raw_atomic_add(i, v);
106 }
107
108 /**
109 * atomic_add_return() - atomic add with full
110 * @i: int value to add
111 * @v: pointer to atomic_t
112 *
113 * Atomically updates @v to (@v + @i) with ful
114 *
115 * Unsafe to use in noinstr code; use raw_atom
116 *
117 * Return: The updated value of @v.
118 */
119 static __always_inline int
120 atomic_add_return(int i, atomic_t *v)
121 {
122 kcsan_mb();
123 instrument_atomic_read_write(v, sizeof
124 return raw_atomic_add_return(i, v);
125 }
126
127 /**
128 * atomic_add_return_acquire() - atomic add wi
129 * @i: int value to add
130 * @v: pointer to atomic_t
131 *
132 * Atomically updates @v to (@v + @i) with acq
133 *
134 * Unsafe to use in noinstr code; use raw_atom
135 *
136 * Return: The updated value of @v.
137 */
138 static __always_inline int
139 atomic_add_return_acquire(int i, atomic_t *v)
140 {
141 instrument_atomic_read_write(v, sizeof
142 return raw_atomic_add_return_acquire(i
143 }
144
145 /**
146 * atomic_add_return_release() - atomic add wi
147 * @i: int value to add
148 * @v: pointer to atomic_t
149 *
150 * Atomically updates @v to (@v + @i) with rel
151 *
152 * Unsafe to use in noinstr code; use raw_atom
153 *
154 * Return: The updated value of @v.
155 */
156 static __always_inline int
157 atomic_add_return_release(int i, atomic_t *v)
158 {
159 kcsan_release();
160 instrument_atomic_read_write(v, sizeof
161 return raw_atomic_add_return_release(i
162 }
163
164 /**
165 * atomic_add_return_relaxed() - atomic add wi
166 * @i: int value to add
167 * @v: pointer to atomic_t
168 *
169 * Atomically updates @v to (@v + @i) with rel
170 *
171 * Unsafe to use in noinstr code; use raw_atom
172 *
173 * Return: The updated value of @v.
174 */
175 static __always_inline int
176 atomic_add_return_relaxed(int i, atomic_t *v)
177 {
178 instrument_atomic_read_write(v, sizeof
179 return raw_atomic_add_return_relaxed(i
180 }
181
182 /**
183 * atomic_fetch_add() - atomic add with full o
184 * @i: int value to add
185 * @v: pointer to atomic_t
186 *
187 * Atomically updates @v to (@v + @i) with ful
188 *
189 * Unsafe to use in noinstr code; use raw_atom
190 *
191 * Return: The original value of @v.
192 */
193 static __always_inline int
194 atomic_fetch_add(int i, atomic_t *v)
195 {
196 kcsan_mb();
197 instrument_atomic_read_write(v, sizeof
198 return raw_atomic_fetch_add(i, v);
199 }
200
201 /**
202 * atomic_fetch_add_acquire() - atomic add wit
203 * @i: int value to add
204 * @v: pointer to atomic_t
205 *
206 * Atomically updates @v to (@v + @i) with acq
207 *
208 * Unsafe to use in noinstr code; use raw_atom
209 *
210 * Return: The original value of @v.
211 */
212 static __always_inline int
213 atomic_fetch_add_acquire(int i, atomic_t *v)
214 {
215 instrument_atomic_read_write(v, sizeof
216 return raw_atomic_fetch_add_acquire(i,
217 }
218
219 /**
220 * atomic_fetch_add_release() - atomic add wit
221 * @i: int value to add
222 * @v: pointer to atomic_t
223 *
224 * Atomically updates @v to (@v + @i) with rel
225 *
226 * Unsafe to use in noinstr code; use raw_atom
227 *
228 * Return: The original value of @v.
229 */
230 static __always_inline int
231 atomic_fetch_add_release(int i, atomic_t *v)
232 {
233 kcsan_release();
234 instrument_atomic_read_write(v, sizeof
235 return raw_atomic_fetch_add_release(i,
236 }
237
238 /**
239 * atomic_fetch_add_relaxed() - atomic add wit
240 * @i: int value to add
241 * @v: pointer to atomic_t
242 *
243 * Atomically updates @v to (@v + @i) with rel
244 *
245 * Unsafe to use in noinstr code; use raw_atom
246 *
247 * Return: The original value of @v.
248 */
249 static __always_inline int
250 atomic_fetch_add_relaxed(int i, atomic_t *v)
251 {
252 instrument_atomic_read_write(v, sizeof
253 return raw_atomic_fetch_add_relaxed(i,
254 }
255
256 /**
257 * atomic_sub() - atomic subtract with relaxed
258 * @i: int value to subtract
259 * @v: pointer to atomic_t
260 *
261 * Atomically updates @v to (@v - @i) with rel
262 *
263 * Unsafe to use in noinstr code; use raw_atom
264 *
265 * Return: Nothing.
266 */
267 static __always_inline void
268 atomic_sub(int i, atomic_t *v)
269 {
270 instrument_atomic_read_write(v, sizeof
271 raw_atomic_sub(i, v);
272 }
273
274 /**
275 * atomic_sub_return() - atomic subtract with
276 * @i: int value to subtract
277 * @v: pointer to atomic_t
278 *
279 * Atomically updates @v to (@v - @i) with ful
280 *
281 * Unsafe to use in noinstr code; use raw_atom
282 *
283 * Return: The updated value of @v.
284 */
285 static __always_inline int
286 atomic_sub_return(int i, atomic_t *v)
287 {
288 kcsan_mb();
289 instrument_atomic_read_write(v, sizeof
290 return raw_atomic_sub_return(i, v);
291 }
292
293 /**
294 * atomic_sub_return_acquire() - atomic subtra
295 * @i: int value to subtract
296 * @v: pointer to atomic_t
297 *
298 * Atomically updates @v to (@v - @i) with acq
299 *
300 * Unsafe to use in noinstr code; use raw_atom
301 *
302 * Return: The updated value of @v.
303 */
304 static __always_inline int
305 atomic_sub_return_acquire(int i, atomic_t *v)
306 {
307 instrument_atomic_read_write(v, sizeof
308 return raw_atomic_sub_return_acquire(i
309 }
310
311 /**
312 * atomic_sub_return_release() - atomic subtra
313 * @i: int value to subtract
314 * @v: pointer to atomic_t
315 *
316 * Atomically updates @v to (@v - @i) with rel
317 *
318 * Unsafe to use in noinstr code; use raw_atom
319 *
320 * Return: The updated value of @v.
321 */
322 static __always_inline int
323 atomic_sub_return_release(int i, atomic_t *v)
324 {
325 kcsan_release();
326 instrument_atomic_read_write(v, sizeof
327 return raw_atomic_sub_return_release(i
328 }
329
330 /**
331 * atomic_sub_return_relaxed() - atomic subtra
332 * @i: int value to subtract
333 * @v: pointer to atomic_t
334 *
335 * Atomically updates @v to (@v - @i) with rel
336 *
337 * Unsafe to use in noinstr code; use raw_atom
338 *
339 * Return: The updated value of @v.
340 */
341 static __always_inline int
342 atomic_sub_return_relaxed(int i, atomic_t *v)
343 {
344 instrument_atomic_read_write(v, sizeof
345 return raw_atomic_sub_return_relaxed(i
346 }
347
348 /**
349 * atomic_fetch_sub() - atomic subtract with f
350 * @i: int value to subtract
351 * @v: pointer to atomic_t
352 *
353 * Atomically updates @v to (@v - @i) with ful
354 *
355 * Unsafe to use in noinstr code; use raw_atom
356 *
357 * Return: The original value of @v.
358 */
359 static __always_inline int
360 atomic_fetch_sub(int i, atomic_t *v)
361 {
362 kcsan_mb();
363 instrument_atomic_read_write(v, sizeof
364 return raw_atomic_fetch_sub(i, v);
365 }
366
367 /**
368 * atomic_fetch_sub_acquire() - atomic subtrac
369 * @i: int value to subtract
370 * @v: pointer to atomic_t
371 *
372 * Atomically updates @v to (@v - @i) with acq
373 *
374 * Unsafe to use in noinstr code; use raw_atom
375 *
376 * Return: The original value of @v.
377 */
378 static __always_inline int
379 atomic_fetch_sub_acquire(int i, atomic_t *v)
380 {
381 instrument_atomic_read_write(v, sizeof
382 return raw_atomic_fetch_sub_acquire(i,
383 }
384
385 /**
386 * atomic_fetch_sub_release() - atomic subtrac
387 * @i: int value to subtract
388 * @v: pointer to atomic_t
389 *
390 * Atomically updates @v to (@v - @i) with rel
391 *
392 * Unsafe to use in noinstr code; use raw_atom
393 *
394 * Return: The original value of @v.
395 */
396 static __always_inline int
397 atomic_fetch_sub_release(int i, atomic_t *v)
398 {
399 kcsan_release();
400 instrument_atomic_read_write(v, sizeof
401 return raw_atomic_fetch_sub_release(i,
402 }
403
404 /**
405 * atomic_fetch_sub_relaxed() - atomic subtrac
406 * @i: int value to subtract
407 * @v: pointer to atomic_t
408 *
409 * Atomically updates @v to (@v - @i) with rel
410 *
411 * Unsafe to use in noinstr code; use raw_atom
412 *
413 * Return: The original value of @v.
414 */
415 static __always_inline int
416 atomic_fetch_sub_relaxed(int i, atomic_t *v)
417 {
418 instrument_atomic_read_write(v, sizeof
419 return raw_atomic_fetch_sub_relaxed(i,
420 }
421
422 /**
423 * atomic_inc() - atomic increment with relaxe
424 * @v: pointer to atomic_t
425 *
426 * Atomically updates @v to (@v + 1) with rela
427 *
428 * Unsafe to use in noinstr code; use raw_atom
429 *
430 * Return: Nothing.
431 */
432 static __always_inline void
433 atomic_inc(atomic_t *v)
434 {
435 instrument_atomic_read_write(v, sizeof
436 raw_atomic_inc(v);
437 }
438
439 /**
440 * atomic_inc_return() - atomic increment with
441 * @v: pointer to atomic_t
442 *
443 * Atomically updates @v to (@v + 1) with full
444 *
445 * Unsafe to use in noinstr code; use raw_atom
446 *
447 * Return: The updated value of @v.
448 */
449 static __always_inline int
450 atomic_inc_return(atomic_t *v)
451 {
452 kcsan_mb();
453 instrument_atomic_read_write(v, sizeof
454 return raw_atomic_inc_return(v);
455 }
456
457 /**
458 * atomic_inc_return_acquire() - atomic increm
459 * @v: pointer to atomic_t
460 *
461 * Atomically updates @v to (@v + 1) with acqu
462 *
463 * Unsafe to use in noinstr code; use raw_atom
464 *
465 * Return: The updated value of @v.
466 */
467 static __always_inline int
468 atomic_inc_return_acquire(atomic_t *v)
469 {
470 instrument_atomic_read_write(v, sizeof
471 return raw_atomic_inc_return_acquire(v
472 }
473
474 /**
475 * atomic_inc_return_release() - atomic increm
476 * @v: pointer to atomic_t
477 *
478 * Atomically updates @v to (@v + 1) with rele
479 *
480 * Unsafe to use in noinstr code; use raw_atom
481 *
482 * Return: The updated value of @v.
483 */
484 static __always_inline int
485 atomic_inc_return_release(atomic_t *v)
486 {
487 kcsan_release();
488 instrument_atomic_read_write(v, sizeof
489 return raw_atomic_inc_return_release(v
490 }
491
492 /**
493 * atomic_inc_return_relaxed() - atomic increm
494 * @v: pointer to atomic_t
495 *
496 * Atomically updates @v to (@v + 1) with rela
497 *
498 * Unsafe to use in noinstr code; use raw_atom
499 *
500 * Return: The updated value of @v.
501 */
502 static __always_inline int
503 atomic_inc_return_relaxed(atomic_t *v)
504 {
505 instrument_atomic_read_write(v, sizeof
506 return raw_atomic_inc_return_relaxed(v
507 }
508
509 /**
510 * atomic_fetch_inc() - atomic increment with
511 * @v: pointer to atomic_t
512 *
513 * Atomically updates @v to (@v + 1) with full
514 *
515 * Unsafe to use in noinstr code; use raw_atom
516 *
517 * Return: The original value of @v.
518 */
519 static __always_inline int
520 atomic_fetch_inc(atomic_t *v)
521 {
522 kcsan_mb();
523 instrument_atomic_read_write(v, sizeof
524 return raw_atomic_fetch_inc(v);
525 }
526
527 /**
528 * atomic_fetch_inc_acquire() - atomic increme
529 * @v: pointer to atomic_t
530 *
531 * Atomically updates @v to (@v + 1) with acqu
532 *
533 * Unsafe to use in noinstr code; use raw_atom
534 *
535 * Return: The original value of @v.
536 */
537 static __always_inline int
538 atomic_fetch_inc_acquire(atomic_t *v)
539 {
540 instrument_atomic_read_write(v, sizeof
541 return raw_atomic_fetch_inc_acquire(v)
542 }
543
544 /**
545 * atomic_fetch_inc_release() - atomic increme
546 * @v: pointer to atomic_t
547 *
548 * Atomically updates @v to (@v + 1) with rele
549 *
550 * Unsafe to use in noinstr code; use raw_atom
551 *
552 * Return: The original value of @v.
553 */
554 static __always_inline int
555 atomic_fetch_inc_release(atomic_t *v)
556 {
557 kcsan_release();
558 instrument_atomic_read_write(v, sizeof
559 return raw_atomic_fetch_inc_release(v)
560 }
561
562 /**
563 * atomic_fetch_inc_relaxed() - atomic increme
564 * @v: pointer to atomic_t
565 *
566 * Atomically updates @v to (@v + 1) with rela
567 *
568 * Unsafe to use in noinstr code; use raw_atom
569 *
570 * Return: The original value of @v.
571 */
572 static __always_inline int
573 atomic_fetch_inc_relaxed(atomic_t *v)
574 {
575 instrument_atomic_read_write(v, sizeof
576 return raw_atomic_fetch_inc_relaxed(v)
577 }
578
579 /**
580 * atomic_dec() - atomic decrement with relaxe
581 * @v: pointer to atomic_t
582 *
583 * Atomically updates @v to (@v - 1) with rela
584 *
585 * Unsafe to use in noinstr code; use raw_atom
586 *
587 * Return: Nothing.
588 */
589 static __always_inline void
590 atomic_dec(atomic_t *v)
591 {
592 instrument_atomic_read_write(v, sizeof
593 raw_atomic_dec(v);
594 }
595
596 /**
597 * atomic_dec_return() - atomic decrement with
598 * @v: pointer to atomic_t
599 *
600 * Atomically updates @v to (@v - 1) with full
601 *
602 * Unsafe to use in noinstr code; use raw_atom
603 *
604 * Return: The updated value of @v.
605 */
606 static __always_inline int
607 atomic_dec_return(atomic_t *v)
608 {
609 kcsan_mb();
610 instrument_atomic_read_write(v, sizeof
611 return raw_atomic_dec_return(v);
612 }
613
614 /**
615 * atomic_dec_return_acquire() - atomic decrem
616 * @v: pointer to atomic_t
617 *
618 * Atomically updates @v to (@v - 1) with acqu
619 *
620 * Unsafe to use in noinstr code; use raw_atom
621 *
622 * Return: The updated value of @v.
623 */
624 static __always_inline int
625 atomic_dec_return_acquire(atomic_t *v)
626 {
627 instrument_atomic_read_write(v, sizeof
628 return raw_atomic_dec_return_acquire(v
629 }
630
631 /**
632 * atomic_dec_return_release() - atomic decrem
633 * @v: pointer to atomic_t
634 *
635 * Atomically updates @v to (@v - 1) with rele
636 *
637 * Unsafe to use in noinstr code; use raw_atom
638 *
639 * Return: The updated value of @v.
640 */
641 static __always_inline int
642 atomic_dec_return_release(atomic_t *v)
643 {
644 kcsan_release();
645 instrument_atomic_read_write(v, sizeof
646 return raw_atomic_dec_return_release(v
647 }
648
649 /**
650 * atomic_dec_return_relaxed() - atomic decrem
651 * @v: pointer to atomic_t
652 *
653 * Atomically updates @v to (@v - 1) with rela
654 *
655 * Unsafe to use in noinstr code; use raw_atom
656 *
657 * Return: The updated value of @v.
658 */
659 static __always_inline int
660 atomic_dec_return_relaxed(atomic_t *v)
661 {
662 instrument_atomic_read_write(v, sizeof
663 return raw_atomic_dec_return_relaxed(v
664 }
665
666 /**
667 * atomic_fetch_dec() - atomic decrement with
668 * @v: pointer to atomic_t
669 *
670 * Atomically updates @v to (@v - 1) with full
671 *
672 * Unsafe to use in noinstr code; use raw_atom
673 *
674 * Return: The original value of @v.
675 */
676 static __always_inline int
677 atomic_fetch_dec(atomic_t *v)
678 {
679 kcsan_mb();
680 instrument_atomic_read_write(v, sizeof
681 return raw_atomic_fetch_dec(v);
682 }
683
684 /**
685 * atomic_fetch_dec_acquire() - atomic decreme
686 * @v: pointer to atomic_t
687 *
688 * Atomically updates @v to (@v - 1) with acqu
689 *
690 * Unsafe to use in noinstr code; use raw_atom
691 *
692 * Return: The original value of @v.
693 */
694 static __always_inline int
695 atomic_fetch_dec_acquire(atomic_t *v)
696 {
697 instrument_atomic_read_write(v, sizeof
698 return raw_atomic_fetch_dec_acquire(v)
699 }
700
701 /**
702 * atomic_fetch_dec_release() - atomic decreme
703 * @v: pointer to atomic_t
704 *
705 * Atomically updates @v to (@v - 1) with rele
706 *
707 * Unsafe to use in noinstr code; use raw_atom
708 *
709 * Return: The original value of @v.
710 */
711 static __always_inline int
712 atomic_fetch_dec_release(atomic_t *v)
713 {
714 kcsan_release();
715 instrument_atomic_read_write(v, sizeof
716 return raw_atomic_fetch_dec_release(v)
717 }
718
719 /**
720 * atomic_fetch_dec_relaxed() - atomic decreme
721 * @v: pointer to atomic_t
722 *
723 * Atomically updates @v to (@v - 1) with rela
724 *
725 * Unsafe to use in noinstr code; use raw_atom
726 *
727 * Return: The original value of @v.
728 */
729 static __always_inline int
730 atomic_fetch_dec_relaxed(atomic_t *v)
731 {
732 instrument_atomic_read_write(v, sizeof
733 return raw_atomic_fetch_dec_relaxed(v)
734 }
735
736 /**
737 * atomic_and() - atomic bitwise AND with rela
738 * @i: int value
739 * @v: pointer to atomic_t
740 *
741 * Atomically updates @v to (@v & @i) with rel
742 *
743 * Unsafe to use in noinstr code; use raw_atom
744 *
745 * Return: Nothing.
746 */
747 static __always_inline void
748 atomic_and(int i, atomic_t *v)
749 {
750 instrument_atomic_read_write(v, sizeof
751 raw_atomic_and(i, v);
752 }
753
754 /**
755 * atomic_fetch_and() - atomic bitwise AND wit
756 * @i: int value
757 * @v: pointer to atomic_t
758 *
759 * Atomically updates @v to (@v & @i) with ful
760 *
761 * Unsafe to use in noinstr code; use raw_atom
762 *
763 * Return: The original value of @v.
764 */
765 static __always_inline int
766 atomic_fetch_and(int i, atomic_t *v)
767 {
768 kcsan_mb();
769 instrument_atomic_read_write(v, sizeof
770 return raw_atomic_fetch_and(i, v);
771 }
772
773 /**
774 * atomic_fetch_and_acquire() - atomic bitwise
775 * @i: int value
776 * @v: pointer to atomic_t
777 *
778 * Atomically updates @v to (@v & @i) with acq
779 *
780 * Unsafe to use in noinstr code; use raw_atom
781 *
782 * Return: The original value of @v.
783 */
784 static __always_inline int
785 atomic_fetch_and_acquire(int i, atomic_t *v)
786 {
787 instrument_atomic_read_write(v, sizeof
788 return raw_atomic_fetch_and_acquire(i,
789 }
790
791 /**
792 * atomic_fetch_and_release() - atomic bitwise
793 * @i: int value
794 * @v: pointer to atomic_t
795 *
796 * Atomically updates @v to (@v & @i) with rel
797 *
798 * Unsafe to use in noinstr code; use raw_atom
799 *
800 * Return: The original value of @v.
801 */
802 static __always_inline int
803 atomic_fetch_and_release(int i, atomic_t *v)
804 {
805 kcsan_release();
806 instrument_atomic_read_write(v, sizeof
807 return raw_atomic_fetch_and_release(i,
808 }
809
810 /**
811 * atomic_fetch_and_relaxed() - atomic bitwise
812 * @i: int value
813 * @v: pointer to atomic_t
814 *
815 * Atomically updates @v to (@v & @i) with rel
816 *
817 * Unsafe to use in noinstr code; use raw_atom
818 *
819 * Return: The original value of @v.
820 */
821 static __always_inline int
822 atomic_fetch_and_relaxed(int i, atomic_t *v)
823 {
824 instrument_atomic_read_write(v, sizeof
825 return raw_atomic_fetch_and_relaxed(i,
826 }
827
828 /**
829 * atomic_andnot() - atomic bitwise AND NOT wi
830 * @i: int value
831 * @v: pointer to atomic_t
832 *
833 * Atomically updates @v to (@v & ~@i) with re
834 *
835 * Unsafe to use in noinstr code; use raw_atom
836 *
837 * Return: Nothing.
838 */
839 static __always_inline void
840 atomic_andnot(int i, atomic_t *v)
841 {
842 instrument_atomic_read_write(v, sizeof
843 raw_atomic_andnot(i, v);
844 }
845
846 /**
847 * atomic_fetch_andnot() - atomic bitwise AND
848 * @i: int value
849 * @v: pointer to atomic_t
850 *
851 * Atomically updates @v to (@v & ~@i) with fu
852 *
853 * Unsafe to use in noinstr code; use raw_atom
854 *
855 * Return: The original value of @v.
856 */
857 static __always_inline int
858 atomic_fetch_andnot(int i, atomic_t *v)
859 {
860 kcsan_mb();
861 instrument_atomic_read_write(v, sizeof
862 return raw_atomic_fetch_andnot(i, v);
863 }
864
865 /**
866 * atomic_fetch_andnot_acquire() - atomic bitw
867 * @i: int value
868 * @v: pointer to atomic_t
869 *
870 * Atomically updates @v to (@v & ~@i) with ac
871 *
872 * Unsafe to use in noinstr code; use raw_atom
873 *
874 * Return: The original value of @v.
875 */
876 static __always_inline int
877 atomic_fetch_andnot_acquire(int i, atomic_t *v
878 {
879 instrument_atomic_read_write(v, sizeof
880 return raw_atomic_fetch_andnot_acquire
881 }
882
883 /**
884 * atomic_fetch_andnot_release() - atomic bitw
885 * @i: int value
886 * @v: pointer to atomic_t
887 *
888 * Atomically updates @v to (@v & ~@i) with re
889 *
890 * Unsafe to use in noinstr code; use raw_atom
891 *
892 * Return: The original value of @v.
893 */
894 static __always_inline int
895 atomic_fetch_andnot_release(int i, atomic_t *v
896 {
897 kcsan_release();
898 instrument_atomic_read_write(v, sizeof
899 return raw_atomic_fetch_andnot_release
900 }
901
902 /**
903 * atomic_fetch_andnot_relaxed() - atomic bitw
904 * @i: int value
905 * @v: pointer to atomic_t
906 *
907 * Atomically updates @v to (@v & ~@i) with re
908 *
909 * Unsafe to use in noinstr code; use raw_atom
910 *
911 * Return: The original value of @v.
912 */
913 static __always_inline int
914 atomic_fetch_andnot_relaxed(int i, atomic_t *v
915 {
916 instrument_atomic_read_write(v, sizeof
917 return raw_atomic_fetch_andnot_relaxed
918 }
919
920 /**
921 * atomic_or() - atomic bitwise OR with relaxe
922 * @i: int value
923 * @v: pointer to atomic_t
924 *
925 * Atomically updates @v to (@v | @i) with rel
926 *
927 * Unsafe to use in noinstr code; use raw_atom
928 *
929 * Return: Nothing.
930 */
931 static __always_inline void
932 atomic_or(int i, atomic_t *v)
933 {
934 instrument_atomic_read_write(v, sizeof
935 raw_atomic_or(i, v);
936 }
937
938 /**
939 * atomic_fetch_or() - atomic bitwise OR with
940 * @i: int value
941 * @v: pointer to atomic_t
942 *
943 * Atomically updates @v to (@v | @i) with ful
944 *
945 * Unsafe to use in noinstr code; use raw_atom
946 *
947 * Return: The original value of @v.
948 */
949 static __always_inline int
950 atomic_fetch_or(int i, atomic_t *v)
951 {
952 kcsan_mb();
953 instrument_atomic_read_write(v, sizeof
954 return raw_atomic_fetch_or(i, v);
955 }
956
957 /**
958 * atomic_fetch_or_acquire() - atomic bitwise
959 * @i: int value
960 * @v: pointer to atomic_t
961 *
962 * Atomically updates @v to (@v | @i) with acq
963 *
964 * Unsafe to use in noinstr code; use raw_atom
965 *
966 * Return: The original value of @v.
967 */
968 static __always_inline int
969 atomic_fetch_or_acquire(int i, atomic_t *v)
970 {
971 instrument_atomic_read_write(v, sizeof
972 return raw_atomic_fetch_or_acquire(i,
973 }
974
975 /**
976 * atomic_fetch_or_release() - atomic bitwise
977 * @i: int value
978 * @v: pointer to atomic_t
979 *
980 * Atomically updates @v to (@v | @i) with rel
981 *
982 * Unsafe to use in noinstr code; use raw_atom
983 *
984 * Return: The original value of @v.
985 */
986 static __always_inline int
987 atomic_fetch_or_release(int i, atomic_t *v)
988 {
989 kcsan_release();
990 instrument_atomic_read_write(v, sizeof
991 return raw_atomic_fetch_or_release(i,
992 }
993
994 /**
995 * atomic_fetch_or_relaxed() - atomic bitwise
996 * @i: int value
997 * @v: pointer to atomic_t
998 *
999 * Atomically updates @v to (@v | @i) with rel
1000 *
1001 * Unsafe to use in noinstr code; use raw_ato
1002 *
1003 * Return: The original value of @v.
1004 */
1005 static __always_inline int
1006 atomic_fetch_or_relaxed(int i, atomic_t *v)
1007 {
1008 instrument_atomic_read_write(v, sizeo
1009 return raw_atomic_fetch_or_relaxed(i,
1010 }
1011
1012 /**
1013 * atomic_xor() - atomic bitwise XOR with rel
1014 * @i: int value
1015 * @v: pointer to atomic_t
1016 *
1017 * Atomically updates @v to (@v ^ @i) with re
1018 *
1019 * Unsafe to use in noinstr code; use raw_ato
1020 *
1021 * Return: Nothing.
1022 */
1023 static __always_inline void
1024 atomic_xor(int i, atomic_t *v)
1025 {
1026 instrument_atomic_read_write(v, sizeo
1027 raw_atomic_xor(i, v);
1028 }
1029
1030 /**
1031 * atomic_fetch_xor() - atomic bitwise XOR wi
1032 * @i: int value
1033 * @v: pointer to atomic_t
1034 *
1035 * Atomically updates @v to (@v ^ @i) with fu
1036 *
1037 * Unsafe to use in noinstr code; use raw_ato
1038 *
1039 * Return: The original value of @v.
1040 */
1041 static __always_inline int
1042 atomic_fetch_xor(int i, atomic_t *v)
1043 {
1044 kcsan_mb();
1045 instrument_atomic_read_write(v, sizeo
1046 return raw_atomic_fetch_xor(i, v);
1047 }
1048
1049 /**
1050 * atomic_fetch_xor_acquire() - atomic bitwis
1051 * @i: int value
1052 * @v: pointer to atomic_t
1053 *
1054 * Atomically updates @v to (@v ^ @i) with ac
1055 *
1056 * Unsafe to use in noinstr code; use raw_ato
1057 *
1058 * Return: The original value of @v.
1059 */
1060 static __always_inline int
1061 atomic_fetch_xor_acquire(int i, atomic_t *v)
1062 {
1063 instrument_atomic_read_write(v, sizeo
1064 return raw_atomic_fetch_xor_acquire(i
1065 }
1066
1067 /**
1068 * atomic_fetch_xor_release() - atomic bitwis
1069 * @i: int value
1070 * @v: pointer to atomic_t
1071 *
1072 * Atomically updates @v to (@v ^ @i) with re
1073 *
1074 * Unsafe to use in noinstr code; use raw_ato
1075 *
1076 * Return: The original value of @v.
1077 */
1078 static __always_inline int
1079 atomic_fetch_xor_release(int i, atomic_t *v)
1080 {
1081 kcsan_release();
1082 instrument_atomic_read_write(v, sizeo
1083 return raw_atomic_fetch_xor_release(i
1084 }
1085
1086 /**
1087 * atomic_fetch_xor_relaxed() - atomic bitwis
1088 * @i: int value
1089 * @v: pointer to atomic_t
1090 *
1091 * Atomically updates @v to (@v ^ @i) with re
1092 *
1093 * Unsafe to use in noinstr code; use raw_ato
1094 *
1095 * Return: The original value of @v.
1096 */
1097 static __always_inline int
1098 atomic_fetch_xor_relaxed(int i, atomic_t *v)
1099 {
1100 instrument_atomic_read_write(v, sizeo
1101 return raw_atomic_fetch_xor_relaxed(i
1102 }
1103
1104 /**
1105 * atomic_xchg() - atomic exchange with full
1106 * @v: pointer to atomic_t
1107 * @new: int value to assign
1108 *
1109 * Atomically updates @v to @new with full or
1110 *
1111 * Unsafe to use in noinstr code; use raw_ato
1112 *
1113 * Return: The original value of @v.
1114 */
1115 static __always_inline int
1116 atomic_xchg(atomic_t *v, int new)
1117 {
1118 kcsan_mb();
1119 instrument_atomic_read_write(v, sizeo
1120 return raw_atomic_xchg(v, new);
1121 }
1122
1123 /**
1124 * atomic_xchg_acquire() - atomic exchange wi
1125 * @v: pointer to atomic_t
1126 * @new: int value to assign
1127 *
1128 * Atomically updates @v to @new with acquire
1129 *
1130 * Unsafe to use in noinstr code; use raw_ato
1131 *
1132 * Return: The original value of @v.
1133 */
1134 static __always_inline int
1135 atomic_xchg_acquire(atomic_t *v, int new)
1136 {
1137 instrument_atomic_read_write(v, sizeo
1138 return raw_atomic_xchg_acquire(v, new
1139 }
1140
1141 /**
1142 * atomic_xchg_release() - atomic exchange wi
1143 * @v: pointer to atomic_t
1144 * @new: int value to assign
1145 *
1146 * Atomically updates @v to @new with release
1147 *
1148 * Unsafe to use in noinstr code; use raw_ato
1149 *
1150 * Return: The original value of @v.
1151 */
1152 static __always_inline int
1153 atomic_xchg_release(atomic_t *v, int new)
1154 {
1155 kcsan_release();
1156 instrument_atomic_read_write(v, sizeo
1157 return raw_atomic_xchg_release(v, new
1158 }
1159
1160 /**
1161 * atomic_xchg_relaxed() - atomic exchange wi
1162 * @v: pointer to atomic_t
1163 * @new: int value to assign
1164 *
1165 * Atomically updates @v to @new with relaxed
1166 *
1167 * Unsafe to use in noinstr code; use raw_ato
1168 *
1169 * Return: The original value of @v.
1170 */
1171 static __always_inline int
1172 atomic_xchg_relaxed(atomic_t *v, int new)
1173 {
1174 instrument_atomic_read_write(v, sizeo
1175 return raw_atomic_xchg_relaxed(v, new
1176 }
1177
1178 /**
1179 * atomic_cmpxchg() - atomic compare and exch
1180 * @v: pointer to atomic_t
1181 * @old: int value to compare with
1182 * @new: int value to assign
1183 *
1184 * If (@v == @old), atomically updates @v to
1185 * Otherwise, @v is not modified and relaxed
1186 *
1187 * Unsafe to use in noinstr code; use raw_ato
1188 *
1189 * Return: The original value of @v.
1190 */
1191 static __always_inline int
1192 atomic_cmpxchg(atomic_t *v, int old, int new)
1193 {
1194 kcsan_mb();
1195 instrument_atomic_read_write(v, sizeo
1196 return raw_atomic_cmpxchg(v, old, new
1197 }
1198
1199 /**
1200 * atomic_cmpxchg_acquire() - atomic compare
1201 * @v: pointer to atomic_t
1202 * @old: int value to compare with
1203 * @new: int value to assign
1204 *
1205 * If (@v == @old), atomically updates @v to
1206 * Otherwise, @v is not modified and relaxed
1207 *
1208 * Unsafe to use in noinstr code; use raw_ato
1209 *
1210 * Return: The original value of @v.
1211 */
1212 static __always_inline int
1213 atomic_cmpxchg_acquire(atomic_t *v, int old,
1214 {
1215 instrument_atomic_read_write(v, sizeo
1216 return raw_atomic_cmpxchg_acquire(v,
1217 }
1218
1219 /**
1220 * atomic_cmpxchg_release() - atomic compare
1221 * @v: pointer to atomic_t
1222 * @old: int value to compare with
1223 * @new: int value to assign
1224 *
1225 * If (@v == @old), atomically updates @v to
1226 * Otherwise, @v is not modified and relaxed
1227 *
1228 * Unsafe to use in noinstr code; use raw_ato
1229 *
1230 * Return: The original value of @v.
1231 */
1232 static __always_inline int
1233 atomic_cmpxchg_release(atomic_t *v, int old,
1234 {
1235 kcsan_release();
1236 instrument_atomic_read_write(v, sizeo
1237 return raw_atomic_cmpxchg_release(v,
1238 }
1239
1240 /**
1241 * atomic_cmpxchg_relaxed() - atomic compare
1242 * @v: pointer to atomic_t
1243 * @old: int value to compare with
1244 * @new: int value to assign
1245 *
1246 * If (@v == @old), atomically updates @v to
1247 * Otherwise, @v is not modified and relaxed
1248 *
1249 * Unsafe to use in noinstr code; use raw_ato
1250 *
1251 * Return: The original value of @v.
1252 */
1253 static __always_inline int
1254 atomic_cmpxchg_relaxed(atomic_t *v, int old,
1255 {
1256 instrument_atomic_read_write(v, sizeo
1257 return raw_atomic_cmpxchg_relaxed(v,
1258 }
1259
1260 /**
1261 * atomic_try_cmpxchg() - atomic compare and
1262 * @v: pointer to atomic_t
1263 * @old: pointer to int value to compare with
1264 * @new: int value to assign
1265 *
1266 * If (@v == @old), atomically updates @v to
1267 * Otherwise, @v is not modified, @old is upd
1268 * and relaxed ordering is provided.
1269 *
1270 * Unsafe to use in noinstr code; use raw_ato
1271 *
1272 * Return: @true if the exchange occured, @fa
1273 */
1274 static __always_inline bool
1275 atomic_try_cmpxchg(atomic_t *v, int *old, int
1276 {
1277 kcsan_mb();
1278 instrument_atomic_read_write(v, sizeo
1279 instrument_atomic_read_write(old, siz
1280 return raw_atomic_try_cmpxchg(v, old,
1281 }
1282
1283 /**
1284 * atomic_try_cmpxchg_acquire() - atomic comp
1285 * @v: pointer to atomic_t
1286 * @old: pointer to int value to compare with
1287 * @new: int value to assign
1288 *
1289 * If (@v == @old), atomically updates @v to
1290 * Otherwise, @v is not modified, @old is upd
1291 * and relaxed ordering is provided.
1292 *
1293 * Unsafe to use in noinstr code; use raw_ato
1294 *
1295 * Return: @true if the exchange occured, @fa
1296 */
1297 static __always_inline bool
1298 atomic_try_cmpxchg_acquire(atomic_t *v, int *
1299 {
1300 instrument_atomic_read_write(v, sizeo
1301 instrument_atomic_read_write(old, siz
1302 return raw_atomic_try_cmpxchg_acquire
1303 }
1304
1305 /**
1306 * atomic_try_cmpxchg_release() - atomic comp
1307 * @v: pointer to atomic_t
1308 * @old: pointer to int value to compare with
1309 * @new: int value to assign
1310 *
1311 * If (@v == @old), atomically updates @v to
1312 * Otherwise, @v is not modified, @old is upd
1313 * and relaxed ordering is provided.
1314 *
1315 * Unsafe to use in noinstr code; use raw_ato
1316 *
1317 * Return: @true if the exchange occured, @fa
1318 */
1319 static __always_inline bool
1320 atomic_try_cmpxchg_release(atomic_t *v, int *
1321 {
1322 kcsan_release();
1323 instrument_atomic_read_write(v, sizeo
1324 instrument_atomic_read_write(old, siz
1325 return raw_atomic_try_cmpxchg_release
1326 }
1327
1328 /**
1329 * atomic_try_cmpxchg_relaxed() - atomic comp
1330 * @v: pointer to atomic_t
1331 * @old: pointer to int value to compare with
1332 * @new: int value to assign
1333 *
1334 * If (@v == @old), atomically updates @v to
1335 * Otherwise, @v is not modified, @old is upd
1336 * and relaxed ordering is provided.
1337 *
1338 * Unsafe to use in noinstr code; use raw_ato
1339 *
1340 * Return: @true if the exchange occured, @fa
1341 */
1342 static __always_inline bool
1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int *
1344 {
1345 instrument_atomic_read_write(v, sizeo
1346 instrument_atomic_read_write(old, siz
1347 return raw_atomic_try_cmpxchg_relaxed
1348 }
1349
1350 /**
1351 * atomic_sub_and_test() - atomic subtract an
1352 * @i: int value to subtract
1353 * @v: pointer to atomic_t
1354 *
1355 * Atomically updates @v to (@v - @i) with fu
1356 *
1357 * Unsafe to use in noinstr code; use raw_ato
1358 *
1359 * Return: @true if the resulting value of @v
1360 */
1361 static __always_inline bool
1362 atomic_sub_and_test(int i, atomic_t *v)
1363 {
1364 kcsan_mb();
1365 instrument_atomic_read_write(v, sizeo
1366 return raw_atomic_sub_and_test(i, v);
1367 }
1368
1369 /**
1370 * atomic_dec_and_test() - atomic decrement a
1371 * @v: pointer to atomic_t
1372 *
1373 * Atomically updates @v to (@v - 1) with ful
1374 *
1375 * Unsafe to use in noinstr code; use raw_ato
1376 *
1377 * Return: @true if the resulting value of @v
1378 */
1379 static __always_inline bool
1380 atomic_dec_and_test(atomic_t *v)
1381 {
1382 kcsan_mb();
1383 instrument_atomic_read_write(v, sizeo
1384 return raw_atomic_dec_and_test(v);
1385 }
1386
1387 /**
1388 * atomic_inc_and_test() - atomic increment a
1389 * @v: pointer to atomic_t
1390 *
1391 * Atomically updates @v to (@v + 1) with ful
1392 *
1393 * Unsafe to use in noinstr code; use raw_ato
1394 *
1395 * Return: @true if the resulting value of @v
1396 */
1397 static __always_inline bool
1398 atomic_inc_and_test(atomic_t *v)
1399 {
1400 kcsan_mb();
1401 instrument_atomic_read_write(v, sizeo
1402 return raw_atomic_inc_and_test(v);
1403 }
1404
1405 /**
1406 * atomic_add_negative() - atomic add and tes
1407 * @i: int value to add
1408 * @v: pointer to atomic_t
1409 *
1410 * Atomically updates @v to (@v + @i) with fu
1411 *
1412 * Unsafe to use in noinstr code; use raw_ato
1413 *
1414 * Return: @true if the resulting value of @v
1415 */
1416 static __always_inline bool
1417 atomic_add_negative(int i, atomic_t *v)
1418 {
1419 kcsan_mb();
1420 instrument_atomic_read_write(v, sizeo
1421 return raw_atomic_add_negative(i, v);
1422 }
1423
1424 /**
1425 * atomic_add_negative_acquire() - atomic add
1426 * @i: int value to add
1427 * @v: pointer to atomic_t
1428 *
1429 * Atomically updates @v to (@v + @i) with ac
1430 *
1431 * Unsafe to use in noinstr code; use raw_ato
1432 *
1433 * Return: @true if the resulting value of @v
1434 */
1435 static __always_inline bool
1436 atomic_add_negative_acquire(int i, atomic_t *
1437 {
1438 instrument_atomic_read_write(v, sizeo
1439 return raw_atomic_add_negative_acquir
1440 }
1441
1442 /**
1443 * atomic_add_negative_release() - atomic add
1444 * @i: int value to add
1445 * @v: pointer to atomic_t
1446 *
1447 * Atomically updates @v to (@v + @i) with re
1448 *
1449 * Unsafe to use in noinstr code; use raw_ato
1450 *
1451 * Return: @true if the resulting value of @v
1452 */
1453 static __always_inline bool
1454 atomic_add_negative_release(int i, atomic_t *
1455 {
1456 kcsan_release();
1457 instrument_atomic_read_write(v, sizeo
1458 return raw_atomic_add_negative_releas
1459 }
1460
1461 /**
1462 * atomic_add_negative_relaxed() - atomic add
1463 * @i: int value to add
1464 * @v: pointer to atomic_t
1465 *
1466 * Atomically updates @v to (@v + @i) with re
1467 *
1468 * Unsafe to use in noinstr code; use raw_ato
1469 *
1470 * Return: @true if the resulting value of @v
1471 */
1472 static __always_inline bool
1473 atomic_add_negative_relaxed(int i, atomic_t *
1474 {
1475 instrument_atomic_read_write(v, sizeo
1476 return raw_atomic_add_negative_relaxe
1477 }
1478
1479 /**
1480 * atomic_fetch_add_unless() - atomic add unl
1481 * @v: pointer to atomic_t
1482 * @a: int value to add
1483 * @u: int value to compare with
1484 *
1485 * If (@v != @u), atomically updates @v to (@
1486 * Otherwise, @v is not modified and relaxed
1487 *
1488 * Unsafe to use in noinstr code; use raw_ato
1489 *
1490 * Return: The original value of @v.
1491 */
1492 static __always_inline int
1493 atomic_fetch_add_unless(atomic_t *v, int a, i
1494 {
1495 kcsan_mb();
1496 instrument_atomic_read_write(v, sizeo
1497 return raw_atomic_fetch_add_unless(v,
1498 }
1499
1500 /**
1501 * atomic_add_unless() - atomic add unless va
1502 * @v: pointer to atomic_t
1503 * @a: int value to add
1504 * @u: int value to compare with
1505 *
1506 * If (@v != @u), atomically updates @v to (@
1507 * Otherwise, @v is not modified and relaxed
1508 *
1509 * Unsafe to use in noinstr code; use raw_ato
1510 *
1511 * Return: @true if @v was updated, @false ot
1512 */
1513 static __always_inline bool
1514 atomic_add_unless(atomic_t *v, int a, int u)
1515 {
1516 kcsan_mb();
1517 instrument_atomic_read_write(v, sizeo
1518 return raw_atomic_add_unless(v, a, u)
1519 }
1520
1521 /**
1522 * atomic_inc_not_zero() - atomic increment u
1523 * @v: pointer to atomic_t
1524 *
1525 * If (@v != 0), atomically updates @v to (@v
1526 * Otherwise, @v is not modified and relaxed
1527 *
1528 * Unsafe to use in noinstr code; use raw_ato
1529 *
1530 * Return: @true if @v was updated, @false ot
1531 */
1532 static __always_inline bool
1533 atomic_inc_not_zero(atomic_t *v)
1534 {
1535 kcsan_mb();
1536 instrument_atomic_read_write(v, sizeo
1537 return raw_atomic_inc_not_zero(v);
1538 }
1539
1540 /**
1541 * atomic_inc_unless_negative() - atomic incr
1542 * @v: pointer to atomic_t
1543 *
1544 * If (@v >= 0), atomically updates @v to (@v
1545 * Otherwise, @v is not modified and relaxed
1546 *
1547 * Unsafe to use in noinstr code; use raw_ato
1548 *
1549 * Return: @true if @v was updated, @false ot
1550 */
1551 static __always_inline bool
1552 atomic_inc_unless_negative(atomic_t *v)
1553 {
1554 kcsan_mb();
1555 instrument_atomic_read_write(v, sizeo
1556 return raw_atomic_inc_unless_negative
1557 }
1558
1559 /**
1560 * atomic_dec_unless_positive() - atomic decr
1561 * @v: pointer to atomic_t
1562 *
1563 * If (@v <= 0), atomically updates @v to (@v
1564 * Otherwise, @v is not modified and relaxed
1565 *
1566 * Unsafe to use in noinstr code; use raw_ato
1567 *
1568 * Return: @true if @v was updated, @false ot
1569 */
1570 static __always_inline bool
1571 atomic_dec_unless_positive(atomic_t *v)
1572 {
1573 kcsan_mb();
1574 instrument_atomic_read_write(v, sizeo
1575 return raw_atomic_dec_unless_positive
1576 }
1577
1578 /**
1579 * atomic_dec_if_positive() - atomic decremen
1580 * @v: pointer to atomic_t
1581 *
1582 * If (@v > 0), atomically updates @v to (@v
1583 * Otherwise, @v is not modified and relaxed
1584 *
1585 * Unsafe to use in noinstr code; use raw_ato
1586 *
1587 * Return: The old value of (@v - 1), regardl
1588 */
1589 static __always_inline int
1590 atomic_dec_if_positive(atomic_t *v)
1591 {
1592 kcsan_mb();
1593 instrument_atomic_read_write(v, sizeo
1594 return raw_atomic_dec_if_positive(v);
1595 }
1596
1597 /**
1598 * atomic64_read() - atomic load with relaxed
1599 * @v: pointer to atomic64_t
1600 *
1601 * Atomically loads the value of @v with rela
1602 *
1603 * Unsafe to use in noinstr code; use raw_ato
1604 *
1605 * Return: The value loaded from @v.
1606 */
1607 static __always_inline s64
1608 atomic64_read(const atomic64_t *v)
1609 {
1610 instrument_atomic_read(v, sizeof(*v))
1611 return raw_atomic64_read(v);
1612 }
1613
1614 /**
1615 * atomic64_read_acquire() - atomic load with
1616 * @v: pointer to atomic64_t
1617 *
1618 * Atomically loads the value of @v with acqu
1619 *
1620 * Unsafe to use in noinstr code; use raw_ato
1621 *
1622 * Return: The value loaded from @v.
1623 */
1624 static __always_inline s64
1625 atomic64_read_acquire(const atomic64_t *v)
1626 {
1627 instrument_atomic_read(v, sizeof(*v))
1628 return raw_atomic64_read_acquire(v);
1629 }
1630
1631 /**
1632 * atomic64_set() - atomic set with relaxed o
1633 * @v: pointer to atomic64_t
1634 * @i: s64 value to assign
1635 *
1636 * Atomically sets @v to @i with relaxed orde
1637 *
1638 * Unsafe to use in noinstr code; use raw_ato
1639 *
1640 * Return: Nothing.
1641 */
1642 static __always_inline void
1643 atomic64_set(atomic64_t *v, s64 i)
1644 {
1645 instrument_atomic_write(v, sizeof(*v)
1646 raw_atomic64_set(v, i);
1647 }
1648
1649 /**
1650 * atomic64_set_release() - atomic set with r
1651 * @v: pointer to atomic64_t
1652 * @i: s64 value to assign
1653 *
1654 * Atomically sets @v to @i with release orde
1655 *
1656 * Unsafe to use in noinstr code; use raw_ato
1657 *
1658 * Return: Nothing.
1659 */
1660 static __always_inline void
1661 atomic64_set_release(atomic64_t *v, s64 i)
1662 {
1663 kcsan_release();
1664 instrument_atomic_write(v, sizeof(*v)
1665 raw_atomic64_set_release(v, i);
1666 }
1667
1668 /**
1669 * atomic64_add() - atomic add with relaxed o
1670 * @i: s64 value to add
1671 * @v: pointer to atomic64_t
1672 *
1673 * Atomically updates @v to (@v + @i) with re
1674 *
1675 * Unsafe to use in noinstr code; use raw_ato
1676 *
1677 * Return: Nothing.
1678 */
1679 static __always_inline void
1680 atomic64_add(s64 i, atomic64_t *v)
1681 {
1682 instrument_atomic_read_write(v, sizeo
1683 raw_atomic64_add(i, v);
1684 }
1685
1686 /**
1687 * atomic64_add_return() - atomic add with fu
1688 * @i: s64 value to add
1689 * @v: pointer to atomic64_t
1690 *
1691 * Atomically updates @v to (@v + @i) with fu
1692 *
1693 * Unsafe to use in noinstr code; use raw_ato
1694 *
1695 * Return: The updated value of @v.
1696 */
1697 static __always_inline s64
1698 atomic64_add_return(s64 i, atomic64_t *v)
1699 {
1700 kcsan_mb();
1701 instrument_atomic_read_write(v, sizeo
1702 return raw_atomic64_add_return(i, v);
1703 }
1704
1705 /**
1706 * atomic64_add_return_acquire() - atomic add
1707 * @i: s64 value to add
1708 * @v: pointer to atomic64_t
1709 *
1710 * Atomically updates @v to (@v + @i) with ac
1711 *
1712 * Unsafe to use in noinstr code; use raw_ato
1713 *
1714 * Return: The updated value of @v.
1715 */
1716 static __always_inline s64
1717 atomic64_add_return_acquire(s64 i, atomic64_t
1718 {
1719 instrument_atomic_read_write(v, sizeo
1720 return raw_atomic64_add_return_acquir
1721 }
1722
1723 /**
1724 * atomic64_add_return_release() - atomic add
1725 * @i: s64 value to add
1726 * @v: pointer to atomic64_t
1727 *
1728 * Atomically updates @v to (@v + @i) with re
1729 *
1730 * Unsafe to use in noinstr code; use raw_ato
1731 *
1732 * Return: The updated value of @v.
1733 */
1734 static __always_inline s64
1735 atomic64_add_return_release(s64 i, atomic64_t
1736 {
1737 kcsan_release();
1738 instrument_atomic_read_write(v, sizeo
1739 return raw_atomic64_add_return_releas
1740 }
1741
1742 /**
1743 * atomic64_add_return_relaxed() - atomic add
1744 * @i: s64 value to add
1745 * @v: pointer to atomic64_t
1746 *
1747 * Atomically updates @v to (@v + @i) with re
1748 *
1749 * Unsafe to use in noinstr code; use raw_ato
1750 *
1751 * Return: The updated value of @v.
1752 */
1753 static __always_inline s64
1754 atomic64_add_return_relaxed(s64 i, atomic64_t
1755 {
1756 instrument_atomic_read_write(v, sizeo
1757 return raw_atomic64_add_return_relaxe
1758 }
1759
1760 /**
1761 * atomic64_fetch_add() - atomic add with ful
1762 * @i: s64 value to add
1763 * @v: pointer to atomic64_t
1764 *
1765 * Atomically updates @v to (@v + @i) with fu
1766 *
1767 * Unsafe to use in noinstr code; use raw_ato
1768 *
1769 * Return: The original value of @v.
1770 */
1771 static __always_inline s64
1772 atomic64_fetch_add(s64 i, atomic64_t *v)
1773 {
1774 kcsan_mb();
1775 instrument_atomic_read_write(v, sizeo
1776 return raw_atomic64_fetch_add(i, v);
1777 }
1778
1779 /**
1780 * atomic64_fetch_add_acquire() - atomic add
1781 * @i: s64 value to add
1782 * @v: pointer to atomic64_t
1783 *
1784 * Atomically updates @v to (@v + @i) with ac
1785 *
1786 * Unsafe to use in noinstr code; use raw_ato
1787 *
1788 * Return: The original value of @v.
1789 */
1790 static __always_inline s64
1791 atomic64_fetch_add_acquire(s64 i, atomic64_t
1792 {
1793 instrument_atomic_read_write(v, sizeo
1794 return raw_atomic64_fetch_add_acquire
1795 }
1796
1797 /**
1798 * atomic64_fetch_add_release() - atomic add
1799 * @i: s64 value to add
1800 * @v: pointer to atomic64_t
1801 *
1802 * Atomically updates @v to (@v + @i) with re
1803 *
1804 * Unsafe to use in noinstr code; use raw_ato
1805 *
1806 * Return: The original value of @v.
1807 */
1808 static __always_inline s64
1809 atomic64_fetch_add_release(s64 i, atomic64_t
1810 {
1811 kcsan_release();
1812 instrument_atomic_read_write(v, sizeo
1813 return raw_atomic64_fetch_add_release
1814 }
1815
1816 /**
1817 * atomic64_fetch_add_relaxed() - atomic add
1818 * @i: s64 value to add
1819 * @v: pointer to atomic64_t
1820 *
1821 * Atomically updates @v to (@v + @i) with re
1822 *
1823 * Unsafe to use in noinstr code; use raw_ato
1824 *
1825 * Return: The original value of @v.
1826 */
1827 static __always_inline s64
1828 atomic64_fetch_add_relaxed(s64 i, atomic64_t
1829 {
1830 instrument_atomic_read_write(v, sizeo
1831 return raw_atomic64_fetch_add_relaxed
1832 }
1833
1834 /**
1835 * atomic64_sub() - atomic subtract with rela
1836 * @i: s64 value to subtract
1837 * @v: pointer to atomic64_t
1838 *
1839 * Atomically updates @v to (@v - @i) with re
1840 *
1841 * Unsafe to use in noinstr code; use raw_ato
1842 *
1843 * Return: Nothing.
1844 */
1845 static __always_inline void
1846 atomic64_sub(s64 i, atomic64_t *v)
1847 {
1848 instrument_atomic_read_write(v, sizeo
1849 raw_atomic64_sub(i, v);
1850 }
1851
1852 /**
1853 * atomic64_sub_return() - atomic subtract wi
1854 * @i: s64 value to subtract
1855 * @v: pointer to atomic64_t
1856 *
1857 * Atomically updates @v to (@v - @i) with fu
1858 *
1859 * Unsafe to use in noinstr code; use raw_ato
1860 *
1861 * Return: The updated value of @v.
1862 */
1863 static __always_inline s64
1864 atomic64_sub_return(s64 i, atomic64_t *v)
1865 {
1866 kcsan_mb();
1867 instrument_atomic_read_write(v, sizeo
1868 return raw_atomic64_sub_return(i, v);
1869 }
1870
1871 /**
1872 * atomic64_sub_return_acquire() - atomic sub
1873 * @i: s64 value to subtract
1874 * @v: pointer to atomic64_t
1875 *
1876 * Atomically updates @v to (@v - @i) with ac
1877 *
1878 * Unsafe to use in noinstr code; use raw_ato
1879 *
1880 * Return: The updated value of @v.
1881 */
1882 static __always_inline s64
1883 atomic64_sub_return_acquire(s64 i, atomic64_t
1884 {
1885 instrument_atomic_read_write(v, sizeo
1886 return raw_atomic64_sub_return_acquir
1887 }
1888
1889 /**
1890 * atomic64_sub_return_release() - atomic sub
1891 * @i: s64 value to subtract
1892 * @v: pointer to atomic64_t
1893 *
1894 * Atomically updates @v to (@v - @i) with re
1895 *
1896 * Unsafe to use in noinstr code; use raw_ato
1897 *
1898 * Return: The updated value of @v.
1899 */
1900 static __always_inline s64
1901 atomic64_sub_return_release(s64 i, atomic64_t
1902 {
1903 kcsan_release();
1904 instrument_atomic_read_write(v, sizeo
1905 return raw_atomic64_sub_return_releas
1906 }
1907
1908 /**
1909 * atomic64_sub_return_relaxed() - atomic sub
1910 * @i: s64 value to subtract
1911 * @v: pointer to atomic64_t
1912 *
1913 * Atomically updates @v to (@v - @i) with re
1914 *
1915 * Unsafe to use in noinstr code; use raw_ato
1916 *
1917 * Return: The updated value of @v.
1918 */
1919 static __always_inline s64
1920 atomic64_sub_return_relaxed(s64 i, atomic64_t
1921 {
1922 instrument_atomic_read_write(v, sizeo
1923 return raw_atomic64_sub_return_relaxe
1924 }
1925
1926 /**
1927 * atomic64_fetch_sub() - atomic subtract wit
1928 * @i: s64 value to subtract
1929 * @v: pointer to atomic64_t
1930 *
1931 * Atomically updates @v to (@v - @i) with fu
1932 *
1933 * Unsafe to use in noinstr code; use raw_ato
1934 *
1935 * Return: The original value of @v.
1936 */
1937 static __always_inline s64
1938 atomic64_fetch_sub(s64 i, atomic64_t *v)
1939 {
1940 kcsan_mb();
1941 instrument_atomic_read_write(v, sizeo
1942 return raw_atomic64_fetch_sub(i, v);
1943 }
1944
1945 /**
1946 * atomic64_fetch_sub_acquire() - atomic subt
1947 * @i: s64 value to subtract
1948 * @v: pointer to atomic64_t
1949 *
1950 * Atomically updates @v to (@v - @i) with ac
1951 *
1952 * Unsafe to use in noinstr code; use raw_ato
1953 *
1954 * Return: The original value of @v.
1955 */
1956 static __always_inline s64
1957 atomic64_fetch_sub_acquire(s64 i, atomic64_t
1958 {
1959 instrument_atomic_read_write(v, sizeo
1960 return raw_atomic64_fetch_sub_acquire
1961 }
1962
1963 /**
1964 * atomic64_fetch_sub_release() - atomic subt
1965 * @i: s64 value to subtract
1966 * @v: pointer to atomic64_t
1967 *
1968 * Atomically updates @v to (@v - @i) with re
1969 *
1970 * Unsafe to use in noinstr code; use raw_ato
1971 *
1972 * Return: The original value of @v.
1973 */
1974 static __always_inline s64
1975 atomic64_fetch_sub_release(s64 i, atomic64_t
1976 {
1977 kcsan_release();
1978 instrument_atomic_read_write(v, sizeo
1979 return raw_atomic64_fetch_sub_release
1980 }
1981
1982 /**
1983 * atomic64_fetch_sub_relaxed() - atomic subt
1984 * @i: s64 value to subtract
1985 * @v: pointer to atomic64_t
1986 *
1987 * Atomically updates @v to (@v - @i) with re
1988 *
1989 * Unsafe to use in noinstr code; use raw_ato
1990 *
1991 * Return: The original value of @v.
1992 */
1993 static __always_inline s64
1994 atomic64_fetch_sub_relaxed(s64 i, atomic64_t
1995 {
1996 instrument_atomic_read_write(v, sizeo
1997 return raw_atomic64_fetch_sub_relaxed
1998 }
1999
2000 /**
2001 * atomic64_inc() - atomic increment with rel
2002 * @v: pointer to atomic64_t
2003 *
2004 * Atomically updates @v to (@v + 1) with rel
2005 *
2006 * Unsafe to use in noinstr code; use raw_ato
2007 *
2008 * Return: Nothing.
2009 */
2010 static __always_inline void
2011 atomic64_inc(atomic64_t *v)
2012 {
2013 instrument_atomic_read_write(v, sizeo
2014 raw_atomic64_inc(v);
2015 }
2016
2017 /**
2018 * atomic64_inc_return() - atomic increment w
2019 * @v: pointer to atomic64_t
2020 *
2021 * Atomically updates @v to (@v + 1) with ful
2022 *
2023 * Unsafe to use in noinstr code; use raw_ato
2024 *
2025 * Return: The updated value of @v.
2026 */
2027 static __always_inline s64
2028 atomic64_inc_return(atomic64_t *v)
2029 {
2030 kcsan_mb();
2031 instrument_atomic_read_write(v, sizeo
2032 return raw_atomic64_inc_return(v);
2033 }
2034
2035 /**
2036 * atomic64_inc_return_acquire() - atomic inc
2037 * @v: pointer to atomic64_t
2038 *
2039 * Atomically updates @v to (@v + 1) with acq
2040 *
2041 * Unsafe to use in noinstr code; use raw_ato
2042 *
2043 * Return: The updated value of @v.
2044 */
2045 static __always_inline s64
2046 atomic64_inc_return_acquire(atomic64_t *v)
2047 {
2048 instrument_atomic_read_write(v, sizeo
2049 return raw_atomic64_inc_return_acquir
2050 }
2051
2052 /**
2053 * atomic64_inc_return_release() - atomic inc
2054 * @v: pointer to atomic64_t
2055 *
2056 * Atomically updates @v to (@v + 1) with rel
2057 *
2058 * Unsafe to use in noinstr code; use raw_ato
2059 *
2060 * Return: The updated value of @v.
2061 */
2062 static __always_inline s64
2063 atomic64_inc_return_release(atomic64_t *v)
2064 {
2065 kcsan_release();
2066 instrument_atomic_read_write(v, sizeo
2067 return raw_atomic64_inc_return_releas
2068 }
2069
2070 /**
2071 * atomic64_inc_return_relaxed() - atomic inc
2072 * @v: pointer to atomic64_t
2073 *
2074 * Atomically updates @v to (@v + 1) with rel
2075 *
2076 * Unsafe to use in noinstr code; use raw_ato
2077 *
2078 * Return: The updated value of @v.
2079 */
2080 static __always_inline s64
2081 atomic64_inc_return_relaxed(atomic64_t *v)
2082 {
2083 instrument_atomic_read_write(v, sizeo
2084 return raw_atomic64_inc_return_relaxe
2085 }
2086
2087 /**
2088 * atomic64_fetch_inc() - atomic increment wi
2089 * @v: pointer to atomic64_t
2090 *
2091 * Atomically updates @v to (@v + 1) with ful
2092 *
2093 * Unsafe to use in noinstr code; use raw_ato
2094 *
2095 * Return: The original value of @v.
2096 */
2097 static __always_inline s64
2098 atomic64_fetch_inc(atomic64_t *v)
2099 {
2100 kcsan_mb();
2101 instrument_atomic_read_write(v, sizeo
2102 return raw_atomic64_fetch_inc(v);
2103 }
2104
2105 /**
2106 * atomic64_fetch_inc_acquire() - atomic incr
2107 * @v: pointer to atomic64_t
2108 *
2109 * Atomically updates @v to (@v + 1) with acq
2110 *
2111 * Unsafe to use in noinstr code; use raw_ato
2112 *
2113 * Return: The original value of @v.
2114 */
2115 static __always_inline s64
2116 atomic64_fetch_inc_acquire(atomic64_t *v)
2117 {
2118 instrument_atomic_read_write(v, sizeo
2119 return raw_atomic64_fetch_inc_acquire
2120 }
2121
2122 /**
2123 * atomic64_fetch_inc_release() - atomic incr
2124 * @v: pointer to atomic64_t
2125 *
2126 * Atomically updates @v to (@v + 1) with rel
2127 *
2128 * Unsafe to use in noinstr code; use raw_ato
2129 *
2130 * Return: The original value of @v.
2131 */
2132 static __always_inline s64
2133 atomic64_fetch_inc_release(atomic64_t *v)
2134 {
2135 kcsan_release();
2136 instrument_atomic_read_write(v, sizeo
2137 return raw_atomic64_fetch_inc_release
2138 }
2139
2140 /**
2141 * atomic64_fetch_inc_relaxed() - atomic incr
2142 * @v: pointer to atomic64_t
2143 *
2144 * Atomically updates @v to (@v + 1) with rel
2145 *
2146 * Unsafe to use in noinstr code; use raw_ato
2147 *
2148 * Return: The original value of @v.
2149 */
2150 static __always_inline s64
2151 atomic64_fetch_inc_relaxed(atomic64_t *v)
2152 {
2153 instrument_atomic_read_write(v, sizeo
2154 return raw_atomic64_fetch_inc_relaxed
2155 }
2156
2157 /**
2158 * atomic64_dec() - atomic decrement with rel
2159 * @v: pointer to atomic64_t
2160 *
2161 * Atomically updates @v to (@v - 1) with rel
2162 *
2163 * Unsafe to use in noinstr code; use raw_ato
2164 *
2165 * Return: Nothing.
2166 */
2167 static __always_inline void
2168 atomic64_dec(atomic64_t *v)
2169 {
2170 instrument_atomic_read_write(v, sizeo
2171 raw_atomic64_dec(v);
2172 }
2173
2174 /**
2175 * atomic64_dec_return() - atomic decrement w
2176 * @v: pointer to atomic64_t
2177 *
2178 * Atomically updates @v to (@v - 1) with ful
2179 *
2180 * Unsafe to use in noinstr code; use raw_ato
2181 *
2182 * Return: The updated value of @v.
2183 */
2184 static __always_inline s64
2185 atomic64_dec_return(atomic64_t *v)
2186 {
2187 kcsan_mb();
2188 instrument_atomic_read_write(v, sizeo
2189 return raw_atomic64_dec_return(v);
2190 }
2191
2192 /**
2193 * atomic64_dec_return_acquire() - atomic dec
2194 * @v: pointer to atomic64_t
2195 *
2196 * Atomically updates @v to (@v - 1) with acq
2197 *
2198 * Unsafe to use in noinstr code; use raw_ato
2199 *
2200 * Return: The updated value of @v.
2201 */
2202 static __always_inline s64
2203 atomic64_dec_return_acquire(atomic64_t *v)
2204 {
2205 instrument_atomic_read_write(v, sizeo
2206 return raw_atomic64_dec_return_acquir
2207 }
2208
2209 /**
2210 * atomic64_dec_return_release() - atomic dec
2211 * @v: pointer to atomic64_t
2212 *
2213 * Atomically updates @v to (@v - 1) with rel
2214 *
2215 * Unsafe to use in noinstr code; use raw_ato
2216 *
2217 * Return: The updated value of @v.
2218 */
2219 static __always_inline s64
2220 atomic64_dec_return_release(atomic64_t *v)
2221 {
2222 kcsan_release();
2223 instrument_atomic_read_write(v, sizeo
2224 return raw_atomic64_dec_return_releas
2225 }
2226
2227 /**
2228 * atomic64_dec_return_relaxed() - atomic dec
2229 * @v: pointer to atomic64_t
2230 *
2231 * Atomically updates @v to (@v - 1) with rel
2232 *
2233 * Unsafe to use in noinstr code; use raw_ato
2234 *
2235 * Return: The updated value of @v.
2236 */
2237 static __always_inline s64
2238 atomic64_dec_return_relaxed(atomic64_t *v)
2239 {
2240 instrument_atomic_read_write(v, sizeo
2241 return raw_atomic64_dec_return_relaxe
2242 }
2243
2244 /**
2245 * atomic64_fetch_dec() - atomic decrement wi
2246 * @v: pointer to atomic64_t
2247 *
2248 * Atomically updates @v to (@v - 1) with ful
2249 *
2250 * Unsafe to use in noinstr code; use raw_ato
2251 *
2252 * Return: The original value of @v.
2253 */
2254 static __always_inline s64
2255 atomic64_fetch_dec(atomic64_t *v)
2256 {
2257 kcsan_mb();
2258 instrument_atomic_read_write(v, sizeo
2259 return raw_atomic64_fetch_dec(v);
2260 }
2261
2262 /**
2263 * atomic64_fetch_dec_acquire() - atomic decr
2264 * @v: pointer to atomic64_t
2265 *
2266 * Atomically updates @v to (@v - 1) with acq
2267 *
2268 * Unsafe to use in noinstr code; use raw_ato
2269 *
2270 * Return: The original value of @v.
2271 */
2272 static __always_inline s64
2273 atomic64_fetch_dec_acquire(atomic64_t *v)
2274 {
2275 instrument_atomic_read_write(v, sizeo
2276 return raw_atomic64_fetch_dec_acquire
2277 }
2278
2279 /**
2280 * atomic64_fetch_dec_release() - atomic decr
2281 * @v: pointer to atomic64_t
2282 *
2283 * Atomically updates @v to (@v - 1) with rel
2284 *
2285 * Unsafe to use in noinstr code; use raw_ato
2286 *
2287 * Return: The original value of @v.
2288 */
2289 static __always_inline s64
2290 atomic64_fetch_dec_release(atomic64_t *v)
2291 {
2292 kcsan_release();
2293 instrument_atomic_read_write(v, sizeo
2294 return raw_atomic64_fetch_dec_release
2295 }
2296
2297 /**
2298 * atomic64_fetch_dec_relaxed() - atomic decr
2299 * @v: pointer to atomic64_t
2300 *
2301 * Atomically updates @v to (@v - 1) with rel
2302 *
2303 * Unsafe to use in noinstr code; use raw_ato
2304 *
2305 * Return: The original value of @v.
2306 */
2307 static __always_inline s64
2308 atomic64_fetch_dec_relaxed(atomic64_t *v)
2309 {
2310 instrument_atomic_read_write(v, sizeo
2311 return raw_atomic64_fetch_dec_relaxed
2312 }
2313
2314 /**
2315 * atomic64_and() - atomic bitwise AND with r
2316 * @i: s64 value
2317 * @v: pointer to atomic64_t
2318 *
2319 * Atomically updates @v to (@v & @i) with re
2320 *
2321 * Unsafe to use in noinstr code; use raw_ato
2322 *
2323 * Return: Nothing.
2324 */
2325 static __always_inline void
2326 atomic64_and(s64 i, atomic64_t *v)
2327 {
2328 instrument_atomic_read_write(v, sizeo
2329 raw_atomic64_and(i, v);
2330 }
2331
2332 /**
2333 * atomic64_fetch_and() - atomic bitwise AND
2334 * @i: s64 value
2335 * @v: pointer to atomic64_t
2336 *
2337 * Atomically updates @v to (@v & @i) with fu
2338 *
2339 * Unsafe to use in noinstr code; use raw_ato
2340 *
2341 * Return: The original value of @v.
2342 */
2343 static __always_inline s64
2344 atomic64_fetch_and(s64 i, atomic64_t *v)
2345 {
2346 kcsan_mb();
2347 instrument_atomic_read_write(v, sizeo
2348 return raw_atomic64_fetch_and(i, v);
2349 }
2350
2351 /**
2352 * atomic64_fetch_and_acquire() - atomic bitw
2353 * @i: s64 value
2354 * @v: pointer to atomic64_t
2355 *
2356 * Atomically updates @v to (@v & @i) with ac
2357 *
2358 * Unsafe to use in noinstr code; use raw_ato
2359 *
2360 * Return: The original value of @v.
2361 */
2362 static __always_inline s64
2363 atomic64_fetch_and_acquire(s64 i, atomic64_t
2364 {
2365 instrument_atomic_read_write(v, sizeo
2366 return raw_atomic64_fetch_and_acquire
2367 }
2368
2369 /**
2370 * atomic64_fetch_and_release() - atomic bitw
2371 * @i: s64 value
2372 * @v: pointer to atomic64_t
2373 *
2374 * Atomically updates @v to (@v & @i) with re
2375 *
2376 * Unsafe to use in noinstr code; use raw_ato
2377 *
2378 * Return: The original value of @v.
2379 */
2380 static __always_inline s64
2381 atomic64_fetch_and_release(s64 i, atomic64_t
2382 {
2383 kcsan_release();
2384 instrument_atomic_read_write(v, sizeo
2385 return raw_atomic64_fetch_and_release
2386 }
2387
2388 /**
2389 * atomic64_fetch_and_relaxed() - atomic bitw
2390 * @i: s64 value
2391 * @v: pointer to atomic64_t
2392 *
2393 * Atomically updates @v to (@v & @i) with re
2394 *
2395 * Unsafe to use in noinstr code; use raw_ato
2396 *
2397 * Return: The original value of @v.
2398 */
2399 static __always_inline s64
2400 atomic64_fetch_and_relaxed(s64 i, atomic64_t
2401 {
2402 instrument_atomic_read_write(v, sizeo
2403 return raw_atomic64_fetch_and_relaxed
2404 }
2405
2406 /**
2407 * atomic64_andnot() - atomic bitwise AND NOT
2408 * @i: s64 value
2409 * @v: pointer to atomic64_t
2410 *
2411 * Atomically updates @v to (@v & ~@i) with r
2412 *
2413 * Unsafe to use in noinstr code; use raw_ato
2414 *
2415 * Return: Nothing.
2416 */
2417 static __always_inline void
2418 atomic64_andnot(s64 i, atomic64_t *v)
2419 {
2420 instrument_atomic_read_write(v, sizeo
2421 raw_atomic64_andnot(i, v);
2422 }
2423
2424 /**
2425 * atomic64_fetch_andnot() - atomic bitwise A
2426 * @i: s64 value
2427 * @v: pointer to atomic64_t
2428 *
2429 * Atomically updates @v to (@v & ~@i) with f
2430 *
2431 * Unsafe to use in noinstr code; use raw_ato
2432 *
2433 * Return: The original value of @v.
2434 */
2435 static __always_inline s64
2436 atomic64_fetch_andnot(s64 i, atomic64_t *v)
2437 {
2438 kcsan_mb();
2439 instrument_atomic_read_write(v, sizeo
2440 return raw_atomic64_fetch_andnot(i, v
2441 }
2442
2443 /**
2444 * atomic64_fetch_andnot_acquire() - atomic b
2445 * @i: s64 value
2446 * @v: pointer to atomic64_t
2447 *
2448 * Atomically updates @v to (@v & ~@i) with a
2449 *
2450 * Unsafe to use in noinstr code; use raw_ato
2451 *
2452 * Return: The original value of @v.
2453 */
2454 static __always_inline s64
2455 atomic64_fetch_andnot_acquire(s64 i, atomic64
2456 {
2457 instrument_atomic_read_write(v, sizeo
2458 return raw_atomic64_fetch_andnot_acqu
2459 }
2460
2461 /**
2462 * atomic64_fetch_andnot_release() - atomic b
2463 * @i: s64 value
2464 * @v: pointer to atomic64_t
2465 *
2466 * Atomically updates @v to (@v & ~@i) with r
2467 *
2468 * Unsafe to use in noinstr code; use raw_ato
2469 *
2470 * Return: The original value of @v.
2471 */
2472 static __always_inline s64
2473 atomic64_fetch_andnot_release(s64 i, atomic64
2474 {
2475 kcsan_release();
2476 instrument_atomic_read_write(v, sizeo
2477 return raw_atomic64_fetch_andnot_rele
2478 }
2479
2480 /**
2481 * atomic64_fetch_andnot_relaxed() - atomic b
2482 * @i: s64 value
2483 * @v: pointer to atomic64_t
2484 *
2485 * Atomically updates @v to (@v & ~@i) with r
2486 *
2487 * Unsafe to use in noinstr code; use raw_ato
2488 *
2489 * Return: The original value of @v.
2490 */
2491 static __always_inline s64
2492 atomic64_fetch_andnot_relaxed(s64 i, atomic64
2493 {
2494 instrument_atomic_read_write(v, sizeo
2495 return raw_atomic64_fetch_andnot_rela
2496 }
2497
2498 /**
2499 * atomic64_or() - atomic bitwise OR with rel
2500 * @i: s64 value
2501 * @v: pointer to atomic64_t
2502 *
2503 * Atomically updates @v to (@v | @i) with re
2504 *
2505 * Unsafe to use in noinstr code; use raw_ato
2506 *
2507 * Return: Nothing.
2508 */
2509 static __always_inline void
2510 atomic64_or(s64 i, atomic64_t *v)
2511 {
2512 instrument_atomic_read_write(v, sizeo
2513 raw_atomic64_or(i, v);
2514 }
2515
2516 /**
2517 * atomic64_fetch_or() - atomic bitwise OR wi
2518 * @i: s64 value
2519 * @v: pointer to atomic64_t
2520 *
2521 * Atomically updates @v to (@v | @i) with fu
2522 *
2523 * Unsafe to use in noinstr code; use raw_ato
2524 *
2525 * Return: The original value of @v.
2526 */
2527 static __always_inline s64
2528 atomic64_fetch_or(s64 i, atomic64_t *v)
2529 {
2530 kcsan_mb();
2531 instrument_atomic_read_write(v, sizeo
2532 return raw_atomic64_fetch_or(i, v);
2533 }
2534
2535 /**
2536 * atomic64_fetch_or_acquire() - atomic bitwi
2537 * @i: s64 value
2538 * @v: pointer to atomic64_t
2539 *
2540 * Atomically updates @v to (@v | @i) with ac
2541 *
2542 * Unsafe to use in noinstr code; use raw_ato
2543 *
2544 * Return: The original value of @v.
2545 */
2546 static __always_inline s64
2547 atomic64_fetch_or_acquire(s64 i, atomic64_t *
2548 {
2549 instrument_atomic_read_write(v, sizeo
2550 return raw_atomic64_fetch_or_acquire(
2551 }
2552
2553 /**
2554 * atomic64_fetch_or_release() - atomic bitwi
2555 * @i: s64 value
2556 * @v: pointer to atomic64_t
2557 *
2558 * Atomically updates @v to (@v | @i) with re
2559 *
2560 * Unsafe to use in noinstr code; use raw_ato
2561 *
2562 * Return: The original value of @v.
2563 */
2564 static __always_inline s64
2565 atomic64_fetch_or_release(s64 i, atomic64_t *
2566 {
2567 kcsan_release();
2568 instrument_atomic_read_write(v, sizeo
2569 return raw_atomic64_fetch_or_release(
2570 }
2571
2572 /**
2573 * atomic64_fetch_or_relaxed() - atomic bitwi
2574 * @i: s64 value
2575 * @v: pointer to atomic64_t
2576 *
2577 * Atomically updates @v to (@v | @i) with re
2578 *
2579 * Unsafe to use in noinstr code; use raw_ato
2580 *
2581 * Return: The original value of @v.
2582 */
2583 static __always_inline s64
2584 atomic64_fetch_or_relaxed(s64 i, atomic64_t *
2585 {
2586 instrument_atomic_read_write(v, sizeo
2587 return raw_atomic64_fetch_or_relaxed(
2588 }
2589
2590 /**
2591 * atomic64_xor() - atomic bitwise XOR with r
2592 * @i: s64 value
2593 * @v: pointer to atomic64_t
2594 *
2595 * Atomically updates @v to (@v ^ @i) with re
2596 *
2597 * Unsafe to use in noinstr code; use raw_ato
2598 *
2599 * Return: Nothing.
2600 */
2601 static __always_inline void
2602 atomic64_xor(s64 i, atomic64_t *v)
2603 {
2604 instrument_atomic_read_write(v, sizeo
2605 raw_atomic64_xor(i, v);
2606 }
2607
2608 /**
2609 * atomic64_fetch_xor() - atomic bitwise XOR
2610 * @i: s64 value
2611 * @v: pointer to atomic64_t
2612 *
2613 * Atomically updates @v to (@v ^ @i) with fu
2614 *
2615 * Unsafe to use in noinstr code; use raw_ato
2616 *
2617 * Return: The original value of @v.
2618 */
2619 static __always_inline s64
2620 atomic64_fetch_xor(s64 i, atomic64_t *v)
2621 {
2622 kcsan_mb();
2623 instrument_atomic_read_write(v, sizeo
2624 return raw_atomic64_fetch_xor(i, v);
2625 }
2626
2627 /**
2628 * atomic64_fetch_xor_acquire() - atomic bitw
2629 * @i: s64 value
2630 * @v: pointer to atomic64_t
2631 *
2632 * Atomically updates @v to (@v ^ @i) with ac
2633 *
2634 * Unsafe to use in noinstr code; use raw_ato
2635 *
2636 * Return: The original value of @v.
2637 */
2638 static __always_inline s64
2639 atomic64_fetch_xor_acquire(s64 i, atomic64_t
2640 {
2641 instrument_atomic_read_write(v, sizeo
2642 return raw_atomic64_fetch_xor_acquire
2643 }
2644
2645 /**
2646 * atomic64_fetch_xor_release() - atomic bitw
2647 * @i: s64 value
2648 * @v: pointer to atomic64_t
2649 *
2650 * Atomically updates @v to (@v ^ @i) with re
2651 *
2652 * Unsafe to use in noinstr code; use raw_ato
2653 *
2654 * Return: The original value of @v.
2655 */
2656 static __always_inline s64
2657 atomic64_fetch_xor_release(s64 i, atomic64_t
2658 {
2659 kcsan_release();
2660 instrument_atomic_read_write(v, sizeo
2661 return raw_atomic64_fetch_xor_release
2662 }
2663
2664 /**
2665 * atomic64_fetch_xor_relaxed() - atomic bitw
2666 * @i: s64 value
2667 * @v: pointer to atomic64_t
2668 *
2669 * Atomically updates @v to (@v ^ @i) with re
2670 *
2671 * Unsafe to use in noinstr code; use raw_ato
2672 *
2673 * Return: The original value of @v.
2674 */
2675 static __always_inline s64
2676 atomic64_fetch_xor_relaxed(s64 i, atomic64_t
2677 {
2678 instrument_atomic_read_write(v, sizeo
2679 return raw_atomic64_fetch_xor_relaxed
2680 }
2681
2682 /**
2683 * atomic64_xchg() - atomic exchange with ful
2684 * @v: pointer to atomic64_t
2685 * @new: s64 value to assign
2686 *
2687 * Atomically updates @v to @new with full or
2688 *
2689 * Unsafe to use in noinstr code; use raw_ato
2690 *
2691 * Return: The original value of @v.
2692 */
2693 static __always_inline s64
2694 atomic64_xchg(atomic64_t *v, s64 new)
2695 {
2696 kcsan_mb();
2697 instrument_atomic_read_write(v, sizeo
2698 return raw_atomic64_xchg(v, new);
2699 }
2700
2701 /**
2702 * atomic64_xchg_acquire() - atomic exchange
2703 * @v: pointer to atomic64_t
2704 * @new: s64 value to assign
2705 *
2706 * Atomically updates @v to @new with acquire
2707 *
2708 * Unsafe to use in noinstr code; use raw_ato
2709 *
2710 * Return: The original value of @v.
2711 */
2712 static __always_inline s64
2713 atomic64_xchg_acquire(atomic64_t *v, s64 new)
2714 {
2715 instrument_atomic_read_write(v, sizeo
2716 return raw_atomic64_xchg_acquire(v, n
2717 }
2718
2719 /**
2720 * atomic64_xchg_release() - atomic exchange
2721 * @v: pointer to atomic64_t
2722 * @new: s64 value to assign
2723 *
2724 * Atomically updates @v to @new with release
2725 *
2726 * Unsafe to use in noinstr code; use raw_ato
2727 *
2728 * Return: The original value of @v.
2729 */
2730 static __always_inline s64
2731 atomic64_xchg_release(atomic64_t *v, s64 new)
2732 {
2733 kcsan_release();
2734 instrument_atomic_read_write(v, sizeo
2735 return raw_atomic64_xchg_release(v, n
2736 }
2737
2738 /**
2739 * atomic64_xchg_relaxed() - atomic exchange
2740 * @v: pointer to atomic64_t
2741 * @new: s64 value to assign
2742 *
2743 * Atomically updates @v to @new with relaxed
2744 *
2745 * Unsafe to use in noinstr code; use raw_ato
2746 *
2747 * Return: The original value of @v.
2748 */
2749 static __always_inline s64
2750 atomic64_xchg_relaxed(atomic64_t *v, s64 new)
2751 {
2752 instrument_atomic_read_write(v, sizeo
2753 return raw_atomic64_xchg_relaxed(v, n
2754 }
2755
2756 /**
2757 * atomic64_cmpxchg() - atomic compare and ex
2758 * @v: pointer to atomic64_t
2759 * @old: s64 value to compare with
2760 * @new: s64 value to assign
2761 *
2762 * If (@v == @old), atomically updates @v to
2763 * Otherwise, @v is not modified and relaxed
2764 *
2765 * Unsafe to use in noinstr code; use raw_ato
2766 *
2767 * Return: The original value of @v.
2768 */
2769 static __always_inline s64
2770 atomic64_cmpxchg(atomic64_t *v, s64 old, s64
2771 {
2772 kcsan_mb();
2773 instrument_atomic_read_write(v, sizeo
2774 return raw_atomic64_cmpxchg(v, old, n
2775 }
2776
2777 /**
2778 * atomic64_cmpxchg_acquire() - atomic compar
2779 * @v: pointer to atomic64_t
2780 * @old: s64 value to compare with
2781 * @new: s64 value to assign
2782 *
2783 * If (@v == @old), atomically updates @v to
2784 * Otherwise, @v is not modified and relaxed
2785 *
2786 * Unsafe to use in noinstr code; use raw_ato
2787 *
2788 * Return: The original value of @v.
2789 */
2790 static __always_inline s64
2791 atomic64_cmpxchg_acquire(atomic64_t *v, s64 o
2792 {
2793 instrument_atomic_read_write(v, sizeo
2794 return raw_atomic64_cmpxchg_acquire(v
2795 }
2796
2797 /**
2798 * atomic64_cmpxchg_release() - atomic compar
2799 * @v: pointer to atomic64_t
2800 * @old: s64 value to compare with
2801 * @new: s64 value to assign
2802 *
2803 * If (@v == @old), atomically updates @v to
2804 * Otherwise, @v is not modified and relaxed
2805 *
2806 * Unsafe to use in noinstr code; use raw_ato
2807 *
2808 * Return: The original value of @v.
2809 */
2810 static __always_inline s64
2811 atomic64_cmpxchg_release(atomic64_t *v, s64 o
2812 {
2813 kcsan_release();
2814 instrument_atomic_read_write(v, sizeo
2815 return raw_atomic64_cmpxchg_release(v
2816 }
2817
2818 /**
2819 * atomic64_cmpxchg_relaxed() - atomic compar
2820 * @v: pointer to atomic64_t
2821 * @old: s64 value to compare with
2822 * @new: s64 value to assign
2823 *
2824 * If (@v == @old), atomically updates @v to
2825 * Otherwise, @v is not modified and relaxed
2826 *
2827 * Unsafe to use in noinstr code; use raw_ato
2828 *
2829 * Return: The original value of @v.
2830 */
2831 static __always_inline s64
2832 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 o
2833 {
2834 instrument_atomic_read_write(v, sizeo
2835 return raw_atomic64_cmpxchg_relaxed(v
2836 }
2837
2838 /**
2839 * atomic64_try_cmpxchg() - atomic compare an
2840 * @v: pointer to atomic64_t
2841 * @old: pointer to s64 value to compare with
2842 * @new: s64 value to assign
2843 *
2844 * If (@v == @old), atomically updates @v to
2845 * Otherwise, @v is not modified, @old is upd
2846 * and relaxed ordering is provided.
2847 *
2848 * Unsafe to use in noinstr code; use raw_ato
2849 *
2850 * Return: @true if the exchange occured, @fa
2851 */
2852 static __always_inline bool
2853 atomic64_try_cmpxchg(atomic64_t *v, s64 *old,
2854 {
2855 kcsan_mb();
2856 instrument_atomic_read_write(v, sizeo
2857 instrument_atomic_read_write(old, siz
2858 return raw_atomic64_try_cmpxchg(v, ol
2859 }
2860
2861 /**
2862 * atomic64_try_cmpxchg_acquire() - atomic co
2863 * @v: pointer to atomic64_t
2864 * @old: pointer to s64 value to compare with
2865 * @new: s64 value to assign
2866 *
2867 * If (@v == @old), atomically updates @v to
2868 * Otherwise, @v is not modified, @old is upd
2869 * and relaxed ordering is provided.
2870 *
2871 * Unsafe to use in noinstr code; use raw_ato
2872 *
2873 * Return: @true if the exchange occured, @fa
2874 */
2875 static __always_inline bool
2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s
2877 {
2878 instrument_atomic_read_write(v, sizeo
2879 instrument_atomic_read_write(old, siz
2880 return raw_atomic64_try_cmpxchg_acqui
2881 }
2882
2883 /**
2884 * atomic64_try_cmpxchg_release() - atomic co
2885 * @v: pointer to atomic64_t
2886 * @old: pointer to s64 value to compare with
2887 * @new: s64 value to assign
2888 *
2889 * If (@v == @old), atomically updates @v to
2890 * Otherwise, @v is not modified, @old is upd
2891 * and relaxed ordering is provided.
2892 *
2893 * Unsafe to use in noinstr code; use raw_ato
2894 *
2895 * Return: @true if the exchange occured, @fa
2896 */
2897 static __always_inline bool
2898 atomic64_try_cmpxchg_release(atomic64_t *v, s
2899 {
2900 kcsan_release();
2901 instrument_atomic_read_write(v, sizeo
2902 instrument_atomic_read_write(old, siz
2903 return raw_atomic64_try_cmpxchg_relea
2904 }
2905
2906 /**
2907 * atomic64_try_cmpxchg_relaxed() - atomic co
2908 * @v: pointer to atomic64_t
2909 * @old: pointer to s64 value to compare with
2910 * @new: s64 value to assign
2911 *
2912 * If (@v == @old), atomically updates @v to
2913 * Otherwise, @v is not modified, @old is upd
2914 * and relaxed ordering is provided.
2915 *
2916 * Unsafe to use in noinstr code; use raw_ato
2917 *
2918 * Return: @true if the exchange occured, @fa
2919 */
2920 static __always_inline bool
2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s
2922 {
2923 instrument_atomic_read_write(v, sizeo
2924 instrument_atomic_read_write(old, siz
2925 return raw_atomic64_try_cmpxchg_relax
2926 }
2927
2928 /**
2929 * atomic64_sub_and_test() - atomic subtract
2930 * @i: s64 value to subtract
2931 * @v: pointer to atomic64_t
2932 *
2933 * Atomically updates @v to (@v - @i) with fu
2934 *
2935 * Unsafe to use in noinstr code; use raw_ato
2936 *
2937 * Return: @true if the resulting value of @v
2938 */
2939 static __always_inline bool
2940 atomic64_sub_and_test(s64 i, atomic64_t *v)
2941 {
2942 kcsan_mb();
2943 instrument_atomic_read_write(v, sizeo
2944 return raw_atomic64_sub_and_test(i, v
2945 }
2946
2947 /**
2948 * atomic64_dec_and_test() - atomic decrement
2949 * @v: pointer to atomic64_t
2950 *
2951 * Atomically updates @v to (@v - 1) with ful
2952 *
2953 * Unsafe to use in noinstr code; use raw_ato
2954 *
2955 * Return: @true if the resulting value of @v
2956 */
2957 static __always_inline bool
2958 atomic64_dec_and_test(atomic64_t *v)
2959 {
2960 kcsan_mb();
2961 instrument_atomic_read_write(v, sizeo
2962 return raw_atomic64_dec_and_test(v);
2963 }
2964
2965 /**
2966 * atomic64_inc_and_test() - atomic increment
2967 * @v: pointer to atomic64_t
2968 *
2969 * Atomically updates @v to (@v + 1) with ful
2970 *
2971 * Unsafe to use in noinstr code; use raw_ato
2972 *
2973 * Return: @true if the resulting value of @v
2974 */
2975 static __always_inline bool
2976 atomic64_inc_and_test(atomic64_t *v)
2977 {
2978 kcsan_mb();
2979 instrument_atomic_read_write(v, sizeo
2980 return raw_atomic64_inc_and_test(v);
2981 }
2982
2983 /**
2984 * atomic64_add_negative() - atomic add and t
2985 * @i: s64 value to add
2986 * @v: pointer to atomic64_t
2987 *
2988 * Atomically updates @v to (@v + @i) with fu
2989 *
2990 * Unsafe to use in noinstr code; use raw_ato
2991 *
2992 * Return: @true if the resulting value of @v
2993 */
2994 static __always_inline bool
2995 atomic64_add_negative(s64 i, atomic64_t *v)
2996 {
2997 kcsan_mb();
2998 instrument_atomic_read_write(v, sizeo
2999 return raw_atomic64_add_negative(i, v
3000 }
3001
3002 /**
3003 * atomic64_add_negative_acquire() - atomic a
3004 * @i: s64 value to add
3005 * @v: pointer to atomic64_t
3006 *
3007 * Atomically updates @v to (@v + @i) with ac
3008 *
3009 * Unsafe to use in noinstr code; use raw_ato
3010 *
3011 * Return: @true if the resulting value of @v
3012 */
3013 static __always_inline bool
3014 atomic64_add_negative_acquire(s64 i, atomic64
3015 {
3016 instrument_atomic_read_write(v, sizeo
3017 return raw_atomic64_add_negative_acqu
3018 }
3019
3020 /**
3021 * atomic64_add_negative_release() - atomic a
3022 * @i: s64 value to add
3023 * @v: pointer to atomic64_t
3024 *
3025 * Atomically updates @v to (@v + @i) with re
3026 *
3027 * Unsafe to use in noinstr code; use raw_ato
3028 *
3029 * Return: @true if the resulting value of @v
3030 */
3031 static __always_inline bool
3032 atomic64_add_negative_release(s64 i, atomic64
3033 {
3034 kcsan_release();
3035 instrument_atomic_read_write(v, sizeo
3036 return raw_atomic64_add_negative_rele
3037 }
3038
3039 /**
3040 * atomic64_add_negative_relaxed() - atomic a
3041 * @i: s64 value to add
3042 * @v: pointer to atomic64_t
3043 *
3044 * Atomically updates @v to (@v + @i) with re
3045 *
3046 * Unsafe to use in noinstr code; use raw_ato
3047 *
3048 * Return: @true if the resulting value of @v
3049 */
3050 static __always_inline bool
3051 atomic64_add_negative_relaxed(s64 i, atomic64
3052 {
3053 instrument_atomic_read_write(v, sizeo
3054 return raw_atomic64_add_negative_rela
3055 }
3056
3057 /**
3058 * atomic64_fetch_add_unless() - atomic add u
3059 * @v: pointer to atomic64_t
3060 * @a: s64 value to add
3061 * @u: s64 value to compare with
3062 *
3063 * If (@v != @u), atomically updates @v to (@
3064 * Otherwise, @v is not modified and relaxed
3065 *
3066 * Unsafe to use in noinstr code; use raw_ato
3067 *
3068 * Return: The original value of @v.
3069 */
3070 static __always_inline s64
3071 atomic64_fetch_add_unless(atomic64_t *v, s64
3072 {
3073 kcsan_mb();
3074 instrument_atomic_read_write(v, sizeo
3075 return raw_atomic64_fetch_add_unless(
3076 }
3077
3078 /**
3079 * atomic64_add_unless() - atomic add unless
3080 * @v: pointer to atomic64_t
3081 * @a: s64 value to add
3082 * @u: s64 value to compare with
3083 *
3084 * If (@v != @u), atomically updates @v to (@
3085 * Otherwise, @v is not modified and relaxed
3086 *
3087 * Unsafe to use in noinstr code; use raw_ato
3088 *
3089 * Return: @true if @v was updated, @false ot
3090 */
3091 static __always_inline bool
3092 atomic64_add_unless(atomic64_t *v, s64 a, s64
3093 {
3094 kcsan_mb();
3095 instrument_atomic_read_write(v, sizeo
3096 return raw_atomic64_add_unless(v, a,
3097 }
3098
3099 /**
3100 * atomic64_inc_not_zero() - atomic increment
3101 * @v: pointer to atomic64_t
3102 *
3103 * If (@v != 0), atomically updates @v to (@v
3104 * Otherwise, @v is not modified and relaxed
3105 *
3106 * Unsafe to use in noinstr code; use raw_ato
3107 *
3108 * Return: @true if @v was updated, @false ot
3109 */
3110 static __always_inline bool
3111 atomic64_inc_not_zero(atomic64_t *v)
3112 {
3113 kcsan_mb();
3114 instrument_atomic_read_write(v, sizeo
3115 return raw_atomic64_inc_not_zero(v);
3116 }
3117
3118 /**
3119 * atomic64_inc_unless_negative() - atomic in
3120 * @v: pointer to atomic64_t
3121 *
3122 * If (@v >= 0), atomically updates @v to (@v
3123 * Otherwise, @v is not modified and relaxed
3124 *
3125 * Unsafe to use in noinstr code; use raw_ato
3126 *
3127 * Return: @true if @v was updated, @false ot
3128 */
3129 static __always_inline bool
3130 atomic64_inc_unless_negative(atomic64_t *v)
3131 {
3132 kcsan_mb();
3133 instrument_atomic_read_write(v, sizeo
3134 return raw_atomic64_inc_unless_negati
3135 }
3136
3137 /**
3138 * atomic64_dec_unless_positive() - atomic de
3139 * @v: pointer to atomic64_t
3140 *
3141 * If (@v <= 0), atomically updates @v to (@v
3142 * Otherwise, @v is not modified and relaxed
3143 *
3144 * Unsafe to use in noinstr code; use raw_ato
3145 *
3146 * Return: @true if @v was updated, @false ot
3147 */
3148 static __always_inline bool
3149 atomic64_dec_unless_positive(atomic64_t *v)
3150 {
3151 kcsan_mb();
3152 instrument_atomic_read_write(v, sizeo
3153 return raw_atomic64_dec_unless_positi
3154 }
3155
3156 /**
3157 * atomic64_dec_if_positive() - atomic decrem
3158 * @v: pointer to atomic64_t
3159 *
3160 * If (@v > 0), atomically updates @v to (@v
3161 * Otherwise, @v is not modified and relaxed
3162 *
3163 * Unsafe to use in noinstr code; use raw_ato
3164 *
3165 * Return: The old value of (@v - 1), regardl
3166 */
3167 static __always_inline s64
3168 atomic64_dec_if_positive(atomic64_t *v)
3169 {
3170 kcsan_mb();
3171 instrument_atomic_read_write(v, sizeo
3172 return raw_atomic64_dec_if_positive(v
3173 }
3174
3175 /**
3176 * atomic_long_read() - atomic load with rela
3177 * @v: pointer to atomic_long_t
3178 *
3179 * Atomically loads the value of @v with rela
3180 *
3181 * Unsafe to use in noinstr code; use raw_ato
3182 *
3183 * Return: The value loaded from @v.
3184 */
3185 static __always_inline long
3186 atomic_long_read(const atomic_long_t *v)
3187 {
3188 instrument_atomic_read(v, sizeof(*v))
3189 return raw_atomic_long_read(v);
3190 }
3191
3192 /**
3193 * atomic_long_read_acquire() - atomic load w
3194 * @v: pointer to atomic_long_t
3195 *
3196 * Atomically loads the value of @v with acqu
3197 *
3198 * Unsafe to use in noinstr code; use raw_ato
3199 *
3200 * Return: The value loaded from @v.
3201 */
3202 static __always_inline long
3203 atomic_long_read_acquire(const atomic_long_t
3204 {
3205 instrument_atomic_read(v, sizeof(*v))
3206 return raw_atomic_long_read_acquire(v
3207 }
3208
3209 /**
3210 * atomic_long_set() - atomic set with relaxe
3211 * @v: pointer to atomic_long_t
3212 * @i: long value to assign
3213 *
3214 * Atomically sets @v to @i with relaxed orde
3215 *
3216 * Unsafe to use in noinstr code; use raw_ato
3217 *
3218 * Return: Nothing.
3219 */
3220 static __always_inline void
3221 atomic_long_set(atomic_long_t *v, long i)
3222 {
3223 instrument_atomic_write(v, sizeof(*v)
3224 raw_atomic_long_set(v, i);
3225 }
3226
3227 /**
3228 * atomic_long_set_release() - atomic set wit
3229 * @v: pointer to atomic_long_t
3230 * @i: long value to assign
3231 *
3232 * Atomically sets @v to @i with release orde
3233 *
3234 * Unsafe to use in noinstr code; use raw_ato
3235 *
3236 * Return: Nothing.
3237 */
3238 static __always_inline void
3239 atomic_long_set_release(atomic_long_t *v, lon
3240 {
3241 kcsan_release();
3242 instrument_atomic_write(v, sizeof(*v)
3243 raw_atomic_long_set_release(v, i);
3244 }
3245
3246 /**
3247 * atomic_long_add() - atomic add with relaxe
3248 * @i: long value to add
3249 * @v: pointer to atomic_long_t
3250 *
3251 * Atomically updates @v to (@v + @i) with re
3252 *
3253 * Unsafe to use in noinstr code; use raw_ato
3254 *
3255 * Return: Nothing.
3256 */
3257 static __always_inline void
3258 atomic_long_add(long i, atomic_long_t *v)
3259 {
3260 instrument_atomic_read_write(v, sizeo
3261 raw_atomic_long_add(i, v);
3262 }
3263
3264 /**
3265 * atomic_long_add_return() - atomic add with
3266 * @i: long value to add
3267 * @v: pointer to atomic_long_t
3268 *
3269 * Atomically updates @v to (@v + @i) with fu
3270 *
3271 * Unsafe to use in noinstr code; use raw_ato
3272 *
3273 * Return: The updated value of @v.
3274 */
3275 static __always_inline long
3276 atomic_long_add_return(long i, atomic_long_t
3277 {
3278 kcsan_mb();
3279 instrument_atomic_read_write(v, sizeo
3280 return raw_atomic_long_add_return(i,
3281 }
3282
3283 /**
3284 * atomic_long_add_return_acquire() - atomic
3285 * @i: long value to add
3286 * @v: pointer to atomic_long_t
3287 *
3288 * Atomically updates @v to (@v + @i) with ac
3289 *
3290 * Unsafe to use in noinstr code; use raw_ato
3291 *
3292 * Return: The updated value of @v.
3293 */
3294 static __always_inline long
3295 atomic_long_add_return_acquire(long i, atomic
3296 {
3297 instrument_atomic_read_write(v, sizeo
3298 return raw_atomic_long_add_return_acq
3299 }
3300
3301 /**
3302 * atomic_long_add_return_release() - atomic
3303 * @i: long value to add
3304 * @v: pointer to atomic_long_t
3305 *
3306 * Atomically updates @v to (@v + @i) with re
3307 *
3308 * Unsafe to use in noinstr code; use raw_ato
3309 *
3310 * Return: The updated value of @v.
3311 */
3312 static __always_inline long
3313 atomic_long_add_return_release(long i, atomic
3314 {
3315 kcsan_release();
3316 instrument_atomic_read_write(v, sizeo
3317 return raw_atomic_long_add_return_rel
3318 }
3319
3320 /**
3321 * atomic_long_add_return_relaxed() - atomic
3322 * @i: long value to add
3323 * @v: pointer to atomic_long_t
3324 *
3325 * Atomically updates @v to (@v + @i) with re
3326 *
3327 * Unsafe to use in noinstr code; use raw_ato
3328 *
3329 * Return: The updated value of @v.
3330 */
3331 static __always_inline long
3332 atomic_long_add_return_relaxed(long i, atomic
3333 {
3334 instrument_atomic_read_write(v, sizeo
3335 return raw_atomic_long_add_return_rel
3336 }
3337
3338 /**
3339 * atomic_long_fetch_add() - atomic add with
3340 * @i: long value to add
3341 * @v: pointer to atomic_long_t
3342 *
3343 * Atomically updates @v to (@v + @i) with fu
3344 *
3345 * Unsafe to use in noinstr code; use raw_ato
3346 *
3347 * Return: The original value of @v.
3348 */
3349 static __always_inline long
3350 atomic_long_fetch_add(long i, atomic_long_t *
3351 {
3352 kcsan_mb();
3353 instrument_atomic_read_write(v, sizeo
3354 return raw_atomic_long_fetch_add(i, v
3355 }
3356
3357 /**
3358 * atomic_long_fetch_add_acquire() - atomic a
3359 * @i: long value to add
3360 * @v: pointer to atomic_long_t
3361 *
3362 * Atomically updates @v to (@v + @i) with ac
3363 *
3364 * Unsafe to use in noinstr code; use raw_ato
3365 *
3366 * Return: The original value of @v.
3367 */
3368 static __always_inline long
3369 atomic_long_fetch_add_acquire(long i, atomic_
3370 {
3371 instrument_atomic_read_write(v, sizeo
3372 return raw_atomic_long_fetch_add_acqu
3373 }
3374
3375 /**
3376 * atomic_long_fetch_add_release() - atomic a
3377 * @i: long value to add
3378 * @v: pointer to atomic_long_t
3379 *
3380 * Atomically updates @v to (@v + @i) with re
3381 *
3382 * Unsafe to use in noinstr code; use raw_ato
3383 *
3384 * Return: The original value of @v.
3385 */
3386 static __always_inline long
3387 atomic_long_fetch_add_release(long i, atomic_
3388 {
3389 kcsan_release();
3390 instrument_atomic_read_write(v, sizeo
3391 return raw_atomic_long_fetch_add_rele
3392 }
3393
3394 /**
3395 * atomic_long_fetch_add_relaxed() - atomic a
3396 * @i: long value to add
3397 * @v: pointer to atomic_long_t
3398 *
3399 * Atomically updates @v to (@v + @i) with re
3400 *
3401 * Unsafe to use in noinstr code; use raw_ato
3402 *
3403 * Return: The original value of @v.
3404 */
3405 static __always_inline long
3406 atomic_long_fetch_add_relaxed(long i, atomic_
3407 {
3408 instrument_atomic_read_write(v, sizeo
3409 return raw_atomic_long_fetch_add_rela
3410 }
3411
3412 /**
3413 * atomic_long_sub() - atomic subtract with r
3414 * @i: long value to subtract
3415 * @v: pointer to atomic_long_t
3416 *
3417 * Atomically updates @v to (@v - @i) with re
3418 *
3419 * Unsafe to use in noinstr code; use raw_ato
3420 *
3421 * Return: Nothing.
3422 */
3423 static __always_inline void
3424 atomic_long_sub(long i, atomic_long_t *v)
3425 {
3426 instrument_atomic_read_write(v, sizeo
3427 raw_atomic_long_sub(i, v);
3428 }
3429
3430 /**
3431 * atomic_long_sub_return() - atomic subtract
3432 * @i: long value to subtract
3433 * @v: pointer to atomic_long_t
3434 *
3435 * Atomically updates @v to (@v - @i) with fu
3436 *
3437 * Unsafe to use in noinstr code; use raw_ato
3438 *
3439 * Return: The updated value of @v.
3440 */
3441 static __always_inline long
3442 atomic_long_sub_return(long i, atomic_long_t
3443 {
3444 kcsan_mb();
3445 instrument_atomic_read_write(v, sizeo
3446 return raw_atomic_long_sub_return(i,
3447 }
3448
3449 /**
3450 * atomic_long_sub_return_acquire() - atomic
3451 * @i: long value to subtract
3452 * @v: pointer to atomic_long_t
3453 *
3454 * Atomically updates @v to (@v - @i) with ac
3455 *
3456 * Unsafe to use in noinstr code; use raw_ato
3457 *
3458 * Return: The updated value of @v.
3459 */
3460 static __always_inline long
3461 atomic_long_sub_return_acquire(long i, atomic
3462 {
3463 instrument_atomic_read_write(v, sizeo
3464 return raw_atomic_long_sub_return_acq
3465 }
3466
3467 /**
3468 * atomic_long_sub_return_release() - atomic
3469 * @i: long value to subtract
3470 * @v: pointer to atomic_long_t
3471 *
3472 * Atomically updates @v to (@v - @i) with re
3473 *
3474 * Unsafe to use in noinstr code; use raw_ato
3475 *
3476 * Return: The updated value of @v.
3477 */
3478 static __always_inline long
3479 atomic_long_sub_return_release(long i, atomic
3480 {
3481 kcsan_release();
3482 instrument_atomic_read_write(v, sizeo
3483 return raw_atomic_long_sub_return_rel
3484 }
3485
3486 /**
3487 * atomic_long_sub_return_relaxed() - atomic
3488 * @i: long value to subtract
3489 * @v: pointer to atomic_long_t
3490 *
3491 * Atomically updates @v to (@v - @i) with re
3492 *
3493 * Unsafe to use in noinstr code; use raw_ato
3494 *
3495 * Return: The updated value of @v.
3496 */
3497 static __always_inline long
3498 atomic_long_sub_return_relaxed(long i, atomic
3499 {
3500 instrument_atomic_read_write(v, sizeo
3501 return raw_atomic_long_sub_return_rel
3502 }
3503
3504 /**
3505 * atomic_long_fetch_sub() - atomic subtract
3506 * @i: long value to subtract
3507 * @v: pointer to atomic_long_t
3508 *
3509 * Atomically updates @v to (@v - @i) with fu
3510 *
3511 * Unsafe to use in noinstr code; use raw_ato
3512 *
3513 * Return: The original value of @v.
3514 */
3515 static __always_inline long
3516 atomic_long_fetch_sub(long i, atomic_long_t *
3517 {
3518 kcsan_mb();
3519 instrument_atomic_read_write(v, sizeo
3520 return raw_atomic_long_fetch_sub(i, v
3521 }
3522
3523 /**
3524 * atomic_long_fetch_sub_acquire() - atomic s
3525 * @i: long value to subtract
3526 * @v: pointer to atomic_long_t
3527 *
3528 * Atomically updates @v to (@v - @i) with ac
3529 *
3530 * Unsafe to use in noinstr code; use raw_ato
3531 *
3532 * Return: The original value of @v.
3533 */
3534 static __always_inline long
3535 atomic_long_fetch_sub_acquire(long i, atomic_
3536 {
3537 instrument_atomic_read_write(v, sizeo
3538 return raw_atomic_long_fetch_sub_acqu
3539 }
3540
3541 /**
3542 * atomic_long_fetch_sub_release() - atomic s
3543 * @i: long value to subtract
3544 * @v: pointer to atomic_long_t
3545 *
3546 * Atomically updates @v to (@v - @i) with re
3547 *
3548 * Unsafe to use in noinstr code; use raw_ato
3549 *
3550 * Return: The original value of @v.
3551 */
3552 static __always_inline long
3553 atomic_long_fetch_sub_release(long i, atomic_
3554 {
3555 kcsan_release();
3556 instrument_atomic_read_write(v, sizeo
3557 return raw_atomic_long_fetch_sub_rele
3558 }
3559
3560 /**
3561 * atomic_long_fetch_sub_relaxed() - atomic s
3562 * @i: long value to subtract
3563 * @v: pointer to atomic_long_t
3564 *
3565 * Atomically updates @v to (@v - @i) with re
3566 *
3567 * Unsafe to use in noinstr code; use raw_ato
3568 *
3569 * Return: The original value of @v.
3570 */
3571 static __always_inline long
3572 atomic_long_fetch_sub_relaxed(long i, atomic_
3573 {
3574 instrument_atomic_read_write(v, sizeo
3575 return raw_atomic_long_fetch_sub_rela
3576 }
3577
3578 /**
3579 * atomic_long_inc() - atomic increment with
3580 * @v: pointer to atomic_long_t
3581 *
3582 * Atomically updates @v to (@v + 1) with rel
3583 *
3584 * Unsafe to use in noinstr code; use raw_ato
3585 *
3586 * Return: Nothing.
3587 */
3588 static __always_inline void
3589 atomic_long_inc(atomic_long_t *v)
3590 {
3591 instrument_atomic_read_write(v, sizeo
3592 raw_atomic_long_inc(v);
3593 }
3594
3595 /**
3596 * atomic_long_inc_return() - atomic incremen
3597 * @v: pointer to atomic_long_t
3598 *
3599 * Atomically updates @v to (@v + 1) with ful
3600 *
3601 * Unsafe to use in noinstr code; use raw_ato
3602 *
3603 * Return: The updated value of @v.
3604 */
3605 static __always_inline long
3606 atomic_long_inc_return(atomic_long_t *v)
3607 {
3608 kcsan_mb();
3609 instrument_atomic_read_write(v, sizeo
3610 return raw_atomic_long_inc_return(v);
3611 }
3612
3613 /**
3614 * atomic_long_inc_return_acquire() - atomic
3615 * @v: pointer to atomic_long_t
3616 *
3617 * Atomically updates @v to (@v + 1) with acq
3618 *
3619 * Unsafe to use in noinstr code; use raw_ato
3620 *
3621 * Return: The updated value of @v.
3622 */
3623 static __always_inline long
3624 atomic_long_inc_return_acquire(atomic_long_t
3625 {
3626 instrument_atomic_read_write(v, sizeo
3627 return raw_atomic_long_inc_return_acq
3628 }
3629
3630 /**
3631 * atomic_long_inc_return_release() - atomic
3632 * @v: pointer to atomic_long_t
3633 *
3634 * Atomically updates @v to (@v + 1) with rel
3635 *
3636 * Unsafe to use in noinstr code; use raw_ato
3637 *
3638 * Return: The updated value of @v.
3639 */
3640 static __always_inline long
3641 atomic_long_inc_return_release(atomic_long_t
3642 {
3643 kcsan_release();
3644 instrument_atomic_read_write(v, sizeo
3645 return raw_atomic_long_inc_return_rel
3646 }
3647
3648 /**
3649 * atomic_long_inc_return_relaxed() - atomic
3650 * @v: pointer to atomic_long_t
3651 *
3652 * Atomically updates @v to (@v + 1) with rel
3653 *
3654 * Unsafe to use in noinstr code; use raw_ato
3655 *
3656 * Return: The updated value of @v.
3657 */
3658 static __always_inline long
3659 atomic_long_inc_return_relaxed(atomic_long_t
3660 {
3661 instrument_atomic_read_write(v, sizeo
3662 return raw_atomic_long_inc_return_rel
3663 }
3664
3665 /**
3666 * atomic_long_fetch_inc() - atomic increment
3667 * @v: pointer to atomic_long_t
3668 *
3669 * Atomically updates @v to (@v + 1) with ful
3670 *
3671 * Unsafe to use in noinstr code; use raw_ato
3672 *
3673 * Return: The original value of @v.
3674 */
3675 static __always_inline long
3676 atomic_long_fetch_inc(atomic_long_t *v)
3677 {
3678 kcsan_mb();
3679 instrument_atomic_read_write(v, sizeo
3680 return raw_atomic_long_fetch_inc(v);
3681 }
3682
3683 /**
3684 * atomic_long_fetch_inc_acquire() - atomic i
3685 * @v: pointer to atomic_long_t
3686 *
3687 * Atomically updates @v to (@v + 1) with acq
3688 *
3689 * Unsafe to use in noinstr code; use raw_ato
3690 *
3691 * Return: The original value of @v.
3692 */
3693 static __always_inline long
3694 atomic_long_fetch_inc_acquire(atomic_long_t *
3695 {
3696 instrument_atomic_read_write(v, sizeo
3697 return raw_atomic_long_fetch_inc_acqu
3698 }
3699
3700 /**
3701 * atomic_long_fetch_inc_release() - atomic i
3702 * @v: pointer to atomic_long_t
3703 *
3704 * Atomically updates @v to (@v + 1) with rel
3705 *
3706 * Unsafe to use in noinstr code; use raw_ato
3707 *
3708 * Return: The original value of @v.
3709 */
3710 static __always_inline long
3711 atomic_long_fetch_inc_release(atomic_long_t *
3712 {
3713 kcsan_release();
3714 instrument_atomic_read_write(v, sizeo
3715 return raw_atomic_long_fetch_inc_rele
3716 }
3717
3718 /**
3719 * atomic_long_fetch_inc_relaxed() - atomic i
3720 * @v: pointer to atomic_long_t
3721 *
3722 * Atomically updates @v to (@v + 1) with rel
3723 *
3724 * Unsafe to use in noinstr code; use raw_ato
3725 *
3726 * Return: The original value of @v.
3727 */
3728 static __always_inline long
3729 atomic_long_fetch_inc_relaxed(atomic_long_t *
3730 {
3731 instrument_atomic_read_write(v, sizeo
3732 return raw_atomic_long_fetch_inc_rela
3733 }
3734
3735 /**
3736 * atomic_long_dec() - atomic decrement with
3737 * @v: pointer to atomic_long_t
3738 *
3739 * Atomically updates @v to (@v - 1) with rel
3740 *
3741 * Unsafe to use in noinstr code; use raw_ato
3742 *
3743 * Return: Nothing.
3744 */
3745 static __always_inline void
3746 atomic_long_dec(atomic_long_t *v)
3747 {
3748 instrument_atomic_read_write(v, sizeo
3749 raw_atomic_long_dec(v);
3750 }
3751
3752 /**
3753 * atomic_long_dec_return() - atomic decremen
3754 * @v: pointer to atomic_long_t
3755 *
3756 * Atomically updates @v to (@v - 1) with ful
3757 *
3758 * Unsafe to use in noinstr code; use raw_ato
3759 *
3760 * Return: The updated value of @v.
3761 */
3762 static __always_inline long
3763 atomic_long_dec_return(atomic_long_t *v)
3764 {
3765 kcsan_mb();
3766 instrument_atomic_read_write(v, sizeo
3767 return raw_atomic_long_dec_return(v);
3768 }
3769
3770 /**
3771 * atomic_long_dec_return_acquire() - atomic
3772 * @v: pointer to atomic_long_t
3773 *
3774 * Atomically updates @v to (@v - 1) with acq
3775 *
3776 * Unsafe to use in noinstr code; use raw_ato
3777 *
3778 * Return: The updated value of @v.
3779 */
3780 static __always_inline long
3781 atomic_long_dec_return_acquire(atomic_long_t
3782 {
3783 instrument_atomic_read_write(v, sizeo
3784 return raw_atomic_long_dec_return_acq
3785 }
3786
3787 /**
3788 * atomic_long_dec_return_release() - atomic
3789 * @v: pointer to atomic_long_t
3790 *
3791 * Atomically updates @v to (@v - 1) with rel
3792 *
3793 * Unsafe to use in noinstr code; use raw_ato
3794 *
3795 * Return: The updated value of @v.
3796 */
3797 static __always_inline long
3798 atomic_long_dec_return_release(atomic_long_t
3799 {
3800 kcsan_release();
3801 instrument_atomic_read_write(v, sizeo
3802 return raw_atomic_long_dec_return_rel
3803 }
3804
3805 /**
3806 * atomic_long_dec_return_relaxed() - atomic
3807 * @v: pointer to atomic_long_t
3808 *
3809 * Atomically updates @v to (@v - 1) with rel
3810 *
3811 * Unsafe to use in noinstr code; use raw_ato
3812 *
3813 * Return: The updated value of @v.
3814 */
3815 static __always_inline long
3816 atomic_long_dec_return_relaxed(atomic_long_t
3817 {
3818 instrument_atomic_read_write(v, sizeo
3819 return raw_atomic_long_dec_return_rel
3820 }
3821
3822 /**
3823 * atomic_long_fetch_dec() - atomic decrement
3824 * @v: pointer to atomic_long_t
3825 *
3826 * Atomically updates @v to (@v - 1) with ful
3827 *
3828 * Unsafe to use in noinstr code; use raw_ato
3829 *
3830 * Return: The original value of @v.
3831 */
3832 static __always_inline long
3833 atomic_long_fetch_dec(atomic_long_t *v)
3834 {
3835 kcsan_mb();
3836 instrument_atomic_read_write(v, sizeo
3837 return raw_atomic_long_fetch_dec(v);
3838 }
3839
3840 /**
3841 * atomic_long_fetch_dec_acquire() - atomic d
3842 * @v: pointer to atomic_long_t
3843 *
3844 * Atomically updates @v to (@v - 1) with acq
3845 *
3846 * Unsafe to use in noinstr code; use raw_ato
3847 *
3848 * Return: The original value of @v.
3849 */
3850 static __always_inline long
3851 atomic_long_fetch_dec_acquire(atomic_long_t *
3852 {
3853 instrument_atomic_read_write(v, sizeo
3854 return raw_atomic_long_fetch_dec_acqu
3855 }
3856
3857 /**
3858 * atomic_long_fetch_dec_release() - atomic d
3859 * @v: pointer to atomic_long_t
3860 *
3861 * Atomically updates @v to (@v - 1) with rel
3862 *
3863 * Unsafe to use in noinstr code; use raw_ato
3864 *
3865 * Return: The original value of @v.
3866 */
3867 static __always_inline long
3868 atomic_long_fetch_dec_release(atomic_long_t *
3869 {
3870 kcsan_release();
3871 instrument_atomic_read_write(v, sizeo
3872 return raw_atomic_long_fetch_dec_rele
3873 }
3874
3875 /**
3876 * atomic_long_fetch_dec_relaxed() - atomic d
3877 * @v: pointer to atomic_long_t
3878 *
3879 * Atomically updates @v to (@v - 1) with rel
3880 *
3881 * Unsafe to use in noinstr code; use raw_ato
3882 *
3883 * Return: The original value of @v.
3884 */
3885 static __always_inline long
3886 atomic_long_fetch_dec_relaxed(atomic_long_t *
3887 {
3888 instrument_atomic_read_write(v, sizeo
3889 return raw_atomic_long_fetch_dec_rela
3890 }
3891
3892 /**
3893 * atomic_long_and() - atomic bitwise AND wit
3894 * @i: long value
3895 * @v: pointer to atomic_long_t
3896 *
3897 * Atomically updates @v to (@v & @i) with re
3898 *
3899 * Unsafe to use in noinstr code; use raw_ato
3900 *
3901 * Return: Nothing.
3902 */
3903 static __always_inline void
3904 atomic_long_and(long i, atomic_long_t *v)
3905 {
3906 instrument_atomic_read_write(v, sizeo
3907 raw_atomic_long_and(i, v);
3908 }
3909
3910 /**
3911 * atomic_long_fetch_and() - atomic bitwise A
3912 * @i: long value
3913 * @v: pointer to atomic_long_t
3914 *
3915 * Atomically updates @v to (@v & @i) with fu
3916 *
3917 * Unsafe to use in noinstr code; use raw_ato
3918 *
3919 * Return: The original value of @v.
3920 */
3921 static __always_inline long
3922 atomic_long_fetch_and(long i, atomic_long_t *
3923 {
3924 kcsan_mb();
3925 instrument_atomic_read_write(v, sizeo
3926 return raw_atomic_long_fetch_and(i, v
3927 }
3928
3929 /**
3930 * atomic_long_fetch_and_acquire() - atomic b
3931 * @i: long value
3932 * @v: pointer to atomic_long_t
3933 *
3934 * Atomically updates @v to (@v & @i) with ac
3935 *
3936 * Unsafe to use in noinstr code; use raw_ato
3937 *
3938 * Return: The original value of @v.
3939 */
3940 static __always_inline long
3941 atomic_long_fetch_and_acquire(long i, atomic_
3942 {
3943 instrument_atomic_read_write(v, sizeo
3944 return raw_atomic_long_fetch_and_acqu
3945 }
3946
3947 /**
3948 * atomic_long_fetch_and_release() - atomic b
3949 * @i: long value
3950 * @v: pointer to atomic_long_t
3951 *
3952 * Atomically updates @v to (@v & @i) with re
3953 *
3954 * Unsafe to use in noinstr code; use raw_ato
3955 *
3956 * Return: The original value of @v.
3957 */
3958 static __always_inline long
3959 atomic_long_fetch_and_release(long i, atomic_
3960 {
3961 kcsan_release();
3962 instrument_atomic_read_write(v, sizeo
3963 return raw_atomic_long_fetch_and_rele
3964 }
3965
3966 /**
3967 * atomic_long_fetch_and_relaxed() - atomic b
3968 * @i: long value
3969 * @v: pointer to atomic_long_t
3970 *
3971 * Atomically updates @v to (@v & @i) with re
3972 *
3973 * Unsafe to use in noinstr code; use raw_ato
3974 *
3975 * Return: The original value of @v.
3976 */
3977 static __always_inline long
3978 atomic_long_fetch_and_relaxed(long i, atomic_
3979 {
3980 instrument_atomic_read_write(v, sizeo
3981 return raw_atomic_long_fetch_and_rela
3982 }
3983
3984 /**
3985 * atomic_long_andnot() - atomic bitwise AND
3986 * @i: long value
3987 * @v: pointer to atomic_long_t
3988 *
3989 * Atomically updates @v to (@v & ~@i) with r
3990 *
3991 * Unsafe to use in noinstr code; use raw_ato
3992 *
3993 * Return: Nothing.
3994 */
3995 static __always_inline void
3996 atomic_long_andnot(long i, atomic_long_t *v)
3997 {
3998 instrument_atomic_read_write(v, sizeo
3999 raw_atomic_long_andnot(i, v);
4000 }
4001
4002 /**
4003 * atomic_long_fetch_andnot() - atomic bitwis
4004 * @i: long value
4005 * @v: pointer to atomic_long_t
4006 *
4007 * Atomically updates @v to (@v & ~@i) with f
4008 *
4009 * Unsafe to use in noinstr code; use raw_ato
4010 *
4011 * Return: The original value of @v.
4012 */
4013 static __always_inline long
4014 atomic_long_fetch_andnot(long i, atomic_long_
4015 {
4016 kcsan_mb();
4017 instrument_atomic_read_write(v, sizeo
4018 return raw_atomic_long_fetch_andnot(i
4019 }
4020
4021 /**
4022 * atomic_long_fetch_andnot_acquire() - atomi
4023 * @i: long value
4024 * @v: pointer to atomic_long_t
4025 *
4026 * Atomically updates @v to (@v & ~@i) with a
4027 *
4028 * Unsafe to use in noinstr code; use raw_ato
4029 *
4030 * Return: The original value of @v.
4031 */
4032 static __always_inline long
4033 atomic_long_fetch_andnot_acquire(long i, atom
4034 {
4035 instrument_atomic_read_write(v, sizeo
4036 return raw_atomic_long_fetch_andnot_a
4037 }
4038
4039 /**
4040 * atomic_long_fetch_andnot_release() - atomi
4041 * @i: long value
4042 * @v: pointer to atomic_long_t
4043 *
4044 * Atomically updates @v to (@v & ~@i) with r
4045 *
4046 * Unsafe to use in noinstr code; use raw_ato
4047 *
4048 * Return: The original value of @v.
4049 */
4050 static __always_inline long
4051 atomic_long_fetch_andnot_release(long i, atom
4052 {
4053 kcsan_release();
4054 instrument_atomic_read_write(v, sizeo
4055 return raw_atomic_long_fetch_andnot_r
4056 }
4057
4058 /**
4059 * atomic_long_fetch_andnot_relaxed() - atomi
4060 * @i: long value
4061 * @v: pointer to atomic_long_t
4062 *
4063 * Atomically updates @v to (@v & ~@i) with r
4064 *
4065 * Unsafe to use in noinstr code; use raw_ato
4066 *
4067 * Return: The original value of @v.
4068 */
4069 static __always_inline long
4070 atomic_long_fetch_andnot_relaxed(long i, atom
4071 {
4072 instrument_atomic_read_write(v, sizeo
4073 return raw_atomic_long_fetch_andnot_r
4074 }
4075
4076 /**
4077 * atomic_long_or() - atomic bitwise OR with
4078 * @i: long value
4079 * @v: pointer to atomic_long_t
4080 *
4081 * Atomically updates @v to (@v | @i) with re
4082 *
4083 * Unsafe to use in noinstr code; use raw_ato
4084 *
4085 * Return: Nothing.
4086 */
4087 static __always_inline void
4088 atomic_long_or(long i, atomic_long_t *v)
4089 {
4090 instrument_atomic_read_write(v, sizeo
4091 raw_atomic_long_or(i, v);
4092 }
4093
4094 /**
4095 * atomic_long_fetch_or() - atomic bitwise OR
4096 * @i: long value
4097 * @v: pointer to atomic_long_t
4098 *
4099 * Atomically updates @v to (@v | @i) with fu
4100 *
4101 * Unsafe to use in noinstr code; use raw_ato
4102 *
4103 * Return: The original value of @v.
4104 */
4105 static __always_inline long
4106 atomic_long_fetch_or(long i, atomic_long_t *v
4107 {
4108 kcsan_mb();
4109 instrument_atomic_read_write(v, sizeo
4110 return raw_atomic_long_fetch_or(i, v)
4111 }
4112
4113 /**
4114 * atomic_long_fetch_or_acquire() - atomic bi
4115 * @i: long value
4116 * @v: pointer to atomic_long_t
4117 *
4118 * Atomically updates @v to (@v | @i) with ac
4119 *
4120 * Unsafe to use in noinstr code; use raw_ato
4121 *
4122 * Return: The original value of @v.
4123 */
4124 static __always_inline long
4125 atomic_long_fetch_or_acquire(long i, atomic_l
4126 {
4127 instrument_atomic_read_write(v, sizeo
4128 return raw_atomic_long_fetch_or_acqui
4129 }
4130
4131 /**
4132 * atomic_long_fetch_or_release() - atomic bi
4133 * @i: long value
4134 * @v: pointer to atomic_long_t
4135 *
4136 * Atomically updates @v to (@v | @i) with re
4137 *
4138 * Unsafe to use in noinstr code; use raw_ato
4139 *
4140 * Return: The original value of @v.
4141 */
4142 static __always_inline long
4143 atomic_long_fetch_or_release(long i, atomic_l
4144 {
4145 kcsan_release();
4146 instrument_atomic_read_write(v, sizeo
4147 return raw_atomic_long_fetch_or_relea
4148 }
4149
4150 /**
4151 * atomic_long_fetch_or_relaxed() - atomic bi
4152 * @i: long value
4153 * @v: pointer to atomic_long_t
4154 *
4155 * Atomically updates @v to (@v | @i) with re
4156 *
4157 * Unsafe to use in noinstr code; use raw_ato
4158 *
4159 * Return: The original value of @v.
4160 */
4161 static __always_inline long
4162 atomic_long_fetch_or_relaxed(long i, atomic_l
4163 {
4164 instrument_atomic_read_write(v, sizeo
4165 return raw_atomic_long_fetch_or_relax
4166 }
4167
4168 /**
4169 * atomic_long_xor() - atomic bitwise XOR wit
4170 * @i: long value
4171 * @v: pointer to atomic_long_t
4172 *
4173 * Atomically updates @v to (@v ^ @i) with re
4174 *
4175 * Unsafe to use in noinstr code; use raw_ato
4176 *
4177 * Return: Nothing.
4178 */
4179 static __always_inline void
4180 atomic_long_xor(long i, atomic_long_t *v)
4181 {
4182 instrument_atomic_read_write(v, sizeo
4183 raw_atomic_long_xor(i, v);
4184 }
4185
4186 /**
4187 * atomic_long_fetch_xor() - atomic bitwise X
4188 * @i: long value
4189 * @v: pointer to atomic_long_t
4190 *
4191 * Atomically updates @v to (@v ^ @i) with fu
4192 *
4193 * Unsafe to use in noinstr code; use raw_ato
4194 *
4195 * Return: The original value of @v.
4196 */
4197 static __always_inline long
4198 atomic_long_fetch_xor(long i, atomic_long_t *
4199 {
4200 kcsan_mb();
4201 instrument_atomic_read_write(v, sizeo
4202 return raw_atomic_long_fetch_xor(i, v
4203 }
4204
4205 /**
4206 * atomic_long_fetch_xor_acquire() - atomic b
4207 * @i: long value
4208 * @v: pointer to atomic_long_t
4209 *
4210 * Atomically updates @v to (@v ^ @i) with ac
4211 *
4212 * Unsafe to use in noinstr code; use raw_ato
4213 *
4214 * Return: The original value of @v.
4215 */
4216 static __always_inline long
4217 atomic_long_fetch_xor_acquire(long i, atomic_
4218 {
4219 instrument_atomic_read_write(v, sizeo
4220 return raw_atomic_long_fetch_xor_acqu
4221 }
4222
4223 /**
4224 * atomic_long_fetch_xor_release() - atomic b
4225 * @i: long value
4226 * @v: pointer to atomic_long_t
4227 *
4228 * Atomically updates @v to (@v ^ @i) with re
4229 *
4230 * Unsafe to use in noinstr code; use raw_ato
4231 *
4232 * Return: The original value of @v.
4233 */
4234 static __always_inline long
4235 atomic_long_fetch_xor_release(long i, atomic_
4236 {
4237 kcsan_release();
4238 instrument_atomic_read_write(v, sizeo
4239 return raw_atomic_long_fetch_xor_rele
4240 }
4241
4242 /**
4243 * atomic_long_fetch_xor_relaxed() - atomic b
4244 * @i: long value
4245 * @v: pointer to atomic_long_t
4246 *
4247 * Atomically updates @v to (@v ^ @i) with re
4248 *
4249 * Unsafe to use in noinstr code; use raw_ato
4250 *
4251 * Return: The original value of @v.
4252 */
4253 static __always_inline long
4254 atomic_long_fetch_xor_relaxed(long i, atomic_
4255 {
4256 instrument_atomic_read_write(v, sizeo
4257 return raw_atomic_long_fetch_xor_rela
4258 }
4259
4260 /**
4261 * atomic_long_xchg() - atomic exchange with
4262 * @v: pointer to atomic_long_t
4263 * @new: long value to assign
4264 *
4265 * Atomically updates @v to @new with full or
4266 *
4267 * Unsafe to use in noinstr code; use raw_ato
4268 *
4269 * Return: The original value of @v.
4270 */
4271 static __always_inline long
4272 atomic_long_xchg(atomic_long_t *v, long new)
4273 {
4274 kcsan_mb();
4275 instrument_atomic_read_write(v, sizeo
4276 return raw_atomic_long_xchg(v, new);
4277 }
4278
4279 /**
4280 * atomic_long_xchg_acquire() - atomic exchan
4281 * @v: pointer to atomic_long_t
4282 * @new: long value to assign
4283 *
4284 * Atomically updates @v to @new with acquire
4285 *
4286 * Unsafe to use in noinstr code; use raw_ato
4287 *
4288 * Return: The original value of @v.
4289 */
4290 static __always_inline long
4291 atomic_long_xchg_acquire(atomic_long_t *v, lo
4292 {
4293 instrument_atomic_read_write(v, sizeo
4294 return raw_atomic_long_xchg_acquire(v
4295 }
4296
4297 /**
4298 * atomic_long_xchg_release() - atomic exchan
4299 * @v: pointer to atomic_long_t
4300 * @new: long value to assign
4301 *
4302 * Atomically updates @v to @new with release
4303 *
4304 * Unsafe to use in noinstr code; use raw_ato
4305 *
4306 * Return: The original value of @v.
4307 */
4308 static __always_inline long
4309 atomic_long_xchg_release(atomic_long_t *v, lo
4310 {
4311 kcsan_release();
4312 instrument_atomic_read_write(v, sizeo
4313 return raw_atomic_long_xchg_release(v
4314 }
4315
4316 /**
4317 * atomic_long_xchg_relaxed() - atomic exchan
4318 * @v: pointer to atomic_long_t
4319 * @new: long value to assign
4320 *
4321 * Atomically updates @v to @new with relaxed
4322 *
4323 * Unsafe to use in noinstr code; use raw_ato
4324 *
4325 * Return: The original value of @v.
4326 */
4327 static __always_inline long
4328 atomic_long_xchg_relaxed(atomic_long_t *v, lo
4329 {
4330 instrument_atomic_read_write(v, sizeo
4331 return raw_atomic_long_xchg_relaxed(v
4332 }
4333
4334 /**
4335 * atomic_long_cmpxchg() - atomic compare and
4336 * @v: pointer to atomic_long_t
4337 * @old: long value to compare with
4338 * @new: long value to assign
4339 *
4340 * If (@v == @old), atomically updates @v to
4341 * Otherwise, @v is not modified and relaxed
4342 *
4343 * Unsafe to use in noinstr code; use raw_ato
4344 *
4345 * Return: The original value of @v.
4346 */
4347 static __always_inline long
4348 atomic_long_cmpxchg(atomic_long_t *v, long ol
4349 {
4350 kcsan_mb();
4351 instrument_atomic_read_write(v, sizeo
4352 return raw_atomic_long_cmpxchg(v, old
4353 }
4354
4355 /**
4356 * atomic_long_cmpxchg_acquire() - atomic com
4357 * @v: pointer to atomic_long_t
4358 * @old: long value to compare with
4359 * @new: long value to assign
4360 *
4361 * If (@v == @old), atomically updates @v to
4362 * Otherwise, @v is not modified and relaxed
4363 *
4364 * Unsafe to use in noinstr code; use raw_ato
4365 *
4366 * Return: The original value of @v.
4367 */
4368 static __always_inline long
4369 atomic_long_cmpxchg_acquire(atomic_long_t *v,
4370 {
4371 instrument_atomic_read_write(v, sizeo
4372 return raw_atomic_long_cmpxchg_acquir
4373 }
4374
4375 /**
4376 * atomic_long_cmpxchg_release() - atomic com
4377 * @v: pointer to atomic_long_t
4378 * @old: long value to compare with
4379 * @new: long value to assign
4380 *
4381 * If (@v == @old), atomically updates @v to
4382 * Otherwise, @v is not modified and relaxed
4383 *
4384 * Unsafe to use in noinstr code; use raw_ato
4385 *
4386 * Return: The original value of @v.
4387 */
4388 static __always_inline long
4389 atomic_long_cmpxchg_release(atomic_long_t *v,
4390 {
4391 kcsan_release();
4392 instrument_atomic_read_write(v, sizeo
4393 return raw_atomic_long_cmpxchg_releas
4394 }
4395
4396 /**
4397 * atomic_long_cmpxchg_relaxed() - atomic com
4398 * @v: pointer to atomic_long_t
4399 * @old: long value to compare with
4400 * @new: long value to assign
4401 *
4402 * If (@v == @old), atomically updates @v to
4403 * Otherwise, @v is not modified and relaxed
4404 *
4405 * Unsafe to use in noinstr code; use raw_ato
4406 *
4407 * Return: The original value of @v.
4408 */
4409 static __always_inline long
4410 atomic_long_cmpxchg_relaxed(atomic_long_t *v,
4411 {
4412 instrument_atomic_read_write(v, sizeo
4413 return raw_atomic_long_cmpxchg_relaxe
4414 }
4415
4416 /**
4417 * atomic_long_try_cmpxchg() - atomic compare
4418 * @v: pointer to atomic_long_t
4419 * @old: pointer to long value to compare wit
4420 * @new: long value to assign
4421 *
4422 * If (@v == @old), atomically updates @v to
4423 * Otherwise, @v is not modified, @old is upd
4424 * and relaxed ordering is provided.
4425 *
4426 * Unsafe to use in noinstr code; use raw_ato
4427 *
4428 * Return: @true if the exchange occured, @fa
4429 */
4430 static __always_inline bool
4431 atomic_long_try_cmpxchg(atomic_long_t *v, lon
4432 {
4433 kcsan_mb();
4434 instrument_atomic_read_write(v, sizeo
4435 instrument_atomic_read_write(old, siz
4436 return raw_atomic_long_try_cmpxchg(v,
4437 }
4438
4439 /**
4440 * atomic_long_try_cmpxchg_acquire() - atomic
4441 * @v: pointer to atomic_long_t
4442 * @old: pointer to long value to compare wit
4443 * @new: long value to assign
4444 *
4445 * If (@v == @old), atomically updates @v to
4446 * Otherwise, @v is not modified, @old is upd
4447 * and relaxed ordering is provided.
4448 *
4449 * Unsafe to use in noinstr code; use raw_ato
4450 *
4451 * Return: @true if the exchange occured, @fa
4452 */
4453 static __always_inline bool
4454 atomic_long_try_cmpxchg_acquire(atomic_long_t
4455 {
4456 instrument_atomic_read_write(v, sizeo
4457 instrument_atomic_read_write(old, siz
4458 return raw_atomic_long_try_cmpxchg_ac
4459 }
4460
4461 /**
4462 * atomic_long_try_cmpxchg_release() - atomic
4463 * @v: pointer to atomic_long_t
4464 * @old: pointer to long value to compare wit
4465 * @new: long value to assign
4466 *
4467 * If (@v == @old), atomically updates @v to
4468 * Otherwise, @v is not modified, @old is upd
4469 * and relaxed ordering is provided.
4470 *
4471 * Unsafe to use in noinstr code; use raw_ato
4472 *
4473 * Return: @true if the exchange occured, @fa
4474 */
4475 static __always_inline bool
4476 atomic_long_try_cmpxchg_release(atomic_long_t
4477 {
4478 kcsan_release();
4479 instrument_atomic_read_write(v, sizeo
4480 instrument_atomic_read_write(old, siz
4481 return raw_atomic_long_try_cmpxchg_re
4482 }
4483
4484 /**
4485 * atomic_long_try_cmpxchg_relaxed() - atomic
4486 * @v: pointer to atomic_long_t
4487 * @old: pointer to long value to compare wit
4488 * @new: long value to assign
4489 *
4490 * If (@v == @old), atomically updates @v to
4491 * Otherwise, @v is not modified, @old is upd
4492 * and relaxed ordering is provided.
4493 *
4494 * Unsafe to use in noinstr code; use raw_ato
4495 *
4496 * Return: @true if the exchange occured, @fa
4497 */
4498 static __always_inline bool
4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t
4500 {
4501 instrument_atomic_read_write(v, sizeo
4502 instrument_atomic_read_write(old, siz
4503 return raw_atomic_long_try_cmpxchg_re
4504 }
4505
4506 /**
4507 * atomic_long_sub_and_test() - atomic subtra
4508 * @i: long value to subtract
4509 * @v: pointer to atomic_long_t
4510 *
4511 * Atomically updates @v to (@v - @i) with fu
4512 *
4513 * Unsafe to use in noinstr code; use raw_ato
4514 *
4515 * Return: @true if the resulting value of @v
4516 */
4517 static __always_inline bool
4518 atomic_long_sub_and_test(long i, atomic_long_
4519 {
4520 kcsan_mb();
4521 instrument_atomic_read_write(v, sizeo
4522 return raw_atomic_long_sub_and_test(i
4523 }
4524
4525 /**
4526 * atomic_long_dec_and_test() - atomic decrem
4527 * @v: pointer to atomic_long_t
4528 *
4529 * Atomically updates @v to (@v - 1) with ful
4530 *
4531 * Unsafe to use in noinstr code; use raw_ato
4532 *
4533 * Return: @true if the resulting value of @v
4534 */
4535 static __always_inline bool
4536 atomic_long_dec_and_test(atomic_long_t *v)
4537 {
4538 kcsan_mb();
4539 instrument_atomic_read_write(v, sizeo
4540 return raw_atomic_long_dec_and_test(v
4541 }
4542
4543 /**
4544 * atomic_long_inc_and_test() - atomic increm
4545 * @v: pointer to atomic_long_t
4546 *
4547 * Atomically updates @v to (@v + 1) with ful
4548 *
4549 * Unsafe to use in noinstr code; use raw_ato
4550 *
4551 * Return: @true if the resulting value of @v
4552 */
4553 static __always_inline bool
4554 atomic_long_inc_and_test(atomic_long_t *v)
4555 {
4556 kcsan_mb();
4557 instrument_atomic_read_write(v, sizeo
4558 return raw_atomic_long_inc_and_test(v
4559 }
4560
4561 /**
4562 * atomic_long_add_negative() - atomic add an
4563 * @i: long value to add
4564 * @v: pointer to atomic_long_t
4565 *
4566 * Atomically updates @v to (@v + @i) with fu
4567 *
4568 * Unsafe to use in noinstr code; use raw_ato
4569 *
4570 * Return: @true if the resulting value of @v
4571 */
4572 static __always_inline bool
4573 atomic_long_add_negative(long i, atomic_long_
4574 {
4575 kcsan_mb();
4576 instrument_atomic_read_write(v, sizeo
4577 return raw_atomic_long_add_negative(i
4578 }
4579
4580 /**
4581 * atomic_long_add_negative_acquire() - atomi
4582 * @i: long value to add
4583 * @v: pointer to atomic_long_t
4584 *
4585 * Atomically updates @v to (@v + @i) with ac
4586 *
4587 * Unsafe to use in noinstr code; use raw_ato
4588 *
4589 * Return: @true if the resulting value of @v
4590 */
4591 static __always_inline bool
4592 atomic_long_add_negative_acquire(long i, atom
4593 {
4594 instrument_atomic_read_write(v, sizeo
4595 return raw_atomic_long_add_negative_a
4596 }
4597
4598 /**
4599 * atomic_long_add_negative_release() - atomi
4600 * @i: long value to add
4601 * @v: pointer to atomic_long_t
4602 *
4603 * Atomically updates @v to (@v + @i) with re
4604 *
4605 * Unsafe to use in noinstr code; use raw_ato
4606 *
4607 * Return: @true if the resulting value of @v
4608 */
4609 static __always_inline bool
4610 atomic_long_add_negative_release(long i, atom
4611 {
4612 kcsan_release();
4613 instrument_atomic_read_write(v, sizeo
4614 return raw_atomic_long_add_negative_r
4615 }
4616
4617 /**
4618 * atomic_long_add_negative_relaxed() - atomi
4619 * @i: long value to add
4620 * @v: pointer to atomic_long_t
4621 *
4622 * Atomically updates @v to (@v + @i) with re
4623 *
4624 * Unsafe to use in noinstr code; use raw_ato
4625 *
4626 * Return: @true if the resulting value of @v
4627 */
4628 static __always_inline bool
4629 atomic_long_add_negative_relaxed(long i, atom
4630 {
4631 instrument_atomic_read_write(v, sizeo
4632 return raw_atomic_long_add_negative_r
4633 }
4634
4635 /**
4636 * atomic_long_fetch_add_unless() - atomic ad
4637 * @v: pointer to atomic_long_t
4638 * @a: long value to add
4639 * @u: long value to compare with
4640 *
4641 * If (@v != @u), atomically updates @v to (@
4642 * Otherwise, @v is not modified and relaxed
4643 *
4644 * Unsafe to use in noinstr code; use raw_ato
4645 *
4646 * Return: The original value of @v.
4647 */
4648 static __always_inline long
4649 atomic_long_fetch_add_unless(atomic_long_t *v
4650 {
4651 kcsan_mb();
4652 instrument_atomic_read_write(v, sizeo
4653 return raw_atomic_long_fetch_add_unle
4654 }
4655
4656 /**
4657 * atomic_long_add_unless() - atomic add unle
4658 * @v: pointer to atomic_long_t
4659 * @a: long value to add
4660 * @u: long value to compare with
4661 *
4662 * If (@v != @u), atomically updates @v to (@
4663 * Otherwise, @v is not modified and relaxed
4664 *
4665 * Unsafe to use in noinstr code; use raw_ato
4666 *
4667 * Return: @true if @v was updated, @false ot
4668 */
4669 static __always_inline bool
4670 atomic_long_add_unless(atomic_long_t *v, long
4671 {
4672 kcsan_mb();
4673 instrument_atomic_read_write(v, sizeo
4674 return raw_atomic_long_add_unless(v,
4675 }
4676
4677 /**
4678 * atomic_long_inc_not_zero() - atomic increm
4679 * @v: pointer to atomic_long_t
4680 *
4681 * If (@v != 0), atomically updates @v to (@v
4682 * Otherwise, @v is not modified and relaxed
4683 *
4684 * Unsafe to use in noinstr code; use raw_ato
4685 *
4686 * Return: @true if @v was updated, @false ot
4687 */
4688 static __always_inline bool
4689 atomic_long_inc_not_zero(atomic_long_t *v)
4690 {
4691 kcsan_mb();
4692 instrument_atomic_read_write(v, sizeo
4693 return raw_atomic_long_inc_not_zero(v
4694 }
4695
4696 /**
4697 * atomic_long_inc_unless_negative() - atomic
4698 * @v: pointer to atomic_long_t
4699 *
4700 * If (@v >= 0), atomically updates @v to (@v
4701 * Otherwise, @v is not modified and relaxed
4702 *
4703 * Unsafe to use in noinstr code; use raw_ato
4704 *
4705 * Return: @true if @v was updated, @false ot
4706 */
4707 static __always_inline bool
4708 atomic_long_inc_unless_negative(atomic_long_t
4709 {
4710 kcsan_mb();
4711 instrument_atomic_read_write(v, sizeo
4712 return raw_atomic_long_inc_unless_neg
4713 }
4714
4715 /**
4716 * atomic_long_dec_unless_positive() - atomic
4717 * @v: pointer to atomic_long_t
4718 *
4719 * If (@v <= 0), atomically updates @v to (@v
4720 * Otherwise, @v is not modified and relaxed
4721 *
4722 * Unsafe to use in noinstr code; use raw_ato
4723 *
4724 * Return: @true if @v was updated, @false ot
4725 */
4726 static __always_inline bool
4727 atomic_long_dec_unless_positive(atomic_long_t
4728 {
4729 kcsan_mb();
4730 instrument_atomic_read_write(v, sizeo
4731 return raw_atomic_long_dec_unless_pos
4732 }
4733
4734 /**
4735 * atomic_long_dec_if_positive() - atomic dec
4736 * @v: pointer to atomic_long_t
4737 *
4738 * If (@v > 0), atomically updates @v to (@v
4739 * Otherwise, @v is not modified and relaxed
4740 *
4741 * Unsafe to use in noinstr code; use raw_ato
4742 *
4743 * Return: The old value of (@v - 1), regardl
4744 */
4745 static __always_inline long
4746 atomic_long_dec_if_positive(atomic_long_t *v)
4747 {
4748 kcsan_mb();
4749 instrument_atomic_read_write(v, sizeo
4750 return raw_atomic_long_dec_if_positiv
4751 }
4752
4753 #define xchg(ptr, ...) \
4754 ({ \
4755 typeof(ptr) __ai_ptr = (ptr); \
4756 kcsan_mb(); \
4757 instrument_atomic_read_write(__ai_ptr
4758 raw_xchg(__ai_ptr, __VA_ARGS__); \
4759 })
4760
4761 #define xchg_acquire(ptr, ...) \
4762 ({ \
4763 typeof(ptr) __ai_ptr = (ptr); \
4764 instrument_atomic_read_write(__ai_ptr
4765 raw_xchg_acquire(__ai_ptr, __VA_ARGS_
4766 })
4767
4768 #define xchg_release(ptr, ...) \
4769 ({ \
4770 typeof(ptr) __ai_ptr = (ptr); \
4771 kcsan_release(); \
4772 instrument_atomic_read_write(__ai_ptr
4773 raw_xchg_release(__ai_ptr, __VA_ARGS_
4774 })
4775
4776 #define xchg_relaxed(ptr, ...) \
4777 ({ \
4778 typeof(ptr) __ai_ptr = (ptr); \
4779 instrument_atomic_read_write(__ai_ptr
4780 raw_xchg_relaxed(__ai_ptr, __VA_ARGS_
4781 })
4782
4783 #define cmpxchg(ptr, ...) \
4784 ({ \
4785 typeof(ptr) __ai_ptr = (ptr); \
4786 kcsan_mb(); \
4787 instrument_atomic_read_write(__ai_ptr
4788 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
4789 })
4790
4791 #define cmpxchg_acquire(ptr, ...) \
4792 ({ \
4793 typeof(ptr) __ai_ptr = (ptr); \
4794 instrument_atomic_read_write(__ai_ptr
4795 raw_cmpxchg_acquire(__ai_ptr, __VA_AR
4796 })
4797
4798 #define cmpxchg_release(ptr, ...) \
4799 ({ \
4800 typeof(ptr) __ai_ptr = (ptr); \
4801 kcsan_release(); \
4802 instrument_atomic_read_write(__ai_ptr
4803 raw_cmpxchg_release(__ai_ptr, __VA_AR
4804 })
4805
4806 #define cmpxchg_relaxed(ptr, ...) \
4807 ({ \
4808 typeof(ptr) __ai_ptr = (ptr); \
4809 instrument_atomic_read_write(__ai_ptr
4810 raw_cmpxchg_relaxed(__ai_ptr, __VA_AR
4811 })
4812
4813 #define cmpxchg64(ptr, ...) \
4814 ({ \
4815 typeof(ptr) __ai_ptr = (ptr); \
4816 kcsan_mb(); \
4817 instrument_atomic_read_write(__ai_ptr
4818 raw_cmpxchg64(__ai_ptr, __VA_ARGS__);
4819 })
4820
4821 #define cmpxchg64_acquire(ptr, ...) \
4822 ({ \
4823 typeof(ptr) __ai_ptr = (ptr); \
4824 instrument_atomic_read_write(__ai_ptr
4825 raw_cmpxchg64_acquire(__ai_ptr, __VA_
4826 })
4827
4828 #define cmpxchg64_release(ptr, ...) \
4829 ({ \
4830 typeof(ptr) __ai_ptr = (ptr); \
4831 kcsan_release(); \
4832 instrument_atomic_read_write(__ai_ptr
4833 raw_cmpxchg64_release(__ai_ptr, __VA_
4834 })
4835
4836 #define cmpxchg64_relaxed(ptr, ...) \
4837 ({ \
4838 typeof(ptr) __ai_ptr = (ptr); \
4839 instrument_atomic_read_write(__ai_ptr
4840 raw_cmpxchg64_relaxed(__ai_ptr, __VA_
4841 })
4842
4843 #define cmpxchg128(ptr, ...) \
4844 ({ \
4845 typeof(ptr) __ai_ptr = (ptr); \
4846 kcsan_mb(); \
4847 instrument_atomic_read_write(__ai_ptr
4848 raw_cmpxchg128(__ai_ptr, __VA_ARGS__)
4849 })
4850
4851 #define cmpxchg128_acquire(ptr, ...) \
4852 ({ \
4853 typeof(ptr) __ai_ptr = (ptr); \
4854 instrument_atomic_read_write(__ai_ptr
4855 raw_cmpxchg128_acquire(__ai_ptr, __VA
4856 })
4857
4858 #define cmpxchg128_release(ptr, ...) \
4859 ({ \
4860 typeof(ptr) __ai_ptr = (ptr); \
4861 kcsan_release(); \
4862 instrument_atomic_read_write(__ai_ptr
4863 raw_cmpxchg128_release(__ai_ptr, __VA
4864 })
4865
4866 #define cmpxchg128_relaxed(ptr, ...) \
4867 ({ \
4868 typeof(ptr) __ai_ptr = (ptr); \
4869 instrument_atomic_read_write(__ai_ptr
4870 raw_cmpxchg128_relaxed(__ai_ptr, __VA
4871 })
4872
4873 #define try_cmpxchg(ptr, oldp, ...) \
4874 ({ \
4875 typeof(ptr) __ai_ptr = (ptr); \
4876 typeof(oldp) __ai_oldp = (oldp); \
4877 kcsan_mb(); \
4878 instrument_atomic_read_write(__ai_ptr
4879 instrument_read_write(__ai_oldp, size
4880 raw_try_cmpxchg(__ai_ptr, __ai_oldp,
4881 })
4882
4883 #define try_cmpxchg_acquire(ptr, oldp, ...) \
4884 ({ \
4885 typeof(ptr) __ai_ptr = (ptr); \
4886 typeof(oldp) __ai_oldp = (oldp); \
4887 instrument_atomic_read_write(__ai_ptr
4888 instrument_read_write(__ai_oldp, size
4889 raw_try_cmpxchg_acquire(__ai_ptr, __a
4890 })
4891
4892 #define try_cmpxchg_release(ptr, oldp, ...) \
4893 ({ \
4894 typeof(ptr) __ai_ptr = (ptr); \
4895 typeof(oldp) __ai_oldp = (oldp); \
4896 kcsan_release(); \
4897 instrument_atomic_read_write(__ai_ptr
4898 instrument_read_write(__ai_oldp, size
4899 raw_try_cmpxchg_release(__ai_ptr, __a
4900 })
4901
4902 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
4903 ({ \
4904 typeof(ptr) __ai_ptr = (ptr); \
4905 typeof(oldp) __ai_oldp = (oldp); \
4906 instrument_atomic_read_write(__ai_ptr
4907 instrument_read_write(__ai_oldp, size
4908 raw_try_cmpxchg_relaxed(__ai_ptr, __a
4909 })
4910
4911 #define try_cmpxchg64(ptr, oldp, ...) \
4912 ({ \
4913 typeof(ptr) __ai_ptr = (ptr); \
4914 typeof(oldp) __ai_oldp = (oldp); \
4915 kcsan_mb(); \
4916 instrument_atomic_read_write(__ai_ptr
4917 instrument_read_write(__ai_oldp, size
4918 raw_try_cmpxchg64(__ai_ptr, __ai_oldp
4919 })
4920
4921 #define try_cmpxchg64_acquire(ptr, oldp, ...)
4922 ({ \
4923 typeof(ptr) __ai_ptr = (ptr); \
4924 typeof(oldp) __ai_oldp = (oldp); \
4925 instrument_atomic_read_write(__ai_ptr
4926 instrument_read_write(__ai_oldp, size
4927 raw_try_cmpxchg64_acquire(__ai_ptr, _
4928 })
4929
4930 #define try_cmpxchg64_release(ptr, oldp, ...)
4931 ({ \
4932 typeof(ptr) __ai_ptr = (ptr); \
4933 typeof(oldp) __ai_oldp = (oldp); \
4934 kcsan_release(); \
4935 instrument_atomic_read_write(__ai_ptr
4936 instrument_read_write(__ai_oldp, size
4937 raw_try_cmpxchg64_release(__ai_ptr, _
4938 })
4939
4940 #define try_cmpxchg64_relaxed(ptr, oldp, ...)
4941 ({ \
4942 typeof(ptr) __ai_ptr = (ptr); \
4943 typeof(oldp) __ai_oldp = (oldp); \
4944 instrument_atomic_read_write(__ai_ptr
4945 instrument_read_write(__ai_oldp, size
4946 raw_try_cmpxchg64_relaxed(__ai_ptr, _
4947 })
4948
4949 #define try_cmpxchg128(ptr, oldp, ...) \
4950 ({ \
4951 typeof(ptr) __ai_ptr = (ptr); \
4952 typeof(oldp) __ai_oldp = (oldp); \
4953 kcsan_mb(); \
4954 instrument_atomic_read_write(__ai_ptr
4955 instrument_read_write(__ai_oldp, size
4956 raw_try_cmpxchg128(__ai_ptr, __ai_old
4957 })
4958
4959 #define try_cmpxchg128_acquire(ptr, oldp, ...
4960 ({ \
4961 typeof(ptr) __ai_ptr = (ptr); \
4962 typeof(oldp) __ai_oldp = (oldp); \
4963 instrument_atomic_read_write(__ai_ptr
4964 instrument_read_write(__ai_oldp, size
4965 raw_try_cmpxchg128_acquire(__ai_ptr,
4966 })
4967
4968 #define try_cmpxchg128_release(ptr, oldp, ...
4969 ({ \
4970 typeof(ptr) __ai_ptr = (ptr); \
4971 typeof(oldp) __ai_oldp = (oldp); \
4972 kcsan_release(); \
4973 instrument_atomic_read_write(__ai_ptr
4974 instrument_read_write(__ai_oldp, size
4975 raw_try_cmpxchg128_release(__ai_ptr,
4976 })
4977
4978 #define try_cmpxchg128_relaxed(ptr, oldp, ...
4979 ({ \
4980 typeof(ptr) __ai_ptr = (ptr); \
4981 typeof(oldp) __ai_oldp = (oldp); \
4982 instrument_atomic_read_write(__ai_ptr
4983 instrument_read_write(__ai_oldp, size
4984 raw_try_cmpxchg128_relaxed(__ai_ptr,
4985 })
4986
4987 #define cmpxchg_local(ptr, ...) \
4988 ({ \
4989 typeof(ptr) __ai_ptr = (ptr); \
4990 instrument_atomic_read_write(__ai_ptr
4991 raw_cmpxchg_local(__ai_ptr, __VA_ARGS
4992 })
4993
4994 #define cmpxchg64_local(ptr, ...) \
4995 ({ \
4996 typeof(ptr) __ai_ptr = (ptr); \
4997 instrument_atomic_read_write(__ai_ptr
4998 raw_cmpxchg64_local(__ai_ptr, __VA_AR
4999 })
5000
5001 #define cmpxchg128_local(ptr, ...) \
5002 ({ \
5003 typeof(ptr) __ai_ptr = (ptr); \
5004 instrument_atomic_read_write(__ai_ptr
5005 raw_cmpxchg128_local(__ai_ptr, __VA_A
5006 })
5007
5008 #define sync_cmpxchg(ptr, ...) \
5009 ({ \
5010 typeof(ptr) __ai_ptr = (ptr); \
5011 kcsan_mb(); \
5012 instrument_atomic_read_write(__ai_ptr
5013 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS_
5014 })
5015
5016 #define try_cmpxchg_local(ptr, oldp, ...) \
5017 ({ \
5018 typeof(ptr) __ai_ptr = (ptr); \
5019 typeof(oldp) __ai_oldp = (oldp); \
5020 instrument_atomic_read_write(__ai_ptr
5021 instrument_read_write(__ai_oldp, size
5022 raw_try_cmpxchg_local(__ai_ptr, __ai_
5023 })
5024
5025 #define try_cmpxchg64_local(ptr, oldp, ...) \
5026 ({ \
5027 typeof(ptr) __ai_ptr = (ptr); \
5028 typeof(oldp) __ai_oldp = (oldp); \
5029 instrument_atomic_read_write(__ai_ptr
5030 instrument_read_write(__ai_oldp, size
5031 raw_try_cmpxchg64_local(__ai_ptr, __a
5032 })
5033
5034 #define try_cmpxchg128_local(ptr, oldp, ...)
5035 ({ \
5036 typeof(ptr) __ai_ptr = (ptr); \
5037 typeof(oldp) __ai_oldp = (oldp); \
5038 instrument_atomic_read_write(__ai_ptr
5039 instrument_read_write(__ai_oldp, size
5040 raw_try_cmpxchg128_local(__ai_ptr, __
5041 })
5042
5043 #define sync_try_cmpxchg(ptr, ...) \
5044 ({ \
5045 typeof(ptr) __ai_ptr = (ptr); \
5046 kcsan_mb(); \
5047 instrument_atomic_read_write(__ai_ptr
5048 raw_sync_try_cmpxchg(__ai_ptr, __VA_A
5049 })
5050
5051
5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
5053 // 8829b337928e9508259079d32581775ececd415b
5054
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.