1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
3 // Generated by scripts/atomic/gen-atomic-inst 3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY 4 // DO NOT MODIFY THIS FILE DIRECTLY
5 5
6 /* 6 /*
7 * This file provoides atomic operations with !! 7 * This file provides wrappers with KASAN instrumentation for atomic operations.
8 * KASAN, KCSAN), which should be used unless !! 8 * To use this functionality an arch's atomic.h file needs to define all
9 * instrumentation. Where it is necessary to a !! 9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 * raw_atomic*() operations should be used. !! 10 * this file at the end. This file provides atomic_read() that forwards to
>> 11 * arch_atomic_read() for actual atomic operation.
>> 12 * Note: if an arch atomic operation is implemented by means of other atomic
>> 13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
>> 14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
>> 15 * double instrumentation.
11 */ 16 */
12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H 17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
13 #define _LINUX_ATOMIC_INSTRUMENTED_H 18 #define _LINUX_ATOMIC_INSTRUMENTED_H
14 19
15 #include <linux/build_bug.h> 20 #include <linux/build_bug.h>
16 #include <linux/compiler.h> 21 #include <linux/compiler.h>
17 #include <linux/instrumented.h> 22 #include <linux/instrumented.h>
18 23
19 /** <<
20 * atomic_read() - atomic load with relaxed or <<
21 * @v: pointer to atomic_t <<
22 * <<
23 * Atomically loads the value of @v with relax <<
24 * <<
25 * Unsafe to use in noinstr code; use raw_atom <<
26 * <<
27 * Return: The value loaded from @v. <<
28 */ <<
29 static __always_inline int 24 static __always_inline int
30 atomic_read(const atomic_t *v) 25 atomic_read(const atomic_t *v)
31 { 26 {
32 instrument_atomic_read(v, sizeof(*v)); 27 instrument_atomic_read(v, sizeof(*v));
33 return raw_atomic_read(v); !! 28 return arch_atomic_read(v);
34 } 29 }
35 30
36 /** <<
37 * atomic_read_acquire() - atomic load with ac <<
38 * @v: pointer to atomic_t <<
39 * <<
40 * Atomically loads the value of @v with acqui <<
41 * <<
42 * Unsafe to use in noinstr code; use raw_atom <<
43 * <<
44 * Return: The value loaded from @v. <<
45 */ <<
46 static __always_inline int 31 static __always_inline int
47 atomic_read_acquire(const atomic_t *v) 32 atomic_read_acquire(const atomic_t *v)
48 { 33 {
49 instrument_atomic_read(v, sizeof(*v)); 34 instrument_atomic_read(v, sizeof(*v));
50 return raw_atomic_read_acquire(v); !! 35 return arch_atomic_read_acquire(v);
51 } 36 }
52 37
53 /** <<
54 * atomic_set() - atomic set with relaxed orde <<
55 * @v: pointer to atomic_t <<
56 * @i: int value to assign <<
57 * <<
58 * Atomically sets @v to @i with relaxed order <<
59 * <<
60 * Unsafe to use in noinstr code; use raw_atom <<
61 * <<
62 * Return: Nothing. <<
63 */ <<
64 static __always_inline void 38 static __always_inline void
65 atomic_set(atomic_t *v, int i) 39 atomic_set(atomic_t *v, int i)
66 { 40 {
67 instrument_atomic_write(v, sizeof(*v)) 41 instrument_atomic_write(v, sizeof(*v));
68 raw_atomic_set(v, i); !! 42 arch_atomic_set(v, i);
69 } 43 }
70 44
71 /** <<
72 * atomic_set_release() - atomic set with rele <<
73 * @v: pointer to atomic_t <<
74 * @i: int value to assign <<
75 * <<
76 * Atomically sets @v to @i with release order <<
77 * <<
78 * Unsafe to use in noinstr code; use raw_atom <<
79 * <<
80 * Return: Nothing. <<
81 */ <<
82 static __always_inline void 45 static __always_inline void
83 atomic_set_release(atomic_t *v, int i) 46 atomic_set_release(atomic_t *v, int i)
84 { 47 {
85 kcsan_release(); 48 kcsan_release();
86 instrument_atomic_write(v, sizeof(*v)) 49 instrument_atomic_write(v, sizeof(*v));
87 raw_atomic_set_release(v, i); !! 50 arch_atomic_set_release(v, i);
88 } 51 }
89 52
90 /** <<
91 * atomic_add() - atomic add with relaxed orde <<
92 * @i: int value to add <<
93 * @v: pointer to atomic_t <<
94 * <<
95 * Atomically updates @v to (@v + @i) with rel <<
96 * <<
97 * Unsafe to use in noinstr code; use raw_atom <<
98 * <<
99 * Return: Nothing. <<
100 */ <<
101 static __always_inline void 53 static __always_inline void
102 atomic_add(int i, atomic_t *v) 54 atomic_add(int i, atomic_t *v)
103 { 55 {
104 instrument_atomic_read_write(v, sizeof 56 instrument_atomic_read_write(v, sizeof(*v));
105 raw_atomic_add(i, v); !! 57 arch_atomic_add(i, v);
106 } 58 }
107 59
108 /** <<
109 * atomic_add_return() - atomic add with full <<
110 * @i: int value to add <<
111 * @v: pointer to atomic_t <<
112 * <<
113 * Atomically updates @v to (@v + @i) with ful <<
114 * <<
115 * Unsafe to use in noinstr code; use raw_atom <<
116 * <<
117 * Return: The updated value of @v. <<
118 */ <<
119 static __always_inline int 60 static __always_inline int
120 atomic_add_return(int i, atomic_t *v) 61 atomic_add_return(int i, atomic_t *v)
121 { 62 {
122 kcsan_mb(); 63 kcsan_mb();
123 instrument_atomic_read_write(v, sizeof 64 instrument_atomic_read_write(v, sizeof(*v));
124 return raw_atomic_add_return(i, v); !! 65 return arch_atomic_add_return(i, v);
125 } 66 }
126 67
127 /** <<
128 * atomic_add_return_acquire() - atomic add wi <<
129 * @i: int value to add <<
130 * @v: pointer to atomic_t <<
131 * <<
132 * Atomically updates @v to (@v + @i) with acq <<
133 * <<
134 * Unsafe to use in noinstr code; use raw_atom <<
135 * <<
136 * Return: The updated value of @v. <<
137 */ <<
138 static __always_inline int 68 static __always_inline int
139 atomic_add_return_acquire(int i, atomic_t *v) 69 atomic_add_return_acquire(int i, atomic_t *v)
140 { 70 {
141 instrument_atomic_read_write(v, sizeof 71 instrument_atomic_read_write(v, sizeof(*v));
142 return raw_atomic_add_return_acquire(i !! 72 return arch_atomic_add_return_acquire(i, v);
143 } 73 }
144 74
145 /** <<
146 * atomic_add_return_release() - atomic add wi <<
147 * @i: int value to add <<
148 * @v: pointer to atomic_t <<
149 * <<
150 * Atomically updates @v to (@v + @i) with rel <<
151 * <<
152 * Unsafe to use in noinstr code; use raw_atom <<
153 * <<
154 * Return: The updated value of @v. <<
155 */ <<
156 static __always_inline int 75 static __always_inline int
157 atomic_add_return_release(int i, atomic_t *v) 76 atomic_add_return_release(int i, atomic_t *v)
158 { 77 {
159 kcsan_release(); 78 kcsan_release();
160 instrument_atomic_read_write(v, sizeof 79 instrument_atomic_read_write(v, sizeof(*v));
161 return raw_atomic_add_return_release(i !! 80 return arch_atomic_add_return_release(i, v);
162 } 81 }
163 82
164 /** <<
165 * atomic_add_return_relaxed() - atomic add wi <<
166 * @i: int value to add <<
167 * @v: pointer to atomic_t <<
168 * <<
169 * Atomically updates @v to (@v + @i) with rel <<
170 * <<
171 * Unsafe to use in noinstr code; use raw_atom <<
172 * <<
173 * Return: The updated value of @v. <<
174 */ <<
175 static __always_inline int 83 static __always_inline int
176 atomic_add_return_relaxed(int i, atomic_t *v) 84 atomic_add_return_relaxed(int i, atomic_t *v)
177 { 85 {
178 instrument_atomic_read_write(v, sizeof 86 instrument_atomic_read_write(v, sizeof(*v));
179 return raw_atomic_add_return_relaxed(i !! 87 return arch_atomic_add_return_relaxed(i, v);
180 } 88 }
181 89
182 /** <<
183 * atomic_fetch_add() - atomic add with full o <<
184 * @i: int value to add <<
185 * @v: pointer to atomic_t <<
186 * <<
187 * Atomically updates @v to (@v + @i) with ful <<
188 * <<
189 * Unsafe to use in noinstr code; use raw_atom <<
190 * <<
191 * Return: The original value of @v. <<
192 */ <<
193 static __always_inline int 90 static __always_inline int
194 atomic_fetch_add(int i, atomic_t *v) 91 atomic_fetch_add(int i, atomic_t *v)
195 { 92 {
196 kcsan_mb(); 93 kcsan_mb();
197 instrument_atomic_read_write(v, sizeof 94 instrument_atomic_read_write(v, sizeof(*v));
198 return raw_atomic_fetch_add(i, v); !! 95 return arch_atomic_fetch_add(i, v);
199 } 96 }
200 97
201 /** <<
202 * atomic_fetch_add_acquire() - atomic add wit <<
203 * @i: int value to add <<
204 * @v: pointer to atomic_t <<
205 * <<
206 * Atomically updates @v to (@v + @i) with acq <<
207 * <<
208 * Unsafe to use in noinstr code; use raw_atom <<
209 * <<
210 * Return: The original value of @v. <<
211 */ <<
212 static __always_inline int 98 static __always_inline int
213 atomic_fetch_add_acquire(int i, atomic_t *v) 99 atomic_fetch_add_acquire(int i, atomic_t *v)
214 { 100 {
215 instrument_atomic_read_write(v, sizeof 101 instrument_atomic_read_write(v, sizeof(*v));
216 return raw_atomic_fetch_add_acquire(i, !! 102 return arch_atomic_fetch_add_acquire(i, v);
217 } 103 }
218 104
219 /** <<
220 * atomic_fetch_add_release() - atomic add wit <<
221 * @i: int value to add <<
222 * @v: pointer to atomic_t <<
223 * <<
224 * Atomically updates @v to (@v + @i) with rel <<
225 * <<
226 * Unsafe to use in noinstr code; use raw_atom <<
227 * <<
228 * Return: The original value of @v. <<
229 */ <<
230 static __always_inline int 105 static __always_inline int
231 atomic_fetch_add_release(int i, atomic_t *v) 106 atomic_fetch_add_release(int i, atomic_t *v)
232 { 107 {
233 kcsan_release(); 108 kcsan_release();
234 instrument_atomic_read_write(v, sizeof 109 instrument_atomic_read_write(v, sizeof(*v));
235 return raw_atomic_fetch_add_release(i, !! 110 return arch_atomic_fetch_add_release(i, v);
236 } 111 }
237 112
238 /** <<
239 * atomic_fetch_add_relaxed() - atomic add wit <<
240 * @i: int value to add <<
241 * @v: pointer to atomic_t <<
242 * <<
243 * Atomically updates @v to (@v + @i) with rel <<
244 * <<
245 * Unsafe to use in noinstr code; use raw_atom <<
246 * <<
247 * Return: The original value of @v. <<
248 */ <<
249 static __always_inline int 113 static __always_inline int
250 atomic_fetch_add_relaxed(int i, atomic_t *v) 114 atomic_fetch_add_relaxed(int i, atomic_t *v)
251 { 115 {
252 instrument_atomic_read_write(v, sizeof 116 instrument_atomic_read_write(v, sizeof(*v));
253 return raw_atomic_fetch_add_relaxed(i, !! 117 return arch_atomic_fetch_add_relaxed(i, v);
254 } 118 }
255 119
256 /** <<
257 * atomic_sub() - atomic subtract with relaxed <<
258 * @i: int value to subtract <<
259 * @v: pointer to atomic_t <<
260 * <<
261 * Atomically updates @v to (@v - @i) with rel <<
262 * <<
263 * Unsafe to use in noinstr code; use raw_atom <<
264 * <<
265 * Return: Nothing. <<
266 */ <<
267 static __always_inline void 120 static __always_inline void
268 atomic_sub(int i, atomic_t *v) 121 atomic_sub(int i, atomic_t *v)
269 { 122 {
270 instrument_atomic_read_write(v, sizeof 123 instrument_atomic_read_write(v, sizeof(*v));
271 raw_atomic_sub(i, v); !! 124 arch_atomic_sub(i, v);
272 } 125 }
273 126
274 /** <<
275 * atomic_sub_return() - atomic subtract with <<
276 * @i: int value to subtract <<
277 * @v: pointer to atomic_t <<
278 * <<
279 * Atomically updates @v to (@v - @i) with ful <<
280 * <<
281 * Unsafe to use in noinstr code; use raw_atom <<
282 * <<
283 * Return: The updated value of @v. <<
284 */ <<
285 static __always_inline int 127 static __always_inline int
286 atomic_sub_return(int i, atomic_t *v) 128 atomic_sub_return(int i, atomic_t *v)
287 { 129 {
288 kcsan_mb(); 130 kcsan_mb();
289 instrument_atomic_read_write(v, sizeof 131 instrument_atomic_read_write(v, sizeof(*v));
290 return raw_atomic_sub_return(i, v); !! 132 return arch_atomic_sub_return(i, v);
291 } 133 }
292 134
293 /** <<
294 * atomic_sub_return_acquire() - atomic subtra <<
295 * @i: int value to subtract <<
296 * @v: pointer to atomic_t <<
297 * <<
298 * Atomically updates @v to (@v - @i) with acq <<
299 * <<
300 * Unsafe to use in noinstr code; use raw_atom <<
301 * <<
302 * Return: The updated value of @v. <<
303 */ <<
304 static __always_inline int 135 static __always_inline int
305 atomic_sub_return_acquire(int i, atomic_t *v) 136 atomic_sub_return_acquire(int i, atomic_t *v)
306 { 137 {
307 instrument_atomic_read_write(v, sizeof 138 instrument_atomic_read_write(v, sizeof(*v));
308 return raw_atomic_sub_return_acquire(i !! 139 return arch_atomic_sub_return_acquire(i, v);
309 } 140 }
310 141
311 /** <<
312 * atomic_sub_return_release() - atomic subtra <<
313 * @i: int value to subtract <<
314 * @v: pointer to atomic_t <<
315 * <<
316 * Atomically updates @v to (@v - @i) with rel <<
317 * <<
318 * Unsafe to use in noinstr code; use raw_atom <<
319 * <<
320 * Return: The updated value of @v. <<
321 */ <<
322 static __always_inline int 142 static __always_inline int
323 atomic_sub_return_release(int i, atomic_t *v) 143 atomic_sub_return_release(int i, atomic_t *v)
324 { 144 {
325 kcsan_release(); 145 kcsan_release();
326 instrument_atomic_read_write(v, sizeof 146 instrument_atomic_read_write(v, sizeof(*v));
327 return raw_atomic_sub_return_release(i !! 147 return arch_atomic_sub_return_release(i, v);
328 } 148 }
329 149
330 /** <<
331 * atomic_sub_return_relaxed() - atomic subtra <<
332 * @i: int value to subtract <<
333 * @v: pointer to atomic_t <<
334 * <<
335 * Atomically updates @v to (@v - @i) with rel <<
336 * <<
337 * Unsafe to use in noinstr code; use raw_atom <<
338 * <<
339 * Return: The updated value of @v. <<
340 */ <<
341 static __always_inline int 150 static __always_inline int
342 atomic_sub_return_relaxed(int i, atomic_t *v) 151 atomic_sub_return_relaxed(int i, atomic_t *v)
343 { 152 {
344 instrument_atomic_read_write(v, sizeof 153 instrument_atomic_read_write(v, sizeof(*v));
345 return raw_atomic_sub_return_relaxed(i !! 154 return arch_atomic_sub_return_relaxed(i, v);
346 } 155 }
347 156
348 /** <<
349 * atomic_fetch_sub() - atomic subtract with f <<
350 * @i: int value to subtract <<
351 * @v: pointer to atomic_t <<
352 * <<
353 * Atomically updates @v to (@v - @i) with ful <<
354 * <<
355 * Unsafe to use in noinstr code; use raw_atom <<
356 * <<
357 * Return: The original value of @v. <<
358 */ <<
359 static __always_inline int 157 static __always_inline int
360 atomic_fetch_sub(int i, atomic_t *v) 158 atomic_fetch_sub(int i, atomic_t *v)
361 { 159 {
362 kcsan_mb(); 160 kcsan_mb();
363 instrument_atomic_read_write(v, sizeof 161 instrument_atomic_read_write(v, sizeof(*v));
364 return raw_atomic_fetch_sub(i, v); !! 162 return arch_atomic_fetch_sub(i, v);
365 } 163 }
366 164
367 /** <<
368 * atomic_fetch_sub_acquire() - atomic subtrac <<
369 * @i: int value to subtract <<
370 * @v: pointer to atomic_t <<
371 * <<
372 * Atomically updates @v to (@v - @i) with acq <<
373 * <<
374 * Unsafe to use in noinstr code; use raw_atom <<
375 * <<
376 * Return: The original value of @v. <<
377 */ <<
378 static __always_inline int 165 static __always_inline int
379 atomic_fetch_sub_acquire(int i, atomic_t *v) 166 atomic_fetch_sub_acquire(int i, atomic_t *v)
380 { 167 {
381 instrument_atomic_read_write(v, sizeof 168 instrument_atomic_read_write(v, sizeof(*v));
382 return raw_atomic_fetch_sub_acquire(i, !! 169 return arch_atomic_fetch_sub_acquire(i, v);
383 } 170 }
384 171
385 /** <<
386 * atomic_fetch_sub_release() - atomic subtrac <<
387 * @i: int value to subtract <<
388 * @v: pointer to atomic_t <<
389 * <<
390 * Atomically updates @v to (@v - @i) with rel <<
391 * <<
392 * Unsafe to use in noinstr code; use raw_atom <<
393 * <<
394 * Return: The original value of @v. <<
395 */ <<
396 static __always_inline int 172 static __always_inline int
397 atomic_fetch_sub_release(int i, atomic_t *v) 173 atomic_fetch_sub_release(int i, atomic_t *v)
398 { 174 {
399 kcsan_release(); 175 kcsan_release();
400 instrument_atomic_read_write(v, sizeof 176 instrument_atomic_read_write(v, sizeof(*v));
401 return raw_atomic_fetch_sub_release(i, !! 177 return arch_atomic_fetch_sub_release(i, v);
402 } 178 }
403 179
404 /** <<
405 * atomic_fetch_sub_relaxed() - atomic subtrac <<
406 * @i: int value to subtract <<
407 * @v: pointer to atomic_t <<
408 * <<
409 * Atomically updates @v to (@v - @i) with rel <<
410 * <<
411 * Unsafe to use in noinstr code; use raw_atom <<
412 * <<
413 * Return: The original value of @v. <<
414 */ <<
415 static __always_inline int 180 static __always_inline int
416 atomic_fetch_sub_relaxed(int i, atomic_t *v) 181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
417 { 182 {
418 instrument_atomic_read_write(v, sizeof 183 instrument_atomic_read_write(v, sizeof(*v));
419 return raw_atomic_fetch_sub_relaxed(i, !! 184 return arch_atomic_fetch_sub_relaxed(i, v);
420 } 185 }
421 186
422 /** <<
423 * atomic_inc() - atomic increment with relaxe <<
424 * @v: pointer to atomic_t <<
425 * <<
426 * Atomically updates @v to (@v + 1) with rela <<
427 * <<
428 * Unsafe to use in noinstr code; use raw_atom <<
429 * <<
430 * Return: Nothing. <<
431 */ <<
432 static __always_inline void 187 static __always_inline void
433 atomic_inc(atomic_t *v) 188 atomic_inc(atomic_t *v)
434 { 189 {
435 instrument_atomic_read_write(v, sizeof 190 instrument_atomic_read_write(v, sizeof(*v));
436 raw_atomic_inc(v); !! 191 arch_atomic_inc(v);
437 } 192 }
438 193
439 /** <<
440 * atomic_inc_return() - atomic increment with <<
441 * @v: pointer to atomic_t <<
442 * <<
443 * Atomically updates @v to (@v + 1) with full <<
444 * <<
445 * Unsafe to use in noinstr code; use raw_atom <<
446 * <<
447 * Return: The updated value of @v. <<
448 */ <<
449 static __always_inline int 194 static __always_inline int
450 atomic_inc_return(atomic_t *v) 195 atomic_inc_return(atomic_t *v)
451 { 196 {
452 kcsan_mb(); 197 kcsan_mb();
453 instrument_atomic_read_write(v, sizeof 198 instrument_atomic_read_write(v, sizeof(*v));
454 return raw_atomic_inc_return(v); !! 199 return arch_atomic_inc_return(v);
455 } 200 }
456 201
457 /** <<
458 * atomic_inc_return_acquire() - atomic increm <<
459 * @v: pointer to atomic_t <<
460 * <<
461 * Atomically updates @v to (@v + 1) with acqu <<
462 * <<
463 * Unsafe to use in noinstr code; use raw_atom <<
464 * <<
465 * Return: The updated value of @v. <<
466 */ <<
467 static __always_inline int 202 static __always_inline int
468 atomic_inc_return_acquire(atomic_t *v) 203 atomic_inc_return_acquire(atomic_t *v)
469 { 204 {
470 instrument_atomic_read_write(v, sizeof 205 instrument_atomic_read_write(v, sizeof(*v));
471 return raw_atomic_inc_return_acquire(v !! 206 return arch_atomic_inc_return_acquire(v);
472 } 207 }
473 208
474 /** <<
475 * atomic_inc_return_release() - atomic increm <<
476 * @v: pointer to atomic_t <<
477 * <<
478 * Atomically updates @v to (@v + 1) with rele <<
479 * <<
480 * Unsafe to use in noinstr code; use raw_atom <<
481 * <<
482 * Return: The updated value of @v. <<
483 */ <<
484 static __always_inline int 209 static __always_inline int
485 atomic_inc_return_release(atomic_t *v) 210 atomic_inc_return_release(atomic_t *v)
486 { 211 {
487 kcsan_release(); 212 kcsan_release();
488 instrument_atomic_read_write(v, sizeof 213 instrument_atomic_read_write(v, sizeof(*v));
489 return raw_atomic_inc_return_release(v !! 214 return arch_atomic_inc_return_release(v);
490 } 215 }
491 216
492 /** <<
493 * atomic_inc_return_relaxed() - atomic increm <<
494 * @v: pointer to atomic_t <<
495 * <<
496 * Atomically updates @v to (@v + 1) with rela <<
497 * <<
498 * Unsafe to use in noinstr code; use raw_atom <<
499 * <<
500 * Return: The updated value of @v. <<
501 */ <<
502 static __always_inline int 217 static __always_inline int
503 atomic_inc_return_relaxed(atomic_t *v) 218 atomic_inc_return_relaxed(atomic_t *v)
504 { 219 {
505 instrument_atomic_read_write(v, sizeof 220 instrument_atomic_read_write(v, sizeof(*v));
506 return raw_atomic_inc_return_relaxed(v !! 221 return arch_atomic_inc_return_relaxed(v);
507 } 222 }
508 223
509 /** <<
510 * atomic_fetch_inc() - atomic increment with <<
511 * @v: pointer to atomic_t <<
512 * <<
513 * Atomically updates @v to (@v + 1) with full <<
514 * <<
515 * Unsafe to use in noinstr code; use raw_atom <<
516 * <<
517 * Return: The original value of @v. <<
518 */ <<
519 static __always_inline int 224 static __always_inline int
520 atomic_fetch_inc(atomic_t *v) 225 atomic_fetch_inc(atomic_t *v)
521 { 226 {
522 kcsan_mb(); 227 kcsan_mb();
523 instrument_atomic_read_write(v, sizeof 228 instrument_atomic_read_write(v, sizeof(*v));
524 return raw_atomic_fetch_inc(v); !! 229 return arch_atomic_fetch_inc(v);
525 } 230 }
526 231
527 /** <<
528 * atomic_fetch_inc_acquire() - atomic increme <<
529 * @v: pointer to atomic_t <<
530 * <<
531 * Atomically updates @v to (@v + 1) with acqu <<
532 * <<
533 * Unsafe to use in noinstr code; use raw_atom <<
534 * <<
535 * Return: The original value of @v. <<
536 */ <<
537 static __always_inline int 232 static __always_inline int
538 atomic_fetch_inc_acquire(atomic_t *v) 233 atomic_fetch_inc_acquire(atomic_t *v)
539 { 234 {
540 instrument_atomic_read_write(v, sizeof 235 instrument_atomic_read_write(v, sizeof(*v));
541 return raw_atomic_fetch_inc_acquire(v) !! 236 return arch_atomic_fetch_inc_acquire(v);
542 } 237 }
543 238
544 /** <<
545 * atomic_fetch_inc_release() - atomic increme <<
546 * @v: pointer to atomic_t <<
547 * <<
548 * Atomically updates @v to (@v + 1) with rele <<
549 * <<
550 * Unsafe to use in noinstr code; use raw_atom <<
551 * <<
552 * Return: The original value of @v. <<
553 */ <<
554 static __always_inline int 239 static __always_inline int
555 atomic_fetch_inc_release(atomic_t *v) 240 atomic_fetch_inc_release(atomic_t *v)
556 { 241 {
557 kcsan_release(); 242 kcsan_release();
558 instrument_atomic_read_write(v, sizeof 243 instrument_atomic_read_write(v, sizeof(*v));
559 return raw_atomic_fetch_inc_release(v) !! 244 return arch_atomic_fetch_inc_release(v);
560 } 245 }
561 246
562 /** <<
563 * atomic_fetch_inc_relaxed() - atomic increme <<
564 * @v: pointer to atomic_t <<
565 * <<
566 * Atomically updates @v to (@v + 1) with rela <<
567 * <<
568 * Unsafe to use in noinstr code; use raw_atom <<
569 * <<
570 * Return: The original value of @v. <<
571 */ <<
572 static __always_inline int 247 static __always_inline int
573 atomic_fetch_inc_relaxed(atomic_t *v) 248 atomic_fetch_inc_relaxed(atomic_t *v)
574 { 249 {
575 instrument_atomic_read_write(v, sizeof 250 instrument_atomic_read_write(v, sizeof(*v));
576 return raw_atomic_fetch_inc_relaxed(v) !! 251 return arch_atomic_fetch_inc_relaxed(v);
577 } 252 }
578 253
579 /** <<
580 * atomic_dec() - atomic decrement with relaxe <<
581 * @v: pointer to atomic_t <<
582 * <<
583 * Atomically updates @v to (@v - 1) with rela <<
584 * <<
585 * Unsafe to use in noinstr code; use raw_atom <<
586 * <<
587 * Return: Nothing. <<
588 */ <<
589 static __always_inline void 254 static __always_inline void
590 atomic_dec(atomic_t *v) 255 atomic_dec(atomic_t *v)
591 { 256 {
592 instrument_atomic_read_write(v, sizeof 257 instrument_atomic_read_write(v, sizeof(*v));
593 raw_atomic_dec(v); !! 258 arch_atomic_dec(v);
594 } 259 }
595 260
596 /** <<
597 * atomic_dec_return() - atomic decrement with <<
598 * @v: pointer to atomic_t <<
599 * <<
600 * Atomically updates @v to (@v - 1) with full <<
601 * <<
602 * Unsafe to use in noinstr code; use raw_atom <<
603 * <<
604 * Return: The updated value of @v. <<
605 */ <<
606 static __always_inline int 261 static __always_inline int
607 atomic_dec_return(atomic_t *v) 262 atomic_dec_return(atomic_t *v)
608 { 263 {
609 kcsan_mb(); 264 kcsan_mb();
610 instrument_atomic_read_write(v, sizeof 265 instrument_atomic_read_write(v, sizeof(*v));
611 return raw_atomic_dec_return(v); !! 266 return arch_atomic_dec_return(v);
612 } 267 }
613 268
614 /** <<
615 * atomic_dec_return_acquire() - atomic decrem <<
616 * @v: pointer to atomic_t <<
617 * <<
618 * Atomically updates @v to (@v - 1) with acqu <<
619 * <<
620 * Unsafe to use in noinstr code; use raw_atom <<
621 * <<
622 * Return: The updated value of @v. <<
623 */ <<
624 static __always_inline int 269 static __always_inline int
625 atomic_dec_return_acquire(atomic_t *v) 270 atomic_dec_return_acquire(atomic_t *v)
626 { 271 {
627 instrument_atomic_read_write(v, sizeof 272 instrument_atomic_read_write(v, sizeof(*v));
628 return raw_atomic_dec_return_acquire(v !! 273 return arch_atomic_dec_return_acquire(v);
629 } 274 }
630 275
631 /** <<
632 * atomic_dec_return_release() - atomic decrem <<
633 * @v: pointer to atomic_t <<
634 * <<
635 * Atomically updates @v to (@v - 1) with rele <<
636 * <<
637 * Unsafe to use in noinstr code; use raw_atom <<
638 * <<
639 * Return: The updated value of @v. <<
640 */ <<
641 static __always_inline int 276 static __always_inline int
642 atomic_dec_return_release(atomic_t *v) 277 atomic_dec_return_release(atomic_t *v)
643 { 278 {
644 kcsan_release(); 279 kcsan_release();
645 instrument_atomic_read_write(v, sizeof 280 instrument_atomic_read_write(v, sizeof(*v));
646 return raw_atomic_dec_return_release(v !! 281 return arch_atomic_dec_return_release(v);
647 } 282 }
648 283
649 /** <<
650 * atomic_dec_return_relaxed() - atomic decrem <<
651 * @v: pointer to atomic_t <<
652 * <<
653 * Atomically updates @v to (@v - 1) with rela <<
654 * <<
655 * Unsafe to use in noinstr code; use raw_atom <<
656 * <<
657 * Return: The updated value of @v. <<
658 */ <<
659 static __always_inline int 284 static __always_inline int
660 atomic_dec_return_relaxed(atomic_t *v) 285 atomic_dec_return_relaxed(atomic_t *v)
661 { 286 {
662 instrument_atomic_read_write(v, sizeof 287 instrument_atomic_read_write(v, sizeof(*v));
663 return raw_atomic_dec_return_relaxed(v !! 288 return arch_atomic_dec_return_relaxed(v);
664 } 289 }
665 290
666 /** <<
667 * atomic_fetch_dec() - atomic decrement with <<
668 * @v: pointer to atomic_t <<
669 * <<
670 * Atomically updates @v to (@v - 1) with full <<
671 * <<
672 * Unsafe to use in noinstr code; use raw_atom <<
673 * <<
674 * Return: The original value of @v. <<
675 */ <<
676 static __always_inline int 291 static __always_inline int
677 atomic_fetch_dec(atomic_t *v) 292 atomic_fetch_dec(atomic_t *v)
678 { 293 {
679 kcsan_mb(); 294 kcsan_mb();
680 instrument_atomic_read_write(v, sizeof 295 instrument_atomic_read_write(v, sizeof(*v));
681 return raw_atomic_fetch_dec(v); !! 296 return arch_atomic_fetch_dec(v);
682 } 297 }
683 298
684 /** <<
685 * atomic_fetch_dec_acquire() - atomic decreme <<
686 * @v: pointer to atomic_t <<
687 * <<
688 * Atomically updates @v to (@v - 1) with acqu <<
689 * <<
690 * Unsafe to use in noinstr code; use raw_atom <<
691 * <<
692 * Return: The original value of @v. <<
693 */ <<
694 static __always_inline int 299 static __always_inline int
695 atomic_fetch_dec_acquire(atomic_t *v) 300 atomic_fetch_dec_acquire(atomic_t *v)
696 { 301 {
697 instrument_atomic_read_write(v, sizeof 302 instrument_atomic_read_write(v, sizeof(*v));
698 return raw_atomic_fetch_dec_acquire(v) !! 303 return arch_atomic_fetch_dec_acquire(v);
699 } 304 }
700 305
701 /** <<
702 * atomic_fetch_dec_release() - atomic decreme <<
703 * @v: pointer to atomic_t <<
704 * <<
705 * Atomically updates @v to (@v - 1) with rele <<
706 * <<
707 * Unsafe to use in noinstr code; use raw_atom <<
708 * <<
709 * Return: The original value of @v. <<
710 */ <<
711 static __always_inline int 306 static __always_inline int
712 atomic_fetch_dec_release(atomic_t *v) 307 atomic_fetch_dec_release(atomic_t *v)
713 { 308 {
714 kcsan_release(); 309 kcsan_release();
715 instrument_atomic_read_write(v, sizeof 310 instrument_atomic_read_write(v, sizeof(*v));
716 return raw_atomic_fetch_dec_release(v) !! 311 return arch_atomic_fetch_dec_release(v);
717 } 312 }
718 313
719 /** <<
720 * atomic_fetch_dec_relaxed() - atomic decreme <<
721 * @v: pointer to atomic_t <<
722 * <<
723 * Atomically updates @v to (@v - 1) with rela <<
724 * <<
725 * Unsafe to use in noinstr code; use raw_atom <<
726 * <<
727 * Return: The original value of @v. <<
728 */ <<
729 static __always_inline int 314 static __always_inline int
730 atomic_fetch_dec_relaxed(atomic_t *v) 315 atomic_fetch_dec_relaxed(atomic_t *v)
731 { 316 {
732 instrument_atomic_read_write(v, sizeof 317 instrument_atomic_read_write(v, sizeof(*v));
733 return raw_atomic_fetch_dec_relaxed(v) !! 318 return arch_atomic_fetch_dec_relaxed(v);
734 } 319 }
735 320
736 /** <<
737 * atomic_and() - atomic bitwise AND with rela <<
738 * @i: int value <<
739 * @v: pointer to atomic_t <<
740 * <<
741 * Atomically updates @v to (@v & @i) with rel <<
742 * <<
743 * Unsafe to use in noinstr code; use raw_atom <<
744 * <<
745 * Return: Nothing. <<
746 */ <<
747 static __always_inline void 321 static __always_inline void
748 atomic_and(int i, atomic_t *v) 322 atomic_and(int i, atomic_t *v)
749 { 323 {
750 instrument_atomic_read_write(v, sizeof 324 instrument_atomic_read_write(v, sizeof(*v));
751 raw_atomic_and(i, v); !! 325 arch_atomic_and(i, v);
752 } 326 }
753 327
754 /** <<
755 * atomic_fetch_and() - atomic bitwise AND wit <<
756 * @i: int value <<
757 * @v: pointer to atomic_t <<
758 * <<
759 * Atomically updates @v to (@v & @i) with ful <<
760 * <<
761 * Unsafe to use in noinstr code; use raw_atom <<
762 * <<
763 * Return: The original value of @v. <<
764 */ <<
765 static __always_inline int 328 static __always_inline int
766 atomic_fetch_and(int i, atomic_t *v) 329 atomic_fetch_and(int i, atomic_t *v)
767 { 330 {
768 kcsan_mb(); 331 kcsan_mb();
769 instrument_atomic_read_write(v, sizeof 332 instrument_atomic_read_write(v, sizeof(*v));
770 return raw_atomic_fetch_and(i, v); !! 333 return arch_atomic_fetch_and(i, v);
771 } 334 }
772 335
773 /** <<
774 * atomic_fetch_and_acquire() - atomic bitwise <<
775 * @i: int value <<
776 * @v: pointer to atomic_t <<
777 * <<
778 * Atomically updates @v to (@v & @i) with acq <<
779 * <<
780 * Unsafe to use in noinstr code; use raw_atom <<
781 * <<
782 * Return: The original value of @v. <<
783 */ <<
784 static __always_inline int 336 static __always_inline int
785 atomic_fetch_and_acquire(int i, atomic_t *v) 337 atomic_fetch_and_acquire(int i, atomic_t *v)
786 { 338 {
787 instrument_atomic_read_write(v, sizeof 339 instrument_atomic_read_write(v, sizeof(*v));
788 return raw_atomic_fetch_and_acquire(i, !! 340 return arch_atomic_fetch_and_acquire(i, v);
789 } 341 }
790 342
791 /** <<
792 * atomic_fetch_and_release() - atomic bitwise <<
793 * @i: int value <<
794 * @v: pointer to atomic_t <<
795 * <<
796 * Atomically updates @v to (@v & @i) with rel <<
797 * <<
798 * Unsafe to use in noinstr code; use raw_atom <<
799 * <<
800 * Return: The original value of @v. <<
801 */ <<
802 static __always_inline int 343 static __always_inline int
803 atomic_fetch_and_release(int i, atomic_t *v) 344 atomic_fetch_and_release(int i, atomic_t *v)
804 { 345 {
805 kcsan_release(); 346 kcsan_release();
806 instrument_atomic_read_write(v, sizeof 347 instrument_atomic_read_write(v, sizeof(*v));
807 return raw_atomic_fetch_and_release(i, !! 348 return arch_atomic_fetch_and_release(i, v);
808 } 349 }
809 350
810 /** <<
811 * atomic_fetch_and_relaxed() - atomic bitwise <<
812 * @i: int value <<
813 * @v: pointer to atomic_t <<
814 * <<
815 * Atomically updates @v to (@v & @i) with rel <<
816 * <<
817 * Unsafe to use in noinstr code; use raw_atom <<
818 * <<
819 * Return: The original value of @v. <<
820 */ <<
821 static __always_inline int 351 static __always_inline int
822 atomic_fetch_and_relaxed(int i, atomic_t *v) 352 atomic_fetch_and_relaxed(int i, atomic_t *v)
823 { 353 {
824 instrument_atomic_read_write(v, sizeof 354 instrument_atomic_read_write(v, sizeof(*v));
825 return raw_atomic_fetch_and_relaxed(i, !! 355 return arch_atomic_fetch_and_relaxed(i, v);
826 } 356 }
827 357
828 /** <<
829 * atomic_andnot() - atomic bitwise AND NOT wi <<
830 * @i: int value <<
831 * @v: pointer to atomic_t <<
832 * <<
833 * Atomically updates @v to (@v & ~@i) with re <<
834 * <<
835 * Unsafe to use in noinstr code; use raw_atom <<
836 * <<
837 * Return: Nothing. <<
838 */ <<
839 static __always_inline void 358 static __always_inline void
840 atomic_andnot(int i, atomic_t *v) 359 atomic_andnot(int i, atomic_t *v)
841 { 360 {
842 instrument_atomic_read_write(v, sizeof 361 instrument_atomic_read_write(v, sizeof(*v));
843 raw_atomic_andnot(i, v); !! 362 arch_atomic_andnot(i, v);
844 } 363 }
845 364
846 /** <<
847 * atomic_fetch_andnot() - atomic bitwise AND <<
848 * @i: int value <<
849 * @v: pointer to atomic_t <<
850 * <<
851 * Atomically updates @v to (@v & ~@i) with fu <<
852 * <<
853 * Unsafe to use in noinstr code; use raw_atom <<
854 * <<
855 * Return: The original value of @v. <<
856 */ <<
857 static __always_inline int 365 static __always_inline int
858 atomic_fetch_andnot(int i, atomic_t *v) 366 atomic_fetch_andnot(int i, atomic_t *v)
859 { 367 {
860 kcsan_mb(); 368 kcsan_mb();
861 instrument_atomic_read_write(v, sizeof 369 instrument_atomic_read_write(v, sizeof(*v));
862 return raw_atomic_fetch_andnot(i, v); !! 370 return arch_atomic_fetch_andnot(i, v);
863 } 371 }
864 372
865 /** <<
866 * atomic_fetch_andnot_acquire() - atomic bitw <<
867 * @i: int value <<
868 * @v: pointer to atomic_t <<
869 * <<
870 * Atomically updates @v to (@v & ~@i) with ac <<
871 * <<
872 * Unsafe to use in noinstr code; use raw_atom <<
873 * <<
874 * Return: The original value of @v. <<
875 */ <<
876 static __always_inline int 373 static __always_inline int
877 atomic_fetch_andnot_acquire(int i, atomic_t *v 374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
878 { 375 {
879 instrument_atomic_read_write(v, sizeof 376 instrument_atomic_read_write(v, sizeof(*v));
880 return raw_atomic_fetch_andnot_acquire !! 377 return arch_atomic_fetch_andnot_acquire(i, v);
881 } 378 }
882 379
883 /** <<
884 * atomic_fetch_andnot_release() - atomic bitw <<
885 * @i: int value <<
886 * @v: pointer to atomic_t <<
887 * <<
888 * Atomically updates @v to (@v & ~@i) with re <<
889 * <<
890 * Unsafe to use in noinstr code; use raw_atom <<
891 * <<
892 * Return: The original value of @v. <<
893 */ <<
894 static __always_inline int 380 static __always_inline int
895 atomic_fetch_andnot_release(int i, atomic_t *v 381 atomic_fetch_andnot_release(int i, atomic_t *v)
896 { 382 {
897 kcsan_release(); 383 kcsan_release();
898 instrument_atomic_read_write(v, sizeof 384 instrument_atomic_read_write(v, sizeof(*v));
899 return raw_atomic_fetch_andnot_release !! 385 return arch_atomic_fetch_andnot_release(i, v);
900 } 386 }
901 387
902 /** <<
903 * atomic_fetch_andnot_relaxed() - atomic bitw <<
904 * @i: int value <<
905 * @v: pointer to atomic_t <<
906 * <<
907 * Atomically updates @v to (@v & ~@i) with re <<
908 * <<
909 * Unsafe to use in noinstr code; use raw_atom <<
910 * <<
911 * Return: The original value of @v. <<
912 */ <<
913 static __always_inline int 388 static __always_inline int
914 atomic_fetch_andnot_relaxed(int i, atomic_t *v 389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
915 { 390 {
916 instrument_atomic_read_write(v, sizeof 391 instrument_atomic_read_write(v, sizeof(*v));
917 return raw_atomic_fetch_andnot_relaxed !! 392 return arch_atomic_fetch_andnot_relaxed(i, v);
918 } 393 }
919 394
920 /** <<
921 * atomic_or() - atomic bitwise OR with relaxe <<
922 * @i: int value <<
923 * @v: pointer to atomic_t <<
924 * <<
925 * Atomically updates @v to (@v | @i) with rel <<
926 * <<
927 * Unsafe to use in noinstr code; use raw_atom <<
928 * <<
929 * Return: Nothing. <<
930 */ <<
931 static __always_inline void 395 static __always_inline void
932 atomic_or(int i, atomic_t *v) 396 atomic_or(int i, atomic_t *v)
933 { 397 {
934 instrument_atomic_read_write(v, sizeof 398 instrument_atomic_read_write(v, sizeof(*v));
935 raw_atomic_or(i, v); !! 399 arch_atomic_or(i, v);
936 } 400 }
937 401
938 /** <<
939 * atomic_fetch_or() - atomic bitwise OR with <<
940 * @i: int value <<
941 * @v: pointer to atomic_t <<
942 * <<
943 * Atomically updates @v to (@v | @i) with ful <<
944 * <<
945 * Unsafe to use in noinstr code; use raw_atom <<
946 * <<
947 * Return: The original value of @v. <<
948 */ <<
949 static __always_inline int 402 static __always_inline int
950 atomic_fetch_or(int i, atomic_t *v) 403 atomic_fetch_or(int i, atomic_t *v)
951 { 404 {
952 kcsan_mb(); 405 kcsan_mb();
953 instrument_atomic_read_write(v, sizeof 406 instrument_atomic_read_write(v, sizeof(*v));
954 return raw_atomic_fetch_or(i, v); !! 407 return arch_atomic_fetch_or(i, v);
955 } 408 }
956 409
957 /** <<
958 * atomic_fetch_or_acquire() - atomic bitwise <<
959 * @i: int value <<
960 * @v: pointer to atomic_t <<
961 * <<
962 * Atomically updates @v to (@v | @i) with acq <<
963 * <<
964 * Unsafe to use in noinstr code; use raw_atom <<
965 * <<
966 * Return: The original value of @v. <<
967 */ <<
968 static __always_inline int 410 static __always_inline int
969 atomic_fetch_or_acquire(int i, atomic_t *v) 411 atomic_fetch_or_acquire(int i, atomic_t *v)
970 { 412 {
971 instrument_atomic_read_write(v, sizeof 413 instrument_atomic_read_write(v, sizeof(*v));
972 return raw_atomic_fetch_or_acquire(i, !! 414 return arch_atomic_fetch_or_acquire(i, v);
973 } 415 }
974 416
975 /** <<
976 * atomic_fetch_or_release() - atomic bitwise <<
977 * @i: int value <<
978 * @v: pointer to atomic_t <<
979 * <<
980 * Atomically updates @v to (@v | @i) with rel <<
981 * <<
982 * Unsafe to use in noinstr code; use raw_atom <<
983 * <<
984 * Return: The original value of @v. <<
985 */ <<
986 static __always_inline int 417 static __always_inline int
987 atomic_fetch_or_release(int i, atomic_t *v) 418 atomic_fetch_or_release(int i, atomic_t *v)
988 { 419 {
989 kcsan_release(); 420 kcsan_release();
990 instrument_atomic_read_write(v, sizeof 421 instrument_atomic_read_write(v, sizeof(*v));
991 return raw_atomic_fetch_or_release(i, !! 422 return arch_atomic_fetch_or_release(i, v);
992 } 423 }
993 424
994 /** <<
995 * atomic_fetch_or_relaxed() - atomic bitwise <<
996 * @i: int value <<
997 * @v: pointer to atomic_t <<
998 * <<
999 * Atomically updates @v to (@v | @i) with rel <<
1000 * <<
1001 * Unsafe to use in noinstr code; use raw_ato <<
1002 * <<
1003 * Return: The original value of @v. <<
1004 */ <<
1005 static __always_inline int 425 static __always_inline int
1006 atomic_fetch_or_relaxed(int i, atomic_t *v) 426 atomic_fetch_or_relaxed(int i, atomic_t *v)
1007 { 427 {
1008 instrument_atomic_read_write(v, sizeo 428 instrument_atomic_read_write(v, sizeof(*v));
1009 return raw_atomic_fetch_or_relaxed(i, !! 429 return arch_atomic_fetch_or_relaxed(i, v);
1010 } 430 }
1011 431
1012 /** <<
1013 * atomic_xor() - atomic bitwise XOR with rel <<
1014 * @i: int value <<
1015 * @v: pointer to atomic_t <<
1016 * <<
1017 * Atomically updates @v to (@v ^ @i) with re <<
1018 * <<
1019 * Unsafe to use in noinstr code; use raw_ato <<
1020 * <<
1021 * Return: Nothing. <<
1022 */ <<
1023 static __always_inline void 432 static __always_inline void
1024 atomic_xor(int i, atomic_t *v) 433 atomic_xor(int i, atomic_t *v)
1025 { 434 {
1026 instrument_atomic_read_write(v, sizeo 435 instrument_atomic_read_write(v, sizeof(*v));
1027 raw_atomic_xor(i, v); !! 436 arch_atomic_xor(i, v);
1028 } 437 }
1029 438
1030 /** <<
1031 * atomic_fetch_xor() - atomic bitwise XOR wi <<
1032 * @i: int value <<
1033 * @v: pointer to atomic_t <<
1034 * <<
1035 * Atomically updates @v to (@v ^ @i) with fu <<
1036 * <<
1037 * Unsafe to use in noinstr code; use raw_ato <<
1038 * <<
1039 * Return: The original value of @v. <<
1040 */ <<
1041 static __always_inline int 439 static __always_inline int
1042 atomic_fetch_xor(int i, atomic_t *v) 440 atomic_fetch_xor(int i, atomic_t *v)
1043 { 441 {
1044 kcsan_mb(); 442 kcsan_mb();
1045 instrument_atomic_read_write(v, sizeo 443 instrument_atomic_read_write(v, sizeof(*v));
1046 return raw_atomic_fetch_xor(i, v); !! 444 return arch_atomic_fetch_xor(i, v);
1047 } 445 }
1048 446
1049 /** <<
1050 * atomic_fetch_xor_acquire() - atomic bitwis <<
1051 * @i: int value <<
1052 * @v: pointer to atomic_t <<
1053 * <<
1054 * Atomically updates @v to (@v ^ @i) with ac <<
1055 * <<
1056 * Unsafe to use in noinstr code; use raw_ato <<
1057 * <<
1058 * Return: The original value of @v. <<
1059 */ <<
1060 static __always_inline int 447 static __always_inline int
1061 atomic_fetch_xor_acquire(int i, atomic_t *v) 448 atomic_fetch_xor_acquire(int i, atomic_t *v)
1062 { 449 {
1063 instrument_atomic_read_write(v, sizeo 450 instrument_atomic_read_write(v, sizeof(*v));
1064 return raw_atomic_fetch_xor_acquire(i !! 451 return arch_atomic_fetch_xor_acquire(i, v);
1065 } 452 }
1066 453
1067 /** <<
1068 * atomic_fetch_xor_release() - atomic bitwis <<
1069 * @i: int value <<
1070 * @v: pointer to atomic_t <<
1071 * <<
1072 * Atomically updates @v to (@v ^ @i) with re <<
1073 * <<
1074 * Unsafe to use in noinstr code; use raw_ato <<
1075 * <<
1076 * Return: The original value of @v. <<
1077 */ <<
1078 static __always_inline int 454 static __always_inline int
1079 atomic_fetch_xor_release(int i, atomic_t *v) 455 atomic_fetch_xor_release(int i, atomic_t *v)
1080 { 456 {
1081 kcsan_release(); 457 kcsan_release();
1082 instrument_atomic_read_write(v, sizeo 458 instrument_atomic_read_write(v, sizeof(*v));
1083 return raw_atomic_fetch_xor_release(i !! 459 return arch_atomic_fetch_xor_release(i, v);
1084 } 460 }
1085 461
1086 /** <<
1087 * atomic_fetch_xor_relaxed() - atomic bitwis <<
1088 * @i: int value <<
1089 * @v: pointer to atomic_t <<
1090 * <<
1091 * Atomically updates @v to (@v ^ @i) with re <<
1092 * <<
1093 * Unsafe to use in noinstr code; use raw_ato <<
1094 * <<
1095 * Return: The original value of @v. <<
1096 */ <<
1097 static __always_inline int 462 static __always_inline int
1098 atomic_fetch_xor_relaxed(int i, atomic_t *v) 463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
1099 { 464 {
1100 instrument_atomic_read_write(v, sizeo 465 instrument_atomic_read_write(v, sizeof(*v));
1101 return raw_atomic_fetch_xor_relaxed(i !! 466 return arch_atomic_fetch_xor_relaxed(i, v);
1102 } 467 }
1103 468
1104 /** <<
1105 * atomic_xchg() - atomic exchange with full <<
1106 * @v: pointer to atomic_t <<
1107 * @new: int value to assign <<
1108 * <<
1109 * Atomically updates @v to @new with full or <<
1110 * <<
1111 * Unsafe to use in noinstr code; use raw_ato <<
1112 * <<
1113 * Return: The original value of @v. <<
1114 */ <<
1115 static __always_inline int 469 static __always_inline int
1116 atomic_xchg(atomic_t *v, int new) !! 470 atomic_xchg(atomic_t *v, int i)
1117 { 471 {
1118 kcsan_mb(); 472 kcsan_mb();
1119 instrument_atomic_read_write(v, sizeo 473 instrument_atomic_read_write(v, sizeof(*v));
1120 return raw_atomic_xchg(v, new); !! 474 return arch_atomic_xchg(v, i);
1121 } 475 }
1122 476
1123 /** <<
1124 * atomic_xchg_acquire() - atomic exchange wi <<
1125 * @v: pointer to atomic_t <<
1126 * @new: int value to assign <<
1127 * <<
1128 * Atomically updates @v to @new with acquire <<
1129 * <<
1130 * Unsafe to use in noinstr code; use raw_ato <<
1131 * <<
1132 * Return: The original value of @v. <<
1133 */ <<
1134 static __always_inline int 477 static __always_inline int
1135 atomic_xchg_acquire(atomic_t *v, int new) !! 478 atomic_xchg_acquire(atomic_t *v, int i)
1136 { 479 {
1137 instrument_atomic_read_write(v, sizeo 480 instrument_atomic_read_write(v, sizeof(*v));
1138 return raw_atomic_xchg_acquire(v, new !! 481 return arch_atomic_xchg_acquire(v, i);
1139 } 482 }
1140 483
1141 /** <<
1142 * atomic_xchg_release() - atomic exchange wi <<
1143 * @v: pointer to atomic_t <<
1144 * @new: int value to assign <<
1145 * <<
1146 * Atomically updates @v to @new with release <<
1147 * <<
1148 * Unsafe to use in noinstr code; use raw_ato <<
1149 * <<
1150 * Return: The original value of @v. <<
1151 */ <<
1152 static __always_inline int 484 static __always_inline int
1153 atomic_xchg_release(atomic_t *v, int new) !! 485 atomic_xchg_release(atomic_t *v, int i)
1154 { 486 {
1155 kcsan_release(); 487 kcsan_release();
1156 instrument_atomic_read_write(v, sizeo 488 instrument_atomic_read_write(v, sizeof(*v));
1157 return raw_atomic_xchg_release(v, new !! 489 return arch_atomic_xchg_release(v, i);
1158 } 490 }
1159 491
1160 /** <<
1161 * atomic_xchg_relaxed() - atomic exchange wi <<
1162 * @v: pointer to atomic_t <<
1163 * @new: int value to assign <<
1164 * <<
1165 * Atomically updates @v to @new with relaxed <<
1166 * <<
1167 * Unsafe to use in noinstr code; use raw_ato <<
1168 * <<
1169 * Return: The original value of @v. <<
1170 */ <<
1171 static __always_inline int 492 static __always_inline int
1172 atomic_xchg_relaxed(atomic_t *v, int new) !! 493 atomic_xchg_relaxed(atomic_t *v, int i)
1173 { 494 {
1174 instrument_atomic_read_write(v, sizeo 495 instrument_atomic_read_write(v, sizeof(*v));
1175 return raw_atomic_xchg_relaxed(v, new !! 496 return arch_atomic_xchg_relaxed(v, i);
1176 } 497 }
1177 498
1178 /** <<
1179 * atomic_cmpxchg() - atomic compare and exch <<
1180 * @v: pointer to atomic_t <<
1181 * @old: int value to compare with <<
1182 * @new: int value to assign <<
1183 * <<
1184 * If (@v == @old), atomically updates @v to <<
1185 * Otherwise, @v is not modified and relaxed <<
1186 * <<
1187 * Unsafe to use in noinstr code; use raw_ato <<
1188 * <<
1189 * Return: The original value of @v. <<
1190 */ <<
1191 static __always_inline int 499 static __always_inline int
1192 atomic_cmpxchg(atomic_t *v, int old, int new) 500 atomic_cmpxchg(atomic_t *v, int old, int new)
1193 { 501 {
1194 kcsan_mb(); 502 kcsan_mb();
1195 instrument_atomic_read_write(v, sizeo 503 instrument_atomic_read_write(v, sizeof(*v));
1196 return raw_atomic_cmpxchg(v, old, new !! 504 return arch_atomic_cmpxchg(v, old, new);
1197 } 505 }
1198 506
1199 /** <<
1200 * atomic_cmpxchg_acquire() - atomic compare <<
1201 * @v: pointer to atomic_t <<
1202 * @old: int value to compare with <<
1203 * @new: int value to assign <<
1204 * <<
1205 * If (@v == @old), atomically updates @v to <<
1206 * Otherwise, @v is not modified and relaxed <<
1207 * <<
1208 * Unsafe to use in noinstr code; use raw_ato <<
1209 * <<
1210 * Return: The original value of @v. <<
1211 */ <<
1212 static __always_inline int 507 static __always_inline int
1213 atomic_cmpxchg_acquire(atomic_t *v, int old, 508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1214 { 509 {
1215 instrument_atomic_read_write(v, sizeo 510 instrument_atomic_read_write(v, sizeof(*v));
1216 return raw_atomic_cmpxchg_acquire(v, !! 511 return arch_atomic_cmpxchg_acquire(v, old, new);
1217 } 512 }
1218 513
1219 /** <<
1220 * atomic_cmpxchg_release() - atomic compare <<
1221 * @v: pointer to atomic_t <<
1222 * @old: int value to compare with <<
1223 * @new: int value to assign <<
1224 * <<
1225 * If (@v == @old), atomically updates @v to <<
1226 * Otherwise, @v is not modified and relaxed <<
1227 * <<
1228 * Unsafe to use in noinstr code; use raw_ato <<
1229 * <<
1230 * Return: The original value of @v. <<
1231 */ <<
1232 static __always_inline int 514 static __always_inline int
1233 atomic_cmpxchg_release(atomic_t *v, int old, 515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
1234 { 516 {
1235 kcsan_release(); 517 kcsan_release();
1236 instrument_atomic_read_write(v, sizeo 518 instrument_atomic_read_write(v, sizeof(*v));
1237 return raw_atomic_cmpxchg_release(v, !! 519 return arch_atomic_cmpxchg_release(v, old, new);
1238 } 520 }
1239 521
1240 /** <<
1241 * atomic_cmpxchg_relaxed() - atomic compare <<
1242 * @v: pointer to atomic_t <<
1243 * @old: int value to compare with <<
1244 * @new: int value to assign <<
1245 * <<
1246 * If (@v == @old), atomically updates @v to <<
1247 * Otherwise, @v is not modified and relaxed <<
1248 * <<
1249 * Unsafe to use in noinstr code; use raw_ato <<
1250 * <<
1251 * Return: The original value of @v. <<
1252 */ <<
1253 static __always_inline int 522 static __always_inline int
1254 atomic_cmpxchg_relaxed(atomic_t *v, int old, 523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
1255 { 524 {
1256 instrument_atomic_read_write(v, sizeo 525 instrument_atomic_read_write(v, sizeof(*v));
1257 return raw_atomic_cmpxchg_relaxed(v, !! 526 return arch_atomic_cmpxchg_relaxed(v, old, new);
1258 } 527 }
1259 528
1260 /** <<
1261 * atomic_try_cmpxchg() - atomic compare and <<
1262 * @v: pointer to atomic_t <<
1263 * @old: pointer to int value to compare with <<
1264 * @new: int value to assign <<
1265 * <<
1266 * If (@v == @old), atomically updates @v to <<
1267 * Otherwise, @v is not modified, @old is upd <<
1268 * and relaxed ordering is provided. <<
1269 * <<
1270 * Unsafe to use in noinstr code; use raw_ato <<
1271 * <<
1272 * Return: @true if the exchange occured, @fa <<
1273 */ <<
1274 static __always_inline bool 529 static __always_inline bool
1275 atomic_try_cmpxchg(atomic_t *v, int *old, int 530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1276 { 531 {
1277 kcsan_mb(); 532 kcsan_mb();
1278 instrument_atomic_read_write(v, sizeo 533 instrument_atomic_read_write(v, sizeof(*v));
1279 instrument_atomic_read_write(old, siz 534 instrument_atomic_read_write(old, sizeof(*old));
1280 return raw_atomic_try_cmpxchg(v, old, !! 535 return arch_atomic_try_cmpxchg(v, old, new);
1281 } 536 }
1282 537
1283 /** <<
1284 * atomic_try_cmpxchg_acquire() - atomic comp <<
1285 * @v: pointer to atomic_t <<
1286 * @old: pointer to int value to compare with <<
1287 * @new: int value to assign <<
1288 * <<
1289 * If (@v == @old), atomically updates @v to <<
1290 * Otherwise, @v is not modified, @old is upd <<
1291 * and relaxed ordering is provided. <<
1292 * <<
1293 * Unsafe to use in noinstr code; use raw_ato <<
1294 * <<
1295 * Return: @true if the exchange occured, @fa <<
1296 */ <<
1297 static __always_inline bool 538 static __always_inline bool
1298 atomic_try_cmpxchg_acquire(atomic_t *v, int * 539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1299 { 540 {
1300 instrument_atomic_read_write(v, sizeo 541 instrument_atomic_read_write(v, sizeof(*v));
1301 instrument_atomic_read_write(old, siz 542 instrument_atomic_read_write(old, sizeof(*old));
1302 return raw_atomic_try_cmpxchg_acquire !! 543 return arch_atomic_try_cmpxchg_acquire(v, old, new);
1303 } 544 }
1304 545
1305 /** <<
1306 * atomic_try_cmpxchg_release() - atomic comp <<
1307 * @v: pointer to atomic_t <<
1308 * @old: pointer to int value to compare with <<
1309 * @new: int value to assign <<
1310 * <<
1311 * If (@v == @old), atomically updates @v to <<
1312 * Otherwise, @v is not modified, @old is upd <<
1313 * and relaxed ordering is provided. <<
1314 * <<
1315 * Unsafe to use in noinstr code; use raw_ato <<
1316 * <<
1317 * Return: @true if the exchange occured, @fa <<
1318 */ <<
1319 static __always_inline bool 546 static __always_inline bool
1320 atomic_try_cmpxchg_release(atomic_t *v, int * 547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1321 { 548 {
1322 kcsan_release(); 549 kcsan_release();
1323 instrument_atomic_read_write(v, sizeo 550 instrument_atomic_read_write(v, sizeof(*v));
1324 instrument_atomic_read_write(old, siz 551 instrument_atomic_read_write(old, sizeof(*old));
1325 return raw_atomic_try_cmpxchg_release !! 552 return arch_atomic_try_cmpxchg_release(v, old, new);
1326 } 553 }
1327 554
1328 /** <<
1329 * atomic_try_cmpxchg_relaxed() - atomic comp <<
1330 * @v: pointer to atomic_t <<
1331 * @old: pointer to int value to compare with <<
1332 * @new: int value to assign <<
1333 * <<
1334 * If (@v == @old), atomically updates @v to <<
1335 * Otherwise, @v is not modified, @old is upd <<
1336 * and relaxed ordering is provided. <<
1337 * <<
1338 * Unsafe to use in noinstr code; use raw_ato <<
1339 * <<
1340 * Return: @true if the exchange occured, @fa <<
1341 */ <<
1342 static __always_inline bool 555 static __always_inline bool
1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int * 556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1344 { 557 {
1345 instrument_atomic_read_write(v, sizeo 558 instrument_atomic_read_write(v, sizeof(*v));
1346 instrument_atomic_read_write(old, siz 559 instrument_atomic_read_write(old, sizeof(*old));
1347 return raw_atomic_try_cmpxchg_relaxed !! 560 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1348 } 561 }
1349 562
1350 /** <<
1351 * atomic_sub_and_test() - atomic subtract an <<
1352 * @i: int value to subtract <<
1353 * @v: pointer to atomic_t <<
1354 * <<
1355 * Atomically updates @v to (@v - @i) with fu <<
1356 * <<
1357 * Unsafe to use in noinstr code; use raw_ato <<
1358 * <<
1359 * Return: @true if the resulting value of @v <<
1360 */ <<
1361 static __always_inline bool 563 static __always_inline bool
1362 atomic_sub_and_test(int i, atomic_t *v) 564 atomic_sub_and_test(int i, atomic_t *v)
1363 { 565 {
1364 kcsan_mb(); 566 kcsan_mb();
1365 instrument_atomic_read_write(v, sizeo 567 instrument_atomic_read_write(v, sizeof(*v));
1366 return raw_atomic_sub_and_test(i, v); !! 568 return arch_atomic_sub_and_test(i, v);
1367 } 569 }
1368 570
1369 /** <<
1370 * atomic_dec_and_test() - atomic decrement a <<
1371 * @v: pointer to atomic_t <<
1372 * <<
1373 * Atomically updates @v to (@v - 1) with ful <<
1374 * <<
1375 * Unsafe to use in noinstr code; use raw_ato <<
1376 * <<
1377 * Return: @true if the resulting value of @v <<
1378 */ <<
1379 static __always_inline bool 571 static __always_inline bool
1380 atomic_dec_and_test(atomic_t *v) 572 atomic_dec_and_test(atomic_t *v)
1381 { 573 {
1382 kcsan_mb(); 574 kcsan_mb();
1383 instrument_atomic_read_write(v, sizeo 575 instrument_atomic_read_write(v, sizeof(*v));
1384 return raw_atomic_dec_and_test(v); !! 576 return arch_atomic_dec_and_test(v);
1385 } 577 }
1386 578
1387 /** <<
1388 * atomic_inc_and_test() - atomic increment a <<
1389 * @v: pointer to atomic_t <<
1390 * <<
1391 * Atomically updates @v to (@v + 1) with ful <<
1392 * <<
1393 * Unsafe to use in noinstr code; use raw_ato <<
1394 * <<
1395 * Return: @true if the resulting value of @v <<
1396 */ <<
1397 static __always_inline bool 579 static __always_inline bool
1398 atomic_inc_and_test(atomic_t *v) 580 atomic_inc_and_test(atomic_t *v)
1399 { 581 {
1400 kcsan_mb(); 582 kcsan_mb();
1401 instrument_atomic_read_write(v, sizeo 583 instrument_atomic_read_write(v, sizeof(*v));
1402 return raw_atomic_inc_and_test(v); !! 584 return arch_atomic_inc_and_test(v);
1403 } 585 }
1404 586
1405 /** <<
1406 * atomic_add_negative() - atomic add and tes <<
1407 * @i: int value to add <<
1408 * @v: pointer to atomic_t <<
1409 * <<
1410 * Atomically updates @v to (@v + @i) with fu <<
1411 * <<
1412 * Unsafe to use in noinstr code; use raw_ato <<
1413 * <<
1414 * Return: @true if the resulting value of @v <<
1415 */ <<
1416 static __always_inline bool 587 static __always_inline bool
1417 atomic_add_negative(int i, atomic_t *v) 588 atomic_add_negative(int i, atomic_t *v)
1418 { 589 {
1419 kcsan_mb(); 590 kcsan_mb();
1420 instrument_atomic_read_write(v, sizeo 591 instrument_atomic_read_write(v, sizeof(*v));
1421 return raw_atomic_add_negative(i, v); !! 592 return arch_atomic_add_negative(i, v);
1422 } <<
1423 <<
1424 /** <<
1425 * atomic_add_negative_acquire() - atomic add <<
1426 * @i: int value to add <<
1427 * @v: pointer to atomic_t <<
1428 * <<
1429 * Atomically updates @v to (@v + @i) with ac <<
1430 * <<
1431 * Unsafe to use in noinstr code; use raw_ato <<
1432 * <<
1433 * Return: @true if the resulting value of @v <<
1434 */ <<
1435 static __always_inline bool <<
1436 atomic_add_negative_acquire(int i, atomic_t * <<
1437 { <<
1438 instrument_atomic_read_write(v, sizeo <<
1439 return raw_atomic_add_negative_acquir <<
1440 } <<
1441 <<
1442 /** <<
1443 * atomic_add_negative_release() - atomic add <<
1444 * @i: int value to add <<
1445 * @v: pointer to atomic_t <<
1446 * <<
1447 * Atomically updates @v to (@v + @i) with re <<
1448 * <<
1449 * Unsafe to use in noinstr code; use raw_ato <<
1450 * <<
1451 * Return: @true if the resulting value of @v <<
1452 */ <<
1453 static __always_inline bool <<
1454 atomic_add_negative_release(int i, atomic_t * <<
1455 { <<
1456 kcsan_release(); <<
1457 instrument_atomic_read_write(v, sizeo <<
1458 return raw_atomic_add_negative_releas <<
1459 } <<
1460 <<
1461 /** <<
1462 * atomic_add_negative_relaxed() - atomic add <<
1463 * @i: int value to add <<
1464 * @v: pointer to atomic_t <<
1465 * <<
1466 * Atomically updates @v to (@v + @i) with re <<
1467 * <<
1468 * Unsafe to use in noinstr code; use raw_ato <<
1469 * <<
1470 * Return: @true if the resulting value of @v <<
1471 */ <<
1472 static __always_inline bool <<
1473 atomic_add_negative_relaxed(int i, atomic_t * <<
1474 { <<
1475 instrument_atomic_read_write(v, sizeo <<
1476 return raw_atomic_add_negative_relaxe <<
1477 } 593 }
1478 594
1479 /** <<
1480 * atomic_fetch_add_unless() - atomic add unl <<
1481 * @v: pointer to atomic_t <<
1482 * @a: int value to add <<
1483 * @u: int value to compare with <<
1484 * <<
1485 * If (@v != @u), atomically updates @v to (@ <<
1486 * Otherwise, @v is not modified and relaxed <<
1487 * <<
1488 * Unsafe to use in noinstr code; use raw_ato <<
1489 * <<
1490 * Return: The original value of @v. <<
1491 */ <<
1492 static __always_inline int 595 static __always_inline int
1493 atomic_fetch_add_unless(atomic_t *v, int a, i 596 atomic_fetch_add_unless(atomic_t *v, int a, int u)
1494 { 597 {
1495 kcsan_mb(); 598 kcsan_mb();
1496 instrument_atomic_read_write(v, sizeo 599 instrument_atomic_read_write(v, sizeof(*v));
1497 return raw_atomic_fetch_add_unless(v, !! 600 return arch_atomic_fetch_add_unless(v, a, u);
1498 } 601 }
1499 602
1500 /** <<
1501 * atomic_add_unless() - atomic add unless va <<
1502 * @v: pointer to atomic_t <<
1503 * @a: int value to add <<
1504 * @u: int value to compare with <<
1505 * <<
1506 * If (@v != @u), atomically updates @v to (@ <<
1507 * Otherwise, @v is not modified and relaxed <<
1508 * <<
1509 * Unsafe to use in noinstr code; use raw_ato <<
1510 * <<
1511 * Return: @true if @v was updated, @false ot <<
1512 */ <<
1513 static __always_inline bool 603 static __always_inline bool
1514 atomic_add_unless(atomic_t *v, int a, int u) 604 atomic_add_unless(atomic_t *v, int a, int u)
1515 { 605 {
1516 kcsan_mb(); 606 kcsan_mb();
1517 instrument_atomic_read_write(v, sizeo 607 instrument_atomic_read_write(v, sizeof(*v));
1518 return raw_atomic_add_unless(v, a, u) !! 608 return arch_atomic_add_unless(v, a, u);
1519 } 609 }
1520 610
1521 /** <<
1522 * atomic_inc_not_zero() - atomic increment u <<
1523 * @v: pointer to atomic_t <<
1524 * <<
1525 * If (@v != 0), atomically updates @v to (@v <<
1526 * Otherwise, @v is not modified and relaxed <<
1527 * <<
1528 * Unsafe to use in noinstr code; use raw_ato <<
1529 * <<
1530 * Return: @true if @v was updated, @false ot <<
1531 */ <<
1532 static __always_inline bool 611 static __always_inline bool
1533 atomic_inc_not_zero(atomic_t *v) 612 atomic_inc_not_zero(atomic_t *v)
1534 { 613 {
1535 kcsan_mb(); 614 kcsan_mb();
1536 instrument_atomic_read_write(v, sizeo 615 instrument_atomic_read_write(v, sizeof(*v));
1537 return raw_atomic_inc_not_zero(v); !! 616 return arch_atomic_inc_not_zero(v);
1538 } 617 }
1539 618
1540 /** <<
1541 * atomic_inc_unless_negative() - atomic incr <<
1542 * @v: pointer to atomic_t <<
1543 * <<
1544 * If (@v >= 0), atomically updates @v to (@v <<
1545 * Otherwise, @v is not modified and relaxed <<
1546 * <<
1547 * Unsafe to use in noinstr code; use raw_ato <<
1548 * <<
1549 * Return: @true if @v was updated, @false ot <<
1550 */ <<
1551 static __always_inline bool 619 static __always_inline bool
1552 atomic_inc_unless_negative(atomic_t *v) 620 atomic_inc_unless_negative(atomic_t *v)
1553 { 621 {
1554 kcsan_mb(); 622 kcsan_mb();
1555 instrument_atomic_read_write(v, sizeo 623 instrument_atomic_read_write(v, sizeof(*v));
1556 return raw_atomic_inc_unless_negative !! 624 return arch_atomic_inc_unless_negative(v);
1557 } 625 }
1558 626
1559 /** <<
1560 * atomic_dec_unless_positive() - atomic decr <<
1561 * @v: pointer to atomic_t <<
1562 * <<
1563 * If (@v <= 0), atomically updates @v to (@v <<
1564 * Otherwise, @v is not modified and relaxed <<
1565 * <<
1566 * Unsafe to use in noinstr code; use raw_ato <<
1567 * <<
1568 * Return: @true if @v was updated, @false ot <<
1569 */ <<
1570 static __always_inline bool 627 static __always_inline bool
1571 atomic_dec_unless_positive(atomic_t *v) 628 atomic_dec_unless_positive(atomic_t *v)
1572 { 629 {
1573 kcsan_mb(); 630 kcsan_mb();
1574 instrument_atomic_read_write(v, sizeo 631 instrument_atomic_read_write(v, sizeof(*v));
1575 return raw_atomic_dec_unless_positive !! 632 return arch_atomic_dec_unless_positive(v);
1576 } 633 }
1577 634
1578 /** <<
1579 * atomic_dec_if_positive() - atomic decremen <<
1580 * @v: pointer to atomic_t <<
1581 * <<
1582 * If (@v > 0), atomically updates @v to (@v <<
1583 * Otherwise, @v is not modified and relaxed <<
1584 * <<
1585 * Unsafe to use in noinstr code; use raw_ato <<
1586 * <<
1587 * Return: The old value of (@v - 1), regardl <<
1588 */ <<
1589 static __always_inline int 635 static __always_inline int
1590 atomic_dec_if_positive(atomic_t *v) 636 atomic_dec_if_positive(atomic_t *v)
1591 { 637 {
1592 kcsan_mb(); 638 kcsan_mb();
1593 instrument_atomic_read_write(v, sizeo 639 instrument_atomic_read_write(v, sizeof(*v));
1594 return raw_atomic_dec_if_positive(v); !! 640 return arch_atomic_dec_if_positive(v);
1595 } 641 }
1596 642
1597 /** <<
1598 * atomic64_read() - atomic load with relaxed <<
1599 * @v: pointer to atomic64_t <<
1600 * <<
1601 * Atomically loads the value of @v with rela <<
1602 * <<
1603 * Unsafe to use in noinstr code; use raw_ato <<
1604 * <<
1605 * Return: The value loaded from @v. <<
1606 */ <<
1607 static __always_inline s64 643 static __always_inline s64
1608 atomic64_read(const atomic64_t *v) 644 atomic64_read(const atomic64_t *v)
1609 { 645 {
1610 instrument_atomic_read(v, sizeof(*v)) 646 instrument_atomic_read(v, sizeof(*v));
1611 return raw_atomic64_read(v); !! 647 return arch_atomic64_read(v);
1612 } 648 }
1613 649
1614 /** <<
1615 * atomic64_read_acquire() - atomic load with <<
1616 * @v: pointer to atomic64_t <<
1617 * <<
1618 * Atomically loads the value of @v with acqu <<
1619 * <<
1620 * Unsafe to use in noinstr code; use raw_ato <<
1621 * <<
1622 * Return: The value loaded from @v. <<
1623 */ <<
1624 static __always_inline s64 650 static __always_inline s64
1625 atomic64_read_acquire(const atomic64_t *v) 651 atomic64_read_acquire(const atomic64_t *v)
1626 { 652 {
1627 instrument_atomic_read(v, sizeof(*v)) 653 instrument_atomic_read(v, sizeof(*v));
1628 return raw_atomic64_read_acquire(v); !! 654 return arch_atomic64_read_acquire(v);
1629 } 655 }
1630 656
1631 /** <<
1632 * atomic64_set() - atomic set with relaxed o <<
1633 * @v: pointer to atomic64_t <<
1634 * @i: s64 value to assign <<
1635 * <<
1636 * Atomically sets @v to @i with relaxed orde <<
1637 * <<
1638 * Unsafe to use in noinstr code; use raw_ato <<
1639 * <<
1640 * Return: Nothing. <<
1641 */ <<
1642 static __always_inline void 657 static __always_inline void
1643 atomic64_set(atomic64_t *v, s64 i) 658 atomic64_set(atomic64_t *v, s64 i)
1644 { 659 {
1645 instrument_atomic_write(v, sizeof(*v) 660 instrument_atomic_write(v, sizeof(*v));
1646 raw_atomic64_set(v, i); !! 661 arch_atomic64_set(v, i);
1647 } 662 }
1648 663
1649 /** <<
1650 * atomic64_set_release() - atomic set with r <<
1651 * @v: pointer to atomic64_t <<
1652 * @i: s64 value to assign <<
1653 * <<
1654 * Atomically sets @v to @i with release orde <<
1655 * <<
1656 * Unsafe to use in noinstr code; use raw_ato <<
1657 * <<
1658 * Return: Nothing. <<
1659 */ <<
1660 static __always_inline void 664 static __always_inline void
1661 atomic64_set_release(atomic64_t *v, s64 i) 665 atomic64_set_release(atomic64_t *v, s64 i)
1662 { 666 {
1663 kcsan_release(); 667 kcsan_release();
1664 instrument_atomic_write(v, sizeof(*v) 668 instrument_atomic_write(v, sizeof(*v));
1665 raw_atomic64_set_release(v, i); !! 669 arch_atomic64_set_release(v, i);
1666 } 670 }
1667 671
1668 /** <<
1669 * atomic64_add() - atomic add with relaxed o <<
1670 * @i: s64 value to add <<
1671 * @v: pointer to atomic64_t <<
1672 * <<
1673 * Atomically updates @v to (@v + @i) with re <<
1674 * <<
1675 * Unsafe to use in noinstr code; use raw_ato <<
1676 * <<
1677 * Return: Nothing. <<
1678 */ <<
1679 static __always_inline void 672 static __always_inline void
1680 atomic64_add(s64 i, atomic64_t *v) 673 atomic64_add(s64 i, atomic64_t *v)
1681 { 674 {
1682 instrument_atomic_read_write(v, sizeo 675 instrument_atomic_read_write(v, sizeof(*v));
1683 raw_atomic64_add(i, v); !! 676 arch_atomic64_add(i, v);
1684 } 677 }
1685 678
1686 /** <<
1687 * atomic64_add_return() - atomic add with fu <<
1688 * @i: s64 value to add <<
1689 * @v: pointer to atomic64_t <<
1690 * <<
1691 * Atomically updates @v to (@v + @i) with fu <<
1692 * <<
1693 * Unsafe to use in noinstr code; use raw_ato <<
1694 * <<
1695 * Return: The updated value of @v. <<
1696 */ <<
1697 static __always_inline s64 679 static __always_inline s64
1698 atomic64_add_return(s64 i, atomic64_t *v) 680 atomic64_add_return(s64 i, atomic64_t *v)
1699 { 681 {
1700 kcsan_mb(); 682 kcsan_mb();
1701 instrument_atomic_read_write(v, sizeo 683 instrument_atomic_read_write(v, sizeof(*v));
1702 return raw_atomic64_add_return(i, v); !! 684 return arch_atomic64_add_return(i, v);
1703 } 685 }
1704 686
1705 /** <<
1706 * atomic64_add_return_acquire() - atomic add <<
1707 * @i: s64 value to add <<
1708 * @v: pointer to atomic64_t <<
1709 * <<
1710 * Atomically updates @v to (@v + @i) with ac <<
1711 * <<
1712 * Unsafe to use in noinstr code; use raw_ato <<
1713 * <<
1714 * Return: The updated value of @v. <<
1715 */ <<
1716 static __always_inline s64 687 static __always_inline s64
1717 atomic64_add_return_acquire(s64 i, atomic64_t 688 atomic64_add_return_acquire(s64 i, atomic64_t *v)
1718 { 689 {
1719 instrument_atomic_read_write(v, sizeo 690 instrument_atomic_read_write(v, sizeof(*v));
1720 return raw_atomic64_add_return_acquir !! 691 return arch_atomic64_add_return_acquire(i, v);
1721 } 692 }
1722 693
1723 /** <<
1724 * atomic64_add_return_release() - atomic add <<
1725 * @i: s64 value to add <<
1726 * @v: pointer to atomic64_t <<
1727 * <<
1728 * Atomically updates @v to (@v + @i) with re <<
1729 * <<
1730 * Unsafe to use in noinstr code; use raw_ato <<
1731 * <<
1732 * Return: The updated value of @v. <<
1733 */ <<
1734 static __always_inline s64 694 static __always_inline s64
1735 atomic64_add_return_release(s64 i, atomic64_t 695 atomic64_add_return_release(s64 i, atomic64_t *v)
1736 { 696 {
1737 kcsan_release(); 697 kcsan_release();
1738 instrument_atomic_read_write(v, sizeo 698 instrument_atomic_read_write(v, sizeof(*v));
1739 return raw_atomic64_add_return_releas !! 699 return arch_atomic64_add_return_release(i, v);
1740 } 700 }
1741 701
1742 /** <<
1743 * atomic64_add_return_relaxed() - atomic add <<
1744 * @i: s64 value to add <<
1745 * @v: pointer to atomic64_t <<
1746 * <<
1747 * Atomically updates @v to (@v + @i) with re <<
1748 * <<
1749 * Unsafe to use in noinstr code; use raw_ato <<
1750 * <<
1751 * Return: The updated value of @v. <<
1752 */ <<
1753 static __always_inline s64 702 static __always_inline s64
1754 atomic64_add_return_relaxed(s64 i, atomic64_t 703 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
1755 { 704 {
1756 instrument_atomic_read_write(v, sizeo 705 instrument_atomic_read_write(v, sizeof(*v));
1757 return raw_atomic64_add_return_relaxe !! 706 return arch_atomic64_add_return_relaxed(i, v);
1758 } 707 }
1759 708
1760 /** <<
1761 * atomic64_fetch_add() - atomic add with ful <<
1762 * @i: s64 value to add <<
1763 * @v: pointer to atomic64_t <<
1764 * <<
1765 * Atomically updates @v to (@v + @i) with fu <<
1766 * <<
1767 * Unsafe to use in noinstr code; use raw_ato <<
1768 * <<
1769 * Return: The original value of @v. <<
1770 */ <<
1771 static __always_inline s64 709 static __always_inline s64
1772 atomic64_fetch_add(s64 i, atomic64_t *v) 710 atomic64_fetch_add(s64 i, atomic64_t *v)
1773 { 711 {
1774 kcsan_mb(); 712 kcsan_mb();
1775 instrument_atomic_read_write(v, sizeo 713 instrument_atomic_read_write(v, sizeof(*v));
1776 return raw_atomic64_fetch_add(i, v); !! 714 return arch_atomic64_fetch_add(i, v);
1777 } 715 }
1778 716
1779 /** <<
1780 * atomic64_fetch_add_acquire() - atomic add <<
1781 * @i: s64 value to add <<
1782 * @v: pointer to atomic64_t <<
1783 * <<
1784 * Atomically updates @v to (@v + @i) with ac <<
1785 * <<
1786 * Unsafe to use in noinstr code; use raw_ato <<
1787 * <<
1788 * Return: The original value of @v. <<
1789 */ <<
1790 static __always_inline s64 717 static __always_inline s64
1791 atomic64_fetch_add_acquire(s64 i, atomic64_t 718 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1792 { 719 {
1793 instrument_atomic_read_write(v, sizeo 720 instrument_atomic_read_write(v, sizeof(*v));
1794 return raw_atomic64_fetch_add_acquire !! 721 return arch_atomic64_fetch_add_acquire(i, v);
1795 } 722 }
1796 723
1797 /** <<
1798 * atomic64_fetch_add_release() - atomic add <<
1799 * @i: s64 value to add <<
1800 * @v: pointer to atomic64_t <<
1801 * <<
1802 * Atomically updates @v to (@v + @i) with re <<
1803 * <<
1804 * Unsafe to use in noinstr code; use raw_ato <<
1805 * <<
1806 * Return: The original value of @v. <<
1807 */ <<
1808 static __always_inline s64 724 static __always_inline s64
1809 atomic64_fetch_add_release(s64 i, atomic64_t 725 atomic64_fetch_add_release(s64 i, atomic64_t *v)
1810 { 726 {
1811 kcsan_release(); 727 kcsan_release();
1812 instrument_atomic_read_write(v, sizeo 728 instrument_atomic_read_write(v, sizeof(*v));
1813 return raw_atomic64_fetch_add_release !! 729 return arch_atomic64_fetch_add_release(i, v);
1814 } 730 }
1815 731
1816 /** <<
1817 * atomic64_fetch_add_relaxed() - atomic add <<
1818 * @i: s64 value to add <<
1819 * @v: pointer to atomic64_t <<
1820 * <<
1821 * Atomically updates @v to (@v + @i) with re <<
1822 * <<
1823 * Unsafe to use in noinstr code; use raw_ato <<
1824 * <<
1825 * Return: The original value of @v. <<
1826 */ <<
1827 static __always_inline s64 732 static __always_inline s64
1828 atomic64_fetch_add_relaxed(s64 i, atomic64_t 733 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
1829 { 734 {
1830 instrument_atomic_read_write(v, sizeo 735 instrument_atomic_read_write(v, sizeof(*v));
1831 return raw_atomic64_fetch_add_relaxed !! 736 return arch_atomic64_fetch_add_relaxed(i, v);
1832 } 737 }
1833 738
1834 /** <<
1835 * atomic64_sub() - atomic subtract with rela <<
1836 * @i: s64 value to subtract <<
1837 * @v: pointer to atomic64_t <<
1838 * <<
1839 * Atomically updates @v to (@v - @i) with re <<
1840 * <<
1841 * Unsafe to use in noinstr code; use raw_ato <<
1842 * <<
1843 * Return: Nothing. <<
1844 */ <<
1845 static __always_inline void 739 static __always_inline void
1846 atomic64_sub(s64 i, atomic64_t *v) 740 atomic64_sub(s64 i, atomic64_t *v)
1847 { 741 {
1848 instrument_atomic_read_write(v, sizeo 742 instrument_atomic_read_write(v, sizeof(*v));
1849 raw_atomic64_sub(i, v); !! 743 arch_atomic64_sub(i, v);
1850 } 744 }
1851 745
1852 /** <<
1853 * atomic64_sub_return() - atomic subtract wi <<
1854 * @i: s64 value to subtract <<
1855 * @v: pointer to atomic64_t <<
1856 * <<
1857 * Atomically updates @v to (@v - @i) with fu <<
1858 * <<
1859 * Unsafe to use in noinstr code; use raw_ato <<
1860 * <<
1861 * Return: The updated value of @v. <<
1862 */ <<
1863 static __always_inline s64 746 static __always_inline s64
1864 atomic64_sub_return(s64 i, atomic64_t *v) 747 atomic64_sub_return(s64 i, atomic64_t *v)
1865 { 748 {
1866 kcsan_mb(); 749 kcsan_mb();
1867 instrument_atomic_read_write(v, sizeo 750 instrument_atomic_read_write(v, sizeof(*v));
1868 return raw_atomic64_sub_return(i, v); !! 751 return arch_atomic64_sub_return(i, v);
1869 } 752 }
1870 753
1871 /** <<
1872 * atomic64_sub_return_acquire() - atomic sub <<
1873 * @i: s64 value to subtract <<
1874 * @v: pointer to atomic64_t <<
1875 * <<
1876 * Atomically updates @v to (@v - @i) with ac <<
1877 * <<
1878 * Unsafe to use in noinstr code; use raw_ato <<
1879 * <<
1880 * Return: The updated value of @v. <<
1881 */ <<
1882 static __always_inline s64 754 static __always_inline s64
1883 atomic64_sub_return_acquire(s64 i, atomic64_t 755 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1884 { 756 {
1885 instrument_atomic_read_write(v, sizeo 757 instrument_atomic_read_write(v, sizeof(*v));
1886 return raw_atomic64_sub_return_acquir !! 758 return arch_atomic64_sub_return_acquire(i, v);
1887 } 759 }
1888 760
1889 /** <<
1890 * atomic64_sub_return_release() - atomic sub <<
1891 * @i: s64 value to subtract <<
1892 * @v: pointer to atomic64_t <<
1893 * <<
1894 * Atomically updates @v to (@v - @i) with re <<
1895 * <<
1896 * Unsafe to use in noinstr code; use raw_ato <<
1897 * <<
1898 * Return: The updated value of @v. <<
1899 */ <<
1900 static __always_inline s64 761 static __always_inline s64
1901 atomic64_sub_return_release(s64 i, atomic64_t 762 atomic64_sub_return_release(s64 i, atomic64_t *v)
1902 { 763 {
1903 kcsan_release(); 764 kcsan_release();
1904 instrument_atomic_read_write(v, sizeo 765 instrument_atomic_read_write(v, sizeof(*v));
1905 return raw_atomic64_sub_return_releas !! 766 return arch_atomic64_sub_return_release(i, v);
1906 } 767 }
1907 768
1908 /** <<
1909 * atomic64_sub_return_relaxed() - atomic sub <<
1910 * @i: s64 value to subtract <<
1911 * @v: pointer to atomic64_t <<
1912 * <<
1913 * Atomically updates @v to (@v - @i) with re <<
1914 * <<
1915 * Unsafe to use in noinstr code; use raw_ato <<
1916 * <<
1917 * Return: The updated value of @v. <<
1918 */ <<
1919 static __always_inline s64 769 static __always_inline s64
1920 atomic64_sub_return_relaxed(s64 i, atomic64_t 770 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
1921 { 771 {
1922 instrument_atomic_read_write(v, sizeo 772 instrument_atomic_read_write(v, sizeof(*v));
1923 return raw_atomic64_sub_return_relaxe !! 773 return arch_atomic64_sub_return_relaxed(i, v);
1924 } 774 }
1925 775
1926 /** <<
1927 * atomic64_fetch_sub() - atomic subtract wit <<
1928 * @i: s64 value to subtract <<
1929 * @v: pointer to atomic64_t <<
1930 * <<
1931 * Atomically updates @v to (@v - @i) with fu <<
1932 * <<
1933 * Unsafe to use in noinstr code; use raw_ato <<
1934 * <<
1935 * Return: The original value of @v. <<
1936 */ <<
1937 static __always_inline s64 776 static __always_inline s64
1938 atomic64_fetch_sub(s64 i, atomic64_t *v) 777 atomic64_fetch_sub(s64 i, atomic64_t *v)
1939 { 778 {
1940 kcsan_mb(); 779 kcsan_mb();
1941 instrument_atomic_read_write(v, sizeo 780 instrument_atomic_read_write(v, sizeof(*v));
1942 return raw_atomic64_fetch_sub(i, v); !! 781 return arch_atomic64_fetch_sub(i, v);
1943 } 782 }
1944 783
1945 /** <<
1946 * atomic64_fetch_sub_acquire() - atomic subt <<
1947 * @i: s64 value to subtract <<
1948 * @v: pointer to atomic64_t <<
1949 * <<
1950 * Atomically updates @v to (@v - @i) with ac <<
1951 * <<
1952 * Unsafe to use in noinstr code; use raw_ato <<
1953 * <<
1954 * Return: The original value of @v. <<
1955 */ <<
1956 static __always_inline s64 784 static __always_inline s64
1957 atomic64_fetch_sub_acquire(s64 i, atomic64_t 785 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1958 { 786 {
1959 instrument_atomic_read_write(v, sizeo 787 instrument_atomic_read_write(v, sizeof(*v));
1960 return raw_atomic64_fetch_sub_acquire !! 788 return arch_atomic64_fetch_sub_acquire(i, v);
1961 } 789 }
1962 790
1963 /** <<
1964 * atomic64_fetch_sub_release() - atomic subt <<
1965 * @i: s64 value to subtract <<
1966 * @v: pointer to atomic64_t <<
1967 * <<
1968 * Atomically updates @v to (@v - @i) with re <<
1969 * <<
1970 * Unsafe to use in noinstr code; use raw_ato <<
1971 * <<
1972 * Return: The original value of @v. <<
1973 */ <<
1974 static __always_inline s64 791 static __always_inline s64
1975 atomic64_fetch_sub_release(s64 i, atomic64_t 792 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1976 { 793 {
1977 kcsan_release(); 794 kcsan_release();
1978 instrument_atomic_read_write(v, sizeo 795 instrument_atomic_read_write(v, sizeof(*v));
1979 return raw_atomic64_fetch_sub_release !! 796 return arch_atomic64_fetch_sub_release(i, v);
1980 } 797 }
1981 798
1982 /** <<
1983 * atomic64_fetch_sub_relaxed() - atomic subt <<
1984 * @i: s64 value to subtract <<
1985 * @v: pointer to atomic64_t <<
1986 * <<
1987 * Atomically updates @v to (@v - @i) with re <<
1988 * <<
1989 * Unsafe to use in noinstr code; use raw_ato <<
1990 * <<
1991 * Return: The original value of @v. <<
1992 */ <<
1993 static __always_inline s64 799 static __always_inline s64
1994 atomic64_fetch_sub_relaxed(s64 i, atomic64_t 800 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1995 { 801 {
1996 instrument_atomic_read_write(v, sizeo 802 instrument_atomic_read_write(v, sizeof(*v));
1997 return raw_atomic64_fetch_sub_relaxed !! 803 return arch_atomic64_fetch_sub_relaxed(i, v);
1998 } 804 }
1999 805
2000 /** <<
2001 * atomic64_inc() - atomic increment with rel <<
2002 * @v: pointer to atomic64_t <<
2003 * <<
2004 * Atomically updates @v to (@v + 1) with rel <<
2005 * <<
2006 * Unsafe to use in noinstr code; use raw_ato <<
2007 * <<
2008 * Return: Nothing. <<
2009 */ <<
2010 static __always_inline void 806 static __always_inline void
2011 atomic64_inc(atomic64_t *v) 807 atomic64_inc(atomic64_t *v)
2012 { 808 {
2013 instrument_atomic_read_write(v, sizeo 809 instrument_atomic_read_write(v, sizeof(*v));
2014 raw_atomic64_inc(v); !! 810 arch_atomic64_inc(v);
2015 } 811 }
2016 812
2017 /** <<
2018 * atomic64_inc_return() - atomic increment w <<
2019 * @v: pointer to atomic64_t <<
2020 * <<
2021 * Atomically updates @v to (@v + 1) with ful <<
2022 * <<
2023 * Unsafe to use in noinstr code; use raw_ato <<
2024 * <<
2025 * Return: The updated value of @v. <<
2026 */ <<
2027 static __always_inline s64 813 static __always_inline s64
2028 atomic64_inc_return(atomic64_t *v) 814 atomic64_inc_return(atomic64_t *v)
2029 { 815 {
2030 kcsan_mb(); 816 kcsan_mb();
2031 instrument_atomic_read_write(v, sizeo 817 instrument_atomic_read_write(v, sizeof(*v));
2032 return raw_atomic64_inc_return(v); !! 818 return arch_atomic64_inc_return(v);
2033 } 819 }
2034 820
2035 /** <<
2036 * atomic64_inc_return_acquire() - atomic inc <<
2037 * @v: pointer to atomic64_t <<
2038 * <<
2039 * Atomically updates @v to (@v + 1) with acq <<
2040 * <<
2041 * Unsafe to use in noinstr code; use raw_ato <<
2042 * <<
2043 * Return: The updated value of @v. <<
2044 */ <<
2045 static __always_inline s64 821 static __always_inline s64
2046 atomic64_inc_return_acquire(atomic64_t *v) 822 atomic64_inc_return_acquire(atomic64_t *v)
2047 { 823 {
2048 instrument_atomic_read_write(v, sizeo 824 instrument_atomic_read_write(v, sizeof(*v));
2049 return raw_atomic64_inc_return_acquir !! 825 return arch_atomic64_inc_return_acquire(v);
2050 } 826 }
2051 827
2052 /** <<
2053 * atomic64_inc_return_release() - atomic inc <<
2054 * @v: pointer to atomic64_t <<
2055 * <<
2056 * Atomically updates @v to (@v + 1) with rel <<
2057 * <<
2058 * Unsafe to use in noinstr code; use raw_ato <<
2059 * <<
2060 * Return: The updated value of @v. <<
2061 */ <<
2062 static __always_inline s64 828 static __always_inline s64
2063 atomic64_inc_return_release(atomic64_t *v) 829 atomic64_inc_return_release(atomic64_t *v)
2064 { 830 {
2065 kcsan_release(); 831 kcsan_release();
2066 instrument_atomic_read_write(v, sizeo 832 instrument_atomic_read_write(v, sizeof(*v));
2067 return raw_atomic64_inc_return_releas !! 833 return arch_atomic64_inc_return_release(v);
2068 } 834 }
2069 835
2070 /** <<
2071 * atomic64_inc_return_relaxed() - atomic inc <<
2072 * @v: pointer to atomic64_t <<
2073 * <<
2074 * Atomically updates @v to (@v + 1) with rel <<
2075 * <<
2076 * Unsafe to use in noinstr code; use raw_ato <<
2077 * <<
2078 * Return: The updated value of @v. <<
2079 */ <<
2080 static __always_inline s64 836 static __always_inline s64
2081 atomic64_inc_return_relaxed(atomic64_t *v) 837 atomic64_inc_return_relaxed(atomic64_t *v)
2082 { 838 {
2083 instrument_atomic_read_write(v, sizeo 839 instrument_atomic_read_write(v, sizeof(*v));
2084 return raw_atomic64_inc_return_relaxe !! 840 return arch_atomic64_inc_return_relaxed(v);
2085 } 841 }
2086 842
2087 /** <<
2088 * atomic64_fetch_inc() - atomic increment wi <<
2089 * @v: pointer to atomic64_t <<
2090 * <<
2091 * Atomically updates @v to (@v + 1) with ful <<
2092 * <<
2093 * Unsafe to use in noinstr code; use raw_ato <<
2094 * <<
2095 * Return: The original value of @v. <<
2096 */ <<
2097 static __always_inline s64 843 static __always_inline s64
2098 atomic64_fetch_inc(atomic64_t *v) 844 atomic64_fetch_inc(atomic64_t *v)
2099 { 845 {
2100 kcsan_mb(); 846 kcsan_mb();
2101 instrument_atomic_read_write(v, sizeo 847 instrument_atomic_read_write(v, sizeof(*v));
2102 return raw_atomic64_fetch_inc(v); !! 848 return arch_atomic64_fetch_inc(v);
2103 } 849 }
2104 850
2105 /** <<
2106 * atomic64_fetch_inc_acquire() - atomic incr <<
2107 * @v: pointer to atomic64_t <<
2108 * <<
2109 * Atomically updates @v to (@v + 1) with acq <<
2110 * <<
2111 * Unsafe to use in noinstr code; use raw_ato <<
2112 * <<
2113 * Return: The original value of @v. <<
2114 */ <<
2115 static __always_inline s64 851 static __always_inline s64
2116 atomic64_fetch_inc_acquire(atomic64_t *v) 852 atomic64_fetch_inc_acquire(atomic64_t *v)
2117 { 853 {
2118 instrument_atomic_read_write(v, sizeo 854 instrument_atomic_read_write(v, sizeof(*v));
2119 return raw_atomic64_fetch_inc_acquire !! 855 return arch_atomic64_fetch_inc_acquire(v);
2120 } 856 }
2121 857
2122 /** <<
2123 * atomic64_fetch_inc_release() - atomic incr <<
2124 * @v: pointer to atomic64_t <<
2125 * <<
2126 * Atomically updates @v to (@v + 1) with rel <<
2127 * <<
2128 * Unsafe to use in noinstr code; use raw_ato <<
2129 * <<
2130 * Return: The original value of @v. <<
2131 */ <<
2132 static __always_inline s64 858 static __always_inline s64
2133 atomic64_fetch_inc_release(atomic64_t *v) 859 atomic64_fetch_inc_release(atomic64_t *v)
2134 { 860 {
2135 kcsan_release(); 861 kcsan_release();
2136 instrument_atomic_read_write(v, sizeo 862 instrument_atomic_read_write(v, sizeof(*v));
2137 return raw_atomic64_fetch_inc_release !! 863 return arch_atomic64_fetch_inc_release(v);
2138 } 864 }
2139 865
2140 /** <<
2141 * atomic64_fetch_inc_relaxed() - atomic incr <<
2142 * @v: pointer to atomic64_t <<
2143 * <<
2144 * Atomically updates @v to (@v + 1) with rel <<
2145 * <<
2146 * Unsafe to use in noinstr code; use raw_ato <<
2147 * <<
2148 * Return: The original value of @v. <<
2149 */ <<
2150 static __always_inline s64 866 static __always_inline s64
2151 atomic64_fetch_inc_relaxed(atomic64_t *v) 867 atomic64_fetch_inc_relaxed(atomic64_t *v)
2152 { 868 {
2153 instrument_atomic_read_write(v, sizeo 869 instrument_atomic_read_write(v, sizeof(*v));
2154 return raw_atomic64_fetch_inc_relaxed !! 870 return arch_atomic64_fetch_inc_relaxed(v);
2155 } 871 }
2156 872
2157 /** <<
2158 * atomic64_dec() - atomic decrement with rel <<
2159 * @v: pointer to atomic64_t <<
2160 * <<
2161 * Atomically updates @v to (@v - 1) with rel <<
2162 * <<
2163 * Unsafe to use in noinstr code; use raw_ato <<
2164 * <<
2165 * Return: Nothing. <<
2166 */ <<
2167 static __always_inline void 873 static __always_inline void
2168 atomic64_dec(atomic64_t *v) 874 atomic64_dec(atomic64_t *v)
2169 { 875 {
2170 instrument_atomic_read_write(v, sizeo 876 instrument_atomic_read_write(v, sizeof(*v));
2171 raw_atomic64_dec(v); !! 877 arch_atomic64_dec(v);
2172 } 878 }
2173 879
2174 /** <<
2175 * atomic64_dec_return() - atomic decrement w <<
2176 * @v: pointer to atomic64_t <<
2177 * <<
2178 * Atomically updates @v to (@v - 1) with ful <<
2179 * <<
2180 * Unsafe to use in noinstr code; use raw_ato <<
2181 * <<
2182 * Return: The updated value of @v. <<
2183 */ <<
2184 static __always_inline s64 880 static __always_inline s64
2185 atomic64_dec_return(atomic64_t *v) 881 atomic64_dec_return(atomic64_t *v)
2186 { 882 {
2187 kcsan_mb(); 883 kcsan_mb();
2188 instrument_atomic_read_write(v, sizeo 884 instrument_atomic_read_write(v, sizeof(*v));
2189 return raw_atomic64_dec_return(v); !! 885 return arch_atomic64_dec_return(v);
2190 } 886 }
2191 887
2192 /** <<
2193 * atomic64_dec_return_acquire() - atomic dec <<
2194 * @v: pointer to atomic64_t <<
2195 * <<
2196 * Atomically updates @v to (@v - 1) with acq <<
2197 * <<
2198 * Unsafe to use in noinstr code; use raw_ato <<
2199 * <<
2200 * Return: The updated value of @v. <<
2201 */ <<
2202 static __always_inline s64 888 static __always_inline s64
2203 atomic64_dec_return_acquire(atomic64_t *v) 889 atomic64_dec_return_acquire(atomic64_t *v)
2204 { 890 {
2205 instrument_atomic_read_write(v, sizeo 891 instrument_atomic_read_write(v, sizeof(*v));
2206 return raw_atomic64_dec_return_acquir !! 892 return arch_atomic64_dec_return_acquire(v);
2207 } 893 }
2208 894
2209 /** <<
2210 * atomic64_dec_return_release() - atomic dec <<
2211 * @v: pointer to atomic64_t <<
2212 * <<
2213 * Atomically updates @v to (@v - 1) with rel <<
2214 * <<
2215 * Unsafe to use in noinstr code; use raw_ato <<
2216 * <<
2217 * Return: The updated value of @v. <<
2218 */ <<
2219 static __always_inline s64 895 static __always_inline s64
2220 atomic64_dec_return_release(atomic64_t *v) 896 atomic64_dec_return_release(atomic64_t *v)
2221 { 897 {
2222 kcsan_release(); 898 kcsan_release();
2223 instrument_atomic_read_write(v, sizeo 899 instrument_atomic_read_write(v, sizeof(*v));
2224 return raw_atomic64_dec_return_releas !! 900 return arch_atomic64_dec_return_release(v);
2225 } 901 }
2226 902
2227 /** <<
2228 * atomic64_dec_return_relaxed() - atomic dec <<
2229 * @v: pointer to atomic64_t <<
2230 * <<
2231 * Atomically updates @v to (@v - 1) with rel <<
2232 * <<
2233 * Unsafe to use in noinstr code; use raw_ato <<
2234 * <<
2235 * Return: The updated value of @v. <<
2236 */ <<
2237 static __always_inline s64 903 static __always_inline s64
2238 atomic64_dec_return_relaxed(atomic64_t *v) 904 atomic64_dec_return_relaxed(atomic64_t *v)
2239 { 905 {
2240 instrument_atomic_read_write(v, sizeo 906 instrument_atomic_read_write(v, sizeof(*v));
2241 return raw_atomic64_dec_return_relaxe !! 907 return arch_atomic64_dec_return_relaxed(v);
2242 } 908 }
2243 909
2244 /** <<
2245 * atomic64_fetch_dec() - atomic decrement wi <<
2246 * @v: pointer to atomic64_t <<
2247 * <<
2248 * Atomically updates @v to (@v - 1) with ful <<
2249 * <<
2250 * Unsafe to use in noinstr code; use raw_ato <<
2251 * <<
2252 * Return: The original value of @v. <<
2253 */ <<
2254 static __always_inline s64 910 static __always_inline s64
2255 atomic64_fetch_dec(atomic64_t *v) 911 atomic64_fetch_dec(atomic64_t *v)
2256 { 912 {
2257 kcsan_mb(); 913 kcsan_mb();
2258 instrument_atomic_read_write(v, sizeo 914 instrument_atomic_read_write(v, sizeof(*v));
2259 return raw_atomic64_fetch_dec(v); !! 915 return arch_atomic64_fetch_dec(v);
2260 } 916 }
2261 917
2262 /** <<
2263 * atomic64_fetch_dec_acquire() - atomic decr <<
2264 * @v: pointer to atomic64_t <<
2265 * <<
2266 * Atomically updates @v to (@v - 1) with acq <<
2267 * <<
2268 * Unsafe to use in noinstr code; use raw_ato <<
2269 * <<
2270 * Return: The original value of @v. <<
2271 */ <<
2272 static __always_inline s64 918 static __always_inline s64
2273 atomic64_fetch_dec_acquire(atomic64_t *v) 919 atomic64_fetch_dec_acquire(atomic64_t *v)
2274 { 920 {
2275 instrument_atomic_read_write(v, sizeo 921 instrument_atomic_read_write(v, sizeof(*v));
2276 return raw_atomic64_fetch_dec_acquire !! 922 return arch_atomic64_fetch_dec_acquire(v);
2277 } 923 }
2278 924
2279 /** <<
2280 * atomic64_fetch_dec_release() - atomic decr <<
2281 * @v: pointer to atomic64_t <<
2282 * <<
2283 * Atomically updates @v to (@v - 1) with rel <<
2284 * <<
2285 * Unsafe to use in noinstr code; use raw_ato <<
2286 * <<
2287 * Return: The original value of @v. <<
2288 */ <<
2289 static __always_inline s64 925 static __always_inline s64
2290 atomic64_fetch_dec_release(atomic64_t *v) 926 atomic64_fetch_dec_release(atomic64_t *v)
2291 { 927 {
2292 kcsan_release(); 928 kcsan_release();
2293 instrument_atomic_read_write(v, sizeo 929 instrument_atomic_read_write(v, sizeof(*v));
2294 return raw_atomic64_fetch_dec_release !! 930 return arch_atomic64_fetch_dec_release(v);
2295 } 931 }
2296 932
2297 /** <<
2298 * atomic64_fetch_dec_relaxed() - atomic decr <<
2299 * @v: pointer to atomic64_t <<
2300 * <<
2301 * Atomically updates @v to (@v - 1) with rel <<
2302 * <<
2303 * Unsafe to use in noinstr code; use raw_ato <<
2304 * <<
2305 * Return: The original value of @v. <<
2306 */ <<
2307 static __always_inline s64 933 static __always_inline s64
2308 atomic64_fetch_dec_relaxed(atomic64_t *v) 934 atomic64_fetch_dec_relaxed(atomic64_t *v)
2309 { 935 {
2310 instrument_atomic_read_write(v, sizeo 936 instrument_atomic_read_write(v, sizeof(*v));
2311 return raw_atomic64_fetch_dec_relaxed !! 937 return arch_atomic64_fetch_dec_relaxed(v);
2312 } 938 }
2313 939
2314 /** <<
2315 * atomic64_and() - atomic bitwise AND with r <<
2316 * @i: s64 value <<
2317 * @v: pointer to atomic64_t <<
2318 * <<
2319 * Atomically updates @v to (@v & @i) with re <<
2320 * <<
2321 * Unsafe to use in noinstr code; use raw_ato <<
2322 * <<
2323 * Return: Nothing. <<
2324 */ <<
2325 static __always_inline void 940 static __always_inline void
2326 atomic64_and(s64 i, atomic64_t *v) 941 atomic64_and(s64 i, atomic64_t *v)
2327 { 942 {
2328 instrument_atomic_read_write(v, sizeo 943 instrument_atomic_read_write(v, sizeof(*v));
2329 raw_atomic64_and(i, v); !! 944 arch_atomic64_and(i, v);
2330 } 945 }
2331 946
2332 /** <<
2333 * atomic64_fetch_and() - atomic bitwise AND <<
2334 * @i: s64 value <<
2335 * @v: pointer to atomic64_t <<
2336 * <<
2337 * Atomically updates @v to (@v & @i) with fu <<
2338 * <<
2339 * Unsafe to use in noinstr code; use raw_ato <<
2340 * <<
2341 * Return: The original value of @v. <<
2342 */ <<
2343 static __always_inline s64 947 static __always_inline s64
2344 atomic64_fetch_and(s64 i, atomic64_t *v) 948 atomic64_fetch_and(s64 i, atomic64_t *v)
2345 { 949 {
2346 kcsan_mb(); 950 kcsan_mb();
2347 instrument_atomic_read_write(v, sizeo 951 instrument_atomic_read_write(v, sizeof(*v));
2348 return raw_atomic64_fetch_and(i, v); !! 952 return arch_atomic64_fetch_and(i, v);
2349 } 953 }
2350 954
2351 /** <<
2352 * atomic64_fetch_and_acquire() - atomic bitw <<
2353 * @i: s64 value <<
2354 * @v: pointer to atomic64_t <<
2355 * <<
2356 * Atomically updates @v to (@v & @i) with ac <<
2357 * <<
2358 * Unsafe to use in noinstr code; use raw_ato <<
2359 * <<
2360 * Return: The original value of @v. <<
2361 */ <<
2362 static __always_inline s64 955 static __always_inline s64
2363 atomic64_fetch_and_acquire(s64 i, atomic64_t 956 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2364 { 957 {
2365 instrument_atomic_read_write(v, sizeo 958 instrument_atomic_read_write(v, sizeof(*v));
2366 return raw_atomic64_fetch_and_acquire !! 959 return arch_atomic64_fetch_and_acquire(i, v);
2367 } 960 }
2368 961
2369 /** <<
2370 * atomic64_fetch_and_release() - atomic bitw <<
2371 * @i: s64 value <<
2372 * @v: pointer to atomic64_t <<
2373 * <<
2374 * Atomically updates @v to (@v & @i) with re <<
2375 * <<
2376 * Unsafe to use in noinstr code; use raw_ato <<
2377 * <<
2378 * Return: The original value of @v. <<
2379 */ <<
2380 static __always_inline s64 962 static __always_inline s64
2381 atomic64_fetch_and_release(s64 i, atomic64_t 963 atomic64_fetch_and_release(s64 i, atomic64_t *v)
2382 { 964 {
2383 kcsan_release(); 965 kcsan_release();
2384 instrument_atomic_read_write(v, sizeo 966 instrument_atomic_read_write(v, sizeof(*v));
2385 return raw_atomic64_fetch_and_release !! 967 return arch_atomic64_fetch_and_release(i, v);
2386 } 968 }
2387 969
2388 /** <<
2389 * atomic64_fetch_and_relaxed() - atomic bitw <<
2390 * @i: s64 value <<
2391 * @v: pointer to atomic64_t <<
2392 * <<
2393 * Atomically updates @v to (@v & @i) with re <<
2394 * <<
2395 * Unsafe to use in noinstr code; use raw_ato <<
2396 * <<
2397 * Return: The original value of @v. <<
2398 */ <<
2399 static __always_inline s64 970 static __always_inline s64
2400 atomic64_fetch_and_relaxed(s64 i, atomic64_t 971 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
2401 { 972 {
2402 instrument_atomic_read_write(v, sizeo 973 instrument_atomic_read_write(v, sizeof(*v));
2403 return raw_atomic64_fetch_and_relaxed !! 974 return arch_atomic64_fetch_and_relaxed(i, v);
2404 } 975 }
2405 976
2406 /** <<
2407 * atomic64_andnot() - atomic bitwise AND NOT <<
2408 * @i: s64 value <<
2409 * @v: pointer to atomic64_t <<
2410 * <<
2411 * Atomically updates @v to (@v & ~@i) with r <<
2412 * <<
2413 * Unsafe to use in noinstr code; use raw_ato <<
2414 * <<
2415 * Return: Nothing. <<
2416 */ <<
2417 static __always_inline void 977 static __always_inline void
2418 atomic64_andnot(s64 i, atomic64_t *v) 978 atomic64_andnot(s64 i, atomic64_t *v)
2419 { 979 {
2420 instrument_atomic_read_write(v, sizeo 980 instrument_atomic_read_write(v, sizeof(*v));
2421 raw_atomic64_andnot(i, v); !! 981 arch_atomic64_andnot(i, v);
2422 } 982 }
2423 983
2424 /** <<
2425 * atomic64_fetch_andnot() - atomic bitwise A <<
2426 * @i: s64 value <<
2427 * @v: pointer to atomic64_t <<
2428 * <<
2429 * Atomically updates @v to (@v & ~@i) with f <<
2430 * <<
2431 * Unsafe to use in noinstr code; use raw_ato <<
2432 * <<
2433 * Return: The original value of @v. <<
2434 */ <<
2435 static __always_inline s64 984 static __always_inline s64
2436 atomic64_fetch_andnot(s64 i, atomic64_t *v) 985 atomic64_fetch_andnot(s64 i, atomic64_t *v)
2437 { 986 {
2438 kcsan_mb(); 987 kcsan_mb();
2439 instrument_atomic_read_write(v, sizeo 988 instrument_atomic_read_write(v, sizeof(*v));
2440 return raw_atomic64_fetch_andnot(i, v !! 989 return arch_atomic64_fetch_andnot(i, v);
2441 } 990 }
2442 991
2443 /** <<
2444 * atomic64_fetch_andnot_acquire() - atomic b <<
2445 * @i: s64 value <<
2446 * @v: pointer to atomic64_t <<
2447 * <<
2448 * Atomically updates @v to (@v & ~@i) with a <<
2449 * <<
2450 * Unsafe to use in noinstr code; use raw_ato <<
2451 * <<
2452 * Return: The original value of @v. <<
2453 */ <<
2454 static __always_inline s64 992 static __always_inline s64
2455 atomic64_fetch_andnot_acquire(s64 i, atomic64 993 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2456 { 994 {
2457 instrument_atomic_read_write(v, sizeo 995 instrument_atomic_read_write(v, sizeof(*v));
2458 return raw_atomic64_fetch_andnot_acqu !! 996 return arch_atomic64_fetch_andnot_acquire(i, v);
2459 } 997 }
2460 998
2461 /** <<
2462 * atomic64_fetch_andnot_release() - atomic b <<
2463 * @i: s64 value <<
2464 * @v: pointer to atomic64_t <<
2465 * <<
2466 * Atomically updates @v to (@v & ~@i) with r <<
2467 * <<
2468 * Unsafe to use in noinstr code; use raw_ato <<
2469 * <<
2470 * Return: The original value of @v. <<
2471 */ <<
2472 static __always_inline s64 999 static __always_inline s64
2473 atomic64_fetch_andnot_release(s64 i, atomic64 1000 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2474 { 1001 {
2475 kcsan_release(); 1002 kcsan_release();
2476 instrument_atomic_read_write(v, sizeo 1003 instrument_atomic_read_write(v, sizeof(*v));
2477 return raw_atomic64_fetch_andnot_rele !! 1004 return arch_atomic64_fetch_andnot_release(i, v);
2478 } 1005 }
2479 1006
2480 /** <<
2481 * atomic64_fetch_andnot_relaxed() - atomic b <<
2482 * @i: s64 value <<
2483 * @v: pointer to atomic64_t <<
2484 * <<
2485 * Atomically updates @v to (@v & ~@i) with r <<
2486 * <<
2487 * Unsafe to use in noinstr code; use raw_ato <<
2488 * <<
2489 * Return: The original value of @v. <<
2490 */ <<
2491 static __always_inline s64 1007 static __always_inline s64
2492 atomic64_fetch_andnot_relaxed(s64 i, atomic64 1008 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2493 { 1009 {
2494 instrument_atomic_read_write(v, sizeo 1010 instrument_atomic_read_write(v, sizeof(*v));
2495 return raw_atomic64_fetch_andnot_rela !! 1011 return arch_atomic64_fetch_andnot_relaxed(i, v);
2496 } 1012 }
2497 1013
2498 /** <<
2499 * atomic64_or() - atomic bitwise OR with rel <<
2500 * @i: s64 value <<
2501 * @v: pointer to atomic64_t <<
2502 * <<
2503 * Atomically updates @v to (@v | @i) with re <<
2504 * <<
2505 * Unsafe to use in noinstr code; use raw_ato <<
2506 * <<
2507 * Return: Nothing. <<
2508 */ <<
2509 static __always_inline void 1014 static __always_inline void
2510 atomic64_or(s64 i, atomic64_t *v) 1015 atomic64_or(s64 i, atomic64_t *v)
2511 { 1016 {
2512 instrument_atomic_read_write(v, sizeo 1017 instrument_atomic_read_write(v, sizeof(*v));
2513 raw_atomic64_or(i, v); !! 1018 arch_atomic64_or(i, v);
2514 } 1019 }
2515 1020
2516 /** <<
2517 * atomic64_fetch_or() - atomic bitwise OR wi <<
2518 * @i: s64 value <<
2519 * @v: pointer to atomic64_t <<
2520 * <<
2521 * Atomically updates @v to (@v | @i) with fu <<
2522 * <<
2523 * Unsafe to use in noinstr code; use raw_ato <<
2524 * <<
2525 * Return: The original value of @v. <<
2526 */ <<
2527 static __always_inline s64 1021 static __always_inline s64
2528 atomic64_fetch_or(s64 i, atomic64_t *v) 1022 atomic64_fetch_or(s64 i, atomic64_t *v)
2529 { 1023 {
2530 kcsan_mb(); 1024 kcsan_mb();
2531 instrument_atomic_read_write(v, sizeo 1025 instrument_atomic_read_write(v, sizeof(*v));
2532 return raw_atomic64_fetch_or(i, v); !! 1026 return arch_atomic64_fetch_or(i, v);
2533 } 1027 }
2534 1028
2535 /** <<
2536 * atomic64_fetch_or_acquire() - atomic bitwi <<
2537 * @i: s64 value <<
2538 * @v: pointer to atomic64_t <<
2539 * <<
2540 * Atomically updates @v to (@v | @i) with ac <<
2541 * <<
2542 * Unsafe to use in noinstr code; use raw_ato <<
2543 * <<
2544 * Return: The original value of @v. <<
2545 */ <<
2546 static __always_inline s64 1029 static __always_inline s64
2547 atomic64_fetch_or_acquire(s64 i, atomic64_t * 1030 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2548 { 1031 {
2549 instrument_atomic_read_write(v, sizeo 1032 instrument_atomic_read_write(v, sizeof(*v));
2550 return raw_atomic64_fetch_or_acquire( !! 1033 return arch_atomic64_fetch_or_acquire(i, v);
2551 } 1034 }
2552 1035
2553 /** <<
2554 * atomic64_fetch_or_release() - atomic bitwi <<
2555 * @i: s64 value <<
2556 * @v: pointer to atomic64_t <<
2557 * <<
2558 * Atomically updates @v to (@v | @i) with re <<
2559 * <<
2560 * Unsafe to use in noinstr code; use raw_ato <<
2561 * <<
2562 * Return: The original value of @v. <<
2563 */ <<
2564 static __always_inline s64 1036 static __always_inline s64
2565 atomic64_fetch_or_release(s64 i, atomic64_t * 1037 atomic64_fetch_or_release(s64 i, atomic64_t *v)
2566 { 1038 {
2567 kcsan_release(); 1039 kcsan_release();
2568 instrument_atomic_read_write(v, sizeo 1040 instrument_atomic_read_write(v, sizeof(*v));
2569 return raw_atomic64_fetch_or_release( !! 1041 return arch_atomic64_fetch_or_release(i, v);
2570 } 1042 }
2571 1043
2572 /** <<
2573 * atomic64_fetch_or_relaxed() - atomic bitwi <<
2574 * @i: s64 value <<
2575 * @v: pointer to atomic64_t <<
2576 * <<
2577 * Atomically updates @v to (@v | @i) with re <<
2578 * <<
2579 * Unsafe to use in noinstr code; use raw_ato <<
2580 * <<
2581 * Return: The original value of @v. <<
2582 */ <<
2583 static __always_inline s64 1044 static __always_inline s64
2584 atomic64_fetch_or_relaxed(s64 i, atomic64_t * 1045 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
2585 { 1046 {
2586 instrument_atomic_read_write(v, sizeo 1047 instrument_atomic_read_write(v, sizeof(*v));
2587 return raw_atomic64_fetch_or_relaxed( !! 1048 return arch_atomic64_fetch_or_relaxed(i, v);
2588 } 1049 }
2589 1050
2590 /** <<
2591 * atomic64_xor() - atomic bitwise XOR with r <<
2592 * @i: s64 value <<
2593 * @v: pointer to atomic64_t <<
2594 * <<
2595 * Atomically updates @v to (@v ^ @i) with re <<
2596 * <<
2597 * Unsafe to use in noinstr code; use raw_ato <<
2598 * <<
2599 * Return: Nothing. <<
2600 */ <<
2601 static __always_inline void 1051 static __always_inline void
2602 atomic64_xor(s64 i, atomic64_t *v) 1052 atomic64_xor(s64 i, atomic64_t *v)
2603 { 1053 {
2604 instrument_atomic_read_write(v, sizeo 1054 instrument_atomic_read_write(v, sizeof(*v));
2605 raw_atomic64_xor(i, v); !! 1055 arch_atomic64_xor(i, v);
2606 } 1056 }
2607 1057
2608 /** <<
2609 * atomic64_fetch_xor() - atomic bitwise XOR <<
2610 * @i: s64 value <<
2611 * @v: pointer to atomic64_t <<
2612 * <<
2613 * Atomically updates @v to (@v ^ @i) with fu <<
2614 * <<
2615 * Unsafe to use in noinstr code; use raw_ato <<
2616 * <<
2617 * Return: The original value of @v. <<
2618 */ <<
2619 static __always_inline s64 1058 static __always_inline s64
2620 atomic64_fetch_xor(s64 i, atomic64_t *v) 1059 atomic64_fetch_xor(s64 i, atomic64_t *v)
2621 { 1060 {
2622 kcsan_mb(); 1061 kcsan_mb();
2623 instrument_atomic_read_write(v, sizeo 1062 instrument_atomic_read_write(v, sizeof(*v));
2624 return raw_atomic64_fetch_xor(i, v); !! 1063 return arch_atomic64_fetch_xor(i, v);
2625 } 1064 }
2626 1065
2627 /** <<
2628 * atomic64_fetch_xor_acquire() - atomic bitw <<
2629 * @i: s64 value <<
2630 * @v: pointer to atomic64_t <<
2631 * <<
2632 * Atomically updates @v to (@v ^ @i) with ac <<
2633 * <<
2634 * Unsafe to use in noinstr code; use raw_ato <<
2635 * <<
2636 * Return: The original value of @v. <<
2637 */ <<
2638 static __always_inline s64 1066 static __always_inline s64
2639 atomic64_fetch_xor_acquire(s64 i, atomic64_t 1067 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2640 { 1068 {
2641 instrument_atomic_read_write(v, sizeo 1069 instrument_atomic_read_write(v, sizeof(*v));
2642 return raw_atomic64_fetch_xor_acquire !! 1070 return arch_atomic64_fetch_xor_acquire(i, v);
2643 } 1071 }
2644 1072
2645 /** <<
2646 * atomic64_fetch_xor_release() - atomic bitw <<
2647 * @i: s64 value <<
2648 * @v: pointer to atomic64_t <<
2649 * <<
2650 * Atomically updates @v to (@v ^ @i) with re <<
2651 * <<
2652 * Unsafe to use in noinstr code; use raw_ato <<
2653 * <<
2654 * Return: The original value of @v. <<
2655 */ <<
2656 static __always_inline s64 1073 static __always_inline s64
2657 atomic64_fetch_xor_release(s64 i, atomic64_t 1074 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2658 { 1075 {
2659 kcsan_release(); 1076 kcsan_release();
2660 instrument_atomic_read_write(v, sizeo 1077 instrument_atomic_read_write(v, sizeof(*v));
2661 return raw_atomic64_fetch_xor_release !! 1078 return arch_atomic64_fetch_xor_release(i, v);
2662 } 1079 }
2663 1080
2664 /** <<
2665 * atomic64_fetch_xor_relaxed() - atomic bitw <<
2666 * @i: s64 value <<
2667 * @v: pointer to atomic64_t <<
2668 * <<
2669 * Atomically updates @v to (@v ^ @i) with re <<
2670 * <<
2671 * Unsafe to use in noinstr code; use raw_ato <<
2672 * <<
2673 * Return: The original value of @v. <<
2674 */ <<
2675 static __always_inline s64 1081 static __always_inline s64
2676 atomic64_fetch_xor_relaxed(s64 i, atomic64_t 1082 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
2677 { 1083 {
2678 instrument_atomic_read_write(v, sizeo 1084 instrument_atomic_read_write(v, sizeof(*v));
2679 return raw_atomic64_fetch_xor_relaxed !! 1085 return arch_atomic64_fetch_xor_relaxed(i, v);
2680 } 1086 }
2681 1087
2682 /** <<
2683 * atomic64_xchg() - atomic exchange with ful <<
2684 * @v: pointer to atomic64_t <<
2685 * @new: s64 value to assign <<
2686 * <<
2687 * Atomically updates @v to @new with full or <<
2688 * <<
2689 * Unsafe to use in noinstr code; use raw_ato <<
2690 * <<
2691 * Return: The original value of @v. <<
2692 */ <<
2693 static __always_inline s64 1088 static __always_inline s64
2694 atomic64_xchg(atomic64_t *v, s64 new) !! 1089 atomic64_xchg(atomic64_t *v, s64 i)
2695 { 1090 {
2696 kcsan_mb(); 1091 kcsan_mb();
2697 instrument_atomic_read_write(v, sizeo 1092 instrument_atomic_read_write(v, sizeof(*v));
2698 return raw_atomic64_xchg(v, new); !! 1093 return arch_atomic64_xchg(v, i);
2699 } 1094 }
2700 1095
2701 /** <<
2702 * atomic64_xchg_acquire() - atomic exchange <<
2703 * @v: pointer to atomic64_t <<
2704 * @new: s64 value to assign <<
2705 * <<
2706 * Atomically updates @v to @new with acquire <<
2707 * <<
2708 * Unsafe to use in noinstr code; use raw_ato <<
2709 * <<
2710 * Return: The original value of @v. <<
2711 */ <<
2712 static __always_inline s64 1096 static __always_inline s64
2713 atomic64_xchg_acquire(atomic64_t *v, s64 new) !! 1097 atomic64_xchg_acquire(atomic64_t *v, s64 i)
2714 { 1098 {
2715 instrument_atomic_read_write(v, sizeo 1099 instrument_atomic_read_write(v, sizeof(*v));
2716 return raw_atomic64_xchg_acquire(v, n !! 1100 return arch_atomic64_xchg_acquire(v, i);
2717 } 1101 }
2718 1102
2719 /** <<
2720 * atomic64_xchg_release() - atomic exchange <<
2721 * @v: pointer to atomic64_t <<
2722 * @new: s64 value to assign <<
2723 * <<
2724 * Atomically updates @v to @new with release <<
2725 * <<
2726 * Unsafe to use in noinstr code; use raw_ato <<
2727 * <<
2728 * Return: The original value of @v. <<
2729 */ <<
2730 static __always_inline s64 1103 static __always_inline s64
2731 atomic64_xchg_release(atomic64_t *v, s64 new) !! 1104 atomic64_xchg_release(atomic64_t *v, s64 i)
2732 { 1105 {
2733 kcsan_release(); 1106 kcsan_release();
2734 instrument_atomic_read_write(v, sizeo 1107 instrument_atomic_read_write(v, sizeof(*v));
2735 return raw_atomic64_xchg_release(v, n !! 1108 return arch_atomic64_xchg_release(v, i);
2736 } 1109 }
2737 1110
2738 /** <<
2739 * atomic64_xchg_relaxed() - atomic exchange <<
2740 * @v: pointer to atomic64_t <<
2741 * @new: s64 value to assign <<
2742 * <<
2743 * Atomically updates @v to @new with relaxed <<
2744 * <<
2745 * Unsafe to use in noinstr code; use raw_ato <<
2746 * <<
2747 * Return: The original value of @v. <<
2748 */ <<
2749 static __always_inline s64 1111 static __always_inline s64
2750 atomic64_xchg_relaxed(atomic64_t *v, s64 new) !! 1112 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
2751 { 1113 {
2752 instrument_atomic_read_write(v, sizeo 1114 instrument_atomic_read_write(v, sizeof(*v));
2753 return raw_atomic64_xchg_relaxed(v, n !! 1115 return arch_atomic64_xchg_relaxed(v, i);
2754 } 1116 }
2755 1117
2756 /** <<
2757 * atomic64_cmpxchg() - atomic compare and ex <<
2758 * @v: pointer to atomic64_t <<
2759 * @old: s64 value to compare with <<
2760 * @new: s64 value to assign <<
2761 * <<
2762 * If (@v == @old), atomically updates @v to <<
2763 * Otherwise, @v is not modified and relaxed <<
2764 * <<
2765 * Unsafe to use in noinstr code; use raw_ato <<
2766 * <<
2767 * Return: The original value of @v. <<
2768 */ <<
2769 static __always_inline s64 1118 static __always_inline s64
2770 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 1119 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2771 { 1120 {
2772 kcsan_mb(); 1121 kcsan_mb();
2773 instrument_atomic_read_write(v, sizeo 1122 instrument_atomic_read_write(v, sizeof(*v));
2774 return raw_atomic64_cmpxchg(v, old, n !! 1123 return arch_atomic64_cmpxchg(v, old, new);
2775 } 1124 }
2776 1125
2777 /** <<
2778 * atomic64_cmpxchg_acquire() - atomic compar <<
2779 * @v: pointer to atomic64_t <<
2780 * @old: s64 value to compare with <<
2781 * @new: s64 value to assign <<
2782 * <<
2783 * If (@v == @old), atomically updates @v to <<
2784 * Otherwise, @v is not modified and relaxed <<
2785 * <<
2786 * Unsafe to use in noinstr code; use raw_ato <<
2787 * <<
2788 * Return: The original value of @v. <<
2789 */ <<
2790 static __always_inline s64 1126 static __always_inline s64
2791 atomic64_cmpxchg_acquire(atomic64_t *v, s64 o 1127 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2792 { 1128 {
2793 instrument_atomic_read_write(v, sizeo 1129 instrument_atomic_read_write(v, sizeof(*v));
2794 return raw_atomic64_cmpxchg_acquire(v !! 1130 return arch_atomic64_cmpxchg_acquire(v, old, new);
2795 } 1131 }
2796 1132
2797 /** <<
2798 * atomic64_cmpxchg_release() - atomic compar <<
2799 * @v: pointer to atomic64_t <<
2800 * @old: s64 value to compare with <<
2801 * @new: s64 value to assign <<
2802 * <<
2803 * If (@v == @old), atomically updates @v to <<
2804 * Otherwise, @v is not modified and relaxed <<
2805 * <<
2806 * Unsafe to use in noinstr code; use raw_ato <<
2807 * <<
2808 * Return: The original value of @v. <<
2809 */ <<
2810 static __always_inline s64 1133 static __always_inline s64
2811 atomic64_cmpxchg_release(atomic64_t *v, s64 o 1134 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2812 { 1135 {
2813 kcsan_release(); 1136 kcsan_release();
2814 instrument_atomic_read_write(v, sizeo 1137 instrument_atomic_read_write(v, sizeof(*v));
2815 return raw_atomic64_cmpxchg_release(v !! 1138 return arch_atomic64_cmpxchg_release(v, old, new);
2816 } 1139 }
2817 1140
2818 /** <<
2819 * atomic64_cmpxchg_relaxed() - atomic compar <<
2820 * @v: pointer to atomic64_t <<
2821 * @old: s64 value to compare with <<
2822 * @new: s64 value to assign <<
2823 * <<
2824 * If (@v == @old), atomically updates @v to <<
2825 * Otherwise, @v is not modified and relaxed <<
2826 * <<
2827 * Unsafe to use in noinstr code; use raw_ato <<
2828 * <<
2829 * Return: The original value of @v. <<
2830 */ <<
2831 static __always_inline s64 1141 static __always_inline s64
2832 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 o 1142 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
2833 { 1143 {
2834 instrument_atomic_read_write(v, sizeo 1144 instrument_atomic_read_write(v, sizeof(*v));
2835 return raw_atomic64_cmpxchg_relaxed(v !! 1145 return arch_atomic64_cmpxchg_relaxed(v, old, new);
2836 } 1146 }
2837 1147
2838 /** <<
2839 * atomic64_try_cmpxchg() - atomic compare an <<
2840 * @v: pointer to atomic64_t <<
2841 * @old: pointer to s64 value to compare with <<
2842 * @new: s64 value to assign <<
2843 * <<
2844 * If (@v == @old), atomically updates @v to <<
2845 * Otherwise, @v is not modified, @old is upd <<
2846 * and relaxed ordering is provided. <<
2847 * <<
2848 * Unsafe to use in noinstr code; use raw_ato <<
2849 * <<
2850 * Return: @true if the exchange occured, @fa <<
2851 */ <<
2852 static __always_inline bool 1148 static __always_inline bool
2853 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, 1149 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2854 { 1150 {
2855 kcsan_mb(); 1151 kcsan_mb();
2856 instrument_atomic_read_write(v, sizeo 1152 instrument_atomic_read_write(v, sizeof(*v));
2857 instrument_atomic_read_write(old, siz 1153 instrument_atomic_read_write(old, sizeof(*old));
2858 return raw_atomic64_try_cmpxchg(v, ol !! 1154 return arch_atomic64_try_cmpxchg(v, old, new);
2859 } 1155 }
2860 1156
2861 /** <<
2862 * atomic64_try_cmpxchg_acquire() - atomic co <<
2863 * @v: pointer to atomic64_t <<
2864 * @old: pointer to s64 value to compare with <<
2865 * @new: s64 value to assign <<
2866 * <<
2867 * If (@v == @old), atomically updates @v to <<
2868 * Otherwise, @v is not modified, @old is upd <<
2869 * and relaxed ordering is provided. <<
2870 * <<
2871 * Unsafe to use in noinstr code; use raw_ato <<
2872 * <<
2873 * Return: @true if the exchange occured, @fa <<
2874 */ <<
2875 static __always_inline bool 1157 static __always_inline bool
2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s 1158 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2877 { 1159 {
2878 instrument_atomic_read_write(v, sizeo 1160 instrument_atomic_read_write(v, sizeof(*v));
2879 instrument_atomic_read_write(old, siz 1161 instrument_atomic_read_write(old, sizeof(*old));
2880 return raw_atomic64_try_cmpxchg_acqui !! 1162 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
2881 } 1163 }
2882 1164
2883 /** <<
2884 * atomic64_try_cmpxchg_release() - atomic co <<
2885 * @v: pointer to atomic64_t <<
2886 * @old: pointer to s64 value to compare with <<
2887 * @new: s64 value to assign <<
2888 * <<
2889 * If (@v == @old), atomically updates @v to <<
2890 * Otherwise, @v is not modified, @old is upd <<
2891 * and relaxed ordering is provided. <<
2892 * <<
2893 * Unsafe to use in noinstr code; use raw_ato <<
2894 * <<
2895 * Return: @true if the exchange occured, @fa <<
2896 */ <<
2897 static __always_inline bool 1165 static __always_inline bool
2898 atomic64_try_cmpxchg_release(atomic64_t *v, s 1166 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2899 { 1167 {
2900 kcsan_release(); 1168 kcsan_release();
2901 instrument_atomic_read_write(v, sizeo 1169 instrument_atomic_read_write(v, sizeof(*v));
2902 instrument_atomic_read_write(old, siz 1170 instrument_atomic_read_write(old, sizeof(*old));
2903 return raw_atomic64_try_cmpxchg_relea !! 1171 return arch_atomic64_try_cmpxchg_release(v, old, new);
2904 } 1172 }
2905 1173
2906 /** <<
2907 * atomic64_try_cmpxchg_relaxed() - atomic co <<
2908 * @v: pointer to atomic64_t <<
2909 * @old: pointer to s64 value to compare with <<
2910 * @new: s64 value to assign <<
2911 * <<
2912 * If (@v == @old), atomically updates @v to <<
2913 * Otherwise, @v is not modified, @old is upd <<
2914 * and relaxed ordering is provided. <<
2915 * <<
2916 * Unsafe to use in noinstr code; use raw_ato <<
2917 * <<
2918 * Return: @true if the exchange occured, @fa <<
2919 */ <<
2920 static __always_inline bool 1174 static __always_inline bool
2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s 1175 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2922 { 1176 {
2923 instrument_atomic_read_write(v, sizeo 1177 instrument_atomic_read_write(v, sizeof(*v));
2924 instrument_atomic_read_write(old, siz 1178 instrument_atomic_read_write(old, sizeof(*old));
2925 return raw_atomic64_try_cmpxchg_relax !! 1179 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2926 } 1180 }
2927 1181
2928 /** <<
2929 * atomic64_sub_and_test() - atomic subtract <<
2930 * @i: s64 value to subtract <<
2931 * @v: pointer to atomic64_t <<
2932 * <<
2933 * Atomically updates @v to (@v - @i) with fu <<
2934 * <<
2935 * Unsafe to use in noinstr code; use raw_ato <<
2936 * <<
2937 * Return: @true if the resulting value of @v <<
2938 */ <<
2939 static __always_inline bool 1182 static __always_inline bool
2940 atomic64_sub_and_test(s64 i, atomic64_t *v) 1183 atomic64_sub_and_test(s64 i, atomic64_t *v)
2941 { 1184 {
2942 kcsan_mb(); 1185 kcsan_mb();
2943 instrument_atomic_read_write(v, sizeo 1186 instrument_atomic_read_write(v, sizeof(*v));
2944 return raw_atomic64_sub_and_test(i, v !! 1187 return arch_atomic64_sub_and_test(i, v);
2945 } 1188 }
2946 1189
2947 /** <<
2948 * atomic64_dec_and_test() - atomic decrement <<
2949 * @v: pointer to atomic64_t <<
2950 * <<
2951 * Atomically updates @v to (@v - 1) with ful <<
2952 * <<
2953 * Unsafe to use in noinstr code; use raw_ato <<
2954 * <<
2955 * Return: @true if the resulting value of @v <<
2956 */ <<
2957 static __always_inline bool 1190 static __always_inline bool
2958 atomic64_dec_and_test(atomic64_t *v) 1191 atomic64_dec_and_test(atomic64_t *v)
2959 { 1192 {
2960 kcsan_mb(); 1193 kcsan_mb();
2961 instrument_atomic_read_write(v, sizeo 1194 instrument_atomic_read_write(v, sizeof(*v));
2962 return raw_atomic64_dec_and_test(v); !! 1195 return arch_atomic64_dec_and_test(v);
2963 } 1196 }
2964 1197
2965 /** <<
2966 * atomic64_inc_and_test() - atomic increment <<
2967 * @v: pointer to atomic64_t <<
2968 * <<
2969 * Atomically updates @v to (@v + 1) with ful <<
2970 * <<
2971 * Unsafe to use in noinstr code; use raw_ato <<
2972 * <<
2973 * Return: @true if the resulting value of @v <<
2974 */ <<
2975 static __always_inline bool 1198 static __always_inline bool
2976 atomic64_inc_and_test(atomic64_t *v) 1199 atomic64_inc_and_test(atomic64_t *v)
2977 { 1200 {
2978 kcsan_mb(); 1201 kcsan_mb();
2979 instrument_atomic_read_write(v, sizeo 1202 instrument_atomic_read_write(v, sizeof(*v));
2980 return raw_atomic64_inc_and_test(v); !! 1203 return arch_atomic64_inc_and_test(v);
2981 } 1204 }
2982 1205
2983 /** <<
2984 * atomic64_add_negative() - atomic add and t <<
2985 * @i: s64 value to add <<
2986 * @v: pointer to atomic64_t <<
2987 * <<
2988 * Atomically updates @v to (@v + @i) with fu <<
2989 * <<
2990 * Unsafe to use in noinstr code; use raw_ato <<
2991 * <<
2992 * Return: @true if the resulting value of @v <<
2993 */ <<
2994 static __always_inline bool 1206 static __always_inline bool
2995 atomic64_add_negative(s64 i, atomic64_t *v) 1207 atomic64_add_negative(s64 i, atomic64_t *v)
2996 { 1208 {
2997 kcsan_mb(); 1209 kcsan_mb();
2998 instrument_atomic_read_write(v, sizeo 1210 instrument_atomic_read_write(v, sizeof(*v));
2999 return raw_atomic64_add_negative(i, v !! 1211 return arch_atomic64_add_negative(i, v);
3000 } <<
3001 <<
3002 /** <<
3003 * atomic64_add_negative_acquire() - atomic a <<
3004 * @i: s64 value to add <<
3005 * @v: pointer to atomic64_t <<
3006 * <<
3007 * Atomically updates @v to (@v + @i) with ac <<
3008 * <<
3009 * Unsafe to use in noinstr code; use raw_ato <<
3010 * <<
3011 * Return: @true if the resulting value of @v <<
3012 */ <<
3013 static __always_inline bool <<
3014 atomic64_add_negative_acquire(s64 i, atomic64 <<
3015 { <<
3016 instrument_atomic_read_write(v, sizeo <<
3017 return raw_atomic64_add_negative_acqu <<
3018 } <<
3019 <<
3020 /** <<
3021 * atomic64_add_negative_release() - atomic a <<
3022 * @i: s64 value to add <<
3023 * @v: pointer to atomic64_t <<
3024 * <<
3025 * Atomically updates @v to (@v + @i) with re <<
3026 * <<
3027 * Unsafe to use in noinstr code; use raw_ato <<
3028 * <<
3029 * Return: @true if the resulting value of @v <<
3030 */ <<
3031 static __always_inline bool <<
3032 atomic64_add_negative_release(s64 i, atomic64 <<
3033 { <<
3034 kcsan_release(); <<
3035 instrument_atomic_read_write(v, sizeo <<
3036 return raw_atomic64_add_negative_rele <<
3037 } 1212 }
3038 1213
3039 /** <<
3040 * atomic64_add_negative_relaxed() - atomic a <<
3041 * @i: s64 value to add <<
3042 * @v: pointer to atomic64_t <<
3043 * <<
3044 * Atomically updates @v to (@v + @i) with re <<
3045 * <<
3046 * Unsafe to use in noinstr code; use raw_ato <<
3047 * <<
3048 * Return: @true if the resulting value of @v <<
3049 */ <<
3050 static __always_inline bool <<
3051 atomic64_add_negative_relaxed(s64 i, atomic64 <<
3052 { <<
3053 instrument_atomic_read_write(v, sizeo <<
3054 return raw_atomic64_add_negative_rela <<
3055 } <<
3056 <<
3057 /** <<
3058 * atomic64_fetch_add_unless() - atomic add u <<
3059 * @v: pointer to atomic64_t <<
3060 * @a: s64 value to add <<
3061 * @u: s64 value to compare with <<
3062 * <<
3063 * If (@v != @u), atomically updates @v to (@ <<
3064 * Otherwise, @v is not modified and relaxed <<
3065 * <<
3066 * Unsafe to use in noinstr code; use raw_ato <<
3067 * <<
3068 * Return: The original value of @v. <<
3069 */ <<
3070 static __always_inline s64 1214 static __always_inline s64
3071 atomic64_fetch_add_unless(atomic64_t *v, s64 1215 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
3072 { 1216 {
3073 kcsan_mb(); 1217 kcsan_mb();
3074 instrument_atomic_read_write(v, sizeo 1218 instrument_atomic_read_write(v, sizeof(*v));
3075 return raw_atomic64_fetch_add_unless( !! 1219 return arch_atomic64_fetch_add_unless(v, a, u);
3076 } 1220 }
3077 1221
3078 /** <<
3079 * atomic64_add_unless() - atomic add unless <<
3080 * @v: pointer to atomic64_t <<
3081 * @a: s64 value to add <<
3082 * @u: s64 value to compare with <<
3083 * <<
3084 * If (@v != @u), atomically updates @v to (@ <<
3085 * Otherwise, @v is not modified and relaxed <<
3086 * <<
3087 * Unsafe to use in noinstr code; use raw_ato <<
3088 * <<
3089 * Return: @true if @v was updated, @false ot <<
3090 */ <<
3091 static __always_inline bool 1222 static __always_inline bool
3092 atomic64_add_unless(atomic64_t *v, s64 a, s64 1223 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
3093 { 1224 {
3094 kcsan_mb(); 1225 kcsan_mb();
3095 instrument_atomic_read_write(v, sizeo 1226 instrument_atomic_read_write(v, sizeof(*v));
3096 return raw_atomic64_add_unless(v, a, !! 1227 return arch_atomic64_add_unless(v, a, u);
3097 } 1228 }
3098 1229
3099 /** <<
3100 * atomic64_inc_not_zero() - atomic increment <<
3101 * @v: pointer to atomic64_t <<
3102 * <<
3103 * If (@v != 0), atomically updates @v to (@v <<
3104 * Otherwise, @v is not modified and relaxed <<
3105 * <<
3106 * Unsafe to use in noinstr code; use raw_ato <<
3107 * <<
3108 * Return: @true if @v was updated, @false ot <<
3109 */ <<
3110 static __always_inline bool 1230 static __always_inline bool
3111 atomic64_inc_not_zero(atomic64_t *v) 1231 atomic64_inc_not_zero(atomic64_t *v)
3112 { 1232 {
3113 kcsan_mb(); 1233 kcsan_mb();
3114 instrument_atomic_read_write(v, sizeo 1234 instrument_atomic_read_write(v, sizeof(*v));
3115 return raw_atomic64_inc_not_zero(v); !! 1235 return arch_atomic64_inc_not_zero(v);
3116 } 1236 }
3117 1237
3118 /** <<
3119 * atomic64_inc_unless_negative() - atomic in <<
3120 * @v: pointer to atomic64_t <<
3121 * <<
3122 * If (@v >= 0), atomically updates @v to (@v <<
3123 * Otherwise, @v is not modified and relaxed <<
3124 * <<
3125 * Unsafe to use in noinstr code; use raw_ato <<
3126 * <<
3127 * Return: @true if @v was updated, @false ot <<
3128 */ <<
3129 static __always_inline bool 1238 static __always_inline bool
3130 atomic64_inc_unless_negative(atomic64_t *v) 1239 atomic64_inc_unless_negative(atomic64_t *v)
3131 { 1240 {
3132 kcsan_mb(); 1241 kcsan_mb();
3133 instrument_atomic_read_write(v, sizeo 1242 instrument_atomic_read_write(v, sizeof(*v));
3134 return raw_atomic64_inc_unless_negati !! 1243 return arch_atomic64_inc_unless_negative(v);
3135 } 1244 }
3136 1245
3137 /** <<
3138 * atomic64_dec_unless_positive() - atomic de <<
3139 * @v: pointer to atomic64_t <<
3140 * <<
3141 * If (@v <= 0), atomically updates @v to (@v <<
3142 * Otherwise, @v is not modified and relaxed <<
3143 * <<
3144 * Unsafe to use in noinstr code; use raw_ato <<
3145 * <<
3146 * Return: @true if @v was updated, @false ot <<
3147 */ <<
3148 static __always_inline bool 1246 static __always_inline bool
3149 atomic64_dec_unless_positive(atomic64_t *v) 1247 atomic64_dec_unless_positive(atomic64_t *v)
3150 { 1248 {
3151 kcsan_mb(); 1249 kcsan_mb();
3152 instrument_atomic_read_write(v, sizeo 1250 instrument_atomic_read_write(v, sizeof(*v));
3153 return raw_atomic64_dec_unless_positi !! 1251 return arch_atomic64_dec_unless_positive(v);
3154 } 1252 }
3155 1253
3156 /** <<
3157 * atomic64_dec_if_positive() - atomic decrem <<
3158 * @v: pointer to atomic64_t <<
3159 * <<
3160 * If (@v > 0), atomically updates @v to (@v <<
3161 * Otherwise, @v is not modified and relaxed <<
3162 * <<
3163 * Unsafe to use in noinstr code; use raw_ato <<
3164 * <<
3165 * Return: The old value of (@v - 1), regardl <<
3166 */ <<
3167 static __always_inline s64 1254 static __always_inline s64
3168 atomic64_dec_if_positive(atomic64_t *v) 1255 atomic64_dec_if_positive(atomic64_t *v)
3169 { 1256 {
3170 kcsan_mb(); 1257 kcsan_mb();
3171 instrument_atomic_read_write(v, sizeo 1258 instrument_atomic_read_write(v, sizeof(*v));
3172 return raw_atomic64_dec_if_positive(v !! 1259 return arch_atomic64_dec_if_positive(v);
3173 } 1260 }
3174 1261
3175 /** <<
3176 * atomic_long_read() - atomic load with rela <<
3177 * @v: pointer to atomic_long_t <<
3178 * <<
3179 * Atomically loads the value of @v with rela <<
3180 * <<
3181 * Unsafe to use in noinstr code; use raw_ato <<
3182 * <<
3183 * Return: The value loaded from @v. <<
3184 */ <<
3185 static __always_inline long 1262 static __always_inline long
3186 atomic_long_read(const atomic_long_t *v) 1263 atomic_long_read(const atomic_long_t *v)
3187 { 1264 {
3188 instrument_atomic_read(v, sizeof(*v)) 1265 instrument_atomic_read(v, sizeof(*v));
3189 return raw_atomic_long_read(v); !! 1266 return arch_atomic_long_read(v);
3190 } 1267 }
3191 1268
3192 /** <<
3193 * atomic_long_read_acquire() - atomic load w <<
3194 * @v: pointer to atomic_long_t <<
3195 * <<
3196 * Atomically loads the value of @v with acqu <<
3197 * <<
3198 * Unsafe to use in noinstr code; use raw_ato <<
3199 * <<
3200 * Return: The value loaded from @v. <<
3201 */ <<
3202 static __always_inline long 1269 static __always_inline long
3203 atomic_long_read_acquire(const atomic_long_t 1270 atomic_long_read_acquire(const atomic_long_t *v)
3204 { 1271 {
3205 instrument_atomic_read(v, sizeof(*v)) 1272 instrument_atomic_read(v, sizeof(*v));
3206 return raw_atomic_long_read_acquire(v !! 1273 return arch_atomic_long_read_acquire(v);
3207 } 1274 }
3208 1275
3209 /** <<
3210 * atomic_long_set() - atomic set with relaxe <<
3211 * @v: pointer to atomic_long_t <<
3212 * @i: long value to assign <<
3213 * <<
3214 * Atomically sets @v to @i with relaxed orde <<
3215 * <<
3216 * Unsafe to use in noinstr code; use raw_ato <<
3217 * <<
3218 * Return: Nothing. <<
3219 */ <<
3220 static __always_inline void 1276 static __always_inline void
3221 atomic_long_set(atomic_long_t *v, long i) 1277 atomic_long_set(atomic_long_t *v, long i)
3222 { 1278 {
3223 instrument_atomic_write(v, sizeof(*v) 1279 instrument_atomic_write(v, sizeof(*v));
3224 raw_atomic_long_set(v, i); !! 1280 arch_atomic_long_set(v, i);
3225 } 1281 }
3226 1282
3227 /** <<
3228 * atomic_long_set_release() - atomic set wit <<
3229 * @v: pointer to atomic_long_t <<
3230 * @i: long value to assign <<
3231 * <<
3232 * Atomically sets @v to @i with release orde <<
3233 * <<
3234 * Unsafe to use in noinstr code; use raw_ato <<
3235 * <<
3236 * Return: Nothing. <<
3237 */ <<
3238 static __always_inline void 1283 static __always_inline void
3239 atomic_long_set_release(atomic_long_t *v, lon 1284 atomic_long_set_release(atomic_long_t *v, long i)
3240 { 1285 {
3241 kcsan_release(); 1286 kcsan_release();
3242 instrument_atomic_write(v, sizeof(*v) 1287 instrument_atomic_write(v, sizeof(*v));
3243 raw_atomic_long_set_release(v, i); !! 1288 arch_atomic_long_set_release(v, i);
3244 } 1289 }
3245 1290
3246 /** <<
3247 * atomic_long_add() - atomic add with relaxe <<
3248 * @i: long value to add <<
3249 * @v: pointer to atomic_long_t <<
3250 * <<
3251 * Atomically updates @v to (@v + @i) with re <<
3252 * <<
3253 * Unsafe to use in noinstr code; use raw_ato <<
3254 * <<
3255 * Return: Nothing. <<
3256 */ <<
3257 static __always_inline void 1291 static __always_inline void
3258 atomic_long_add(long i, atomic_long_t *v) 1292 atomic_long_add(long i, atomic_long_t *v)
3259 { 1293 {
3260 instrument_atomic_read_write(v, sizeo 1294 instrument_atomic_read_write(v, sizeof(*v));
3261 raw_atomic_long_add(i, v); !! 1295 arch_atomic_long_add(i, v);
3262 } 1296 }
3263 1297
3264 /** <<
3265 * atomic_long_add_return() - atomic add with <<
3266 * @i: long value to add <<
3267 * @v: pointer to atomic_long_t <<
3268 * <<
3269 * Atomically updates @v to (@v + @i) with fu <<
3270 * <<
3271 * Unsafe to use in noinstr code; use raw_ato <<
3272 * <<
3273 * Return: The updated value of @v. <<
3274 */ <<
3275 static __always_inline long 1298 static __always_inline long
3276 atomic_long_add_return(long i, atomic_long_t 1299 atomic_long_add_return(long i, atomic_long_t *v)
3277 { 1300 {
3278 kcsan_mb(); 1301 kcsan_mb();
3279 instrument_atomic_read_write(v, sizeo 1302 instrument_atomic_read_write(v, sizeof(*v));
3280 return raw_atomic_long_add_return(i, !! 1303 return arch_atomic_long_add_return(i, v);
3281 } 1304 }
3282 1305
3283 /** <<
3284 * atomic_long_add_return_acquire() - atomic <<
3285 * @i: long value to add <<
3286 * @v: pointer to atomic_long_t <<
3287 * <<
3288 * Atomically updates @v to (@v + @i) with ac <<
3289 * <<
3290 * Unsafe to use in noinstr code; use raw_ato <<
3291 * <<
3292 * Return: The updated value of @v. <<
3293 */ <<
3294 static __always_inline long 1306 static __always_inline long
3295 atomic_long_add_return_acquire(long i, atomic 1307 atomic_long_add_return_acquire(long i, atomic_long_t *v)
3296 { 1308 {
3297 instrument_atomic_read_write(v, sizeo 1309 instrument_atomic_read_write(v, sizeof(*v));
3298 return raw_atomic_long_add_return_acq !! 1310 return arch_atomic_long_add_return_acquire(i, v);
3299 } 1311 }
3300 1312
3301 /** <<
3302 * atomic_long_add_return_release() - atomic <<
3303 * @i: long value to add <<
3304 * @v: pointer to atomic_long_t <<
3305 * <<
3306 * Atomically updates @v to (@v + @i) with re <<
3307 * <<
3308 * Unsafe to use in noinstr code; use raw_ato <<
3309 * <<
3310 * Return: The updated value of @v. <<
3311 */ <<
3312 static __always_inline long 1313 static __always_inline long
3313 atomic_long_add_return_release(long i, atomic 1314 atomic_long_add_return_release(long i, atomic_long_t *v)
3314 { 1315 {
3315 kcsan_release(); 1316 kcsan_release();
3316 instrument_atomic_read_write(v, sizeo 1317 instrument_atomic_read_write(v, sizeof(*v));
3317 return raw_atomic_long_add_return_rel !! 1318 return arch_atomic_long_add_return_release(i, v);
3318 } 1319 }
3319 1320
3320 /** <<
3321 * atomic_long_add_return_relaxed() - atomic <<
3322 * @i: long value to add <<
3323 * @v: pointer to atomic_long_t <<
3324 * <<
3325 * Atomically updates @v to (@v + @i) with re <<
3326 * <<
3327 * Unsafe to use in noinstr code; use raw_ato <<
3328 * <<
3329 * Return: The updated value of @v. <<
3330 */ <<
3331 static __always_inline long 1321 static __always_inline long
3332 atomic_long_add_return_relaxed(long i, atomic 1322 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
3333 { 1323 {
3334 instrument_atomic_read_write(v, sizeo 1324 instrument_atomic_read_write(v, sizeof(*v));
3335 return raw_atomic_long_add_return_rel !! 1325 return arch_atomic_long_add_return_relaxed(i, v);
3336 } 1326 }
3337 1327
3338 /** <<
3339 * atomic_long_fetch_add() - atomic add with <<
3340 * @i: long value to add <<
3341 * @v: pointer to atomic_long_t <<
3342 * <<
3343 * Atomically updates @v to (@v + @i) with fu <<
3344 * <<
3345 * Unsafe to use in noinstr code; use raw_ato <<
3346 * <<
3347 * Return: The original value of @v. <<
3348 */ <<
3349 static __always_inline long 1328 static __always_inline long
3350 atomic_long_fetch_add(long i, atomic_long_t * 1329 atomic_long_fetch_add(long i, atomic_long_t *v)
3351 { 1330 {
3352 kcsan_mb(); 1331 kcsan_mb();
3353 instrument_atomic_read_write(v, sizeo 1332 instrument_atomic_read_write(v, sizeof(*v));
3354 return raw_atomic_long_fetch_add(i, v !! 1333 return arch_atomic_long_fetch_add(i, v);
3355 } 1334 }
3356 1335
3357 /** <<
3358 * atomic_long_fetch_add_acquire() - atomic a <<
3359 * @i: long value to add <<
3360 * @v: pointer to atomic_long_t <<
3361 * <<
3362 * Atomically updates @v to (@v + @i) with ac <<
3363 * <<
3364 * Unsafe to use in noinstr code; use raw_ato <<
3365 * <<
3366 * Return: The original value of @v. <<
3367 */ <<
3368 static __always_inline long 1336 static __always_inline long
3369 atomic_long_fetch_add_acquire(long i, atomic_ 1337 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
3370 { 1338 {
3371 instrument_atomic_read_write(v, sizeo 1339 instrument_atomic_read_write(v, sizeof(*v));
3372 return raw_atomic_long_fetch_add_acqu !! 1340 return arch_atomic_long_fetch_add_acquire(i, v);
3373 } 1341 }
3374 1342
3375 /** <<
3376 * atomic_long_fetch_add_release() - atomic a <<
3377 * @i: long value to add <<
3378 * @v: pointer to atomic_long_t <<
3379 * <<
3380 * Atomically updates @v to (@v + @i) with re <<
3381 * <<
3382 * Unsafe to use in noinstr code; use raw_ato <<
3383 * <<
3384 * Return: The original value of @v. <<
3385 */ <<
3386 static __always_inline long 1343 static __always_inline long
3387 atomic_long_fetch_add_release(long i, atomic_ 1344 atomic_long_fetch_add_release(long i, atomic_long_t *v)
3388 { 1345 {
3389 kcsan_release(); 1346 kcsan_release();
3390 instrument_atomic_read_write(v, sizeo 1347 instrument_atomic_read_write(v, sizeof(*v));
3391 return raw_atomic_long_fetch_add_rele !! 1348 return arch_atomic_long_fetch_add_release(i, v);
3392 } 1349 }
3393 1350
3394 /** <<
3395 * atomic_long_fetch_add_relaxed() - atomic a <<
3396 * @i: long value to add <<
3397 * @v: pointer to atomic_long_t <<
3398 * <<
3399 * Atomically updates @v to (@v + @i) with re <<
3400 * <<
3401 * Unsafe to use in noinstr code; use raw_ato <<
3402 * <<
3403 * Return: The original value of @v. <<
3404 */ <<
3405 static __always_inline long 1351 static __always_inline long
3406 atomic_long_fetch_add_relaxed(long i, atomic_ 1352 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
3407 { 1353 {
3408 instrument_atomic_read_write(v, sizeo 1354 instrument_atomic_read_write(v, sizeof(*v));
3409 return raw_atomic_long_fetch_add_rela !! 1355 return arch_atomic_long_fetch_add_relaxed(i, v);
3410 } 1356 }
3411 1357
3412 /** <<
3413 * atomic_long_sub() - atomic subtract with r <<
3414 * @i: long value to subtract <<
3415 * @v: pointer to atomic_long_t <<
3416 * <<
3417 * Atomically updates @v to (@v - @i) with re <<
3418 * <<
3419 * Unsafe to use in noinstr code; use raw_ato <<
3420 * <<
3421 * Return: Nothing. <<
3422 */ <<
3423 static __always_inline void 1358 static __always_inline void
3424 atomic_long_sub(long i, atomic_long_t *v) 1359 atomic_long_sub(long i, atomic_long_t *v)
3425 { 1360 {
3426 instrument_atomic_read_write(v, sizeo 1361 instrument_atomic_read_write(v, sizeof(*v));
3427 raw_atomic_long_sub(i, v); !! 1362 arch_atomic_long_sub(i, v);
3428 } 1363 }
3429 1364
3430 /** <<
3431 * atomic_long_sub_return() - atomic subtract <<
3432 * @i: long value to subtract <<
3433 * @v: pointer to atomic_long_t <<
3434 * <<
3435 * Atomically updates @v to (@v - @i) with fu <<
3436 * <<
3437 * Unsafe to use in noinstr code; use raw_ato <<
3438 * <<
3439 * Return: The updated value of @v. <<
3440 */ <<
3441 static __always_inline long 1365 static __always_inline long
3442 atomic_long_sub_return(long i, atomic_long_t 1366 atomic_long_sub_return(long i, atomic_long_t *v)
3443 { 1367 {
3444 kcsan_mb(); 1368 kcsan_mb();
3445 instrument_atomic_read_write(v, sizeo 1369 instrument_atomic_read_write(v, sizeof(*v));
3446 return raw_atomic_long_sub_return(i, !! 1370 return arch_atomic_long_sub_return(i, v);
3447 } 1371 }
3448 1372
3449 /** <<
3450 * atomic_long_sub_return_acquire() - atomic <<
3451 * @i: long value to subtract <<
3452 * @v: pointer to atomic_long_t <<
3453 * <<
3454 * Atomically updates @v to (@v - @i) with ac <<
3455 * <<
3456 * Unsafe to use in noinstr code; use raw_ato <<
3457 * <<
3458 * Return: The updated value of @v. <<
3459 */ <<
3460 static __always_inline long 1373 static __always_inline long
3461 atomic_long_sub_return_acquire(long i, atomic 1374 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
3462 { 1375 {
3463 instrument_atomic_read_write(v, sizeo 1376 instrument_atomic_read_write(v, sizeof(*v));
3464 return raw_atomic_long_sub_return_acq !! 1377 return arch_atomic_long_sub_return_acquire(i, v);
3465 } 1378 }
3466 1379
3467 /** <<
3468 * atomic_long_sub_return_release() - atomic <<
3469 * @i: long value to subtract <<
3470 * @v: pointer to atomic_long_t <<
3471 * <<
3472 * Atomically updates @v to (@v - @i) with re <<
3473 * <<
3474 * Unsafe to use in noinstr code; use raw_ato <<
3475 * <<
3476 * Return: The updated value of @v. <<
3477 */ <<
3478 static __always_inline long 1380 static __always_inline long
3479 atomic_long_sub_return_release(long i, atomic 1381 atomic_long_sub_return_release(long i, atomic_long_t *v)
3480 { 1382 {
3481 kcsan_release(); 1383 kcsan_release();
3482 instrument_atomic_read_write(v, sizeo 1384 instrument_atomic_read_write(v, sizeof(*v));
3483 return raw_atomic_long_sub_return_rel !! 1385 return arch_atomic_long_sub_return_release(i, v);
3484 } 1386 }
3485 1387
3486 /** <<
3487 * atomic_long_sub_return_relaxed() - atomic <<
3488 * @i: long value to subtract <<
3489 * @v: pointer to atomic_long_t <<
3490 * <<
3491 * Atomically updates @v to (@v - @i) with re <<
3492 * <<
3493 * Unsafe to use in noinstr code; use raw_ato <<
3494 * <<
3495 * Return: The updated value of @v. <<
3496 */ <<
3497 static __always_inline long 1388 static __always_inline long
3498 atomic_long_sub_return_relaxed(long i, atomic 1389 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
3499 { 1390 {
3500 instrument_atomic_read_write(v, sizeo 1391 instrument_atomic_read_write(v, sizeof(*v));
3501 return raw_atomic_long_sub_return_rel !! 1392 return arch_atomic_long_sub_return_relaxed(i, v);
3502 } 1393 }
3503 1394
3504 /** <<
3505 * atomic_long_fetch_sub() - atomic subtract <<
3506 * @i: long value to subtract <<
3507 * @v: pointer to atomic_long_t <<
3508 * <<
3509 * Atomically updates @v to (@v - @i) with fu <<
3510 * <<
3511 * Unsafe to use in noinstr code; use raw_ato <<
3512 * <<
3513 * Return: The original value of @v. <<
3514 */ <<
3515 static __always_inline long 1395 static __always_inline long
3516 atomic_long_fetch_sub(long i, atomic_long_t * 1396 atomic_long_fetch_sub(long i, atomic_long_t *v)
3517 { 1397 {
3518 kcsan_mb(); 1398 kcsan_mb();
3519 instrument_atomic_read_write(v, sizeo 1399 instrument_atomic_read_write(v, sizeof(*v));
3520 return raw_atomic_long_fetch_sub(i, v !! 1400 return arch_atomic_long_fetch_sub(i, v);
3521 } 1401 }
3522 1402
3523 /** <<
3524 * atomic_long_fetch_sub_acquire() - atomic s <<
3525 * @i: long value to subtract <<
3526 * @v: pointer to atomic_long_t <<
3527 * <<
3528 * Atomically updates @v to (@v - @i) with ac <<
3529 * <<
3530 * Unsafe to use in noinstr code; use raw_ato <<
3531 * <<
3532 * Return: The original value of @v. <<
3533 */ <<
3534 static __always_inline long 1403 static __always_inline long
3535 atomic_long_fetch_sub_acquire(long i, atomic_ 1404 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
3536 { 1405 {
3537 instrument_atomic_read_write(v, sizeo 1406 instrument_atomic_read_write(v, sizeof(*v));
3538 return raw_atomic_long_fetch_sub_acqu !! 1407 return arch_atomic_long_fetch_sub_acquire(i, v);
3539 } 1408 }
3540 1409
3541 /** <<
3542 * atomic_long_fetch_sub_release() - atomic s <<
3543 * @i: long value to subtract <<
3544 * @v: pointer to atomic_long_t <<
3545 * <<
3546 * Atomically updates @v to (@v - @i) with re <<
3547 * <<
3548 * Unsafe to use in noinstr code; use raw_ato <<
3549 * <<
3550 * Return: The original value of @v. <<
3551 */ <<
3552 static __always_inline long 1410 static __always_inline long
3553 atomic_long_fetch_sub_release(long i, atomic_ 1411 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
3554 { 1412 {
3555 kcsan_release(); 1413 kcsan_release();
3556 instrument_atomic_read_write(v, sizeo 1414 instrument_atomic_read_write(v, sizeof(*v));
3557 return raw_atomic_long_fetch_sub_rele !! 1415 return arch_atomic_long_fetch_sub_release(i, v);
3558 } 1416 }
3559 1417
3560 /** <<
3561 * atomic_long_fetch_sub_relaxed() - atomic s <<
3562 * @i: long value to subtract <<
3563 * @v: pointer to atomic_long_t <<
3564 * <<
3565 * Atomically updates @v to (@v - @i) with re <<
3566 * <<
3567 * Unsafe to use in noinstr code; use raw_ato <<
3568 * <<
3569 * Return: The original value of @v. <<
3570 */ <<
3571 static __always_inline long 1418 static __always_inline long
3572 atomic_long_fetch_sub_relaxed(long i, atomic_ 1419 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
3573 { 1420 {
3574 instrument_atomic_read_write(v, sizeo 1421 instrument_atomic_read_write(v, sizeof(*v));
3575 return raw_atomic_long_fetch_sub_rela !! 1422 return arch_atomic_long_fetch_sub_relaxed(i, v);
3576 } 1423 }
3577 1424
3578 /** <<
3579 * atomic_long_inc() - atomic increment with <<
3580 * @v: pointer to atomic_long_t <<
3581 * <<
3582 * Atomically updates @v to (@v + 1) with rel <<
3583 * <<
3584 * Unsafe to use in noinstr code; use raw_ato <<
3585 * <<
3586 * Return: Nothing. <<
3587 */ <<
3588 static __always_inline void 1425 static __always_inline void
3589 atomic_long_inc(atomic_long_t *v) 1426 atomic_long_inc(atomic_long_t *v)
3590 { 1427 {
3591 instrument_atomic_read_write(v, sizeo 1428 instrument_atomic_read_write(v, sizeof(*v));
3592 raw_atomic_long_inc(v); !! 1429 arch_atomic_long_inc(v);
3593 } 1430 }
3594 1431
3595 /** <<
3596 * atomic_long_inc_return() - atomic incremen <<
3597 * @v: pointer to atomic_long_t <<
3598 * <<
3599 * Atomically updates @v to (@v + 1) with ful <<
3600 * <<
3601 * Unsafe to use in noinstr code; use raw_ato <<
3602 * <<
3603 * Return: The updated value of @v. <<
3604 */ <<
3605 static __always_inline long 1432 static __always_inline long
3606 atomic_long_inc_return(atomic_long_t *v) 1433 atomic_long_inc_return(atomic_long_t *v)
3607 { 1434 {
3608 kcsan_mb(); 1435 kcsan_mb();
3609 instrument_atomic_read_write(v, sizeo 1436 instrument_atomic_read_write(v, sizeof(*v));
3610 return raw_atomic_long_inc_return(v); !! 1437 return arch_atomic_long_inc_return(v);
3611 } 1438 }
3612 1439
3613 /** <<
3614 * atomic_long_inc_return_acquire() - atomic <<
3615 * @v: pointer to atomic_long_t <<
3616 * <<
3617 * Atomically updates @v to (@v + 1) with acq <<
3618 * <<
3619 * Unsafe to use in noinstr code; use raw_ato <<
3620 * <<
3621 * Return: The updated value of @v. <<
3622 */ <<
3623 static __always_inline long 1440 static __always_inline long
3624 atomic_long_inc_return_acquire(atomic_long_t 1441 atomic_long_inc_return_acquire(atomic_long_t *v)
3625 { 1442 {
3626 instrument_atomic_read_write(v, sizeo 1443 instrument_atomic_read_write(v, sizeof(*v));
3627 return raw_atomic_long_inc_return_acq !! 1444 return arch_atomic_long_inc_return_acquire(v);
3628 } 1445 }
3629 1446
3630 /** <<
3631 * atomic_long_inc_return_release() - atomic <<
3632 * @v: pointer to atomic_long_t <<
3633 * <<
3634 * Atomically updates @v to (@v + 1) with rel <<
3635 * <<
3636 * Unsafe to use in noinstr code; use raw_ato <<
3637 * <<
3638 * Return: The updated value of @v. <<
3639 */ <<
3640 static __always_inline long 1447 static __always_inline long
3641 atomic_long_inc_return_release(atomic_long_t 1448 atomic_long_inc_return_release(atomic_long_t *v)
3642 { 1449 {
3643 kcsan_release(); 1450 kcsan_release();
3644 instrument_atomic_read_write(v, sizeo 1451 instrument_atomic_read_write(v, sizeof(*v));
3645 return raw_atomic_long_inc_return_rel !! 1452 return arch_atomic_long_inc_return_release(v);
3646 } 1453 }
3647 1454
3648 /** <<
3649 * atomic_long_inc_return_relaxed() - atomic <<
3650 * @v: pointer to atomic_long_t <<
3651 * <<
3652 * Atomically updates @v to (@v + 1) with rel <<
3653 * <<
3654 * Unsafe to use in noinstr code; use raw_ato <<
3655 * <<
3656 * Return: The updated value of @v. <<
3657 */ <<
3658 static __always_inline long 1455 static __always_inline long
3659 atomic_long_inc_return_relaxed(atomic_long_t 1456 atomic_long_inc_return_relaxed(atomic_long_t *v)
3660 { 1457 {
3661 instrument_atomic_read_write(v, sizeo 1458 instrument_atomic_read_write(v, sizeof(*v));
3662 return raw_atomic_long_inc_return_rel !! 1459 return arch_atomic_long_inc_return_relaxed(v);
3663 } 1460 }
3664 1461
3665 /** <<
3666 * atomic_long_fetch_inc() - atomic increment <<
3667 * @v: pointer to atomic_long_t <<
3668 * <<
3669 * Atomically updates @v to (@v + 1) with ful <<
3670 * <<
3671 * Unsafe to use in noinstr code; use raw_ato <<
3672 * <<
3673 * Return: The original value of @v. <<
3674 */ <<
3675 static __always_inline long 1462 static __always_inline long
3676 atomic_long_fetch_inc(atomic_long_t *v) 1463 atomic_long_fetch_inc(atomic_long_t *v)
3677 { 1464 {
3678 kcsan_mb(); 1465 kcsan_mb();
3679 instrument_atomic_read_write(v, sizeo 1466 instrument_atomic_read_write(v, sizeof(*v));
3680 return raw_atomic_long_fetch_inc(v); !! 1467 return arch_atomic_long_fetch_inc(v);
3681 } 1468 }
3682 1469
3683 /** <<
3684 * atomic_long_fetch_inc_acquire() - atomic i <<
3685 * @v: pointer to atomic_long_t <<
3686 * <<
3687 * Atomically updates @v to (@v + 1) with acq <<
3688 * <<
3689 * Unsafe to use in noinstr code; use raw_ato <<
3690 * <<
3691 * Return: The original value of @v. <<
3692 */ <<
3693 static __always_inline long 1470 static __always_inline long
3694 atomic_long_fetch_inc_acquire(atomic_long_t * 1471 atomic_long_fetch_inc_acquire(atomic_long_t *v)
3695 { 1472 {
3696 instrument_atomic_read_write(v, sizeo 1473 instrument_atomic_read_write(v, sizeof(*v));
3697 return raw_atomic_long_fetch_inc_acqu !! 1474 return arch_atomic_long_fetch_inc_acquire(v);
3698 } 1475 }
3699 1476
3700 /** <<
3701 * atomic_long_fetch_inc_release() - atomic i <<
3702 * @v: pointer to atomic_long_t <<
3703 * <<
3704 * Atomically updates @v to (@v + 1) with rel <<
3705 * <<
3706 * Unsafe to use in noinstr code; use raw_ato <<
3707 * <<
3708 * Return: The original value of @v. <<
3709 */ <<
3710 static __always_inline long 1477 static __always_inline long
3711 atomic_long_fetch_inc_release(atomic_long_t * 1478 atomic_long_fetch_inc_release(atomic_long_t *v)
3712 { 1479 {
3713 kcsan_release(); 1480 kcsan_release();
3714 instrument_atomic_read_write(v, sizeo 1481 instrument_atomic_read_write(v, sizeof(*v));
3715 return raw_atomic_long_fetch_inc_rele !! 1482 return arch_atomic_long_fetch_inc_release(v);
3716 } 1483 }
3717 1484
3718 /** <<
3719 * atomic_long_fetch_inc_relaxed() - atomic i <<
3720 * @v: pointer to atomic_long_t <<
3721 * <<
3722 * Atomically updates @v to (@v + 1) with rel <<
3723 * <<
3724 * Unsafe to use in noinstr code; use raw_ato <<
3725 * <<
3726 * Return: The original value of @v. <<
3727 */ <<
3728 static __always_inline long 1485 static __always_inline long
3729 atomic_long_fetch_inc_relaxed(atomic_long_t * 1486 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
3730 { 1487 {
3731 instrument_atomic_read_write(v, sizeo 1488 instrument_atomic_read_write(v, sizeof(*v));
3732 return raw_atomic_long_fetch_inc_rela !! 1489 return arch_atomic_long_fetch_inc_relaxed(v);
3733 } 1490 }
3734 1491
3735 /** <<
3736 * atomic_long_dec() - atomic decrement with <<
3737 * @v: pointer to atomic_long_t <<
3738 * <<
3739 * Atomically updates @v to (@v - 1) with rel <<
3740 * <<
3741 * Unsafe to use in noinstr code; use raw_ato <<
3742 * <<
3743 * Return: Nothing. <<
3744 */ <<
3745 static __always_inline void 1492 static __always_inline void
3746 atomic_long_dec(atomic_long_t *v) 1493 atomic_long_dec(atomic_long_t *v)
3747 { 1494 {
3748 instrument_atomic_read_write(v, sizeo 1495 instrument_atomic_read_write(v, sizeof(*v));
3749 raw_atomic_long_dec(v); !! 1496 arch_atomic_long_dec(v);
3750 } 1497 }
3751 1498
3752 /** <<
3753 * atomic_long_dec_return() - atomic decremen <<
3754 * @v: pointer to atomic_long_t <<
3755 * <<
3756 * Atomically updates @v to (@v - 1) with ful <<
3757 * <<
3758 * Unsafe to use in noinstr code; use raw_ato <<
3759 * <<
3760 * Return: The updated value of @v. <<
3761 */ <<
3762 static __always_inline long 1499 static __always_inline long
3763 atomic_long_dec_return(atomic_long_t *v) 1500 atomic_long_dec_return(atomic_long_t *v)
3764 { 1501 {
3765 kcsan_mb(); 1502 kcsan_mb();
3766 instrument_atomic_read_write(v, sizeo 1503 instrument_atomic_read_write(v, sizeof(*v));
3767 return raw_atomic_long_dec_return(v); !! 1504 return arch_atomic_long_dec_return(v);
3768 } 1505 }
3769 1506
3770 /** <<
3771 * atomic_long_dec_return_acquire() - atomic <<
3772 * @v: pointer to atomic_long_t <<
3773 * <<
3774 * Atomically updates @v to (@v - 1) with acq <<
3775 * <<
3776 * Unsafe to use in noinstr code; use raw_ato <<
3777 * <<
3778 * Return: The updated value of @v. <<
3779 */ <<
3780 static __always_inline long 1507 static __always_inline long
3781 atomic_long_dec_return_acquire(atomic_long_t 1508 atomic_long_dec_return_acquire(atomic_long_t *v)
3782 { 1509 {
3783 instrument_atomic_read_write(v, sizeo 1510 instrument_atomic_read_write(v, sizeof(*v));
3784 return raw_atomic_long_dec_return_acq !! 1511 return arch_atomic_long_dec_return_acquire(v);
3785 } 1512 }
3786 1513
3787 /** <<
3788 * atomic_long_dec_return_release() - atomic <<
3789 * @v: pointer to atomic_long_t <<
3790 * <<
3791 * Atomically updates @v to (@v - 1) with rel <<
3792 * <<
3793 * Unsafe to use in noinstr code; use raw_ato <<
3794 * <<
3795 * Return: The updated value of @v. <<
3796 */ <<
3797 static __always_inline long 1514 static __always_inline long
3798 atomic_long_dec_return_release(atomic_long_t 1515 atomic_long_dec_return_release(atomic_long_t *v)
3799 { 1516 {
3800 kcsan_release(); 1517 kcsan_release();
3801 instrument_atomic_read_write(v, sizeo 1518 instrument_atomic_read_write(v, sizeof(*v));
3802 return raw_atomic_long_dec_return_rel !! 1519 return arch_atomic_long_dec_return_release(v);
3803 } 1520 }
3804 1521
3805 /** <<
3806 * atomic_long_dec_return_relaxed() - atomic <<
3807 * @v: pointer to atomic_long_t <<
3808 * <<
3809 * Atomically updates @v to (@v - 1) with rel <<
3810 * <<
3811 * Unsafe to use in noinstr code; use raw_ato <<
3812 * <<
3813 * Return: The updated value of @v. <<
3814 */ <<
3815 static __always_inline long 1522 static __always_inline long
3816 atomic_long_dec_return_relaxed(atomic_long_t 1523 atomic_long_dec_return_relaxed(atomic_long_t *v)
3817 { 1524 {
3818 instrument_atomic_read_write(v, sizeo 1525 instrument_atomic_read_write(v, sizeof(*v));
3819 return raw_atomic_long_dec_return_rel !! 1526 return arch_atomic_long_dec_return_relaxed(v);
3820 } 1527 }
3821 1528
3822 /** <<
3823 * atomic_long_fetch_dec() - atomic decrement <<
3824 * @v: pointer to atomic_long_t <<
3825 * <<
3826 * Atomically updates @v to (@v - 1) with ful <<
3827 * <<
3828 * Unsafe to use in noinstr code; use raw_ato <<
3829 * <<
3830 * Return: The original value of @v. <<
3831 */ <<
3832 static __always_inline long 1529 static __always_inline long
3833 atomic_long_fetch_dec(atomic_long_t *v) 1530 atomic_long_fetch_dec(atomic_long_t *v)
3834 { 1531 {
3835 kcsan_mb(); 1532 kcsan_mb();
3836 instrument_atomic_read_write(v, sizeo 1533 instrument_atomic_read_write(v, sizeof(*v));
3837 return raw_atomic_long_fetch_dec(v); !! 1534 return arch_atomic_long_fetch_dec(v);
3838 } 1535 }
3839 1536
3840 /** <<
3841 * atomic_long_fetch_dec_acquire() - atomic d <<
3842 * @v: pointer to atomic_long_t <<
3843 * <<
3844 * Atomically updates @v to (@v - 1) with acq <<
3845 * <<
3846 * Unsafe to use in noinstr code; use raw_ato <<
3847 * <<
3848 * Return: The original value of @v. <<
3849 */ <<
3850 static __always_inline long 1537 static __always_inline long
3851 atomic_long_fetch_dec_acquire(atomic_long_t * 1538 atomic_long_fetch_dec_acquire(atomic_long_t *v)
3852 { 1539 {
3853 instrument_atomic_read_write(v, sizeo 1540 instrument_atomic_read_write(v, sizeof(*v));
3854 return raw_atomic_long_fetch_dec_acqu !! 1541 return arch_atomic_long_fetch_dec_acquire(v);
3855 } 1542 }
3856 1543
3857 /** <<
3858 * atomic_long_fetch_dec_release() - atomic d <<
3859 * @v: pointer to atomic_long_t <<
3860 * <<
3861 * Atomically updates @v to (@v - 1) with rel <<
3862 * <<
3863 * Unsafe to use in noinstr code; use raw_ato <<
3864 * <<
3865 * Return: The original value of @v. <<
3866 */ <<
3867 static __always_inline long 1544 static __always_inline long
3868 atomic_long_fetch_dec_release(atomic_long_t * 1545 atomic_long_fetch_dec_release(atomic_long_t *v)
3869 { 1546 {
3870 kcsan_release(); 1547 kcsan_release();
3871 instrument_atomic_read_write(v, sizeo 1548 instrument_atomic_read_write(v, sizeof(*v));
3872 return raw_atomic_long_fetch_dec_rele !! 1549 return arch_atomic_long_fetch_dec_release(v);
3873 } 1550 }
3874 1551
3875 /** <<
3876 * atomic_long_fetch_dec_relaxed() - atomic d <<
3877 * @v: pointer to atomic_long_t <<
3878 * <<
3879 * Atomically updates @v to (@v - 1) with rel <<
3880 * <<
3881 * Unsafe to use in noinstr code; use raw_ato <<
3882 * <<
3883 * Return: The original value of @v. <<
3884 */ <<
3885 static __always_inline long 1552 static __always_inline long
3886 atomic_long_fetch_dec_relaxed(atomic_long_t * 1553 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
3887 { 1554 {
3888 instrument_atomic_read_write(v, sizeo 1555 instrument_atomic_read_write(v, sizeof(*v));
3889 return raw_atomic_long_fetch_dec_rela !! 1556 return arch_atomic_long_fetch_dec_relaxed(v);
3890 } 1557 }
3891 1558
3892 /** <<
3893 * atomic_long_and() - atomic bitwise AND wit <<
3894 * @i: long value <<
3895 * @v: pointer to atomic_long_t <<
3896 * <<
3897 * Atomically updates @v to (@v & @i) with re <<
3898 * <<
3899 * Unsafe to use in noinstr code; use raw_ato <<
3900 * <<
3901 * Return: Nothing. <<
3902 */ <<
3903 static __always_inline void 1559 static __always_inline void
3904 atomic_long_and(long i, atomic_long_t *v) 1560 atomic_long_and(long i, atomic_long_t *v)
3905 { 1561 {
3906 instrument_atomic_read_write(v, sizeo 1562 instrument_atomic_read_write(v, sizeof(*v));
3907 raw_atomic_long_and(i, v); !! 1563 arch_atomic_long_and(i, v);
3908 } 1564 }
3909 1565
3910 /** <<
3911 * atomic_long_fetch_and() - atomic bitwise A <<
3912 * @i: long value <<
3913 * @v: pointer to atomic_long_t <<
3914 * <<
3915 * Atomically updates @v to (@v & @i) with fu <<
3916 * <<
3917 * Unsafe to use in noinstr code; use raw_ato <<
3918 * <<
3919 * Return: The original value of @v. <<
3920 */ <<
3921 static __always_inline long 1566 static __always_inline long
3922 atomic_long_fetch_and(long i, atomic_long_t * 1567 atomic_long_fetch_and(long i, atomic_long_t *v)
3923 { 1568 {
3924 kcsan_mb(); 1569 kcsan_mb();
3925 instrument_atomic_read_write(v, sizeo 1570 instrument_atomic_read_write(v, sizeof(*v));
3926 return raw_atomic_long_fetch_and(i, v !! 1571 return arch_atomic_long_fetch_and(i, v);
3927 } 1572 }
3928 1573
3929 /** <<
3930 * atomic_long_fetch_and_acquire() - atomic b <<
3931 * @i: long value <<
3932 * @v: pointer to atomic_long_t <<
3933 * <<
3934 * Atomically updates @v to (@v & @i) with ac <<
3935 * <<
3936 * Unsafe to use in noinstr code; use raw_ato <<
3937 * <<
3938 * Return: The original value of @v. <<
3939 */ <<
3940 static __always_inline long 1574 static __always_inline long
3941 atomic_long_fetch_and_acquire(long i, atomic_ 1575 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
3942 { 1576 {
3943 instrument_atomic_read_write(v, sizeo 1577 instrument_atomic_read_write(v, sizeof(*v));
3944 return raw_atomic_long_fetch_and_acqu !! 1578 return arch_atomic_long_fetch_and_acquire(i, v);
3945 } 1579 }
3946 1580
3947 /** <<
3948 * atomic_long_fetch_and_release() - atomic b <<
3949 * @i: long value <<
3950 * @v: pointer to atomic_long_t <<
3951 * <<
3952 * Atomically updates @v to (@v & @i) with re <<
3953 * <<
3954 * Unsafe to use in noinstr code; use raw_ato <<
3955 * <<
3956 * Return: The original value of @v. <<
3957 */ <<
3958 static __always_inline long 1581 static __always_inline long
3959 atomic_long_fetch_and_release(long i, atomic_ 1582 atomic_long_fetch_and_release(long i, atomic_long_t *v)
3960 { 1583 {
3961 kcsan_release(); 1584 kcsan_release();
3962 instrument_atomic_read_write(v, sizeo 1585 instrument_atomic_read_write(v, sizeof(*v));
3963 return raw_atomic_long_fetch_and_rele !! 1586 return arch_atomic_long_fetch_and_release(i, v);
3964 } 1587 }
3965 1588
3966 /** <<
3967 * atomic_long_fetch_and_relaxed() - atomic b <<
3968 * @i: long value <<
3969 * @v: pointer to atomic_long_t <<
3970 * <<
3971 * Atomically updates @v to (@v & @i) with re <<
3972 * <<
3973 * Unsafe to use in noinstr code; use raw_ato <<
3974 * <<
3975 * Return: The original value of @v. <<
3976 */ <<
3977 static __always_inline long 1589 static __always_inline long
3978 atomic_long_fetch_and_relaxed(long i, atomic_ 1590 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
3979 { 1591 {
3980 instrument_atomic_read_write(v, sizeo 1592 instrument_atomic_read_write(v, sizeof(*v));
3981 return raw_atomic_long_fetch_and_rela !! 1593 return arch_atomic_long_fetch_and_relaxed(i, v);
3982 } 1594 }
3983 1595
3984 /** <<
3985 * atomic_long_andnot() - atomic bitwise AND <<
3986 * @i: long value <<
3987 * @v: pointer to atomic_long_t <<
3988 * <<
3989 * Atomically updates @v to (@v & ~@i) with r <<
3990 * <<
3991 * Unsafe to use in noinstr code; use raw_ato <<
3992 * <<
3993 * Return: Nothing. <<
3994 */ <<
3995 static __always_inline void 1596 static __always_inline void
3996 atomic_long_andnot(long i, atomic_long_t *v) 1597 atomic_long_andnot(long i, atomic_long_t *v)
3997 { 1598 {
3998 instrument_atomic_read_write(v, sizeo 1599 instrument_atomic_read_write(v, sizeof(*v));
3999 raw_atomic_long_andnot(i, v); !! 1600 arch_atomic_long_andnot(i, v);
4000 } 1601 }
4001 1602
4002 /** <<
4003 * atomic_long_fetch_andnot() - atomic bitwis <<
4004 * @i: long value <<
4005 * @v: pointer to atomic_long_t <<
4006 * <<
4007 * Atomically updates @v to (@v & ~@i) with f <<
4008 * <<
4009 * Unsafe to use in noinstr code; use raw_ato <<
4010 * <<
4011 * Return: The original value of @v. <<
4012 */ <<
4013 static __always_inline long 1603 static __always_inline long
4014 atomic_long_fetch_andnot(long i, atomic_long_ 1604 atomic_long_fetch_andnot(long i, atomic_long_t *v)
4015 { 1605 {
4016 kcsan_mb(); 1606 kcsan_mb();
4017 instrument_atomic_read_write(v, sizeo 1607 instrument_atomic_read_write(v, sizeof(*v));
4018 return raw_atomic_long_fetch_andnot(i !! 1608 return arch_atomic_long_fetch_andnot(i, v);
4019 } 1609 }
4020 1610
4021 /** <<
4022 * atomic_long_fetch_andnot_acquire() - atomi <<
4023 * @i: long value <<
4024 * @v: pointer to atomic_long_t <<
4025 * <<
4026 * Atomically updates @v to (@v & ~@i) with a <<
4027 * <<
4028 * Unsafe to use in noinstr code; use raw_ato <<
4029 * <<
4030 * Return: The original value of @v. <<
4031 */ <<
4032 static __always_inline long 1611 static __always_inline long
4033 atomic_long_fetch_andnot_acquire(long i, atom 1612 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
4034 { 1613 {
4035 instrument_atomic_read_write(v, sizeo 1614 instrument_atomic_read_write(v, sizeof(*v));
4036 return raw_atomic_long_fetch_andnot_a !! 1615 return arch_atomic_long_fetch_andnot_acquire(i, v);
4037 } 1616 }
4038 1617
4039 /** <<
4040 * atomic_long_fetch_andnot_release() - atomi <<
4041 * @i: long value <<
4042 * @v: pointer to atomic_long_t <<
4043 * <<
4044 * Atomically updates @v to (@v & ~@i) with r <<
4045 * <<
4046 * Unsafe to use in noinstr code; use raw_ato <<
4047 * <<
4048 * Return: The original value of @v. <<
4049 */ <<
4050 static __always_inline long 1618 static __always_inline long
4051 atomic_long_fetch_andnot_release(long i, atom 1619 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
4052 { 1620 {
4053 kcsan_release(); 1621 kcsan_release();
4054 instrument_atomic_read_write(v, sizeo 1622 instrument_atomic_read_write(v, sizeof(*v));
4055 return raw_atomic_long_fetch_andnot_r !! 1623 return arch_atomic_long_fetch_andnot_release(i, v);
4056 } 1624 }
4057 1625
4058 /** <<
4059 * atomic_long_fetch_andnot_relaxed() - atomi <<
4060 * @i: long value <<
4061 * @v: pointer to atomic_long_t <<
4062 * <<
4063 * Atomically updates @v to (@v & ~@i) with r <<
4064 * <<
4065 * Unsafe to use in noinstr code; use raw_ato <<
4066 * <<
4067 * Return: The original value of @v. <<
4068 */ <<
4069 static __always_inline long 1626 static __always_inline long
4070 atomic_long_fetch_andnot_relaxed(long i, atom 1627 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
4071 { 1628 {
4072 instrument_atomic_read_write(v, sizeo 1629 instrument_atomic_read_write(v, sizeof(*v));
4073 return raw_atomic_long_fetch_andnot_r !! 1630 return arch_atomic_long_fetch_andnot_relaxed(i, v);
4074 } 1631 }
4075 1632
4076 /** <<
4077 * atomic_long_or() - atomic bitwise OR with <<
4078 * @i: long value <<
4079 * @v: pointer to atomic_long_t <<
4080 * <<
4081 * Atomically updates @v to (@v | @i) with re <<
4082 * <<
4083 * Unsafe to use in noinstr code; use raw_ato <<
4084 * <<
4085 * Return: Nothing. <<
4086 */ <<
4087 static __always_inline void 1633 static __always_inline void
4088 atomic_long_or(long i, atomic_long_t *v) 1634 atomic_long_or(long i, atomic_long_t *v)
4089 { 1635 {
4090 instrument_atomic_read_write(v, sizeo 1636 instrument_atomic_read_write(v, sizeof(*v));
4091 raw_atomic_long_or(i, v); !! 1637 arch_atomic_long_or(i, v);
4092 } 1638 }
4093 1639
4094 /** <<
4095 * atomic_long_fetch_or() - atomic bitwise OR <<
4096 * @i: long value <<
4097 * @v: pointer to atomic_long_t <<
4098 * <<
4099 * Atomically updates @v to (@v | @i) with fu <<
4100 * <<
4101 * Unsafe to use in noinstr code; use raw_ato <<
4102 * <<
4103 * Return: The original value of @v. <<
4104 */ <<
4105 static __always_inline long 1640 static __always_inline long
4106 atomic_long_fetch_or(long i, atomic_long_t *v 1641 atomic_long_fetch_or(long i, atomic_long_t *v)
4107 { 1642 {
4108 kcsan_mb(); 1643 kcsan_mb();
4109 instrument_atomic_read_write(v, sizeo 1644 instrument_atomic_read_write(v, sizeof(*v));
4110 return raw_atomic_long_fetch_or(i, v) !! 1645 return arch_atomic_long_fetch_or(i, v);
4111 } 1646 }
4112 1647
4113 /** <<
4114 * atomic_long_fetch_or_acquire() - atomic bi <<
4115 * @i: long value <<
4116 * @v: pointer to atomic_long_t <<
4117 * <<
4118 * Atomically updates @v to (@v | @i) with ac <<
4119 * <<
4120 * Unsafe to use in noinstr code; use raw_ato <<
4121 * <<
4122 * Return: The original value of @v. <<
4123 */ <<
4124 static __always_inline long 1648 static __always_inline long
4125 atomic_long_fetch_or_acquire(long i, atomic_l 1649 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
4126 { 1650 {
4127 instrument_atomic_read_write(v, sizeo 1651 instrument_atomic_read_write(v, sizeof(*v));
4128 return raw_atomic_long_fetch_or_acqui !! 1652 return arch_atomic_long_fetch_or_acquire(i, v);
4129 } 1653 }
4130 1654
4131 /** <<
4132 * atomic_long_fetch_or_release() - atomic bi <<
4133 * @i: long value <<
4134 * @v: pointer to atomic_long_t <<
4135 * <<
4136 * Atomically updates @v to (@v | @i) with re <<
4137 * <<
4138 * Unsafe to use in noinstr code; use raw_ato <<
4139 * <<
4140 * Return: The original value of @v. <<
4141 */ <<
4142 static __always_inline long 1655 static __always_inline long
4143 atomic_long_fetch_or_release(long i, atomic_l 1656 atomic_long_fetch_or_release(long i, atomic_long_t *v)
4144 { 1657 {
4145 kcsan_release(); 1658 kcsan_release();
4146 instrument_atomic_read_write(v, sizeo 1659 instrument_atomic_read_write(v, sizeof(*v));
4147 return raw_atomic_long_fetch_or_relea !! 1660 return arch_atomic_long_fetch_or_release(i, v);
4148 } 1661 }
4149 1662
4150 /** <<
4151 * atomic_long_fetch_or_relaxed() - atomic bi <<
4152 * @i: long value <<
4153 * @v: pointer to atomic_long_t <<
4154 * <<
4155 * Atomically updates @v to (@v | @i) with re <<
4156 * <<
4157 * Unsafe to use in noinstr code; use raw_ato <<
4158 * <<
4159 * Return: The original value of @v. <<
4160 */ <<
4161 static __always_inline long 1663 static __always_inline long
4162 atomic_long_fetch_or_relaxed(long i, atomic_l 1664 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
4163 { 1665 {
4164 instrument_atomic_read_write(v, sizeo 1666 instrument_atomic_read_write(v, sizeof(*v));
4165 return raw_atomic_long_fetch_or_relax !! 1667 return arch_atomic_long_fetch_or_relaxed(i, v);
4166 } 1668 }
4167 1669
4168 /** <<
4169 * atomic_long_xor() - atomic bitwise XOR wit <<
4170 * @i: long value <<
4171 * @v: pointer to atomic_long_t <<
4172 * <<
4173 * Atomically updates @v to (@v ^ @i) with re <<
4174 * <<
4175 * Unsafe to use in noinstr code; use raw_ato <<
4176 * <<
4177 * Return: Nothing. <<
4178 */ <<
4179 static __always_inline void 1670 static __always_inline void
4180 atomic_long_xor(long i, atomic_long_t *v) 1671 atomic_long_xor(long i, atomic_long_t *v)
4181 { 1672 {
4182 instrument_atomic_read_write(v, sizeo 1673 instrument_atomic_read_write(v, sizeof(*v));
4183 raw_atomic_long_xor(i, v); !! 1674 arch_atomic_long_xor(i, v);
4184 } 1675 }
4185 1676
4186 /** <<
4187 * atomic_long_fetch_xor() - atomic bitwise X <<
4188 * @i: long value <<
4189 * @v: pointer to atomic_long_t <<
4190 * <<
4191 * Atomically updates @v to (@v ^ @i) with fu <<
4192 * <<
4193 * Unsafe to use in noinstr code; use raw_ato <<
4194 * <<
4195 * Return: The original value of @v. <<
4196 */ <<
4197 static __always_inline long 1677 static __always_inline long
4198 atomic_long_fetch_xor(long i, atomic_long_t * 1678 atomic_long_fetch_xor(long i, atomic_long_t *v)
4199 { 1679 {
4200 kcsan_mb(); 1680 kcsan_mb();
4201 instrument_atomic_read_write(v, sizeo 1681 instrument_atomic_read_write(v, sizeof(*v));
4202 return raw_atomic_long_fetch_xor(i, v !! 1682 return arch_atomic_long_fetch_xor(i, v);
4203 } 1683 }
4204 1684
4205 /** <<
4206 * atomic_long_fetch_xor_acquire() - atomic b <<
4207 * @i: long value <<
4208 * @v: pointer to atomic_long_t <<
4209 * <<
4210 * Atomically updates @v to (@v ^ @i) with ac <<
4211 * <<
4212 * Unsafe to use in noinstr code; use raw_ato <<
4213 * <<
4214 * Return: The original value of @v. <<
4215 */ <<
4216 static __always_inline long 1685 static __always_inline long
4217 atomic_long_fetch_xor_acquire(long i, atomic_ 1686 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
4218 { 1687 {
4219 instrument_atomic_read_write(v, sizeo 1688 instrument_atomic_read_write(v, sizeof(*v));
4220 return raw_atomic_long_fetch_xor_acqu !! 1689 return arch_atomic_long_fetch_xor_acquire(i, v);
4221 } 1690 }
4222 1691
4223 /** <<
4224 * atomic_long_fetch_xor_release() - atomic b <<
4225 * @i: long value <<
4226 * @v: pointer to atomic_long_t <<
4227 * <<
4228 * Atomically updates @v to (@v ^ @i) with re <<
4229 * <<
4230 * Unsafe to use in noinstr code; use raw_ato <<
4231 * <<
4232 * Return: The original value of @v. <<
4233 */ <<
4234 static __always_inline long 1692 static __always_inline long
4235 atomic_long_fetch_xor_release(long i, atomic_ 1693 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
4236 { 1694 {
4237 kcsan_release(); 1695 kcsan_release();
4238 instrument_atomic_read_write(v, sizeo 1696 instrument_atomic_read_write(v, sizeof(*v));
4239 return raw_atomic_long_fetch_xor_rele !! 1697 return arch_atomic_long_fetch_xor_release(i, v);
4240 } 1698 }
4241 1699
4242 /** <<
4243 * atomic_long_fetch_xor_relaxed() - atomic b <<
4244 * @i: long value <<
4245 * @v: pointer to atomic_long_t <<
4246 * <<
4247 * Atomically updates @v to (@v ^ @i) with re <<
4248 * <<
4249 * Unsafe to use in noinstr code; use raw_ato <<
4250 * <<
4251 * Return: The original value of @v. <<
4252 */ <<
4253 static __always_inline long 1700 static __always_inline long
4254 atomic_long_fetch_xor_relaxed(long i, atomic_ 1701 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
4255 { 1702 {
4256 instrument_atomic_read_write(v, sizeo 1703 instrument_atomic_read_write(v, sizeof(*v));
4257 return raw_atomic_long_fetch_xor_rela !! 1704 return arch_atomic_long_fetch_xor_relaxed(i, v);
4258 } 1705 }
4259 1706
4260 /** <<
4261 * atomic_long_xchg() - atomic exchange with <<
4262 * @v: pointer to atomic_long_t <<
4263 * @new: long value to assign <<
4264 * <<
4265 * Atomically updates @v to @new with full or <<
4266 * <<
4267 * Unsafe to use in noinstr code; use raw_ato <<
4268 * <<
4269 * Return: The original value of @v. <<
4270 */ <<
4271 static __always_inline long 1707 static __always_inline long
4272 atomic_long_xchg(atomic_long_t *v, long new) !! 1708 atomic_long_xchg(atomic_long_t *v, long i)
4273 { 1709 {
4274 kcsan_mb(); 1710 kcsan_mb();
4275 instrument_atomic_read_write(v, sizeo 1711 instrument_atomic_read_write(v, sizeof(*v));
4276 return raw_atomic_long_xchg(v, new); !! 1712 return arch_atomic_long_xchg(v, i);
4277 } 1713 }
4278 1714
4279 /** <<
4280 * atomic_long_xchg_acquire() - atomic exchan <<
4281 * @v: pointer to atomic_long_t <<
4282 * @new: long value to assign <<
4283 * <<
4284 * Atomically updates @v to @new with acquire <<
4285 * <<
4286 * Unsafe to use in noinstr code; use raw_ato <<
4287 * <<
4288 * Return: The original value of @v. <<
4289 */ <<
4290 static __always_inline long 1715 static __always_inline long
4291 atomic_long_xchg_acquire(atomic_long_t *v, lo !! 1716 atomic_long_xchg_acquire(atomic_long_t *v, long i)
4292 { 1717 {
4293 instrument_atomic_read_write(v, sizeo 1718 instrument_atomic_read_write(v, sizeof(*v));
4294 return raw_atomic_long_xchg_acquire(v !! 1719 return arch_atomic_long_xchg_acquire(v, i);
4295 } 1720 }
4296 1721
4297 /** <<
4298 * atomic_long_xchg_release() - atomic exchan <<
4299 * @v: pointer to atomic_long_t <<
4300 * @new: long value to assign <<
4301 * <<
4302 * Atomically updates @v to @new with release <<
4303 * <<
4304 * Unsafe to use in noinstr code; use raw_ato <<
4305 * <<
4306 * Return: The original value of @v. <<
4307 */ <<
4308 static __always_inline long 1722 static __always_inline long
4309 atomic_long_xchg_release(atomic_long_t *v, lo !! 1723 atomic_long_xchg_release(atomic_long_t *v, long i)
4310 { 1724 {
4311 kcsan_release(); 1725 kcsan_release();
4312 instrument_atomic_read_write(v, sizeo 1726 instrument_atomic_read_write(v, sizeof(*v));
4313 return raw_atomic_long_xchg_release(v !! 1727 return arch_atomic_long_xchg_release(v, i);
4314 } 1728 }
4315 1729
4316 /** <<
4317 * atomic_long_xchg_relaxed() - atomic exchan <<
4318 * @v: pointer to atomic_long_t <<
4319 * @new: long value to assign <<
4320 * <<
4321 * Atomically updates @v to @new with relaxed <<
4322 * <<
4323 * Unsafe to use in noinstr code; use raw_ato <<
4324 * <<
4325 * Return: The original value of @v. <<
4326 */ <<
4327 static __always_inline long 1730 static __always_inline long
4328 atomic_long_xchg_relaxed(atomic_long_t *v, lo !! 1731 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
4329 { 1732 {
4330 instrument_atomic_read_write(v, sizeo 1733 instrument_atomic_read_write(v, sizeof(*v));
4331 return raw_atomic_long_xchg_relaxed(v !! 1734 return arch_atomic_long_xchg_relaxed(v, i);
4332 } 1735 }
4333 1736
4334 /** <<
4335 * atomic_long_cmpxchg() - atomic compare and <<
4336 * @v: pointer to atomic_long_t <<
4337 * @old: long value to compare with <<
4338 * @new: long value to assign <<
4339 * <<
4340 * If (@v == @old), atomically updates @v to <<
4341 * Otherwise, @v is not modified and relaxed <<
4342 * <<
4343 * Unsafe to use in noinstr code; use raw_ato <<
4344 * <<
4345 * Return: The original value of @v. <<
4346 */ <<
4347 static __always_inline long 1737 static __always_inline long
4348 atomic_long_cmpxchg(atomic_long_t *v, long ol 1738 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
4349 { 1739 {
4350 kcsan_mb(); 1740 kcsan_mb();
4351 instrument_atomic_read_write(v, sizeo 1741 instrument_atomic_read_write(v, sizeof(*v));
4352 return raw_atomic_long_cmpxchg(v, old !! 1742 return arch_atomic_long_cmpxchg(v, old, new);
4353 } 1743 }
4354 1744
4355 /** <<
4356 * atomic_long_cmpxchg_acquire() - atomic com <<
4357 * @v: pointer to atomic_long_t <<
4358 * @old: long value to compare with <<
4359 * @new: long value to assign <<
4360 * <<
4361 * If (@v == @old), atomically updates @v to <<
4362 * Otherwise, @v is not modified and relaxed <<
4363 * <<
4364 * Unsafe to use in noinstr code; use raw_ato <<
4365 * <<
4366 * Return: The original value of @v. <<
4367 */ <<
4368 static __always_inline long 1745 static __always_inline long
4369 atomic_long_cmpxchg_acquire(atomic_long_t *v, 1746 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
4370 { 1747 {
4371 instrument_atomic_read_write(v, sizeo 1748 instrument_atomic_read_write(v, sizeof(*v));
4372 return raw_atomic_long_cmpxchg_acquir !! 1749 return arch_atomic_long_cmpxchg_acquire(v, old, new);
4373 } 1750 }
4374 1751
4375 /** <<
4376 * atomic_long_cmpxchg_release() - atomic com <<
4377 * @v: pointer to atomic_long_t <<
4378 * @old: long value to compare with <<
4379 * @new: long value to assign <<
4380 * <<
4381 * If (@v == @old), atomically updates @v to <<
4382 * Otherwise, @v is not modified and relaxed <<
4383 * <<
4384 * Unsafe to use in noinstr code; use raw_ato <<
4385 * <<
4386 * Return: The original value of @v. <<
4387 */ <<
4388 static __always_inline long 1752 static __always_inline long
4389 atomic_long_cmpxchg_release(atomic_long_t *v, 1753 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
4390 { 1754 {
4391 kcsan_release(); 1755 kcsan_release();
4392 instrument_atomic_read_write(v, sizeo 1756 instrument_atomic_read_write(v, sizeof(*v));
4393 return raw_atomic_long_cmpxchg_releas !! 1757 return arch_atomic_long_cmpxchg_release(v, old, new);
4394 } 1758 }
4395 1759
4396 /** <<
4397 * atomic_long_cmpxchg_relaxed() - atomic com <<
4398 * @v: pointer to atomic_long_t <<
4399 * @old: long value to compare with <<
4400 * @new: long value to assign <<
4401 * <<
4402 * If (@v == @old), atomically updates @v to <<
4403 * Otherwise, @v is not modified and relaxed <<
4404 * <<
4405 * Unsafe to use in noinstr code; use raw_ato <<
4406 * <<
4407 * Return: The original value of @v. <<
4408 */ <<
4409 static __always_inline long 1760 static __always_inline long
4410 atomic_long_cmpxchg_relaxed(atomic_long_t *v, 1761 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
4411 { 1762 {
4412 instrument_atomic_read_write(v, sizeo 1763 instrument_atomic_read_write(v, sizeof(*v));
4413 return raw_atomic_long_cmpxchg_relaxe !! 1764 return arch_atomic_long_cmpxchg_relaxed(v, old, new);
4414 } 1765 }
4415 1766
4416 /** <<
4417 * atomic_long_try_cmpxchg() - atomic compare <<
4418 * @v: pointer to atomic_long_t <<
4419 * @old: pointer to long value to compare wit <<
4420 * @new: long value to assign <<
4421 * <<
4422 * If (@v == @old), atomically updates @v to <<
4423 * Otherwise, @v is not modified, @old is upd <<
4424 * and relaxed ordering is provided. <<
4425 * <<
4426 * Unsafe to use in noinstr code; use raw_ato <<
4427 * <<
4428 * Return: @true if the exchange occured, @fa <<
4429 */ <<
4430 static __always_inline bool 1767 static __always_inline bool
4431 atomic_long_try_cmpxchg(atomic_long_t *v, lon 1768 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
4432 { 1769 {
4433 kcsan_mb(); 1770 kcsan_mb();
4434 instrument_atomic_read_write(v, sizeo 1771 instrument_atomic_read_write(v, sizeof(*v));
4435 instrument_atomic_read_write(old, siz 1772 instrument_atomic_read_write(old, sizeof(*old));
4436 return raw_atomic_long_try_cmpxchg(v, !! 1773 return arch_atomic_long_try_cmpxchg(v, old, new);
4437 } 1774 }
4438 1775
4439 /** <<
4440 * atomic_long_try_cmpxchg_acquire() - atomic <<
4441 * @v: pointer to atomic_long_t <<
4442 * @old: pointer to long value to compare wit <<
4443 * @new: long value to assign <<
4444 * <<
4445 * If (@v == @old), atomically updates @v to <<
4446 * Otherwise, @v is not modified, @old is upd <<
4447 * and relaxed ordering is provided. <<
4448 * <<
4449 * Unsafe to use in noinstr code; use raw_ato <<
4450 * <<
4451 * Return: @true if the exchange occured, @fa <<
4452 */ <<
4453 static __always_inline bool 1776 static __always_inline bool
4454 atomic_long_try_cmpxchg_acquire(atomic_long_t 1777 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
4455 { 1778 {
4456 instrument_atomic_read_write(v, sizeo 1779 instrument_atomic_read_write(v, sizeof(*v));
4457 instrument_atomic_read_write(old, siz 1780 instrument_atomic_read_write(old, sizeof(*old));
4458 return raw_atomic_long_try_cmpxchg_ac !! 1781 return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
4459 } 1782 }
4460 1783
4461 /** <<
4462 * atomic_long_try_cmpxchg_release() - atomic <<
4463 * @v: pointer to atomic_long_t <<
4464 * @old: pointer to long value to compare wit <<
4465 * @new: long value to assign <<
4466 * <<
4467 * If (@v == @old), atomically updates @v to <<
4468 * Otherwise, @v is not modified, @old is upd <<
4469 * and relaxed ordering is provided. <<
4470 * <<
4471 * Unsafe to use in noinstr code; use raw_ato <<
4472 * <<
4473 * Return: @true if the exchange occured, @fa <<
4474 */ <<
4475 static __always_inline bool 1784 static __always_inline bool
4476 atomic_long_try_cmpxchg_release(atomic_long_t 1785 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
4477 { 1786 {
4478 kcsan_release(); 1787 kcsan_release();
4479 instrument_atomic_read_write(v, sizeo 1788 instrument_atomic_read_write(v, sizeof(*v));
4480 instrument_atomic_read_write(old, siz 1789 instrument_atomic_read_write(old, sizeof(*old));
4481 return raw_atomic_long_try_cmpxchg_re !! 1790 return arch_atomic_long_try_cmpxchg_release(v, old, new);
4482 } 1791 }
4483 1792
4484 /** <<
4485 * atomic_long_try_cmpxchg_relaxed() - atomic <<
4486 * @v: pointer to atomic_long_t <<
4487 * @old: pointer to long value to compare wit <<
4488 * @new: long value to assign <<
4489 * <<
4490 * If (@v == @old), atomically updates @v to <<
4491 * Otherwise, @v is not modified, @old is upd <<
4492 * and relaxed ordering is provided. <<
4493 * <<
4494 * Unsafe to use in noinstr code; use raw_ato <<
4495 * <<
4496 * Return: @true if the exchange occured, @fa <<
4497 */ <<
4498 static __always_inline bool 1793 static __always_inline bool
4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t 1794 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
4500 { 1795 {
4501 instrument_atomic_read_write(v, sizeo 1796 instrument_atomic_read_write(v, sizeof(*v));
4502 instrument_atomic_read_write(old, siz 1797 instrument_atomic_read_write(old, sizeof(*old));
4503 return raw_atomic_long_try_cmpxchg_re !! 1798 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
4504 } 1799 }
4505 1800
4506 /** <<
4507 * atomic_long_sub_and_test() - atomic subtra <<
4508 * @i: long value to subtract <<
4509 * @v: pointer to atomic_long_t <<
4510 * <<
4511 * Atomically updates @v to (@v - @i) with fu <<
4512 * <<
4513 * Unsafe to use in noinstr code; use raw_ato <<
4514 * <<
4515 * Return: @true if the resulting value of @v <<
4516 */ <<
4517 static __always_inline bool 1801 static __always_inline bool
4518 atomic_long_sub_and_test(long i, atomic_long_ 1802 atomic_long_sub_and_test(long i, atomic_long_t *v)
4519 { 1803 {
4520 kcsan_mb(); 1804 kcsan_mb();
4521 instrument_atomic_read_write(v, sizeo 1805 instrument_atomic_read_write(v, sizeof(*v));
4522 return raw_atomic_long_sub_and_test(i !! 1806 return arch_atomic_long_sub_and_test(i, v);
4523 } 1807 }
4524 1808
4525 /** <<
4526 * atomic_long_dec_and_test() - atomic decrem <<
4527 * @v: pointer to atomic_long_t <<
4528 * <<
4529 * Atomically updates @v to (@v - 1) with ful <<
4530 * <<
4531 * Unsafe to use in noinstr code; use raw_ato <<
4532 * <<
4533 * Return: @true if the resulting value of @v <<
4534 */ <<
4535 static __always_inline bool 1809 static __always_inline bool
4536 atomic_long_dec_and_test(atomic_long_t *v) 1810 atomic_long_dec_and_test(atomic_long_t *v)
4537 { 1811 {
4538 kcsan_mb(); 1812 kcsan_mb();
4539 instrument_atomic_read_write(v, sizeo 1813 instrument_atomic_read_write(v, sizeof(*v));
4540 return raw_atomic_long_dec_and_test(v !! 1814 return arch_atomic_long_dec_and_test(v);
4541 } 1815 }
4542 1816
4543 /** <<
4544 * atomic_long_inc_and_test() - atomic increm <<
4545 * @v: pointer to atomic_long_t <<
4546 * <<
4547 * Atomically updates @v to (@v + 1) with ful <<
4548 * <<
4549 * Unsafe to use in noinstr code; use raw_ato <<
4550 * <<
4551 * Return: @true if the resulting value of @v <<
4552 */ <<
4553 static __always_inline bool 1817 static __always_inline bool
4554 atomic_long_inc_and_test(atomic_long_t *v) 1818 atomic_long_inc_and_test(atomic_long_t *v)
4555 { 1819 {
4556 kcsan_mb(); 1820 kcsan_mb();
4557 instrument_atomic_read_write(v, sizeo 1821 instrument_atomic_read_write(v, sizeof(*v));
4558 return raw_atomic_long_inc_and_test(v !! 1822 return arch_atomic_long_inc_and_test(v);
4559 } 1823 }
4560 1824
4561 /** <<
4562 * atomic_long_add_negative() - atomic add an <<
4563 * @i: long value to add <<
4564 * @v: pointer to atomic_long_t <<
4565 * <<
4566 * Atomically updates @v to (@v + @i) with fu <<
4567 * <<
4568 * Unsafe to use in noinstr code; use raw_ato <<
4569 * <<
4570 * Return: @true if the resulting value of @v <<
4571 */ <<
4572 static __always_inline bool 1825 static __always_inline bool
4573 atomic_long_add_negative(long i, atomic_long_ 1826 atomic_long_add_negative(long i, atomic_long_t *v)
4574 { 1827 {
4575 kcsan_mb(); 1828 kcsan_mb();
4576 instrument_atomic_read_write(v, sizeo 1829 instrument_atomic_read_write(v, sizeof(*v));
4577 return raw_atomic_long_add_negative(i !! 1830 return arch_atomic_long_add_negative(i, v);
4578 } 1831 }
4579 1832
4580 /** <<
4581 * atomic_long_add_negative_acquire() - atomi <<
4582 * @i: long value to add <<
4583 * @v: pointer to atomic_long_t <<
4584 * <<
4585 * Atomically updates @v to (@v + @i) with ac <<
4586 * <<
4587 * Unsafe to use in noinstr code; use raw_ato <<
4588 * <<
4589 * Return: @true if the resulting value of @v <<
4590 */ <<
4591 static __always_inline bool <<
4592 atomic_long_add_negative_acquire(long i, atom <<
4593 { <<
4594 instrument_atomic_read_write(v, sizeo <<
4595 return raw_atomic_long_add_negative_a <<
4596 } <<
4597 <<
4598 /** <<
4599 * atomic_long_add_negative_release() - atomi <<
4600 * @i: long value to add <<
4601 * @v: pointer to atomic_long_t <<
4602 * <<
4603 * Atomically updates @v to (@v + @i) with re <<
4604 * <<
4605 * Unsafe to use in noinstr code; use raw_ato <<
4606 * <<
4607 * Return: @true if the resulting value of @v <<
4608 */ <<
4609 static __always_inline bool <<
4610 atomic_long_add_negative_release(long i, atom <<
4611 { <<
4612 kcsan_release(); <<
4613 instrument_atomic_read_write(v, sizeo <<
4614 return raw_atomic_long_add_negative_r <<
4615 } <<
4616 <<
4617 /** <<
4618 * atomic_long_add_negative_relaxed() - atomi <<
4619 * @i: long value to add <<
4620 * @v: pointer to atomic_long_t <<
4621 * <<
4622 * Atomically updates @v to (@v + @i) with re <<
4623 * <<
4624 * Unsafe to use in noinstr code; use raw_ato <<
4625 * <<
4626 * Return: @true if the resulting value of @v <<
4627 */ <<
4628 static __always_inline bool <<
4629 atomic_long_add_negative_relaxed(long i, atom <<
4630 { <<
4631 instrument_atomic_read_write(v, sizeo <<
4632 return raw_atomic_long_add_negative_r <<
4633 } <<
4634 <<
4635 /** <<
4636 * atomic_long_fetch_add_unless() - atomic ad <<
4637 * @v: pointer to atomic_long_t <<
4638 * @a: long value to add <<
4639 * @u: long value to compare with <<
4640 * <<
4641 * If (@v != @u), atomically updates @v to (@ <<
4642 * Otherwise, @v is not modified and relaxed <<
4643 * <<
4644 * Unsafe to use in noinstr code; use raw_ato <<
4645 * <<
4646 * Return: The original value of @v. <<
4647 */ <<
4648 static __always_inline long 1833 static __always_inline long
4649 atomic_long_fetch_add_unless(atomic_long_t *v 1834 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
4650 { 1835 {
4651 kcsan_mb(); 1836 kcsan_mb();
4652 instrument_atomic_read_write(v, sizeo 1837 instrument_atomic_read_write(v, sizeof(*v));
4653 return raw_atomic_long_fetch_add_unle !! 1838 return arch_atomic_long_fetch_add_unless(v, a, u);
4654 } 1839 }
4655 1840
4656 /** <<
4657 * atomic_long_add_unless() - atomic add unle <<
4658 * @v: pointer to atomic_long_t <<
4659 * @a: long value to add <<
4660 * @u: long value to compare with <<
4661 * <<
4662 * If (@v != @u), atomically updates @v to (@ <<
4663 * Otherwise, @v is not modified and relaxed <<
4664 * <<
4665 * Unsafe to use in noinstr code; use raw_ato <<
4666 * <<
4667 * Return: @true if @v was updated, @false ot <<
4668 */ <<
4669 static __always_inline bool 1841 static __always_inline bool
4670 atomic_long_add_unless(atomic_long_t *v, long 1842 atomic_long_add_unless(atomic_long_t *v, long a, long u)
4671 { 1843 {
4672 kcsan_mb(); 1844 kcsan_mb();
4673 instrument_atomic_read_write(v, sizeo 1845 instrument_atomic_read_write(v, sizeof(*v));
4674 return raw_atomic_long_add_unless(v, !! 1846 return arch_atomic_long_add_unless(v, a, u);
4675 } 1847 }
4676 1848
4677 /** <<
4678 * atomic_long_inc_not_zero() - atomic increm <<
4679 * @v: pointer to atomic_long_t <<
4680 * <<
4681 * If (@v != 0), atomically updates @v to (@v <<
4682 * Otherwise, @v is not modified and relaxed <<
4683 * <<
4684 * Unsafe to use in noinstr code; use raw_ato <<
4685 * <<
4686 * Return: @true if @v was updated, @false ot <<
4687 */ <<
4688 static __always_inline bool 1849 static __always_inline bool
4689 atomic_long_inc_not_zero(atomic_long_t *v) 1850 atomic_long_inc_not_zero(atomic_long_t *v)
4690 { 1851 {
4691 kcsan_mb(); 1852 kcsan_mb();
4692 instrument_atomic_read_write(v, sizeo 1853 instrument_atomic_read_write(v, sizeof(*v));
4693 return raw_atomic_long_inc_not_zero(v !! 1854 return arch_atomic_long_inc_not_zero(v);
4694 } 1855 }
4695 1856
4696 /** <<
4697 * atomic_long_inc_unless_negative() - atomic <<
4698 * @v: pointer to atomic_long_t <<
4699 * <<
4700 * If (@v >= 0), atomically updates @v to (@v <<
4701 * Otherwise, @v is not modified and relaxed <<
4702 * <<
4703 * Unsafe to use in noinstr code; use raw_ato <<
4704 * <<
4705 * Return: @true if @v was updated, @false ot <<
4706 */ <<
4707 static __always_inline bool 1857 static __always_inline bool
4708 atomic_long_inc_unless_negative(atomic_long_t 1858 atomic_long_inc_unless_negative(atomic_long_t *v)
4709 { 1859 {
4710 kcsan_mb(); 1860 kcsan_mb();
4711 instrument_atomic_read_write(v, sizeo 1861 instrument_atomic_read_write(v, sizeof(*v));
4712 return raw_atomic_long_inc_unless_neg !! 1862 return arch_atomic_long_inc_unless_negative(v);
4713 } 1863 }
4714 1864
4715 /** <<
4716 * atomic_long_dec_unless_positive() - atomic <<
4717 * @v: pointer to atomic_long_t <<
4718 * <<
4719 * If (@v <= 0), atomically updates @v to (@v <<
4720 * Otherwise, @v is not modified and relaxed <<
4721 * <<
4722 * Unsafe to use in noinstr code; use raw_ato <<
4723 * <<
4724 * Return: @true if @v was updated, @false ot <<
4725 */ <<
4726 static __always_inline bool 1865 static __always_inline bool
4727 atomic_long_dec_unless_positive(atomic_long_t 1866 atomic_long_dec_unless_positive(atomic_long_t *v)
4728 { 1867 {
4729 kcsan_mb(); 1868 kcsan_mb();
4730 instrument_atomic_read_write(v, sizeo 1869 instrument_atomic_read_write(v, sizeof(*v));
4731 return raw_atomic_long_dec_unless_pos !! 1870 return arch_atomic_long_dec_unless_positive(v);
4732 } 1871 }
4733 1872
4734 /** <<
4735 * atomic_long_dec_if_positive() - atomic dec <<
4736 * @v: pointer to atomic_long_t <<
4737 * <<
4738 * If (@v > 0), atomically updates @v to (@v <<
4739 * Otherwise, @v is not modified and relaxed <<
4740 * <<
4741 * Unsafe to use in noinstr code; use raw_ato <<
4742 * <<
4743 * Return: The old value of (@v - 1), regardl <<
4744 */ <<
4745 static __always_inline long 1873 static __always_inline long
4746 atomic_long_dec_if_positive(atomic_long_t *v) 1874 atomic_long_dec_if_positive(atomic_long_t *v)
4747 { 1875 {
4748 kcsan_mb(); 1876 kcsan_mb();
4749 instrument_atomic_read_write(v, sizeo 1877 instrument_atomic_read_write(v, sizeof(*v));
4750 return raw_atomic_long_dec_if_positiv !! 1878 return arch_atomic_long_dec_if_positive(v);
4751 } 1879 }
4752 1880
4753 #define xchg(ptr, ...) \ 1881 #define xchg(ptr, ...) \
4754 ({ \ 1882 ({ \
4755 typeof(ptr) __ai_ptr = (ptr); \ 1883 typeof(ptr) __ai_ptr = (ptr); \
4756 kcsan_mb(); \ 1884 kcsan_mb(); \
4757 instrument_atomic_read_write(__ai_ptr !! 1885 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4758 raw_xchg(__ai_ptr, __VA_ARGS__); \ !! 1886 arch_xchg(__ai_ptr, __VA_ARGS__); \
4759 }) 1887 })
4760 1888
4761 #define xchg_acquire(ptr, ...) \ 1889 #define xchg_acquire(ptr, ...) \
4762 ({ \ 1890 ({ \
4763 typeof(ptr) __ai_ptr = (ptr); \ 1891 typeof(ptr) __ai_ptr = (ptr); \
4764 instrument_atomic_read_write(__ai_ptr !! 1892 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4765 raw_xchg_acquire(__ai_ptr, __VA_ARGS_ !! 1893 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
4766 }) 1894 })
4767 1895
4768 #define xchg_release(ptr, ...) \ 1896 #define xchg_release(ptr, ...) \
4769 ({ \ 1897 ({ \
4770 typeof(ptr) __ai_ptr = (ptr); \ 1898 typeof(ptr) __ai_ptr = (ptr); \
4771 kcsan_release(); \ 1899 kcsan_release(); \
4772 instrument_atomic_read_write(__ai_ptr !! 1900 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4773 raw_xchg_release(__ai_ptr, __VA_ARGS_ !! 1901 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
4774 }) 1902 })
4775 1903
4776 #define xchg_relaxed(ptr, ...) \ 1904 #define xchg_relaxed(ptr, ...) \
4777 ({ \ 1905 ({ \
4778 typeof(ptr) __ai_ptr = (ptr); \ 1906 typeof(ptr) __ai_ptr = (ptr); \
4779 instrument_atomic_read_write(__ai_ptr !! 1907 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4780 raw_xchg_relaxed(__ai_ptr, __VA_ARGS_ !! 1908 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
4781 }) 1909 })
4782 1910
4783 #define cmpxchg(ptr, ...) \ 1911 #define cmpxchg(ptr, ...) \
4784 ({ \ 1912 ({ \
4785 typeof(ptr) __ai_ptr = (ptr); \ 1913 typeof(ptr) __ai_ptr = (ptr); \
4786 kcsan_mb(); \ 1914 kcsan_mb(); \
4787 instrument_atomic_read_write(__ai_ptr !! 1915 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4788 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \ !! 1916 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
4789 }) 1917 })
4790 1918
4791 #define cmpxchg_acquire(ptr, ...) \ 1919 #define cmpxchg_acquire(ptr, ...) \
4792 ({ \ 1920 ({ \
4793 typeof(ptr) __ai_ptr = (ptr); \ 1921 typeof(ptr) __ai_ptr = (ptr); \
4794 instrument_atomic_read_write(__ai_ptr !! 1922 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4795 raw_cmpxchg_acquire(__ai_ptr, __VA_AR !! 1923 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
4796 }) 1924 })
4797 1925
4798 #define cmpxchg_release(ptr, ...) \ 1926 #define cmpxchg_release(ptr, ...) \
4799 ({ \ 1927 ({ \
4800 typeof(ptr) __ai_ptr = (ptr); \ 1928 typeof(ptr) __ai_ptr = (ptr); \
4801 kcsan_release(); \ 1929 kcsan_release(); \
4802 instrument_atomic_read_write(__ai_ptr !! 1930 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4803 raw_cmpxchg_release(__ai_ptr, __VA_AR !! 1931 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
4804 }) 1932 })
4805 1933
4806 #define cmpxchg_relaxed(ptr, ...) \ 1934 #define cmpxchg_relaxed(ptr, ...) \
4807 ({ \ 1935 ({ \
4808 typeof(ptr) __ai_ptr = (ptr); \ 1936 typeof(ptr) __ai_ptr = (ptr); \
4809 instrument_atomic_read_write(__ai_ptr !! 1937 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4810 raw_cmpxchg_relaxed(__ai_ptr, __VA_AR !! 1938 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
4811 }) 1939 })
4812 1940
4813 #define cmpxchg64(ptr, ...) \ 1941 #define cmpxchg64(ptr, ...) \
4814 ({ \ 1942 ({ \
4815 typeof(ptr) __ai_ptr = (ptr); \ 1943 typeof(ptr) __ai_ptr = (ptr); \
4816 kcsan_mb(); \ 1944 kcsan_mb(); \
4817 instrument_atomic_read_write(__ai_ptr !! 1945 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4818 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); !! 1946 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
4819 }) 1947 })
4820 1948
4821 #define cmpxchg64_acquire(ptr, ...) \ 1949 #define cmpxchg64_acquire(ptr, ...) \
4822 ({ \ 1950 ({ \
4823 typeof(ptr) __ai_ptr = (ptr); \ 1951 typeof(ptr) __ai_ptr = (ptr); \
4824 instrument_atomic_read_write(__ai_ptr !! 1952 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4825 raw_cmpxchg64_acquire(__ai_ptr, __VA_ !! 1953 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
4826 }) 1954 })
4827 1955
4828 #define cmpxchg64_release(ptr, ...) \ 1956 #define cmpxchg64_release(ptr, ...) \
4829 ({ \ 1957 ({ \
4830 typeof(ptr) __ai_ptr = (ptr); \ 1958 typeof(ptr) __ai_ptr = (ptr); \
4831 kcsan_release(); \ 1959 kcsan_release(); \
4832 instrument_atomic_read_write(__ai_ptr !! 1960 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4833 raw_cmpxchg64_release(__ai_ptr, __VA_ !! 1961 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
4834 }) 1962 })
4835 1963
4836 #define cmpxchg64_relaxed(ptr, ...) \ 1964 #define cmpxchg64_relaxed(ptr, ...) \
4837 ({ \ 1965 ({ \
4838 typeof(ptr) __ai_ptr = (ptr); \ 1966 typeof(ptr) __ai_ptr = (ptr); \
4839 instrument_atomic_read_write(__ai_ptr !! 1967 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4840 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ !! 1968 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
4841 }) <<
4842 <<
4843 #define cmpxchg128(ptr, ...) \ <<
4844 ({ \ <<
4845 typeof(ptr) __ai_ptr = (ptr); \ <<
4846 kcsan_mb(); \ <<
4847 instrument_atomic_read_write(__ai_ptr <<
4848 raw_cmpxchg128(__ai_ptr, __VA_ARGS__) <<
4849 }) <<
4850 <<
4851 #define cmpxchg128_acquire(ptr, ...) \ <<
4852 ({ \ <<
4853 typeof(ptr) __ai_ptr = (ptr); \ <<
4854 instrument_atomic_read_write(__ai_ptr <<
4855 raw_cmpxchg128_acquire(__ai_ptr, __VA <<
4856 }) <<
4857 <<
4858 #define cmpxchg128_release(ptr, ...) \ <<
4859 ({ \ <<
4860 typeof(ptr) __ai_ptr = (ptr); \ <<
4861 kcsan_release(); \ <<
4862 instrument_atomic_read_write(__ai_ptr <<
4863 raw_cmpxchg128_release(__ai_ptr, __VA <<
4864 }) <<
4865 <<
4866 #define cmpxchg128_relaxed(ptr, ...) \ <<
4867 ({ \ <<
4868 typeof(ptr) __ai_ptr = (ptr); \ <<
4869 instrument_atomic_read_write(__ai_ptr <<
4870 raw_cmpxchg128_relaxed(__ai_ptr, __VA <<
4871 }) 1969 })
4872 1970
4873 #define try_cmpxchg(ptr, oldp, ...) \ 1971 #define try_cmpxchg(ptr, oldp, ...) \
4874 ({ \ 1972 ({ \
4875 typeof(ptr) __ai_ptr = (ptr); \ 1973 typeof(ptr) __ai_ptr = (ptr); \
4876 typeof(oldp) __ai_oldp = (oldp); \ 1974 typeof(oldp) __ai_oldp = (oldp); \
4877 kcsan_mb(); \ 1975 kcsan_mb(); \
4878 instrument_atomic_read_write(__ai_ptr !! 1976 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4879 instrument_read_write(__ai_oldp, size !! 1977 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4880 raw_try_cmpxchg(__ai_ptr, __ai_oldp, !! 1978 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4881 }) 1979 })
4882 1980
4883 #define try_cmpxchg_acquire(ptr, oldp, ...) \ 1981 #define try_cmpxchg_acquire(ptr, oldp, ...) \
4884 ({ \ 1982 ({ \
4885 typeof(ptr) __ai_ptr = (ptr); \ 1983 typeof(ptr) __ai_ptr = (ptr); \
4886 typeof(oldp) __ai_oldp = (oldp); \ 1984 typeof(oldp) __ai_oldp = (oldp); \
4887 instrument_atomic_read_write(__ai_ptr !! 1985 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4888 instrument_read_write(__ai_oldp, size !! 1986 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4889 raw_try_cmpxchg_acquire(__ai_ptr, __a !! 1987 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4890 }) 1988 })
4891 1989
4892 #define try_cmpxchg_release(ptr, oldp, ...) \ 1990 #define try_cmpxchg_release(ptr, oldp, ...) \
4893 ({ \ 1991 ({ \
4894 typeof(ptr) __ai_ptr = (ptr); \ 1992 typeof(ptr) __ai_ptr = (ptr); \
4895 typeof(oldp) __ai_oldp = (oldp); \ 1993 typeof(oldp) __ai_oldp = (oldp); \
4896 kcsan_release(); \ 1994 kcsan_release(); \
4897 instrument_atomic_read_write(__ai_ptr !! 1995 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4898 instrument_read_write(__ai_oldp, size !! 1996 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4899 raw_try_cmpxchg_release(__ai_ptr, __a !! 1997 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4900 }) 1998 })
4901 1999
4902 #define try_cmpxchg_relaxed(ptr, oldp, ...) \ 2000 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
4903 ({ \ 2001 ({ \
4904 typeof(ptr) __ai_ptr = (ptr); \ 2002 typeof(ptr) __ai_ptr = (ptr); \
4905 typeof(oldp) __ai_oldp = (oldp); \ 2003 typeof(oldp) __ai_oldp = (oldp); \
4906 instrument_atomic_read_write(__ai_ptr !! 2004 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4907 instrument_read_write(__ai_oldp, size !! 2005 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4908 raw_try_cmpxchg_relaxed(__ai_ptr, __a !! 2006 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4909 }) 2007 })
4910 2008
4911 #define try_cmpxchg64(ptr, oldp, ...) \ 2009 #define try_cmpxchg64(ptr, oldp, ...) \
4912 ({ \ 2010 ({ \
4913 typeof(ptr) __ai_ptr = (ptr); \ 2011 typeof(ptr) __ai_ptr = (ptr); \
4914 typeof(oldp) __ai_oldp = (oldp); \ 2012 typeof(oldp) __ai_oldp = (oldp); \
4915 kcsan_mb(); \ 2013 kcsan_mb(); \
4916 instrument_atomic_read_write(__ai_ptr !! 2014 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4917 instrument_read_write(__ai_oldp, size !! 2015 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4918 raw_try_cmpxchg64(__ai_ptr, __ai_oldp !! 2016 arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4919 }) 2017 })
4920 2018
4921 #define try_cmpxchg64_acquire(ptr, oldp, ...) 2019 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
4922 ({ \ 2020 ({ \
4923 typeof(ptr) __ai_ptr = (ptr); \ 2021 typeof(ptr) __ai_ptr = (ptr); \
4924 typeof(oldp) __ai_oldp = (oldp); \ 2022 typeof(oldp) __ai_oldp = (oldp); \
4925 instrument_atomic_read_write(__ai_ptr !! 2023 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4926 instrument_read_write(__ai_oldp, size !! 2024 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4927 raw_try_cmpxchg64_acquire(__ai_ptr, _ !! 2025 arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4928 }) 2026 })
4929 2027
4930 #define try_cmpxchg64_release(ptr, oldp, ...) 2028 #define try_cmpxchg64_release(ptr, oldp, ...) \
4931 ({ \ 2029 ({ \
4932 typeof(ptr) __ai_ptr = (ptr); \ 2030 typeof(ptr) __ai_ptr = (ptr); \
4933 typeof(oldp) __ai_oldp = (oldp); \ 2031 typeof(oldp) __ai_oldp = (oldp); \
4934 kcsan_release(); \ 2032 kcsan_release(); \
4935 instrument_atomic_read_write(__ai_ptr !! 2033 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4936 instrument_read_write(__ai_oldp, size !! 2034 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4937 raw_try_cmpxchg64_release(__ai_ptr, _ !! 2035 arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4938 }) 2036 })
4939 2037
4940 #define try_cmpxchg64_relaxed(ptr, oldp, ...) 2038 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
4941 ({ \ 2039 ({ \
4942 typeof(ptr) __ai_ptr = (ptr); \ 2040 typeof(ptr) __ai_ptr = (ptr); \
4943 typeof(oldp) __ai_oldp = (oldp); \ 2041 typeof(oldp) __ai_oldp = (oldp); \
4944 instrument_atomic_read_write(__ai_ptr !! 2042 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4945 instrument_read_write(__ai_oldp, size !! 2043 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
4946 raw_try_cmpxchg64_relaxed(__ai_ptr, _ !! 2044 arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4947 }) <<
4948 <<
4949 #define try_cmpxchg128(ptr, oldp, ...) \ <<
4950 ({ \ <<
4951 typeof(ptr) __ai_ptr = (ptr); \ <<
4952 typeof(oldp) __ai_oldp = (oldp); \ <<
4953 kcsan_mb(); \ <<
4954 instrument_atomic_read_write(__ai_ptr <<
4955 instrument_read_write(__ai_oldp, size <<
4956 raw_try_cmpxchg128(__ai_ptr, __ai_old <<
4957 }) <<
4958 <<
4959 #define try_cmpxchg128_acquire(ptr, oldp, ... <<
4960 ({ \ <<
4961 typeof(ptr) __ai_ptr = (ptr); \ <<
4962 typeof(oldp) __ai_oldp = (oldp); \ <<
4963 instrument_atomic_read_write(__ai_ptr <<
4964 instrument_read_write(__ai_oldp, size <<
4965 raw_try_cmpxchg128_acquire(__ai_ptr, <<
4966 }) <<
4967 <<
4968 #define try_cmpxchg128_release(ptr, oldp, ... <<
4969 ({ \ <<
4970 typeof(ptr) __ai_ptr = (ptr); \ <<
4971 typeof(oldp) __ai_oldp = (oldp); \ <<
4972 kcsan_release(); \ <<
4973 instrument_atomic_read_write(__ai_ptr <<
4974 instrument_read_write(__ai_oldp, size <<
4975 raw_try_cmpxchg128_release(__ai_ptr, <<
4976 }) <<
4977 <<
4978 #define try_cmpxchg128_relaxed(ptr, oldp, ... <<
4979 ({ \ <<
4980 typeof(ptr) __ai_ptr = (ptr); \ <<
4981 typeof(oldp) __ai_oldp = (oldp); \ <<
4982 instrument_atomic_read_write(__ai_ptr <<
4983 instrument_read_write(__ai_oldp, size <<
4984 raw_try_cmpxchg128_relaxed(__ai_ptr, <<
4985 }) 2045 })
4986 2046
4987 #define cmpxchg_local(ptr, ...) \ 2047 #define cmpxchg_local(ptr, ...) \
4988 ({ \ 2048 ({ \
4989 typeof(ptr) __ai_ptr = (ptr); \ 2049 typeof(ptr) __ai_ptr = (ptr); \
4990 instrument_atomic_read_write(__ai_ptr !! 2050 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4991 raw_cmpxchg_local(__ai_ptr, __VA_ARGS !! 2051 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
4992 }) 2052 })
4993 2053
4994 #define cmpxchg64_local(ptr, ...) \ 2054 #define cmpxchg64_local(ptr, ...) \
4995 ({ \ 2055 ({ \
4996 typeof(ptr) __ai_ptr = (ptr); \ 2056 typeof(ptr) __ai_ptr = (ptr); \
4997 instrument_atomic_read_write(__ai_ptr !! 2057 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
4998 raw_cmpxchg64_local(__ai_ptr, __VA_AR !! 2058 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
4999 }) <<
5000 <<
5001 #define cmpxchg128_local(ptr, ...) \ <<
5002 ({ \ <<
5003 typeof(ptr) __ai_ptr = (ptr); \ <<
5004 instrument_atomic_read_write(__ai_ptr <<
5005 raw_cmpxchg128_local(__ai_ptr, __VA_A <<
5006 }) 2059 })
5007 2060
5008 #define sync_cmpxchg(ptr, ...) \ 2061 #define sync_cmpxchg(ptr, ...) \
5009 ({ \ 2062 ({ \
5010 typeof(ptr) __ai_ptr = (ptr); \ 2063 typeof(ptr) __ai_ptr = (ptr); \
5011 kcsan_mb(); \ 2064 kcsan_mb(); \
5012 instrument_atomic_read_write(__ai_ptr !! 2065 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
5013 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS_ !! 2066 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
5014 }) <<
5015 <<
5016 #define try_cmpxchg_local(ptr, oldp, ...) \ <<
5017 ({ \ <<
5018 typeof(ptr) __ai_ptr = (ptr); \ <<
5019 typeof(oldp) __ai_oldp = (oldp); \ <<
5020 instrument_atomic_read_write(__ai_ptr <<
5021 instrument_read_write(__ai_oldp, size <<
5022 raw_try_cmpxchg_local(__ai_ptr, __ai_ <<
5023 }) 2067 })
5024 2068
5025 #define try_cmpxchg64_local(ptr, oldp, ...) \ !! 2069 #define cmpxchg_double(ptr, ...) \
5026 ({ \ 2070 ({ \
5027 typeof(ptr) __ai_ptr = (ptr); \ 2071 typeof(ptr) __ai_ptr = (ptr); \
5028 typeof(oldp) __ai_oldp = (oldp); \ !! 2072 kcsan_mb(); \
5029 instrument_atomic_read_write(__ai_ptr !! 2073 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
5030 instrument_read_write(__ai_oldp, size !! 2074 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
5031 raw_try_cmpxchg64_local(__ai_ptr, __a <<
5032 }) 2075 })
5033 2076
5034 #define try_cmpxchg128_local(ptr, oldp, ...) <<
5035 ({ \ <<
5036 typeof(ptr) __ai_ptr = (ptr); \ <<
5037 typeof(oldp) __ai_oldp = (oldp); \ <<
5038 instrument_atomic_read_write(__ai_ptr <<
5039 instrument_read_write(__ai_oldp, size <<
5040 raw_try_cmpxchg128_local(__ai_ptr, __ <<
5041 }) <<
5042 2077
5043 #define sync_try_cmpxchg(ptr, ...) \ !! 2078 #define cmpxchg_double_local(ptr, ...) \
5044 ({ \ 2079 ({ \
5045 typeof(ptr) __ai_ptr = (ptr); \ 2080 typeof(ptr) __ai_ptr = (ptr); \
5046 kcsan_mb(); \ !! 2081 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
5047 instrument_atomic_read_write(__ai_ptr !! 2082 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
5048 raw_sync_try_cmpxchg(__ai_ptr, __VA_A <<
5049 }) 2083 })
5050 2084
5051 <<
5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ 2085 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
5053 // 8829b337928e9508259079d32581775ececd415b !! 2086 // 764f741eb77a7ad565dc8d99ce2837d5542e8aee
5054 2087
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.