1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
3 // Generated by scripts/atomic/gen-atomic-inst 3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY 4 // DO NOT MODIFY THIS FILE DIRECTLY
5 5
6 /* 6 /*
7 * This file provoides atomic operations with 7 * This file provoides atomic operations with explicit instrumentation (e.g.
8 * KASAN, KCSAN), which should be used unless 8 * KASAN, KCSAN), which should be used unless it is necessary to avoid
9 * instrumentation. Where it is necessary to a 9 * instrumentation. Where it is necessary to aovid instrumenation, the
10 * raw_atomic*() operations should be used. 10 * raw_atomic*() operations should be used.
11 */ 11 */
12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H 12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
13 #define _LINUX_ATOMIC_INSTRUMENTED_H 13 #define _LINUX_ATOMIC_INSTRUMENTED_H
14 14
15 #include <linux/build_bug.h> 15 #include <linux/build_bug.h>
16 #include <linux/compiler.h> 16 #include <linux/compiler.h>
17 #include <linux/instrumented.h> 17 #include <linux/instrumented.h>
18 18
19 /** 19 /**
20 * atomic_read() - atomic load with relaxed or 20 * atomic_read() - atomic load with relaxed ordering
21 * @v: pointer to atomic_t 21 * @v: pointer to atomic_t
22 * 22 *
23 * Atomically loads the value of @v with relax 23 * Atomically loads the value of @v with relaxed ordering.
24 * 24 *
25 * Unsafe to use in noinstr code; use raw_atom 25 * Unsafe to use in noinstr code; use raw_atomic_read() there.
26 * 26 *
27 * Return: The value loaded from @v. 27 * Return: The value loaded from @v.
28 */ 28 */
29 static __always_inline int 29 static __always_inline int
30 atomic_read(const atomic_t *v) 30 atomic_read(const atomic_t *v)
31 { 31 {
32 instrument_atomic_read(v, sizeof(*v)); 32 instrument_atomic_read(v, sizeof(*v));
33 return raw_atomic_read(v); 33 return raw_atomic_read(v);
34 } 34 }
35 35
36 /** 36 /**
37 * atomic_read_acquire() - atomic load with ac 37 * atomic_read_acquire() - atomic load with acquire ordering
38 * @v: pointer to atomic_t 38 * @v: pointer to atomic_t
39 * 39 *
40 * Atomically loads the value of @v with acqui 40 * Atomically loads the value of @v with acquire ordering.
41 * 41 *
42 * Unsafe to use in noinstr code; use raw_atom 42 * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
43 * 43 *
44 * Return: The value loaded from @v. 44 * Return: The value loaded from @v.
45 */ 45 */
46 static __always_inline int 46 static __always_inline int
47 atomic_read_acquire(const atomic_t *v) 47 atomic_read_acquire(const atomic_t *v)
48 { 48 {
49 instrument_atomic_read(v, sizeof(*v)); 49 instrument_atomic_read(v, sizeof(*v));
50 return raw_atomic_read_acquire(v); 50 return raw_atomic_read_acquire(v);
51 } 51 }
52 52
53 /** 53 /**
54 * atomic_set() - atomic set with relaxed orde 54 * atomic_set() - atomic set with relaxed ordering
55 * @v: pointer to atomic_t 55 * @v: pointer to atomic_t
56 * @i: int value to assign 56 * @i: int value to assign
57 * 57 *
58 * Atomically sets @v to @i with relaxed order 58 * Atomically sets @v to @i with relaxed ordering.
59 * 59 *
60 * Unsafe to use in noinstr code; use raw_atom 60 * Unsafe to use in noinstr code; use raw_atomic_set() there.
61 * 61 *
62 * Return: Nothing. 62 * Return: Nothing.
63 */ 63 */
64 static __always_inline void 64 static __always_inline void
65 atomic_set(atomic_t *v, int i) 65 atomic_set(atomic_t *v, int i)
66 { 66 {
67 instrument_atomic_write(v, sizeof(*v)) 67 instrument_atomic_write(v, sizeof(*v));
68 raw_atomic_set(v, i); 68 raw_atomic_set(v, i);
69 } 69 }
70 70
71 /** 71 /**
72 * atomic_set_release() - atomic set with rele 72 * atomic_set_release() - atomic set with release ordering
73 * @v: pointer to atomic_t 73 * @v: pointer to atomic_t
74 * @i: int value to assign 74 * @i: int value to assign
75 * 75 *
76 * Atomically sets @v to @i with release order 76 * Atomically sets @v to @i with release ordering.
77 * 77 *
78 * Unsafe to use in noinstr code; use raw_atom 78 * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
79 * 79 *
80 * Return: Nothing. 80 * Return: Nothing.
81 */ 81 */
82 static __always_inline void 82 static __always_inline void
83 atomic_set_release(atomic_t *v, int i) 83 atomic_set_release(atomic_t *v, int i)
84 { 84 {
85 kcsan_release(); 85 kcsan_release();
86 instrument_atomic_write(v, sizeof(*v)) 86 instrument_atomic_write(v, sizeof(*v));
87 raw_atomic_set_release(v, i); 87 raw_atomic_set_release(v, i);
88 } 88 }
89 89
90 /** 90 /**
91 * atomic_add() - atomic add with relaxed orde 91 * atomic_add() - atomic add with relaxed ordering
92 * @i: int value to add 92 * @i: int value to add
93 * @v: pointer to atomic_t 93 * @v: pointer to atomic_t
94 * 94 *
95 * Atomically updates @v to (@v + @i) with rel 95 * Atomically updates @v to (@v + @i) with relaxed ordering.
96 * 96 *
97 * Unsafe to use in noinstr code; use raw_atom 97 * Unsafe to use in noinstr code; use raw_atomic_add() there.
98 * 98 *
99 * Return: Nothing. 99 * Return: Nothing.
100 */ 100 */
101 static __always_inline void 101 static __always_inline void
102 atomic_add(int i, atomic_t *v) 102 atomic_add(int i, atomic_t *v)
103 { 103 {
104 instrument_atomic_read_write(v, sizeof 104 instrument_atomic_read_write(v, sizeof(*v));
105 raw_atomic_add(i, v); 105 raw_atomic_add(i, v);
106 } 106 }
107 107
108 /** 108 /**
109 * atomic_add_return() - atomic add with full 109 * atomic_add_return() - atomic add with full ordering
110 * @i: int value to add 110 * @i: int value to add
111 * @v: pointer to atomic_t 111 * @v: pointer to atomic_t
112 * 112 *
113 * Atomically updates @v to (@v + @i) with ful 113 * Atomically updates @v to (@v + @i) with full ordering.
114 * 114 *
115 * Unsafe to use in noinstr code; use raw_atom 115 * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
116 * 116 *
117 * Return: The updated value of @v. 117 * Return: The updated value of @v.
118 */ 118 */
119 static __always_inline int 119 static __always_inline int
120 atomic_add_return(int i, atomic_t *v) 120 atomic_add_return(int i, atomic_t *v)
121 { 121 {
122 kcsan_mb(); 122 kcsan_mb();
123 instrument_atomic_read_write(v, sizeof 123 instrument_atomic_read_write(v, sizeof(*v));
124 return raw_atomic_add_return(i, v); 124 return raw_atomic_add_return(i, v);
125 } 125 }
126 126
127 /** 127 /**
128 * atomic_add_return_acquire() - atomic add wi 128 * atomic_add_return_acquire() - atomic add with acquire ordering
129 * @i: int value to add 129 * @i: int value to add
130 * @v: pointer to atomic_t 130 * @v: pointer to atomic_t
131 * 131 *
132 * Atomically updates @v to (@v + @i) with acq 132 * Atomically updates @v to (@v + @i) with acquire ordering.
133 * 133 *
134 * Unsafe to use in noinstr code; use raw_atom 134 * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
135 * 135 *
136 * Return: The updated value of @v. 136 * Return: The updated value of @v.
137 */ 137 */
138 static __always_inline int 138 static __always_inline int
139 atomic_add_return_acquire(int i, atomic_t *v) 139 atomic_add_return_acquire(int i, atomic_t *v)
140 { 140 {
141 instrument_atomic_read_write(v, sizeof 141 instrument_atomic_read_write(v, sizeof(*v));
142 return raw_atomic_add_return_acquire(i 142 return raw_atomic_add_return_acquire(i, v);
143 } 143 }
144 144
145 /** 145 /**
146 * atomic_add_return_release() - atomic add wi 146 * atomic_add_return_release() - atomic add with release ordering
147 * @i: int value to add 147 * @i: int value to add
148 * @v: pointer to atomic_t 148 * @v: pointer to atomic_t
149 * 149 *
150 * Atomically updates @v to (@v + @i) with rel 150 * Atomically updates @v to (@v + @i) with release ordering.
151 * 151 *
152 * Unsafe to use in noinstr code; use raw_atom 152 * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
153 * 153 *
154 * Return: The updated value of @v. 154 * Return: The updated value of @v.
155 */ 155 */
156 static __always_inline int 156 static __always_inline int
157 atomic_add_return_release(int i, atomic_t *v) 157 atomic_add_return_release(int i, atomic_t *v)
158 { 158 {
159 kcsan_release(); 159 kcsan_release();
160 instrument_atomic_read_write(v, sizeof 160 instrument_atomic_read_write(v, sizeof(*v));
161 return raw_atomic_add_return_release(i 161 return raw_atomic_add_return_release(i, v);
162 } 162 }
163 163
164 /** 164 /**
165 * atomic_add_return_relaxed() - atomic add wi 165 * atomic_add_return_relaxed() - atomic add with relaxed ordering
166 * @i: int value to add 166 * @i: int value to add
167 * @v: pointer to atomic_t 167 * @v: pointer to atomic_t
168 * 168 *
169 * Atomically updates @v to (@v + @i) with rel 169 * Atomically updates @v to (@v + @i) with relaxed ordering.
170 * 170 *
171 * Unsafe to use in noinstr code; use raw_atom 171 * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
172 * 172 *
173 * Return: The updated value of @v. 173 * Return: The updated value of @v.
174 */ 174 */
175 static __always_inline int 175 static __always_inline int
176 atomic_add_return_relaxed(int i, atomic_t *v) 176 atomic_add_return_relaxed(int i, atomic_t *v)
177 { 177 {
178 instrument_atomic_read_write(v, sizeof 178 instrument_atomic_read_write(v, sizeof(*v));
179 return raw_atomic_add_return_relaxed(i 179 return raw_atomic_add_return_relaxed(i, v);
180 } 180 }
181 181
182 /** 182 /**
183 * atomic_fetch_add() - atomic add with full o 183 * atomic_fetch_add() - atomic add with full ordering
184 * @i: int value to add 184 * @i: int value to add
185 * @v: pointer to atomic_t 185 * @v: pointer to atomic_t
186 * 186 *
187 * Atomically updates @v to (@v + @i) with ful 187 * Atomically updates @v to (@v + @i) with full ordering.
188 * 188 *
189 * Unsafe to use in noinstr code; use raw_atom 189 * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
190 * 190 *
191 * Return: The original value of @v. 191 * Return: The original value of @v.
192 */ 192 */
193 static __always_inline int 193 static __always_inline int
194 atomic_fetch_add(int i, atomic_t *v) 194 atomic_fetch_add(int i, atomic_t *v)
195 { 195 {
196 kcsan_mb(); 196 kcsan_mb();
197 instrument_atomic_read_write(v, sizeof 197 instrument_atomic_read_write(v, sizeof(*v));
198 return raw_atomic_fetch_add(i, v); 198 return raw_atomic_fetch_add(i, v);
199 } 199 }
200 200
201 /** 201 /**
202 * atomic_fetch_add_acquire() - atomic add wit 202 * atomic_fetch_add_acquire() - atomic add with acquire ordering
203 * @i: int value to add 203 * @i: int value to add
204 * @v: pointer to atomic_t 204 * @v: pointer to atomic_t
205 * 205 *
206 * Atomically updates @v to (@v + @i) with acq 206 * Atomically updates @v to (@v + @i) with acquire ordering.
207 * 207 *
208 * Unsafe to use in noinstr code; use raw_atom 208 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
209 * 209 *
210 * Return: The original value of @v. 210 * Return: The original value of @v.
211 */ 211 */
212 static __always_inline int 212 static __always_inline int
213 atomic_fetch_add_acquire(int i, atomic_t *v) 213 atomic_fetch_add_acquire(int i, atomic_t *v)
214 { 214 {
215 instrument_atomic_read_write(v, sizeof 215 instrument_atomic_read_write(v, sizeof(*v));
216 return raw_atomic_fetch_add_acquire(i, 216 return raw_atomic_fetch_add_acquire(i, v);
217 } 217 }
218 218
219 /** 219 /**
220 * atomic_fetch_add_release() - atomic add wit 220 * atomic_fetch_add_release() - atomic add with release ordering
221 * @i: int value to add 221 * @i: int value to add
222 * @v: pointer to atomic_t 222 * @v: pointer to atomic_t
223 * 223 *
224 * Atomically updates @v to (@v + @i) with rel 224 * Atomically updates @v to (@v + @i) with release ordering.
225 * 225 *
226 * Unsafe to use in noinstr code; use raw_atom 226 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
227 * 227 *
228 * Return: The original value of @v. 228 * Return: The original value of @v.
229 */ 229 */
230 static __always_inline int 230 static __always_inline int
231 atomic_fetch_add_release(int i, atomic_t *v) 231 atomic_fetch_add_release(int i, atomic_t *v)
232 { 232 {
233 kcsan_release(); 233 kcsan_release();
234 instrument_atomic_read_write(v, sizeof 234 instrument_atomic_read_write(v, sizeof(*v));
235 return raw_atomic_fetch_add_release(i, 235 return raw_atomic_fetch_add_release(i, v);
236 } 236 }
237 237
238 /** 238 /**
239 * atomic_fetch_add_relaxed() - atomic add wit 239 * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
240 * @i: int value to add 240 * @i: int value to add
241 * @v: pointer to atomic_t 241 * @v: pointer to atomic_t
242 * 242 *
243 * Atomically updates @v to (@v + @i) with rel 243 * Atomically updates @v to (@v + @i) with relaxed ordering.
244 * 244 *
245 * Unsafe to use in noinstr code; use raw_atom 245 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
246 * 246 *
247 * Return: The original value of @v. 247 * Return: The original value of @v.
248 */ 248 */
249 static __always_inline int 249 static __always_inline int
250 atomic_fetch_add_relaxed(int i, atomic_t *v) 250 atomic_fetch_add_relaxed(int i, atomic_t *v)
251 { 251 {
252 instrument_atomic_read_write(v, sizeof 252 instrument_atomic_read_write(v, sizeof(*v));
253 return raw_atomic_fetch_add_relaxed(i, 253 return raw_atomic_fetch_add_relaxed(i, v);
254 } 254 }
255 255
256 /** 256 /**
257 * atomic_sub() - atomic subtract with relaxed 257 * atomic_sub() - atomic subtract with relaxed ordering
258 * @i: int value to subtract 258 * @i: int value to subtract
259 * @v: pointer to atomic_t 259 * @v: pointer to atomic_t
260 * 260 *
261 * Atomically updates @v to (@v - @i) with rel 261 * Atomically updates @v to (@v - @i) with relaxed ordering.
262 * 262 *
263 * Unsafe to use in noinstr code; use raw_atom 263 * Unsafe to use in noinstr code; use raw_atomic_sub() there.
264 * 264 *
265 * Return: Nothing. 265 * Return: Nothing.
266 */ 266 */
267 static __always_inline void 267 static __always_inline void
268 atomic_sub(int i, atomic_t *v) 268 atomic_sub(int i, atomic_t *v)
269 { 269 {
270 instrument_atomic_read_write(v, sizeof 270 instrument_atomic_read_write(v, sizeof(*v));
271 raw_atomic_sub(i, v); 271 raw_atomic_sub(i, v);
272 } 272 }
273 273
274 /** 274 /**
275 * atomic_sub_return() - atomic subtract with 275 * atomic_sub_return() - atomic subtract with full ordering
276 * @i: int value to subtract 276 * @i: int value to subtract
277 * @v: pointer to atomic_t 277 * @v: pointer to atomic_t
278 * 278 *
279 * Atomically updates @v to (@v - @i) with ful 279 * Atomically updates @v to (@v - @i) with full ordering.
280 * 280 *
281 * Unsafe to use in noinstr code; use raw_atom 281 * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
282 * 282 *
283 * Return: The updated value of @v. 283 * Return: The updated value of @v.
284 */ 284 */
285 static __always_inline int 285 static __always_inline int
286 atomic_sub_return(int i, atomic_t *v) 286 atomic_sub_return(int i, atomic_t *v)
287 { 287 {
288 kcsan_mb(); 288 kcsan_mb();
289 instrument_atomic_read_write(v, sizeof 289 instrument_atomic_read_write(v, sizeof(*v));
290 return raw_atomic_sub_return(i, v); 290 return raw_atomic_sub_return(i, v);
291 } 291 }
292 292
293 /** 293 /**
294 * atomic_sub_return_acquire() - atomic subtra 294 * atomic_sub_return_acquire() - atomic subtract with acquire ordering
295 * @i: int value to subtract 295 * @i: int value to subtract
296 * @v: pointer to atomic_t 296 * @v: pointer to atomic_t
297 * 297 *
298 * Atomically updates @v to (@v - @i) with acq 298 * Atomically updates @v to (@v - @i) with acquire ordering.
299 * 299 *
300 * Unsafe to use in noinstr code; use raw_atom 300 * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
301 * 301 *
302 * Return: The updated value of @v. 302 * Return: The updated value of @v.
303 */ 303 */
304 static __always_inline int 304 static __always_inline int
305 atomic_sub_return_acquire(int i, atomic_t *v) 305 atomic_sub_return_acquire(int i, atomic_t *v)
306 { 306 {
307 instrument_atomic_read_write(v, sizeof 307 instrument_atomic_read_write(v, sizeof(*v));
308 return raw_atomic_sub_return_acquire(i 308 return raw_atomic_sub_return_acquire(i, v);
309 } 309 }
310 310
311 /** 311 /**
312 * atomic_sub_return_release() - atomic subtra 312 * atomic_sub_return_release() - atomic subtract with release ordering
313 * @i: int value to subtract 313 * @i: int value to subtract
314 * @v: pointer to atomic_t 314 * @v: pointer to atomic_t
315 * 315 *
316 * Atomically updates @v to (@v - @i) with rel 316 * Atomically updates @v to (@v - @i) with release ordering.
317 * 317 *
318 * Unsafe to use in noinstr code; use raw_atom 318 * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
319 * 319 *
320 * Return: The updated value of @v. 320 * Return: The updated value of @v.
321 */ 321 */
322 static __always_inline int 322 static __always_inline int
323 atomic_sub_return_release(int i, atomic_t *v) 323 atomic_sub_return_release(int i, atomic_t *v)
324 { 324 {
325 kcsan_release(); 325 kcsan_release();
326 instrument_atomic_read_write(v, sizeof 326 instrument_atomic_read_write(v, sizeof(*v));
327 return raw_atomic_sub_return_release(i 327 return raw_atomic_sub_return_release(i, v);
328 } 328 }
329 329
330 /** 330 /**
331 * atomic_sub_return_relaxed() - atomic subtra 331 * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
332 * @i: int value to subtract 332 * @i: int value to subtract
333 * @v: pointer to atomic_t 333 * @v: pointer to atomic_t
334 * 334 *
335 * Atomically updates @v to (@v - @i) with rel 335 * Atomically updates @v to (@v - @i) with relaxed ordering.
336 * 336 *
337 * Unsafe to use in noinstr code; use raw_atom 337 * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
338 * 338 *
339 * Return: The updated value of @v. 339 * Return: The updated value of @v.
340 */ 340 */
341 static __always_inline int 341 static __always_inline int
342 atomic_sub_return_relaxed(int i, atomic_t *v) 342 atomic_sub_return_relaxed(int i, atomic_t *v)
343 { 343 {
344 instrument_atomic_read_write(v, sizeof 344 instrument_atomic_read_write(v, sizeof(*v));
345 return raw_atomic_sub_return_relaxed(i 345 return raw_atomic_sub_return_relaxed(i, v);
346 } 346 }
347 347
348 /** 348 /**
349 * atomic_fetch_sub() - atomic subtract with f 349 * atomic_fetch_sub() - atomic subtract with full ordering
350 * @i: int value to subtract 350 * @i: int value to subtract
351 * @v: pointer to atomic_t 351 * @v: pointer to atomic_t
352 * 352 *
353 * Atomically updates @v to (@v - @i) with ful 353 * Atomically updates @v to (@v - @i) with full ordering.
354 * 354 *
355 * Unsafe to use in noinstr code; use raw_atom 355 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
356 * 356 *
357 * Return: The original value of @v. 357 * Return: The original value of @v.
358 */ 358 */
359 static __always_inline int 359 static __always_inline int
360 atomic_fetch_sub(int i, atomic_t *v) 360 atomic_fetch_sub(int i, atomic_t *v)
361 { 361 {
362 kcsan_mb(); 362 kcsan_mb();
363 instrument_atomic_read_write(v, sizeof 363 instrument_atomic_read_write(v, sizeof(*v));
364 return raw_atomic_fetch_sub(i, v); 364 return raw_atomic_fetch_sub(i, v);
365 } 365 }
366 366
367 /** 367 /**
368 * atomic_fetch_sub_acquire() - atomic subtrac 368 * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
369 * @i: int value to subtract 369 * @i: int value to subtract
370 * @v: pointer to atomic_t 370 * @v: pointer to atomic_t
371 * 371 *
372 * Atomically updates @v to (@v - @i) with acq 372 * Atomically updates @v to (@v - @i) with acquire ordering.
373 * 373 *
374 * Unsafe to use in noinstr code; use raw_atom 374 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
375 * 375 *
376 * Return: The original value of @v. 376 * Return: The original value of @v.
377 */ 377 */
378 static __always_inline int 378 static __always_inline int
379 atomic_fetch_sub_acquire(int i, atomic_t *v) 379 atomic_fetch_sub_acquire(int i, atomic_t *v)
380 { 380 {
381 instrument_atomic_read_write(v, sizeof 381 instrument_atomic_read_write(v, sizeof(*v));
382 return raw_atomic_fetch_sub_acquire(i, 382 return raw_atomic_fetch_sub_acquire(i, v);
383 } 383 }
384 384
385 /** 385 /**
386 * atomic_fetch_sub_release() - atomic subtrac 386 * atomic_fetch_sub_release() - atomic subtract with release ordering
387 * @i: int value to subtract 387 * @i: int value to subtract
388 * @v: pointer to atomic_t 388 * @v: pointer to atomic_t
389 * 389 *
390 * Atomically updates @v to (@v - @i) with rel 390 * Atomically updates @v to (@v - @i) with release ordering.
391 * 391 *
392 * Unsafe to use in noinstr code; use raw_atom 392 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
393 * 393 *
394 * Return: The original value of @v. 394 * Return: The original value of @v.
395 */ 395 */
396 static __always_inline int 396 static __always_inline int
397 atomic_fetch_sub_release(int i, atomic_t *v) 397 atomic_fetch_sub_release(int i, atomic_t *v)
398 { 398 {
399 kcsan_release(); 399 kcsan_release();
400 instrument_atomic_read_write(v, sizeof 400 instrument_atomic_read_write(v, sizeof(*v));
401 return raw_atomic_fetch_sub_release(i, 401 return raw_atomic_fetch_sub_release(i, v);
402 } 402 }
403 403
404 /** 404 /**
405 * atomic_fetch_sub_relaxed() - atomic subtrac 405 * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
406 * @i: int value to subtract 406 * @i: int value to subtract
407 * @v: pointer to atomic_t 407 * @v: pointer to atomic_t
408 * 408 *
409 * Atomically updates @v to (@v - @i) with rel 409 * Atomically updates @v to (@v - @i) with relaxed ordering.
410 * 410 *
411 * Unsafe to use in noinstr code; use raw_atom 411 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
412 * 412 *
413 * Return: The original value of @v. 413 * Return: The original value of @v.
414 */ 414 */
415 static __always_inline int 415 static __always_inline int
416 atomic_fetch_sub_relaxed(int i, atomic_t *v) 416 atomic_fetch_sub_relaxed(int i, atomic_t *v)
417 { 417 {
418 instrument_atomic_read_write(v, sizeof 418 instrument_atomic_read_write(v, sizeof(*v));
419 return raw_atomic_fetch_sub_relaxed(i, 419 return raw_atomic_fetch_sub_relaxed(i, v);
420 } 420 }
421 421
422 /** 422 /**
423 * atomic_inc() - atomic increment with relaxe 423 * atomic_inc() - atomic increment with relaxed ordering
424 * @v: pointer to atomic_t 424 * @v: pointer to atomic_t
425 * 425 *
426 * Atomically updates @v to (@v + 1) with rela 426 * Atomically updates @v to (@v + 1) with relaxed ordering.
427 * 427 *
428 * Unsafe to use in noinstr code; use raw_atom 428 * Unsafe to use in noinstr code; use raw_atomic_inc() there.
429 * 429 *
430 * Return: Nothing. 430 * Return: Nothing.
431 */ 431 */
432 static __always_inline void 432 static __always_inline void
433 atomic_inc(atomic_t *v) 433 atomic_inc(atomic_t *v)
434 { 434 {
435 instrument_atomic_read_write(v, sizeof 435 instrument_atomic_read_write(v, sizeof(*v));
436 raw_atomic_inc(v); 436 raw_atomic_inc(v);
437 } 437 }
438 438
439 /** 439 /**
440 * atomic_inc_return() - atomic increment with 440 * atomic_inc_return() - atomic increment with full ordering
441 * @v: pointer to atomic_t 441 * @v: pointer to atomic_t
442 * 442 *
443 * Atomically updates @v to (@v + 1) with full 443 * Atomically updates @v to (@v + 1) with full ordering.
444 * 444 *
445 * Unsafe to use in noinstr code; use raw_atom 445 * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
446 * 446 *
447 * Return: The updated value of @v. 447 * Return: The updated value of @v.
448 */ 448 */
449 static __always_inline int 449 static __always_inline int
450 atomic_inc_return(atomic_t *v) 450 atomic_inc_return(atomic_t *v)
451 { 451 {
452 kcsan_mb(); 452 kcsan_mb();
453 instrument_atomic_read_write(v, sizeof 453 instrument_atomic_read_write(v, sizeof(*v));
454 return raw_atomic_inc_return(v); 454 return raw_atomic_inc_return(v);
455 } 455 }
456 456
457 /** 457 /**
458 * atomic_inc_return_acquire() - atomic increm 458 * atomic_inc_return_acquire() - atomic increment with acquire ordering
459 * @v: pointer to atomic_t 459 * @v: pointer to atomic_t
460 * 460 *
461 * Atomically updates @v to (@v + 1) with acqu 461 * Atomically updates @v to (@v + 1) with acquire ordering.
462 * 462 *
463 * Unsafe to use in noinstr code; use raw_atom 463 * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
464 * 464 *
465 * Return: The updated value of @v. 465 * Return: The updated value of @v.
466 */ 466 */
467 static __always_inline int 467 static __always_inline int
468 atomic_inc_return_acquire(atomic_t *v) 468 atomic_inc_return_acquire(atomic_t *v)
469 { 469 {
470 instrument_atomic_read_write(v, sizeof 470 instrument_atomic_read_write(v, sizeof(*v));
471 return raw_atomic_inc_return_acquire(v 471 return raw_atomic_inc_return_acquire(v);
472 } 472 }
473 473
474 /** 474 /**
475 * atomic_inc_return_release() - atomic increm 475 * atomic_inc_return_release() - atomic increment with release ordering
476 * @v: pointer to atomic_t 476 * @v: pointer to atomic_t
477 * 477 *
478 * Atomically updates @v to (@v + 1) with rele 478 * Atomically updates @v to (@v + 1) with release ordering.
479 * 479 *
480 * Unsafe to use in noinstr code; use raw_atom 480 * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
481 * 481 *
482 * Return: The updated value of @v. 482 * Return: The updated value of @v.
483 */ 483 */
484 static __always_inline int 484 static __always_inline int
485 atomic_inc_return_release(atomic_t *v) 485 atomic_inc_return_release(atomic_t *v)
486 { 486 {
487 kcsan_release(); 487 kcsan_release();
488 instrument_atomic_read_write(v, sizeof 488 instrument_atomic_read_write(v, sizeof(*v));
489 return raw_atomic_inc_return_release(v 489 return raw_atomic_inc_return_release(v);
490 } 490 }
491 491
492 /** 492 /**
493 * atomic_inc_return_relaxed() - atomic increm 493 * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
494 * @v: pointer to atomic_t 494 * @v: pointer to atomic_t
495 * 495 *
496 * Atomically updates @v to (@v + 1) with rela 496 * Atomically updates @v to (@v + 1) with relaxed ordering.
497 * 497 *
498 * Unsafe to use in noinstr code; use raw_atom 498 * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
499 * 499 *
500 * Return: The updated value of @v. 500 * Return: The updated value of @v.
501 */ 501 */
502 static __always_inline int 502 static __always_inline int
503 atomic_inc_return_relaxed(atomic_t *v) 503 atomic_inc_return_relaxed(atomic_t *v)
504 { 504 {
505 instrument_atomic_read_write(v, sizeof 505 instrument_atomic_read_write(v, sizeof(*v));
506 return raw_atomic_inc_return_relaxed(v 506 return raw_atomic_inc_return_relaxed(v);
507 } 507 }
508 508
509 /** 509 /**
510 * atomic_fetch_inc() - atomic increment with 510 * atomic_fetch_inc() - atomic increment with full ordering
511 * @v: pointer to atomic_t 511 * @v: pointer to atomic_t
512 * 512 *
513 * Atomically updates @v to (@v + 1) with full 513 * Atomically updates @v to (@v + 1) with full ordering.
514 * 514 *
515 * Unsafe to use in noinstr code; use raw_atom 515 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
516 * 516 *
517 * Return: The original value of @v. 517 * Return: The original value of @v.
518 */ 518 */
519 static __always_inline int 519 static __always_inline int
520 atomic_fetch_inc(atomic_t *v) 520 atomic_fetch_inc(atomic_t *v)
521 { 521 {
522 kcsan_mb(); 522 kcsan_mb();
523 instrument_atomic_read_write(v, sizeof 523 instrument_atomic_read_write(v, sizeof(*v));
524 return raw_atomic_fetch_inc(v); 524 return raw_atomic_fetch_inc(v);
525 } 525 }
526 526
527 /** 527 /**
528 * atomic_fetch_inc_acquire() - atomic increme 528 * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
529 * @v: pointer to atomic_t 529 * @v: pointer to atomic_t
530 * 530 *
531 * Atomically updates @v to (@v + 1) with acqu 531 * Atomically updates @v to (@v + 1) with acquire ordering.
532 * 532 *
533 * Unsafe to use in noinstr code; use raw_atom 533 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
534 * 534 *
535 * Return: The original value of @v. 535 * Return: The original value of @v.
536 */ 536 */
537 static __always_inline int 537 static __always_inline int
538 atomic_fetch_inc_acquire(atomic_t *v) 538 atomic_fetch_inc_acquire(atomic_t *v)
539 { 539 {
540 instrument_atomic_read_write(v, sizeof 540 instrument_atomic_read_write(v, sizeof(*v));
541 return raw_atomic_fetch_inc_acquire(v) 541 return raw_atomic_fetch_inc_acquire(v);
542 } 542 }
543 543
544 /** 544 /**
545 * atomic_fetch_inc_release() - atomic increme 545 * atomic_fetch_inc_release() - atomic increment with release ordering
546 * @v: pointer to atomic_t 546 * @v: pointer to atomic_t
547 * 547 *
548 * Atomically updates @v to (@v + 1) with rele 548 * Atomically updates @v to (@v + 1) with release ordering.
549 * 549 *
550 * Unsafe to use in noinstr code; use raw_atom 550 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
551 * 551 *
552 * Return: The original value of @v. 552 * Return: The original value of @v.
553 */ 553 */
554 static __always_inline int 554 static __always_inline int
555 atomic_fetch_inc_release(atomic_t *v) 555 atomic_fetch_inc_release(atomic_t *v)
556 { 556 {
557 kcsan_release(); 557 kcsan_release();
558 instrument_atomic_read_write(v, sizeof 558 instrument_atomic_read_write(v, sizeof(*v));
559 return raw_atomic_fetch_inc_release(v) 559 return raw_atomic_fetch_inc_release(v);
560 } 560 }
561 561
562 /** 562 /**
563 * atomic_fetch_inc_relaxed() - atomic increme 563 * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
564 * @v: pointer to atomic_t 564 * @v: pointer to atomic_t
565 * 565 *
566 * Atomically updates @v to (@v + 1) with rela 566 * Atomically updates @v to (@v + 1) with relaxed ordering.
567 * 567 *
568 * Unsafe to use in noinstr code; use raw_atom 568 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
569 * 569 *
570 * Return: The original value of @v. 570 * Return: The original value of @v.
571 */ 571 */
572 static __always_inline int 572 static __always_inline int
573 atomic_fetch_inc_relaxed(atomic_t *v) 573 atomic_fetch_inc_relaxed(atomic_t *v)
574 { 574 {
575 instrument_atomic_read_write(v, sizeof 575 instrument_atomic_read_write(v, sizeof(*v));
576 return raw_atomic_fetch_inc_relaxed(v) 576 return raw_atomic_fetch_inc_relaxed(v);
577 } 577 }
578 578
579 /** 579 /**
580 * atomic_dec() - atomic decrement with relaxe 580 * atomic_dec() - atomic decrement with relaxed ordering
581 * @v: pointer to atomic_t 581 * @v: pointer to atomic_t
582 * 582 *
583 * Atomically updates @v to (@v - 1) with rela 583 * Atomically updates @v to (@v - 1) with relaxed ordering.
584 * 584 *
585 * Unsafe to use in noinstr code; use raw_atom 585 * Unsafe to use in noinstr code; use raw_atomic_dec() there.
586 * 586 *
587 * Return: Nothing. 587 * Return: Nothing.
588 */ 588 */
589 static __always_inline void 589 static __always_inline void
590 atomic_dec(atomic_t *v) 590 atomic_dec(atomic_t *v)
591 { 591 {
592 instrument_atomic_read_write(v, sizeof 592 instrument_atomic_read_write(v, sizeof(*v));
593 raw_atomic_dec(v); 593 raw_atomic_dec(v);
594 } 594 }
595 595
596 /** 596 /**
597 * atomic_dec_return() - atomic decrement with 597 * atomic_dec_return() - atomic decrement with full ordering
598 * @v: pointer to atomic_t 598 * @v: pointer to atomic_t
599 * 599 *
600 * Atomically updates @v to (@v - 1) with full 600 * Atomically updates @v to (@v - 1) with full ordering.
601 * 601 *
602 * Unsafe to use in noinstr code; use raw_atom 602 * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
603 * 603 *
604 * Return: The updated value of @v. 604 * Return: The updated value of @v.
605 */ 605 */
606 static __always_inline int 606 static __always_inline int
607 atomic_dec_return(atomic_t *v) 607 atomic_dec_return(atomic_t *v)
608 { 608 {
609 kcsan_mb(); 609 kcsan_mb();
610 instrument_atomic_read_write(v, sizeof 610 instrument_atomic_read_write(v, sizeof(*v));
611 return raw_atomic_dec_return(v); 611 return raw_atomic_dec_return(v);
612 } 612 }
613 613
614 /** 614 /**
615 * atomic_dec_return_acquire() - atomic decrem 615 * atomic_dec_return_acquire() - atomic decrement with acquire ordering
616 * @v: pointer to atomic_t 616 * @v: pointer to atomic_t
617 * 617 *
618 * Atomically updates @v to (@v - 1) with acqu 618 * Atomically updates @v to (@v - 1) with acquire ordering.
619 * 619 *
620 * Unsafe to use in noinstr code; use raw_atom 620 * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
621 * 621 *
622 * Return: The updated value of @v. 622 * Return: The updated value of @v.
623 */ 623 */
624 static __always_inline int 624 static __always_inline int
625 atomic_dec_return_acquire(atomic_t *v) 625 atomic_dec_return_acquire(atomic_t *v)
626 { 626 {
627 instrument_atomic_read_write(v, sizeof 627 instrument_atomic_read_write(v, sizeof(*v));
628 return raw_atomic_dec_return_acquire(v 628 return raw_atomic_dec_return_acquire(v);
629 } 629 }
630 630
631 /** 631 /**
632 * atomic_dec_return_release() - atomic decrem 632 * atomic_dec_return_release() - atomic decrement with release ordering
633 * @v: pointer to atomic_t 633 * @v: pointer to atomic_t
634 * 634 *
635 * Atomically updates @v to (@v - 1) with rele 635 * Atomically updates @v to (@v - 1) with release ordering.
636 * 636 *
637 * Unsafe to use in noinstr code; use raw_atom 637 * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
638 * 638 *
639 * Return: The updated value of @v. 639 * Return: The updated value of @v.
640 */ 640 */
641 static __always_inline int 641 static __always_inline int
642 atomic_dec_return_release(atomic_t *v) 642 atomic_dec_return_release(atomic_t *v)
643 { 643 {
644 kcsan_release(); 644 kcsan_release();
645 instrument_atomic_read_write(v, sizeof 645 instrument_atomic_read_write(v, sizeof(*v));
646 return raw_atomic_dec_return_release(v 646 return raw_atomic_dec_return_release(v);
647 } 647 }
648 648
649 /** 649 /**
650 * atomic_dec_return_relaxed() - atomic decrem 650 * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
651 * @v: pointer to atomic_t 651 * @v: pointer to atomic_t
652 * 652 *
653 * Atomically updates @v to (@v - 1) with rela 653 * Atomically updates @v to (@v - 1) with relaxed ordering.
654 * 654 *
655 * Unsafe to use in noinstr code; use raw_atom 655 * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
656 * 656 *
657 * Return: The updated value of @v. 657 * Return: The updated value of @v.
658 */ 658 */
659 static __always_inline int 659 static __always_inline int
660 atomic_dec_return_relaxed(atomic_t *v) 660 atomic_dec_return_relaxed(atomic_t *v)
661 { 661 {
662 instrument_atomic_read_write(v, sizeof 662 instrument_atomic_read_write(v, sizeof(*v));
663 return raw_atomic_dec_return_relaxed(v 663 return raw_atomic_dec_return_relaxed(v);
664 } 664 }
665 665
666 /** 666 /**
667 * atomic_fetch_dec() - atomic decrement with 667 * atomic_fetch_dec() - atomic decrement with full ordering
668 * @v: pointer to atomic_t 668 * @v: pointer to atomic_t
669 * 669 *
670 * Atomically updates @v to (@v - 1) with full 670 * Atomically updates @v to (@v - 1) with full ordering.
671 * 671 *
672 * Unsafe to use in noinstr code; use raw_atom 672 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
673 * 673 *
674 * Return: The original value of @v. 674 * Return: The original value of @v.
675 */ 675 */
676 static __always_inline int 676 static __always_inline int
677 atomic_fetch_dec(atomic_t *v) 677 atomic_fetch_dec(atomic_t *v)
678 { 678 {
679 kcsan_mb(); 679 kcsan_mb();
680 instrument_atomic_read_write(v, sizeof 680 instrument_atomic_read_write(v, sizeof(*v));
681 return raw_atomic_fetch_dec(v); 681 return raw_atomic_fetch_dec(v);
682 } 682 }
683 683
684 /** 684 /**
685 * atomic_fetch_dec_acquire() - atomic decreme 685 * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
686 * @v: pointer to atomic_t 686 * @v: pointer to atomic_t
687 * 687 *
688 * Atomically updates @v to (@v - 1) with acqu 688 * Atomically updates @v to (@v - 1) with acquire ordering.
689 * 689 *
690 * Unsafe to use in noinstr code; use raw_atom 690 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
691 * 691 *
692 * Return: The original value of @v. 692 * Return: The original value of @v.
693 */ 693 */
694 static __always_inline int 694 static __always_inline int
695 atomic_fetch_dec_acquire(atomic_t *v) 695 atomic_fetch_dec_acquire(atomic_t *v)
696 { 696 {
697 instrument_atomic_read_write(v, sizeof 697 instrument_atomic_read_write(v, sizeof(*v));
698 return raw_atomic_fetch_dec_acquire(v) 698 return raw_atomic_fetch_dec_acquire(v);
699 } 699 }
700 700
701 /** 701 /**
702 * atomic_fetch_dec_release() - atomic decreme 702 * atomic_fetch_dec_release() - atomic decrement with release ordering
703 * @v: pointer to atomic_t 703 * @v: pointer to atomic_t
704 * 704 *
705 * Atomically updates @v to (@v - 1) with rele 705 * Atomically updates @v to (@v - 1) with release ordering.
706 * 706 *
707 * Unsafe to use in noinstr code; use raw_atom 707 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
708 * 708 *
709 * Return: The original value of @v. 709 * Return: The original value of @v.
710 */ 710 */
711 static __always_inline int 711 static __always_inline int
712 atomic_fetch_dec_release(atomic_t *v) 712 atomic_fetch_dec_release(atomic_t *v)
713 { 713 {
714 kcsan_release(); 714 kcsan_release();
715 instrument_atomic_read_write(v, sizeof 715 instrument_atomic_read_write(v, sizeof(*v));
716 return raw_atomic_fetch_dec_release(v) 716 return raw_atomic_fetch_dec_release(v);
717 } 717 }
718 718
719 /** 719 /**
720 * atomic_fetch_dec_relaxed() - atomic decreme 720 * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
721 * @v: pointer to atomic_t 721 * @v: pointer to atomic_t
722 * 722 *
723 * Atomically updates @v to (@v - 1) with rela 723 * Atomically updates @v to (@v - 1) with relaxed ordering.
724 * 724 *
725 * Unsafe to use in noinstr code; use raw_atom 725 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
726 * 726 *
727 * Return: The original value of @v. 727 * Return: The original value of @v.
728 */ 728 */
729 static __always_inline int 729 static __always_inline int
730 atomic_fetch_dec_relaxed(atomic_t *v) 730 atomic_fetch_dec_relaxed(atomic_t *v)
731 { 731 {
732 instrument_atomic_read_write(v, sizeof 732 instrument_atomic_read_write(v, sizeof(*v));
733 return raw_atomic_fetch_dec_relaxed(v) 733 return raw_atomic_fetch_dec_relaxed(v);
734 } 734 }
735 735
736 /** 736 /**
737 * atomic_and() - atomic bitwise AND with rela 737 * atomic_and() - atomic bitwise AND with relaxed ordering
738 * @i: int value 738 * @i: int value
739 * @v: pointer to atomic_t 739 * @v: pointer to atomic_t
740 * 740 *
741 * Atomically updates @v to (@v & @i) with rel 741 * Atomically updates @v to (@v & @i) with relaxed ordering.
742 * 742 *
743 * Unsafe to use in noinstr code; use raw_atom 743 * Unsafe to use in noinstr code; use raw_atomic_and() there.
744 * 744 *
745 * Return: Nothing. 745 * Return: Nothing.
746 */ 746 */
747 static __always_inline void 747 static __always_inline void
748 atomic_and(int i, atomic_t *v) 748 atomic_and(int i, atomic_t *v)
749 { 749 {
750 instrument_atomic_read_write(v, sizeof 750 instrument_atomic_read_write(v, sizeof(*v));
751 raw_atomic_and(i, v); 751 raw_atomic_and(i, v);
752 } 752 }
753 753
754 /** 754 /**
755 * atomic_fetch_and() - atomic bitwise AND wit 755 * atomic_fetch_and() - atomic bitwise AND with full ordering
756 * @i: int value 756 * @i: int value
757 * @v: pointer to atomic_t 757 * @v: pointer to atomic_t
758 * 758 *
759 * Atomically updates @v to (@v & @i) with ful 759 * Atomically updates @v to (@v & @i) with full ordering.
760 * 760 *
761 * Unsafe to use in noinstr code; use raw_atom 761 * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
762 * 762 *
763 * Return: The original value of @v. 763 * Return: The original value of @v.
764 */ 764 */
765 static __always_inline int 765 static __always_inline int
766 atomic_fetch_and(int i, atomic_t *v) 766 atomic_fetch_and(int i, atomic_t *v)
767 { 767 {
768 kcsan_mb(); 768 kcsan_mb();
769 instrument_atomic_read_write(v, sizeof 769 instrument_atomic_read_write(v, sizeof(*v));
770 return raw_atomic_fetch_and(i, v); 770 return raw_atomic_fetch_and(i, v);
771 } 771 }
772 772
773 /** 773 /**
774 * atomic_fetch_and_acquire() - atomic bitwise 774 * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
775 * @i: int value 775 * @i: int value
776 * @v: pointer to atomic_t 776 * @v: pointer to atomic_t
777 * 777 *
778 * Atomically updates @v to (@v & @i) with acq 778 * Atomically updates @v to (@v & @i) with acquire ordering.
779 * 779 *
780 * Unsafe to use in noinstr code; use raw_atom 780 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
781 * 781 *
782 * Return: The original value of @v. 782 * Return: The original value of @v.
783 */ 783 */
784 static __always_inline int 784 static __always_inline int
785 atomic_fetch_and_acquire(int i, atomic_t *v) 785 atomic_fetch_and_acquire(int i, atomic_t *v)
786 { 786 {
787 instrument_atomic_read_write(v, sizeof 787 instrument_atomic_read_write(v, sizeof(*v));
788 return raw_atomic_fetch_and_acquire(i, 788 return raw_atomic_fetch_and_acquire(i, v);
789 } 789 }
790 790
791 /** 791 /**
792 * atomic_fetch_and_release() - atomic bitwise 792 * atomic_fetch_and_release() - atomic bitwise AND with release ordering
793 * @i: int value 793 * @i: int value
794 * @v: pointer to atomic_t 794 * @v: pointer to atomic_t
795 * 795 *
796 * Atomically updates @v to (@v & @i) with rel 796 * Atomically updates @v to (@v & @i) with release ordering.
797 * 797 *
798 * Unsafe to use in noinstr code; use raw_atom 798 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
799 * 799 *
800 * Return: The original value of @v. 800 * Return: The original value of @v.
801 */ 801 */
802 static __always_inline int 802 static __always_inline int
803 atomic_fetch_and_release(int i, atomic_t *v) 803 atomic_fetch_and_release(int i, atomic_t *v)
804 { 804 {
805 kcsan_release(); 805 kcsan_release();
806 instrument_atomic_read_write(v, sizeof 806 instrument_atomic_read_write(v, sizeof(*v));
807 return raw_atomic_fetch_and_release(i, 807 return raw_atomic_fetch_and_release(i, v);
808 } 808 }
809 809
810 /** 810 /**
811 * atomic_fetch_and_relaxed() - atomic bitwise 811 * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
812 * @i: int value 812 * @i: int value
813 * @v: pointer to atomic_t 813 * @v: pointer to atomic_t
814 * 814 *
815 * Atomically updates @v to (@v & @i) with rel 815 * Atomically updates @v to (@v & @i) with relaxed ordering.
816 * 816 *
817 * Unsafe to use in noinstr code; use raw_atom 817 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
818 * 818 *
819 * Return: The original value of @v. 819 * Return: The original value of @v.
820 */ 820 */
821 static __always_inline int 821 static __always_inline int
822 atomic_fetch_and_relaxed(int i, atomic_t *v) 822 atomic_fetch_and_relaxed(int i, atomic_t *v)
823 { 823 {
824 instrument_atomic_read_write(v, sizeof 824 instrument_atomic_read_write(v, sizeof(*v));
825 return raw_atomic_fetch_and_relaxed(i, 825 return raw_atomic_fetch_and_relaxed(i, v);
826 } 826 }
827 827
828 /** 828 /**
829 * atomic_andnot() - atomic bitwise AND NOT wi 829 * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
830 * @i: int value 830 * @i: int value
831 * @v: pointer to atomic_t 831 * @v: pointer to atomic_t
832 * 832 *
833 * Atomically updates @v to (@v & ~@i) with re 833 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
834 * 834 *
835 * Unsafe to use in noinstr code; use raw_atom 835 * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
836 * 836 *
837 * Return: Nothing. 837 * Return: Nothing.
838 */ 838 */
839 static __always_inline void 839 static __always_inline void
840 atomic_andnot(int i, atomic_t *v) 840 atomic_andnot(int i, atomic_t *v)
841 { 841 {
842 instrument_atomic_read_write(v, sizeof 842 instrument_atomic_read_write(v, sizeof(*v));
843 raw_atomic_andnot(i, v); 843 raw_atomic_andnot(i, v);
844 } 844 }
845 845
846 /** 846 /**
847 * atomic_fetch_andnot() - atomic bitwise AND 847 * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
848 * @i: int value 848 * @i: int value
849 * @v: pointer to atomic_t 849 * @v: pointer to atomic_t
850 * 850 *
851 * Atomically updates @v to (@v & ~@i) with fu 851 * Atomically updates @v to (@v & ~@i) with full ordering.
852 * 852 *
853 * Unsafe to use in noinstr code; use raw_atom 853 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
854 * 854 *
855 * Return: The original value of @v. 855 * Return: The original value of @v.
856 */ 856 */
857 static __always_inline int 857 static __always_inline int
858 atomic_fetch_andnot(int i, atomic_t *v) 858 atomic_fetch_andnot(int i, atomic_t *v)
859 { 859 {
860 kcsan_mb(); 860 kcsan_mb();
861 instrument_atomic_read_write(v, sizeof 861 instrument_atomic_read_write(v, sizeof(*v));
862 return raw_atomic_fetch_andnot(i, v); 862 return raw_atomic_fetch_andnot(i, v);
863 } 863 }
864 864
865 /** 865 /**
866 * atomic_fetch_andnot_acquire() - atomic bitw 866 * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
867 * @i: int value 867 * @i: int value
868 * @v: pointer to atomic_t 868 * @v: pointer to atomic_t
869 * 869 *
870 * Atomically updates @v to (@v & ~@i) with ac 870 * Atomically updates @v to (@v & ~@i) with acquire ordering.
871 * 871 *
872 * Unsafe to use in noinstr code; use raw_atom 872 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
873 * 873 *
874 * Return: The original value of @v. 874 * Return: The original value of @v.
875 */ 875 */
876 static __always_inline int 876 static __always_inline int
877 atomic_fetch_andnot_acquire(int i, atomic_t *v 877 atomic_fetch_andnot_acquire(int i, atomic_t *v)
878 { 878 {
879 instrument_atomic_read_write(v, sizeof 879 instrument_atomic_read_write(v, sizeof(*v));
880 return raw_atomic_fetch_andnot_acquire 880 return raw_atomic_fetch_andnot_acquire(i, v);
881 } 881 }
882 882
883 /** 883 /**
884 * atomic_fetch_andnot_release() - atomic bitw 884 * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
885 * @i: int value 885 * @i: int value
886 * @v: pointer to atomic_t 886 * @v: pointer to atomic_t
887 * 887 *
888 * Atomically updates @v to (@v & ~@i) with re 888 * Atomically updates @v to (@v & ~@i) with release ordering.
889 * 889 *
890 * Unsafe to use in noinstr code; use raw_atom 890 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
891 * 891 *
892 * Return: The original value of @v. 892 * Return: The original value of @v.
893 */ 893 */
894 static __always_inline int 894 static __always_inline int
895 atomic_fetch_andnot_release(int i, atomic_t *v 895 atomic_fetch_andnot_release(int i, atomic_t *v)
896 { 896 {
897 kcsan_release(); 897 kcsan_release();
898 instrument_atomic_read_write(v, sizeof 898 instrument_atomic_read_write(v, sizeof(*v));
899 return raw_atomic_fetch_andnot_release 899 return raw_atomic_fetch_andnot_release(i, v);
900 } 900 }
901 901
902 /** 902 /**
903 * atomic_fetch_andnot_relaxed() - atomic bitw 903 * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
904 * @i: int value 904 * @i: int value
905 * @v: pointer to atomic_t 905 * @v: pointer to atomic_t
906 * 906 *
907 * Atomically updates @v to (@v & ~@i) with re 907 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
908 * 908 *
909 * Unsafe to use in noinstr code; use raw_atom 909 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
910 * 910 *
911 * Return: The original value of @v. 911 * Return: The original value of @v.
912 */ 912 */
913 static __always_inline int 913 static __always_inline int
914 atomic_fetch_andnot_relaxed(int i, atomic_t *v 914 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
915 { 915 {
916 instrument_atomic_read_write(v, sizeof 916 instrument_atomic_read_write(v, sizeof(*v));
917 return raw_atomic_fetch_andnot_relaxed 917 return raw_atomic_fetch_andnot_relaxed(i, v);
918 } 918 }
919 919
920 /** 920 /**
921 * atomic_or() - atomic bitwise OR with relaxe 921 * atomic_or() - atomic bitwise OR with relaxed ordering
922 * @i: int value 922 * @i: int value
923 * @v: pointer to atomic_t 923 * @v: pointer to atomic_t
924 * 924 *
925 * Atomically updates @v to (@v | @i) with rel 925 * Atomically updates @v to (@v | @i) with relaxed ordering.
926 * 926 *
927 * Unsafe to use in noinstr code; use raw_atom 927 * Unsafe to use in noinstr code; use raw_atomic_or() there.
928 * 928 *
929 * Return: Nothing. 929 * Return: Nothing.
930 */ 930 */
931 static __always_inline void 931 static __always_inline void
932 atomic_or(int i, atomic_t *v) 932 atomic_or(int i, atomic_t *v)
933 { 933 {
934 instrument_atomic_read_write(v, sizeof 934 instrument_atomic_read_write(v, sizeof(*v));
935 raw_atomic_or(i, v); 935 raw_atomic_or(i, v);
936 } 936 }
937 937
938 /** 938 /**
939 * atomic_fetch_or() - atomic bitwise OR with 939 * atomic_fetch_or() - atomic bitwise OR with full ordering
940 * @i: int value 940 * @i: int value
941 * @v: pointer to atomic_t 941 * @v: pointer to atomic_t
942 * 942 *
943 * Atomically updates @v to (@v | @i) with ful 943 * Atomically updates @v to (@v | @i) with full ordering.
944 * 944 *
945 * Unsafe to use in noinstr code; use raw_atom 945 * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
946 * 946 *
947 * Return: The original value of @v. 947 * Return: The original value of @v.
948 */ 948 */
949 static __always_inline int 949 static __always_inline int
950 atomic_fetch_or(int i, atomic_t *v) 950 atomic_fetch_or(int i, atomic_t *v)
951 { 951 {
952 kcsan_mb(); 952 kcsan_mb();
953 instrument_atomic_read_write(v, sizeof 953 instrument_atomic_read_write(v, sizeof(*v));
954 return raw_atomic_fetch_or(i, v); 954 return raw_atomic_fetch_or(i, v);
955 } 955 }
956 956
957 /** 957 /**
958 * atomic_fetch_or_acquire() - atomic bitwise 958 * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
959 * @i: int value 959 * @i: int value
960 * @v: pointer to atomic_t 960 * @v: pointer to atomic_t
961 * 961 *
962 * Atomically updates @v to (@v | @i) with acq 962 * Atomically updates @v to (@v | @i) with acquire ordering.
963 * 963 *
964 * Unsafe to use in noinstr code; use raw_atom 964 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
965 * 965 *
966 * Return: The original value of @v. 966 * Return: The original value of @v.
967 */ 967 */
968 static __always_inline int 968 static __always_inline int
969 atomic_fetch_or_acquire(int i, atomic_t *v) 969 atomic_fetch_or_acquire(int i, atomic_t *v)
970 { 970 {
971 instrument_atomic_read_write(v, sizeof 971 instrument_atomic_read_write(v, sizeof(*v));
972 return raw_atomic_fetch_or_acquire(i, 972 return raw_atomic_fetch_or_acquire(i, v);
973 } 973 }
974 974
975 /** 975 /**
976 * atomic_fetch_or_release() - atomic bitwise 976 * atomic_fetch_or_release() - atomic bitwise OR with release ordering
977 * @i: int value 977 * @i: int value
978 * @v: pointer to atomic_t 978 * @v: pointer to atomic_t
979 * 979 *
980 * Atomically updates @v to (@v | @i) with rel 980 * Atomically updates @v to (@v | @i) with release ordering.
981 * 981 *
982 * Unsafe to use in noinstr code; use raw_atom 982 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
983 * 983 *
984 * Return: The original value of @v. 984 * Return: The original value of @v.
985 */ 985 */
986 static __always_inline int 986 static __always_inline int
987 atomic_fetch_or_release(int i, atomic_t *v) 987 atomic_fetch_or_release(int i, atomic_t *v)
988 { 988 {
989 kcsan_release(); 989 kcsan_release();
990 instrument_atomic_read_write(v, sizeof 990 instrument_atomic_read_write(v, sizeof(*v));
991 return raw_atomic_fetch_or_release(i, 991 return raw_atomic_fetch_or_release(i, v);
992 } 992 }
993 993
994 /** 994 /**
995 * atomic_fetch_or_relaxed() - atomic bitwise 995 * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
996 * @i: int value 996 * @i: int value
997 * @v: pointer to atomic_t 997 * @v: pointer to atomic_t
998 * 998 *
999 * Atomically updates @v to (@v | @i) with rel 999 * Atomically updates @v to (@v | @i) with relaxed ordering.
1000 * 1000 *
1001 * Unsafe to use in noinstr code; use raw_ato 1001 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
1002 * 1002 *
1003 * Return: The original value of @v. 1003 * Return: The original value of @v.
1004 */ 1004 */
1005 static __always_inline int 1005 static __always_inline int
1006 atomic_fetch_or_relaxed(int i, atomic_t *v) 1006 atomic_fetch_or_relaxed(int i, atomic_t *v)
1007 { 1007 {
1008 instrument_atomic_read_write(v, sizeo 1008 instrument_atomic_read_write(v, sizeof(*v));
1009 return raw_atomic_fetch_or_relaxed(i, 1009 return raw_atomic_fetch_or_relaxed(i, v);
1010 } 1010 }
1011 1011
1012 /** 1012 /**
1013 * atomic_xor() - atomic bitwise XOR with rel 1013 * atomic_xor() - atomic bitwise XOR with relaxed ordering
1014 * @i: int value 1014 * @i: int value
1015 * @v: pointer to atomic_t 1015 * @v: pointer to atomic_t
1016 * 1016 *
1017 * Atomically updates @v to (@v ^ @i) with re 1017 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1018 * 1018 *
1019 * Unsafe to use in noinstr code; use raw_ato 1019 * Unsafe to use in noinstr code; use raw_atomic_xor() there.
1020 * 1020 *
1021 * Return: Nothing. 1021 * Return: Nothing.
1022 */ 1022 */
1023 static __always_inline void 1023 static __always_inline void
1024 atomic_xor(int i, atomic_t *v) 1024 atomic_xor(int i, atomic_t *v)
1025 { 1025 {
1026 instrument_atomic_read_write(v, sizeo 1026 instrument_atomic_read_write(v, sizeof(*v));
1027 raw_atomic_xor(i, v); 1027 raw_atomic_xor(i, v);
1028 } 1028 }
1029 1029
1030 /** 1030 /**
1031 * atomic_fetch_xor() - atomic bitwise XOR wi 1031 * atomic_fetch_xor() - atomic bitwise XOR with full ordering
1032 * @i: int value 1032 * @i: int value
1033 * @v: pointer to atomic_t 1033 * @v: pointer to atomic_t
1034 * 1034 *
1035 * Atomically updates @v to (@v ^ @i) with fu 1035 * Atomically updates @v to (@v ^ @i) with full ordering.
1036 * 1036 *
1037 * Unsafe to use in noinstr code; use raw_ato 1037 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
1038 * 1038 *
1039 * Return: The original value of @v. 1039 * Return: The original value of @v.
1040 */ 1040 */
1041 static __always_inline int 1041 static __always_inline int
1042 atomic_fetch_xor(int i, atomic_t *v) 1042 atomic_fetch_xor(int i, atomic_t *v)
1043 { 1043 {
1044 kcsan_mb(); 1044 kcsan_mb();
1045 instrument_atomic_read_write(v, sizeo 1045 instrument_atomic_read_write(v, sizeof(*v));
1046 return raw_atomic_fetch_xor(i, v); 1046 return raw_atomic_fetch_xor(i, v);
1047 } 1047 }
1048 1048
1049 /** 1049 /**
1050 * atomic_fetch_xor_acquire() - atomic bitwis 1050 * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1051 * @i: int value 1051 * @i: int value
1052 * @v: pointer to atomic_t 1052 * @v: pointer to atomic_t
1053 * 1053 *
1054 * Atomically updates @v to (@v ^ @i) with ac 1054 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1055 * 1055 *
1056 * Unsafe to use in noinstr code; use raw_ato 1056 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
1057 * 1057 *
1058 * Return: The original value of @v. 1058 * Return: The original value of @v.
1059 */ 1059 */
1060 static __always_inline int 1060 static __always_inline int
1061 atomic_fetch_xor_acquire(int i, atomic_t *v) 1061 atomic_fetch_xor_acquire(int i, atomic_t *v)
1062 { 1062 {
1063 instrument_atomic_read_write(v, sizeo 1063 instrument_atomic_read_write(v, sizeof(*v));
1064 return raw_atomic_fetch_xor_acquire(i 1064 return raw_atomic_fetch_xor_acquire(i, v);
1065 } 1065 }
1066 1066
1067 /** 1067 /**
1068 * atomic_fetch_xor_release() - atomic bitwis 1068 * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1069 * @i: int value 1069 * @i: int value
1070 * @v: pointer to atomic_t 1070 * @v: pointer to atomic_t
1071 * 1071 *
1072 * Atomically updates @v to (@v ^ @i) with re 1072 * Atomically updates @v to (@v ^ @i) with release ordering.
1073 * 1073 *
1074 * Unsafe to use in noinstr code; use raw_ato 1074 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
1075 * 1075 *
1076 * Return: The original value of @v. 1076 * Return: The original value of @v.
1077 */ 1077 */
1078 static __always_inline int 1078 static __always_inline int
1079 atomic_fetch_xor_release(int i, atomic_t *v) 1079 atomic_fetch_xor_release(int i, atomic_t *v)
1080 { 1080 {
1081 kcsan_release(); 1081 kcsan_release();
1082 instrument_atomic_read_write(v, sizeo 1082 instrument_atomic_read_write(v, sizeof(*v));
1083 return raw_atomic_fetch_xor_release(i 1083 return raw_atomic_fetch_xor_release(i, v);
1084 } 1084 }
1085 1085
1086 /** 1086 /**
1087 * atomic_fetch_xor_relaxed() - atomic bitwis 1087 * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1088 * @i: int value 1088 * @i: int value
1089 * @v: pointer to atomic_t 1089 * @v: pointer to atomic_t
1090 * 1090 *
1091 * Atomically updates @v to (@v ^ @i) with re 1091 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1092 * 1092 *
1093 * Unsafe to use in noinstr code; use raw_ato 1093 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
1094 * 1094 *
1095 * Return: The original value of @v. 1095 * Return: The original value of @v.
1096 */ 1096 */
1097 static __always_inline int 1097 static __always_inline int
1098 atomic_fetch_xor_relaxed(int i, atomic_t *v) 1098 atomic_fetch_xor_relaxed(int i, atomic_t *v)
1099 { 1099 {
1100 instrument_atomic_read_write(v, sizeo 1100 instrument_atomic_read_write(v, sizeof(*v));
1101 return raw_atomic_fetch_xor_relaxed(i 1101 return raw_atomic_fetch_xor_relaxed(i, v);
1102 } 1102 }
1103 1103
1104 /** 1104 /**
1105 * atomic_xchg() - atomic exchange with full 1105 * atomic_xchg() - atomic exchange with full ordering
1106 * @v: pointer to atomic_t 1106 * @v: pointer to atomic_t
1107 * @new: int value to assign 1107 * @new: int value to assign
1108 * 1108 *
1109 * Atomically updates @v to @new with full or 1109 * Atomically updates @v to @new with full ordering.
1110 * 1110 *
1111 * Unsafe to use in noinstr code; use raw_ato 1111 * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
1112 * 1112 *
1113 * Return: The original value of @v. 1113 * Return: The original value of @v.
1114 */ 1114 */
1115 static __always_inline int 1115 static __always_inline int
1116 atomic_xchg(atomic_t *v, int new) 1116 atomic_xchg(atomic_t *v, int new)
1117 { 1117 {
1118 kcsan_mb(); 1118 kcsan_mb();
1119 instrument_atomic_read_write(v, sizeo 1119 instrument_atomic_read_write(v, sizeof(*v));
1120 return raw_atomic_xchg(v, new); 1120 return raw_atomic_xchg(v, new);
1121 } 1121 }
1122 1122
1123 /** 1123 /**
1124 * atomic_xchg_acquire() - atomic exchange wi 1124 * atomic_xchg_acquire() - atomic exchange with acquire ordering
1125 * @v: pointer to atomic_t 1125 * @v: pointer to atomic_t
1126 * @new: int value to assign 1126 * @new: int value to assign
1127 * 1127 *
1128 * Atomically updates @v to @new with acquire 1128 * Atomically updates @v to @new with acquire ordering.
1129 * 1129 *
1130 * Unsafe to use in noinstr code; use raw_ato 1130 * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
1131 * 1131 *
1132 * Return: The original value of @v. 1132 * Return: The original value of @v.
1133 */ 1133 */
1134 static __always_inline int 1134 static __always_inline int
1135 atomic_xchg_acquire(atomic_t *v, int new) 1135 atomic_xchg_acquire(atomic_t *v, int new)
1136 { 1136 {
1137 instrument_atomic_read_write(v, sizeo 1137 instrument_atomic_read_write(v, sizeof(*v));
1138 return raw_atomic_xchg_acquire(v, new 1138 return raw_atomic_xchg_acquire(v, new);
1139 } 1139 }
1140 1140
1141 /** 1141 /**
1142 * atomic_xchg_release() - atomic exchange wi 1142 * atomic_xchg_release() - atomic exchange with release ordering
1143 * @v: pointer to atomic_t 1143 * @v: pointer to atomic_t
1144 * @new: int value to assign 1144 * @new: int value to assign
1145 * 1145 *
1146 * Atomically updates @v to @new with release 1146 * Atomically updates @v to @new with release ordering.
1147 * 1147 *
1148 * Unsafe to use in noinstr code; use raw_ato 1148 * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
1149 * 1149 *
1150 * Return: The original value of @v. 1150 * Return: The original value of @v.
1151 */ 1151 */
1152 static __always_inline int 1152 static __always_inline int
1153 atomic_xchg_release(atomic_t *v, int new) 1153 atomic_xchg_release(atomic_t *v, int new)
1154 { 1154 {
1155 kcsan_release(); 1155 kcsan_release();
1156 instrument_atomic_read_write(v, sizeo 1156 instrument_atomic_read_write(v, sizeof(*v));
1157 return raw_atomic_xchg_release(v, new 1157 return raw_atomic_xchg_release(v, new);
1158 } 1158 }
1159 1159
1160 /** 1160 /**
1161 * atomic_xchg_relaxed() - atomic exchange wi 1161 * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1162 * @v: pointer to atomic_t 1162 * @v: pointer to atomic_t
1163 * @new: int value to assign 1163 * @new: int value to assign
1164 * 1164 *
1165 * Atomically updates @v to @new with relaxed 1165 * Atomically updates @v to @new with relaxed ordering.
1166 * 1166 *
1167 * Unsafe to use in noinstr code; use raw_ato 1167 * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
1168 * 1168 *
1169 * Return: The original value of @v. 1169 * Return: The original value of @v.
1170 */ 1170 */
1171 static __always_inline int 1171 static __always_inline int
1172 atomic_xchg_relaxed(atomic_t *v, int new) 1172 atomic_xchg_relaxed(atomic_t *v, int new)
1173 { 1173 {
1174 instrument_atomic_read_write(v, sizeo 1174 instrument_atomic_read_write(v, sizeof(*v));
1175 return raw_atomic_xchg_relaxed(v, new 1175 return raw_atomic_xchg_relaxed(v, new);
1176 } 1176 }
1177 1177
1178 /** 1178 /**
1179 * atomic_cmpxchg() - atomic compare and exch 1179 * atomic_cmpxchg() - atomic compare and exchange with full ordering
1180 * @v: pointer to atomic_t 1180 * @v: pointer to atomic_t
1181 * @old: int value to compare with 1181 * @old: int value to compare with
1182 * @new: int value to assign 1182 * @new: int value to assign
1183 * 1183 *
1184 * If (@v == @old), atomically updates @v to 1184 * If (@v == @old), atomically updates @v to @new with full ordering.
1185 * Otherwise, @v is not modified and relaxed 1185 * Otherwise, @v is not modified and relaxed ordering is provided.
1186 * 1186 *
1187 * Unsafe to use in noinstr code; use raw_ato 1187 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
1188 * 1188 *
1189 * Return: The original value of @v. 1189 * Return: The original value of @v.
1190 */ 1190 */
1191 static __always_inline int 1191 static __always_inline int
1192 atomic_cmpxchg(atomic_t *v, int old, int new) 1192 atomic_cmpxchg(atomic_t *v, int old, int new)
1193 { 1193 {
1194 kcsan_mb(); 1194 kcsan_mb();
1195 instrument_atomic_read_write(v, sizeo 1195 instrument_atomic_read_write(v, sizeof(*v));
1196 return raw_atomic_cmpxchg(v, old, new 1196 return raw_atomic_cmpxchg(v, old, new);
1197 } 1197 }
1198 1198
1199 /** 1199 /**
1200 * atomic_cmpxchg_acquire() - atomic compare 1200 * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1201 * @v: pointer to atomic_t 1201 * @v: pointer to atomic_t
1202 * @old: int value to compare with 1202 * @old: int value to compare with
1203 * @new: int value to assign 1203 * @new: int value to assign
1204 * 1204 *
1205 * If (@v == @old), atomically updates @v to 1205 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1206 * Otherwise, @v is not modified and relaxed 1206 * Otherwise, @v is not modified and relaxed ordering is provided.
1207 * 1207 *
1208 * Unsafe to use in noinstr code; use raw_ato 1208 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
1209 * 1209 *
1210 * Return: The original value of @v. 1210 * Return: The original value of @v.
1211 */ 1211 */
1212 static __always_inline int 1212 static __always_inline int
1213 atomic_cmpxchg_acquire(atomic_t *v, int old, 1213 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1214 { 1214 {
1215 instrument_atomic_read_write(v, sizeo 1215 instrument_atomic_read_write(v, sizeof(*v));
1216 return raw_atomic_cmpxchg_acquire(v, 1216 return raw_atomic_cmpxchg_acquire(v, old, new);
1217 } 1217 }
1218 1218
1219 /** 1219 /**
1220 * atomic_cmpxchg_release() - atomic compare 1220 * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
1221 * @v: pointer to atomic_t 1221 * @v: pointer to atomic_t
1222 * @old: int value to compare with 1222 * @old: int value to compare with
1223 * @new: int value to assign 1223 * @new: int value to assign
1224 * 1224 *
1225 * If (@v == @old), atomically updates @v to 1225 * If (@v == @old), atomically updates @v to @new with release ordering.
1226 * Otherwise, @v is not modified and relaxed 1226 * Otherwise, @v is not modified and relaxed ordering is provided.
1227 * 1227 *
1228 * Unsafe to use in noinstr code; use raw_ato 1228 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
1229 * 1229 *
1230 * Return: The original value of @v. 1230 * Return: The original value of @v.
1231 */ 1231 */
1232 static __always_inline int 1232 static __always_inline int
1233 atomic_cmpxchg_release(atomic_t *v, int old, 1233 atomic_cmpxchg_release(atomic_t *v, int old, int new)
1234 { 1234 {
1235 kcsan_release(); 1235 kcsan_release();
1236 instrument_atomic_read_write(v, sizeo 1236 instrument_atomic_read_write(v, sizeof(*v));
1237 return raw_atomic_cmpxchg_release(v, 1237 return raw_atomic_cmpxchg_release(v, old, new);
1238 } 1238 }
1239 1239
1240 /** 1240 /**
1241 * atomic_cmpxchg_relaxed() - atomic compare 1241 * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1242 * @v: pointer to atomic_t 1242 * @v: pointer to atomic_t
1243 * @old: int value to compare with 1243 * @old: int value to compare with
1244 * @new: int value to assign 1244 * @new: int value to assign
1245 * 1245 *
1246 * If (@v == @old), atomically updates @v to 1246 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1247 * Otherwise, @v is not modified and relaxed 1247 * Otherwise, @v is not modified and relaxed ordering is provided.
1248 * 1248 *
1249 * Unsafe to use in noinstr code; use raw_ato 1249 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
1250 * 1250 *
1251 * Return: The original value of @v. 1251 * Return: The original value of @v.
1252 */ 1252 */
1253 static __always_inline int 1253 static __always_inline int
1254 atomic_cmpxchg_relaxed(atomic_t *v, int old, 1254 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
1255 { 1255 {
1256 instrument_atomic_read_write(v, sizeo 1256 instrument_atomic_read_write(v, sizeof(*v));
1257 return raw_atomic_cmpxchg_relaxed(v, 1257 return raw_atomic_cmpxchg_relaxed(v, old, new);
1258 } 1258 }
1259 1259
1260 /** 1260 /**
1261 * atomic_try_cmpxchg() - atomic compare and 1261 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
1262 * @v: pointer to atomic_t 1262 * @v: pointer to atomic_t
1263 * @old: pointer to int value to compare with 1263 * @old: pointer to int value to compare with
1264 * @new: int value to assign 1264 * @new: int value to assign
1265 * 1265 *
1266 * If (@v == @old), atomically updates @v to 1266 * If (@v == @old), atomically updates @v to @new with full ordering.
1267 * Otherwise, @v is not modified, @old is upd 1267 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1268 * and relaxed ordering is provided. 1268 * and relaxed ordering is provided.
1269 * 1269 *
1270 * Unsafe to use in noinstr code; use raw_ato 1270 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
1271 * 1271 *
1272 * Return: @true if the exchange occured, @fa 1272 * Return: @true if the exchange occured, @false otherwise.
1273 */ 1273 */
1274 static __always_inline bool 1274 static __always_inline bool
1275 atomic_try_cmpxchg(atomic_t *v, int *old, int 1275 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1276 { 1276 {
1277 kcsan_mb(); 1277 kcsan_mb();
1278 instrument_atomic_read_write(v, sizeo 1278 instrument_atomic_read_write(v, sizeof(*v));
1279 instrument_atomic_read_write(old, siz 1279 instrument_atomic_read_write(old, sizeof(*old));
1280 return raw_atomic_try_cmpxchg(v, old, 1280 return raw_atomic_try_cmpxchg(v, old, new);
1281 } 1281 }
1282 1282
1283 /** 1283 /**
1284 * atomic_try_cmpxchg_acquire() - atomic comp 1284 * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1285 * @v: pointer to atomic_t 1285 * @v: pointer to atomic_t
1286 * @old: pointer to int value to compare with 1286 * @old: pointer to int value to compare with
1287 * @new: int value to assign 1287 * @new: int value to assign
1288 * 1288 *
1289 * If (@v == @old), atomically updates @v to 1289 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1290 * Otherwise, @v is not modified, @old is upd 1290 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1291 * and relaxed ordering is provided. 1291 * and relaxed ordering is provided.
1292 * 1292 *
1293 * Unsafe to use in noinstr code; use raw_ato 1293 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
1294 * 1294 *
1295 * Return: @true if the exchange occured, @fa 1295 * Return: @true if the exchange occured, @false otherwise.
1296 */ 1296 */
1297 static __always_inline bool 1297 static __always_inline bool
1298 atomic_try_cmpxchg_acquire(atomic_t *v, int * 1298 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1299 { 1299 {
1300 instrument_atomic_read_write(v, sizeo 1300 instrument_atomic_read_write(v, sizeof(*v));
1301 instrument_atomic_read_write(old, siz 1301 instrument_atomic_read_write(old, sizeof(*old));
1302 return raw_atomic_try_cmpxchg_acquire 1302 return raw_atomic_try_cmpxchg_acquire(v, old, new);
1303 } 1303 }
1304 1304
1305 /** 1305 /**
1306 * atomic_try_cmpxchg_release() - atomic comp 1306 * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
1307 * @v: pointer to atomic_t 1307 * @v: pointer to atomic_t
1308 * @old: pointer to int value to compare with 1308 * @old: pointer to int value to compare with
1309 * @new: int value to assign 1309 * @new: int value to assign
1310 * 1310 *
1311 * If (@v == @old), atomically updates @v to 1311 * If (@v == @old), atomically updates @v to @new with release ordering.
1312 * Otherwise, @v is not modified, @old is upd 1312 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1313 * and relaxed ordering is provided. 1313 * and relaxed ordering is provided.
1314 * 1314 *
1315 * Unsafe to use in noinstr code; use raw_ato 1315 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
1316 * 1316 *
1317 * Return: @true if the exchange occured, @fa 1317 * Return: @true if the exchange occured, @false otherwise.
1318 */ 1318 */
1319 static __always_inline bool 1319 static __always_inline bool
1320 atomic_try_cmpxchg_release(atomic_t *v, int * 1320 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1321 { 1321 {
1322 kcsan_release(); 1322 kcsan_release();
1323 instrument_atomic_read_write(v, sizeo 1323 instrument_atomic_read_write(v, sizeof(*v));
1324 instrument_atomic_read_write(old, siz 1324 instrument_atomic_read_write(old, sizeof(*old));
1325 return raw_atomic_try_cmpxchg_release 1325 return raw_atomic_try_cmpxchg_release(v, old, new);
1326 } 1326 }
1327 1327
1328 /** 1328 /**
1329 * atomic_try_cmpxchg_relaxed() - atomic comp 1329 * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1330 * @v: pointer to atomic_t 1330 * @v: pointer to atomic_t
1331 * @old: pointer to int value to compare with 1331 * @old: pointer to int value to compare with
1332 * @new: int value to assign 1332 * @new: int value to assign
1333 * 1333 *
1334 * If (@v == @old), atomically updates @v to 1334 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1335 * Otherwise, @v is not modified, @old is upd 1335 * Otherwise, @v is not modified, @old is updated to the current value of @v,
1336 * and relaxed ordering is provided. 1336 * and relaxed ordering is provided.
1337 * 1337 *
1338 * Unsafe to use in noinstr code; use raw_ato 1338 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
1339 * 1339 *
1340 * Return: @true if the exchange occured, @fa 1340 * Return: @true if the exchange occured, @false otherwise.
1341 */ 1341 */
1342 static __always_inline bool 1342 static __always_inline bool
1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int * 1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1344 { 1344 {
1345 instrument_atomic_read_write(v, sizeo 1345 instrument_atomic_read_write(v, sizeof(*v));
1346 instrument_atomic_read_write(old, siz 1346 instrument_atomic_read_write(old, sizeof(*old));
1347 return raw_atomic_try_cmpxchg_relaxed 1347 return raw_atomic_try_cmpxchg_relaxed(v, old, new);
1348 } 1348 }
1349 1349
1350 /** 1350 /**
1351 * atomic_sub_and_test() - atomic subtract an 1351 * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
1352 * @i: int value to subtract 1352 * @i: int value to subtract
1353 * @v: pointer to atomic_t 1353 * @v: pointer to atomic_t
1354 * 1354 *
1355 * Atomically updates @v to (@v - @i) with fu 1355 * Atomically updates @v to (@v - @i) with full ordering.
1356 * 1356 *
1357 * Unsafe to use in noinstr code; use raw_ato 1357 * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
1358 * 1358 *
1359 * Return: @true if the resulting value of @v 1359 * Return: @true if the resulting value of @v is zero, @false otherwise.
1360 */ 1360 */
1361 static __always_inline bool 1361 static __always_inline bool
1362 atomic_sub_and_test(int i, atomic_t *v) 1362 atomic_sub_and_test(int i, atomic_t *v)
1363 { 1363 {
1364 kcsan_mb(); 1364 kcsan_mb();
1365 instrument_atomic_read_write(v, sizeo 1365 instrument_atomic_read_write(v, sizeof(*v));
1366 return raw_atomic_sub_and_test(i, v); 1366 return raw_atomic_sub_and_test(i, v);
1367 } 1367 }
1368 1368
1369 /** 1369 /**
1370 * atomic_dec_and_test() - atomic decrement a 1370 * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
1371 * @v: pointer to atomic_t 1371 * @v: pointer to atomic_t
1372 * 1372 *
1373 * Atomically updates @v to (@v - 1) with ful 1373 * Atomically updates @v to (@v - 1) with full ordering.
1374 * 1374 *
1375 * Unsafe to use in noinstr code; use raw_ato 1375 * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
1376 * 1376 *
1377 * Return: @true if the resulting value of @v 1377 * Return: @true if the resulting value of @v is zero, @false otherwise.
1378 */ 1378 */
1379 static __always_inline bool 1379 static __always_inline bool
1380 atomic_dec_and_test(atomic_t *v) 1380 atomic_dec_and_test(atomic_t *v)
1381 { 1381 {
1382 kcsan_mb(); 1382 kcsan_mb();
1383 instrument_atomic_read_write(v, sizeo 1383 instrument_atomic_read_write(v, sizeof(*v));
1384 return raw_atomic_dec_and_test(v); 1384 return raw_atomic_dec_and_test(v);
1385 } 1385 }
1386 1386
1387 /** 1387 /**
1388 * atomic_inc_and_test() - atomic increment a 1388 * atomic_inc_and_test() - atomic increment and test if zero with full ordering
1389 * @v: pointer to atomic_t 1389 * @v: pointer to atomic_t
1390 * 1390 *
1391 * Atomically updates @v to (@v + 1) with ful 1391 * Atomically updates @v to (@v + 1) with full ordering.
1392 * 1392 *
1393 * Unsafe to use in noinstr code; use raw_ato 1393 * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
1394 * 1394 *
1395 * Return: @true if the resulting value of @v 1395 * Return: @true if the resulting value of @v is zero, @false otherwise.
1396 */ 1396 */
1397 static __always_inline bool 1397 static __always_inline bool
1398 atomic_inc_and_test(atomic_t *v) 1398 atomic_inc_and_test(atomic_t *v)
1399 { 1399 {
1400 kcsan_mb(); 1400 kcsan_mb();
1401 instrument_atomic_read_write(v, sizeo 1401 instrument_atomic_read_write(v, sizeof(*v));
1402 return raw_atomic_inc_and_test(v); 1402 return raw_atomic_inc_and_test(v);
1403 } 1403 }
1404 1404
1405 /** 1405 /**
1406 * atomic_add_negative() - atomic add and tes 1406 * atomic_add_negative() - atomic add and test if negative with full ordering
1407 * @i: int value to add 1407 * @i: int value to add
1408 * @v: pointer to atomic_t 1408 * @v: pointer to atomic_t
1409 * 1409 *
1410 * Atomically updates @v to (@v + @i) with fu 1410 * Atomically updates @v to (@v + @i) with full ordering.
1411 * 1411 *
1412 * Unsafe to use in noinstr code; use raw_ato 1412 * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
1413 * 1413 *
1414 * Return: @true if the resulting value of @v 1414 * Return: @true if the resulting value of @v is negative, @false otherwise.
1415 */ 1415 */
1416 static __always_inline bool 1416 static __always_inline bool
1417 atomic_add_negative(int i, atomic_t *v) 1417 atomic_add_negative(int i, atomic_t *v)
1418 { 1418 {
1419 kcsan_mb(); 1419 kcsan_mb();
1420 instrument_atomic_read_write(v, sizeo 1420 instrument_atomic_read_write(v, sizeof(*v));
1421 return raw_atomic_add_negative(i, v); 1421 return raw_atomic_add_negative(i, v);
1422 } 1422 }
1423 1423
1424 /** 1424 /**
1425 * atomic_add_negative_acquire() - atomic add 1425 * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
1426 * @i: int value to add 1426 * @i: int value to add
1427 * @v: pointer to atomic_t 1427 * @v: pointer to atomic_t
1428 * 1428 *
1429 * Atomically updates @v to (@v + @i) with ac 1429 * Atomically updates @v to (@v + @i) with acquire ordering.
1430 * 1430 *
1431 * Unsafe to use in noinstr code; use raw_ato 1431 * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
1432 * 1432 *
1433 * Return: @true if the resulting value of @v 1433 * Return: @true if the resulting value of @v is negative, @false otherwise.
1434 */ 1434 */
1435 static __always_inline bool 1435 static __always_inline bool
1436 atomic_add_negative_acquire(int i, atomic_t * 1436 atomic_add_negative_acquire(int i, atomic_t *v)
1437 { 1437 {
1438 instrument_atomic_read_write(v, sizeo 1438 instrument_atomic_read_write(v, sizeof(*v));
1439 return raw_atomic_add_negative_acquir 1439 return raw_atomic_add_negative_acquire(i, v);
1440 } 1440 }
1441 1441
1442 /** 1442 /**
1443 * atomic_add_negative_release() - atomic add 1443 * atomic_add_negative_release() - atomic add and test if negative with release ordering
1444 * @i: int value to add 1444 * @i: int value to add
1445 * @v: pointer to atomic_t 1445 * @v: pointer to atomic_t
1446 * 1446 *
1447 * Atomically updates @v to (@v + @i) with re 1447 * Atomically updates @v to (@v + @i) with release ordering.
1448 * 1448 *
1449 * Unsafe to use in noinstr code; use raw_ato 1449 * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
1450 * 1450 *
1451 * Return: @true if the resulting value of @v 1451 * Return: @true if the resulting value of @v is negative, @false otherwise.
1452 */ 1452 */
1453 static __always_inline bool 1453 static __always_inline bool
1454 atomic_add_negative_release(int i, atomic_t * 1454 atomic_add_negative_release(int i, atomic_t *v)
1455 { 1455 {
1456 kcsan_release(); 1456 kcsan_release();
1457 instrument_atomic_read_write(v, sizeo 1457 instrument_atomic_read_write(v, sizeof(*v));
1458 return raw_atomic_add_negative_releas 1458 return raw_atomic_add_negative_release(i, v);
1459 } 1459 }
1460 1460
1461 /** 1461 /**
1462 * atomic_add_negative_relaxed() - atomic add 1462 * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
1463 * @i: int value to add 1463 * @i: int value to add
1464 * @v: pointer to atomic_t 1464 * @v: pointer to atomic_t
1465 * 1465 *
1466 * Atomically updates @v to (@v + @i) with re 1466 * Atomically updates @v to (@v + @i) with relaxed ordering.
1467 * 1467 *
1468 * Unsafe to use in noinstr code; use raw_ato 1468 * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
1469 * 1469 *
1470 * Return: @true if the resulting value of @v 1470 * Return: @true if the resulting value of @v is negative, @false otherwise.
1471 */ 1471 */
1472 static __always_inline bool 1472 static __always_inline bool
1473 atomic_add_negative_relaxed(int i, atomic_t * 1473 atomic_add_negative_relaxed(int i, atomic_t *v)
1474 { 1474 {
1475 instrument_atomic_read_write(v, sizeo 1475 instrument_atomic_read_write(v, sizeof(*v));
1476 return raw_atomic_add_negative_relaxe 1476 return raw_atomic_add_negative_relaxed(i, v);
1477 } 1477 }
1478 1478
1479 /** 1479 /**
1480 * atomic_fetch_add_unless() - atomic add unl 1480 * atomic_fetch_add_unless() - atomic add unless value with full ordering
1481 * @v: pointer to atomic_t 1481 * @v: pointer to atomic_t
1482 * @a: int value to add 1482 * @a: int value to add
1483 * @u: int value to compare with 1483 * @u: int value to compare with
1484 * 1484 *
1485 * If (@v != @u), atomically updates @v to (@ 1485 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1486 * Otherwise, @v is not modified and relaxed 1486 * Otherwise, @v is not modified and relaxed ordering is provided.
1487 * 1487 *
1488 * Unsafe to use in noinstr code; use raw_ato 1488 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
1489 * 1489 *
1490 * Return: The original value of @v. 1490 * Return: The original value of @v.
1491 */ 1491 */
1492 static __always_inline int 1492 static __always_inline int
1493 atomic_fetch_add_unless(atomic_t *v, int a, i 1493 atomic_fetch_add_unless(atomic_t *v, int a, int u)
1494 { 1494 {
1495 kcsan_mb(); 1495 kcsan_mb();
1496 instrument_atomic_read_write(v, sizeo 1496 instrument_atomic_read_write(v, sizeof(*v));
1497 return raw_atomic_fetch_add_unless(v, 1497 return raw_atomic_fetch_add_unless(v, a, u);
1498 } 1498 }
1499 1499
1500 /** 1500 /**
1501 * atomic_add_unless() - atomic add unless va 1501 * atomic_add_unless() - atomic add unless value with full ordering
1502 * @v: pointer to atomic_t 1502 * @v: pointer to atomic_t
1503 * @a: int value to add 1503 * @a: int value to add
1504 * @u: int value to compare with 1504 * @u: int value to compare with
1505 * 1505 *
1506 * If (@v != @u), atomically updates @v to (@ 1506 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1507 * Otherwise, @v is not modified and relaxed 1507 * Otherwise, @v is not modified and relaxed ordering is provided.
1508 * 1508 *
1509 * Unsafe to use in noinstr code; use raw_ato 1509 * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
1510 * 1510 *
1511 * Return: @true if @v was updated, @false ot 1511 * Return: @true if @v was updated, @false otherwise.
1512 */ 1512 */
1513 static __always_inline bool 1513 static __always_inline bool
1514 atomic_add_unless(atomic_t *v, int a, int u) 1514 atomic_add_unless(atomic_t *v, int a, int u)
1515 { 1515 {
1516 kcsan_mb(); 1516 kcsan_mb();
1517 instrument_atomic_read_write(v, sizeo 1517 instrument_atomic_read_write(v, sizeof(*v));
1518 return raw_atomic_add_unless(v, a, u) 1518 return raw_atomic_add_unless(v, a, u);
1519 } 1519 }
1520 1520
1521 /** 1521 /**
1522 * atomic_inc_not_zero() - atomic increment u 1522 * atomic_inc_not_zero() - atomic increment unless zero with full ordering
1523 * @v: pointer to atomic_t 1523 * @v: pointer to atomic_t
1524 * 1524 *
1525 * If (@v != 0), atomically updates @v to (@v 1525 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
1526 * Otherwise, @v is not modified and relaxed 1526 * Otherwise, @v is not modified and relaxed ordering is provided.
1527 * 1527 *
1528 * Unsafe to use in noinstr code; use raw_ato 1528 * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
1529 * 1529 *
1530 * Return: @true if @v was updated, @false ot 1530 * Return: @true if @v was updated, @false otherwise.
1531 */ 1531 */
1532 static __always_inline bool 1532 static __always_inline bool
1533 atomic_inc_not_zero(atomic_t *v) 1533 atomic_inc_not_zero(atomic_t *v)
1534 { 1534 {
1535 kcsan_mb(); 1535 kcsan_mb();
1536 instrument_atomic_read_write(v, sizeo 1536 instrument_atomic_read_write(v, sizeof(*v));
1537 return raw_atomic_inc_not_zero(v); 1537 return raw_atomic_inc_not_zero(v);
1538 } 1538 }
1539 1539
1540 /** 1540 /**
1541 * atomic_inc_unless_negative() - atomic incr 1541 * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
1542 * @v: pointer to atomic_t 1542 * @v: pointer to atomic_t
1543 * 1543 *
1544 * If (@v >= 0), atomically updates @v to (@v 1544 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
1545 * Otherwise, @v is not modified and relaxed 1545 * Otherwise, @v is not modified and relaxed ordering is provided.
1546 * 1546 *
1547 * Unsafe to use in noinstr code; use raw_ato 1547 * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
1548 * 1548 *
1549 * Return: @true if @v was updated, @false ot 1549 * Return: @true if @v was updated, @false otherwise.
1550 */ 1550 */
1551 static __always_inline bool 1551 static __always_inline bool
1552 atomic_inc_unless_negative(atomic_t *v) 1552 atomic_inc_unless_negative(atomic_t *v)
1553 { 1553 {
1554 kcsan_mb(); 1554 kcsan_mb();
1555 instrument_atomic_read_write(v, sizeo 1555 instrument_atomic_read_write(v, sizeof(*v));
1556 return raw_atomic_inc_unless_negative 1556 return raw_atomic_inc_unless_negative(v);
1557 } 1557 }
1558 1558
1559 /** 1559 /**
1560 * atomic_dec_unless_positive() - atomic decr 1560 * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
1561 * @v: pointer to atomic_t 1561 * @v: pointer to atomic_t
1562 * 1562 *
1563 * If (@v <= 0), atomically updates @v to (@v 1563 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
1564 * Otherwise, @v is not modified and relaxed 1564 * Otherwise, @v is not modified and relaxed ordering is provided.
1565 * 1565 *
1566 * Unsafe to use in noinstr code; use raw_ato 1566 * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
1567 * 1567 *
1568 * Return: @true if @v was updated, @false ot 1568 * Return: @true if @v was updated, @false otherwise.
1569 */ 1569 */
1570 static __always_inline bool 1570 static __always_inline bool
1571 atomic_dec_unless_positive(atomic_t *v) 1571 atomic_dec_unless_positive(atomic_t *v)
1572 { 1572 {
1573 kcsan_mb(); 1573 kcsan_mb();
1574 instrument_atomic_read_write(v, sizeo 1574 instrument_atomic_read_write(v, sizeof(*v));
1575 return raw_atomic_dec_unless_positive 1575 return raw_atomic_dec_unless_positive(v);
1576 } 1576 }
1577 1577
1578 /** 1578 /**
1579 * atomic_dec_if_positive() - atomic decremen 1579 * atomic_dec_if_positive() - atomic decrement if positive with full ordering
1580 * @v: pointer to atomic_t 1580 * @v: pointer to atomic_t
1581 * 1581 *
1582 * If (@v > 0), atomically updates @v to (@v 1582 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
1583 * Otherwise, @v is not modified and relaxed 1583 * Otherwise, @v is not modified and relaxed ordering is provided.
1584 * 1584 *
1585 * Unsafe to use in noinstr code; use raw_ato 1585 * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
1586 * 1586 *
1587 * Return: The old value of (@v - 1), regardl 1587 * Return: The old value of (@v - 1), regardless of whether @v was updated.
1588 */ 1588 */
1589 static __always_inline int 1589 static __always_inline int
1590 atomic_dec_if_positive(atomic_t *v) 1590 atomic_dec_if_positive(atomic_t *v)
1591 { 1591 {
1592 kcsan_mb(); 1592 kcsan_mb();
1593 instrument_atomic_read_write(v, sizeo 1593 instrument_atomic_read_write(v, sizeof(*v));
1594 return raw_atomic_dec_if_positive(v); 1594 return raw_atomic_dec_if_positive(v);
1595 } 1595 }
1596 1596
1597 /** 1597 /**
1598 * atomic64_read() - atomic load with relaxed 1598 * atomic64_read() - atomic load with relaxed ordering
1599 * @v: pointer to atomic64_t 1599 * @v: pointer to atomic64_t
1600 * 1600 *
1601 * Atomically loads the value of @v with rela 1601 * Atomically loads the value of @v with relaxed ordering.
1602 * 1602 *
1603 * Unsafe to use in noinstr code; use raw_ato 1603 * Unsafe to use in noinstr code; use raw_atomic64_read() there.
1604 * 1604 *
1605 * Return: The value loaded from @v. 1605 * Return: The value loaded from @v.
1606 */ 1606 */
1607 static __always_inline s64 1607 static __always_inline s64
1608 atomic64_read(const atomic64_t *v) 1608 atomic64_read(const atomic64_t *v)
1609 { 1609 {
1610 instrument_atomic_read(v, sizeof(*v)) 1610 instrument_atomic_read(v, sizeof(*v));
1611 return raw_atomic64_read(v); 1611 return raw_atomic64_read(v);
1612 } 1612 }
1613 1613
1614 /** 1614 /**
1615 * atomic64_read_acquire() - atomic load with 1615 * atomic64_read_acquire() - atomic load with acquire ordering
1616 * @v: pointer to atomic64_t 1616 * @v: pointer to atomic64_t
1617 * 1617 *
1618 * Atomically loads the value of @v with acqu 1618 * Atomically loads the value of @v with acquire ordering.
1619 * 1619 *
1620 * Unsafe to use in noinstr code; use raw_ato 1620 * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
1621 * 1621 *
1622 * Return: The value loaded from @v. 1622 * Return: The value loaded from @v.
1623 */ 1623 */
1624 static __always_inline s64 1624 static __always_inline s64
1625 atomic64_read_acquire(const atomic64_t *v) 1625 atomic64_read_acquire(const atomic64_t *v)
1626 { 1626 {
1627 instrument_atomic_read(v, sizeof(*v)) 1627 instrument_atomic_read(v, sizeof(*v));
1628 return raw_atomic64_read_acquire(v); 1628 return raw_atomic64_read_acquire(v);
1629 } 1629 }
1630 1630
1631 /** 1631 /**
1632 * atomic64_set() - atomic set with relaxed o 1632 * atomic64_set() - atomic set with relaxed ordering
1633 * @v: pointer to atomic64_t 1633 * @v: pointer to atomic64_t
1634 * @i: s64 value to assign 1634 * @i: s64 value to assign
1635 * 1635 *
1636 * Atomically sets @v to @i with relaxed orde 1636 * Atomically sets @v to @i with relaxed ordering.
1637 * 1637 *
1638 * Unsafe to use in noinstr code; use raw_ato 1638 * Unsafe to use in noinstr code; use raw_atomic64_set() there.
1639 * 1639 *
1640 * Return: Nothing. 1640 * Return: Nothing.
1641 */ 1641 */
1642 static __always_inline void 1642 static __always_inline void
1643 atomic64_set(atomic64_t *v, s64 i) 1643 atomic64_set(atomic64_t *v, s64 i)
1644 { 1644 {
1645 instrument_atomic_write(v, sizeof(*v) 1645 instrument_atomic_write(v, sizeof(*v));
1646 raw_atomic64_set(v, i); 1646 raw_atomic64_set(v, i);
1647 } 1647 }
1648 1648
1649 /** 1649 /**
1650 * atomic64_set_release() - atomic set with r 1650 * atomic64_set_release() - atomic set with release ordering
1651 * @v: pointer to atomic64_t 1651 * @v: pointer to atomic64_t
1652 * @i: s64 value to assign 1652 * @i: s64 value to assign
1653 * 1653 *
1654 * Atomically sets @v to @i with release orde 1654 * Atomically sets @v to @i with release ordering.
1655 * 1655 *
1656 * Unsafe to use in noinstr code; use raw_ato 1656 * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
1657 * 1657 *
1658 * Return: Nothing. 1658 * Return: Nothing.
1659 */ 1659 */
1660 static __always_inline void 1660 static __always_inline void
1661 atomic64_set_release(atomic64_t *v, s64 i) 1661 atomic64_set_release(atomic64_t *v, s64 i)
1662 { 1662 {
1663 kcsan_release(); 1663 kcsan_release();
1664 instrument_atomic_write(v, sizeof(*v) 1664 instrument_atomic_write(v, sizeof(*v));
1665 raw_atomic64_set_release(v, i); 1665 raw_atomic64_set_release(v, i);
1666 } 1666 }
1667 1667
1668 /** 1668 /**
1669 * atomic64_add() - atomic add with relaxed o 1669 * atomic64_add() - atomic add with relaxed ordering
1670 * @i: s64 value to add 1670 * @i: s64 value to add
1671 * @v: pointer to atomic64_t 1671 * @v: pointer to atomic64_t
1672 * 1672 *
1673 * Atomically updates @v to (@v + @i) with re 1673 * Atomically updates @v to (@v + @i) with relaxed ordering.
1674 * 1674 *
1675 * Unsafe to use in noinstr code; use raw_ato 1675 * Unsafe to use in noinstr code; use raw_atomic64_add() there.
1676 * 1676 *
1677 * Return: Nothing. 1677 * Return: Nothing.
1678 */ 1678 */
1679 static __always_inline void 1679 static __always_inline void
1680 atomic64_add(s64 i, atomic64_t *v) 1680 atomic64_add(s64 i, atomic64_t *v)
1681 { 1681 {
1682 instrument_atomic_read_write(v, sizeo 1682 instrument_atomic_read_write(v, sizeof(*v));
1683 raw_atomic64_add(i, v); 1683 raw_atomic64_add(i, v);
1684 } 1684 }
1685 1685
1686 /** 1686 /**
1687 * atomic64_add_return() - atomic add with fu 1687 * atomic64_add_return() - atomic add with full ordering
1688 * @i: s64 value to add 1688 * @i: s64 value to add
1689 * @v: pointer to atomic64_t 1689 * @v: pointer to atomic64_t
1690 * 1690 *
1691 * Atomically updates @v to (@v + @i) with fu 1691 * Atomically updates @v to (@v + @i) with full ordering.
1692 * 1692 *
1693 * Unsafe to use in noinstr code; use raw_ato 1693 * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
1694 * 1694 *
1695 * Return: The updated value of @v. 1695 * Return: The updated value of @v.
1696 */ 1696 */
1697 static __always_inline s64 1697 static __always_inline s64
1698 atomic64_add_return(s64 i, atomic64_t *v) 1698 atomic64_add_return(s64 i, atomic64_t *v)
1699 { 1699 {
1700 kcsan_mb(); 1700 kcsan_mb();
1701 instrument_atomic_read_write(v, sizeo 1701 instrument_atomic_read_write(v, sizeof(*v));
1702 return raw_atomic64_add_return(i, v); 1702 return raw_atomic64_add_return(i, v);
1703 } 1703 }
1704 1704
1705 /** 1705 /**
1706 * atomic64_add_return_acquire() - atomic add 1706 * atomic64_add_return_acquire() - atomic add with acquire ordering
1707 * @i: s64 value to add 1707 * @i: s64 value to add
1708 * @v: pointer to atomic64_t 1708 * @v: pointer to atomic64_t
1709 * 1709 *
1710 * Atomically updates @v to (@v + @i) with ac 1710 * Atomically updates @v to (@v + @i) with acquire ordering.
1711 * 1711 *
1712 * Unsafe to use in noinstr code; use raw_ato 1712 * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
1713 * 1713 *
1714 * Return: The updated value of @v. 1714 * Return: The updated value of @v.
1715 */ 1715 */
1716 static __always_inline s64 1716 static __always_inline s64
1717 atomic64_add_return_acquire(s64 i, atomic64_t 1717 atomic64_add_return_acquire(s64 i, atomic64_t *v)
1718 { 1718 {
1719 instrument_atomic_read_write(v, sizeo 1719 instrument_atomic_read_write(v, sizeof(*v));
1720 return raw_atomic64_add_return_acquir 1720 return raw_atomic64_add_return_acquire(i, v);
1721 } 1721 }
1722 1722
1723 /** 1723 /**
1724 * atomic64_add_return_release() - atomic add 1724 * atomic64_add_return_release() - atomic add with release ordering
1725 * @i: s64 value to add 1725 * @i: s64 value to add
1726 * @v: pointer to atomic64_t 1726 * @v: pointer to atomic64_t
1727 * 1727 *
1728 * Atomically updates @v to (@v + @i) with re 1728 * Atomically updates @v to (@v + @i) with release ordering.
1729 * 1729 *
1730 * Unsafe to use in noinstr code; use raw_ato 1730 * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
1731 * 1731 *
1732 * Return: The updated value of @v. 1732 * Return: The updated value of @v.
1733 */ 1733 */
1734 static __always_inline s64 1734 static __always_inline s64
1735 atomic64_add_return_release(s64 i, atomic64_t 1735 atomic64_add_return_release(s64 i, atomic64_t *v)
1736 { 1736 {
1737 kcsan_release(); 1737 kcsan_release();
1738 instrument_atomic_read_write(v, sizeo 1738 instrument_atomic_read_write(v, sizeof(*v));
1739 return raw_atomic64_add_return_releas 1739 return raw_atomic64_add_return_release(i, v);
1740 } 1740 }
1741 1741
1742 /** 1742 /**
1743 * atomic64_add_return_relaxed() - atomic add 1743 * atomic64_add_return_relaxed() - atomic add with relaxed ordering
1744 * @i: s64 value to add 1744 * @i: s64 value to add
1745 * @v: pointer to atomic64_t 1745 * @v: pointer to atomic64_t
1746 * 1746 *
1747 * Atomically updates @v to (@v + @i) with re 1747 * Atomically updates @v to (@v + @i) with relaxed ordering.
1748 * 1748 *
1749 * Unsafe to use in noinstr code; use raw_ato 1749 * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
1750 * 1750 *
1751 * Return: The updated value of @v. 1751 * Return: The updated value of @v.
1752 */ 1752 */
1753 static __always_inline s64 1753 static __always_inline s64
1754 atomic64_add_return_relaxed(s64 i, atomic64_t 1754 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
1755 { 1755 {
1756 instrument_atomic_read_write(v, sizeo 1756 instrument_atomic_read_write(v, sizeof(*v));
1757 return raw_atomic64_add_return_relaxe 1757 return raw_atomic64_add_return_relaxed(i, v);
1758 } 1758 }
1759 1759
1760 /** 1760 /**
1761 * atomic64_fetch_add() - atomic add with ful 1761 * atomic64_fetch_add() - atomic add with full ordering
1762 * @i: s64 value to add 1762 * @i: s64 value to add
1763 * @v: pointer to atomic64_t 1763 * @v: pointer to atomic64_t
1764 * 1764 *
1765 * Atomically updates @v to (@v + @i) with fu 1765 * Atomically updates @v to (@v + @i) with full ordering.
1766 * 1766 *
1767 * Unsafe to use in noinstr code; use raw_ato 1767 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
1768 * 1768 *
1769 * Return: The original value of @v. 1769 * Return: The original value of @v.
1770 */ 1770 */
1771 static __always_inline s64 1771 static __always_inline s64
1772 atomic64_fetch_add(s64 i, atomic64_t *v) 1772 atomic64_fetch_add(s64 i, atomic64_t *v)
1773 { 1773 {
1774 kcsan_mb(); 1774 kcsan_mb();
1775 instrument_atomic_read_write(v, sizeo 1775 instrument_atomic_read_write(v, sizeof(*v));
1776 return raw_atomic64_fetch_add(i, v); 1776 return raw_atomic64_fetch_add(i, v);
1777 } 1777 }
1778 1778
1779 /** 1779 /**
1780 * atomic64_fetch_add_acquire() - atomic add 1780 * atomic64_fetch_add_acquire() - atomic add with acquire ordering
1781 * @i: s64 value to add 1781 * @i: s64 value to add
1782 * @v: pointer to atomic64_t 1782 * @v: pointer to atomic64_t
1783 * 1783 *
1784 * Atomically updates @v to (@v + @i) with ac 1784 * Atomically updates @v to (@v + @i) with acquire ordering.
1785 * 1785 *
1786 * Unsafe to use in noinstr code; use raw_ato 1786 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
1787 * 1787 *
1788 * Return: The original value of @v. 1788 * Return: The original value of @v.
1789 */ 1789 */
1790 static __always_inline s64 1790 static __always_inline s64
1791 atomic64_fetch_add_acquire(s64 i, atomic64_t 1791 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1792 { 1792 {
1793 instrument_atomic_read_write(v, sizeo 1793 instrument_atomic_read_write(v, sizeof(*v));
1794 return raw_atomic64_fetch_add_acquire 1794 return raw_atomic64_fetch_add_acquire(i, v);
1795 } 1795 }
1796 1796
1797 /** 1797 /**
1798 * atomic64_fetch_add_release() - atomic add 1798 * atomic64_fetch_add_release() - atomic add with release ordering
1799 * @i: s64 value to add 1799 * @i: s64 value to add
1800 * @v: pointer to atomic64_t 1800 * @v: pointer to atomic64_t
1801 * 1801 *
1802 * Atomically updates @v to (@v + @i) with re 1802 * Atomically updates @v to (@v + @i) with release ordering.
1803 * 1803 *
1804 * Unsafe to use in noinstr code; use raw_ato 1804 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
1805 * 1805 *
1806 * Return: The original value of @v. 1806 * Return: The original value of @v.
1807 */ 1807 */
1808 static __always_inline s64 1808 static __always_inline s64
1809 atomic64_fetch_add_release(s64 i, atomic64_t 1809 atomic64_fetch_add_release(s64 i, atomic64_t *v)
1810 { 1810 {
1811 kcsan_release(); 1811 kcsan_release();
1812 instrument_atomic_read_write(v, sizeo 1812 instrument_atomic_read_write(v, sizeof(*v));
1813 return raw_atomic64_fetch_add_release 1813 return raw_atomic64_fetch_add_release(i, v);
1814 } 1814 }
1815 1815
1816 /** 1816 /**
1817 * atomic64_fetch_add_relaxed() - atomic add 1817 * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
1818 * @i: s64 value to add 1818 * @i: s64 value to add
1819 * @v: pointer to atomic64_t 1819 * @v: pointer to atomic64_t
1820 * 1820 *
1821 * Atomically updates @v to (@v + @i) with re 1821 * Atomically updates @v to (@v + @i) with relaxed ordering.
1822 * 1822 *
1823 * Unsafe to use in noinstr code; use raw_ato 1823 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
1824 * 1824 *
1825 * Return: The original value of @v. 1825 * Return: The original value of @v.
1826 */ 1826 */
1827 static __always_inline s64 1827 static __always_inline s64
1828 atomic64_fetch_add_relaxed(s64 i, atomic64_t 1828 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
1829 { 1829 {
1830 instrument_atomic_read_write(v, sizeo 1830 instrument_atomic_read_write(v, sizeof(*v));
1831 return raw_atomic64_fetch_add_relaxed 1831 return raw_atomic64_fetch_add_relaxed(i, v);
1832 } 1832 }
1833 1833
1834 /** 1834 /**
1835 * atomic64_sub() - atomic subtract with rela 1835 * atomic64_sub() - atomic subtract with relaxed ordering
1836 * @i: s64 value to subtract 1836 * @i: s64 value to subtract
1837 * @v: pointer to atomic64_t 1837 * @v: pointer to atomic64_t
1838 * 1838 *
1839 * Atomically updates @v to (@v - @i) with re 1839 * Atomically updates @v to (@v - @i) with relaxed ordering.
1840 * 1840 *
1841 * Unsafe to use in noinstr code; use raw_ato 1841 * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
1842 * 1842 *
1843 * Return: Nothing. 1843 * Return: Nothing.
1844 */ 1844 */
1845 static __always_inline void 1845 static __always_inline void
1846 atomic64_sub(s64 i, atomic64_t *v) 1846 atomic64_sub(s64 i, atomic64_t *v)
1847 { 1847 {
1848 instrument_atomic_read_write(v, sizeo 1848 instrument_atomic_read_write(v, sizeof(*v));
1849 raw_atomic64_sub(i, v); 1849 raw_atomic64_sub(i, v);
1850 } 1850 }
1851 1851
1852 /** 1852 /**
1853 * atomic64_sub_return() - atomic subtract wi 1853 * atomic64_sub_return() - atomic subtract with full ordering
1854 * @i: s64 value to subtract 1854 * @i: s64 value to subtract
1855 * @v: pointer to atomic64_t 1855 * @v: pointer to atomic64_t
1856 * 1856 *
1857 * Atomically updates @v to (@v - @i) with fu 1857 * Atomically updates @v to (@v - @i) with full ordering.
1858 * 1858 *
1859 * Unsafe to use in noinstr code; use raw_ato 1859 * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
1860 * 1860 *
1861 * Return: The updated value of @v. 1861 * Return: The updated value of @v.
1862 */ 1862 */
1863 static __always_inline s64 1863 static __always_inline s64
1864 atomic64_sub_return(s64 i, atomic64_t *v) 1864 atomic64_sub_return(s64 i, atomic64_t *v)
1865 { 1865 {
1866 kcsan_mb(); 1866 kcsan_mb();
1867 instrument_atomic_read_write(v, sizeo 1867 instrument_atomic_read_write(v, sizeof(*v));
1868 return raw_atomic64_sub_return(i, v); 1868 return raw_atomic64_sub_return(i, v);
1869 } 1869 }
1870 1870
1871 /** 1871 /**
1872 * atomic64_sub_return_acquire() - atomic sub 1872 * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
1873 * @i: s64 value to subtract 1873 * @i: s64 value to subtract
1874 * @v: pointer to atomic64_t 1874 * @v: pointer to atomic64_t
1875 * 1875 *
1876 * Atomically updates @v to (@v - @i) with ac 1876 * Atomically updates @v to (@v - @i) with acquire ordering.
1877 * 1877 *
1878 * Unsafe to use in noinstr code; use raw_ato 1878 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
1879 * 1879 *
1880 * Return: The updated value of @v. 1880 * Return: The updated value of @v.
1881 */ 1881 */
1882 static __always_inline s64 1882 static __always_inline s64
1883 atomic64_sub_return_acquire(s64 i, atomic64_t 1883 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1884 { 1884 {
1885 instrument_atomic_read_write(v, sizeo 1885 instrument_atomic_read_write(v, sizeof(*v));
1886 return raw_atomic64_sub_return_acquir 1886 return raw_atomic64_sub_return_acquire(i, v);
1887 } 1887 }
1888 1888
1889 /** 1889 /**
1890 * atomic64_sub_return_release() - atomic sub 1890 * atomic64_sub_return_release() - atomic subtract with release ordering
1891 * @i: s64 value to subtract 1891 * @i: s64 value to subtract
1892 * @v: pointer to atomic64_t 1892 * @v: pointer to atomic64_t
1893 * 1893 *
1894 * Atomically updates @v to (@v - @i) with re 1894 * Atomically updates @v to (@v - @i) with release ordering.
1895 * 1895 *
1896 * Unsafe to use in noinstr code; use raw_ato 1896 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
1897 * 1897 *
1898 * Return: The updated value of @v. 1898 * Return: The updated value of @v.
1899 */ 1899 */
1900 static __always_inline s64 1900 static __always_inline s64
1901 atomic64_sub_return_release(s64 i, atomic64_t 1901 atomic64_sub_return_release(s64 i, atomic64_t *v)
1902 { 1902 {
1903 kcsan_release(); 1903 kcsan_release();
1904 instrument_atomic_read_write(v, sizeo 1904 instrument_atomic_read_write(v, sizeof(*v));
1905 return raw_atomic64_sub_return_releas 1905 return raw_atomic64_sub_return_release(i, v);
1906 } 1906 }
1907 1907
1908 /** 1908 /**
1909 * atomic64_sub_return_relaxed() - atomic sub 1909 * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
1910 * @i: s64 value to subtract 1910 * @i: s64 value to subtract
1911 * @v: pointer to atomic64_t 1911 * @v: pointer to atomic64_t
1912 * 1912 *
1913 * Atomically updates @v to (@v - @i) with re 1913 * Atomically updates @v to (@v - @i) with relaxed ordering.
1914 * 1914 *
1915 * Unsafe to use in noinstr code; use raw_ato 1915 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
1916 * 1916 *
1917 * Return: The updated value of @v. 1917 * Return: The updated value of @v.
1918 */ 1918 */
1919 static __always_inline s64 1919 static __always_inline s64
1920 atomic64_sub_return_relaxed(s64 i, atomic64_t 1920 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
1921 { 1921 {
1922 instrument_atomic_read_write(v, sizeo 1922 instrument_atomic_read_write(v, sizeof(*v));
1923 return raw_atomic64_sub_return_relaxe 1923 return raw_atomic64_sub_return_relaxed(i, v);
1924 } 1924 }
1925 1925
1926 /** 1926 /**
1927 * atomic64_fetch_sub() - atomic subtract wit 1927 * atomic64_fetch_sub() - atomic subtract with full ordering
1928 * @i: s64 value to subtract 1928 * @i: s64 value to subtract
1929 * @v: pointer to atomic64_t 1929 * @v: pointer to atomic64_t
1930 * 1930 *
1931 * Atomically updates @v to (@v - @i) with fu 1931 * Atomically updates @v to (@v - @i) with full ordering.
1932 * 1932 *
1933 * Unsafe to use in noinstr code; use raw_ato 1933 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
1934 * 1934 *
1935 * Return: The original value of @v. 1935 * Return: The original value of @v.
1936 */ 1936 */
1937 static __always_inline s64 1937 static __always_inline s64
1938 atomic64_fetch_sub(s64 i, atomic64_t *v) 1938 atomic64_fetch_sub(s64 i, atomic64_t *v)
1939 { 1939 {
1940 kcsan_mb(); 1940 kcsan_mb();
1941 instrument_atomic_read_write(v, sizeo 1941 instrument_atomic_read_write(v, sizeof(*v));
1942 return raw_atomic64_fetch_sub(i, v); 1942 return raw_atomic64_fetch_sub(i, v);
1943 } 1943 }
1944 1944
1945 /** 1945 /**
1946 * atomic64_fetch_sub_acquire() - atomic subt 1946 * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
1947 * @i: s64 value to subtract 1947 * @i: s64 value to subtract
1948 * @v: pointer to atomic64_t 1948 * @v: pointer to atomic64_t
1949 * 1949 *
1950 * Atomically updates @v to (@v - @i) with ac 1950 * Atomically updates @v to (@v - @i) with acquire ordering.
1951 * 1951 *
1952 * Unsafe to use in noinstr code; use raw_ato 1952 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
1953 * 1953 *
1954 * Return: The original value of @v. 1954 * Return: The original value of @v.
1955 */ 1955 */
1956 static __always_inline s64 1956 static __always_inline s64
1957 atomic64_fetch_sub_acquire(s64 i, atomic64_t 1957 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1958 { 1958 {
1959 instrument_atomic_read_write(v, sizeo 1959 instrument_atomic_read_write(v, sizeof(*v));
1960 return raw_atomic64_fetch_sub_acquire 1960 return raw_atomic64_fetch_sub_acquire(i, v);
1961 } 1961 }
1962 1962
1963 /** 1963 /**
1964 * atomic64_fetch_sub_release() - atomic subt 1964 * atomic64_fetch_sub_release() - atomic subtract with release ordering
1965 * @i: s64 value to subtract 1965 * @i: s64 value to subtract
1966 * @v: pointer to atomic64_t 1966 * @v: pointer to atomic64_t
1967 * 1967 *
1968 * Atomically updates @v to (@v - @i) with re 1968 * Atomically updates @v to (@v - @i) with release ordering.
1969 * 1969 *
1970 * Unsafe to use in noinstr code; use raw_ato 1970 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
1971 * 1971 *
1972 * Return: The original value of @v. 1972 * Return: The original value of @v.
1973 */ 1973 */
1974 static __always_inline s64 1974 static __always_inline s64
1975 atomic64_fetch_sub_release(s64 i, atomic64_t 1975 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1976 { 1976 {
1977 kcsan_release(); 1977 kcsan_release();
1978 instrument_atomic_read_write(v, sizeo 1978 instrument_atomic_read_write(v, sizeof(*v));
1979 return raw_atomic64_fetch_sub_release 1979 return raw_atomic64_fetch_sub_release(i, v);
1980 } 1980 }
1981 1981
1982 /** 1982 /**
1983 * atomic64_fetch_sub_relaxed() - atomic subt 1983 * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
1984 * @i: s64 value to subtract 1984 * @i: s64 value to subtract
1985 * @v: pointer to atomic64_t 1985 * @v: pointer to atomic64_t
1986 * 1986 *
1987 * Atomically updates @v to (@v - @i) with re 1987 * Atomically updates @v to (@v - @i) with relaxed ordering.
1988 * 1988 *
1989 * Unsafe to use in noinstr code; use raw_ato 1989 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
1990 * 1990 *
1991 * Return: The original value of @v. 1991 * Return: The original value of @v.
1992 */ 1992 */
1993 static __always_inline s64 1993 static __always_inline s64
1994 atomic64_fetch_sub_relaxed(s64 i, atomic64_t 1994 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1995 { 1995 {
1996 instrument_atomic_read_write(v, sizeo 1996 instrument_atomic_read_write(v, sizeof(*v));
1997 return raw_atomic64_fetch_sub_relaxed 1997 return raw_atomic64_fetch_sub_relaxed(i, v);
1998 } 1998 }
1999 1999
2000 /** 2000 /**
2001 * atomic64_inc() - atomic increment with rel 2001 * atomic64_inc() - atomic increment with relaxed ordering
2002 * @v: pointer to atomic64_t 2002 * @v: pointer to atomic64_t
2003 * 2003 *
2004 * Atomically updates @v to (@v + 1) with rel 2004 * Atomically updates @v to (@v + 1) with relaxed ordering.
2005 * 2005 *
2006 * Unsafe to use in noinstr code; use raw_ato 2006 * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
2007 * 2007 *
2008 * Return: Nothing. 2008 * Return: Nothing.
2009 */ 2009 */
2010 static __always_inline void 2010 static __always_inline void
2011 atomic64_inc(atomic64_t *v) 2011 atomic64_inc(atomic64_t *v)
2012 { 2012 {
2013 instrument_atomic_read_write(v, sizeo 2013 instrument_atomic_read_write(v, sizeof(*v));
2014 raw_atomic64_inc(v); 2014 raw_atomic64_inc(v);
2015 } 2015 }
2016 2016
2017 /** 2017 /**
2018 * atomic64_inc_return() - atomic increment w 2018 * atomic64_inc_return() - atomic increment with full ordering
2019 * @v: pointer to atomic64_t 2019 * @v: pointer to atomic64_t
2020 * 2020 *
2021 * Atomically updates @v to (@v + 1) with ful 2021 * Atomically updates @v to (@v + 1) with full ordering.
2022 * 2022 *
2023 * Unsafe to use in noinstr code; use raw_ato 2023 * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
2024 * 2024 *
2025 * Return: The updated value of @v. 2025 * Return: The updated value of @v.
2026 */ 2026 */
2027 static __always_inline s64 2027 static __always_inline s64
2028 atomic64_inc_return(atomic64_t *v) 2028 atomic64_inc_return(atomic64_t *v)
2029 { 2029 {
2030 kcsan_mb(); 2030 kcsan_mb();
2031 instrument_atomic_read_write(v, sizeo 2031 instrument_atomic_read_write(v, sizeof(*v));
2032 return raw_atomic64_inc_return(v); 2032 return raw_atomic64_inc_return(v);
2033 } 2033 }
2034 2034
2035 /** 2035 /**
2036 * atomic64_inc_return_acquire() - atomic inc 2036 * atomic64_inc_return_acquire() - atomic increment with acquire ordering
2037 * @v: pointer to atomic64_t 2037 * @v: pointer to atomic64_t
2038 * 2038 *
2039 * Atomically updates @v to (@v + 1) with acq 2039 * Atomically updates @v to (@v + 1) with acquire ordering.
2040 * 2040 *
2041 * Unsafe to use in noinstr code; use raw_ato 2041 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
2042 * 2042 *
2043 * Return: The updated value of @v. 2043 * Return: The updated value of @v.
2044 */ 2044 */
2045 static __always_inline s64 2045 static __always_inline s64
2046 atomic64_inc_return_acquire(atomic64_t *v) 2046 atomic64_inc_return_acquire(atomic64_t *v)
2047 { 2047 {
2048 instrument_atomic_read_write(v, sizeo 2048 instrument_atomic_read_write(v, sizeof(*v));
2049 return raw_atomic64_inc_return_acquir 2049 return raw_atomic64_inc_return_acquire(v);
2050 } 2050 }
2051 2051
2052 /** 2052 /**
2053 * atomic64_inc_return_release() - atomic inc 2053 * atomic64_inc_return_release() - atomic increment with release ordering
2054 * @v: pointer to atomic64_t 2054 * @v: pointer to atomic64_t
2055 * 2055 *
2056 * Atomically updates @v to (@v + 1) with rel 2056 * Atomically updates @v to (@v + 1) with release ordering.
2057 * 2057 *
2058 * Unsafe to use in noinstr code; use raw_ato 2058 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
2059 * 2059 *
2060 * Return: The updated value of @v. 2060 * Return: The updated value of @v.
2061 */ 2061 */
2062 static __always_inline s64 2062 static __always_inline s64
2063 atomic64_inc_return_release(atomic64_t *v) 2063 atomic64_inc_return_release(atomic64_t *v)
2064 { 2064 {
2065 kcsan_release(); 2065 kcsan_release();
2066 instrument_atomic_read_write(v, sizeo 2066 instrument_atomic_read_write(v, sizeof(*v));
2067 return raw_atomic64_inc_return_releas 2067 return raw_atomic64_inc_return_release(v);
2068 } 2068 }
2069 2069
2070 /** 2070 /**
2071 * atomic64_inc_return_relaxed() - atomic inc 2071 * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
2072 * @v: pointer to atomic64_t 2072 * @v: pointer to atomic64_t
2073 * 2073 *
2074 * Atomically updates @v to (@v + 1) with rel 2074 * Atomically updates @v to (@v + 1) with relaxed ordering.
2075 * 2075 *
2076 * Unsafe to use in noinstr code; use raw_ato 2076 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
2077 * 2077 *
2078 * Return: The updated value of @v. 2078 * Return: The updated value of @v.
2079 */ 2079 */
2080 static __always_inline s64 2080 static __always_inline s64
2081 atomic64_inc_return_relaxed(atomic64_t *v) 2081 atomic64_inc_return_relaxed(atomic64_t *v)
2082 { 2082 {
2083 instrument_atomic_read_write(v, sizeo 2083 instrument_atomic_read_write(v, sizeof(*v));
2084 return raw_atomic64_inc_return_relaxe 2084 return raw_atomic64_inc_return_relaxed(v);
2085 } 2085 }
2086 2086
2087 /** 2087 /**
2088 * atomic64_fetch_inc() - atomic increment wi 2088 * atomic64_fetch_inc() - atomic increment with full ordering
2089 * @v: pointer to atomic64_t 2089 * @v: pointer to atomic64_t
2090 * 2090 *
2091 * Atomically updates @v to (@v + 1) with ful 2091 * Atomically updates @v to (@v + 1) with full ordering.
2092 * 2092 *
2093 * Unsafe to use in noinstr code; use raw_ato 2093 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
2094 * 2094 *
2095 * Return: The original value of @v. 2095 * Return: The original value of @v.
2096 */ 2096 */
2097 static __always_inline s64 2097 static __always_inline s64
2098 atomic64_fetch_inc(atomic64_t *v) 2098 atomic64_fetch_inc(atomic64_t *v)
2099 { 2099 {
2100 kcsan_mb(); 2100 kcsan_mb();
2101 instrument_atomic_read_write(v, sizeo 2101 instrument_atomic_read_write(v, sizeof(*v));
2102 return raw_atomic64_fetch_inc(v); 2102 return raw_atomic64_fetch_inc(v);
2103 } 2103 }
2104 2104
2105 /** 2105 /**
2106 * atomic64_fetch_inc_acquire() - atomic incr 2106 * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
2107 * @v: pointer to atomic64_t 2107 * @v: pointer to atomic64_t
2108 * 2108 *
2109 * Atomically updates @v to (@v + 1) with acq 2109 * Atomically updates @v to (@v + 1) with acquire ordering.
2110 * 2110 *
2111 * Unsafe to use in noinstr code; use raw_ato 2111 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
2112 * 2112 *
2113 * Return: The original value of @v. 2113 * Return: The original value of @v.
2114 */ 2114 */
2115 static __always_inline s64 2115 static __always_inline s64
2116 atomic64_fetch_inc_acquire(atomic64_t *v) 2116 atomic64_fetch_inc_acquire(atomic64_t *v)
2117 { 2117 {
2118 instrument_atomic_read_write(v, sizeo 2118 instrument_atomic_read_write(v, sizeof(*v));
2119 return raw_atomic64_fetch_inc_acquire 2119 return raw_atomic64_fetch_inc_acquire(v);
2120 } 2120 }
2121 2121
2122 /** 2122 /**
2123 * atomic64_fetch_inc_release() - atomic incr 2123 * atomic64_fetch_inc_release() - atomic increment with release ordering
2124 * @v: pointer to atomic64_t 2124 * @v: pointer to atomic64_t
2125 * 2125 *
2126 * Atomically updates @v to (@v + 1) with rel 2126 * Atomically updates @v to (@v + 1) with release ordering.
2127 * 2127 *
2128 * Unsafe to use in noinstr code; use raw_ato 2128 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
2129 * 2129 *
2130 * Return: The original value of @v. 2130 * Return: The original value of @v.
2131 */ 2131 */
2132 static __always_inline s64 2132 static __always_inline s64
2133 atomic64_fetch_inc_release(atomic64_t *v) 2133 atomic64_fetch_inc_release(atomic64_t *v)
2134 { 2134 {
2135 kcsan_release(); 2135 kcsan_release();
2136 instrument_atomic_read_write(v, sizeo 2136 instrument_atomic_read_write(v, sizeof(*v));
2137 return raw_atomic64_fetch_inc_release 2137 return raw_atomic64_fetch_inc_release(v);
2138 } 2138 }
2139 2139
2140 /** 2140 /**
2141 * atomic64_fetch_inc_relaxed() - atomic incr 2141 * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
2142 * @v: pointer to atomic64_t 2142 * @v: pointer to atomic64_t
2143 * 2143 *
2144 * Atomically updates @v to (@v + 1) with rel 2144 * Atomically updates @v to (@v + 1) with relaxed ordering.
2145 * 2145 *
2146 * Unsafe to use in noinstr code; use raw_ato 2146 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
2147 * 2147 *
2148 * Return: The original value of @v. 2148 * Return: The original value of @v.
2149 */ 2149 */
2150 static __always_inline s64 2150 static __always_inline s64
2151 atomic64_fetch_inc_relaxed(atomic64_t *v) 2151 atomic64_fetch_inc_relaxed(atomic64_t *v)
2152 { 2152 {
2153 instrument_atomic_read_write(v, sizeo 2153 instrument_atomic_read_write(v, sizeof(*v));
2154 return raw_atomic64_fetch_inc_relaxed 2154 return raw_atomic64_fetch_inc_relaxed(v);
2155 } 2155 }
2156 2156
2157 /** 2157 /**
2158 * atomic64_dec() - atomic decrement with rel 2158 * atomic64_dec() - atomic decrement with relaxed ordering
2159 * @v: pointer to atomic64_t 2159 * @v: pointer to atomic64_t
2160 * 2160 *
2161 * Atomically updates @v to (@v - 1) with rel 2161 * Atomically updates @v to (@v - 1) with relaxed ordering.
2162 * 2162 *
2163 * Unsafe to use in noinstr code; use raw_ato 2163 * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
2164 * 2164 *
2165 * Return: Nothing. 2165 * Return: Nothing.
2166 */ 2166 */
2167 static __always_inline void 2167 static __always_inline void
2168 atomic64_dec(atomic64_t *v) 2168 atomic64_dec(atomic64_t *v)
2169 { 2169 {
2170 instrument_atomic_read_write(v, sizeo 2170 instrument_atomic_read_write(v, sizeof(*v));
2171 raw_atomic64_dec(v); 2171 raw_atomic64_dec(v);
2172 } 2172 }
2173 2173
2174 /** 2174 /**
2175 * atomic64_dec_return() - atomic decrement w 2175 * atomic64_dec_return() - atomic decrement with full ordering
2176 * @v: pointer to atomic64_t 2176 * @v: pointer to atomic64_t
2177 * 2177 *
2178 * Atomically updates @v to (@v - 1) with ful 2178 * Atomically updates @v to (@v - 1) with full ordering.
2179 * 2179 *
2180 * Unsafe to use in noinstr code; use raw_ato 2180 * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
2181 * 2181 *
2182 * Return: The updated value of @v. 2182 * Return: The updated value of @v.
2183 */ 2183 */
2184 static __always_inline s64 2184 static __always_inline s64
2185 atomic64_dec_return(atomic64_t *v) 2185 atomic64_dec_return(atomic64_t *v)
2186 { 2186 {
2187 kcsan_mb(); 2187 kcsan_mb();
2188 instrument_atomic_read_write(v, sizeo 2188 instrument_atomic_read_write(v, sizeof(*v));
2189 return raw_atomic64_dec_return(v); 2189 return raw_atomic64_dec_return(v);
2190 } 2190 }
2191 2191
2192 /** 2192 /**
2193 * atomic64_dec_return_acquire() - atomic dec 2193 * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
2194 * @v: pointer to atomic64_t 2194 * @v: pointer to atomic64_t
2195 * 2195 *
2196 * Atomically updates @v to (@v - 1) with acq 2196 * Atomically updates @v to (@v - 1) with acquire ordering.
2197 * 2197 *
2198 * Unsafe to use in noinstr code; use raw_ato 2198 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
2199 * 2199 *
2200 * Return: The updated value of @v. 2200 * Return: The updated value of @v.
2201 */ 2201 */
2202 static __always_inline s64 2202 static __always_inline s64
2203 atomic64_dec_return_acquire(atomic64_t *v) 2203 atomic64_dec_return_acquire(atomic64_t *v)
2204 { 2204 {
2205 instrument_atomic_read_write(v, sizeo 2205 instrument_atomic_read_write(v, sizeof(*v));
2206 return raw_atomic64_dec_return_acquir 2206 return raw_atomic64_dec_return_acquire(v);
2207 } 2207 }
2208 2208
2209 /** 2209 /**
2210 * atomic64_dec_return_release() - atomic dec 2210 * atomic64_dec_return_release() - atomic decrement with release ordering
2211 * @v: pointer to atomic64_t 2211 * @v: pointer to atomic64_t
2212 * 2212 *
2213 * Atomically updates @v to (@v - 1) with rel 2213 * Atomically updates @v to (@v - 1) with release ordering.
2214 * 2214 *
2215 * Unsafe to use in noinstr code; use raw_ato 2215 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
2216 * 2216 *
2217 * Return: The updated value of @v. 2217 * Return: The updated value of @v.
2218 */ 2218 */
2219 static __always_inline s64 2219 static __always_inline s64
2220 atomic64_dec_return_release(atomic64_t *v) 2220 atomic64_dec_return_release(atomic64_t *v)
2221 { 2221 {
2222 kcsan_release(); 2222 kcsan_release();
2223 instrument_atomic_read_write(v, sizeo 2223 instrument_atomic_read_write(v, sizeof(*v));
2224 return raw_atomic64_dec_return_releas 2224 return raw_atomic64_dec_return_release(v);
2225 } 2225 }
2226 2226
2227 /** 2227 /**
2228 * atomic64_dec_return_relaxed() - atomic dec 2228 * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
2229 * @v: pointer to atomic64_t 2229 * @v: pointer to atomic64_t
2230 * 2230 *
2231 * Atomically updates @v to (@v - 1) with rel 2231 * Atomically updates @v to (@v - 1) with relaxed ordering.
2232 * 2232 *
2233 * Unsafe to use in noinstr code; use raw_ato 2233 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
2234 * 2234 *
2235 * Return: The updated value of @v. 2235 * Return: The updated value of @v.
2236 */ 2236 */
2237 static __always_inline s64 2237 static __always_inline s64
2238 atomic64_dec_return_relaxed(atomic64_t *v) 2238 atomic64_dec_return_relaxed(atomic64_t *v)
2239 { 2239 {
2240 instrument_atomic_read_write(v, sizeo 2240 instrument_atomic_read_write(v, sizeof(*v));
2241 return raw_atomic64_dec_return_relaxe 2241 return raw_atomic64_dec_return_relaxed(v);
2242 } 2242 }
2243 2243
2244 /** 2244 /**
2245 * atomic64_fetch_dec() - atomic decrement wi 2245 * atomic64_fetch_dec() - atomic decrement with full ordering
2246 * @v: pointer to atomic64_t 2246 * @v: pointer to atomic64_t
2247 * 2247 *
2248 * Atomically updates @v to (@v - 1) with ful 2248 * Atomically updates @v to (@v - 1) with full ordering.
2249 * 2249 *
2250 * Unsafe to use in noinstr code; use raw_ato 2250 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
2251 * 2251 *
2252 * Return: The original value of @v. 2252 * Return: The original value of @v.
2253 */ 2253 */
2254 static __always_inline s64 2254 static __always_inline s64
2255 atomic64_fetch_dec(atomic64_t *v) 2255 atomic64_fetch_dec(atomic64_t *v)
2256 { 2256 {
2257 kcsan_mb(); 2257 kcsan_mb();
2258 instrument_atomic_read_write(v, sizeo 2258 instrument_atomic_read_write(v, sizeof(*v));
2259 return raw_atomic64_fetch_dec(v); 2259 return raw_atomic64_fetch_dec(v);
2260 } 2260 }
2261 2261
2262 /** 2262 /**
2263 * atomic64_fetch_dec_acquire() - atomic decr 2263 * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
2264 * @v: pointer to atomic64_t 2264 * @v: pointer to atomic64_t
2265 * 2265 *
2266 * Atomically updates @v to (@v - 1) with acq 2266 * Atomically updates @v to (@v - 1) with acquire ordering.
2267 * 2267 *
2268 * Unsafe to use in noinstr code; use raw_ato 2268 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
2269 * 2269 *
2270 * Return: The original value of @v. 2270 * Return: The original value of @v.
2271 */ 2271 */
2272 static __always_inline s64 2272 static __always_inline s64
2273 atomic64_fetch_dec_acquire(atomic64_t *v) 2273 atomic64_fetch_dec_acquire(atomic64_t *v)
2274 { 2274 {
2275 instrument_atomic_read_write(v, sizeo 2275 instrument_atomic_read_write(v, sizeof(*v));
2276 return raw_atomic64_fetch_dec_acquire 2276 return raw_atomic64_fetch_dec_acquire(v);
2277 } 2277 }
2278 2278
2279 /** 2279 /**
2280 * atomic64_fetch_dec_release() - atomic decr 2280 * atomic64_fetch_dec_release() - atomic decrement with release ordering
2281 * @v: pointer to atomic64_t 2281 * @v: pointer to atomic64_t
2282 * 2282 *
2283 * Atomically updates @v to (@v - 1) with rel 2283 * Atomically updates @v to (@v - 1) with release ordering.
2284 * 2284 *
2285 * Unsafe to use in noinstr code; use raw_ato 2285 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
2286 * 2286 *
2287 * Return: The original value of @v. 2287 * Return: The original value of @v.
2288 */ 2288 */
2289 static __always_inline s64 2289 static __always_inline s64
2290 atomic64_fetch_dec_release(atomic64_t *v) 2290 atomic64_fetch_dec_release(atomic64_t *v)
2291 { 2291 {
2292 kcsan_release(); 2292 kcsan_release();
2293 instrument_atomic_read_write(v, sizeo 2293 instrument_atomic_read_write(v, sizeof(*v));
2294 return raw_atomic64_fetch_dec_release 2294 return raw_atomic64_fetch_dec_release(v);
2295 } 2295 }
2296 2296
2297 /** 2297 /**
2298 * atomic64_fetch_dec_relaxed() - atomic decr 2298 * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
2299 * @v: pointer to atomic64_t 2299 * @v: pointer to atomic64_t
2300 * 2300 *
2301 * Atomically updates @v to (@v - 1) with rel 2301 * Atomically updates @v to (@v - 1) with relaxed ordering.
2302 * 2302 *
2303 * Unsafe to use in noinstr code; use raw_ato 2303 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
2304 * 2304 *
2305 * Return: The original value of @v. 2305 * Return: The original value of @v.
2306 */ 2306 */
2307 static __always_inline s64 2307 static __always_inline s64
2308 atomic64_fetch_dec_relaxed(atomic64_t *v) 2308 atomic64_fetch_dec_relaxed(atomic64_t *v)
2309 { 2309 {
2310 instrument_atomic_read_write(v, sizeo 2310 instrument_atomic_read_write(v, sizeof(*v));
2311 return raw_atomic64_fetch_dec_relaxed 2311 return raw_atomic64_fetch_dec_relaxed(v);
2312 } 2312 }
2313 2313
2314 /** 2314 /**
2315 * atomic64_and() - atomic bitwise AND with r 2315 * atomic64_and() - atomic bitwise AND with relaxed ordering
2316 * @i: s64 value 2316 * @i: s64 value
2317 * @v: pointer to atomic64_t 2317 * @v: pointer to atomic64_t
2318 * 2318 *
2319 * Atomically updates @v to (@v & @i) with re 2319 * Atomically updates @v to (@v & @i) with relaxed ordering.
2320 * 2320 *
2321 * Unsafe to use in noinstr code; use raw_ato 2321 * Unsafe to use in noinstr code; use raw_atomic64_and() there.
2322 * 2322 *
2323 * Return: Nothing. 2323 * Return: Nothing.
2324 */ 2324 */
2325 static __always_inline void 2325 static __always_inline void
2326 atomic64_and(s64 i, atomic64_t *v) 2326 atomic64_and(s64 i, atomic64_t *v)
2327 { 2327 {
2328 instrument_atomic_read_write(v, sizeo 2328 instrument_atomic_read_write(v, sizeof(*v));
2329 raw_atomic64_and(i, v); 2329 raw_atomic64_and(i, v);
2330 } 2330 }
2331 2331
2332 /** 2332 /**
2333 * atomic64_fetch_and() - atomic bitwise AND 2333 * atomic64_fetch_and() - atomic bitwise AND with full ordering
2334 * @i: s64 value 2334 * @i: s64 value
2335 * @v: pointer to atomic64_t 2335 * @v: pointer to atomic64_t
2336 * 2336 *
2337 * Atomically updates @v to (@v & @i) with fu 2337 * Atomically updates @v to (@v & @i) with full ordering.
2338 * 2338 *
2339 * Unsafe to use in noinstr code; use raw_ato 2339 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
2340 * 2340 *
2341 * Return: The original value of @v. 2341 * Return: The original value of @v.
2342 */ 2342 */
2343 static __always_inline s64 2343 static __always_inline s64
2344 atomic64_fetch_and(s64 i, atomic64_t *v) 2344 atomic64_fetch_and(s64 i, atomic64_t *v)
2345 { 2345 {
2346 kcsan_mb(); 2346 kcsan_mb();
2347 instrument_atomic_read_write(v, sizeo 2347 instrument_atomic_read_write(v, sizeof(*v));
2348 return raw_atomic64_fetch_and(i, v); 2348 return raw_atomic64_fetch_and(i, v);
2349 } 2349 }
2350 2350
2351 /** 2351 /**
2352 * atomic64_fetch_and_acquire() - atomic bitw 2352 * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
2353 * @i: s64 value 2353 * @i: s64 value
2354 * @v: pointer to atomic64_t 2354 * @v: pointer to atomic64_t
2355 * 2355 *
2356 * Atomically updates @v to (@v & @i) with ac 2356 * Atomically updates @v to (@v & @i) with acquire ordering.
2357 * 2357 *
2358 * Unsafe to use in noinstr code; use raw_ato 2358 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
2359 * 2359 *
2360 * Return: The original value of @v. 2360 * Return: The original value of @v.
2361 */ 2361 */
2362 static __always_inline s64 2362 static __always_inline s64
2363 atomic64_fetch_and_acquire(s64 i, atomic64_t 2363 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2364 { 2364 {
2365 instrument_atomic_read_write(v, sizeo 2365 instrument_atomic_read_write(v, sizeof(*v));
2366 return raw_atomic64_fetch_and_acquire 2366 return raw_atomic64_fetch_and_acquire(i, v);
2367 } 2367 }
2368 2368
2369 /** 2369 /**
2370 * atomic64_fetch_and_release() - atomic bitw 2370 * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
2371 * @i: s64 value 2371 * @i: s64 value
2372 * @v: pointer to atomic64_t 2372 * @v: pointer to atomic64_t
2373 * 2373 *
2374 * Atomically updates @v to (@v & @i) with re 2374 * Atomically updates @v to (@v & @i) with release ordering.
2375 * 2375 *
2376 * Unsafe to use in noinstr code; use raw_ato 2376 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
2377 * 2377 *
2378 * Return: The original value of @v. 2378 * Return: The original value of @v.
2379 */ 2379 */
2380 static __always_inline s64 2380 static __always_inline s64
2381 atomic64_fetch_and_release(s64 i, atomic64_t 2381 atomic64_fetch_and_release(s64 i, atomic64_t *v)
2382 { 2382 {
2383 kcsan_release(); 2383 kcsan_release();
2384 instrument_atomic_read_write(v, sizeo 2384 instrument_atomic_read_write(v, sizeof(*v));
2385 return raw_atomic64_fetch_and_release 2385 return raw_atomic64_fetch_and_release(i, v);
2386 } 2386 }
2387 2387
2388 /** 2388 /**
2389 * atomic64_fetch_and_relaxed() - atomic bitw 2389 * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
2390 * @i: s64 value 2390 * @i: s64 value
2391 * @v: pointer to atomic64_t 2391 * @v: pointer to atomic64_t
2392 * 2392 *
2393 * Atomically updates @v to (@v & @i) with re 2393 * Atomically updates @v to (@v & @i) with relaxed ordering.
2394 * 2394 *
2395 * Unsafe to use in noinstr code; use raw_ato 2395 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
2396 * 2396 *
2397 * Return: The original value of @v. 2397 * Return: The original value of @v.
2398 */ 2398 */
2399 static __always_inline s64 2399 static __always_inline s64
2400 atomic64_fetch_and_relaxed(s64 i, atomic64_t 2400 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
2401 { 2401 {
2402 instrument_atomic_read_write(v, sizeo 2402 instrument_atomic_read_write(v, sizeof(*v));
2403 return raw_atomic64_fetch_and_relaxed 2403 return raw_atomic64_fetch_and_relaxed(i, v);
2404 } 2404 }
2405 2405
2406 /** 2406 /**
2407 * atomic64_andnot() - atomic bitwise AND NOT 2407 * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
2408 * @i: s64 value 2408 * @i: s64 value
2409 * @v: pointer to atomic64_t 2409 * @v: pointer to atomic64_t
2410 * 2410 *
2411 * Atomically updates @v to (@v & ~@i) with r 2411 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2412 * 2412 *
2413 * Unsafe to use in noinstr code; use raw_ato 2413 * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
2414 * 2414 *
2415 * Return: Nothing. 2415 * Return: Nothing.
2416 */ 2416 */
2417 static __always_inline void 2417 static __always_inline void
2418 atomic64_andnot(s64 i, atomic64_t *v) 2418 atomic64_andnot(s64 i, atomic64_t *v)
2419 { 2419 {
2420 instrument_atomic_read_write(v, sizeo 2420 instrument_atomic_read_write(v, sizeof(*v));
2421 raw_atomic64_andnot(i, v); 2421 raw_atomic64_andnot(i, v);
2422 } 2422 }
2423 2423
2424 /** 2424 /**
2425 * atomic64_fetch_andnot() - atomic bitwise A 2425 * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
2426 * @i: s64 value 2426 * @i: s64 value
2427 * @v: pointer to atomic64_t 2427 * @v: pointer to atomic64_t
2428 * 2428 *
2429 * Atomically updates @v to (@v & ~@i) with f 2429 * Atomically updates @v to (@v & ~@i) with full ordering.
2430 * 2430 *
2431 * Unsafe to use in noinstr code; use raw_ato 2431 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
2432 * 2432 *
2433 * Return: The original value of @v. 2433 * Return: The original value of @v.
2434 */ 2434 */
2435 static __always_inline s64 2435 static __always_inline s64
2436 atomic64_fetch_andnot(s64 i, atomic64_t *v) 2436 atomic64_fetch_andnot(s64 i, atomic64_t *v)
2437 { 2437 {
2438 kcsan_mb(); 2438 kcsan_mb();
2439 instrument_atomic_read_write(v, sizeo 2439 instrument_atomic_read_write(v, sizeof(*v));
2440 return raw_atomic64_fetch_andnot(i, v 2440 return raw_atomic64_fetch_andnot(i, v);
2441 } 2441 }
2442 2442
2443 /** 2443 /**
2444 * atomic64_fetch_andnot_acquire() - atomic b 2444 * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
2445 * @i: s64 value 2445 * @i: s64 value
2446 * @v: pointer to atomic64_t 2446 * @v: pointer to atomic64_t
2447 * 2447 *
2448 * Atomically updates @v to (@v & ~@i) with a 2448 * Atomically updates @v to (@v & ~@i) with acquire ordering.
2449 * 2449 *
2450 * Unsafe to use in noinstr code; use raw_ato 2450 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
2451 * 2451 *
2452 * Return: The original value of @v. 2452 * Return: The original value of @v.
2453 */ 2453 */
2454 static __always_inline s64 2454 static __always_inline s64
2455 atomic64_fetch_andnot_acquire(s64 i, atomic64 2455 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2456 { 2456 {
2457 instrument_atomic_read_write(v, sizeo 2457 instrument_atomic_read_write(v, sizeof(*v));
2458 return raw_atomic64_fetch_andnot_acqu 2458 return raw_atomic64_fetch_andnot_acquire(i, v);
2459 } 2459 }
2460 2460
2461 /** 2461 /**
2462 * atomic64_fetch_andnot_release() - atomic b 2462 * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
2463 * @i: s64 value 2463 * @i: s64 value
2464 * @v: pointer to atomic64_t 2464 * @v: pointer to atomic64_t
2465 * 2465 *
2466 * Atomically updates @v to (@v & ~@i) with r 2466 * Atomically updates @v to (@v & ~@i) with release ordering.
2467 * 2467 *
2468 * Unsafe to use in noinstr code; use raw_ato 2468 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
2469 * 2469 *
2470 * Return: The original value of @v. 2470 * Return: The original value of @v.
2471 */ 2471 */
2472 static __always_inline s64 2472 static __always_inline s64
2473 atomic64_fetch_andnot_release(s64 i, atomic64 2473 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2474 { 2474 {
2475 kcsan_release(); 2475 kcsan_release();
2476 instrument_atomic_read_write(v, sizeo 2476 instrument_atomic_read_write(v, sizeof(*v));
2477 return raw_atomic64_fetch_andnot_rele 2477 return raw_atomic64_fetch_andnot_release(i, v);
2478 } 2478 }
2479 2479
2480 /** 2480 /**
2481 * atomic64_fetch_andnot_relaxed() - atomic b 2481 * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
2482 * @i: s64 value 2482 * @i: s64 value
2483 * @v: pointer to atomic64_t 2483 * @v: pointer to atomic64_t
2484 * 2484 *
2485 * Atomically updates @v to (@v & ~@i) with r 2485 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2486 * 2486 *
2487 * Unsafe to use in noinstr code; use raw_ato 2487 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
2488 * 2488 *
2489 * Return: The original value of @v. 2489 * Return: The original value of @v.
2490 */ 2490 */
2491 static __always_inline s64 2491 static __always_inline s64
2492 atomic64_fetch_andnot_relaxed(s64 i, atomic64 2492 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2493 { 2493 {
2494 instrument_atomic_read_write(v, sizeo 2494 instrument_atomic_read_write(v, sizeof(*v));
2495 return raw_atomic64_fetch_andnot_rela 2495 return raw_atomic64_fetch_andnot_relaxed(i, v);
2496 } 2496 }
2497 2497
2498 /** 2498 /**
2499 * atomic64_or() - atomic bitwise OR with rel 2499 * atomic64_or() - atomic bitwise OR with relaxed ordering
2500 * @i: s64 value 2500 * @i: s64 value
2501 * @v: pointer to atomic64_t 2501 * @v: pointer to atomic64_t
2502 * 2502 *
2503 * Atomically updates @v to (@v | @i) with re 2503 * Atomically updates @v to (@v | @i) with relaxed ordering.
2504 * 2504 *
2505 * Unsafe to use in noinstr code; use raw_ato 2505 * Unsafe to use in noinstr code; use raw_atomic64_or() there.
2506 * 2506 *
2507 * Return: Nothing. 2507 * Return: Nothing.
2508 */ 2508 */
2509 static __always_inline void 2509 static __always_inline void
2510 atomic64_or(s64 i, atomic64_t *v) 2510 atomic64_or(s64 i, atomic64_t *v)
2511 { 2511 {
2512 instrument_atomic_read_write(v, sizeo 2512 instrument_atomic_read_write(v, sizeof(*v));
2513 raw_atomic64_or(i, v); 2513 raw_atomic64_or(i, v);
2514 } 2514 }
2515 2515
2516 /** 2516 /**
2517 * atomic64_fetch_or() - atomic bitwise OR wi 2517 * atomic64_fetch_or() - atomic bitwise OR with full ordering
2518 * @i: s64 value 2518 * @i: s64 value
2519 * @v: pointer to atomic64_t 2519 * @v: pointer to atomic64_t
2520 * 2520 *
2521 * Atomically updates @v to (@v | @i) with fu 2521 * Atomically updates @v to (@v | @i) with full ordering.
2522 * 2522 *
2523 * Unsafe to use in noinstr code; use raw_ato 2523 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
2524 * 2524 *
2525 * Return: The original value of @v. 2525 * Return: The original value of @v.
2526 */ 2526 */
2527 static __always_inline s64 2527 static __always_inline s64
2528 atomic64_fetch_or(s64 i, atomic64_t *v) 2528 atomic64_fetch_or(s64 i, atomic64_t *v)
2529 { 2529 {
2530 kcsan_mb(); 2530 kcsan_mb();
2531 instrument_atomic_read_write(v, sizeo 2531 instrument_atomic_read_write(v, sizeof(*v));
2532 return raw_atomic64_fetch_or(i, v); 2532 return raw_atomic64_fetch_or(i, v);
2533 } 2533 }
2534 2534
2535 /** 2535 /**
2536 * atomic64_fetch_or_acquire() - atomic bitwi 2536 * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
2537 * @i: s64 value 2537 * @i: s64 value
2538 * @v: pointer to atomic64_t 2538 * @v: pointer to atomic64_t
2539 * 2539 *
2540 * Atomically updates @v to (@v | @i) with ac 2540 * Atomically updates @v to (@v | @i) with acquire ordering.
2541 * 2541 *
2542 * Unsafe to use in noinstr code; use raw_ato 2542 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
2543 * 2543 *
2544 * Return: The original value of @v. 2544 * Return: The original value of @v.
2545 */ 2545 */
2546 static __always_inline s64 2546 static __always_inline s64
2547 atomic64_fetch_or_acquire(s64 i, atomic64_t * 2547 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2548 { 2548 {
2549 instrument_atomic_read_write(v, sizeo 2549 instrument_atomic_read_write(v, sizeof(*v));
2550 return raw_atomic64_fetch_or_acquire( 2550 return raw_atomic64_fetch_or_acquire(i, v);
2551 } 2551 }
2552 2552
2553 /** 2553 /**
2554 * atomic64_fetch_or_release() - atomic bitwi 2554 * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
2555 * @i: s64 value 2555 * @i: s64 value
2556 * @v: pointer to atomic64_t 2556 * @v: pointer to atomic64_t
2557 * 2557 *
2558 * Atomically updates @v to (@v | @i) with re 2558 * Atomically updates @v to (@v | @i) with release ordering.
2559 * 2559 *
2560 * Unsafe to use in noinstr code; use raw_ato 2560 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
2561 * 2561 *
2562 * Return: The original value of @v. 2562 * Return: The original value of @v.
2563 */ 2563 */
2564 static __always_inline s64 2564 static __always_inline s64
2565 atomic64_fetch_or_release(s64 i, atomic64_t * 2565 atomic64_fetch_or_release(s64 i, atomic64_t *v)
2566 { 2566 {
2567 kcsan_release(); 2567 kcsan_release();
2568 instrument_atomic_read_write(v, sizeo 2568 instrument_atomic_read_write(v, sizeof(*v));
2569 return raw_atomic64_fetch_or_release( 2569 return raw_atomic64_fetch_or_release(i, v);
2570 } 2570 }
2571 2571
2572 /** 2572 /**
2573 * atomic64_fetch_or_relaxed() - atomic bitwi 2573 * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
2574 * @i: s64 value 2574 * @i: s64 value
2575 * @v: pointer to atomic64_t 2575 * @v: pointer to atomic64_t
2576 * 2576 *
2577 * Atomically updates @v to (@v | @i) with re 2577 * Atomically updates @v to (@v | @i) with relaxed ordering.
2578 * 2578 *
2579 * Unsafe to use in noinstr code; use raw_ato 2579 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
2580 * 2580 *
2581 * Return: The original value of @v. 2581 * Return: The original value of @v.
2582 */ 2582 */
2583 static __always_inline s64 2583 static __always_inline s64
2584 atomic64_fetch_or_relaxed(s64 i, atomic64_t * 2584 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
2585 { 2585 {
2586 instrument_atomic_read_write(v, sizeo 2586 instrument_atomic_read_write(v, sizeof(*v));
2587 return raw_atomic64_fetch_or_relaxed( 2587 return raw_atomic64_fetch_or_relaxed(i, v);
2588 } 2588 }
2589 2589
2590 /** 2590 /**
2591 * atomic64_xor() - atomic bitwise XOR with r 2591 * atomic64_xor() - atomic bitwise XOR with relaxed ordering
2592 * @i: s64 value 2592 * @i: s64 value
2593 * @v: pointer to atomic64_t 2593 * @v: pointer to atomic64_t
2594 * 2594 *
2595 * Atomically updates @v to (@v ^ @i) with re 2595 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2596 * 2596 *
2597 * Unsafe to use in noinstr code; use raw_ato 2597 * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
2598 * 2598 *
2599 * Return: Nothing. 2599 * Return: Nothing.
2600 */ 2600 */
2601 static __always_inline void 2601 static __always_inline void
2602 atomic64_xor(s64 i, atomic64_t *v) 2602 atomic64_xor(s64 i, atomic64_t *v)
2603 { 2603 {
2604 instrument_atomic_read_write(v, sizeo 2604 instrument_atomic_read_write(v, sizeof(*v));
2605 raw_atomic64_xor(i, v); 2605 raw_atomic64_xor(i, v);
2606 } 2606 }
2607 2607
2608 /** 2608 /**
2609 * atomic64_fetch_xor() - atomic bitwise XOR 2609 * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
2610 * @i: s64 value 2610 * @i: s64 value
2611 * @v: pointer to atomic64_t 2611 * @v: pointer to atomic64_t
2612 * 2612 *
2613 * Atomically updates @v to (@v ^ @i) with fu 2613 * Atomically updates @v to (@v ^ @i) with full ordering.
2614 * 2614 *
2615 * Unsafe to use in noinstr code; use raw_ato 2615 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
2616 * 2616 *
2617 * Return: The original value of @v. 2617 * Return: The original value of @v.
2618 */ 2618 */
2619 static __always_inline s64 2619 static __always_inline s64
2620 atomic64_fetch_xor(s64 i, atomic64_t *v) 2620 atomic64_fetch_xor(s64 i, atomic64_t *v)
2621 { 2621 {
2622 kcsan_mb(); 2622 kcsan_mb();
2623 instrument_atomic_read_write(v, sizeo 2623 instrument_atomic_read_write(v, sizeof(*v));
2624 return raw_atomic64_fetch_xor(i, v); 2624 return raw_atomic64_fetch_xor(i, v);
2625 } 2625 }
2626 2626
2627 /** 2627 /**
2628 * atomic64_fetch_xor_acquire() - atomic bitw 2628 * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
2629 * @i: s64 value 2629 * @i: s64 value
2630 * @v: pointer to atomic64_t 2630 * @v: pointer to atomic64_t
2631 * 2631 *
2632 * Atomically updates @v to (@v ^ @i) with ac 2632 * Atomically updates @v to (@v ^ @i) with acquire ordering.
2633 * 2633 *
2634 * Unsafe to use in noinstr code; use raw_ato 2634 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
2635 * 2635 *
2636 * Return: The original value of @v. 2636 * Return: The original value of @v.
2637 */ 2637 */
2638 static __always_inline s64 2638 static __always_inline s64
2639 atomic64_fetch_xor_acquire(s64 i, atomic64_t 2639 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2640 { 2640 {
2641 instrument_atomic_read_write(v, sizeo 2641 instrument_atomic_read_write(v, sizeof(*v));
2642 return raw_atomic64_fetch_xor_acquire 2642 return raw_atomic64_fetch_xor_acquire(i, v);
2643 } 2643 }
2644 2644
2645 /** 2645 /**
2646 * atomic64_fetch_xor_release() - atomic bitw 2646 * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
2647 * @i: s64 value 2647 * @i: s64 value
2648 * @v: pointer to atomic64_t 2648 * @v: pointer to atomic64_t
2649 * 2649 *
2650 * Atomically updates @v to (@v ^ @i) with re 2650 * Atomically updates @v to (@v ^ @i) with release ordering.
2651 * 2651 *
2652 * Unsafe to use in noinstr code; use raw_ato 2652 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
2653 * 2653 *
2654 * Return: The original value of @v. 2654 * Return: The original value of @v.
2655 */ 2655 */
2656 static __always_inline s64 2656 static __always_inline s64
2657 atomic64_fetch_xor_release(s64 i, atomic64_t 2657 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2658 { 2658 {
2659 kcsan_release(); 2659 kcsan_release();
2660 instrument_atomic_read_write(v, sizeo 2660 instrument_atomic_read_write(v, sizeof(*v));
2661 return raw_atomic64_fetch_xor_release 2661 return raw_atomic64_fetch_xor_release(i, v);
2662 } 2662 }
2663 2663
2664 /** 2664 /**
2665 * atomic64_fetch_xor_relaxed() - atomic bitw 2665 * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
2666 * @i: s64 value 2666 * @i: s64 value
2667 * @v: pointer to atomic64_t 2667 * @v: pointer to atomic64_t
2668 * 2668 *
2669 * Atomically updates @v to (@v ^ @i) with re 2669 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2670 * 2670 *
2671 * Unsafe to use in noinstr code; use raw_ato 2671 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
2672 * 2672 *
2673 * Return: The original value of @v. 2673 * Return: The original value of @v.
2674 */ 2674 */
2675 static __always_inline s64 2675 static __always_inline s64
2676 atomic64_fetch_xor_relaxed(s64 i, atomic64_t 2676 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
2677 { 2677 {
2678 instrument_atomic_read_write(v, sizeo 2678 instrument_atomic_read_write(v, sizeof(*v));
2679 return raw_atomic64_fetch_xor_relaxed 2679 return raw_atomic64_fetch_xor_relaxed(i, v);
2680 } 2680 }
2681 2681
2682 /** 2682 /**
2683 * atomic64_xchg() - atomic exchange with ful 2683 * atomic64_xchg() - atomic exchange with full ordering
2684 * @v: pointer to atomic64_t 2684 * @v: pointer to atomic64_t
2685 * @new: s64 value to assign 2685 * @new: s64 value to assign
2686 * 2686 *
2687 * Atomically updates @v to @new with full or 2687 * Atomically updates @v to @new with full ordering.
2688 * 2688 *
2689 * Unsafe to use in noinstr code; use raw_ato 2689 * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
2690 * 2690 *
2691 * Return: The original value of @v. 2691 * Return: The original value of @v.
2692 */ 2692 */
2693 static __always_inline s64 2693 static __always_inline s64
2694 atomic64_xchg(atomic64_t *v, s64 new) 2694 atomic64_xchg(atomic64_t *v, s64 new)
2695 { 2695 {
2696 kcsan_mb(); 2696 kcsan_mb();
2697 instrument_atomic_read_write(v, sizeo 2697 instrument_atomic_read_write(v, sizeof(*v));
2698 return raw_atomic64_xchg(v, new); 2698 return raw_atomic64_xchg(v, new);
2699 } 2699 }
2700 2700
2701 /** 2701 /**
2702 * atomic64_xchg_acquire() - atomic exchange 2702 * atomic64_xchg_acquire() - atomic exchange with acquire ordering
2703 * @v: pointer to atomic64_t 2703 * @v: pointer to atomic64_t
2704 * @new: s64 value to assign 2704 * @new: s64 value to assign
2705 * 2705 *
2706 * Atomically updates @v to @new with acquire 2706 * Atomically updates @v to @new with acquire ordering.
2707 * 2707 *
2708 * Unsafe to use in noinstr code; use raw_ato 2708 * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
2709 * 2709 *
2710 * Return: The original value of @v. 2710 * Return: The original value of @v.
2711 */ 2711 */
2712 static __always_inline s64 2712 static __always_inline s64
2713 atomic64_xchg_acquire(atomic64_t *v, s64 new) 2713 atomic64_xchg_acquire(atomic64_t *v, s64 new)
2714 { 2714 {
2715 instrument_atomic_read_write(v, sizeo 2715 instrument_atomic_read_write(v, sizeof(*v));
2716 return raw_atomic64_xchg_acquire(v, n 2716 return raw_atomic64_xchg_acquire(v, new);
2717 } 2717 }
2718 2718
2719 /** 2719 /**
2720 * atomic64_xchg_release() - atomic exchange 2720 * atomic64_xchg_release() - atomic exchange with release ordering
2721 * @v: pointer to atomic64_t 2721 * @v: pointer to atomic64_t
2722 * @new: s64 value to assign 2722 * @new: s64 value to assign
2723 * 2723 *
2724 * Atomically updates @v to @new with release 2724 * Atomically updates @v to @new with release ordering.
2725 * 2725 *
2726 * Unsafe to use in noinstr code; use raw_ato 2726 * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
2727 * 2727 *
2728 * Return: The original value of @v. 2728 * Return: The original value of @v.
2729 */ 2729 */
2730 static __always_inline s64 2730 static __always_inline s64
2731 atomic64_xchg_release(atomic64_t *v, s64 new) 2731 atomic64_xchg_release(atomic64_t *v, s64 new)
2732 { 2732 {
2733 kcsan_release(); 2733 kcsan_release();
2734 instrument_atomic_read_write(v, sizeo 2734 instrument_atomic_read_write(v, sizeof(*v));
2735 return raw_atomic64_xchg_release(v, n 2735 return raw_atomic64_xchg_release(v, new);
2736 } 2736 }
2737 2737
2738 /** 2738 /**
2739 * atomic64_xchg_relaxed() - atomic exchange 2739 * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
2740 * @v: pointer to atomic64_t 2740 * @v: pointer to atomic64_t
2741 * @new: s64 value to assign 2741 * @new: s64 value to assign
2742 * 2742 *
2743 * Atomically updates @v to @new with relaxed 2743 * Atomically updates @v to @new with relaxed ordering.
2744 * 2744 *
2745 * Unsafe to use in noinstr code; use raw_ato 2745 * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
2746 * 2746 *
2747 * Return: The original value of @v. 2747 * Return: The original value of @v.
2748 */ 2748 */
2749 static __always_inline s64 2749 static __always_inline s64
2750 atomic64_xchg_relaxed(atomic64_t *v, s64 new) 2750 atomic64_xchg_relaxed(atomic64_t *v, s64 new)
2751 { 2751 {
2752 instrument_atomic_read_write(v, sizeo 2752 instrument_atomic_read_write(v, sizeof(*v));
2753 return raw_atomic64_xchg_relaxed(v, n 2753 return raw_atomic64_xchg_relaxed(v, new);
2754 } 2754 }
2755 2755
2756 /** 2756 /**
2757 * atomic64_cmpxchg() - atomic compare and ex 2757 * atomic64_cmpxchg() - atomic compare and exchange with full ordering
2758 * @v: pointer to atomic64_t 2758 * @v: pointer to atomic64_t
2759 * @old: s64 value to compare with 2759 * @old: s64 value to compare with
2760 * @new: s64 value to assign 2760 * @new: s64 value to assign
2761 * 2761 *
2762 * If (@v == @old), atomically updates @v to 2762 * If (@v == @old), atomically updates @v to @new with full ordering.
2763 * Otherwise, @v is not modified and relaxed 2763 * Otherwise, @v is not modified and relaxed ordering is provided.
2764 * 2764 *
2765 * Unsafe to use in noinstr code; use raw_ato 2765 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
2766 * 2766 *
2767 * Return: The original value of @v. 2767 * Return: The original value of @v.
2768 */ 2768 */
2769 static __always_inline s64 2769 static __always_inline s64
2770 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 2770 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2771 { 2771 {
2772 kcsan_mb(); 2772 kcsan_mb();
2773 instrument_atomic_read_write(v, sizeo 2773 instrument_atomic_read_write(v, sizeof(*v));
2774 return raw_atomic64_cmpxchg(v, old, n 2774 return raw_atomic64_cmpxchg(v, old, new);
2775 } 2775 }
2776 2776
2777 /** 2777 /**
2778 * atomic64_cmpxchg_acquire() - atomic compar 2778 * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2779 * @v: pointer to atomic64_t 2779 * @v: pointer to atomic64_t
2780 * @old: s64 value to compare with 2780 * @old: s64 value to compare with
2781 * @new: s64 value to assign 2781 * @new: s64 value to assign
2782 * 2782 *
2783 * If (@v == @old), atomically updates @v to 2783 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2784 * Otherwise, @v is not modified and relaxed 2784 * Otherwise, @v is not modified and relaxed ordering is provided.
2785 * 2785 *
2786 * Unsafe to use in noinstr code; use raw_ato 2786 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
2787 * 2787 *
2788 * Return: The original value of @v. 2788 * Return: The original value of @v.
2789 */ 2789 */
2790 static __always_inline s64 2790 static __always_inline s64
2791 atomic64_cmpxchg_acquire(atomic64_t *v, s64 o 2791 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2792 { 2792 {
2793 instrument_atomic_read_write(v, sizeo 2793 instrument_atomic_read_write(v, sizeof(*v));
2794 return raw_atomic64_cmpxchg_acquire(v 2794 return raw_atomic64_cmpxchg_acquire(v, old, new);
2795 } 2795 }
2796 2796
2797 /** 2797 /**
2798 * atomic64_cmpxchg_release() - atomic compar 2798 * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
2799 * @v: pointer to atomic64_t 2799 * @v: pointer to atomic64_t
2800 * @old: s64 value to compare with 2800 * @old: s64 value to compare with
2801 * @new: s64 value to assign 2801 * @new: s64 value to assign
2802 * 2802 *
2803 * If (@v == @old), atomically updates @v to 2803 * If (@v == @old), atomically updates @v to @new with release ordering.
2804 * Otherwise, @v is not modified and relaxed 2804 * Otherwise, @v is not modified and relaxed ordering is provided.
2805 * 2805 *
2806 * Unsafe to use in noinstr code; use raw_ato 2806 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
2807 * 2807 *
2808 * Return: The original value of @v. 2808 * Return: The original value of @v.
2809 */ 2809 */
2810 static __always_inline s64 2810 static __always_inline s64
2811 atomic64_cmpxchg_release(atomic64_t *v, s64 o 2811 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2812 { 2812 {
2813 kcsan_release(); 2813 kcsan_release();
2814 instrument_atomic_read_write(v, sizeo 2814 instrument_atomic_read_write(v, sizeof(*v));
2815 return raw_atomic64_cmpxchg_release(v 2815 return raw_atomic64_cmpxchg_release(v, old, new);
2816 } 2816 }
2817 2817
2818 /** 2818 /**
2819 * atomic64_cmpxchg_relaxed() - atomic compar 2819 * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2820 * @v: pointer to atomic64_t 2820 * @v: pointer to atomic64_t
2821 * @old: s64 value to compare with 2821 * @old: s64 value to compare with
2822 * @new: s64 value to assign 2822 * @new: s64 value to assign
2823 * 2823 *
2824 * If (@v == @old), atomically updates @v to 2824 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2825 * Otherwise, @v is not modified and relaxed 2825 * Otherwise, @v is not modified and relaxed ordering is provided.
2826 * 2826 *
2827 * Unsafe to use in noinstr code; use raw_ato 2827 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
2828 * 2828 *
2829 * Return: The original value of @v. 2829 * Return: The original value of @v.
2830 */ 2830 */
2831 static __always_inline s64 2831 static __always_inline s64
2832 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 o 2832 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
2833 { 2833 {
2834 instrument_atomic_read_write(v, sizeo 2834 instrument_atomic_read_write(v, sizeof(*v));
2835 return raw_atomic64_cmpxchg_relaxed(v 2835 return raw_atomic64_cmpxchg_relaxed(v, old, new);
2836 } 2836 }
2837 2837
2838 /** 2838 /**
2839 * atomic64_try_cmpxchg() - atomic compare an 2839 * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
2840 * @v: pointer to atomic64_t 2840 * @v: pointer to atomic64_t
2841 * @old: pointer to s64 value to compare with 2841 * @old: pointer to s64 value to compare with
2842 * @new: s64 value to assign 2842 * @new: s64 value to assign
2843 * 2843 *
2844 * If (@v == @old), atomically updates @v to 2844 * If (@v == @old), atomically updates @v to @new with full ordering.
2845 * Otherwise, @v is not modified, @old is upd 2845 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2846 * and relaxed ordering is provided. 2846 * and relaxed ordering is provided.
2847 * 2847 *
2848 * Unsafe to use in noinstr code; use raw_ato 2848 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
2849 * 2849 *
2850 * Return: @true if the exchange occured, @fa 2850 * Return: @true if the exchange occured, @false otherwise.
2851 */ 2851 */
2852 static __always_inline bool 2852 static __always_inline bool
2853 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, 2853 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2854 { 2854 {
2855 kcsan_mb(); 2855 kcsan_mb();
2856 instrument_atomic_read_write(v, sizeo 2856 instrument_atomic_read_write(v, sizeof(*v));
2857 instrument_atomic_read_write(old, siz 2857 instrument_atomic_read_write(old, sizeof(*old));
2858 return raw_atomic64_try_cmpxchg(v, ol 2858 return raw_atomic64_try_cmpxchg(v, old, new);
2859 } 2859 }
2860 2860
2861 /** 2861 /**
2862 * atomic64_try_cmpxchg_acquire() - atomic co 2862 * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2863 * @v: pointer to atomic64_t 2863 * @v: pointer to atomic64_t
2864 * @old: pointer to s64 value to compare with 2864 * @old: pointer to s64 value to compare with
2865 * @new: s64 value to assign 2865 * @new: s64 value to assign
2866 * 2866 *
2867 * If (@v == @old), atomically updates @v to 2867 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2868 * Otherwise, @v is not modified, @old is upd 2868 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2869 * and relaxed ordering is provided. 2869 * and relaxed ordering is provided.
2870 * 2870 *
2871 * Unsafe to use in noinstr code; use raw_ato 2871 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
2872 * 2872 *
2873 * Return: @true if the exchange occured, @fa 2873 * Return: @true if the exchange occured, @false otherwise.
2874 */ 2874 */
2875 static __always_inline bool 2875 static __always_inline bool
2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s 2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2877 { 2877 {
2878 instrument_atomic_read_write(v, sizeo 2878 instrument_atomic_read_write(v, sizeof(*v));
2879 instrument_atomic_read_write(old, siz 2879 instrument_atomic_read_write(old, sizeof(*old));
2880 return raw_atomic64_try_cmpxchg_acqui 2880 return raw_atomic64_try_cmpxchg_acquire(v, old, new);
2881 } 2881 }
2882 2882
2883 /** 2883 /**
2884 * atomic64_try_cmpxchg_release() - atomic co 2884 * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
2885 * @v: pointer to atomic64_t 2885 * @v: pointer to atomic64_t
2886 * @old: pointer to s64 value to compare with 2886 * @old: pointer to s64 value to compare with
2887 * @new: s64 value to assign 2887 * @new: s64 value to assign
2888 * 2888 *
2889 * If (@v == @old), atomically updates @v to 2889 * If (@v == @old), atomically updates @v to @new with release ordering.
2890 * Otherwise, @v is not modified, @old is upd 2890 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2891 * and relaxed ordering is provided. 2891 * and relaxed ordering is provided.
2892 * 2892 *
2893 * Unsafe to use in noinstr code; use raw_ato 2893 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
2894 * 2894 *
2895 * Return: @true if the exchange occured, @fa 2895 * Return: @true if the exchange occured, @false otherwise.
2896 */ 2896 */
2897 static __always_inline bool 2897 static __always_inline bool
2898 atomic64_try_cmpxchg_release(atomic64_t *v, s 2898 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2899 { 2899 {
2900 kcsan_release(); 2900 kcsan_release();
2901 instrument_atomic_read_write(v, sizeo 2901 instrument_atomic_read_write(v, sizeof(*v));
2902 instrument_atomic_read_write(old, siz 2902 instrument_atomic_read_write(old, sizeof(*old));
2903 return raw_atomic64_try_cmpxchg_relea 2903 return raw_atomic64_try_cmpxchg_release(v, old, new);
2904 } 2904 }
2905 2905
2906 /** 2906 /**
2907 * atomic64_try_cmpxchg_relaxed() - atomic co 2907 * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2908 * @v: pointer to atomic64_t 2908 * @v: pointer to atomic64_t
2909 * @old: pointer to s64 value to compare with 2909 * @old: pointer to s64 value to compare with
2910 * @new: s64 value to assign 2910 * @new: s64 value to assign
2911 * 2911 *
2912 * If (@v == @old), atomically updates @v to 2912 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2913 * Otherwise, @v is not modified, @old is upd 2913 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2914 * and relaxed ordering is provided. 2914 * and relaxed ordering is provided.
2915 * 2915 *
2916 * Unsafe to use in noinstr code; use raw_ato 2916 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
2917 * 2917 *
2918 * Return: @true if the exchange occured, @fa 2918 * Return: @true if the exchange occured, @false otherwise.
2919 */ 2919 */
2920 static __always_inline bool 2920 static __always_inline bool
2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s 2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2922 { 2922 {
2923 instrument_atomic_read_write(v, sizeo 2923 instrument_atomic_read_write(v, sizeof(*v));
2924 instrument_atomic_read_write(old, siz 2924 instrument_atomic_read_write(old, sizeof(*old));
2925 return raw_atomic64_try_cmpxchg_relax 2925 return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
2926 } 2926 }
2927 2927
2928 /** 2928 /**
2929 * atomic64_sub_and_test() - atomic subtract 2929 * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
2930 * @i: s64 value to subtract 2930 * @i: s64 value to subtract
2931 * @v: pointer to atomic64_t 2931 * @v: pointer to atomic64_t
2932 * 2932 *
2933 * Atomically updates @v to (@v - @i) with fu 2933 * Atomically updates @v to (@v - @i) with full ordering.
2934 * 2934 *
2935 * Unsafe to use in noinstr code; use raw_ato 2935 * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
2936 * 2936 *
2937 * Return: @true if the resulting value of @v 2937 * Return: @true if the resulting value of @v is zero, @false otherwise.
2938 */ 2938 */
2939 static __always_inline bool 2939 static __always_inline bool
2940 atomic64_sub_and_test(s64 i, atomic64_t *v) 2940 atomic64_sub_and_test(s64 i, atomic64_t *v)
2941 { 2941 {
2942 kcsan_mb(); 2942 kcsan_mb();
2943 instrument_atomic_read_write(v, sizeo 2943 instrument_atomic_read_write(v, sizeof(*v));
2944 return raw_atomic64_sub_and_test(i, v 2944 return raw_atomic64_sub_and_test(i, v);
2945 } 2945 }
2946 2946
2947 /** 2947 /**
2948 * atomic64_dec_and_test() - atomic decrement 2948 * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
2949 * @v: pointer to atomic64_t 2949 * @v: pointer to atomic64_t
2950 * 2950 *
2951 * Atomically updates @v to (@v - 1) with ful 2951 * Atomically updates @v to (@v - 1) with full ordering.
2952 * 2952 *
2953 * Unsafe to use in noinstr code; use raw_ato 2953 * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
2954 * 2954 *
2955 * Return: @true if the resulting value of @v 2955 * Return: @true if the resulting value of @v is zero, @false otherwise.
2956 */ 2956 */
2957 static __always_inline bool 2957 static __always_inline bool
2958 atomic64_dec_and_test(atomic64_t *v) 2958 atomic64_dec_and_test(atomic64_t *v)
2959 { 2959 {
2960 kcsan_mb(); 2960 kcsan_mb();
2961 instrument_atomic_read_write(v, sizeo 2961 instrument_atomic_read_write(v, sizeof(*v));
2962 return raw_atomic64_dec_and_test(v); 2962 return raw_atomic64_dec_and_test(v);
2963 } 2963 }
2964 2964
2965 /** 2965 /**
2966 * atomic64_inc_and_test() - atomic increment 2966 * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
2967 * @v: pointer to atomic64_t 2967 * @v: pointer to atomic64_t
2968 * 2968 *
2969 * Atomically updates @v to (@v + 1) with ful 2969 * Atomically updates @v to (@v + 1) with full ordering.
2970 * 2970 *
2971 * Unsafe to use in noinstr code; use raw_ato 2971 * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
2972 * 2972 *
2973 * Return: @true if the resulting value of @v 2973 * Return: @true if the resulting value of @v is zero, @false otherwise.
2974 */ 2974 */
2975 static __always_inline bool 2975 static __always_inline bool
2976 atomic64_inc_and_test(atomic64_t *v) 2976 atomic64_inc_and_test(atomic64_t *v)
2977 { 2977 {
2978 kcsan_mb(); 2978 kcsan_mb();
2979 instrument_atomic_read_write(v, sizeo 2979 instrument_atomic_read_write(v, sizeof(*v));
2980 return raw_atomic64_inc_and_test(v); 2980 return raw_atomic64_inc_and_test(v);
2981 } 2981 }
2982 2982
2983 /** 2983 /**
2984 * atomic64_add_negative() - atomic add and t 2984 * atomic64_add_negative() - atomic add and test if negative with full ordering
2985 * @i: s64 value to add 2985 * @i: s64 value to add
2986 * @v: pointer to atomic64_t 2986 * @v: pointer to atomic64_t
2987 * 2987 *
2988 * Atomically updates @v to (@v + @i) with fu 2988 * Atomically updates @v to (@v + @i) with full ordering.
2989 * 2989 *
2990 * Unsafe to use in noinstr code; use raw_ato 2990 * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
2991 * 2991 *
2992 * Return: @true if the resulting value of @v 2992 * Return: @true if the resulting value of @v is negative, @false otherwise.
2993 */ 2993 */
2994 static __always_inline bool 2994 static __always_inline bool
2995 atomic64_add_negative(s64 i, atomic64_t *v) 2995 atomic64_add_negative(s64 i, atomic64_t *v)
2996 { 2996 {
2997 kcsan_mb(); 2997 kcsan_mb();
2998 instrument_atomic_read_write(v, sizeo 2998 instrument_atomic_read_write(v, sizeof(*v));
2999 return raw_atomic64_add_negative(i, v 2999 return raw_atomic64_add_negative(i, v);
3000 } 3000 }
3001 3001
3002 /** 3002 /**
3003 * atomic64_add_negative_acquire() - atomic a 3003 * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
3004 * @i: s64 value to add 3004 * @i: s64 value to add
3005 * @v: pointer to atomic64_t 3005 * @v: pointer to atomic64_t
3006 * 3006 *
3007 * Atomically updates @v to (@v + @i) with ac 3007 * Atomically updates @v to (@v + @i) with acquire ordering.
3008 * 3008 *
3009 * Unsafe to use in noinstr code; use raw_ato 3009 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
3010 * 3010 *
3011 * Return: @true if the resulting value of @v 3011 * Return: @true if the resulting value of @v is negative, @false otherwise.
3012 */ 3012 */
3013 static __always_inline bool 3013 static __always_inline bool
3014 atomic64_add_negative_acquire(s64 i, atomic64 3014 atomic64_add_negative_acquire(s64 i, atomic64_t *v)
3015 { 3015 {
3016 instrument_atomic_read_write(v, sizeo 3016 instrument_atomic_read_write(v, sizeof(*v));
3017 return raw_atomic64_add_negative_acqu 3017 return raw_atomic64_add_negative_acquire(i, v);
3018 } 3018 }
3019 3019
3020 /** 3020 /**
3021 * atomic64_add_negative_release() - atomic a 3021 * atomic64_add_negative_release() - atomic add and test if negative with release ordering
3022 * @i: s64 value to add 3022 * @i: s64 value to add
3023 * @v: pointer to atomic64_t 3023 * @v: pointer to atomic64_t
3024 * 3024 *
3025 * Atomically updates @v to (@v + @i) with re 3025 * Atomically updates @v to (@v + @i) with release ordering.
3026 * 3026 *
3027 * Unsafe to use in noinstr code; use raw_ato 3027 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
3028 * 3028 *
3029 * Return: @true if the resulting value of @v 3029 * Return: @true if the resulting value of @v is negative, @false otherwise.
3030 */ 3030 */
3031 static __always_inline bool 3031 static __always_inline bool
3032 atomic64_add_negative_release(s64 i, atomic64 3032 atomic64_add_negative_release(s64 i, atomic64_t *v)
3033 { 3033 {
3034 kcsan_release(); 3034 kcsan_release();
3035 instrument_atomic_read_write(v, sizeo 3035 instrument_atomic_read_write(v, sizeof(*v));
3036 return raw_atomic64_add_negative_rele 3036 return raw_atomic64_add_negative_release(i, v);
3037 } 3037 }
3038 3038
3039 /** 3039 /**
3040 * atomic64_add_negative_relaxed() - atomic a 3040 * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
3041 * @i: s64 value to add 3041 * @i: s64 value to add
3042 * @v: pointer to atomic64_t 3042 * @v: pointer to atomic64_t
3043 * 3043 *
3044 * Atomically updates @v to (@v + @i) with re 3044 * Atomically updates @v to (@v + @i) with relaxed ordering.
3045 * 3045 *
3046 * Unsafe to use in noinstr code; use raw_ato 3046 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
3047 * 3047 *
3048 * Return: @true if the resulting value of @v 3048 * Return: @true if the resulting value of @v is negative, @false otherwise.
3049 */ 3049 */
3050 static __always_inline bool 3050 static __always_inline bool
3051 atomic64_add_negative_relaxed(s64 i, atomic64 3051 atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
3052 { 3052 {
3053 instrument_atomic_read_write(v, sizeo 3053 instrument_atomic_read_write(v, sizeof(*v));
3054 return raw_atomic64_add_negative_rela 3054 return raw_atomic64_add_negative_relaxed(i, v);
3055 } 3055 }
3056 3056
3057 /** 3057 /**
3058 * atomic64_fetch_add_unless() - atomic add u 3058 * atomic64_fetch_add_unless() - atomic add unless value with full ordering
3059 * @v: pointer to atomic64_t 3059 * @v: pointer to atomic64_t
3060 * @a: s64 value to add 3060 * @a: s64 value to add
3061 * @u: s64 value to compare with 3061 * @u: s64 value to compare with
3062 * 3062 *
3063 * If (@v != @u), atomically updates @v to (@ 3063 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3064 * Otherwise, @v is not modified and relaxed 3064 * Otherwise, @v is not modified and relaxed ordering is provided.
3065 * 3065 *
3066 * Unsafe to use in noinstr code; use raw_ato 3066 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
3067 * 3067 *
3068 * Return: The original value of @v. 3068 * Return: The original value of @v.
3069 */ 3069 */
3070 static __always_inline s64 3070 static __always_inline s64
3071 atomic64_fetch_add_unless(atomic64_t *v, s64 3071 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
3072 { 3072 {
3073 kcsan_mb(); 3073 kcsan_mb();
3074 instrument_atomic_read_write(v, sizeo 3074 instrument_atomic_read_write(v, sizeof(*v));
3075 return raw_atomic64_fetch_add_unless( 3075 return raw_atomic64_fetch_add_unless(v, a, u);
3076 } 3076 }
3077 3077
3078 /** 3078 /**
3079 * atomic64_add_unless() - atomic add unless 3079 * atomic64_add_unless() - atomic add unless value with full ordering
3080 * @v: pointer to atomic64_t 3080 * @v: pointer to atomic64_t
3081 * @a: s64 value to add 3081 * @a: s64 value to add
3082 * @u: s64 value to compare with 3082 * @u: s64 value to compare with
3083 * 3083 *
3084 * If (@v != @u), atomically updates @v to (@ 3084 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3085 * Otherwise, @v is not modified and relaxed 3085 * Otherwise, @v is not modified and relaxed ordering is provided.
3086 * 3086 *
3087 * Unsafe to use in noinstr code; use raw_ato 3087 * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
3088 * 3088 *
3089 * Return: @true if @v was updated, @false ot 3089 * Return: @true if @v was updated, @false otherwise.
3090 */ 3090 */
3091 static __always_inline bool 3091 static __always_inline bool
3092 atomic64_add_unless(atomic64_t *v, s64 a, s64 3092 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
3093 { 3093 {
3094 kcsan_mb(); 3094 kcsan_mb();
3095 instrument_atomic_read_write(v, sizeo 3095 instrument_atomic_read_write(v, sizeof(*v));
3096 return raw_atomic64_add_unless(v, a, 3096 return raw_atomic64_add_unless(v, a, u);
3097 } 3097 }
3098 3098
3099 /** 3099 /**
3100 * atomic64_inc_not_zero() - atomic increment 3100 * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
3101 * @v: pointer to atomic64_t 3101 * @v: pointer to atomic64_t
3102 * 3102 *
3103 * If (@v != 0), atomically updates @v to (@v 3103 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
3104 * Otherwise, @v is not modified and relaxed 3104 * Otherwise, @v is not modified and relaxed ordering is provided.
3105 * 3105 *
3106 * Unsafe to use in noinstr code; use raw_ato 3106 * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
3107 * 3107 *
3108 * Return: @true if @v was updated, @false ot 3108 * Return: @true if @v was updated, @false otherwise.
3109 */ 3109 */
3110 static __always_inline bool 3110 static __always_inline bool
3111 atomic64_inc_not_zero(atomic64_t *v) 3111 atomic64_inc_not_zero(atomic64_t *v)
3112 { 3112 {
3113 kcsan_mb(); 3113 kcsan_mb();
3114 instrument_atomic_read_write(v, sizeo 3114 instrument_atomic_read_write(v, sizeof(*v));
3115 return raw_atomic64_inc_not_zero(v); 3115 return raw_atomic64_inc_not_zero(v);
3116 } 3116 }
3117 3117
3118 /** 3118 /**
3119 * atomic64_inc_unless_negative() - atomic in 3119 * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
3120 * @v: pointer to atomic64_t 3120 * @v: pointer to atomic64_t
3121 * 3121 *
3122 * If (@v >= 0), atomically updates @v to (@v 3122 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
3123 * Otherwise, @v is not modified and relaxed 3123 * Otherwise, @v is not modified and relaxed ordering is provided.
3124 * 3124 *
3125 * Unsafe to use in noinstr code; use raw_ato 3125 * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
3126 * 3126 *
3127 * Return: @true if @v was updated, @false ot 3127 * Return: @true if @v was updated, @false otherwise.
3128 */ 3128 */
3129 static __always_inline bool 3129 static __always_inline bool
3130 atomic64_inc_unless_negative(atomic64_t *v) 3130 atomic64_inc_unless_negative(atomic64_t *v)
3131 { 3131 {
3132 kcsan_mb(); 3132 kcsan_mb();
3133 instrument_atomic_read_write(v, sizeo 3133 instrument_atomic_read_write(v, sizeof(*v));
3134 return raw_atomic64_inc_unless_negati 3134 return raw_atomic64_inc_unless_negative(v);
3135 } 3135 }
3136 3136
3137 /** 3137 /**
3138 * atomic64_dec_unless_positive() - atomic de 3138 * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
3139 * @v: pointer to atomic64_t 3139 * @v: pointer to atomic64_t
3140 * 3140 *
3141 * If (@v <= 0), atomically updates @v to (@v 3141 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
3142 * Otherwise, @v is not modified and relaxed 3142 * Otherwise, @v is not modified and relaxed ordering is provided.
3143 * 3143 *
3144 * Unsafe to use in noinstr code; use raw_ato 3144 * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
3145 * 3145 *
3146 * Return: @true if @v was updated, @false ot 3146 * Return: @true if @v was updated, @false otherwise.
3147 */ 3147 */
3148 static __always_inline bool 3148 static __always_inline bool
3149 atomic64_dec_unless_positive(atomic64_t *v) 3149 atomic64_dec_unless_positive(atomic64_t *v)
3150 { 3150 {
3151 kcsan_mb(); 3151 kcsan_mb();
3152 instrument_atomic_read_write(v, sizeo 3152 instrument_atomic_read_write(v, sizeof(*v));
3153 return raw_atomic64_dec_unless_positi 3153 return raw_atomic64_dec_unless_positive(v);
3154 } 3154 }
3155 3155
3156 /** 3156 /**
3157 * atomic64_dec_if_positive() - atomic decrem 3157 * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
3158 * @v: pointer to atomic64_t 3158 * @v: pointer to atomic64_t
3159 * 3159 *
3160 * If (@v > 0), atomically updates @v to (@v 3160 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
3161 * Otherwise, @v is not modified and relaxed 3161 * Otherwise, @v is not modified and relaxed ordering is provided.
3162 * 3162 *
3163 * Unsafe to use in noinstr code; use raw_ato 3163 * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
3164 * 3164 *
3165 * Return: The old value of (@v - 1), regardl 3165 * Return: The old value of (@v - 1), regardless of whether @v was updated.
3166 */ 3166 */
3167 static __always_inline s64 3167 static __always_inline s64
3168 atomic64_dec_if_positive(atomic64_t *v) 3168 atomic64_dec_if_positive(atomic64_t *v)
3169 { 3169 {
3170 kcsan_mb(); 3170 kcsan_mb();
3171 instrument_atomic_read_write(v, sizeo 3171 instrument_atomic_read_write(v, sizeof(*v));
3172 return raw_atomic64_dec_if_positive(v 3172 return raw_atomic64_dec_if_positive(v);
3173 } 3173 }
3174 3174
3175 /** 3175 /**
3176 * atomic_long_read() - atomic load with rela 3176 * atomic_long_read() - atomic load with relaxed ordering
3177 * @v: pointer to atomic_long_t 3177 * @v: pointer to atomic_long_t
3178 * 3178 *
3179 * Atomically loads the value of @v with rela 3179 * Atomically loads the value of @v with relaxed ordering.
3180 * 3180 *
3181 * Unsafe to use in noinstr code; use raw_ato 3181 * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
3182 * 3182 *
3183 * Return: The value loaded from @v. 3183 * Return: The value loaded from @v.
3184 */ 3184 */
3185 static __always_inline long 3185 static __always_inline long
3186 atomic_long_read(const atomic_long_t *v) 3186 atomic_long_read(const atomic_long_t *v)
3187 { 3187 {
3188 instrument_atomic_read(v, sizeof(*v)) 3188 instrument_atomic_read(v, sizeof(*v));
3189 return raw_atomic_long_read(v); 3189 return raw_atomic_long_read(v);
3190 } 3190 }
3191 3191
3192 /** 3192 /**
3193 * atomic_long_read_acquire() - atomic load w 3193 * atomic_long_read_acquire() - atomic load with acquire ordering
3194 * @v: pointer to atomic_long_t 3194 * @v: pointer to atomic_long_t
3195 * 3195 *
3196 * Atomically loads the value of @v with acqu 3196 * Atomically loads the value of @v with acquire ordering.
3197 * 3197 *
3198 * Unsafe to use in noinstr code; use raw_ato 3198 * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
3199 * 3199 *
3200 * Return: The value loaded from @v. 3200 * Return: The value loaded from @v.
3201 */ 3201 */
3202 static __always_inline long 3202 static __always_inline long
3203 atomic_long_read_acquire(const atomic_long_t 3203 atomic_long_read_acquire(const atomic_long_t *v)
3204 { 3204 {
3205 instrument_atomic_read(v, sizeof(*v)) 3205 instrument_atomic_read(v, sizeof(*v));
3206 return raw_atomic_long_read_acquire(v 3206 return raw_atomic_long_read_acquire(v);
3207 } 3207 }
3208 3208
3209 /** 3209 /**
3210 * atomic_long_set() - atomic set with relaxe 3210 * atomic_long_set() - atomic set with relaxed ordering
3211 * @v: pointer to atomic_long_t 3211 * @v: pointer to atomic_long_t
3212 * @i: long value to assign 3212 * @i: long value to assign
3213 * 3213 *
3214 * Atomically sets @v to @i with relaxed orde 3214 * Atomically sets @v to @i with relaxed ordering.
3215 * 3215 *
3216 * Unsafe to use in noinstr code; use raw_ato 3216 * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
3217 * 3217 *
3218 * Return: Nothing. 3218 * Return: Nothing.
3219 */ 3219 */
3220 static __always_inline void 3220 static __always_inline void
3221 atomic_long_set(atomic_long_t *v, long i) 3221 atomic_long_set(atomic_long_t *v, long i)
3222 { 3222 {
3223 instrument_atomic_write(v, sizeof(*v) 3223 instrument_atomic_write(v, sizeof(*v));
3224 raw_atomic_long_set(v, i); 3224 raw_atomic_long_set(v, i);
3225 } 3225 }
3226 3226
3227 /** 3227 /**
3228 * atomic_long_set_release() - atomic set wit 3228 * atomic_long_set_release() - atomic set with release ordering
3229 * @v: pointer to atomic_long_t 3229 * @v: pointer to atomic_long_t
3230 * @i: long value to assign 3230 * @i: long value to assign
3231 * 3231 *
3232 * Atomically sets @v to @i with release orde 3232 * Atomically sets @v to @i with release ordering.
3233 * 3233 *
3234 * Unsafe to use in noinstr code; use raw_ato 3234 * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
3235 * 3235 *
3236 * Return: Nothing. 3236 * Return: Nothing.
3237 */ 3237 */
3238 static __always_inline void 3238 static __always_inline void
3239 atomic_long_set_release(atomic_long_t *v, lon 3239 atomic_long_set_release(atomic_long_t *v, long i)
3240 { 3240 {
3241 kcsan_release(); 3241 kcsan_release();
3242 instrument_atomic_write(v, sizeof(*v) 3242 instrument_atomic_write(v, sizeof(*v));
3243 raw_atomic_long_set_release(v, i); 3243 raw_atomic_long_set_release(v, i);
3244 } 3244 }
3245 3245
3246 /** 3246 /**
3247 * atomic_long_add() - atomic add with relaxe 3247 * atomic_long_add() - atomic add with relaxed ordering
3248 * @i: long value to add 3248 * @i: long value to add
3249 * @v: pointer to atomic_long_t 3249 * @v: pointer to atomic_long_t
3250 * 3250 *
3251 * Atomically updates @v to (@v + @i) with re 3251 * Atomically updates @v to (@v + @i) with relaxed ordering.
3252 * 3252 *
3253 * Unsafe to use in noinstr code; use raw_ato 3253 * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
3254 * 3254 *
3255 * Return: Nothing. 3255 * Return: Nothing.
3256 */ 3256 */
3257 static __always_inline void 3257 static __always_inline void
3258 atomic_long_add(long i, atomic_long_t *v) 3258 atomic_long_add(long i, atomic_long_t *v)
3259 { 3259 {
3260 instrument_atomic_read_write(v, sizeo 3260 instrument_atomic_read_write(v, sizeof(*v));
3261 raw_atomic_long_add(i, v); 3261 raw_atomic_long_add(i, v);
3262 } 3262 }
3263 3263
3264 /** 3264 /**
3265 * atomic_long_add_return() - atomic add with 3265 * atomic_long_add_return() - atomic add with full ordering
3266 * @i: long value to add 3266 * @i: long value to add
3267 * @v: pointer to atomic_long_t 3267 * @v: pointer to atomic_long_t
3268 * 3268 *
3269 * Atomically updates @v to (@v + @i) with fu 3269 * Atomically updates @v to (@v + @i) with full ordering.
3270 * 3270 *
3271 * Unsafe to use in noinstr code; use raw_ato 3271 * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
3272 * 3272 *
3273 * Return: The updated value of @v. 3273 * Return: The updated value of @v.
3274 */ 3274 */
3275 static __always_inline long 3275 static __always_inline long
3276 atomic_long_add_return(long i, atomic_long_t 3276 atomic_long_add_return(long i, atomic_long_t *v)
3277 { 3277 {
3278 kcsan_mb(); 3278 kcsan_mb();
3279 instrument_atomic_read_write(v, sizeo 3279 instrument_atomic_read_write(v, sizeof(*v));
3280 return raw_atomic_long_add_return(i, 3280 return raw_atomic_long_add_return(i, v);
3281 } 3281 }
3282 3282
3283 /** 3283 /**
3284 * atomic_long_add_return_acquire() - atomic 3284 * atomic_long_add_return_acquire() - atomic add with acquire ordering
3285 * @i: long value to add 3285 * @i: long value to add
3286 * @v: pointer to atomic_long_t 3286 * @v: pointer to atomic_long_t
3287 * 3287 *
3288 * Atomically updates @v to (@v + @i) with ac 3288 * Atomically updates @v to (@v + @i) with acquire ordering.
3289 * 3289 *
3290 * Unsafe to use in noinstr code; use raw_ato 3290 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
3291 * 3291 *
3292 * Return: The updated value of @v. 3292 * Return: The updated value of @v.
3293 */ 3293 */
3294 static __always_inline long 3294 static __always_inline long
3295 atomic_long_add_return_acquire(long i, atomic 3295 atomic_long_add_return_acquire(long i, atomic_long_t *v)
3296 { 3296 {
3297 instrument_atomic_read_write(v, sizeo 3297 instrument_atomic_read_write(v, sizeof(*v));
3298 return raw_atomic_long_add_return_acq 3298 return raw_atomic_long_add_return_acquire(i, v);
3299 } 3299 }
3300 3300
3301 /** 3301 /**
3302 * atomic_long_add_return_release() - atomic 3302 * atomic_long_add_return_release() - atomic add with release ordering
3303 * @i: long value to add 3303 * @i: long value to add
3304 * @v: pointer to atomic_long_t 3304 * @v: pointer to atomic_long_t
3305 * 3305 *
3306 * Atomically updates @v to (@v + @i) with re 3306 * Atomically updates @v to (@v + @i) with release ordering.
3307 * 3307 *
3308 * Unsafe to use in noinstr code; use raw_ato 3308 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
3309 * 3309 *
3310 * Return: The updated value of @v. 3310 * Return: The updated value of @v.
3311 */ 3311 */
3312 static __always_inline long 3312 static __always_inline long
3313 atomic_long_add_return_release(long i, atomic 3313 atomic_long_add_return_release(long i, atomic_long_t *v)
3314 { 3314 {
3315 kcsan_release(); 3315 kcsan_release();
3316 instrument_atomic_read_write(v, sizeo 3316 instrument_atomic_read_write(v, sizeof(*v));
3317 return raw_atomic_long_add_return_rel 3317 return raw_atomic_long_add_return_release(i, v);
3318 } 3318 }
3319 3319
3320 /** 3320 /**
3321 * atomic_long_add_return_relaxed() - atomic 3321 * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
3322 * @i: long value to add 3322 * @i: long value to add
3323 * @v: pointer to atomic_long_t 3323 * @v: pointer to atomic_long_t
3324 * 3324 *
3325 * Atomically updates @v to (@v + @i) with re 3325 * Atomically updates @v to (@v + @i) with relaxed ordering.
3326 * 3326 *
3327 * Unsafe to use in noinstr code; use raw_ato 3327 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
3328 * 3328 *
3329 * Return: The updated value of @v. 3329 * Return: The updated value of @v.
3330 */ 3330 */
3331 static __always_inline long 3331 static __always_inline long
3332 atomic_long_add_return_relaxed(long i, atomic 3332 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
3333 { 3333 {
3334 instrument_atomic_read_write(v, sizeo 3334 instrument_atomic_read_write(v, sizeof(*v));
3335 return raw_atomic_long_add_return_rel 3335 return raw_atomic_long_add_return_relaxed(i, v);
3336 } 3336 }
3337 3337
3338 /** 3338 /**
3339 * atomic_long_fetch_add() - atomic add with 3339 * atomic_long_fetch_add() - atomic add with full ordering
3340 * @i: long value to add 3340 * @i: long value to add
3341 * @v: pointer to atomic_long_t 3341 * @v: pointer to atomic_long_t
3342 * 3342 *
3343 * Atomically updates @v to (@v + @i) with fu 3343 * Atomically updates @v to (@v + @i) with full ordering.
3344 * 3344 *
3345 * Unsafe to use in noinstr code; use raw_ato 3345 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
3346 * 3346 *
3347 * Return: The original value of @v. 3347 * Return: The original value of @v.
3348 */ 3348 */
3349 static __always_inline long 3349 static __always_inline long
3350 atomic_long_fetch_add(long i, atomic_long_t * 3350 atomic_long_fetch_add(long i, atomic_long_t *v)
3351 { 3351 {
3352 kcsan_mb(); 3352 kcsan_mb();
3353 instrument_atomic_read_write(v, sizeo 3353 instrument_atomic_read_write(v, sizeof(*v));
3354 return raw_atomic_long_fetch_add(i, v 3354 return raw_atomic_long_fetch_add(i, v);
3355 } 3355 }
3356 3356
3357 /** 3357 /**
3358 * atomic_long_fetch_add_acquire() - atomic a 3358 * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
3359 * @i: long value to add 3359 * @i: long value to add
3360 * @v: pointer to atomic_long_t 3360 * @v: pointer to atomic_long_t
3361 * 3361 *
3362 * Atomically updates @v to (@v + @i) with ac 3362 * Atomically updates @v to (@v + @i) with acquire ordering.
3363 * 3363 *
3364 * Unsafe to use in noinstr code; use raw_ato 3364 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
3365 * 3365 *
3366 * Return: The original value of @v. 3366 * Return: The original value of @v.
3367 */ 3367 */
3368 static __always_inline long 3368 static __always_inline long
3369 atomic_long_fetch_add_acquire(long i, atomic_ 3369 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
3370 { 3370 {
3371 instrument_atomic_read_write(v, sizeo 3371 instrument_atomic_read_write(v, sizeof(*v));
3372 return raw_atomic_long_fetch_add_acqu 3372 return raw_atomic_long_fetch_add_acquire(i, v);
3373 } 3373 }
3374 3374
3375 /** 3375 /**
3376 * atomic_long_fetch_add_release() - atomic a 3376 * atomic_long_fetch_add_release() - atomic add with release ordering
3377 * @i: long value to add 3377 * @i: long value to add
3378 * @v: pointer to atomic_long_t 3378 * @v: pointer to atomic_long_t
3379 * 3379 *
3380 * Atomically updates @v to (@v + @i) with re 3380 * Atomically updates @v to (@v + @i) with release ordering.
3381 * 3381 *
3382 * Unsafe to use in noinstr code; use raw_ato 3382 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
3383 * 3383 *
3384 * Return: The original value of @v. 3384 * Return: The original value of @v.
3385 */ 3385 */
3386 static __always_inline long 3386 static __always_inline long
3387 atomic_long_fetch_add_release(long i, atomic_ 3387 atomic_long_fetch_add_release(long i, atomic_long_t *v)
3388 { 3388 {
3389 kcsan_release(); 3389 kcsan_release();
3390 instrument_atomic_read_write(v, sizeo 3390 instrument_atomic_read_write(v, sizeof(*v));
3391 return raw_atomic_long_fetch_add_rele 3391 return raw_atomic_long_fetch_add_release(i, v);
3392 } 3392 }
3393 3393
3394 /** 3394 /**
3395 * atomic_long_fetch_add_relaxed() - atomic a 3395 * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
3396 * @i: long value to add 3396 * @i: long value to add
3397 * @v: pointer to atomic_long_t 3397 * @v: pointer to atomic_long_t
3398 * 3398 *
3399 * Atomically updates @v to (@v + @i) with re 3399 * Atomically updates @v to (@v + @i) with relaxed ordering.
3400 * 3400 *
3401 * Unsafe to use in noinstr code; use raw_ato 3401 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
3402 * 3402 *
3403 * Return: The original value of @v. 3403 * Return: The original value of @v.
3404 */ 3404 */
3405 static __always_inline long 3405 static __always_inline long
3406 atomic_long_fetch_add_relaxed(long i, atomic_ 3406 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
3407 { 3407 {
3408 instrument_atomic_read_write(v, sizeo 3408 instrument_atomic_read_write(v, sizeof(*v));
3409 return raw_atomic_long_fetch_add_rela 3409 return raw_atomic_long_fetch_add_relaxed(i, v);
3410 } 3410 }
3411 3411
3412 /** 3412 /**
3413 * atomic_long_sub() - atomic subtract with r 3413 * atomic_long_sub() - atomic subtract with relaxed ordering
3414 * @i: long value to subtract 3414 * @i: long value to subtract
3415 * @v: pointer to atomic_long_t 3415 * @v: pointer to atomic_long_t
3416 * 3416 *
3417 * Atomically updates @v to (@v - @i) with re 3417 * Atomically updates @v to (@v - @i) with relaxed ordering.
3418 * 3418 *
3419 * Unsafe to use in noinstr code; use raw_ato 3419 * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
3420 * 3420 *
3421 * Return: Nothing. 3421 * Return: Nothing.
3422 */ 3422 */
3423 static __always_inline void 3423 static __always_inline void
3424 atomic_long_sub(long i, atomic_long_t *v) 3424 atomic_long_sub(long i, atomic_long_t *v)
3425 { 3425 {
3426 instrument_atomic_read_write(v, sizeo 3426 instrument_atomic_read_write(v, sizeof(*v));
3427 raw_atomic_long_sub(i, v); 3427 raw_atomic_long_sub(i, v);
3428 } 3428 }
3429 3429
3430 /** 3430 /**
3431 * atomic_long_sub_return() - atomic subtract 3431 * atomic_long_sub_return() - atomic subtract with full ordering
3432 * @i: long value to subtract 3432 * @i: long value to subtract
3433 * @v: pointer to atomic_long_t 3433 * @v: pointer to atomic_long_t
3434 * 3434 *
3435 * Atomically updates @v to (@v - @i) with fu 3435 * Atomically updates @v to (@v - @i) with full ordering.
3436 * 3436 *
3437 * Unsafe to use in noinstr code; use raw_ato 3437 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
3438 * 3438 *
3439 * Return: The updated value of @v. 3439 * Return: The updated value of @v.
3440 */ 3440 */
3441 static __always_inline long 3441 static __always_inline long
3442 atomic_long_sub_return(long i, atomic_long_t 3442 atomic_long_sub_return(long i, atomic_long_t *v)
3443 { 3443 {
3444 kcsan_mb(); 3444 kcsan_mb();
3445 instrument_atomic_read_write(v, sizeo 3445 instrument_atomic_read_write(v, sizeof(*v));
3446 return raw_atomic_long_sub_return(i, 3446 return raw_atomic_long_sub_return(i, v);
3447 } 3447 }
3448 3448
3449 /** 3449 /**
3450 * atomic_long_sub_return_acquire() - atomic 3450 * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
3451 * @i: long value to subtract 3451 * @i: long value to subtract
3452 * @v: pointer to atomic_long_t 3452 * @v: pointer to atomic_long_t
3453 * 3453 *
3454 * Atomically updates @v to (@v - @i) with ac 3454 * Atomically updates @v to (@v - @i) with acquire ordering.
3455 * 3455 *
3456 * Unsafe to use in noinstr code; use raw_ato 3456 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
3457 * 3457 *
3458 * Return: The updated value of @v. 3458 * Return: The updated value of @v.
3459 */ 3459 */
3460 static __always_inline long 3460 static __always_inline long
3461 atomic_long_sub_return_acquire(long i, atomic 3461 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
3462 { 3462 {
3463 instrument_atomic_read_write(v, sizeo 3463 instrument_atomic_read_write(v, sizeof(*v));
3464 return raw_atomic_long_sub_return_acq 3464 return raw_atomic_long_sub_return_acquire(i, v);
3465 } 3465 }
3466 3466
3467 /** 3467 /**
3468 * atomic_long_sub_return_release() - atomic 3468 * atomic_long_sub_return_release() - atomic subtract with release ordering
3469 * @i: long value to subtract 3469 * @i: long value to subtract
3470 * @v: pointer to atomic_long_t 3470 * @v: pointer to atomic_long_t
3471 * 3471 *
3472 * Atomically updates @v to (@v - @i) with re 3472 * Atomically updates @v to (@v - @i) with release ordering.
3473 * 3473 *
3474 * Unsafe to use in noinstr code; use raw_ato 3474 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
3475 * 3475 *
3476 * Return: The updated value of @v. 3476 * Return: The updated value of @v.
3477 */ 3477 */
3478 static __always_inline long 3478 static __always_inline long
3479 atomic_long_sub_return_release(long i, atomic 3479 atomic_long_sub_return_release(long i, atomic_long_t *v)
3480 { 3480 {
3481 kcsan_release(); 3481 kcsan_release();
3482 instrument_atomic_read_write(v, sizeo 3482 instrument_atomic_read_write(v, sizeof(*v));
3483 return raw_atomic_long_sub_return_rel 3483 return raw_atomic_long_sub_return_release(i, v);
3484 } 3484 }
3485 3485
3486 /** 3486 /**
3487 * atomic_long_sub_return_relaxed() - atomic 3487 * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
3488 * @i: long value to subtract 3488 * @i: long value to subtract
3489 * @v: pointer to atomic_long_t 3489 * @v: pointer to atomic_long_t
3490 * 3490 *
3491 * Atomically updates @v to (@v - @i) with re 3491 * Atomically updates @v to (@v - @i) with relaxed ordering.
3492 * 3492 *
3493 * Unsafe to use in noinstr code; use raw_ato 3493 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
3494 * 3494 *
3495 * Return: The updated value of @v. 3495 * Return: The updated value of @v.
3496 */ 3496 */
3497 static __always_inline long 3497 static __always_inline long
3498 atomic_long_sub_return_relaxed(long i, atomic 3498 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
3499 { 3499 {
3500 instrument_atomic_read_write(v, sizeo 3500 instrument_atomic_read_write(v, sizeof(*v));
3501 return raw_atomic_long_sub_return_rel 3501 return raw_atomic_long_sub_return_relaxed(i, v);
3502 } 3502 }
3503 3503
3504 /** 3504 /**
3505 * atomic_long_fetch_sub() - atomic subtract 3505 * atomic_long_fetch_sub() - atomic subtract with full ordering
3506 * @i: long value to subtract 3506 * @i: long value to subtract
3507 * @v: pointer to atomic_long_t 3507 * @v: pointer to atomic_long_t
3508 * 3508 *
3509 * Atomically updates @v to (@v - @i) with fu 3509 * Atomically updates @v to (@v - @i) with full ordering.
3510 * 3510 *
3511 * Unsafe to use in noinstr code; use raw_ato 3511 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
3512 * 3512 *
3513 * Return: The original value of @v. 3513 * Return: The original value of @v.
3514 */ 3514 */
3515 static __always_inline long 3515 static __always_inline long
3516 atomic_long_fetch_sub(long i, atomic_long_t * 3516 atomic_long_fetch_sub(long i, atomic_long_t *v)
3517 { 3517 {
3518 kcsan_mb(); 3518 kcsan_mb();
3519 instrument_atomic_read_write(v, sizeo 3519 instrument_atomic_read_write(v, sizeof(*v));
3520 return raw_atomic_long_fetch_sub(i, v 3520 return raw_atomic_long_fetch_sub(i, v);
3521 } 3521 }
3522 3522
3523 /** 3523 /**
3524 * atomic_long_fetch_sub_acquire() - atomic s 3524 * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
3525 * @i: long value to subtract 3525 * @i: long value to subtract
3526 * @v: pointer to atomic_long_t 3526 * @v: pointer to atomic_long_t
3527 * 3527 *
3528 * Atomically updates @v to (@v - @i) with ac 3528 * Atomically updates @v to (@v - @i) with acquire ordering.
3529 * 3529 *
3530 * Unsafe to use in noinstr code; use raw_ato 3530 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
3531 * 3531 *
3532 * Return: The original value of @v. 3532 * Return: The original value of @v.
3533 */ 3533 */
3534 static __always_inline long 3534 static __always_inline long
3535 atomic_long_fetch_sub_acquire(long i, atomic_ 3535 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
3536 { 3536 {
3537 instrument_atomic_read_write(v, sizeo 3537 instrument_atomic_read_write(v, sizeof(*v));
3538 return raw_atomic_long_fetch_sub_acqu 3538 return raw_atomic_long_fetch_sub_acquire(i, v);
3539 } 3539 }
3540 3540
3541 /** 3541 /**
3542 * atomic_long_fetch_sub_release() - atomic s 3542 * atomic_long_fetch_sub_release() - atomic subtract with release ordering
3543 * @i: long value to subtract 3543 * @i: long value to subtract
3544 * @v: pointer to atomic_long_t 3544 * @v: pointer to atomic_long_t
3545 * 3545 *
3546 * Atomically updates @v to (@v - @i) with re 3546 * Atomically updates @v to (@v - @i) with release ordering.
3547 * 3547 *
3548 * Unsafe to use in noinstr code; use raw_ato 3548 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
3549 * 3549 *
3550 * Return: The original value of @v. 3550 * Return: The original value of @v.
3551 */ 3551 */
3552 static __always_inline long 3552 static __always_inline long
3553 atomic_long_fetch_sub_release(long i, atomic_ 3553 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
3554 { 3554 {
3555 kcsan_release(); 3555 kcsan_release();
3556 instrument_atomic_read_write(v, sizeo 3556 instrument_atomic_read_write(v, sizeof(*v));
3557 return raw_atomic_long_fetch_sub_rele 3557 return raw_atomic_long_fetch_sub_release(i, v);
3558 } 3558 }
3559 3559
3560 /** 3560 /**
3561 * atomic_long_fetch_sub_relaxed() - atomic s 3561 * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3562 * @i: long value to subtract 3562 * @i: long value to subtract
3563 * @v: pointer to atomic_long_t 3563 * @v: pointer to atomic_long_t
3564 * 3564 *
3565 * Atomically updates @v to (@v - @i) with re 3565 * Atomically updates @v to (@v - @i) with relaxed ordering.
3566 * 3566 *
3567 * Unsafe to use in noinstr code; use raw_ato 3567 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
3568 * 3568 *
3569 * Return: The original value of @v. 3569 * Return: The original value of @v.
3570 */ 3570 */
3571 static __always_inline long 3571 static __always_inline long
3572 atomic_long_fetch_sub_relaxed(long i, atomic_ 3572 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
3573 { 3573 {
3574 instrument_atomic_read_write(v, sizeo 3574 instrument_atomic_read_write(v, sizeof(*v));
3575 return raw_atomic_long_fetch_sub_rela 3575 return raw_atomic_long_fetch_sub_relaxed(i, v);
3576 } 3576 }
3577 3577
3578 /** 3578 /**
3579 * atomic_long_inc() - atomic increment with 3579 * atomic_long_inc() - atomic increment with relaxed ordering
3580 * @v: pointer to atomic_long_t 3580 * @v: pointer to atomic_long_t
3581 * 3581 *
3582 * Atomically updates @v to (@v + 1) with rel 3582 * Atomically updates @v to (@v + 1) with relaxed ordering.
3583 * 3583 *
3584 * Unsafe to use in noinstr code; use raw_ato 3584 * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
3585 * 3585 *
3586 * Return: Nothing. 3586 * Return: Nothing.
3587 */ 3587 */
3588 static __always_inline void 3588 static __always_inline void
3589 atomic_long_inc(atomic_long_t *v) 3589 atomic_long_inc(atomic_long_t *v)
3590 { 3590 {
3591 instrument_atomic_read_write(v, sizeo 3591 instrument_atomic_read_write(v, sizeof(*v));
3592 raw_atomic_long_inc(v); 3592 raw_atomic_long_inc(v);
3593 } 3593 }
3594 3594
3595 /** 3595 /**
3596 * atomic_long_inc_return() - atomic incremen 3596 * atomic_long_inc_return() - atomic increment with full ordering
3597 * @v: pointer to atomic_long_t 3597 * @v: pointer to atomic_long_t
3598 * 3598 *
3599 * Atomically updates @v to (@v + 1) with ful 3599 * Atomically updates @v to (@v + 1) with full ordering.
3600 * 3600 *
3601 * Unsafe to use in noinstr code; use raw_ato 3601 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
3602 * 3602 *
3603 * Return: The updated value of @v. 3603 * Return: The updated value of @v.
3604 */ 3604 */
3605 static __always_inline long 3605 static __always_inline long
3606 atomic_long_inc_return(atomic_long_t *v) 3606 atomic_long_inc_return(atomic_long_t *v)
3607 { 3607 {
3608 kcsan_mb(); 3608 kcsan_mb();
3609 instrument_atomic_read_write(v, sizeo 3609 instrument_atomic_read_write(v, sizeof(*v));
3610 return raw_atomic_long_inc_return(v); 3610 return raw_atomic_long_inc_return(v);
3611 } 3611 }
3612 3612
3613 /** 3613 /**
3614 * atomic_long_inc_return_acquire() - atomic 3614 * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
3615 * @v: pointer to atomic_long_t 3615 * @v: pointer to atomic_long_t
3616 * 3616 *
3617 * Atomically updates @v to (@v + 1) with acq 3617 * Atomically updates @v to (@v + 1) with acquire ordering.
3618 * 3618 *
3619 * Unsafe to use in noinstr code; use raw_ato 3619 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
3620 * 3620 *
3621 * Return: The updated value of @v. 3621 * Return: The updated value of @v.
3622 */ 3622 */
3623 static __always_inline long 3623 static __always_inline long
3624 atomic_long_inc_return_acquire(atomic_long_t 3624 atomic_long_inc_return_acquire(atomic_long_t *v)
3625 { 3625 {
3626 instrument_atomic_read_write(v, sizeo 3626 instrument_atomic_read_write(v, sizeof(*v));
3627 return raw_atomic_long_inc_return_acq 3627 return raw_atomic_long_inc_return_acquire(v);
3628 } 3628 }
3629 3629
3630 /** 3630 /**
3631 * atomic_long_inc_return_release() - atomic 3631 * atomic_long_inc_return_release() - atomic increment with release ordering
3632 * @v: pointer to atomic_long_t 3632 * @v: pointer to atomic_long_t
3633 * 3633 *
3634 * Atomically updates @v to (@v + 1) with rel 3634 * Atomically updates @v to (@v + 1) with release ordering.
3635 * 3635 *
3636 * Unsafe to use in noinstr code; use raw_ato 3636 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
3637 * 3637 *
3638 * Return: The updated value of @v. 3638 * Return: The updated value of @v.
3639 */ 3639 */
3640 static __always_inline long 3640 static __always_inline long
3641 atomic_long_inc_return_release(atomic_long_t 3641 atomic_long_inc_return_release(atomic_long_t *v)
3642 { 3642 {
3643 kcsan_release(); 3643 kcsan_release();
3644 instrument_atomic_read_write(v, sizeo 3644 instrument_atomic_read_write(v, sizeof(*v));
3645 return raw_atomic_long_inc_return_rel 3645 return raw_atomic_long_inc_return_release(v);
3646 } 3646 }
3647 3647
3648 /** 3648 /**
3649 * atomic_long_inc_return_relaxed() - atomic 3649 * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
3650 * @v: pointer to atomic_long_t 3650 * @v: pointer to atomic_long_t
3651 * 3651 *
3652 * Atomically updates @v to (@v + 1) with rel 3652 * Atomically updates @v to (@v + 1) with relaxed ordering.
3653 * 3653 *
3654 * Unsafe to use in noinstr code; use raw_ato 3654 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
3655 * 3655 *
3656 * Return: The updated value of @v. 3656 * Return: The updated value of @v.
3657 */ 3657 */
3658 static __always_inline long 3658 static __always_inline long
3659 atomic_long_inc_return_relaxed(atomic_long_t 3659 atomic_long_inc_return_relaxed(atomic_long_t *v)
3660 { 3660 {
3661 instrument_atomic_read_write(v, sizeo 3661 instrument_atomic_read_write(v, sizeof(*v));
3662 return raw_atomic_long_inc_return_rel 3662 return raw_atomic_long_inc_return_relaxed(v);
3663 } 3663 }
3664 3664
3665 /** 3665 /**
3666 * atomic_long_fetch_inc() - atomic increment 3666 * atomic_long_fetch_inc() - atomic increment with full ordering
3667 * @v: pointer to atomic_long_t 3667 * @v: pointer to atomic_long_t
3668 * 3668 *
3669 * Atomically updates @v to (@v + 1) with ful 3669 * Atomically updates @v to (@v + 1) with full ordering.
3670 * 3670 *
3671 * Unsafe to use in noinstr code; use raw_ato 3671 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
3672 * 3672 *
3673 * Return: The original value of @v. 3673 * Return: The original value of @v.
3674 */ 3674 */
3675 static __always_inline long 3675 static __always_inline long
3676 atomic_long_fetch_inc(atomic_long_t *v) 3676 atomic_long_fetch_inc(atomic_long_t *v)
3677 { 3677 {
3678 kcsan_mb(); 3678 kcsan_mb();
3679 instrument_atomic_read_write(v, sizeo 3679 instrument_atomic_read_write(v, sizeof(*v));
3680 return raw_atomic_long_fetch_inc(v); 3680 return raw_atomic_long_fetch_inc(v);
3681 } 3681 }
3682 3682
3683 /** 3683 /**
3684 * atomic_long_fetch_inc_acquire() - atomic i 3684 * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
3685 * @v: pointer to atomic_long_t 3685 * @v: pointer to atomic_long_t
3686 * 3686 *
3687 * Atomically updates @v to (@v + 1) with acq 3687 * Atomically updates @v to (@v + 1) with acquire ordering.
3688 * 3688 *
3689 * Unsafe to use in noinstr code; use raw_ato 3689 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
3690 * 3690 *
3691 * Return: The original value of @v. 3691 * Return: The original value of @v.
3692 */ 3692 */
3693 static __always_inline long 3693 static __always_inline long
3694 atomic_long_fetch_inc_acquire(atomic_long_t * 3694 atomic_long_fetch_inc_acquire(atomic_long_t *v)
3695 { 3695 {
3696 instrument_atomic_read_write(v, sizeo 3696 instrument_atomic_read_write(v, sizeof(*v));
3697 return raw_atomic_long_fetch_inc_acqu 3697 return raw_atomic_long_fetch_inc_acquire(v);
3698 } 3698 }
3699 3699
3700 /** 3700 /**
3701 * atomic_long_fetch_inc_release() - atomic i 3701 * atomic_long_fetch_inc_release() - atomic increment with release ordering
3702 * @v: pointer to atomic_long_t 3702 * @v: pointer to atomic_long_t
3703 * 3703 *
3704 * Atomically updates @v to (@v + 1) with rel 3704 * Atomically updates @v to (@v + 1) with release ordering.
3705 * 3705 *
3706 * Unsafe to use in noinstr code; use raw_ato 3706 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
3707 * 3707 *
3708 * Return: The original value of @v. 3708 * Return: The original value of @v.
3709 */ 3709 */
3710 static __always_inline long 3710 static __always_inline long
3711 atomic_long_fetch_inc_release(atomic_long_t * 3711 atomic_long_fetch_inc_release(atomic_long_t *v)
3712 { 3712 {
3713 kcsan_release(); 3713 kcsan_release();
3714 instrument_atomic_read_write(v, sizeo 3714 instrument_atomic_read_write(v, sizeof(*v));
3715 return raw_atomic_long_fetch_inc_rele 3715 return raw_atomic_long_fetch_inc_release(v);
3716 } 3716 }
3717 3717
3718 /** 3718 /**
3719 * atomic_long_fetch_inc_relaxed() - atomic i 3719 * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
3720 * @v: pointer to atomic_long_t 3720 * @v: pointer to atomic_long_t
3721 * 3721 *
3722 * Atomically updates @v to (@v + 1) with rel 3722 * Atomically updates @v to (@v + 1) with relaxed ordering.
3723 * 3723 *
3724 * Unsafe to use in noinstr code; use raw_ato 3724 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
3725 * 3725 *
3726 * Return: The original value of @v. 3726 * Return: The original value of @v.
3727 */ 3727 */
3728 static __always_inline long 3728 static __always_inline long
3729 atomic_long_fetch_inc_relaxed(atomic_long_t * 3729 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
3730 { 3730 {
3731 instrument_atomic_read_write(v, sizeo 3731 instrument_atomic_read_write(v, sizeof(*v));
3732 return raw_atomic_long_fetch_inc_rela 3732 return raw_atomic_long_fetch_inc_relaxed(v);
3733 } 3733 }
3734 3734
3735 /** 3735 /**
3736 * atomic_long_dec() - atomic decrement with 3736 * atomic_long_dec() - atomic decrement with relaxed ordering
3737 * @v: pointer to atomic_long_t 3737 * @v: pointer to atomic_long_t
3738 * 3738 *
3739 * Atomically updates @v to (@v - 1) with rel 3739 * Atomically updates @v to (@v - 1) with relaxed ordering.
3740 * 3740 *
3741 * Unsafe to use in noinstr code; use raw_ato 3741 * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
3742 * 3742 *
3743 * Return: Nothing. 3743 * Return: Nothing.
3744 */ 3744 */
3745 static __always_inline void 3745 static __always_inline void
3746 atomic_long_dec(atomic_long_t *v) 3746 atomic_long_dec(atomic_long_t *v)
3747 { 3747 {
3748 instrument_atomic_read_write(v, sizeo 3748 instrument_atomic_read_write(v, sizeof(*v));
3749 raw_atomic_long_dec(v); 3749 raw_atomic_long_dec(v);
3750 } 3750 }
3751 3751
3752 /** 3752 /**
3753 * atomic_long_dec_return() - atomic decremen 3753 * atomic_long_dec_return() - atomic decrement with full ordering
3754 * @v: pointer to atomic_long_t 3754 * @v: pointer to atomic_long_t
3755 * 3755 *
3756 * Atomically updates @v to (@v - 1) with ful 3756 * Atomically updates @v to (@v - 1) with full ordering.
3757 * 3757 *
3758 * Unsafe to use in noinstr code; use raw_ato 3758 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
3759 * 3759 *
3760 * Return: The updated value of @v. 3760 * Return: The updated value of @v.
3761 */ 3761 */
3762 static __always_inline long 3762 static __always_inline long
3763 atomic_long_dec_return(atomic_long_t *v) 3763 atomic_long_dec_return(atomic_long_t *v)
3764 { 3764 {
3765 kcsan_mb(); 3765 kcsan_mb();
3766 instrument_atomic_read_write(v, sizeo 3766 instrument_atomic_read_write(v, sizeof(*v));
3767 return raw_atomic_long_dec_return(v); 3767 return raw_atomic_long_dec_return(v);
3768 } 3768 }
3769 3769
3770 /** 3770 /**
3771 * atomic_long_dec_return_acquire() - atomic 3771 * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
3772 * @v: pointer to atomic_long_t 3772 * @v: pointer to atomic_long_t
3773 * 3773 *
3774 * Atomically updates @v to (@v - 1) with acq 3774 * Atomically updates @v to (@v - 1) with acquire ordering.
3775 * 3775 *
3776 * Unsafe to use in noinstr code; use raw_ato 3776 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
3777 * 3777 *
3778 * Return: The updated value of @v. 3778 * Return: The updated value of @v.
3779 */ 3779 */
3780 static __always_inline long 3780 static __always_inline long
3781 atomic_long_dec_return_acquire(atomic_long_t 3781 atomic_long_dec_return_acquire(atomic_long_t *v)
3782 { 3782 {
3783 instrument_atomic_read_write(v, sizeo 3783 instrument_atomic_read_write(v, sizeof(*v));
3784 return raw_atomic_long_dec_return_acq 3784 return raw_atomic_long_dec_return_acquire(v);
3785 } 3785 }
3786 3786
3787 /** 3787 /**
3788 * atomic_long_dec_return_release() - atomic 3788 * atomic_long_dec_return_release() - atomic decrement with release ordering
3789 * @v: pointer to atomic_long_t 3789 * @v: pointer to atomic_long_t
3790 * 3790 *
3791 * Atomically updates @v to (@v - 1) with rel 3791 * Atomically updates @v to (@v - 1) with release ordering.
3792 * 3792 *
3793 * Unsafe to use in noinstr code; use raw_ato 3793 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
3794 * 3794 *
3795 * Return: The updated value of @v. 3795 * Return: The updated value of @v.
3796 */ 3796 */
3797 static __always_inline long 3797 static __always_inline long
3798 atomic_long_dec_return_release(atomic_long_t 3798 atomic_long_dec_return_release(atomic_long_t *v)
3799 { 3799 {
3800 kcsan_release(); 3800 kcsan_release();
3801 instrument_atomic_read_write(v, sizeo 3801 instrument_atomic_read_write(v, sizeof(*v));
3802 return raw_atomic_long_dec_return_rel 3802 return raw_atomic_long_dec_return_release(v);
3803 } 3803 }
3804 3804
3805 /** 3805 /**
3806 * atomic_long_dec_return_relaxed() - atomic 3806 * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
3807 * @v: pointer to atomic_long_t 3807 * @v: pointer to atomic_long_t
3808 * 3808 *
3809 * Atomically updates @v to (@v - 1) with rel 3809 * Atomically updates @v to (@v - 1) with relaxed ordering.
3810 * 3810 *
3811 * Unsafe to use in noinstr code; use raw_ato 3811 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
3812 * 3812 *
3813 * Return: The updated value of @v. 3813 * Return: The updated value of @v.
3814 */ 3814 */
3815 static __always_inline long 3815 static __always_inline long
3816 atomic_long_dec_return_relaxed(atomic_long_t 3816 atomic_long_dec_return_relaxed(atomic_long_t *v)
3817 { 3817 {
3818 instrument_atomic_read_write(v, sizeo 3818 instrument_atomic_read_write(v, sizeof(*v));
3819 return raw_atomic_long_dec_return_rel 3819 return raw_atomic_long_dec_return_relaxed(v);
3820 } 3820 }
3821 3821
3822 /** 3822 /**
3823 * atomic_long_fetch_dec() - atomic decrement 3823 * atomic_long_fetch_dec() - atomic decrement with full ordering
3824 * @v: pointer to atomic_long_t 3824 * @v: pointer to atomic_long_t
3825 * 3825 *
3826 * Atomically updates @v to (@v - 1) with ful 3826 * Atomically updates @v to (@v - 1) with full ordering.
3827 * 3827 *
3828 * Unsafe to use in noinstr code; use raw_ato 3828 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
3829 * 3829 *
3830 * Return: The original value of @v. 3830 * Return: The original value of @v.
3831 */ 3831 */
3832 static __always_inline long 3832 static __always_inline long
3833 atomic_long_fetch_dec(atomic_long_t *v) 3833 atomic_long_fetch_dec(atomic_long_t *v)
3834 { 3834 {
3835 kcsan_mb(); 3835 kcsan_mb();
3836 instrument_atomic_read_write(v, sizeo 3836 instrument_atomic_read_write(v, sizeof(*v));
3837 return raw_atomic_long_fetch_dec(v); 3837 return raw_atomic_long_fetch_dec(v);
3838 } 3838 }
3839 3839
3840 /** 3840 /**
3841 * atomic_long_fetch_dec_acquire() - atomic d 3841 * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
3842 * @v: pointer to atomic_long_t 3842 * @v: pointer to atomic_long_t
3843 * 3843 *
3844 * Atomically updates @v to (@v - 1) with acq 3844 * Atomically updates @v to (@v - 1) with acquire ordering.
3845 * 3845 *
3846 * Unsafe to use in noinstr code; use raw_ato 3846 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
3847 * 3847 *
3848 * Return: The original value of @v. 3848 * Return: The original value of @v.
3849 */ 3849 */
3850 static __always_inline long 3850 static __always_inline long
3851 atomic_long_fetch_dec_acquire(atomic_long_t * 3851 atomic_long_fetch_dec_acquire(atomic_long_t *v)
3852 { 3852 {
3853 instrument_atomic_read_write(v, sizeo 3853 instrument_atomic_read_write(v, sizeof(*v));
3854 return raw_atomic_long_fetch_dec_acqu 3854 return raw_atomic_long_fetch_dec_acquire(v);
3855 } 3855 }
3856 3856
3857 /** 3857 /**
3858 * atomic_long_fetch_dec_release() - atomic d 3858 * atomic_long_fetch_dec_release() - atomic decrement with release ordering
3859 * @v: pointer to atomic_long_t 3859 * @v: pointer to atomic_long_t
3860 * 3860 *
3861 * Atomically updates @v to (@v - 1) with rel 3861 * Atomically updates @v to (@v - 1) with release ordering.
3862 * 3862 *
3863 * Unsafe to use in noinstr code; use raw_ato 3863 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
3864 * 3864 *
3865 * Return: The original value of @v. 3865 * Return: The original value of @v.
3866 */ 3866 */
3867 static __always_inline long 3867 static __always_inline long
3868 atomic_long_fetch_dec_release(atomic_long_t * 3868 atomic_long_fetch_dec_release(atomic_long_t *v)
3869 { 3869 {
3870 kcsan_release(); 3870 kcsan_release();
3871 instrument_atomic_read_write(v, sizeo 3871 instrument_atomic_read_write(v, sizeof(*v));
3872 return raw_atomic_long_fetch_dec_rele 3872 return raw_atomic_long_fetch_dec_release(v);
3873 } 3873 }
3874 3874
3875 /** 3875 /**
3876 * atomic_long_fetch_dec_relaxed() - atomic d 3876 * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3877 * @v: pointer to atomic_long_t 3877 * @v: pointer to atomic_long_t
3878 * 3878 *
3879 * Atomically updates @v to (@v - 1) with rel 3879 * Atomically updates @v to (@v - 1) with relaxed ordering.
3880 * 3880 *
3881 * Unsafe to use in noinstr code; use raw_ato 3881 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
3882 * 3882 *
3883 * Return: The original value of @v. 3883 * Return: The original value of @v.
3884 */ 3884 */
3885 static __always_inline long 3885 static __always_inline long
3886 atomic_long_fetch_dec_relaxed(atomic_long_t * 3886 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
3887 { 3887 {
3888 instrument_atomic_read_write(v, sizeo 3888 instrument_atomic_read_write(v, sizeof(*v));
3889 return raw_atomic_long_fetch_dec_rela 3889 return raw_atomic_long_fetch_dec_relaxed(v);
3890 } 3890 }
3891 3891
3892 /** 3892 /**
3893 * atomic_long_and() - atomic bitwise AND wit 3893 * atomic_long_and() - atomic bitwise AND with relaxed ordering
3894 * @i: long value 3894 * @i: long value
3895 * @v: pointer to atomic_long_t 3895 * @v: pointer to atomic_long_t
3896 * 3896 *
3897 * Atomically updates @v to (@v & @i) with re 3897 * Atomically updates @v to (@v & @i) with relaxed ordering.
3898 * 3898 *
3899 * Unsafe to use in noinstr code; use raw_ato 3899 * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
3900 * 3900 *
3901 * Return: Nothing. 3901 * Return: Nothing.
3902 */ 3902 */
3903 static __always_inline void 3903 static __always_inline void
3904 atomic_long_and(long i, atomic_long_t *v) 3904 atomic_long_and(long i, atomic_long_t *v)
3905 { 3905 {
3906 instrument_atomic_read_write(v, sizeo 3906 instrument_atomic_read_write(v, sizeof(*v));
3907 raw_atomic_long_and(i, v); 3907 raw_atomic_long_and(i, v);
3908 } 3908 }
3909 3909
3910 /** 3910 /**
3911 * atomic_long_fetch_and() - atomic bitwise A 3911 * atomic_long_fetch_and() - atomic bitwise AND with full ordering
3912 * @i: long value 3912 * @i: long value
3913 * @v: pointer to atomic_long_t 3913 * @v: pointer to atomic_long_t
3914 * 3914 *
3915 * Atomically updates @v to (@v & @i) with fu 3915 * Atomically updates @v to (@v & @i) with full ordering.
3916 * 3916 *
3917 * Unsafe to use in noinstr code; use raw_ato 3917 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
3918 * 3918 *
3919 * Return: The original value of @v. 3919 * Return: The original value of @v.
3920 */ 3920 */
3921 static __always_inline long 3921 static __always_inline long
3922 atomic_long_fetch_and(long i, atomic_long_t * 3922 atomic_long_fetch_and(long i, atomic_long_t *v)
3923 { 3923 {
3924 kcsan_mb(); 3924 kcsan_mb();
3925 instrument_atomic_read_write(v, sizeo 3925 instrument_atomic_read_write(v, sizeof(*v));
3926 return raw_atomic_long_fetch_and(i, v 3926 return raw_atomic_long_fetch_and(i, v);
3927 } 3927 }
3928 3928
3929 /** 3929 /**
3930 * atomic_long_fetch_and_acquire() - atomic b 3930 * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3931 * @i: long value 3931 * @i: long value
3932 * @v: pointer to atomic_long_t 3932 * @v: pointer to atomic_long_t
3933 * 3933 *
3934 * Atomically updates @v to (@v & @i) with ac 3934 * Atomically updates @v to (@v & @i) with acquire ordering.
3935 * 3935 *
3936 * Unsafe to use in noinstr code; use raw_ato 3936 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
3937 * 3937 *
3938 * Return: The original value of @v. 3938 * Return: The original value of @v.
3939 */ 3939 */
3940 static __always_inline long 3940 static __always_inline long
3941 atomic_long_fetch_and_acquire(long i, atomic_ 3941 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
3942 { 3942 {
3943 instrument_atomic_read_write(v, sizeo 3943 instrument_atomic_read_write(v, sizeof(*v));
3944 return raw_atomic_long_fetch_and_acqu 3944 return raw_atomic_long_fetch_and_acquire(i, v);
3945 } 3945 }
3946 3946
3947 /** 3947 /**
3948 * atomic_long_fetch_and_release() - atomic b 3948 * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
3949 * @i: long value 3949 * @i: long value
3950 * @v: pointer to atomic_long_t 3950 * @v: pointer to atomic_long_t
3951 * 3951 *
3952 * Atomically updates @v to (@v & @i) with re 3952 * Atomically updates @v to (@v & @i) with release ordering.
3953 * 3953 *
3954 * Unsafe to use in noinstr code; use raw_ato 3954 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
3955 * 3955 *
3956 * Return: The original value of @v. 3956 * Return: The original value of @v.
3957 */ 3957 */
3958 static __always_inline long 3958 static __always_inline long
3959 atomic_long_fetch_and_release(long i, atomic_ 3959 atomic_long_fetch_and_release(long i, atomic_long_t *v)
3960 { 3960 {
3961 kcsan_release(); 3961 kcsan_release();
3962 instrument_atomic_read_write(v, sizeo 3962 instrument_atomic_read_write(v, sizeof(*v));
3963 return raw_atomic_long_fetch_and_rele 3963 return raw_atomic_long_fetch_and_release(i, v);
3964 } 3964 }
3965 3965
3966 /** 3966 /**
3967 * atomic_long_fetch_and_relaxed() - atomic b 3967 * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3968 * @i: long value 3968 * @i: long value
3969 * @v: pointer to atomic_long_t 3969 * @v: pointer to atomic_long_t
3970 * 3970 *
3971 * Atomically updates @v to (@v & @i) with re 3971 * Atomically updates @v to (@v & @i) with relaxed ordering.
3972 * 3972 *
3973 * Unsafe to use in noinstr code; use raw_ato 3973 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
3974 * 3974 *
3975 * Return: The original value of @v. 3975 * Return: The original value of @v.
3976 */ 3976 */
3977 static __always_inline long 3977 static __always_inline long
3978 atomic_long_fetch_and_relaxed(long i, atomic_ 3978 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
3979 { 3979 {
3980 instrument_atomic_read_write(v, sizeo 3980 instrument_atomic_read_write(v, sizeof(*v));
3981 return raw_atomic_long_fetch_and_rela 3981 return raw_atomic_long_fetch_and_relaxed(i, v);
3982 } 3982 }
3983 3983
3984 /** 3984 /**
3985 * atomic_long_andnot() - atomic bitwise AND 3985 * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
3986 * @i: long value 3986 * @i: long value
3987 * @v: pointer to atomic_long_t 3987 * @v: pointer to atomic_long_t
3988 * 3988 *
3989 * Atomically updates @v to (@v & ~@i) with r 3989 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3990 * 3990 *
3991 * Unsafe to use in noinstr code; use raw_ato 3991 * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
3992 * 3992 *
3993 * Return: Nothing. 3993 * Return: Nothing.
3994 */ 3994 */
3995 static __always_inline void 3995 static __always_inline void
3996 atomic_long_andnot(long i, atomic_long_t *v) 3996 atomic_long_andnot(long i, atomic_long_t *v)
3997 { 3997 {
3998 instrument_atomic_read_write(v, sizeo 3998 instrument_atomic_read_write(v, sizeof(*v));
3999 raw_atomic_long_andnot(i, v); 3999 raw_atomic_long_andnot(i, v);
4000 } 4000 }
4001 4001
4002 /** 4002 /**
4003 * atomic_long_fetch_andnot() - atomic bitwis 4003 * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
4004 * @i: long value 4004 * @i: long value
4005 * @v: pointer to atomic_long_t 4005 * @v: pointer to atomic_long_t
4006 * 4006 *
4007 * Atomically updates @v to (@v & ~@i) with f 4007 * Atomically updates @v to (@v & ~@i) with full ordering.
4008 * 4008 *
4009 * Unsafe to use in noinstr code; use raw_ato 4009 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
4010 * 4010 *
4011 * Return: The original value of @v. 4011 * Return: The original value of @v.
4012 */ 4012 */
4013 static __always_inline long 4013 static __always_inline long
4014 atomic_long_fetch_andnot(long i, atomic_long_ 4014 atomic_long_fetch_andnot(long i, atomic_long_t *v)
4015 { 4015 {
4016 kcsan_mb(); 4016 kcsan_mb();
4017 instrument_atomic_read_write(v, sizeo 4017 instrument_atomic_read_write(v, sizeof(*v));
4018 return raw_atomic_long_fetch_andnot(i 4018 return raw_atomic_long_fetch_andnot(i, v);
4019 } 4019 }
4020 4020
4021 /** 4021 /**
4022 * atomic_long_fetch_andnot_acquire() - atomi 4022 * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
4023 * @i: long value 4023 * @i: long value
4024 * @v: pointer to atomic_long_t 4024 * @v: pointer to atomic_long_t
4025 * 4025 *
4026 * Atomically updates @v to (@v & ~@i) with a 4026 * Atomically updates @v to (@v & ~@i) with acquire ordering.
4027 * 4027 *
4028 * Unsafe to use in noinstr code; use raw_ato 4028 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
4029 * 4029 *
4030 * Return: The original value of @v. 4030 * Return: The original value of @v.
4031 */ 4031 */
4032 static __always_inline long 4032 static __always_inline long
4033 atomic_long_fetch_andnot_acquire(long i, atom 4033 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
4034 { 4034 {
4035 instrument_atomic_read_write(v, sizeo 4035 instrument_atomic_read_write(v, sizeof(*v));
4036 return raw_atomic_long_fetch_andnot_a 4036 return raw_atomic_long_fetch_andnot_acquire(i, v);
4037 } 4037 }
4038 4038
4039 /** 4039 /**
4040 * atomic_long_fetch_andnot_release() - atomi 4040 * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
4041 * @i: long value 4041 * @i: long value
4042 * @v: pointer to atomic_long_t 4042 * @v: pointer to atomic_long_t
4043 * 4043 *
4044 * Atomically updates @v to (@v & ~@i) with r 4044 * Atomically updates @v to (@v & ~@i) with release ordering.
4045 * 4045 *
4046 * Unsafe to use in noinstr code; use raw_ato 4046 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
4047 * 4047 *
4048 * Return: The original value of @v. 4048 * Return: The original value of @v.
4049 */ 4049 */
4050 static __always_inline long 4050 static __always_inline long
4051 atomic_long_fetch_andnot_release(long i, atom 4051 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
4052 { 4052 {
4053 kcsan_release(); 4053 kcsan_release();
4054 instrument_atomic_read_write(v, sizeo 4054 instrument_atomic_read_write(v, sizeof(*v));
4055 return raw_atomic_long_fetch_andnot_r 4055 return raw_atomic_long_fetch_andnot_release(i, v);
4056 } 4056 }
4057 4057
4058 /** 4058 /**
4059 * atomic_long_fetch_andnot_relaxed() - atomi 4059 * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
4060 * @i: long value 4060 * @i: long value
4061 * @v: pointer to atomic_long_t 4061 * @v: pointer to atomic_long_t
4062 * 4062 *
4063 * Atomically updates @v to (@v & ~@i) with r 4063 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
4064 * 4064 *
4065 * Unsafe to use in noinstr code; use raw_ato 4065 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
4066 * 4066 *
4067 * Return: The original value of @v. 4067 * Return: The original value of @v.
4068 */ 4068 */
4069 static __always_inline long 4069 static __always_inline long
4070 atomic_long_fetch_andnot_relaxed(long i, atom 4070 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
4071 { 4071 {
4072 instrument_atomic_read_write(v, sizeo 4072 instrument_atomic_read_write(v, sizeof(*v));
4073 return raw_atomic_long_fetch_andnot_r 4073 return raw_atomic_long_fetch_andnot_relaxed(i, v);
4074 } 4074 }
4075 4075
4076 /** 4076 /**
4077 * atomic_long_or() - atomic bitwise OR with 4077 * atomic_long_or() - atomic bitwise OR with relaxed ordering
4078 * @i: long value 4078 * @i: long value
4079 * @v: pointer to atomic_long_t 4079 * @v: pointer to atomic_long_t
4080 * 4080 *
4081 * Atomically updates @v to (@v | @i) with re 4081 * Atomically updates @v to (@v | @i) with relaxed ordering.
4082 * 4082 *
4083 * Unsafe to use in noinstr code; use raw_ato 4083 * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
4084 * 4084 *
4085 * Return: Nothing. 4085 * Return: Nothing.
4086 */ 4086 */
4087 static __always_inline void 4087 static __always_inline void
4088 atomic_long_or(long i, atomic_long_t *v) 4088 atomic_long_or(long i, atomic_long_t *v)
4089 { 4089 {
4090 instrument_atomic_read_write(v, sizeo 4090 instrument_atomic_read_write(v, sizeof(*v));
4091 raw_atomic_long_or(i, v); 4091 raw_atomic_long_or(i, v);
4092 } 4092 }
4093 4093
4094 /** 4094 /**
4095 * atomic_long_fetch_or() - atomic bitwise OR 4095 * atomic_long_fetch_or() - atomic bitwise OR with full ordering
4096 * @i: long value 4096 * @i: long value
4097 * @v: pointer to atomic_long_t 4097 * @v: pointer to atomic_long_t
4098 * 4098 *
4099 * Atomically updates @v to (@v | @i) with fu 4099 * Atomically updates @v to (@v | @i) with full ordering.
4100 * 4100 *
4101 * Unsafe to use in noinstr code; use raw_ato 4101 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
4102 * 4102 *
4103 * Return: The original value of @v. 4103 * Return: The original value of @v.
4104 */ 4104 */
4105 static __always_inline long 4105 static __always_inline long
4106 atomic_long_fetch_or(long i, atomic_long_t *v 4106 atomic_long_fetch_or(long i, atomic_long_t *v)
4107 { 4107 {
4108 kcsan_mb(); 4108 kcsan_mb();
4109 instrument_atomic_read_write(v, sizeo 4109 instrument_atomic_read_write(v, sizeof(*v));
4110 return raw_atomic_long_fetch_or(i, v) 4110 return raw_atomic_long_fetch_or(i, v);
4111 } 4111 }
4112 4112
4113 /** 4113 /**
4114 * atomic_long_fetch_or_acquire() - atomic bi 4114 * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
4115 * @i: long value 4115 * @i: long value
4116 * @v: pointer to atomic_long_t 4116 * @v: pointer to atomic_long_t
4117 * 4117 *
4118 * Atomically updates @v to (@v | @i) with ac 4118 * Atomically updates @v to (@v | @i) with acquire ordering.
4119 * 4119 *
4120 * Unsafe to use in noinstr code; use raw_ato 4120 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
4121 * 4121 *
4122 * Return: The original value of @v. 4122 * Return: The original value of @v.
4123 */ 4123 */
4124 static __always_inline long 4124 static __always_inline long
4125 atomic_long_fetch_or_acquire(long i, atomic_l 4125 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
4126 { 4126 {
4127 instrument_atomic_read_write(v, sizeo 4127 instrument_atomic_read_write(v, sizeof(*v));
4128 return raw_atomic_long_fetch_or_acqui 4128 return raw_atomic_long_fetch_or_acquire(i, v);
4129 } 4129 }
4130 4130
4131 /** 4131 /**
4132 * atomic_long_fetch_or_release() - atomic bi 4132 * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
4133 * @i: long value 4133 * @i: long value
4134 * @v: pointer to atomic_long_t 4134 * @v: pointer to atomic_long_t
4135 * 4135 *
4136 * Atomically updates @v to (@v | @i) with re 4136 * Atomically updates @v to (@v | @i) with release ordering.
4137 * 4137 *
4138 * Unsafe to use in noinstr code; use raw_ato 4138 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
4139 * 4139 *
4140 * Return: The original value of @v. 4140 * Return: The original value of @v.
4141 */ 4141 */
4142 static __always_inline long 4142 static __always_inline long
4143 atomic_long_fetch_or_release(long i, atomic_l 4143 atomic_long_fetch_or_release(long i, atomic_long_t *v)
4144 { 4144 {
4145 kcsan_release(); 4145 kcsan_release();
4146 instrument_atomic_read_write(v, sizeo 4146 instrument_atomic_read_write(v, sizeof(*v));
4147 return raw_atomic_long_fetch_or_relea 4147 return raw_atomic_long_fetch_or_release(i, v);
4148 } 4148 }
4149 4149
4150 /** 4150 /**
4151 * atomic_long_fetch_or_relaxed() - atomic bi 4151 * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
4152 * @i: long value 4152 * @i: long value
4153 * @v: pointer to atomic_long_t 4153 * @v: pointer to atomic_long_t
4154 * 4154 *
4155 * Atomically updates @v to (@v | @i) with re 4155 * Atomically updates @v to (@v | @i) with relaxed ordering.
4156 * 4156 *
4157 * Unsafe to use in noinstr code; use raw_ato 4157 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
4158 * 4158 *
4159 * Return: The original value of @v. 4159 * Return: The original value of @v.
4160 */ 4160 */
4161 static __always_inline long 4161 static __always_inline long
4162 atomic_long_fetch_or_relaxed(long i, atomic_l 4162 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
4163 { 4163 {
4164 instrument_atomic_read_write(v, sizeo 4164 instrument_atomic_read_write(v, sizeof(*v));
4165 return raw_atomic_long_fetch_or_relax 4165 return raw_atomic_long_fetch_or_relaxed(i, v);
4166 } 4166 }
4167 4167
4168 /** 4168 /**
4169 * atomic_long_xor() - atomic bitwise XOR wit 4169 * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
4170 * @i: long value 4170 * @i: long value
4171 * @v: pointer to atomic_long_t 4171 * @v: pointer to atomic_long_t
4172 * 4172 *
4173 * Atomically updates @v to (@v ^ @i) with re 4173 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4174 * 4174 *
4175 * Unsafe to use in noinstr code; use raw_ato 4175 * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
4176 * 4176 *
4177 * Return: Nothing. 4177 * Return: Nothing.
4178 */ 4178 */
4179 static __always_inline void 4179 static __always_inline void
4180 atomic_long_xor(long i, atomic_long_t *v) 4180 atomic_long_xor(long i, atomic_long_t *v)
4181 { 4181 {
4182 instrument_atomic_read_write(v, sizeo 4182 instrument_atomic_read_write(v, sizeof(*v));
4183 raw_atomic_long_xor(i, v); 4183 raw_atomic_long_xor(i, v);
4184 } 4184 }
4185 4185
4186 /** 4186 /**
4187 * atomic_long_fetch_xor() - atomic bitwise X 4187 * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
4188 * @i: long value 4188 * @i: long value
4189 * @v: pointer to atomic_long_t 4189 * @v: pointer to atomic_long_t
4190 * 4190 *
4191 * Atomically updates @v to (@v ^ @i) with fu 4191 * Atomically updates @v to (@v ^ @i) with full ordering.
4192 * 4192 *
4193 * Unsafe to use in noinstr code; use raw_ato 4193 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
4194 * 4194 *
4195 * Return: The original value of @v. 4195 * Return: The original value of @v.
4196 */ 4196 */
4197 static __always_inline long 4197 static __always_inline long
4198 atomic_long_fetch_xor(long i, atomic_long_t * 4198 atomic_long_fetch_xor(long i, atomic_long_t *v)
4199 { 4199 {
4200 kcsan_mb(); 4200 kcsan_mb();
4201 instrument_atomic_read_write(v, sizeo 4201 instrument_atomic_read_write(v, sizeof(*v));
4202 return raw_atomic_long_fetch_xor(i, v 4202 return raw_atomic_long_fetch_xor(i, v);
4203 } 4203 }
4204 4204
4205 /** 4205 /**
4206 * atomic_long_fetch_xor_acquire() - atomic b 4206 * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
4207 * @i: long value 4207 * @i: long value
4208 * @v: pointer to atomic_long_t 4208 * @v: pointer to atomic_long_t
4209 * 4209 *
4210 * Atomically updates @v to (@v ^ @i) with ac 4210 * Atomically updates @v to (@v ^ @i) with acquire ordering.
4211 * 4211 *
4212 * Unsafe to use in noinstr code; use raw_ato 4212 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
4213 * 4213 *
4214 * Return: The original value of @v. 4214 * Return: The original value of @v.
4215 */ 4215 */
4216 static __always_inline long 4216 static __always_inline long
4217 atomic_long_fetch_xor_acquire(long i, atomic_ 4217 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
4218 { 4218 {
4219 instrument_atomic_read_write(v, sizeo 4219 instrument_atomic_read_write(v, sizeof(*v));
4220 return raw_atomic_long_fetch_xor_acqu 4220 return raw_atomic_long_fetch_xor_acquire(i, v);
4221 } 4221 }
4222 4222
4223 /** 4223 /**
4224 * atomic_long_fetch_xor_release() - atomic b 4224 * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
4225 * @i: long value 4225 * @i: long value
4226 * @v: pointer to atomic_long_t 4226 * @v: pointer to atomic_long_t
4227 * 4227 *
4228 * Atomically updates @v to (@v ^ @i) with re 4228 * Atomically updates @v to (@v ^ @i) with release ordering.
4229 * 4229 *
4230 * Unsafe to use in noinstr code; use raw_ato 4230 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
4231 * 4231 *
4232 * Return: The original value of @v. 4232 * Return: The original value of @v.
4233 */ 4233 */
4234 static __always_inline long 4234 static __always_inline long
4235 atomic_long_fetch_xor_release(long i, atomic_ 4235 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
4236 { 4236 {
4237 kcsan_release(); 4237 kcsan_release();
4238 instrument_atomic_read_write(v, sizeo 4238 instrument_atomic_read_write(v, sizeof(*v));
4239 return raw_atomic_long_fetch_xor_rele 4239 return raw_atomic_long_fetch_xor_release(i, v);
4240 } 4240 }
4241 4241
4242 /** 4242 /**
4243 * atomic_long_fetch_xor_relaxed() - atomic b 4243 * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
4244 * @i: long value 4244 * @i: long value
4245 * @v: pointer to atomic_long_t 4245 * @v: pointer to atomic_long_t
4246 * 4246 *
4247 * Atomically updates @v to (@v ^ @i) with re 4247 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4248 * 4248 *
4249 * Unsafe to use in noinstr code; use raw_ato 4249 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
4250 * 4250 *
4251 * Return: The original value of @v. 4251 * Return: The original value of @v.
4252 */ 4252 */
4253 static __always_inline long 4253 static __always_inline long
4254 atomic_long_fetch_xor_relaxed(long i, atomic_ 4254 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
4255 { 4255 {
4256 instrument_atomic_read_write(v, sizeo 4256 instrument_atomic_read_write(v, sizeof(*v));
4257 return raw_atomic_long_fetch_xor_rela 4257 return raw_atomic_long_fetch_xor_relaxed(i, v);
4258 } 4258 }
4259 4259
4260 /** 4260 /**
4261 * atomic_long_xchg() - atomic exchange with 4261 * atomic_long_xchg() - atomic exchange with full ordering
4262 * @v: pointer to atomic_long_t 4262 * @v: pointer to atomic_long_t
4263 * @new: long value to assign 4263 * @new: long value to assign
4264 * 4264 *
4265 * Atomically updates @v to @new with full or 4265 * Atomically updates @v to @new with full ordering.
4266 * 4266 *
4267 * Unsafe to use in noinstr code; use raw_ato 4267 * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
4268 * 4268 *
4269 * Return: The original value of @v. 4269 * Return: The original value of @v.
4270 */ 4270 */
4271 static __always_inline long 4271 static __always_inline long
4272 atomic_long_xchg(atomic_long_t *v, long new) 4272 atomic_long_xchg(atomic_long_t *v, long new)
4273 { 4273 {
4274 kcsan_mb(); 4274 kcsan_mb();
4275 instrument_atomic_read_write(v, sizeo 4275 instrument_atomic_read_write(v, sizeof(*v));
4276 return raw_atomic_long_xchg(v, new); 4276 return raw_atomic_long_xchg(v, new);
4277 } 4277 }
4278 4278
4279 /** 4279 /**
4280 * atomic_long_xchg_acquire() - atomic exchan 4280 * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
4281 * @v: pointer to atomic_long_t 4281 * @v: pointer to atomic_long_t
4282 * @new: long value to assign 4282 * @new: long value to assign
4283 * 4283 *
4284 * Atomically updates @v to @new with acquire 4284 * Atomically updates @v to @new with acquire ordering.
4285 * 4285 *
4286 * Unsafe to use in noinstr code; use raw_ato 4286 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
4287 * 4287 *
4288 * Return: The original value of @v. 4288 * Return: The original value of @v.
4289 */ 4289 */
4290 static __always_inline long 4290 static __always_inline long
4291 atomic_long_xchg_acquire(atomic_long_t *v, lo 4291 atomic_long_xchg_acquire(atomic_long_t *v, long new)
4292 { 4292 {
4293 instrument_atomic_read_write(v, sizeo 4293 instrument_atomic_read_write(v, sizeof(*v));
4294 return raw_atomic_long_xchg_acquire(v 4294 return raw_atomic_long_xchg_acquire(v, new);
4295 } 4295 }
4296 4296
4297 /** 4297 /**
4298 * atomic_long_xchg_release() - atomic exchan 4298 * atomic_long_xchg_release() - atomic exchange with release ordering
4299 * @v: pointer to atomic_long_t 4299 * @v: pointer to atomic_long_t
4300 * @new: long value to assign 4300 * @new: long value to assign
4301 * 4301 *
4302 * Atomically updates @v to @new with release 4302 * Atomically updates @v to @new with release ordering.
4303 * 4303 *
4304 * Unsafe to use in noinstr code; use raw_ato 4304 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
4305 * 4305 *
4306 * Return: The original value of @v. 4306 * Return: The original value of @v.
4307 */ 4307 */
4308 static __always_inline long 4308 static __always_inline long
4309 atomic_long_xchg_release(atomic_long_t *v, lo 4309 atomic_long_xchg_release(atomic_long_t *v, long new)
4310 { 4310 {
4311 kcsan_release(); 4311 kcsan_release();
4312 instrument_atomic_read_write(v, sizeo 4312 instrument_atomic_read_write(v, sizeof(*v));
4313 return raw_atomic_long_xchg_release(v 4313 return raw_atomic_long_xchg_release(v, new);
4314 } 4314 }
4315 4315
4316 /** 4316 /**
4317 * atomic_long_xchg_relaxed() - atomic exchan 4317 * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
4318 * @v: pointer to atomic_long_t 4318 * @v: pointer to atomic_long_t
4319 * @new: long value to assign 4319 * @new: long value to assign
4320 * 4320 *
4321 * Atomically updates @v to @new with relaxed 4321 * Atomically updates @v to @new with relaxed ordering.
4322 * 4322 *
4323 * Unsafe to use in noinstr code; use raw_ato 4323 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
4324 * 4324 *
4325 * Return: The original value of @v. 4325 * Return: The original value of @v.
4326 */ 4326 */
4327 static __always_inline long 4327 static __always_inline long
4328 atomic_long_xchg_relaxed(atomic_long_t *v, lo 4328 atomic_long_xchg_relaxed(atomic_long_t *v, long new)
4329 { 4329 {
4330 instrument_atomic_read_write(v, sizeo 4330 instrument_atomic_read_write(v, sizeof(*v));
4331 return raw_atomic_long_xchg_relaxed(v 4331 return raw_atomic_long_xchg_relaxed(v, new);
4332 } 4332 }
4333 4333
4334 /** 4334 /**
4335 * atomic_long_cmpxchg() - atomic compare and 4335 * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
4336 * @v: pointer to atomic_long_t 4336 * @v: pointer to atomic_long_t
4337 * @old: long value to compare with 4337 * @old: long value to compare with
4338 * @new: long value to assign 4338 * @new: long value to assign
4339 * 4339 *
4340 * If (@v == @old), atomically updates @v to 4340 * If (@v == @old), atomically updates @v to @new with full ordering.
4341 * Otherwise, @v is not modified and relaxed 4341 * Otherwise, @v is not modified and relaxed ordering is provided.
4342 * 4342 *
4343 * Unsafe to use in noinstr code; use raw_ato 4343 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
4344 * 4344 *
4345 * Return: The original value of @v. 4345 * Return: The original value of @v.
4346 */ 4346 */
4347 static __always_inline long 4347 static __always_inline long
4348 atomic_long_cmpxchg(atomic_long_t *v, long ol 4348 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
4349 { 4349 {
4350 kcsan_mb(); 4350 kcsan_mb();
4351 instrument_atomic_read_write(v, sizeo 4351 instrument_atomic_read_write(v, sizeof(*v));
4352 return raw_atomic_long_cmpxchg(v, old 4352 return raw_atomic_long_cmpxchg(v, old, new);
4353 } 4353 }
4354 4354
4355 /** 4355 /**
4356 * atomic_long_cmpxchg_acquire() - atomic com 4356 * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4357 * @v: pointer to atomic_long_t 4357 * @v: pointer to atomic_long_t
4358 * @old: long value to compare with 4358 * @old: long value to compare with
4359 * @new: long value to assign 4359 * @new: long value to assign
4360 * 4360 *
4361 * If (@v == @old), atomically updates @v to 4361 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4362 * Otherwise, @v is not modified and relaxed 4362 * Otherwise, @v is not modified and relaxed ordering is provided.
4363 * 4363 *
4364 * Unsafe to use in noinstr code; use raw_ato 4364 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
4365 * 4365 *
4366 * Return: The original value of @v. 4366 * Return: The original value of @v.
4367 */ 4367 */
4368 static __always_inline long 4368 static __always_inline long
4369 atomic_long_cmpxchg_acquire(atomic_long_t *v, 4369 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
4370 { 4370 {
4371 instrument_atomic_read_write(v, sizeo 4371 instrument_atomic_read_write(v, sizeof(*v));
4372 return raw_atomic_long_cmpxchg_acquir 4372 return raw_atomic_long_cmpxchg_acquire(v, old, new);
4373 } 4373 }
4374 4374
4375 /** 4375 /**
4376 * atomic_long_cmpxchg_release() - atomic com 4376 * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
4377 * @v: pointer to atomic_long_t 4377 * @v: pointer to atomic_long_t
4378 * @old: long value to compare with 4378 * @old: long value to compare with
4379 * @new: long value to assign 4379 * @new: long value to assign
4380 * 4380 *
4381 * If (@v == @old), atomically updates @v to 4381 * If (@v == @old), atomically updates @v to @new with release ordering.
4382 * Otherwise, @v is not modified and relaxed 4382 * Otherwise, @v is not modified and relaxed ordering is provided.
4383 * 4383 *
4384 * Unsafe to use in noinstr code; use raw_ato 4384 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
4385 * 4385 *
4386 * Return: The original value of @v. 4386 * Return: The original value of @v.
4387 */ 4387 */
4388 static __always_inline long 4388 static __always_inline long
4389 atomic_long_cmpxchg_release(atomic_long_t *v, 4389 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
4390 { 4390 {
4391 kcsan_release(); 4391 kcsan_release();
4392 instrument_atomic_read_write(v, sizeo 4392 instrument_atomic_read_write(v, sizeof(*v));
4393 return raw_atomic_long_cmpxchg_releas 4393 return raw_atomic_long_cmpxchg_release(v, old, new);
4394 } 4394 }
4395 4395
4396 /** 4396 /**
4397 * atomic_long_cmpxchg_relaxed() - atomic com 4397 * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4398 * @v: pointer to atomic_long_t 4398 * @v: pointer to atomic_long_t
4399 * @old: long value to compare with 4399 * @old: long value to compare with
4400 * @new: long value to assign 4400 * @new: long value to assign
4401 * 4401 *
4402 * If (@v == @old), atomically updates @v to 4402 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4403 * Otherwise, @v is not modified and relaxed 4403 * Otherwise, @v is not modified and relaxed ordering is provided.
4404 * 4404 *
4405 * Unsafe to use in noinstr code; use raw_ato 4405 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
4406 * 4406 *
4407 * Return: The original value of @v. 4407 * Return: The original value of @v.
4408 */ 4408 */
4409 static __always_inline long 4409 static __always_inline long
4410 atomic_long_cmpxchg_relaxed(atomic_long_t *v, 4410 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
4411 { 4411 {
4412 instrument_atomic_read_write(v, sizeo 4412 instrument_atomic_read_write(v, sizeof(*v));
4413 return raw_atomic_long_cmpxchg_relaxe 4413 return raw_atomic_long_cmpxchg_relaxed(v, old, new);
4414 } 4414 }
4415 4415
4416 /** 4416 /**
4417 * atomic_long_try_cmpxchg() - atomic compare 4417 * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
4418 * @v: pointer to atomic_long_t 4418 * @v: pointer to atomic_long_t
4419 * @old: pointer to long value to compare wit 4419 * @old: pointer to long value to compare with
4420 * @new: long value to assign 4420 * @new: long value to assign
4421 * 4421 *
4422 * If (@v == @old), atomically updates @v to 4422 * If (@v == @old), atomically updates @v to @new with full ordering.
4423 * Otherwise, @v is not modified, @old is upd 4423 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4424 * and relaxed ordering is provided. 4424 * and relaxed ordering is provided.
4425 * 4425 *
4426 * Unsafe to use in noinstr code; use raw_ato 4426 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
4427 * 4427 *
4428 * Return: @true if the exchange occured, @fa 4428 * Return: @true if the exchange occured, @false otherwise.
4429 */ 4429 */
4430 static __always_inline bool 4430 static __always_inline bool
4431 atomic_long_try_cmpxchg(atomic_long_t *v, lon 4431 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
4432 { 4432 {
4433 kcsan_mb(); 4433 kcsan_mb();
4434 instrument_atomic_read_write(v, sizeo 4434 instrument_atomic_read_write(v, sizeof(*v));
4435 instrument_atomic_read_write(old, siz 4435 instrument_atomic_read_write(old, sizeof(*old));
4436 return raw_atomic_long_try_cmpxchg(v, 4436 return raw_atomic_long_try_cmpxchg(v, old, new);
4437 } 4437 }
4438 4438
4439 /** 4439 /**
4440 * atomic_long_try_cmpxchg_acquire() - atomic 4440 * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4441 * @v: pointer to atomic_long_t 4441 * @v: pointer to atomic_long_t
4442 * @old: pointer to long value to compare wit 4442 * @old: pointer to long value to compare with
4443 * @new: long value to assign 4443 * @new: long value to assign
4444 * 4444 *
4445 * If (@v == @old), atomically updates @v to 4445 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4446 * Otherwise, @v is not modified, @old is upd 4446 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4447 * and relaxed ordering is provided. 4447 * and relaxed ordering is provided.
4448 * 4448 *
4449 * Unsafe to use in noinstr code; use raw_ato 4449 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
4450 * 4450 *
4451 * Return: @true if the exchange occured, @fa 4451 * Return: @true if the exchange occured, @false otherwise.
4452 */ 4452 */
4453 static __always_inline bool 4453 static __always_inline bool
4454 atomic_long_try_cmpxchg_acquire(atomic_long_t 4454 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
4455 { 4455 {
4456 instrument_atomic_read_write(v, sizeo 4456 instrument_atomic_read_write(v, sizeof(*v));
4457 instrument_atomic_read_write(old, siz 4457 instrument_atomic_read_write(old, sizeof(*old));
4458 return raw_atomic_long_try_cmpxchg_ac 4458 return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
4459 } 4459 }
4460 4460
4461 /** 4461 /**
4462 * atomic_long_try_cmpxchg_release() - atomic 4462 * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
4463 * @v: pointer to atomic_long_t 4463 * @v: pointer to atomic_long_t
4464 * @old: pointer to long value to compare wit 4464 * @old: pointer to long value to compare with
4465 * @new: long value to assign 4465 * @new: long value to assign
4466 * 4466 *
4467 * If (@v == @old), atomically updates @v to 4467 * If (@v == @old), atomically updates @v to @new with release ordering.
4468 * Otherwise, @v is not modified, @old is upd 4468 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4469 * and relaxed ordering is provided. 4469 * and relaxed ordering is provided.
4470 * 4470 *
4471 * Unsafe to use in noinstr code; use raw_ato 4471 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
4472 * 4472 *
4473 * Return: @true if the exchange occured, @fa 4473 * Return: @true if the exchange occured, @false otherwise.
4474 */ 4474 */
4475 static __always_inline bool 4475 static __always_inline bool
4476 atomic_long_try_cmpxchg_release(atomic_long_t 4476 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
4477 { 4477 {
4478 kcsan_release(); 4478 kcsan_release();
4479 instrument_atomic_read_write(v, sizeo 4479 instrument_atomic_read_write(v, sizeof(*v));
4480 instrument_atomic_read_write(old, siz 4480 instrument_atomic_read_write(old, sizeof(*old));
4481 return raw_atomic_long_try_cmpxchg_re 4481 return raw_atomic_long_try_cmpxchg_release(v, old, new);
4482 } 4482 }
4483 4483
4484 /** 4484 /**
4485 * atomic_long_try_cmpxchg_relaxed() - atomic 4485 * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4486 * @v: pointer to atomic_long_t 4486 * @v: pointer to atomic_long_t
4487 * @old: pointer to long value to compare wit 4487 * @old: pointer to long value to compare with
4488 * @new: long value to assign 4488 * @new: long value to assign
4489 * 4489 *
4490 * If (@v == @old), atomically updates @v to 4490 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4491 * Otherwise, @v is not modified, @old is upd 4491 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4492 * and relaxed ordering is provided. 4492 * and relaxed ordering is provided.
4493 * 4493 *
4494 * Unsafe to use in noinstr code; use raw_ato 4494 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
4495 * 4495 *
4496 * Return: @true if the exchange occured, @fa 4496 * Return: @true if the exchange occured, @false otherwise.
4497 */ 4497 */
4498 static __always_inline bool 4498 static __always_inline bool
4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t 4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
4500 { 4500 {
4501 instrument_atomic_read_write(v, sizeo 4501 instrument_atomic_read_write(v, sizeof(*v));
4502 instrument_atomic_read_write(old, siz 4502 instrument_atomic_read_write(old, sizeof(*old));
4503 return raw_atomic_long_try_cmpxchg_re 4503 return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
4504 } 4504 }
4505 4505
4506 /** 4506 /**
4507 * atomic_long_sub_and_test() - atomic subtra 4507 * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
4508 * @i: long value to subtract 4508 * @i: long value to subtract
4509 * @v: pointer to atomic_long_t 4509 * @v: pointer to atomic_long_t
4510 * 4510 *
4511 * Atomically updates @v to (@v - @i) with fu 4511 * Atomically updates @v to (@v - @i) with full ordering.
4512 * 4512 *
4513 * Unsafe to use in noinstr code; use raw_ato 4513 * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
4514 * 4514 *
4515 * Return: @true if the resulting value of @v 4515 * Return: @true if the resulting value of @v is zero, @false otherwise.
4516 */ 4516 */
4517 static __always_inline bool 4517 static __always_inline bool
4518 atomic_long_sub_and_test(long i, atomic_long_ 4518 atomic_long_sub_and_test(long i, atomic_long_t *v)
4519 { 4519 {
4520 kcsan_mb(); 4520 kcsan_mb();
4521 instrument_atomic_read_write(v, sizeo 4521 instrument_atomic_read_write(v, sizeof(*v));
4522 return raw_atomic_long_sub_and_test(i 4522 return raw_atomic_long_sub_and_test(i, v);
4523 } 4523 }
4524 4524
4525 /** 4525 /**
4526 * atomic_long_dec_and_test() - atomic decrem 4526 * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
4527 * @v: pointer to atomic_long_t 4527 * @v: pointer to atomic_long_t
4528 * 4528 *
4529 * Atomically updates @v to (@v - 1) with ful 4529 * Atomically updates @v to (@v - 1) with full ordering.
4530 * 4530 *
4531 * Unsafe to use in noinstr code; use raw_ato 4531 * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
4532 * 4532 *
4533 * Return: @true if the resulting value of @v 4533 * Return: @true if the resulting value of @v is zero, @false otherwise.
4534 */ 4534 */
4535 static __always_inline bool 4535 static __always_inline bool
4536 atomic_long_dec_and_test(atomic_long_t *v) 4536 atomic_long_dec_and_test(atomic_long_t *v)
4537 { 4537 {
4538 kcsan_mb(); 4538 kcsan_mb();
4539 instrument_atomic_read_write(v, sizeo 4539 instrument_atomic_read_write(v, sizeof(*v));
4540 return raw_atomic_long_dec_and_test(v 4540 return raw_atomic_long_dec_and_test(v);
4541 } 4541 }
4542 4542
4543 /** 4543 /**
4544 * atomic_long_inc_and_test() - atomic increm 4544 * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
4545 * @v: pointer to atomic_long_t 4545 * @v: pointer to atomic_long_t
4546 * 4546 *
4547 * Atomically updates @v to (@v + 1) with ful 4547 * Atomically updates @v to (@v + 1) with full ordering.
4548 * 4548 *
4549 * Unsafe to use in noinstr code; use raw_ato 4549 * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
4550 * 4550 *
4551 * Return: @true if the resulting value of @v 4551 * Return: @true if the resulting value of @v is zero, @false otherwise.
4552 */ 4552 */
4553 static __always_inline bool 4553 static __always_inline bool
4554 atomic_long_inc_and_test(atomic_long_t *v) 4554 atomic_long_inc_and_test(atomic_long_t *v)
4555 { 4555 {
4556 kcsan_mb(); 4556 kcsan_mb();
4557 instrument_atomic_read_write(v, sizeo 4557 instrument_atomic_read_write(v, sizeof(*v));
4558 return raw_atomic_long_inc_and_test(v 4558 return raw_atomic_long_inc_and_test(v);
4559 } 4559 }
4560 4560
4561 /** 4561 /**
4562 * atomic_long_add_negative() - atomic add an 4562 * atomic_long_add_negative() - atomic add and test if negative with full ordering
4563 * @i: long value to add 4563 * @i: long value to add
4564 * @v: pointer to atomic_long_t 4564 * @v: pointer to atomic_long_t
4565 * 4565 *
4566 * Atomically updates @v to (@v + @i) with fu 4566 * Atomically updates @v to (@v + @i) with full ordering.
4567 * 4567 *
4568 * Unsafe to use in noinstr code; use raw_ato 4568 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
4569 * 4569 *
4570 * Return: @true if the resulting value of @v 4570 * Return: @true if the resulting value of @v is negative, @false otherwise.
4571 */ 4571 */
4572 static __always_inline bool 4572 static __always_inline bool
4573 atomic_long_add_negative(long i, atomic_long_ 4573 atomic_long_add_negative(long i, atomic_long_t *v)
4574 { 4574 {
4575 kcsan_mb(); 4575 kcsan_mb();
4576 instrument_atomic_read_write(v, sizeo 4576 instrument_atomic_read_write(v, sizeof(*v));
4577 return raw_atomic_long_add_negative(i 4577 return raw_atomic_long_add_negative(i, v);
4578 } 4578 }
4579 4579
4580 /** 4580 /**
4581 * atomic_long_add_negative_acquire() - atomi 4581 * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
4582 * @i: long value to add 4582 * @i: long value to add
4583 * @v: pointer to atomic_long_t 4583 * @v: pointer to atomic_long_t
4584 * 4584 *
4585 * Atomically updates @v to (@v + @i) with ac 4585 * Atomically updates @v to (@v + @i) with acquire ordering.
4586 * 4586 *
4587 * Unsafe to use in noinstr code; use raw_ato 4587 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
4588 * 4588 *
4589 * Return: @true if the resulting value of @v 4589 * Return: @true if the resulting value of @v is negative, @false otherwise.
4590 */ 4590 */
4591 static __always_inline bool 4591 static __always_inline bool
4592 atomic_long_add_negative_acquire(long i, atom 4592 atomic_long_add_negative_acquire(long i, atomic_long_t *v)
4593 { 4593 {
4594 instrument_atomic_read_write(v, sizeo 4594 instrument_atomic_read_write(v, sizeof(*v));
4595 return raw_atomic_long_add_negative_a 4595 return raw_atomic_long_add_negative_acquire(i, v);
4596 } 4596 }
4597 4597
4598 /** 4598 /**
4599 * atomic_long_add_negative_release() - atomi 4599 * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
4600 * @i: long value to add 4600 * @i: long value to add
4601 * @v: pointer to atomic_long_t 4601 * @v: pointer to atomic_long_t
4602 * 4602 *
4603 * Atomically updates @v to (@v + @i) with re 4603 * Atomically updates @v to (@v + @i) with release ordering.
4604 * 4604 *
4605 * Unsafe to use in noinstr code; use raw_ato 4605 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
4606 * 4606 *
4607 * Return: @true if the resulting value of @v 4607 * Return: @true if the resulting value of @v is negative, @false otherwise.
4608 */ 4608 */
4609 static __always_inline bool 4609 static __always_inline bool
4610 atomic_long_add_negative_release(long i, atom 4610 atomic_long_add_negative_release(long i, atomic_long_t *v)
4611 { 4611 {
4612 kcsan_release(); 4612 kcsan_release();
4613 instrument_atomic_read_write(v, sizeo 4613 instrument_atomic_read_write(v, sizeof(*v));
4614 return raw_atomic_long_add_negative_r 4614 return raw_atomic_long_add_negative_release(i, v);
4615 } 4615 }
4616 4616
4617 /** 4617 /**
4618 * atomic_long_add_negative_relaxed() - atomi 4618 * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4619 * @i: long value to add 4619 * @i: long value to add
4620 * @v: pointer to atomic_long_t 4620 * @v: pointer to atomic_long_t
4621 * 4621 *
4622 * Atomically updates @v to (@v + @i) with re 4622 * Atomically updates @v to (@v + @i) with relaxed ordering.
4623 * 4623 *
4624 * Unsafe to use in noinstr code; use raw_ato 4624 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
4625 * 4625 *
4626 * Return: @true if the resulting value of @v 4626 * Return: @true if the resulting value of @v is negative, @false otherwise.
4627 */ 4627 */
4628 static __always_inline bool 4628 static __always_inline bool
4629 atomic_long_add_negative_relaxed(long i, atom 4629 atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
4630 { 4630 {
4631 instrument_atomic_read_write(v, sizeo 4631 instrument_atomic_read_write(v, sizeof(*v));
4632 return raw_atomic_long_add_negative_r 4632 return raw_atomic_long_add_negative_relaxed(i, v);
4633 } 4633 }
4634 4634
4635 /** 4635 /**
4636 * atomic_long_fetch_add_unless() - atomic ad 4636 * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
4637 * @v: pointer to atomic_long_t 4637 * @v: pointer to atomic_long_t
4638 * @a: long value to add 4638 * @a: long value to add
4639 * @u: long value to compare with 4639 * @u: long value to compare with
4640 * 4640 *
4641 * If (@v != @u), atomically updates @v to (@ 4641 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4642 * Otherwise, @v is not modified and relaxed 4642 * Otherwise, @v is not modified and relaxed ordering is provided.
4643 * 4643 *
4644 * Unsafe to use in noinstr code; use raw_ato 4644 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
4645 * 4645 *
4646 * Return: The original value of @v. 4646 * Return: The original value of @v.
4647 */ 4647 */
4648 static __always_inline long 4648 static __always_inline long
4649 atomic_long_fetch_add_unless(atomic_long_t *v 4649 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
4650 { 4650 {
4651 kcsan_mb(); 4651 kcsan_mb();
4652 instrument_atomic_read_write(v, sizeo 4652 instrument_atomic_read_write(v, sizeof(*v));
4653 return raw_atomic_long_fetch_add_unle 4653 return raw_atomic_long_fetch_add_unless(v, a, u);
4654 } 4654 }
4655 4655
4656 /** 4656 /**
4657 * atomic_long_add_unless() - atomic add unle 4657 * atomic_long_add_unless() - atomic add unless value with full ordering
4658 * @v: pointer to atomic_long_t 4658 * @v: pointer to atomic_long_t
4659 * @a: long value to add 4659 * @a: long value to add
4660 * @u: long value to compare with 4660 * @u: long value to compare with
4661 * 4661 *
4662 * If (@v != @u), atomically updates @v to (@ 4662 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4663 * Otherwise, @v is not modified and relaxed 4663 * Otherwise, @v is not modified and relaxed ordering is provided.
4664 * 4664 *
4665 * Unsafe to use in noinstr code; use raw_ato 4665 * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
4666 * 4666 *
4667 * Return: @true if @v was updated, @false ot 4667 * Return: @true if @v was updated, @false otherwise.
4668 */ 4668 */
4669 static __always_inline bool 4669 static __always_inline bool
4670 atomic_long_add_unless(atomic_long_t *v, long 4670 atomic_long_add_unless(atomic_long_t *v, long a, long u)
4671 { 4671 {
4672 kcsan_mb(); 4672 kcsan_mb();
4673 instrument_atomic_read_write(v, sizeo 4673 instrument_atomic_read_write(v, sizeof(*v));
4674 return raw_atomic_long_add_unless(v, 4674 return raw_atomic_long_add_unless(v, a, u);
4675 } 4675 }
4676 4676
4677 /** 4677 /**
4678 * atomic_long_inc_not_zero() - atomic increm 4678 * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
4679 * @v: pointer to atomic_long_t 4679 * @v: pointer to atomic_long_t
4680 * 4680 *
4681 * If (@v != 0), atomically updates @v to (@v 4681 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4682 * Otherwise, @v is not modified and relaxed 4682 * Otherwise, @v is not modified and relaxed ordering is provided.
4683 * 4683 *
4684 * Unsafe to use in noinstr code; use raw_ato 4684 * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
4685 * 4685 *
4686 * Return: @true if @v was updated, @false ot 4686 * Return: @true if @v was updated, @false otherwise.
4687 */ 4687 */
4688 static __always_inline bool 4688 static __always_inline bool
4689 atomic_long_inc_not_zero(atomic_long_t *v) 4689 atomic_long_inc_not_zero(atomic_long_t *v)
4690 { 4690 {
4691 kcsan_mb(); 4691 kcsan_mb();
4692 instrument_atomic_read_write(v, sizeo 4692 instrument_atomic_read_write(v, sizeof(*v));
4693 return raw_atomic_long_inc_not_zero(v 4693 return raw_atomic_long_inc_not_zero(v);
4694 } 4694 }
4695 4695
4696 /** 4696 /**
4697 * atomic_long_inc_unless_negative() - atomic 4697 * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
4698 * @v: pointer to atomic_long_t 4698 * @v: pointer to atomic_long_t
4699 * 4699 *
4700 * If (@v >= 0), atomically updates @v to (@v 4700 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4701 * Otherwise, @v is not modified and relaxed 4701 * Otherwise, @v is not modified and relaxed ordering is provided.
4702 * 4702 *
4703 * Unsafe to use in noinstr code; use raw_ato 4703 * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
4704 * 4704 *
4705 * Return: @true if @v was updated, @false ot 4705 * Return: @true if @v was updated, @false otherwise.
4706 */ 4706 */
4707 static __always_inline bool 4707 static __always_inline bool
4708 atomic_long_inc_unless_negative(atomic_long_t 4708 atomic_long_inc_unless_negative(atomic_long_t *v)
4709 { 4709 {
4710 kcsan_mb(); 4710 kcsan_mb();
4711 instrument_atomic_read_write(v, sizeo 4711 instrument_atomic_read_write(v, sizeof(*v));
4712 return raw_atomic_long_inc_unless_neg 4712 return raw_atomic_long_inc_unless_negative(v);
4713 } 4713 }
4714 4714
4715 /** 4715 /**
4716 * atomic_long_dec_unless_positive() - atomic 4716 * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
4717 * @v: pointer to atomic_long_t 4717 * @v: pointer to atomic_long_t
4718 * 4718 *
4719 * If (@v <= 0), atomically updates @v to (@v 4719 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4720 * Otherwise, @v is not modified and relaxed 4720 * Otherwise, @v is not modified and relaxed ordering is provided.
4721 * 4721 *
4722 * Unsafe to use in noinstr code; use raw_ato 4722 * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
4723 * 4723 *
4724 * Return: @true if @v was updated, @false ot 4724 * Return: @true if @v was updated, @false otherwise.
4725 */ 4725 */
4726 static __always_inline bool 4726 static __always_inline bool
4727 atomic_long_dec_unless_positive(atomic_long_t 4727 atomic_long_dec_unless_positive(atomic_long_t *v)
4728 { 4728 {
4729 kcsan_mb(); 4729 kcsan_mb();
4730 instrument_atomic_read_write(v, sizeo 4730 instrument_atomic_read_write(v, sizeof(*v));
4731 return raw_atomic_long_dec_unless_pos 4731 return raw_atomic_long_dec_unless_positive(v);
4732 } 4732 }
4733 4733
4734 /** 4734 /**
4735 * atomic_long_dec_if_positive() - atomic dec 4735 * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
4736 * @v: pointer to atomic_long_t 4736 * @v: pointer to atomic_long_t
4737 * 4737 *
4738 * If (@v > 0), atomically updates @v to (@v 4738 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4739 * Otherwise, @v is not modified and relaxed 4739 * Otherwise, @v is not modified and relaxed ordering is provided.
4740 * 4740 *
4741 * Unsafe to use in noinstr code; use raw_ato 4741 * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
4742 * 4742 *
4743 * Return: The old value of (@v - 1), regardl 4743 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4744 */ 4744 */
4745 static __always_inline long 4745 static __always_inline long
4746 atomic_long_dec_if_positive(atomic_long_t *v) 4746 atomic_long_dec_if_positive(atomic_long_t *v)
4747 { 4747 {
4748 kcsan_mb(); 4748 kcsan_mb();
4749 instrument_atomic_read_write(v, sizeo 4749 instrument_atomic_read_write(v, sizeof(*v));
4750 return raw_atomic_long_dec_if_positiv 4750 return raw_atomic_long_dec_if_positive(v);
4751 } 4751 }
4752 4752
4753 #define xchg(ptr, ...) \ 4753 #define xchg(ptr, ...) \
4754 ({ \ 4754 ({ \
4755 typeof(ptr) __ai_ptr = (ptr); \ 4755 typeof(ptr) __ai_ptr = (ptr); \
4756 kcsan_mb(); \ 4756 kcsan_mb(); \
4757 instrument_atomic_read_write(__ai_ptr 4757 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4758 raw_xchg(__ai_ptr, __VA_ARGS__); \ 4758 raw_xchg(__ai_ptr, __VA_ARGS__); \
4759 }) 4759 })
4760 4760
4761 #define xchg_acquire(ptr, ...) \ 4761 #define xchg_acquire(ptr, ...) \
4762 ({ \ 4762 ({ \
4763 typeof(ptr) __ai_ptr = (ptr); \ 4763 typeof(ptr) __ai_ptr = (ptr); \
4764 instrument_atomic_read_write(__ai_ptr 4764 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4765 raw_xchg_acquire(__ai_ptr, __VA_ARGS_ 4765 raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \
4766 }) 4766 })
4767 4767
4768 #define xchg_release(ptr, ...) \ 4768 #define xchg_release(ptr, ...) \
4769 ({ \ 4769 ({ \
4770 typeof(ptr) __ai_ptr = (ptr); \ 4770 typeof(ptr) __ai_ptr = (ptr); \
4771 kcsan_release(); \ 4771 kcsan_release(); \
4772 instrument_atomic_read_write(__ai_ptr 4772 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4773 raw_xchg_release(__ai_ptr, __VA_ARGS_ 4773 raw_xchg_release(__ai_ptr, __VA_ARGS__); \
4774 }) 4774 })
4775 4775
4776 #define xchg_relaxed(ptr, ...) \ 4776 #define xchg_relaxed(ptr, ...) \
4777 ({ \ 4777 ({ \
4778 typeof(ptr) __ai_ptr = (ptr); \ 4778 typeof(ptr) __ai_ptr = (ptr); \
4779 instrument_atomic_read_write(__ai_ptr 4779 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4780 raw_xchg_relaxed(__ai_ptr, __VA_ARGS_ 4780 raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
4781 }) 4781 })
4782 4782
4783 #define cmpxchg(ptr, ...) \ 4783 #define cmpxchg(ptr, ...) \
4784 ({ \ 4784 ({ \
4785 typeof(ptr) __ai_ptr = (ptr); \ 4785 typeof(ptr) __ai_ptr = (ptr); \
4786 kcsan_mb(); \ 4786 kcsan_mb(); \
4787 instrument_atomic_read_write(__ai_ptr 4787 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4788 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \ 4788 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
4789 }) 4789 })
4790 4790
4791 #define cmpxchg_acquire(ptr, ...) \ 4791 #define cmpxchg_acquire(ptr, ...) \
4792 ({ \ 4792 ({ \
4793 typeof(ptr) __ai_ptr = (ptr); \ 4793 typeof(ptr) __ai_ptr = (ptr); \
4794 instrument_atomic_read_write(__ai_ptr 4794 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4795 raw_cmpxchg_acquire(__ai_ptr, __VA_AR 4795 raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
4796 }) 4796 })
4797 4797
4798 #define cmpxchg_release(ptr, ...) \ 4798 #define cmpxchg_release(ptr, ...) \
4799 ({ \ 4799 ({ \
4800 typeof(ptr) __ai_ptr = (ptr); \ 4800 typeof(ptr) __ai_ptr = (ptr); \
4801 kcsan_release(); \ 4801 kcsan_release(); \
4802 instrument_atomic_read_write(__ai_ptr 4802 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4803 raw_cmpxchg_release(__ai_ptr, __VA_AR 4803 raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
4804 }) 4804 })
4805 4805
4806 #define cmpxchg_relaxed(ptr, ...) \ 4806 #define cmpxchg_relaxed(ptr, ...) \
4807 ({ \ 4807 ({ \
4808 typeof(ptr) __ai_ptr = (ptr); \ 4808 typeof(ptr) __ai_ptr = (ptr); \
4809 instrument_atomic_read_write(__ai_ptr 4809 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4810 raw_cmpxchg_relaxed(__ai_ptr, __VA_AR 4810 raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
4811 }) 4811 })
4812 4812
4813 #define cmpxchg64(ptr, ...) \ 4813 #define cmpxchg64(ptr, ...) \
4814 ({ \ 4814 ({ \
4815 typeof(ptr) __ai_ptr = (ptr); \ 4815 typeof(ptr) __ai_ptr = (ptr); \
4816 kcsan_mb(); \ 4816 kcsan_mb(); \
4817 instrument_atomic_read_write(__ai_ptr 4817 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4818 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); 4818 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \
4819 }) 4819 })
4820 4820
4821 #define cmpxchg64_acquire(ptr, ...) \ 4821 #define cmpxchg64_acquire(ptr, ...) \
4822 ({ \ 4822 ({ \
4823 typeof(ptr) __ai_ptr = (ptr); \ 4823 typeof(ptr) __ai_ptr = (ptr); \
4824 instrument_atomic_read_write(__ai_ptr 4824 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4825 raw_cmpxchg64_acquire(__ai_ptr, __VA_ 4825 raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
4826 }) 4826 })
4827 4827
4828 #define cmpxchg64_release(ptr, ...) \ 4828 #define cmpxchg64_release(ptr, ...) \
4829 ({ \ 4829 ({ \
4830 typeof(ptr) __ai_ptr = (ptr); \ 4830 typeof(ptr) __ai_ptr = (ptr); \
4831 kcsan_release(); \ 4831 kcsan_release(); \
4832 instrument_atomic_read_write(__ai_ptr 4832 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4833 raw_cmpxchg64_release(__ai_ptr, __VA_ 4833 raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
4834 }) 4834 })
4835 4835
4836 #define cmpxchg64_relaxed(ptr, ...) \ 4836 #define cmpxchg64_relaxed(ptr, ...) \
4837 ({ \ 4837 ({ \
4838 typeof(ptr) __ai_ptr = (ptr); \ 4838 typeof(ptr) __ai_ptr = (ptr); \
4839 instrument_atomic_read_write(__ai_ptr 4839 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4840 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ 4840 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
4841 }) 4841 })
4842 4842
4843 #define cmpxchg128(ptr, ...) \ 4843 #define cmpxchg128(ptr, ...) \
4844 ({ \ 4844 ({ \
4845 typeof(ptr) __ai_ptr = (ptr); \ 4845 typeof(ptr) __ai_ptr = (ptr); \
4846 kcsan_mb(); \ 4846 kcsan_mb(); \
4847 instrument_atomic_read_write(__ai_ptr 4847 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4848 raw_cmpxchg128(__ai_ptr, __VA_ARGS__) 4848 raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \
4849 }) 4849 })
4850 4850
4851 #define cmpxchg128_acquire(ptr, ...) \ 4851 #define cmpxchg128_acquire(ptr, ...) \
4852 ({ \ 4852 ({ \
4853 typeof(ptr) __ai_ptr = (ptr); \ 4853 typeof(ptr) __ai_ptr = (ptr); \
4854 instrument_atomic_read_write(__ai_ptr 4854 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4855 raw_cmpxchg128_acquire(__ai_ptr, __VA 4855 raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
4856 }) 4856 })
4857 4857
4858 #define cmpxchg128_release(ptr, ...) \ 4858 #define cmpxchg128_release(ptr, ...) \
4859 ({ \ 4859 ({ \
4860 typeof(ptr) __ai_ptr = (ptr); \ 4860 typeof(ptr) __ai_ptr = (ptr); \
4861 kcsan_release(); \ 4861 kcsan_release(); \
4862 instrument_atomic_read_write(__ai_ptr 4862 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4863 raw_cmpxchg128_release(__ai_ptr, __VA 4863 raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
4864 }) 4864 })
4865 4865
4866 #define cmpxchg128_relaxed(ptr, ...) \ 4866 #define cmpxchg128_relaxed(ptr, ...) \
4867 ({ \ 4867 ({ \
4868 typeof(ptr) __ai_ptr = (ptr); \ 4868 typeof(ptr) __ai_ptr = (ptr); \
4869 instrument_atomic_read_write(__ai_ptr 4869 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4870 raw_cmpxchg128_relaxed(__ai_ptr, __VA 4870 raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
4871 }) 4871 })
4872 4872
4873 #define try_cmpxchg(ptr, oldp, ...) \ 4873 #define try_cmpxchg(ptr, oldp, ...) \
4874 ({ \ 4874 ({ \
4875 typeof(ptr) __ai_ptr = (ptr); \ 4875 typeof(ptr) __ai_ptr = (ptr); \
4876 typeof(oldp) __ai_oldp = (oldp); \ 4876 typeof(oldp) __ai_oldp = (oldp); \
4877 kcsan_mb(); \ 4877 kcsan_mb(); \
4878 instrument_atomic_read_write(__ai_ptr 4878 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4879 instrument_read_write(__ai_oldp, size 4879 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4880 raw_try_cmpxchg(__ai_ptr, __ai_oldp, 4880 raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4881 }) 4881 })
4882 4882
4883 #define try_cmpxchg_acquire(ptr, oldp, ...) \ 4883 #define try_cmpxchg_acquire(ptr, oldp, ...) \
4884 ({ \ 4884 ({ \
4885 typeof(ptr) __ai_ptr = (ptr); \ 4885 typeof(ptr) __ai_ptr = (ptr); \
4886 typeof(oldp) __ai_oldp = (oldp); \ 4886 typeof(oldp) __ai_oldp = (oldp); \
4887 instrument_atomic_read_write(__ai_ptr 4887 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4888 instrument_read_write(__ai_oldp, size 4888 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4889 raw_try_cmpxchg_acquire(__ai_ptr, __a 4889 raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4890 }) 4890 })
4891 4891
4892 #define try_cmpxchg_release(ptr, oldp, ...) \ 4892 #define try_cmpxchg_release(ptr, oldp, ...) \
4893 ({ \ 4893 ({ \
4894 typeof(ptr) __ai_ptr = (ptr); \ 4894 typeof(ptr) __ai_ptr = (ptr); \
4895 typeof(oldp) __ai_oldp = (oldp); \ 4895 typeof(oldp) __ai_oldp = (oldp); \
4896 kcsan_release(); \ 4896 kcsan_release(); \
4897 instrument_atomic_read_write(__ai_ptr 4897 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4898 instrument_read_write(__ai_oldp, size 4898 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4899 raw_try_cmpxchg_release(__ai_ptr, __a 4899 raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4900 }) 4900 })
4901 4901
4902 #define try_cmpxchg_relaxed(ptr, oldp, ...) \ 4902 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
4903 ({ \ 4903 ({ \
4904 typeof(ptr) __ai_ptr = (ptr); \ 4904 typeof(ptr) __ai_ptr = (ptr); \
4905 typeof(oldp) __ai_oldp = (oldp); \ 4905 typeof(oldp) __ai_oldp = (oldp); \
4906 instrument_atomic_read_write(__ai_ptr 4906 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4907 instrument_read_write(__ai_oldp, size 4907 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4908 raw_try_cmpxchg_relaxed(__ai_ptr, __a 4908 raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4909 }) 4909 })
4910 4910
4911 #define try_cmpxchg64(ptr, oldp, ...) \ 4911 #define try_cmpxchg64(ptr, oldp, ...) \
4912 ({ \ 4912 ({ \
4913 typeof(ptr) __ai_ptr = (ptr); \ 4913 typeof(ptr) __ai_ptr = (ptr); \
4914 typeof(oldp) __ai_oldp = (oldp); \ 4914 typeof(oldp) __ai_oldp = (oldp); \
4915 kcsan_mb(); \ 4915 kcsan_mb(); \
4916 instrument_atomic_read_write(__ai_ptr 4916 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4917 instrument_read_write(__ai_oldp, size 4917 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4918 raw_try_cmpxchg64(__ai_ptr, __ai_oldp 4918 raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4919 }) 4919 })
4920 4920
4921 #define try_cmpxchg64_acquire(ptr, oldp, ...) 4921 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
4922 ({ \ 4922 ({ \
4923 typeof(ptr) __ai_ptr = (ptr); \ 4923 typeof(ptr) __ai_ptr = (ptr); \
4924 typeof(oldp) __ai_oldp = (oldp); \ 4924 typeof(oldp) __ai_oldp = (oldp); \
4925 instrument_atomic_read_write(__ai_ptr 4925 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4926 instrument_read_write(__ai_oldp, size 4926 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4927 raw_try_cmpxchg64_acquire(__ai_ptr, _ 4927 raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4928 }) 4928 })
4929 4929
4930 #define try_cmpxchg64_release(ptr, oldp, ...) 4930 #define try_cmpxchg64_release(ptr, oldp, ...) \
4931 ({ \ 4931 ({ \
4932 typeof(ptr) __ai_ptr = (ptr); \ 4932 typeof(ptr) __ai_ptr = (ptr); \
4933 typeof(oldp) __ai_oldp = (oldp); \ 4933 typeof(oldp) __ai_oldp = (oldp); \
4934 kcsan_release(); \ 4934 kcsan_release(); \
4935 instrument_atomic_read_write(__ai_ptr 4935 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4936 instrument_read_write(__ai_oldp, size 4936 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4937 raw_try_cmpxchg64_release(__ai_ptr, _ 4937 raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4938 }) 4938 })
4939 4939
4940 #define try_cmpxchg64_relaxed(ptr, oldp, ...) 4940 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
4941 ({ \ 4941 ({ \
4942 typeof(ptr) __ai_ptr = (ptr); \ 4942 typeof(ptr) __ai_ptr = (ptr); \
4943 typeof(oldp) __ai_oldp = (oldp); \ 4943 typeof(oldp) __ai_oldp = (oldp); \
4944 instrument_atomic_read_write(__ai_ptr 4944 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4945 instrument_read_write(__ai_oldp, size 4945 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4946 raw_try_cmpxchg64_relaxed(__ai_ptr, _ 4946 raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4947 }) 4947 })
4948 4948
4949 #define try_cmpxchg128(ptr, oldp, ...) \ 4949 #define try_cmpxchg128(ptr, oldp, ...) \
4950 ({ \ 4950 ({ \
4951 typeof(ptr) __ai_ptr = (ptr); \ 4951 typeof(ptr) __ai_ptr = (ptr); \
4952 typeof(oldp) __ai_oldp = (oldp); \ 4952 typeof(oldp) __ai_oldp = (oldp); \
4953 kcsan_mb(); \ 4953 kcsan_mb(); \
4954 instrument_atomic_read_write(__ai_ptr 4954 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4955 instrument_read_write(__ai_oldp, size 4955 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4956 raw_try_cmpxchg128(__ai_ptr, __ai_old 4956 raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4957 }) 4957 })
4958 4958
4959 #define try_cmpxchg128_acquire(ptr, oldp, ... 4959 #define try_cmpxchg128_acquire(ptr, oldp, ...) \
4960 ({ \ 4960 ({ \
4961 typeof(ptr) __ai_ptr = (ptr); \ 4961 typeof(ptr) __ai_ptr = (ptr); \
4962 typeof(oldp) __ai_oldp = (oldp); \ 4962 typeof(oldp) __ai_oldp = (oldp); \
4963 instrument_atomic_read_write(__ai_ptr 4963 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4964 instrument_read_write(__ai_oldp, size 4964 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4965 raw_try_cmpxchg128_acquire(__ai_ptr, 4965 raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4966 }) 4966 })
4967 4967
4968 #define try_cmpxchg128_release(ptr, oldp, ... 4968 #define try_cmpxchg128_release(ptr, oldp, ...) \
4969 ({ \ 4969 ({ \
4970 typeof(ptr) __ai_ptr = (ptr); \ 4970 typeof(ptr) __ai_ptr = (ptr); \
4971 typeof(oldp) __ai_oldp = (oldp); \ 4971 typeof(oldp) __ai_oldp = (oldp); \
4972 kcsan_release(); \ 4972 kcsan_release(); \
4973 instrument_atomic_read_write(__ai_ptr 4973 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4974 instrument_read_write(__ai_oldp, size 4974 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4975 raw_try_cmpxchg128_release(__ai_ptr, 4975 raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4976 }) 4976 })
4977 4977
4978 #define try_cmpxchg128_relaxed(ptr, oldp, ... 4978 #define try_cmpxchg128_relaxed(ptr, oldp, ...) \
4979 ({ \ 4979 ({ \
4980 typeof(ptr) __ai_ptr = (ptr); \ 4980 typeof(ptr) __ai_ptr = (ptr); \
4981 typeof(oldp) __ai_oldp = (oldp); \ 4981 typeof(oldp) __ai_oldp = (oldp); \
4982 instrument_atomic_read_write(__ai_ptr 4982 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4983 instrument_read_write(__ai_oldp, size 4983 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4984 raw_try_cmpxchg128_relaxed(__ai_ptr, 4984 raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4985 }) 4985 })
4986 4986
4987 #define cmpxchg_local(ptr, ...) \ 4987 #define cmpxchg_local(ptr, ...) \
4988 ({ \ 4988 ({ \
4989 typeof(ptr) __ai_ptr = (ptr); \ 4989 typeof(ptr) __ai_ptr = (ptr); \
4990 instrument_atomic_read_write(__ai_ptr 4990 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4991 raw_cmpxchg_local(__ai_ptr, __VA_ARGS 4991 raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
4992 }) 4992 })
4993 4993
4994 #define cmpxchg64_local(ptr, ...) \ 4994 #define cmpxchg64_local(ptr, ...) \
4995 ({ \ 4995 ({ \
4996 typeof(ptr) __ai_ptr = (ptr); \ 4996 typeof(ptr) __ai_ptr = (ptr); \
4997 instrument_atomic_read_write(__ai_ptr 4997 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4998 raw_cmpxchg64_local(__ai_ptr, __VA_AR 4998 raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
4999 }) 4999 })
5000 5000
5001 #define cmpxchg128_local(ptr, ...) \ 5001 #define cmpxchg128_local(ptr, ...) \
5002 ({ \ 5002 ({ \
5003 typeof(ptr) __ai_ptr = (ptr); \ 5003 typeof(ptr) __ai_ptr = (ptr); \
5004 instrument_atomic_read_write(__ai_ptr 5004 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5005 raw_cmpxchg128_local(__ai_ptr, __VA_A 5005 raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
5006 }) 5006 })
5007 5007
5008 #define sync_cmpxchg(ptr, ...) \ 5008 #define sync_cmpxchg(ptr, ...) \
5009 ({ \ 5009 ({ \
5010 typeof(ptr) __ai_ptr = (ptr); \ 5010 typeof(ptr) __ai_ptr = (ptr); \
5011 kcsan_mb(); \ 5011 kcsan_mb(); \
5012 instrument_atomic_read_write(__ai_ptr 5012 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5013 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS_ 5013 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
5014 }) 5014 })
5015 5015
5016 #define try_cmpxchg_local(ptr, oldp, ...) \ 5016 #define try_cmpxchg_local(ptr, oldp, ...) \
5017 ({ \ 5017 ({ \
5018 typeof(ptr) __ai_ptr = (ptr); \ 5018 typeof(ptr) __ai_ptr = (ptr); \
5019 typeof(oldp) __ai_oldp = (oldp); \ 5019 typeof(oldp) __ai_oldp = (oldp); \
5020 instrument_atomic_read_write(__ai_ptr 5020 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5021 instrument_read_write(__ai_oldp, size 5021 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5022 raw_try_cmpxchg_local(__ai_ptr, __ai_ 5022 raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5023 }) 5023 })
5024 5024
5025 #define try_cmpxchg64_local(ptr, oldp, ...) \ 5025 #define try_cmpxchg64_local(ptr, oldp, ...) \
5026 ({ \ 5026 ({ \
5027 typeof(ptr) __ai_ptr = (ptr); \ 5027 typeof(ptr) __ai_ptr = (ptr); \
5028 typeof(oldp) __ai_oldp = (oldp); \ 5028 typeof(oldp) __ai_oldp = (oldp); \
5029 instrument_atomic_read_write(__ai_ptr 5029 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5030 instrument_read_write(__ai_oldp, size 5030 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5031 raw_try_cmpxchg64_local(__ai_ptr, __a 5031 raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5032 }) 5032 })
5033 5033
5034 #define try_cmpxchg128_local(ptr, oldp, ...) 5034 #define try_cmpxchg128_local(ptr, oldp, ...) \
5035 ({ \ 5035 ({ \
5036 typeof(ptr) __ai_ptr = (ptr); \ 5036 typeof(ptr) __ai_ptr = (ptr); \
5037 typeof(oldp) __ai_oldp = (oldp); \ 5037 typeof(oldp) __ai_oldp = (oldp); \
5038 instrument_atomic_read_write(__ai_ptr 5038 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5039 instrument_read_write(__ai_oldp, size 5039 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5040 raw_try_cmpxchg128_local(__ai_ptr, __ 5040 raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5041 }) 5041 })
5042 5042
5043 #define sync_try_cmpxchg(ptr, ...) \ 5043 #define sync_try_cmpxchg(ptr, ...) \
5044 ({ \ 5044 ({ \
5045 typeof(ptr) __ai_ptr = (ptr); \ 5045 typeof(ptr) __ai_ptr = (ptr); \
5046 kcsan_mb(); \ 5046 kcsan_mb(); \
5047 instrument_atomic_read_write(__ai_ptr 5047 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5048 raw_sync_try_cmpxchg(__ai_ptr, __VA_A 5048 raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \
5049 }) 5049 })
5050 5050
5051 5051
5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ 5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
5053 // 8829b337928e9508259079d32581775ececd415b 5053 // 8829b337928e9508259079d32581775ececd415b
5054 5054
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.