1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
3 // Generated by scripts/atomic/gen-atomic-inst 3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY 4 // DO NOT MODIFY THIS FILE DIRECTLY
5 5
6 /* 6 /*
7 * This file provoides atomic operations with 7 * This file provoides atomic operations with explicit instrumentation (e.g.
8 * KASAN, KCSAN), which should be used unless 8 * KASAN, KCSAN), which should be used unless it is necessary to avoid
9 * instrumentation. Where it is necessary to a 9 * instrumentation. Where it is necessary to aovid instrumenation, the
10 * raw_atomic*() operations should be used. 10 * raw_atomic*() operations should be used.
11 */ 11 */
12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H 12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
13 #define _LINUX_ATOMIC_INSTRUMENTED_H 13 #define _LINUX_ATOMIC_INSTRUMENTED_H
14 14
15 #include <linux/build_bug.h> 15 #include <linux/build_bug.h>
16 #include <linux/compiler.h> 16 #include <linux/compiler.h>
17 #include <linux/instrumented.h> 17 #include <linux/instrumented.h>
18 18
19 /** 19 /**
20 * atomic_read() - atomic load with relaxed or 20 * atomic_read() - atomic load with relaxed ordering
21 * @v: pointer to atomic_t 21 * @v: pointer to atomic_t
22 * 22 *
23 * Atomically loads the value of @v with relax 23 * Atomically loads the value of @v with relaxed ordering.
24 * 24 *
25 * Unsafe to use in noinstr code; use raw_atom 25 * Unsafe to use in noinstr code; use raw_atomic_read() there.
26 * 26 *
27 * Return: The value loaded from @v. 27 * Return: The value loaded from @v.
28 */ 28 */
29 static __always_inline int 29 static __always_inline int
30 atomic_read(const atomic_t *v) 30 atomic_read(const atomic_t *v)
31 { 31 {
32 instrument_atomic_read(v, sizeof(*v)); 32 instrument_atomic_read(v, sizeof(*v));
33 return raw_atomic_read(v); 33 return raw_atomic_read(v);
34 } 34 }
35 35
36 /** 36 /**
37 * atomic_read_acquire() - atomic load with ac 37 * atomic_read_acquire() - atomic load with acquire ordering
38 * @v: pointer to atomic_t 38 * @v: pointer to atomic_t
39 * 39 *
40 * Atomically loads the value of @v with acqui 40 * Atomically loads the value of @v with acquire ordering.
41 * 41 *
42 * Unsafe to use in noinstr code; use raw_atom 42 * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
43 * 43 *
44 * Return: The value loaded from @v. 44 * Return: The value loaded from @v.
45 */ 45 */
46 static __always_inline int 46 static __always_inline int
47 atomic_read_acquire(const atomic_t *v) 47 atomic_read_acquire(const atomic_t *v)
48 { 48 {
49 instrument_atomic_read(v, sizeof(*v)); 49 instrument_atomic_read(v, sizeof(*v));
50 return raw_atomic_read_acquire(v); 50 return raw_atomic_read_acquire(v);
51 } 51 }
52 52
53 /** 53 /**
54 * atomic_set() - atomic set with relaxed orde 54 * atomic_set() - atomic set with relaxed ordering
55 * @v: pointer to atomic_t 55 * @v: pointer to atomic_t
56 * @i: int value to assign 56 * @i: int value to assign
57 * 57 *
58 * Atomically sets @v to @i with relaxed order 58 * Atomically sets @v to @i with relaxed ordering.
59 * 59 *
60 * Unsafe to use in noinstr code; use raw_atom 60 * Unsafe to use in noinstr code; use raw_atomic_set() there.
61 * 61 *
62 * Return: Nothing. 62 * Return: Nothing.
63 */ 63 */
64 static __always_inline void 64 static __always_inline void
65 atomic_set(atomic_t *v, int i) 65 atomic_set(atomic_t *v, int i)
66 { 66 {
67 instrument_atomic_write(v, sizeof(*v)) 67 instrument_atomic_write(v, sizeof(*v));
68 raw_atomic_set(v, i); 68 raw_atomic_set(v, i);
69 } 69 }
70 70
71 /** 71 /**
72 * atomic_set_release() - atomic set with rele 72 * atomic_set_release() - atomic set with release ordering
73 * @v: pointer to atomic_t 73 * @v: pointer to atomic_t
74 * @i: int value to assign 74 * @i: int value to assign
75 * 75 *
76 * Atomically sets @v to @i with release order 76 * Atomically sets @v to @i with release ordering.
77 * 77 *
78 * Unsafe to use in noinstr code; use raw_atom 78 * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
79 * 79 *
80 * Return: Nothing. 80 * Return: Nothing.
81 */ 81 */
82 static __always_inline void 82 static __always_inline void
83 atomic_set_release(atomic_t *v, int i) 83 atomic_set_release(atomic_t *v, int i)
84 { 84 {
85 kcsan_release(); 85 kcsan_release();
86 instrument_atomic_write(v, sizeof(*v)) 86 instrument_atomic_write(v, sizeof(*v));
87 raw_atomic_set_release(v, i); 87 raw_atomic_set_release(v, i);
88 } 88 }
89 89
90 /** 90 /**
91 * atomic_add() - atomic add with relaxed orde 91 * atomic_add() - atomic add with relaxed ordering
92 * @i: int value to add 92 * @i: int value to add
93 * @v: pointer to atomic_t 93 * @v: pointer to atomic_t
94 * 94 *
95 * Atomically updates @v to (@v + @i) with rel 95 * Atomically updates @v to (@v + @i) with relaxed ordering.
96 * 96 *
97 * Unsafe to use in noinstr code; use raw_atom 97 * Unsafe to use in noinstr code; use raw_atomic_add() there.
98 * 98 *
99 * Return: Nothing. 99 * Return: Nothing.
100 */ 100 */
101 static __always_inline void 101 static __always_inline void
102 atomic_add(int i, atomic_t *v) 102 atomic_add(int i, atomic_t *v)
103 { 103 {
104 instrument_atomic_read_write(v, sizeof 104 instrument_atomic_read_write(v, sizeof(*v));
105 raw_atomic_add(i, v); 105 raw_atomic_add(i, v);
106 } 106 }
107 107
108 /** 108 /**
109 * atomic_add_return() - atomic add with full 109 * atomic_add_return() - atomic add with full ordering
110 * @i: int value to add 110 * @i: int value to add
111 * @v: pointer to atomic_t 111 * @v: pointer to atomic_t
112 * 112 *
113 * Atomically updates @v to (@v + @i) with ful 113 * Atomically updates @v to (@v + @i) with full ordering.
114 * 114 *
115 * Unsafe to use in noinstr code; use raw_atom 115 * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
116 * 116 *
117 * Return: The updated value of @v. 117 * Return: The updated value of @v.
118 */ 118 */
119 static __always_inline int 119 static __always_inline int
120 atomic_add_return(int i, atomic_t *v) 120 atomic_add_return(int i, atomic_t *v)
121 { 121 {
122 kcsan_mb(); 122 kcsan_mb();
123 instrument_atomic_read_write(v, sizeof 123 instrument_atomic_read_write(v, sizeof(*v));
124 return raw_atomic_add_return(i, v); 124 return raw_atomic_add_return(i, v);
125 } 125 }
126 126
127 /** 127 /**
128 * atomic_add_return_acquire() - atomic add wi 128 * atomic_add_return_acquire() - atomic add with acquire ordering
129 * @i: int value to add 129 * @i: int value to add
130 * @v: pointer to atomic_t 130 * @v: pointer to atomic_t
131 * 131 *
132 * Atomically updates @v to (@v + @i) with acq 132 * Atomically updates @v to (@v + @i) with acquire ordering.
133 * 133 *
134 * Unsafe to use in noinstr code; use raw_atom 134 * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
135 * 135 *
136 * Return: The updated value of @v. 136 * Return: The updated value of @v.
137 */ 137 */
138 static __always_inline int 138 static __always_inline int
139 atomic_add_return_acquire(int i, atomic_t *v) 139 atomic_add_return_acquire(int i, atomic_t *v)
140 { 140 {
141 instrument_atomic_read_write(v, sizeof 141 instrument_atomic_read_write(v, sizeof(*v));
142 return raw_atomic_add_return_acquire(i 142 return raw_atomic_add_return_acquire(i, v);
143 } 143 }
144 144
145 /** 145 /**
146 * atomic_add_return_release() - atomic add wi 146 * atomic_add_return_release() - atomic add with release ordering
147 * @i: int value to add 147 * @i: int value to add
148 * @v: pointer to atomic_t 148 * @v: pointer to atomic_t
149 * 149 *
150 * Atomically updates @v to (@v + @i) with rel 150 * Atomically updates @v to (@v + @i) with release ordering.
151 * 151 *
152 * Unsafe to use in noinstr code; use raw_atom 152 * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
153 * 153 *
154 * Return: The updated value of @v. 154 * Return: The updated value of @v.
155 */ 155 */
156 static __always_inline int 156 static __always_inline int
157 atomic_add_return_release(int i, atomic_t *v) 157 atomic_add_return_release(int i, atomic_t *v)
158 { 158 {
159 kcsan_release(); 159 kcsan_release();
160 instrument_atomic_read_write(v, sizeof 160 instrument_atomic_read_write(v, sizeof(*v));
161 return raw_atomic_add_return_release(i 161 return raw_atomic_add_return_release(i, v);
162 } 162 }
163 163
164 /** 164 /**
165 * atomic_add_return_relaxed() - atomic add wi 165 * atomic_add_return_relaxed() - atomic add with relaxed ordering
166 * @i: int value to add 166 * @i: int value to add
167 * @v: pointer to atomic_t 167 * @v: pointer to atomic_t
168 * 168 *
169 * Atomically updates @v to (@v + @i) with rel 169 * Atomically updates @v to (@v + @i) with relaxed ordering.
170 * 170 *
171 * Unsafe to use in noinstr code; use raw_atom 171 * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
172 * 172 *
173 * Return: The updated value of @v. 173 * Return: The updated value of @v.
174 */ 174 */
175 static __always_inline int 175 static __always_inline int
176 atomic_add_return_relaxed(int i, atomic_t *v) 176 atomic_add_return_relaxed(int i, atomic_t *v)
177 { 177 {
178 instrument_atomic_read_write(v, sizeof 178 instrument_atomic_read_write(v, sizeof(*v));
179 return raw_atomic_add_return_relaxed(i 179 return raw_atomic_add_return_relaxed(i, v);
180 } 180 }
181 181
182 /** 182 /**
183 * atomic_fetch_add() - atomic add with full o 183 * atomic_fetch_add() - atomic add with full ordering
184 * @i: int value to add 184 * @i: int value to add
185 * @v: pointer to atomic_t 185 * @v: pointer to atomic_t
186 * 186 *
187 * Atomically updates @v to (@v + @i) with ful 187 * Atomically updates @v to (@v + @i) with full ordering.
188 * 188 *
189 * Unsafe to use in noinstr code; use raw_atom 189 * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
190 * 190 *
191 * Return: The original value of @v. 191 * Return: The original value of @v.
192 */ 192 */
193 static __always_inline int 193 static __always_inline int
194 atomic_fetch_add(int i, atomic_t *v) 194 atomic_fetch_add(int i, atomic_t *v)
195 { 195 {
196 kcsan_mb(); 196 kcsan_mb();
197 instrument_atomic_read_write(v, sizeof 197 instrument_atomic_read_write(v, sizeof(*v));
198 return raw_atomic_fetch_add(i, v); 198 return raw_atomic_fetch_add(i, v);
199 } 199 }
200 200
201 /** 201 /**
202 * atomic_fetch_add_acquire() - atomic add wit 202 * atomic_fetch_add_acquire() - atomic add with acquire ordering
203 * @i: int value to add 203 * @i: int value to add
204 * @v: pointer to atomic_t 204 * @v: pointer to atomic_t
205 * 205 *
206 * Atomically updates @v to (@v + @i) with acq 206 * Atomically updates @v to (@v + @i) with acquire ordering.
207 * 207 *
208 * Unsafe to use in noinstr code; use raw_atom 208 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
209 * 209 *
210 * Return: The original value of @v. 210 * Return: The original value of @v.
211 */ 211 */
212 static __always_inline int 212 static __always_inline int
213 atomic_fetch_add_acquire(int i, atomic_t *v) 213 atomic_fetch_add_acquire(int i, atomic_t *v)
214 { 214 {
215 instrument_atomic_read_write(v, sizeof 215 instrument_atomic_read_write(v, sizeof(*v));
216 return raw_atomic_fetch_add_acquire(i, 216 return raw_atomic_fetch_add_acquire(i, v);
217 } 217 }
218 218
219 /** 219 /**
220 * atomic_fetch_add_release() - atomic add wit 220 * atomic_fetch_add_release() - atomic add with release ordering
221 * @i: int value to add 221 * @i: int value to add
222 * @v: pointer to atomic_t 222 * @v: pointer to atomic_t
223 * 223 *
224 * Atomically updates @v to (@v + @i) with rel 224 * Atomically updates @v to (@v + @i) with release ordering.
225 * 225 *
226 * Unsafe to use in noinstr code; use raw_atom 226 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
227 * 227 *
228 * Return: The original value of @v. 228 * Return: The original value of @v.
229 */ 229 */
230 static __always_inline int 230 static __always_inline int
231 atomic_fetch_add_release(int i, atomic_t *v) 231 atomic_fetch_add_release(int i, atomic_t *v)
232 { 232 {
233 kcsan_release(); 233 kcsan_release();
234 instrument_atomic_read_write(v, sizeof 234 instrument_atomic_read_write(v, sizeof(*v));
235 return raw_atomic_fetch_add_release(i, 235 return raw_atomic_fetch_add_release(i, v);
236 } 236 }
237 237
238 /** 238 /**
239 * atomic_fetch_add_relaxed() - atomic add wit 239 * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
240 * @i: int value to add 240 * @i: int value to add
241 * @v: pointer to atomic_t 241 * @v: pointer to atomic_t
242 * 242 *
243 * Atomically updates @v to (@v + @i) with rel 243 * Atomically updates @v to (@v + @i) with relaxed ordering.
244 * 244 *
245 * Unsafe to use in noinstr code; use raw_atom 245 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
246 * 246 *
247 * Return: The original value of @v. 247 * Return: The original value of @v.
248 */ 248 */
249 static __always_inline int 249 static __always_inline int
250 atomic_fetch_add_relaxed(int i, atomic_t *v) 250 atomic_fetch_add_relaxed(int i, atomic_t *v)
251 { 251 {
252 instrument_atomic_read_write(v, sizeof 252 instrument_atomic_read_write(v, sizeof(*v));
253 return raw_atomic_fetch_add_relaxed(i, 253 return raw_atomic_fetch_add_relaxed(i, v);
254 } 254 }
255 255
256 /** 256 /**
257 * atomic_sub() - atomic subtract with relaxed 257 * atomic_sub() - atomic subtract with relaxed ordering
258 * @i: int value to subtract 258 * @i: int value to subtract
259 * @v: pointer to atomic_t 259 * @v: pointer to atomic_t
260 * 260 *
261 * Atomically updates @v to (@v - @i) with rel 261 * Atomically updates @v to (@v - @i) with relaxed ordering.
262 * 262 *
263 * Unsafe to use in noinstr code; use raw_atom 263 * Unsafe to use in noinstr code; use raw_atomic_sub() there.
264 * 264 *
265 * Return: Nothing. 265 * Return: Nothing.
266 */ 266 */
267 static __always_inline void 267 static __always_inline void
268 atomic_sub(int i, atomic_t *v) 268 atomic_sub(int i, atomic_t *v)
269 { 269 {
270 instrument_atomic_read_write(v, sizeof 270 instrument_atomic_read_write(v, sizeof(*v));
271 raw_atomic_sub(i, v); 271 raw_atomic_sub(i, v);
272 } 272 }
273 273
274 /** 274 /**
275 * atomic_sub_return() - atomic subtract with 275 * atomic_sub_return() - atomic subtract with full ordering
276 * @i: int value to subtract 276 * @i: int value to subtract
277 * @v: pointer to atomic_t 277 * @v: pointer to atomic_t
278 * 278 *
279 * Atomically updates @v to (@v - @i) with ful 279 * Atomically updates @v to (@v - @i) with full ordering.
280 * 280 *
281 * Unsafe to use in noinstr code; use raw_atom 281 * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
282 * 282 *
283 * Return: The updated value of @v. 283 * Return: The updated value of @v.
284 */ 284 */
285 static __always_inline int 285 static __always_inline int
286 atomic_sub_return(int i, atomic_t *v) 286 atomic_sub_return(int i, atomic_t *v)
287 { 287 {
288 kcsan_mb(); 288 kcsan_mb();
289 instrument_atomic_read_write(v, sizeof 289 instrument_atomic_read_write(v, sizeof(*v));
290 return raw_atomic_sub_return(i, v); 290 return raw_atomic_sub_return(i, v);
291 } 291 }
292 292
293 /** 293 /**
294 * atomic_sub_return_acquire() - atomic subtra 294 * atomic_sub_return_acquire() - atomic subtract with acquire ordering
295 * @i: int value to subtract 295 * @i: int value to subtract
296 * @v: pointer to atomic_t 296 * @v: pointer to atomic_t
297 * 297 *
298 * Atomically updates @v to (@v - @i) with acq 298 * Atomically updates @v to (@v - @i) with acquire ordering.
299 * 299 *
300 * Unsafe to use in noinstr code; use raw_atom 300 * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
301 * 301 *
302 * Return: The updated value of @v. 302 * Return: The updated value of @v.
303 */ 303 */
304 static __always_inline int 304 static __always_inline int
305 atomic_sub_return_acquire(int i, atomic_t *v) 305 atomic_sub_return_acquire(int i, atomic_t *v)
306 { 306 {
307 instrument_atomic_read_write(v, sizeof 307 instrument_atomic_read_write(v, sizeof(*v));
308 return raw_atomic_sub_return_acquire(i 308 return raw_atomic_sub_return_acquire(i, v);
309 } 309 }
310 310
311 /** 311 /**
312 * atomic_sub_return_release() - atomic subtra 312 * atomic_sub_return_release() - atomic subtract with release ordering
313 * @i: int value to subtract 313 * @i: int value to subtract
314 * @v: pointer to atomic_t 314 * @v: pointer to atomic_t
315 * 315 *
316 * Atomically updates @v to (@v - @i) with rel 316 * Atomically updates @v to (@v - @i) with release ordering.
317 * 317 *
318 * Unsafe to use in noinstr code; use raw_atom 318 * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
319 * 319 *
320 * Return: The updated value of @v. 320 * Return: The updated value of @v.
321 */ 321 */
322 static __always_inline int 322 static __always_inline int
323 atomic_sub_return_release(int i, atomic_t *v) 323 atomic_sub_return_release(int i, atomic_t *v)
324 { 324 {
325 kcsan_release(); 325 kcsan_release();
326 instrument_atomic_read_write(v, sizeof 326 instrument_atomic_read_write(v, sizeof(*v));
327 return raw_atomic_sub_return_release(i 327 return raw_atomic_sub_return_release(i, v);
328 } 328 }
329 329
330 /** 330 /**
331 * atomic_sub_return_relaxed() - atomic subtra 331 * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
332 * @i: int value to subtract 332 * @i: int value to subtract
333 * @v: pointer to atomic_t 333 * @v: pointer to atomic_t
334 * 334 *
335 * Atomically updates @v to (@v - @i) with rel 335 * Atomically updates @v to (@v - @i) with relaxed ordering.
336 * 336 *
337 * Unsafe to use in noinstr code; use raw_atom 337 * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
338 * 338 *
339 * Return: The updated value of @v. 339 * Return: The updated value of @v.
340 */ 340 */
341 static __always_inline int 341 static __always_inline int
342 atomic_sub_return_relaxed(int i, atomic_t *v) 342 atomic_sub_return_relaxed(int i, atomic_t *v)
343 { 343 {
344 instrument_atomic_read_write(v, sizeof 344 instrument_atomic_read_write(v, sizeof(*v));
345 return raw_atomic_sub_return_relaxed(i 345 return raw_atomic_sub_return_relaxed(i, v);
346 } 346 }
347 347
348 /** 348 /**
349 * atomic_fetch_sub() - atomic subtract with f 349 * atomic_fetch_sub() - atomic subtract with full ordering
350 * @i: int value to subtract 350 * @i: int value to subtract
351 * @v: pointer to atomic_t 351 * @v: pointer to atomic_t
352 * 352 *
353 * Atomically updates @v to (@v - @i) with ful 353 * Atomically updates @v to (@v - @i) with full ordering.
354 * 354 *
355 * Unsafe to use in noinstr code; use raw_atom 355 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
356 * 356 *
357 * Return: The original value of @v. 357 * Return: The original value of @v.
358 */ 358 */
359 static __always_inline int 359 static __always_inline int
360 atomic_fetch_sub(int i, atomic_t *v) 360 atomic_fetch_sub(int i, atomic_t *v)
361 { 361 {
362 kcsan_mb(); 362 kcsan_mb();
363 instrument_atomic_read_write(v, sizeof 363 instrument_atomic_read_write(v, sizeof(*v));
364 return raw_atomic_fetch_sub(i, v); 364 return raw_atomic_fetch_sub(i, v);
365 } 365 }
366 366
367 /** 367 /**
368 * atomic_fetch_sub_acquire() - atomic subtrac 368 * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
369 * @i: int value to subtract 369 * @i: int value to subtract
370 * @v: pointer to atomic_t 370 * @v: pointer to atomic_t
371 * 371 *
372 * Atomically updates @v to (@v - @i) with acq 372 * Atomically updates @v to (@v - @i) with acquire ordering.
373 * 373 *
374 * Unsafe to use in noinstr code; use raw_atom 374 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
375 * 375 *
376 * Return: The original value of @v. 376 * Return: The original value of @v.
377 */ 377 */
378 static __always_inline int 378 static __always_inline int
379 atomic_fetch_sub_acquire(int i, atomic_t *v) 379 atomic_fetch_sub_acquire(int i, atomic_t *v)
380 { 380 {
381 instrument_atomic_read_write(v, sizeof 381 instrument_atomic_read_write(v, sizeof(*v));
382 return raw_atomic_fetch_sub_acquire(i, 382 return raw_atomic_fetch_sub_acquire(i, v);
383 } 383 }
384 384
385 /** 385 /**
386 * atomic_fetch_sub_release() - atomic subtrac 386 * atomic_fetch_sub_release() - atomic subtract with release ordering
387 * @i: int value to subtract 387 * @i: int value to subtract
388 * @v: pointer to atomic_t 388 * @v: pointer to atomic_t
389 * 389 *
390 * Atomically updates @v to (@v - @i) with rel 390 * Atomically updates @v to (@v - @i) with release ordering.
391 * 391 *
392 * Unsafe to use in noinstr code; use raw_atom 392 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
393 * 393 *
394 * Return: The original value of @v. 394 * Return: The original value of @v.
395 */ 395 */
396 static __always_inline int 396 static __always_inline int
397 atomic_fetch_sub_release(int i, atomic_t *v) 397 atomic_fetch_sub_release(int i, atomic_t *v)
398 { 398 {
399 kcsan_release(); 399 kcsan_release();
400 instrument_atomic_read_write(v, sizeof 400 instrument_atomic_read_write(v, sizeof(*v));
401 return raw_atomic_fetch_sub_release(i, 401 return raw_atomic_fetch_sub_release(i, v);
402 } 402 }
403 403
404 /** 404 /**
405 * atomic_fetch_sub_relaxed() - atomic subtrac 405 * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
406 * @i: int value to subtract 406 * @i: int value to subtract
407 * @v: pointer to atomic_t 407 * @v: pointer to atomic_t
408 * 408 *
409 * Atomically updates @v to (@v - @i) with rel 409 * Atomically updates @v to (@v - @i) with relaxed ordering.
410 * 410 *
411 * Unsafe to use in noinstr code; use raw_atom 411 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
412 * 412 *
413 * Return: The original value of @v. 413 * Return: The original value of @v.
414 */ 414 */
415 static __always_inline int 415 static __always_inline int
416 atomic_fetch_sub_relaxed(int i, atomic_t *v) 416 atomic_fetch_sub_relaxed(int i, atomic_t *v)
417 { 417 {
418 instrument_atomic_read_write(v, sizeof 418 instrument_atomic_read_write(v, sizeof(*v));
419 return raw_atomic_fetch_sub_relaxed(i, 419 return raw_atomic_fetch_sub_relaxed(i, v);
420 } 420 }
421 421
422 /** 422 /**
423 * atomic_inc() - atomic increment with relaxe 423 * atomic_inc() - atomic increment with relaxed ordering
424 * @v: pointer to atomic_t 424 * @v: pointer to atomic_t
425 * 425 *
426 * Atomically updates @v to (@v + 1) with rela 426 * Atomically updates @v to (@v + 1) with relaxed ordering.
427 * 427 *
428 * Unsafe to use in noinstr code; use raw_atom 428 * Unsafe to use in noinstr code; use raw_atomic_inc() there.
429 * 429 *
430 * Return: Nothing. 430 * Return: Nothing.
431 */ 431 */
432 static __always_inline void 432 static __always_inline void
433 atomic_inc(atomic_t *v) 433 atomic_inc(atomic_t *v)
434 { 434 {
435 instrument_atomic_read_write(v, sizeof 435 instrument_atomic_read_write(v, sizeof(*v));
436 raw_atomic_inc(v); 436 raw_atomic_inc(v);
437 } 437 }
438 438
439 /** 439 /**
440 * atomic_inc_return() - atomic increment with 440 * atomic_inc_return() - atomic increment with full ordering
441 * @v: pointer to atomic_t 441 * @v: pointer to atomic_t
442 * 442 *
443 * Atomically updates @v to (@v + 1) with full 443 * Atomically updates @v to (@v + 1) with full ordering.
444 * 444 *
445 * Unsafe to use in noinstr code; use raw_atom 445 * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
446 * 446 *
447 * Return: The updated value of @v. 447 * Return: The updated value of @v.
448 */ 448 */
449 static __always_inline int 449 static __always_inline int
450 atomic_inc_return(atomic_t *v) 450 atomic_inc_return(atomic_t *v)
451 { 451 {
452 kcsan_mb(); 452 kcsan_mb();
453 instrument_atomic_read_write(v, sizeof 453 instrument_atomic_read_write(v, sizeof(*v));
454 return raw_atomic_inc_return(v); 454 return raw_atomic_inc_return(v);
455 } 455 }
456 456
457 /** 457 /**
458 * atomic_inc_return_acquire() - atomic increm 458 * atomic_inc_return_acquire() - atomic increment with acquire ordering
459 * @v: pointer to atomic_t 459 * @v: pointer to atomic_t
460 * 460 *
461 * Atomically updates @v to (@v + 1) with acqu 461 * Atomically updates @v to (@v + 1) with acquire ordering.
462 * 462 *
463 * Unsafe to use in noinstr code; use raw_atom 463 * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
464 * 464 *
465 * Return: The updated value of @v. 465 * Return: The updated value of @v.
466 */ 466 */
467 static __always_inline int 467 static __always_inline int
468 atomic_inc_return_acquire(atomic_t *v) 468 atomic_inc_return_acquire(atomic_t *v)
469 { 469 {
470 instrument_atomic_read_write(v, sizeof 470 instrument_atomic_read_write(v, sizeof(*v));
471 return raw_atomic_inc_return_acquire(v 471 return raw_atomic_inc_return_acquire(v);
472 } 472 }
473 473
474 /** 474 /**
475 * atomic_inc_return_release() - atomic increm 475 * atomic_inc_return_release() - atomic increment with release ordering
476 * @v: pointer to atomic_t 476 * @v: pointer to atomic_t
477 * 477 *
478 * Atomically updates @v to (@v + 1) with rele 478 * Atomically updates @v to (@v + 1) with release ordering.
479 * 479 *
480 * Unsafe to use in noinstr code; use raw_atom 480 * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
481 * 481 *
482 * Return: The updated value of @v. 482 * Return: The updated value of @v.
483 */ 483 */
484 static __always_inline int 484 static __always_inline int
485 atomic_inc_return_release(atomic_t *v) 485 atomic_inc_return_release(atomic_t *v)
486 { 486 {
487 kcsan_release(); 487 kcsan_release();
488 instrument_atomic_read_write(v, sizeof 488 instrument_atomic_read_write(v, sizeof(*v));
489 return raw_atomic_inc_return_release(v 489 return raw_atomic_inc_return_release(v);
490 } 490 }
491 491
492 /** 492 /**
493 * atomic_inc_return_relaxed() - atomic increm 493 * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
494 * @v: pointer to atomic_t 494 * @v: pointer to atomic_t
495 * 495 *
496 * Atomically updates @v to (@v + 1) with rela 496 * Atomically updates @v to (@v + 1) with relaxed ordering.
497 * 497 *
498 * Unsafe to use in noinstr code; use raw_atom 498 * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
499 * 499 *
500 * Return: The updated value of @v. 500 * Return: The updated value of @v.
501 */ 501 */
502 static __always_inline int 502 static __always_inline int
503 atomic_inc_return_relaxed(atomic_t *v) 503 atomic_inc_return_relaxed(atomic_t *v)
504 { 504 {
505 instrument_atomic_read_write(v, sizeof 505 instrument_atomic_read_write(v, sizeof(*v));
506 return raw_atomic_inc_return_relaxed(v 506 return raw_atomic_inc_return_relaxed(v);
507 } 507 }
508 508
509 /** 509 /**
510 * atomic_fetch_inc() - atomic increment with 510 * atomic_fetch_inc() - atomic increment with full ordering
511 * @v: pointer to atomic_t 511 * @v: pointer to atomic_t
512 * 512 *
513 * Atomically updates @v to (@v + 1) with full 513 * Atomically updates @v to (@v + 1) with full ordering.
514 * 514 *
515 * Unsafe to use in noinstr code; use raw_atom 515 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
516 * 516 *
517 * Return: The original value of @v. 517 * Return: The original value of @v.
518 */ 518 */
519 static __always_inline int 519 static __always_inline int
520 atomic_fetch_inc(atomic_t *v) 520 atomic_fetch_inc(atomic_t *v)
521 { 521 {
522 kcsan_mb(); 522 kcsan_mb();
523 instrument_atomic_read_write(v, sizeof 523 instrument_atomic_read_write(v, sizeof(*v));
524 return raw_atomic_fetch_inc(v); 524 return raw_atomic_fetch_inc(v);
525 } 525 }
526 526
527 /** 527 /**
528 * atomic_fetch_inc_acquire() - atomic increme 528 * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
529 * @v: pointer to atomic_t 529 * @v: pointer to atomic_t
530 * 530 *
531 * Atomically updates @v to (@v + 1) with acqu 531 * Atomically updates @v to (@v + 1) with acquire ordering.
532 * 532 *
533 * Unsafe to use in noinstr code; use raw_atom 533 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
534 * 534 *
535 * Return: The original value of @v. 535 * Return: The original value of @v.
536 */ 536 */
537 static __always_inline int 537 static __always_inline int
538 atomic_fetch_inc_acquire(atomic_t *v) 538 atomic_fetch_inc_acquire(atomic_t *v)
539 { 539 {
540 instrument_atomic_read_write(v, sizeof 540 instrument_atomic_read_write(v, sizeof(*v));
541 return raw_atomic_fetch_inc_acquire(v) 541 return raw_atomic_fetch_inc_acquire(v);
542 } 542 }
543 543
544 /** 544 /**
545 * atomic_fetch_inc_release() - atomic increme 545 * atomic_fetch_inc_release() - atomic increment with release ordering
546 * @v: pointer to atomic_t 546 * @v: pointer to atomic_t
547 * 547 *
548 * Atomically updates @v to (@v + 1) with rele 548 * Atomically updates @v to (@v + 1) with release ordering.
549 * 549 *
550 * Unsafe to use in noinstr code; use raw_atom 550 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
551 * 551 *
552 * Return: The original value of @v. 552 * Return: The original value of @v.
553 */ 553 */
554 static __always_inline int 554 static __always_inline int
555 atomic_fetch_inc_release(atomic_t *v) 555 atomic_fetch_inc_release(atomic_t *v)
556 { 556 {
557 kcsan_release(); 557 kcsan_release();
558 instrument_atomic_read_write(v, sizeof 558 instrument_atomic_read_write(v, sizeof(*v));
559 return raw_atomic_fetch_inc_release(v) 559 return raw_atomic_fetch_inc_release(v);
560 } 560 }
561 561
562 /** 562 /**
563 * atomic_fetch_inc_relaxed() - atomic increme 563 * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
564 * @v: pointer to atomic_t 564 * @v: pointer to atomic_t
565 * 565 *
566 * Atomically updates @v to (@v + 1) with rela 566 * Atomically updates @v to (@v + 1) with relaxed ordering.
567 * 567 *
568 * Unsafe to use in noinstr code; use raw_atom 568 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
569 * 569 *
570 * Return: The original value of @v. 570 * Return: The original value of @v.
571 */ 571 */
572 static __always_inline int 572 static __always_inline int
573 atomic_fetch_inc_relaxed(atomic_t *v) 573 atomic_fetch_inc_relaxed(atomic_t *v)
574 { 574 {
575 instrument_atomic_read_write(v, sizeof 575 instrument_atomic_read_write(v, sizeof(*v));
576 return raw_atomic_fetch_inc_relaxed(v) 576 return raw_atomic_fetch_inc_relaxed(v);
577 } 577 }
578 578
579 /** 579 /**
580 * atomic_dec() - atomic decrement with relaxe 580 * atomic_dec() - atomic decrement with relaxed ordering
581 * @v: pointer to atomic_t 581 * @v: pointer to atomic_t
582 * 582 *
583 * Atomically updates @v to (@v - 1) with rela 583 * Atomically updates @v to (@v - 1) with relaxed ordering.
584 * 584 *
585 * Unsafe to use in noinstr code; use raw_atom 585 * Unsafe to use in noinstr code; use raw_atomic_dec() there.
586 * 586 *
587 * Return: Nothing. 587 * Return: Nothing.
588 */ 588 */
589 static __always_inline void 589 static __always_inline void
590 atomic_dec(atomic_t *v) 590 atomic_dec(atomic_t *v)
591 { 591 {
592 instrument_atomic_read_write(v, sizeof 592 instrument_atomic_read_write(v, sizeof(*v));
593 raw_atomic_dec(v); 593 raw_atomic_dec(v);
594 } 594 }
595 595
596 /** 596 /**
597 * atomic_dec_return() - atomic decrement with 597 * atomic_dec_return() - atomic decrement with full ordering
598 * @v: pointer to atomic_t 598 * @v: pointer to atomic_t
599 * 599 *
600 * Atomically updates @v to (@v - 1) with full 600 * Atomically updates @v to (@v - 1) with full ordering.
601 * 601 *
602 * Unsafe to use in noinstr code; use raw_atom 602 * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
603 * 603 *
604 * Return: The updated value of @v. 604 * Return: The updated value of @v.
605 */ 605 */
606 static __always_inline int 606 static __always_inline int
607 atomic_dec_return(atomic_t *v) 607 atomic_dec_return(atomic_t *v)
608 { 608 {
609 kcsan_mb(); 609 kcsan_mb();
610 instrument_atomic_read_write(v, sizeof 610 instrument_atomic_read_write(v, sizeof(*v));
611 return raw_atomic_dec_return(v); 611 return raw_atomic_dec_return(v);
612 } 612 }
613 613
614 /** 614 /**
615 * atomic_dec_return_acquire() - atomic decrem 615 * atomic_dec_return_acquire() - atomic decrement with acquire ordering
616 * @v: pointer to atomic_t 616 * @v: pointer to atomic_t
617 * 617 *
618 * Atomically updates @v to (@v - 1) with acqu 618 * Atomically updates @v to (@v - 1) with acquire ordering.
619 * 619 *
620 * Unsafe to use in noinstr code; use raw_atom 620 * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
621 * 621 *
622 * Return: The updated value of @v. 622 * Return: The updated value of @v.
623 */ 623 */
624 static __always_inline int 624 static __always_inline int
625 atomic_dec_return_acquire(atomic_t *v) 625 atomic_dec_return_acquire(atomic_t *v)
626 { 626 {
627 instrument_atomic_read_write(v, sizeof 627 instrument_atomic_read_write(v, sizeof(*v));
628 return raw_atomic_dec_return_acquire(v 628 return raw_atomic_dec_return_acquire(v);
629 } 629 }
630 630
631 /** 631 /**
632 * atomic_dec_return_release() - atomic decrem 632 * atomic_dec_return_release() - atomic decrement with release ordering
633 * @v: pointer to atomic_t 633 * @v: pointer to atomic_t
634 * 634 *
635 * Atomically updates @v to (@v - 1) with rele 635 * Atomically updates @v to (@v - 1) with release ordering.
636 * 636 *
637 * Unsafe to use in noinstr code; use raw_atom 637 * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
638 * 638 *
639 * Return: The updated value of @v. 639 * Return: The updated value of @v.
640 */ 640 */
641 static __always_inline int 641 static __always_inline int
642 atomic_dec_return_release(atomic_t *v) 642 atomic_dec_return_release(atomic_t *v)
643 { 643 {
644 kcsan_release(); 644 kcsan_release();
645 instrument_atomic_read_write(v, sizeof 645 instrument_atomic_read_write(v, sizeof(*v));
646 return raw_atomic_dec_return_release(v 646 return raw_atomic_dec_return_release(v);
647 } 647 }
648 648
649 /** 649 /**
650 * atomic_dec_return_relaxed() - atomic decrem 650 * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
651 * @v: pointer to atomic_t 651 * @v: pointer to atomic_t
652 * 652 *
653 * Atomically updates @v to (@v - 1) with rela 653 * Atomically updates @v to (@v - 1) with relaxed ordering.
654 * 654 *
655 * Unsafe to use in noinstr code; use raw_atom 655 * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
656 * 656 *
657 * Return: The updated value of @v. 657 * Return: The updated value of @v.
658 */ 658 */
659 static __always_inline int 659 static __always_inline int
660 atomic_dec_return_relaxed(atomic_t *v) 660 atomic_dec_return_relaxed(atomic_t *v)
661 { 661 {
662 instrument_atomic_read_write(v, sizeof 662 instrument_atomic_read_write(v, sizeof(*v));
663 return raw_atomic_dec_return_relaxed(v 663 return raw_atomic_dec_return_relaxed(v);
664 } 664 }
665 665
666 /** 666 /**
667 * atomic_fetch_dec() - atomic decrement with 667 * atomic_fetch_dec() - atomic decrement with full ordering
668 * @v: pointer to atomic_t 668 * @v: pointer to atomic_t
669 * 669 *
670 * Atomically updates @v to (@v - 1) with full 670 * Atomically updates @v to (@v - 1) with full ordering.
671 * 671 *
672 * Unsafe to use in noinstr code; use raw_atom 672 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
673 * 673 *
674 * Return: The original value of @v. 674 * Return: The original value of @v.
675 */ 675 */
676 static __always_inline int 676 static __always_inline int
677 atomic_fetch_dec(atomic_t *v) 677 atomic_fetch_dec(atomic_t *v)
678 { 678 {
679 kcsan_mb(); 679 kcsan_mb();
680 instrument_atomic_read_write(v, sizeof 680 instrument_atomic_read_write(v, sizeof(*v));
681 return raw_atomic_fetch_dec(v); 681 return raw_atomic_fetch_dec(v);
682 } 682 }
683 683
684 /** 684 /**
685 * atomic_fetch_dec_acquire() - atomic decreme 685 * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
686 * @v: pointer to atomic_t 686 * @v: pointer to atomic_t
687 * 687 *
688 * Atomically updates @v to (@v - 1) with acqu 688 * Atomically updates @v to (@v - 1) with acquire ordering.
689 * 689 *
690 * Unsafe to use in noinstr code; use raw_atom 690 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
691 * 691 *
692 * Return: The original value of @v. 692 * Return: The original value of @v.
693 */ 693 */
694 static __always_inline int 694 static __always_inline int
695 atomic_fetch_dec_acquire(atomic_t *v) 695 atomic_fetch_dec_acquire(atomic_t *v)
696 { 696 {
697 instrument_atomic_read_write(v, sizeof 697 instrument_atomic_read_write(v, sizeof(*v));
698 return raw_atomic_fetch_dec_acquire(v) 698 return raw_atomic_fetch_dec_acquire(v);
699 } 699 }
700 700
701 /** 701 /**
702 * atomic_fetch_dec_release() - atomic decreme 702 * atomic_fetch_dec_release() - atomic decrement with release ordering
703 * @v: pointer to atomic_t 703 * @v: pointer to atomic_t
704 * 704 *
705 * Atomically updates @v to (@v - 1) with rele 705 * Atomically updates @v to (@v - 1) with release ordering.
706 * 706 *
707 * Unsafe to use in noinstr code; use raw_atom 707 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
708 * 708 *
709 * Return: The original value of @v. 709 * Return: The original value of @v.
710 */ 710 */
711 static __always_inline int 711 static __always_inline int
712 atomic_fetch_dec_release(atomic_t *v) 712 atomic_fetch_dec_release(atomic_t *v)
713 { 713 {
714 kcsan_release(); 714 kcsan_release();
715 instrument_atomic_read_write(v, sizeof 715 instrument_atomic_read_write(v, sizeof(*v));
716 return raw_atomic_fetch_dec_release(v) 716 return raw_atomic_fetch_dec_release(v);
717 } 717 }
718 718
719 /** 719 /**
720 * atomic_fetch_dec_relaxed() - atomic decreme 720 * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
721 * @v: pointer to atomic_t 721 * @v: pointer to atomic_t
722 * 722 *
723 * Atomically updates @v to (@v - 1) with rela 723 * Atomically updates @v to (@v - 1) with relaxed ordering.
724 * 724 *
725 * Unsafe to use in noinstr code; use raw_atom 725 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
726 * 726 *
727 * Return: The original value of @v. 727 * Return: The original value of @v.
728 */ 728 */
729 static __always_inline int 729 static __always_inline int
730 atomic_fetch_dec_relaxed(atomic_t *v) 730 atomic_fetch_dec_relaxed(atomic_t *v)
731 { 731 {
732 instrument_atomic_read_write(v, sizeof 732 instrument_atomic_read_write(v, sizeof(*v));
733 return raw_atomic_fetch_dec_relaxed(v) 733 return raw_atomic_fetch_dec_relaxed(v);
734 } 734 }
735 735
736 /** 736 /**
737 * atomic_and() - atomic bitwise AND with rela 737 * atomic_and() - atomic bitwise AND with relaxed ordering
738 * @i: int value 738 * @i: int value
739 * @v: pointer to atomic_t 739 * @v: pointer to atomic_t
740 * 740 *
741 * Atomically updates @v to (@v & @i) with rel 741 * Atomically updates @v to (@v & @i) with relaxed ordering.
742 * 742 *
743 * Unsafe to use in noinstr code; use raw_atom 743 * Unsafe to use in noinstr code; use raw_atomic_and() there.
744 * 744 *
745 * Return: Nothing. 745 * Return: Nothing.
746 */ 746 */
747 static __always_inline void 747 static __always_inline void
748 atomic_and(int i, atomic_t *v) 748 atomic_and(int i, atomic_t *v)
749 { 749 {
750 instrument_atomic_read_write(v, sizeof 750 instrument_atomic_read_write(v, sizeof(*v));
751 raw_atomic_and(i, v); 751 raw_atomic_and(i, v);
752 } 752 }
753 753
754 /** 754 /**
755 * atomic_fetch_and() - atomic bitwise AND wit 755 * atomic_fetch_and() - atomic bitwise AND with full ordering
756 * @i: int value 756 * @i: int value
757 * @v: pointer to atomic_t 757 * @v: pointer to atomic_t
758 * 758 *
759 * Atomically updates @v to (@v & @i) with ful 759 * Atomically updates @v to (@v & @i) with full ordering.
760 * 760 *
761 * Unsafe to use in noinstr code; use raw_atom 761 * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
762 * 762 *
763 * Return: The original value of @v. 763 * Return: The original value of @v.
764 */ 764 */
765 static __always_inline int 765 static __always_inline int
766 atomic_fetch_and(int i, atomic_t *v) 766 atomic_fetch_and(int i, atomic_t *v)
767 { 767 {
768 kcsan_mb(); 768 kcsan_mb();
769 instrument_atomic_read_write(v, sizeof 769 instrument_atomic_read_write(v, sizeof(*v));
770 return raw_atomic_fetch_and(i, v); 770 return raw_atomic_fetch_and(i, v);
771 } 771 }
772 772
773 /** 773 /**
774 * atomic_fetch_and_acquire() - atomic bitwise 774 * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
775 * @i: int value 775 * @i: int value
776 * @v: pointer to atomic_t 776 * @v: pointer to atomic_t
777 * 777 *
778 * Atomically updates @v to (@v & @i) with acq 778 * Atomically updates @v to (@v & @i) with acquire ordering.
779 * 779 *
780 * Unsafe to use in noinstr code; use raw_atom 780 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
781 * 781 *
782 * Return: The original value of @v. 782 * Return: The original value of @v.
783 */ 783 */
784 static __always_inline int 784 static __always_inline int
785 atomic_fetch_and_acquire(int i, atomic_t *v) 785 atomic_fetch_and_acquire(int i, atomic_t *v)
786 { 786 {
787 instrument_atomic_read_write(v, sizeof 787 instrument_atomic_read_write(v, sizeof(*v));
788 return raw_atomic_fetch_and_acquire(i, 788 return raw_atomic_fetch_and_acquire(i, v);
789 } 789 }
790 790
791 /** 791 /**
792 * atomic_fetch_and_release() - atomic bitwise 792 * atomic_fetch_and_release() - atomic bitwise AND with release ordering
793 * @i: int value 793 * @i: int value
794 * @v: pointer to atomic_t 794 * @v: pointer to atomic_t
795 * 795 *
796 * Atomically updates @v to (@v & @i) with rel 796 * Atomically updates @v to (@v & @i) with release ordering.
797 * 797 *
798 * Unsafe to use in noinstr code; use raw_atom 798 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
799 * 799 *
800 * Return: The original value of @v. 800 * Return: The original value of @v.
801 */ 801 */
802 static __always_inline int 802 static __always_inline int
803 atomic_fetch_and_release(int i, atomic_t *v) 803 atomic_fetch_and_release(int i, atomic_t *v)
804 { 804 {
805 kcsan_release(); 805 kcsan_release();
806 instrument_atomic_read_write(v, sizeof 806 instrument_atomic_read_write(v, sizeof(*v));
807 return raw_atomic_fetch_and_release(i, 807 return raw_atomic_fetch_and_release(i, v);
808 } 808 }
809 809
810 /** 810 /**
811 * atomic_fetch_and_relaxed() - atomic bitwise 811 * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
812 * @i: int value 812 * @i: int value
813 * @v: pointer to atomic_t 813 * @v: pointer to atomic_t
814 * 814 *
815 * Atomically updates @v to (@v & @i) with rel 815 * Atomically updates @v to (@v & @i) with relaxed ordering.
816 * 816 *
817 * Unsafe to use in noinstr code; use raw_atom 817 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
818 * 818 *
819 * Return: The original value of @v. 819 * Return: The original value of @v.
820 */ 820 */
821 static __always_inline int 821 static __always_inline int
822 atomic_fetch_and_relaxed(int i, atomic_t *v) 822 atomic_fetch_and_relaxed(int i, atomic_t *v)
823 { 823 {
824 instrument_atomic_read_write(v, sizeof 824 instrument_atomic_read_write(v, sizeof(*v));
825 return raw_atomic_fetch_and_relaxed(i, 825 return raw_atomic_fetch_and_relaxed(i, v);
826 } 826 }
827 827
828 /** 828 /**
829 * atomic_andnot() - atomic bitwise AND NOT wi 829 * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
830 * @i: int value 830 * @i: int value
831 * @v: pointer to atomic_t 831 * @v: pointer to atomic_t
832 * 832 *
833 * Atomically updates @v to (@v & ~@i) with re 833 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
834 * 834 *
835 * Unsafe to use in noinstr code; use raw_atom 835 * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
836 * 836 *
837 * Return: Nothing. 837 * Return: Nothing.
838 */ 838 */
839 static __always_inline void 839 static __always_inline void
840 atomic_andnot(int i, atomic_t *v) 840 atomic_andnot(int i, atomic_t *v)
841 { 841 {
842 instrument_atomic_read_write(v, sizeof 842 instrument_atomic_read_write(v, sizeof(*v));
843 raw_atomic_andnot(i, v); 843 raw_atomic_andnot(i, v);
844 } 844 }
845 845
846 /** 846 /**
847 * atomic_fetch_andnot() - atomic bitwise AND 847 * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
848 * @i: int value 848 * @i: int value
849 * @v: pointer to atomic_t 849 * @v: pointer to atomic_t
850 * 850 *
851 * Atomically updates @v to (@v & ~@i) with fu 851 * Atomically updates @v to (@v & ~@i) with full ordering.
852 * 852 *
853 * Unsafe to use in noinstr code; use raw_atom 853 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
854 * 854 *
855 * Return: The original value of @v. 855 * Return: The original value of @v.
856 */ 856 */
857 static __always_inline int 857 static __always_inline int
858 atomic_fetch_andnot(int i, atomic_t *v) 858 atomic_fetch_andnot(int i, atomic_t *v)
859 { 859 {
860 kcsan_mb(); 860 kcsan_mb();
861 instrument_atomic_read_write(v, sizeof 861 instrument_atomic_read_write(v, sizeof(*v));
862 return raw_atomic_fetch_andnot(i, v); 862 return raw_atomic_fetch_andnot(i, v);
863 } 863 }
864 864
865 /** 865 /**
866 * atomic_fetch_andnot_acquire() - atomic bitw 866 * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
867 * @i: int value 867 * @i: int value
868 * @v: pointer to atomic_t 868 * @v: pointer to atomic_t
869 * 869 *
870 * Atomically updates @v to (@v & ~@i) with ac 870 * Atomically updates @v to (@v & ~@i) with acquire ordering.
871 * 871 *
872 * Unsafe to use in noinstr code; use raw_atom 872 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
873 * 873 *
874 * Return: The original value of @v. 874 * Return: The original value of @v.
875 */ 875 */
876 static __always_inline int 876 static __always_inline int
877 atomic_fetch_andnot_acquire(int i, atomic_t *v 877 atomic_fetch_andnot_acquire(int i, atomic_t *v)
878 { 878 {
879 instrument_atomic_read_write(v, sizeof 879 instrument_atomic_read_write(v, sizeof(*v));
880 return raw_atomic_fetch_andnot_acquire 880 return raw_atomic_fetch_andnot_acquire(i, v);
881 } 881 }
882 882
883 /** 883 /**
884 * atomic_fetch_andnot_release() - atomic bitw 884 * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
885 * @i: int value 885 * @i: int value
886 * @v: pointer to atomic_t 886 * @v: pointer to atomic_t
887 * 887 *
888 * Atomically updates @v to (@v & ~@i) with re 888 * Atomically updates @v to (@v & ~@i) with release ordering.
889 * 889 *
890 * Unsafe to use in noinstr code; use raw_atom 890 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
891 * 891 *
892 * Return: The original value of @v. 892 * Return: The original value of @v.
893 */ 893 */
894 static __always_inline int 894 static __always_inline int
895 atomic_fetch_andnot_release(int i, atomic_t *v 895 atomic_fetch_andnot_release(int i, atomic_t *v)
896 { 896 {
897 kcsan_release(); 897 kcsan_release();
898 instrument_atomic_read_write(v, sizeof 898 instrument_atomic_read_write(v, sizeof(*v));
899 return raw_atomic_fetch_andnot_release 899 return raw_atomic_fetch_andnot_release(i, v);
900 } 900 }
901 901
902 /** 902 /**
903 * atomic_fetch_andnot_relaxed() - atomic bitw 903 * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
904 * @i: int value 904 * @i: int value
905 * @v: pointer to atomic_t 905 * @v: pointer to atomic_t
906 * 906 *
907 * Atomically updates @v to (@v & ~@i) with re 907 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
908 * 908 *
909 * Unsafe to use in noinstr code; use raw_atom 909 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
910 * 910 *
911 * Return: The original value of @v. 911 * Return: The original value of @v.
912 */ 912 */
913 static __always_inline int 913 static __always_inline int
914 atomic_fetch_andnot_relaxed(int i, atomic_t *v 914 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
915 { 915 {
916 instrument_atomic_read_write(v, sizeof 916 instrument_atomic_read_write(v, sizeof(*v));
917 return raw_atomic_fetch_andnot_relaxed 917 return raw_atomic_fetch_andnot_relaxed(i, v);
918 } 918 }
919 919
920 /** 920 /**
921 * atomic_or() - atomic bitwise OR with relaxe 921 * atomic_or() - atomic bitwise OR with relaxed ordering
922 * @i: int value 922 * @i: int value
923 * @v: pointer to atomic_t 923 * @v: pointer to atomic_t
924 * 924 *
925 * Atomically updates @v to (@v | @i) with rel 925 * Atomically updates @v to (@v | @i) with relaxed ordering.
926 * 926 *
927 * Unsafe to use in noinstr code; use raw_atom 927 * Unsafe to use in noinstr code; use raw_atomic_or() there.
928 * 928 *
929 * Return: Nothing. 929 * Return: Nothing.
930 */ 930 */
931 static __always_inline void 931 static __always_inline void
932 atomic_or(int i, atomic_t *v) 932 atomic_or(int i, atomic_t *v)
933 { 933 {
934 instrument_atomic_read_write(v, sizeof 934 instrument_atomic_read_write(v, sizeof(*v));
935 raw_atomic_or(i, v); 935 raw_atomic_or(i, v);
936 } 936 }
937 937
938 /** 938 /**
939 * atomic_fetch_or() - atomic bitwise OR with 939 * atomic_fetch_or() - atomic bitwise OR with full ordering
940 * @i: int value 940 * @i: int value
941 * @v: pointer to atomic_t 941 * @v: pointer to atomic_t
942 * 942 *
943 * Atomically updates @v to (@v | @i) with ful 943 * Atomically updates @v to (@v | @i) with full ordering.
944 * 944 *
945 * Unsafe to use in noinstr code; use raw_atom 945 * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
946 * 946 *
947 * Return: The original value of @v. 947 * Return: The original value of @v.
948 */ 948 */
949 static __always_inline int 949 static __always_inline int
950 atomic_fetch_or(int i, atomic_t *v) 950 atomic_fetch_or(int i, atomic_t *v)
951 { 951 {
952 kcsan_mb(); 952 kcsan_mb();
953 instrument_atomic_read_write(v, sizeof 953 instrument_atomic_read_write(v, sizeof(*v));
954 return raw_atomic_fetch_or(i, v); 954 return raw_atomic_fetch_or(i, v);
955 } 955 }
956 956
957 /** 957 /**
958 * atomic_fetch_or_acquire() - atomic bitwise 958 * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
959 * @i: int value 959 * @i: int value
960 * @v: pointer to atomic_t 960 * @v: pointer to atomic_t
961 * 961 *
962 * Atomically updates @v to (@v | @i) with acq 962 * Atomically updates @v to (@v | @i) with acquire ordering.
963 * 963 *
964 * Unsafe to use in noinstr code; use raw_atom 964 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
965 * 965 *
966 * Return: The original value of @v. 966 * Return: The original value of @v.
967 */ 967 */
968 static __always_inline int 968 static __always_inline int
969 atomic_fetch_or_acquire(int i, atomic_t *v) 969 atomic_fetch_or_acquire(int i, atomic_t *v)
970 { 970 {
971 instrument_atomic_read_write(v, sizeof 971 instrument_atomic_read_write(v, sizeof(*v));
972 return raw_atomic_fetch_or_acquire(i, 972 return raw_atomic_fetch_or_acquire(i, v);
973 } 973 }
974 974
975 /** 975 /**
976 * atomic_fetch_or_release() - atomic bitwise 976 * atomic_fetch_or_release() - atomic bitwise OR with release ordering
977 * @i: int value 977 * @i: int value
978 * @v: pointer to atomic_t 978 * @v: pointer to atomic_t
979 * 979 *
980 * Atomically updates @v to (@v | @i) with rel 980 * Atomically updates @v to (@v | @i) with release ordering.
981 * 981 *
982 * Unsafe to use in noinstr code; use raw_atom 982 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
983 * 983 *
984 * Return: The original value of @v. 984 * Return: The original value of @v.
985 */ 985 */
986 static __always_inline int 986 static __always_inline int
987 atomic_fetch_or_release(int i, atomic_t *v) 987 atomic_fetch_or_release(int i, atomic_t *v)
988 { 988 {
989 kcsan_release(); 989 kcsan_release();
990 instrument_atomic_read_write(v, sizeof 990 instrument_atomic_read_write(v, sizeof(*v));
991 return raw_atomic_fetch_or_release(i, 991 return raw_atomic_fetch_or_release(i, v);
992 } 992 }
993 993
994 /** 994 /**
995 * atomic_fetch_or_relaxed() - atomic bitwise 995 * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
996 * @i: int value 996 * @i: int value
997 * @v: pointer to atomic_t 997 * @v: pointer to atomic_t
998 * 998 *
999 * Atomically updates @v to (@v | @i) with rel 999 * Atomically updates @v to (@v | @i) with relaxed ordering.
1000 * 1000 *
1001 * Unsafe to use in noinstr code; use raw_ato 1001 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
1002 * 1002 *
1003 * Return: The original value of @v. 1003 * Return: The original value of @v.
1004 */ 1004 */
1005 static __always_inline int 1005 static __always_inline int
1006 atomic_fetch_or_relaxed(int i, atomic_t *v) 1006 atomic_fetch_or_relaxed(int i, atomic_t *v)
1007 { 1007 {
1008 instrument_atomic_read_write(v, sizeo 1008 instrument_atomic_read_write(v, sizeof(*v));
1009 return raw_atomic_fetch_or_relaxed(i, 1009 return raw_atomic_fetch_or_relaxed(i, v);
1010 } 1010 }
1011 1011
1012 /** 1012 /**
1013 * atomic_xor() - atomic bitwise XOR with rel 1013 * atomic_xor() - atomic bitwise XOR with relaxed ordering
1014 * @i: int value 1014 * @i: int value
1015 * @v: pointer to atomic_t 1015 * @v: pointer to atomic_t
1016 * 1016 *
1017 * Atomically updates @v to (@v ^ @i) with re 1017 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1018 * 1018 *
1019 * Unsafe to use in noinstr code; use raw_ato 1019 * Unsafe to use in noinstr code; use raw_atomic_xor() there.
1020 * 1020 *
1021 * Return: Nothing. 1021 * Return: Nothing.
1022 */ 1022 */
1023 static __always_inline void 1023 static __always_inline void
1024 atomic_xor(int i, atomic_t *v) 1024 atomic_xor(int i, atomic_t *v)
1025 { 1025 {
1026 instrument_atomic_read_write(v, sizeo 1026 instrument_atomic_read_write(v, sizeof(*v));
1027 raw_atomic_xor(i, v); 1027 raw_atomic_xor(i, v);
1028 } 1028 }
1029 1029
1030 /** 1030 /**
1031 * atomic_fetch_xor() - atomic bitwise XOR wi 1031 * atomic_fetch_xor() - atomic bitwise XOR with full ordering
1032 * @i: int value 1032 * @i: int value
1033 * @v: pointer to atomic_t 1033 * @v: pointer to atomic_t
1034 * 1034 *
1035 * Atomically updates @v to (@v ^ @i) with fu 1035 * Atomically updates @v to (@v ^ @i) with full ordering.
1036 * 1036 *
1037 * Unsafe to use in noinstr code; use raw_ato 1037 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
1038 * 1038 *
1039 * Return: The original value of @v. 1039 * Return: The original value of @v.
1040 */ 1040 */
1041 static __always_inline int 1041 static __always_inline int
1042 atomic_fetch_xor(int i, atomic_t *v) 1042 atomic_fetch_xor(int i, atomic_t *v)
1043 { 1043 {
1044 kcsan_mb(); 1044 kcsan_mb();
1045 instrument_atomic_read_write(v, sizeo 1045 instrument_atomic_read_write(v, sizeof(*v));
1046 return raw_atomic_fetch_xor(i, v); 1046 return raw_atomic_fetch_xor(i, v);
1047 } 1047 }
1048 1048
1049 /** 1049 /**
1050 * atomic_fetch_xor_acquire() - atomic bitwis 1050 * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1051 * @i: int value 1051 * @i: int value
1052 * @v: pointer to atomic_t 1052 * @v: pointer to atomic_t
1053 * 1053 *
1054 * Atomically updates @v to (@v ^ @i) with ac 1054 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1055 * 1055 *
1056 * Unsafe to use in noinstr code; use raw_ato 1056 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
1057 * 1057 *
1058 * Return: The original value of @v. 1058 * Return: The original value of @v.
1059 */ 1059 */
1060 static __always_inline int 1060 static __always_inline int
1061 atomic_fetch_xor_acquire(int i, atomic_t *v) 1061 atomic_fetch_xor_acquire(int i, atomic_t *v)
1062 { 1062 {
1063 instrument_atomic_read_write(v, sizeo 1063 instrument_atomic_read_write(v, sizeof(*v));
1064 return raw_atomic_fetch_xor_acquire(i 1064 return raw_atomic_fetch_xor_acquire(i, v);
1065 } 1065 }
1066 1066
1067 /** 1067 /**
1068 * atomic_fetch_xor_release() - atomic bitwis 1068 * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1069 * @i: int value 1069 * @i: int value
1070 * @v: pointer to atomic_t 1070 * @v: pointer to atomic_t
1071 * 1071 *
1072 * Atomically updates @v to (@v ^ @i) with re 1072 * Atomically updates @v to (@v ^ @i) with release ordering.
1073 * 1073 *
1074 * Unsafe to use in noinstr code; use raw_ato 1074 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
1075 * 1075 *
1076 * Return: The original value of @v. 1076 * Return: The original value of @v.
1077 */ 1077 */
1078 static __always_inline int 1078 static __always_inline int
1079 atomic_fetch_xor_release(int i, atomic_t *v) 1079 atomic_fetch_xor_release(int i, atomic_t *v)
1080 { 1080 {
1081 kcsan_release(); 1081 kcsan_release();
1082 instrument_atomic_read_write(v, sizeo 1082 instrument_atomic_read_write(v, sizeof(*v));
1083 return raw_atomic_fetch_xor_release(i 1083 return raw_atomic_fetch_xor_release(i, v);
1084 } 1084 }
1085 1085
1086 /** 1086 /**
1087 * atomic_fetch_xor_relaxed() - atomic bitwis 1087 * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1088 * @i: int value 1088 * @i: int value
1089 * @v: pointer to atomic_t 1089 * @v: pointer to atomic_t
1090 * 1090 *
1091 * Atomically updates @v to (@v ^ @i) with re 1091 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1092 * 1092 *
1093 * Unsafe to use in noinstr code; use raw_ato 1093 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
1094 * 1094 *
1095 * Return: The original value of @v. 1095 * Return: The original value of @v.
1096 */ 1096 */
1097 static __always_inline int 1097 static __always_inline int
1098 atomic_fetch_xor_relaxed(int i, atomic_t *v) 1098 atomic_fetch_xor_relaxed(int i, atomic_t *v)
1099 { 1099 {
1100 instrument_atomic_read_write(v, sizeo 1100 instrument_atomic_read_write(v, sizeof(*v));
1101 return raw_atomic_fetch_xor_relaxed(i 1101 return raw_atomic_fetch_xor_relaxed(i, v);
1102 } 1102 }
1103 1103
1104 /** 1104 /**
1105 * atomic_xchg() - atomic exchange with full 1105 * atomic_xchg() - atomic exchange with full ordering
1106 * @v: pointer to atomic_t 1106 * @v: pointer to atomic_t
1107 * @new: int value to assign 1107 * @new: int value to assign
1108 * 1108 *
1109 * Atomically updates @v to @new with full or 1109 * Atomically updates @v to @new with full ordering.
1110 * 1110 *
1111 * Unsafe to use in noinstr code; use raw_ato 1111 * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
1112 * 1112 *
1113 * Return: The original value of @v. 1113 * Return: The original value of @v.
1114 */ 1114 */
1115 static __always_inline int 1115 static __always_inline int
1116 atomic_xchg(atomic_t *v, int new) 1116 atomic_xchg(atomic_t *v, int new)
1117 { 1117 {
1118 kcsan_mb(); 1118 kcsan_mb();
1119 instrument_atomic_read_write(v, sizeo 1119 instrument_atomic_read_write(v, sizeof(*v));
1120 return raw_atomic_xchg(v, new); 1120 return raw_atomic_xchg(v, new);
1121 } 1121 }
1122 1122
1123 /** 1123 /**
1124 * atomic_xchg_acquire() - atomic exchange wi 1124 * atomic_xchg_acquire() - atomic exchange with acquire ordering
1125 * @v: pointer to atomic_t 1125 * @v: pointer to atomic_t
1126 * @new: int value to assign 1126 * @new: int value to assign
1127 * 1127 *
1128 * Atomically updates @v to @new with acquire 1128 * Atomically updates @v to @new with acquire ordering.
1129 * 1129 *
1130 * Unsafe to use in noinstr code; use raw_ato 1130 * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
1131 * 1131 *
1132 * Return: The original value of @v. 1132 * Return: The original value of @v.
1133 */ 1133 */
1134 static __always_inline int 1134 static __always_inline int
1135 atomic_xchg_acquire(atomic_t *v, int new) 1135 atomic_xchg_acquire(atomic_t *v, int new)
1136 { 1136 {
1137 instrument_atomic_read_write(v, sizeo 1137 instrument_atomic_read_write(v, sizeof(*v));
1138 return raw_atomic_xchg_acquire(v, new 1138 return raw_atomic_xchg_acquire(v, new);
1139 } 1139 }
1140 1140
1141 /** 1141 /**
1142 * atomic_xchg_release() - atomic exchange wi 1142 * atomic_xchg_release() - atomic exchange with release ordering
1143 * @v: pointer to atomic_t 1143 * @v: pointer to atomic_t
1144 * @new: int value to assign 1144 * @new: int value to assign
1145 * 1145 *
1146 * Atomically updates @v to @new with release 1146 * Atomically updates @v to @new with release ordering.
1147 * 1147 *
1148 * Unsafe to use in noinstr code; use raw_ato 1148 * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
1149 * 1149 *
1150 * Return: The original value of @v. 1150 * Return: The original value of @v.
1151 */ 1151 */
1152 static __always_inline int 1152 static __always_inline int
1153 atomic_xchg_release(atomic_t *v, int new) 1153 atomic_xchg_release(atomic_t *v, int new)
1154 { 1154 {
1155 kcsan_release(); 1155 kcsan_release();
1156 instrument_atomic_read_write(v, sizeo 1156 instrument_atomic_read_write(v, sizeof(*v));
1157 return raw_atomic_xchg_release(v, new 1157 return raw_atomic_xchg_release(v, new);
1158 } 1158 }
1159 1159
1160 /** 1160 /**
1161 * atomic_xchg_relaxed() - atomic exchange wi 1161 * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1162 * @v: pointer to atomic_t 1162 * @v: pointer to atomic_t
1163 * @new: int value to assign 1163 * @new: int value to assign
1164 * 1164 *
1165 * Atomically updates @v to @new with relaxed 1165 * Atomically updates @v to @new with relaxed ordering.
1166 * 1166 *
1167 * Unsafe to use in noinstr code; use raw_ato 1167 * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
1168 * 1168 *
1169 * Return: The original value of @v. 1169 * Return: The original value of @v.
1170 */ 1170 */
1171 static __always_inline int 1171 static __always_inline int
1172 atomic_xchg_relaxed(atomic_t *v, int new) 1172 atomic_xchg_relaxed(atomic_t *v, int new)
1173 { 1173 {
1174 instrument_atomic_read_write(v, sizeo 1174 instrument_atomic_read_write(v, sizeof(*v));
1175 return raw_atomic_xchg_relaxed(v, new 1175 return raw_atomic_xchg_relaxed(v, new);
1176 } 1176 }
1177 1177
1178 /** 1178 /**
1179 * atomic_cmpxchg() - atomic compare and exch 1179 * atomic_cmpxchg() - atomic compare and exchange with full ordering
1180 * @v: pointer to atomic_t 1180 * @v: pointer to atomic_t
1181 * @old: int value to compare with 1181 * @old: int value to compare with
1182 * @new: int value to assign 1182 * @new: int value to assign
1183 * 1183 *
1184 * If (@v == @old), atomically updates @v to 1184 * If (@v == @old), atomically updates @v to @new with full ordering.
1185 * Otherwise, @v is not modified and relaxed <<
1186 * 1185 *
1187 * Unsafe to use in noinstr code; use raw_ato 1186 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
1188 * 1187 *
1189 * Return: The original value of @v. 1188 * Return: The original value of @v.
1190 */ 1189 */
1191 static __always_inline int 1190 static __always_inline int
1192 atomic_cmpxchg(atomic_t *v, int old, int new) 1191 atomic_cmpxchg(atomic_t *v, int old, int new)
1193 { 1192 {
1194 kcsan_mb(); 1193 kcsan_mb();
1195 instrument_atomic_read_write(v, sizeo 1194 instrument_atomic_read_write(v, sizeof(*v));
1196 return raw_atomic_cmpxchg(v, old, new 1195 return raw_atomic_cmpxchg(v, old, new);
1197 } 1196 }
1198 1197
1199 /** 1198 /**
1200 * atomic_cmpxchg_acquire() - atomic compare 1199 * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1201 * @v: pointer to atomic_t 1200 * @v: pointer to atomic_t
1202 * @old: int value to compare with 1201 * @old: int value to compare with
1203 * @new: int value to assign 1202 * @new: int value to assign
1204 * 1203 *
1205 * If (@v == @old), atomically updates @v to 1204 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1206 * Otherwise, @v is not modified and relaxed <<
1207 * 1205 *
1208 * Unsafe to use in noinstr code; use raw_ato 1206 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
1209 * 1207 *
1210 * Return: The original value of @v. 1208 * Return: The original value of @v.
1211 */ 1209 */
1212 static __always_inline int 1210 static __always_inline int
1213 atomic_cmpxchg_acquire(atomic_t *v, int old, 1211 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1214 { 1212 {
1215 instrument_atomic_read_write(v, sizeo 1213 instrument_atomic_read_write(v, sizeof(*v));
1216 return raw_atomic_cmpxchg_acquire(v, 1214 return raw_atomic_cmpxchg_acquire(v, old, new);
1217 } 1215 }
1218 1216
1219 /** 1217 /**
1220 * atomic_cmpxchg_release() - atomic compare 1218 * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
1221 * @v: pointer to atomic_t 1219 * @v: pointer to atomic_t
1222 * @old: int value to compare with 1220 * @old: int value to compare with
1223 * @new: int value to assign 1221 * @new: int value to assign
1224 * 1222 *
1225 * If (@v == @old), atomically updates @v to 1223 * If (@v == @old), atomically updates @v to @new with release ordering.
1226 * Otherwise, @v is not modified and relaxed <<
1227 * 1224 *
1228 * Unsafe to use in noinstr code; use raw_ato 1225 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
1229 * 1226 *
1230 * Return: The original value of @v. 1227 * Return: The original value of @v.
1231 */ 1228 */
1232 static __always_inline int 1229 static __always_inline int
1233 atomic_cmpxchg_release(atomic_t *v, int old, 1230 atomic_cmpxchg_release(atomic_t *v, int old, int new)
1234 { 1231 {
1235 kcsan_release(); 1232 kcsan_release();
1236 instrument_atomic_read_write(v, sizeo 1233 instrument_atomic_read_write(v, sizeof(*v));
1237 return raw_atomic_cmpxchg_release(v, 1234 return raw_atomic_cmpxchg_release(v, old, new);
1238 } 1235 }
1239 1236
1240 /** 1237 /**
1241 * atomic_cmpxchg_relaxed() - atomic compare 1238 * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1242 * @v: pointer to atomic_t 1239 * @v: pointer to atomic_t
1243 * @old: int value to compare with 1240 * @old: int value to compare with
1244 * @new: int value to assign 1241 * @new: int value to assign
1245 * 1242 *
1246 * If (@v == @old), atomically updates @v to 1243 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1247 * Otherwise, @v is not modified and relaxed <<
1248 * 1244 *
1249 * Unsafe to use in noinstr code; use raw_ato 1245 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
1250 * 1246 *
1251 * Return: The original value of @v. 1247 * Return: The original value of @v.
1252 */ 1248 */
1253 static __always_inline int 1249 static __always_inline int
1254 atomic_cmpxchg_relaxed(atomic_t *v, int old, 1250 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
1255 { 1251 {
1256 instrument_atomic_read_write(v, sizeo 1252 instrument_atomic_read_write(v, sizeof(*v));
1257 return raw_atomic_cmpxchg_relaxed(v, 1253 return raw_atomic_cmpxchg_relaxed(v, old, new);
1258 } 1254 }
1259 1255
1260 /** 1256 /**
1261 * atomic_try_cmpxchg() - atomic compare and 1257 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
1262 * @v: pointer to atomic_t 1258 * @v: pointer to atomic_t
1263 * @old: pointer to int value to compare with 1259 * @old: pointer to int value to compare with
1264 * @new: int value to assign 1260 * @new: int value to assign
1265 * 1261 *
1266 * If (@v == @old), atomically updates @v to 1262 * If (@v == @old), atomically updates @v to @new with full ordering.
1267 * Otherwise, @v is not modified, @old is upd !! 1263 * Otherwise, updates @old to the current value of @v.
1268 * and relaxed ordering is provided. <<
1269 * 1264 *
1270 * Unsafe to use in noinstr code; use raw_ato 1265 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
1271 * 1266 *
1272 * Return: @true if the exchange occured, @fa 1267 * Return: @true if the exchange occured, @false otherwise.
1273 */ 1268 */
1274 static __always_inline bool 1269 static __always_inline bool
1275 atomic_try_cmpxchg(atomic_t *v, int *old, int 1270 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1276 { 1271 {
1277 kcsan_mb(); 1272 kcsan_mb();
1278 instrument_atomic_read_write(v, sizeo 1273 instrument_atomic_read_write(v, sizeof(*v));
1279 instrument_atomic_read_write(old, siz 1274 instrument_atomic_read_write(old, sizeof(*old));
1280 return raw_atomic_try_cmpxchg(v, old, 1275 return raw_atomic_try_cmpxchg(v, old, new);
1281 } 1276 }
1282 1277
1283 /** 1278 /**
1284 * atomic_try_cmpxchg_acquire() - atomic comp 1279 * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1285 * @v: pointer to atomic_t 1280 * @v: pointer to atomic_t
1286 * @old: pointer to int value to compare with 1281 * @old: pointer to int value to compare with
1287 * @new: int value to assign 1282 * @new: int value to assign
1288 * 1283 *
1289 * If (@v == @old), atomically updates @v to 1284 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1290 * Otherwise, @v is not modified, @old is upd !! 1285 * Otherwise, updates @old to the current value of @v.
1291 * and relaxed ordering is provided. <<
1292 * 1286 *
1293 * Unsafe to use in noinstr code; use raw_ato 1287 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
1294 * 1288 *
1295 * Return: @true if the exchange occured, @fa 1289 * Return: @true if the exchange occured, @false otherwise.
1296 */ 1290 */
1297 static __always_inline bool 1291 static __always_inline bool
1298 atomic_try_cmpxchg_acquire(atomic_t *v, int * 1292 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1299 { 1293 {
1300 instrument_atomic_read_write(v, sizeo 1294 instrument_atomic_read_write(v, sizeof(*v));
1301 instrument_atomic_read_write(old, siz 1295 instrument_atomic_read_write(old, sizeof(*old));
1302 return raw_atomic_try_cmpxchg_acquire 1296 return raw_atomic_try_cmpxchg_acquire(v, old, new);
1303 } 1297 }
1304 1298
1305 /** 1299 /**
1306 * atomic_try_cmpxchg_release() - atomic comp 1300 * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
1307 * @v: pointer to atomic_t 1301 * @v: pointer to atomic_t
1308 * @old: pointer to int value to compare with 1302 * @old: pointer to int value to compare with
1309 * @new: int value to assign 1303 * @new: int value to assign
1310 * 1304 *
1311 * If (@v == @old), atomically updates @v to 1305 * If (@v == @old), atomically updates @v to @new with release ordering.
1312 * Otherwise, @v is not modified, @old is upd !! 1306 * Otherwise, updates @old to the current value of @v.
1313 * and relaxed ordering is provided. <<
1314 * 1307 *
1315 * Unsafe to use in noinstr code; use raw_ato 1308 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
1316 * 1309 *
1317 * Return: @true if the exchange occured, @fa 1310 * Return: @true if the exchange occured, @false otherwise.
1318 */ 1311 */
1319 static __always_inline bool 1312 static __always_inline bool
1320 atomic_try_cmpxchg_release(atomic_t *v, int * 1313 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1321 { 1314 {
1322 kcsan_release(); 1315 kcsan_release();
1323 instrument_atomic_read_write(v, sizeo 1316 instrument_atomic_read_write(v, sizeof(*v));
1324 instrument_atomic_read_write(old, siz 1317 instrument_atomic_read_write(old, sizeof(*old));
1325 return raw_atomic_try_cmpxchg_release 1318 return raw_atomic_try_cmpxchg_release(v, old, new);
1326 } 1319 }
1327 1320
1328 /** 1321 /**
1329 * atomic_try_cmpxchg_relaxed() - atomic comp 1322 * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1330 * @v: pointer to atomic_t 1323 * @v: pointer to atomic_t
1331 * @old: pointer to int value to compare with 1324 * @old: pointer to int value to compare with
1332 * @new: int value to assign 1325 * @new: int value to assign
1333 * 1326 *
1334 * If (@v == @old), atomically updates @v to 1327 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1335 * Otherwise, @v is not modified, @old is upd !! 1328 * Otherwise, updates @old to the current value of @v.
1336 * and relaxed ordering is provided. <<
1337 * 1329 *
1338 * Unsafe to use in noinstr code; use raw_ato 1330 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
1339 * 1331 *
1340 * Return: @true if the exchange occured, @fa 1332 * Return: @true if the exchange occured, @false otherwise.
1341 */ 1333 */
1342 static __always_inline bool 1334 static __always_inline bool
1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int * 1335 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1344 { 1336 {
1345 instrument_atomic_read_write(v, sizeo 1337 instrument_atomic_read_write(v, sizeof(*v));
1346 instrument_atomic_read_write(old, siz 1338 instrument_atomic_read_write(old, sizeof(*old));
1347 return raw_atomic_try_cmpxchg_relaxed 1339 return raw_atomic_try_cmpxchg_relaxed(v, old, new);
1348 } 1340 }
1349 1341
1350 /** 1342 /**
1351 * atomic_sub_and_test() - atomic subtract an 1343 * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
1352 * @i: int value to subtract !! 1344 * @i: int value to add
1353 * @v: pointer to atomic_t 1345 * @v: pointer to atomic_t
1354 * 1346 *
1355 * Atomically updates @v to (@v - @i) with fu 1347 * Atomically updates @v to (@v - @i) with full ordering.
1356 * 1348 *
1357 * Unsafe to use in noinstr code; use raw_ato 1349 * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
1358 * 1350 *
1359 * Return: @true if the resulting value of @v 1351 * Return: @true if the resulting value of @v is zero, @false otherwise.
1360 */ 1352 */
1361 static __always_inline bool 1353 static __always_inline bool
1362 atomic_sub_and_test(int i, atomic_t *v) 1354 atomic_sub_and_test(int i, atomic_t *v)
1363 { 1355 {
1364 kcsan_mb(); 1356 kcsan_mb();
1365 instrument_atomic_read_write(v, sizeo 1357 instrument_atomic_read_write(v, sizeof(*v));
1366 return raw_atomic_sub_and_test(i, v); 1358 return raw_atomic_sub_and_test(i, v);
1367 } 1359 }
1368 1360
1369 /** 1361 /**
1370 * atomic_dec_and_test() - atomic decrement a 1362 * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
1371 * @v: pointer to atomic_t 1363 * @v: pointer to atomic_t
1372 * 1364 *
1373 * Atomically updates @v to (@v - 1) with ful 1365 * Atomically updates @v to (@v - 1) with full ordering.
1374 * 1366 *
1375 * Unsafe to use in noinstr code; use raw_ato 1367 * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
1376 * 1368 *
1377 * Return: @true if the resulting value of @v 1369 * Return: @true if the resulting value of @v is zero, @false otherwise.
1378 */ 1370 */
1379 static __always_inline bool 1371 static __always_inline bool
1380 atomic_dec_and_test(atomic_t *v) 1372 atomic_dec_and_test(atomic_t *v)
1381 { 1373 {
1382 kcsan_mb(); 1374 kcsan_mb();
1383 instrument_atomic_read_write(v, sizeo 1375 instrument_atomic_read_write(v, sizeof(*v));
1384 return raw_atomic_dec_and_test(v); 1376 return raw_atomic_dec_and_test(v);
1385 } 1377 }
1386 1378
1387 /** 1379 /**
1388 * atomic_inc_and_test() - atomic increment a 1380 * atomic_inc_and_test() - atomic increment and test if zero with full ordering
1389 * @v: pointer to atomic_t 1381 * @v: pointer to atomic_t
1390 * 1382 *
1391 * Atomically updates @v to (@v + 1) with ful 1383 * Atomically updates @v to (@v + 1) with full ordering.
1392 * 1384 *
1393 * Unsafe to use in noinstr code; use raw_ato 1385 * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
1394 * 1386 *
1395 * Return: @true if the resulting value of @v 1387 * Return: @true if the resulting value of @v is zero, @false otherwise.
1396 */ 1388 */
1397 static __always_inline bool 1389 static __always_inline bool
1398 atomic_inc_and_test(atomic_t *v) 1390 atomic_inc_and_test(atomic_t *v)
1399 { 1391 {
1400 kcsan_mb(); 1392 kcsan_mb();
1401 instrument_atomic_read_write(v, sizeo 1393 instrument_atomic_read_write(v, sizeof(*v));
1402 return raw_atomic_inc_and_test(v); 1394 return raw_atomic_inc_and_test(v);
1403 } 1395 }
1404 1396
1405 /** 1397 /**
1406 * atomic_add_negative() - atomic add and tes 1398 * atomic_add_negative() - atomic add and test if negative with full ordering
1407 * @i: int value to add 1399 * @i: int value to add
1408 * @v: pointer to atomic_t 1400 * @v: pointer to atomic_t
1409 * 1401 *
1410 * Atomically updates @v to (@v + @i) with fu 1402 * Atomically updates @v to (@v + @i) with full ordering.
1411 * 1403 *
1412 * Unsafe to use in noinstr code; use raw_ato 1404 * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
1413 * 1405 *
1414 * Return: @true if the resulting value of @v 1406 * Return: @true if the resulting value of @v is negative, @false otherwise.
1415 */ 1407 */
1416 static __always_inline bool 1408 static __always_inline bool
1417 atomic_add_negative(int i, atomic_t *v) 1409 atomic_add_negative(int i, atomic_t *v)
1418 { 1410 {
1419 kcsan_mb(); 1411 kcsan_mb();
1420 instrument_atomic_read_write(v, sizeo 1412 instrument_atomic_read_write(v, sizeof(*v));
1421 return raw_atomic_add_negative(i, v); 1413 return raw_atomic_add_negative(i, v);
1422 } 1414 }
1423 1415
1424 /** 1416 /**
1425 * atomic_add_negative_acquire() - atomic add 1417 * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
1426 * @i: int value to add 1418 * @i: int value to add
1427 * @v: pointer to atomic_t 1419 * @v: pointer to atomic_t
1428 * 1420 *
1429 * Atomically updates @v to (@v + @i) with ac 1421 * Atomically updates @v to (@v + @i) with acquire ordering.
1430 * 1422 *
1431 * Unsafe to use in noinstr code; use raw_ato 1423 * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
1432 * 1424 *
1433 * Return: @true if the resulting value of @v 1425 * Return: @true if the resulting value of @v is negative, @false otherwise.
1434 */ 1426 */
1435 static __always_inline bool 1427 static __always_inline bool
1436 atomic_add_negative_acquire(int i, atomic_t * 1428 atomic_add_negative_acquire(int i, atomic_t *v)
1437 { 1429 {
1438 instrument_atomic_read_write(v, sizeo 1430 instrument_atomic_read_write(v, sizeof(*v));
1439 return raw_atomic_add_negative_acquir 1431 return raw_atomic_add_negative_acquire(i, v);
1440 } 1432 }
1441 1433
1442 /** 1434 /**
1443 * atomic_add_negative_release() - atomic add 1435 * atomic_add_negative_release() - atomic add and test if negative with release ordering
1444 * @i: int value to add 1436 * @i: int value to add
1445 * @v: pointer to atomic_t 1437 * @v: pointer to atomic_t
1446 * 1438 *
1447 * Atomically updates @v to (@v + @i) with re 1439 * Atomically updates @v to (@v + @i) with release ordering.
1448 * 1440 *
1449 * Unsafe to use in noinstr code; use raw_ato 1441 * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
1450 * 1442 *
1451 * Return: @true if the resulting value of @v 1443 * Return: @true if the resulting value of @v is negative, @false otherwise.
1452 */ 1444 */
1453 static __always_inline bool 1445 static __always_inline bool
1454 atomic_add_negative_release(int i, atomic_t * 1446 atomic_add_negative_release(int i, atomic_t *v)
1455 { 1447 {
1456 kcsan_release(); 1448 kcsan_release();
1457 instrument_atomic_read_write(v, sizeo 1449 instrument_atomic_read_write(v, sizeof(*v));
1458 return raw_atomic_add_negative_releas 1450 return raw_atomic_add_negative_release(i, v);
1459 } 1451 }
1460 1452
1461 /** 1453 /**
1462 * atomic_add_negative_relaxed() - atomic add 1454 * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
1463 * @i: int value to add 1455 * @i: int value to add
1464 * @v: pointer to atomic_t 1456 * @v: pointer to atomic_t
1465 * 1457 *
1466 * Atomically updates @v to (@v + @i) with re 1458 * Atomically updates @v to (@v + @i) with relaxed ordering.
1467 * 1459 *
1468 * Unsafe to use in noinstr code; use raw_ato 1460 * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
1469 * 1461 *
1470 * Return: @true if the resulting value of @v 1462 * Return: @true if the resulting value of @v is negative, @false otherwise.
1471 */ 1463 */
1472 static __always_inline bool 1464 static __always_inline bool
1473 atomic_add_negative_relaxed(int i, atomic_t * 1465 atomic_add_negative_relaxed(int i, atomic_t *v)
1474 { 1466 {
1475 instrument_atomic_read_write(v, sizeo 1467 instrument_atomic_read_write(v, sizeof(*v));
1476 return raw_atomic_add_negative_relaxe 1468 return raw_atomic_add_negative_relaxed(i, v);
1477 } 1469 }
1478 1470
1479 /** 1471 /**
1480 * atomic_fetch_add_unless() - atomic add unl 1472 * atomic_fetch_add_unless() - atomic add unless value with full ordering
1481 * @v: pointer to atomic_t 1473 * @v: pointer to atomic_t
1482 * @a: int value to add 1474 * @a: int value to add
1483 * @u: int value to compare with 1475 * @u: int value to compare with
1484 * 1476 *
1485 * If (@v != @u), atomically updates @v to (@ 1477 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1486 * Otherwise, @v is not modified and relaxed <<
1487 * 1478 *
1488 * Unsafe to use in noinstr code; use raw_ato 1479 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
1489 * 1480 *
1490 * Return: The original value of @v. 1481 * Return: The original value of @v.
1491 */ 1482 */
1492 static __always_inline int 1483 static __always_inline int
1493 atomic_fetch_add_unless(atomic_t *v, int a, i 1484 atomic_fetch_add_unless(atomic_t *v, int a, int u)
1494 { 1485 {
1495 kcsan_mb(); 1486 kcsan_mb();
1496 instrument_atomic_read_write(v, sizeo 1487 instrument_atomic_read_write(v, sizeof(*v));
1497 return raw_atomic_fetch_add_unless(v, 1488 return raw_atomic_fetch_add_unless(v, a, u);
1498 } 1489 }
1499 1490
1500 /** 1491 /**
1501 * atomic_add_unless() - atomic add unless va 1492 * atomic_add_unless() - atomic add unless value with full ordering
1502 * @v: pointer to atomic_t 1493 * @v: pointer to atomic_t
1503 * @a: int value to add 1494 * @a: int value to add
1504 * @u: int value to compare with 1495 * @u: int value to compare with
1505 * 1496 *
1506 * If (@v != @u), atomically updates @v to (@ 1497 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1507 * Otherwise, @v is not modified and relaxed <<
1508 * 1498 *
1509 * Unsafe to use in noinstr code; use raw_ato 1499 * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
1510 * 1500 *
1511 * Return: @true if @v was updated, @false ot 1501 * Return: @true if @v was updated, @false otherwise.
1512 */ 1502 */
1513 static __always_inline bool 1503 static __always_inline bool
1514 atomic_add_unless(atomic_t *v, int a, int u) 1504 atomic_add_unless(atomic_t *v, int a, int u)
1515 { 1505 {
1516 kcsan_mb(); 1506 kcsan_mb();
1517 instrument_atomic_read_write(v, sizeo 1507 instrument_atomic_read_write(v, sizeof(*v));
1518 return raw_atomic_add_unless(v, a, u) 1508 return raw_atomic_add_unless(v, a, u);
1519 } 1509 }
1520 1510
1521 /** 1511 /**
1522 * atomic_inc_not_zero() - atomic increment u 1512 * atomic_inc_not_zero() - atomic increment unless zero with full ordering
1523 * @v: pointer to atomic_t 1513 * @v: pointer to atomic_t
1524 * 1514 *
1525 * If (@v != 0), atomically updates @v to (@v 1515 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
1526 * Otherwise, @v is not modified and relaxed <<
1527 * 1516 *
1528 * Unsafe to use in noinstr code; use raw_ato 1517 * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
1529 * 1518 *
1530 * Return: @true if @v was updated, @false ot 1519 * Return: @true if @v was updated, @false otherwise.
1531 */ 1520 */
1532 static __always_inline bool 1521 static __always_inline bool
1533 atomic_inc_not_zero(atomic_t *v) 1522 atomic_inc_not_zero(atomic_t *v)
1534 { 1523 {
1535 kcsan_mb(); 1524 kcsan_mb();
1536 instrument_atomic_read_write(v, sizeo 1525 instrument_atomic_read_write(v, sizeof(*v));
1537 return raw_atomic_inc_not_zero(v); 1526 return raw_atomic_inc_not_zero(v);
1538 } 1527 }
1539 1528
1540 /** 1529 /**
1541 * atomic_inc_unless_negative() - atomic incr 1530 * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
1542 * @v: pointer to atomic_t 1531 * @v: pointer to atomic_t
1543 * 1532 *
1544 * If (@v >= 0), atomically updates @v to (@v 1533 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
1545 * Otherwise, @v is not modified and relaxed <<
1546 * 1534 *
1547 * Unsafe to use in noinstr code; use raw_ato 1535 * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
1548 * 1536 *
1549 * Return: @true if @v was updated, @false ot 1537 * Return: @true if @v was updated, @false otherwise.
1550 */ 1538 */
1551 static __always_inline bool 1539 static __always_inline bool
1552 atomic_inc_unless_negative(atomic_t *v) 1540 atomic_inc_unless_negative(atomic_t *v)
1553 { 1541 {
1554 kcsan_mb(); 1542 kcsan_mb();
1555 instrument_atomic_read_write(v, sizeo 1543 instrument_atomic_read_write(v, sizeof(*v));
1556 return raw_atomic_inc_unless_negative 1544 return raw_atomic_inc_unless_negative(v);
1557 } 1545 }
1558 1546
1559 /** 1547 /**
1560 * atomic_dec_unless_positive() - atomic decr 1548 * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
1561 * @v: pointer to atomic_t 1549 * @v: pointer to atomic_t
1562 * 1550 *
1563 * If (@v <= 0), atomically updates @v to (@v 1551 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
1564 * Otherwise, @v is not modified and relaxed <<
1565 * 1552 *
1566 * Unsafe to use in noinstr code; use raw_ato 1553 * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
1567 * 1554 *
1568 * Return: @true if @v was updated, @false ot 1555 * Return: @true if @v was updated, @false otherwise.
1569 */ 1556 */
1570 static __always_inline bool 1557 static __always_inline bool
1571 atomic_dec_unless_positive(atomic_t *v) 1558 atomic_dec_unless_positive(atomic_t *v)
1572 { 1559 {
1573 kcsan_mb(); 1560 kcsan_mb();
1574 instrument_atomic_read_write(v, sizeo 1561 instrument_atomic_read_write(v, sizeof(*v));
1575 return raw_atomic_dec_unless_positive 1562 return raw_atomic_dec_unless_positive(v);
1576 } 1563 }
1577 1564
1578 /** 1565 /**
1579 * atomic_dec_if_positive() - atomic decremen 1566 * atomic_dec_if_positive() - atomic decrement if positive with full ordering
1580 * @v: pointer to atomic_t 1567 * @v: pointer to atomic_t
1581 * 1568 *
1582 * If (@v > 0), atomically updates @v to (@v 1569 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
1583 * Otherwise, @v is not modified and relaxed <<
1584 * 1570 *
1585 * Unsafe to use in noinstr code; use raw_ato 1571 * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
1586 * 1572 *
1587 * Return: The old value of (@v - 1), regardl 1573 * Return: The old value of (@v - 1), regardless of whether @v was updated.
1588 */ 1574 */
1589 static __always_inline int 1575 static __always_inline int
1590 atomic_dec_if_positive(atomic_t *v) 1576 atomic_dec_if_positive(atomic_t *v)
1591 { 1577 {
1592 kcsan_mb(); 1578 kcsan_mb();
1593 instrument_atomic_read_write(v, sizeo 1579 instrument_atomic_read_write(v, sizeof(*v));
1594 return raw_atomic_dec_if_positive(v); 1580 return raw_atomic_dec_if_positive(v);
1595 } 1581 }
1596 1582
1597 /** 1583 /**
1598 * atomic64_read() - atomic load with relaxed 1584 * atomic64_read() - atomic load with relaxed ordering
1599 * @v: pointer to atomic64_t 1585 * @v: pointer to atomic64_t
1600 * 1586 *
1601 * Atomically loads the value of @v with rela 1587 * Atomically loads the value of @v with relaxed ordering.
1602 * 1588 *
1603 * Unsafe to use in noinstr code; use raw_ato 1589 * Unsafe to use in noinstr code; use raw_atomic64_read() there.
1604 * 1590 *
1605 * Return: The value loaded from @v. 1591 * Return: The value loaded from @v.
1606 */ 1592 */
1607 static __always_inline s64 1593 static __always_inline s64
1608 atomic64_read(const atomic64_t *v) 1594 atomic64_read(const atomic64_t *v)
1609 { 1595 {
1610 instrument_atomic_read(v, sizeof(*v)) 1596 instrument_atomic_read(v, sizeof(*v));
1611 return raw_atomic64_read(v); 1597 return raw_atomic64_read(v);
1612 } 1598 }
1613 1599
1614 /** 1600 /**
1615 * atomic64_read_acquire() - atomic load with 1601 * atomic64_read_acquire() - atomic load with acquire ordering
1616 * @v: pointer to atomic64_t 1602 * @v: pointer to atomic64_t
1617 * 1603 *
1618 * Atomically loads the value of @v with acqu 1604 * Atomically loads the value of @v with acquire ordering.
1619 * 1605 *
1620 * Unsafe to use in noinstr code; use raw_ato 1606 * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
1621 * 1607 *
1622 * Return: The value loaded from @v. 1608 * Return: The value loaded from @v.
1623 */ 1609 */
1624 static __always_inline s64 1610 static __always_inline s64
1625 atomic64_read_acquire(const atomic64_t *v) 1611 atomic64_read_acquire(const atomic64_t *v)
1626 { 1612 {
1627 instrument_atomic_read(v, sizeof(*v)) 1613 instrument_atomic_read(v, sizeof(*v));
1628 return raw_atomic64_read_acquire(v); 1614 return raw_atomic64_read_acquire(v);
1629 } 1615 }
1630 1616
1631 /** 1617 /**
1632 * atomic64_set() - atomic set with relaxed o 1618 * atomic64_set() - atomic set with relaxed ordering
1633 * @v: pointer to atomic64_t 1619 * @v: pointer to atomic64_t
1634 * @i: s64 value to assign 1620 * @i: s64 value to assign
1635 * 1621 *
1636 * Atomically sets @v to @i with relaxed orde 1622 * Atomically sets @v to @i with relaxed ordering.
1637 * 1623 *
1638 * Unsafe to use in noinstr code; use raw_ato 1624 * Unsafe to use in noinstr code; use raw_atomic64_set() there.
1639 * 1625 *
1640 * Return: Nothing. 1626 * Return: Nothing.
1641 */ 1627 */
1642 static __always_inline void 1628 static __always_inline void
1643 atomic64_set(atomic64_t *v, s64 i) 1629 atomic64_set(atomic64_t *v, s64 i)
1644 { 1630 {
1645 instrument_atomic_write(v, sizeof(*v) 1631 instrument_atomic_write(v, sizeof(*v));
1646 raw_atomic64_set(v, i); 1632 raw_atomic64_set(v, i);
1647 } 1633 }
1648 1634
1649 /** 1635 /**
1650 * atomic64_set_release() - atomic set with r 1636 * atomic64_set_release() - atomic set with release ordering
1651 * @v: pointer to atomic64_t 1637 * @v: pointer to atomic64_t
1652 * @i: s64 value to assign 1638 * @i: s64 value to assign
1653 * 1639 *
1654 * Atomically sets @v to @i with release orde 1640 * Atomically sets @v to @i with release ordering.
1655 * 1641 *
1656 * Unsafe to use in noinstr code; use raw_ato 1642 * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
1657 * 1643 *
1658 * Return: Nothing. 1644 * Return: Nothing.
1659 */ 1645 */
1660 static __always_inline void 1646 static __always_inline void
1661 atomic64_set_release(atomic64_t *v, s64 i) 1647 atomic64_set_release(atomic64_t *v, s64 i)
1662 { 1648 {
1663 kcsan_release(); 1649 kcsan_release();
1664 instrument_atomic_write(v, sizeof(*v) 1650 instrument_atomic_write(v, sizeof(*v));
1665 raw_atomic64_set_release(v, i); 1651 raw_atomic64_set_release(v, i);
1666 } 1652 }
1667 1653
1668 /** 1654 /**
1669 * atomic64_add() - atomic add with relaxed o 1655 * atomic64_add() - atomic add with relaxed ordering
1670 * @i: s64 value to add 1656 * @i: s64 value to add
1671 * @v: pointer to atomic64_t 1657 * @v: pointer to atomic64_t
1672 * 1658 *
1673 * Atomically updates @v to (@v + @i) with re 1659 * Atomically updates @v to (@v + @i) with relaxed ordering.
1674 * 1660 *
1675 * Unsafe to use in noinstr code; use raw_ato 1661 * Unsafe to use in noinstr code; use raw_atomic64_add() there.
1676 * 1662 *
1677 * Return: Nothing. 1663 * Return: Nothing.
1678 */ 1664 */
1679 static __always_inline void 1665 static __always_inline void
1680 atomic64_add(s64 i, atomic64_t *v) 1666 atomic64_add(s64 i, atomic64_t *v)
1681 { 1667 {
1682 instrument_atomic_read_write(v, sizeo 1668 instrument_atomic_read_write(v, sizeof(*v));
1683 raw_atomic64_add(i, v); 1669 raw_atomic64_add(i, v);
1684 } 1670 }
1685 1671
1686 /** 1672 /**
1687 * atomic64_add_return() - atomic add with fu 1673 * atomic64_add_return() - atomic add with full ordering
1688 * @i: s64 value to add 1674 * @i: s64 value to add
1689 * @v: pointer to atomic64_t 1675 * @v: pointer to atomic64_t
1690 * 1676 *
1691 * Atomically updates @v to (@v + @i) with fu 1677 * Atomically updates @v to (@v + @i) with full ordering.
1692 * 1678 *
1693 * Unsafe to use in noinstr code; use raw_ato 1679 * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
1694 * 1680 *
1695 * Return: The updated value of @v. 1681 * Return: The updated value of @v.
1696 */ 1682 */
1697 static __always_inline s64 1683 static __always_inline s64
1698 atomic64_add_return(s64 i, atomic64_t *v) 1684 atomic64_add_return(s64 i, atomic64_t *v)
1699 { 1685 {
1700 kcsan_mb(); 1686 kcsan_mb();
1701 instrument_atomic_read_write(v, sizeo 1687 instrument_atomic_read_write(v, sizeof(*v));
1702 return raw_atomic64_add_return(i, v); 1688 return raw_atomic64_add_return(i, v);
1703 } 1689 }
1704 1690
1705 /** 1691 /**
1706 * atomic64_add_return_acquire() - atomic add 1692 * atomic64_add_return_acquire() - atomic add with acquire ordering
1707 * @i: s64 value to add 1693 * @i: s64 value to add
1708 * @v: pointer to atomic64_t 1694 * @v: pointer to atomic64_t
1709 * 1695 *
1710 * Atomically updates @v to (@v + @i) with ac 1696 * Atomically updates @v to (@v + @i) with acquire ordering.
1711 * 1697 *
1712 * Unsafe to use in noinstr code; use raw_ato 1698 * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
1713 * 1699 *
1714 * Return: The updated value of @v. 1700 * Return: The updated value of @v.
1715 */ 1701 */
1716 static __always_inline s64 1702 static __always_inline s64
1717 atomic64_add_return_acquire(s64 i, atomic64_t 1703 atomic64_add_return_acquire(s64 i, atomic64_t *v)
1718 { 1704 {
1719 instrument_atomic_read_write(v, sizeo 1705 instrument_atomic_read_write(v, sizeof(*v));
1720 return raw_atomic64_add_return_acquir 1706 return raw_atomic64_add_return_acquire(i, v);
1721 } 1707 }
1722 1708
1723 /** 1709 /**
1724 * atomic64_add_return_release() - atomic add 1710 * atomic64_add_return_release() - atomic add with release ordering
1725 * @i: s64 value to add 1711 * @i: s64 value to add
1726 * @v: pointer to atomic64_t 1712 * @v: pointer to atomic64_t
1727 * 1713 *
1728 * Atomically updates @v to (@v + @i) with re 1714 * Atomically updates @v to (@v + @i) with release ordering.
1729 * 1715 *
1730 * Unsafe to use in noinstr code; use raw_ato 1716 * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
1731 * 1717 *
1732 * Return: The updated value of @v. 1718 * Return: The updated value of @v.
1733 */ 1719 */
1734 static __always_inline s64 1720 static __always_inline s64
1735 atomic64_add_return_release(s64 i, atomic64_t 1721 atomic64_add_return_release(s64 i, atomic64_t *v)
1736 { 1722 {
1737 kcsan_release(); 1723 kcsan_release();
1738 instrument_atomic_read_write(v, sizeo 1724 instrument_atomic_read_write(v, sizeof(*v));
1739 return raw_atomic64_add_return_releas 1725 return raw_atomic64_add_return_release(i, v);
1740 } 1726 }
1741 1727
1742 /** 1728 /**
1743 * atomic64_add_return_relaxed() - atomic add 1729 * atomic64_add_return_relaxed() - atomic add with relaxed ordering
1744 * @i: s64 value to add 1730 * @i: s64 value to add
1745 * @v: pointer to atomic64_t 1731 * @v: pointer to atomic64_t
1746 * 1732 *
1747 * Atomically updates @v to (@v + @i) with re 1733 * Atomically updates @v to (@v + @i) with relaxed ordering.
1748 * 1734 *
1749 * Unsafe to use in noinstr code; use raw_ato 1735 * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
1750 * 1736 *
1751 * Return: The updated value of @v. 1737 * Return: The updated value of @v.
1752 */ 1738 */
1753 static __always_inline s64 1739 static __always_inline s64
1754 atomic64_add_return_relaxed(s64 i, atomic64_t 1740 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
1755 { 1741 {
1756 instrument_atomic_read_write(v, sizeo 1742 instrument_atomic_read_write(v, sizeof(*v));
1757 return raw_atomic64_add_return_relaxe 1743 return raw_atomic64_add_return_relaxed(i, v);
1758 } 1744 }
1759 1745
1760 /** 1746 /**
1761 * atomic64_fetch_add() - atomic add with ful 1747 * atomic64_fetch_add() - atomic add with full ordering
1762 * @i: s64 value to add 1748 * @i: s64 value to add
1763 * @v: pointer to atomic64_t 1749 * @v: pointer to atomic64_t
1764 * 1750 *
1765 * Atomically updates @v to (@v + @i) with fu 1751 * Atomically updates @v to (@v + @i) with full ordering.
1766 * 1752 *
1767 * Unsafe to use in noinstr code; use raw_ato 1753 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
1768 * 1754 *
1769 * Return: The original value of @v. 1755 * Return: The original value of @v.
1770 */ 1756 */
1771 static __always_inline s64 1757 static __always_inline s64
1772 atomic64_fetch_add(s64 i, atomic64_t *v) 1758 atomic64_fetch_add(s64 i, atomic64_t *v)
1773 { 1759 {
1774 kcsan_mb(); 1760 kcsan_mb();
1775 instrument_atomic_read_write(v, sizeo 1761 instrument_atomic_read_write(v, sizeof(*v));
1776 return raw_atomic64_fetch_add(i, v); 1762 return raw_atomic64_fetch_add(i, v);
1777 } 1763 }
1778 1764
1779 /** 1765 /**
1780 * atomic64_fetch_add_acquire() - atomic add 1766 * atomic64_fetch_add_acquire() - atomic add with acquire ordering
1781 * @i: s64 value to add 1767 * @i: s64 value to add
1782 * @v: pointer to atomic64_t 1768 * @v: pointer to atomic64_t
1783 * 1769 *
1784 * Atomically updates @v to (@v + @i) with ac 1770 * Atomically updates @v to (@v + @i) with acquire ordering.
1785 * 1771 *
1786 * Unsafe to use in noinstr code; use raw_ato 1772 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
1787 * 1773 *
1788 * Return: The original value of @v. 1774 * Return: The original value of @v.
1789 */ 1775 */
1790 static __always_inline s64 1776 static __always_inline s64
1791 atomic64_fetch_add_acquire(s64 i, atomic64_t 1777 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1792 { 1778 {
1793 instrument_atomic_read_write(v, sizeo 1779 instrument_atomic_read_write(v, sizeof(*v));
1794 return raw_atomic64_fetch_add_acquire 1780 return raw_atomic64_fetch_add_acquire(i, v);
1795 } 1781 }
1796 1782
1797 /** 1783 /**
1798 * atomic64_fetch_add_release() - atomic add 1784 * atomic64_fetch_add_release() - atomic add with release ordering
1799 * @i: s64 value to add 1785 * @i: s64 value to add
1800 * @v: pointer to atomic64_t 1786 * @v: pointer to atomic64_t
1801 * 1787 *
1802 * Atomically updates @v to (@v + @i) with re 1788 * Atomically updates @v to (@v + @i) with release ordering.
1803 * 1789 *
1804 * Unsafe to use in noinstr code; use raw_ato 1790 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
1805 * 1791 *
1806 * Return: The original value of @v. 1792 * Return: The original value of @v.
1807 */ 1793 */
1808 static __always_inline s64 1794 static __always_inline s64
1809 atomic64_fetch_add_release(s64 i, atomic64_t 1795 atomic64_fetch_add_release(s64 i, atomic64_t *v)
1810 { 1796 {
1811 kcsan_release(); 1797 kcsan_release();
1812 instrument_atomic_read_write(v, sizeo 1798 instrument_atomic_read_write(v, sizeof(*v));
1813 return raw_atomic64_fetch_add_release 1799 return raw_atomic64_fetch_add_release(i, v);
1814 } 1800 }
1815 1801
1816 /** 1802 /**
1817 * atomic64_fetch_add_relaxed() - atomic add 1803 * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
1818 * @i: s64 value to add 1804 * @i: s64 value to add
1819 * @v: pointer to atomic64_t 1805 * @v: pointer to atomic64_t
1820 * 1806 *
1821 * Atomically updates @v to (@v + @i) with re 1807 * Atomically updates @v to (@v + @i) with relaxed ordering.
1822 * 1808 *
1823 * Unsafe to use in noinstr code; use raw_ato 1809 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
1824 * 1810 *
1825 * Return: The original value of @v. 1811 * Return: The original value of @v.
1826 */ 1812 */
1827 static __always_inline s64 1813 static __always_inline s64
1828 atomic64_fetch_add_relaxed(s64 i, atomic64_t 1814 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
1829 { 1815 {
1830 instrument_atomic_read_write(v, sizeo 1816 instrument_atomic_read_write(v, sizeof(*v));
1831 return raw_atomic64_fetch_add_relaxed 1817 return raw_atomic64_fetch_add_relaxed(i, v);
1832 } 1818 }
1833 1819
1834 /** 1820 /**
1835 * atomic64_sub() - atomic subtract with rela 1821 * atomic64_sub() - atomic subtract with relaxed ordering
1836 * @i: s64 value to subtract 1822 * @i: s64 value to subtract
1837 * @v: pointer to atomic64_t 1823 * @v: pointer to atomic64_t
1838 * 1824 *
1839 * Atomically updates @v to (@v - @i) with re 1825 * Atomically updates @v to (@v - @i) with relaxed ordering.
1840 * 1826 *
1841 * Unsafe to use in noinstr code; use raw_ato 1827 * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
1842 * 1828 *
1843 * Return: Nothing. 1829 * Return: Nothing.
1844 */ 1830 */
1845 static __always_inline void 1831 static __always_inline void
1846 atomic64_sub(s64 i, atomic64_t *v) 1832 atomic64_sub(s64 i, atomic64_t *v)
1847 { 1833 {
1848 instrument_atomic_read_write(v, sizeo 1834 instrument_atomic_read_write(v, sizeof(*v));
1849 raw_atomic64_sub(i, v); 1835 raw_atomic64_sub(i, v);
1850 } 1836 }
1851 1837
1852 /** 1838 /**
1853 * atomic64_sub_return() - atomic subtract wi 1839 * atomic64_sub_return() - atomic subtract with full ordering
1854 * @i: s64 value to subtract 1840 * @i: s64 value to subtract
1855 * @v: pointer to atomic64_t 1841 * @v: pointer to atomic64_t
1856 * 1842 *
1857 * Atomically updates @v to (@v - @i) with fu 1843 * Atomically updates @v to (@v - @i) with full ordering.
1858 * 1844 *
1859 * Unsafe to use in noinstr code; use raw_ato 1845 * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
1860 * 1846 *
1861 * Return: The updated value of @v. 1847 * Return: The updated value of @v.
1862 */ 1848 */
1863 static __always_inline s64 1849 static __always_inline s64
1864 atomic64_sub_return(s64 i, atomic64_t *v) 1850 atomic64_sub_return(s64 i, atomic64_t *v)
1865 { 1851 {
1866 kcsan_mb(); 1852 kcsan_mb();
1867 instrument_atomic_read_write(v, sizeo 1853 instrument_atomic_read_write(v, sizeof(*v));
1868 return raw_atomic64_sub_return(i, v); 1854 return raw_atomic64_sub_return(i, v);
1869 } 1855 }
1870 1856
1871 /** 1857 /**
1872 * atomic64_sub_return_acquire() - atomic sub 1858 * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
1873 * @i: s64 value to subtract 1859 * @i: s64 value to subtract
1874 * @v: pointer to atomic64_t 1860 * @v: pointer to atomic64_t
1875 * 1861 *
1876 * Atomically updates @v to (@v - @i) with ac 1862 * Atomically updates @v to (@v - @i) with acquire ordering.
1877 * 1863 *
1878 * Unsafe to use in noinstr code; use raw_ato 1864 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
1879 * 1865 *
1880 * Return: The updated value of @v. 1866 * Return: The updated value of @v.
1881 */ 1867 */
1882 static __always_inline s64 1868 static __always_inline s64
1883 atomic64_sub_return_acquire(s64 i, atomic64_t 1869 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1884 { 1870 {
1885 instrument_atomic_read_write(v, sizeo 1871 instrument_atomic_read_write(v, sizeof(*v));
1886 return raw_atomic64_sub_return_acquir 1872 return raw_atomic64_sub_return_acquire(i, v);
1887 } 1873 }
1888 1874
1889 /** 1875 /**
1890 * atomic64_sub_return_release() - atomic sub 1876 * atomic64_sub_return_release() - atomic subtract with release ordering
1891 * @i: s64 value to subtract 1877 * @i: s64 value to subtract
1892 * @v: pointer to atomic64_t 1878 * @v: pointer to atomic64_t
1893 * 1879 *
1894 * Atomically updates @v to (@v - @i) with re 1880 * Atomically updates @v to (@v - @i) with release ordering.
1895 * 1881 *
1896 * Unsafe to use in noinstr code; use raw_ato 1882 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
1897 * 1883 *
1898 * Return: The updated value of @v. 1884 * Return: The updated value of @v.
1899 */ 1885 */
1900 static __always_inline s64 1886 static __always_inline s64
1901 atomic64_sub_return_release(s64 i, atomic64_t 1887 atomic64_sub_return_release(s64 i, atomic64_t *v)
1902 { 1888 {
1903 kcsan_release(); 1889 kcsan_release();
1904 instrument_atomic_read_write(v, sizeo 1890 instrument_atomic_read_write(v, sizeof(*v));
1905 return raw_atomic64_sub_return_releas 1891 return raw_atomic64_sub_return_release(i, v);
1906 } 1892 }
1907 1893
1908 /** 1894 /**
1909 * atomic64_sub_return_relaxed() - atomic sub 1895 * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
1910 * @i: s64 value to subtract 1896 * @i: s64 value to subtract
1911 * @v: pointer to atomic64_t 1897 * @v: pointer to atomic64_t
1912 * 1898 *
1913 * Atomically updates @v to (@v - @i) with re 1899 * Atomically updates @v to (@v - @i) with relaxed ordering.
1914 * 1900 *
1915 * Unsafe to use in noinstr code; use raw_ato 1901 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
1916 * 1902 *
1917 * Return: The updated value of @v. 1903 * Return: The updated value of @v.
1918 */ 1904 */
1919 static __always_inline s64 1905 static __always_inline s64
1920 atomic64_sub_return_relaxed(s64 i, atomic64_t 1906 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
1921 { 1907 {
1922 instrument_atomic_read_write(v, sizeo 1908 instrument_atomic_read_write(v, sizeof(*v));
1923 return raw_atomic64_sub_return_relaxe 1909 return raw_atomic64_sub_return_relaxed(i, v);
1924 } 1910 }
1925 1911
1926 /** 1912 /**
1927 * atomic64_fetch_sub() - atomic subtract wit 1913 * atomic64_fetch_sub() - atomic subtract with full ordering
1928 * @i: s64 value to subtract 1914 * @i: s64 value to subtract
1929 * @v: pointer to atomic64_t 1915 * @v: pointer to atomic64_t
1930 * 1916 *
1931 * Atomically updates @v to (@v - @i) with fu 1917 * Atomically updates @v to (@v - @i) with full ordering.
1932 * 1918 *
1933 * Unsafe to use in noinstr code; use raw_ato 1919 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
1934 * 1920 *
1935 * Return: The original value of @v. 1921 * Return: The original value of @v.
1936 */ 1922 */
1937 static __always_inline s64 1923 static __always_inline s64
1938 atomic64_fetch_sub(s64 i, atomic64_t *v) 1924 atomic64_fetch_sub(s64 i, atomic64_t *v)
1939 { 1925 {
1940 kcsan_mb(); 1926 kcsan_mb();
1941 instrument_atomic_read_write(v, sizeo 1927 instrument_atomic_read_write(v, sizeof(*v));
1942 return raw_atomic64_fetch_sub(i, v); 1928 return raw_atomic64_fetch_sub(i, v);
1943 } 1929 }
1944 1930
1945 /** 1931 /**
1946 * atomic64_fetch_sub_acquire() - atomic subt 1932 * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
1947 * @i: s64 value to subtract 1933 * @i: s64 value to subtract
1948 * @v: pointer to atomic64_t 1934 * @v: pointer to atomic64_t
1949 * 1935 *
1950 * Atomically updates @v to (@v - @i) with ac 1936 * Atomically updates @v to (@v - @i) with acquire ordering.
1951 * 1937 *
1952 * Unsafe to use in noinstr code; use raw_ato 1938 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
1953 * 1939 *
1954 * Return: The original value of @v. 1940 * Return: The original value of @v.
1955 */ 1941 */
1956 static __always_inline s64 1942 static __always_inline s64
1957 atomic64_fetch_sub_acquire(s64 i, atomic64_t 1943 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1958 { 1944 {
1959 instrument_atomic_read_write(v, sizeo 1945 instrument_atomic_read_write(v, sizeof(*v));
1960 return raw_atomic64_fetch_sub_acquire 1946 return raw_atomic64_fetch_sub_acquire(i, v);
1961 } 1947 }
1962 1948
1963 /** 1949 /**
1964 * atomic64_fetch_sub_release() - atomic subt 1950 * atomic64_fetch_sub_release() - atomic subtract with release ordering
1965 * @i: s64 value to subtract 1951 * @i: s64 value to subtract
1966 * @v: pointer to atomic64_t 1952 * @v: pointer to atomic64_t
1967 * 1953 *
1968 * Atomically updates @v to (@v - @i) with re 1954 * Atomically updates @v to (@v - @i) with release ordering.
1969 * 1955 *
1970 * Unsafe to use in noinstr code; use raw_ato 1956 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
1971 * 1957 *
1972 * Return: The original value of @v. 1958 * Return: The original value of @v.
1973 */ 1959 */
1974 static __always_inline s64 1960 static __always_inline s64
1975 atomic64_fetch_sub_release(s64 i, atomic64_t 1961 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1976 { 1962 {
1977 kcsan_release(); 1963 kcsan_release();
1978 instrument_atomic_read_write(v, sizeo 1964 instrument_atomic_read_write(v, sizeof(*v));
1979 return raw_atomic64_fetch_sub_release 1965 return raw_atomic64_fetch_sub_release(i, v);
1980 } 1966 }
1981 1967
1982 /** 1968 /**
1983 * atomic64_fetch_sub_relaxed() - atomic subt 1969 * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
1984 * @i: s64 value to subtract 1970 * @i: s64 value to subtract
1985 * @v: pointer to atomic64_t 1971 * @v: pointer to atomic64_t
1986 * 1972 *
1987 * Atomically updates @v to (@v - @i) with re 1973 * Atomically updates @v to (@v - @i) with relaxed ordering.
1988 * 1974 *
1989 * Unsafe to use in noinstr code; use raw_ato 1975 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
1990 * 1976 *
1991 * Return: The original value of @v. 1977 * Return: The original value of @v.
1992 */ 1978 */
1993 static __always_inline s64 1979 static __always_inline s64
1994 atomic64_fetch_sub_relaxed(s64 i, atomic64_t 1980 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1995 { 1981 {
1996 instrument_atomic_read_write(v, sizeo 1982 instrument_atomic_read_write(v, sizeof(*v));
1997 return raw_atomic64_fetch_sub_relaxed 1983 return raw_atomic64_fetch_sub_relaxed(i, v);
1998 } 1984 }
1999 1985
2000 /** 1986 /**
2001 * atomic64_inc() - atomic increment with rel 1987 * atomic64_inc() - atomic increment with relaxed ordering
2002 * @v: pointer to atomic64_t 1988 * @v: pointer to atomic64_t
2003 * 1989 *
2004 * Atomically updates @v to (@v + 1) with rel 1990 * Atomically updates @v to (@v + 1) with relaxed ordering.
2005 * 1991 *
2006 * Unsafe to use in noinstr code; use raw_ato 1992 * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
2007 * 1993 *
2008 * Return: Nothing. 1994 * Return: Nothing.
2009 */ 1995 */
2010 static __always_inline void 1996 static __always_inline void
2011 atomic64_inc(atomic64_t *v) 1997 atomic64_inc(atomic64_t *v)
2012 { 1998 {
2013 instrument_atomic_read_write(v, sizeo 1999 instrument_atomic_read_write(v, sizeof(*v));
2014 raw_atomic64_inc(v); 2000 raw_atomic64_inc(v);
2015 } 2001 }
2016 2002
2017 /** 2003 /**
2018 * atomic64_inc_return() - atomic increment w 2004 * atomic64_inc_return() - atomic increment with full ordering
2019 * @v: pointer to atomic64_t 2005 * @v: pointer to atomic64_t
2020 * 2006 *
2021 * Atomically updates @v to (@v + 1) with ful 2007 * Atomically updates @v to (@v + 1) with full ordering.
2022 * 2008 *
2023 * Unsafe to use in noinstr code; use raw_ato 2009 * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
2024 * 2010 *
2025 * Return: The updated value of @v. 2011 * Return: The updated value of @v.
2026 */ 2012 */
2027 static __always_inline s64 2013 static __always_inline s64
2028 atomic64_inc_return(atomic64_t *v) 2014 atomic64_inc_return(atomic64_t *v)
2029 { 2015 {
2030 kcsan_mb(); 2016 kcsan_mb();
2031 instrument_atomic_read_write(v, sizeo 2017 instrument_atomic_read_write(v, sizeof(*v));
2032 return raw_atomic64_inc_return(v); 2018 return raw_atomic64_inc_return(v);
2033 } 2019 }
2034 2020
2035 /** 2021 /**
2036 * atomic64_inc_return_acquire() - atomic inc 2022 * atomic64_inc_return_acquire() - atomic increment with acquire ordering
2037 * @v: pointer to atomic64_t 2023 * @v: pointer to atomic64_t
2038 * 2024 *
2039 * Atomically updates @v to (@v + 1) with acq 2025 * Atomically updates @v to (@v + 1) with acquire ordering.
2040 * 2026 *
2041 * Unsafe to use in noinstr code; use raw_ato 2027 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
2042 * 2028 *
2043 * Return: The updated value of @v. 2029 * Return: The updated value of @v.
2044 */ 2030 */
2045 static __always_inline s64 2031 static __always_inline s64
2046 atomic64_inc_return_acquire(atomic64_t *v) 2032 atomic64_inc_return_acquire(atomic64_t *v)
2047 { 2033 {
2048 instrument_atomic_read_write(v, sizeo 2034 instrument_atomic_read_write(v, sizeof(*v));
2049 return raw_atomic64_inc_return_acquir 2035 return raw_atomic64_inc_return_acquire(v);
2050 } 2036 }
2051 2037
2052 /** 2038 /**
2053 * atomic64_inc_return_release() - atomic inc 2039 * atomic64_inc_return_release() - atomic increment with release ordering
2054 * @v: pointer to atomic64_t 2040 * @v: pointer to atomic64_t
2055 * 2041 *
2056 * Atomically updates @v to (@v + 1) with rel 2042 * Atomically updates @v to (@v + 1) with release ordering.
2057 * 2043 *
2058 * Unsafe to use in noinstr code; use raw_ato 2044 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
2059 * 2045 *
2060 * Return: The updated value of @v. 2046 * Return: The updated value of @v.
2061 */ 2047 */
2062 static __always_inline s64 2048 static __always_inline s64
2063 atomic64_inc_return_release(atomic64_t *v) 2049 atomic64_inc_return_release(atomic64_t *v)
2064 { 2050 {
2065 kcsan_release(); 2051 kcsan_release();
2066 instrument_atomic_read_write(v, sizeo 2052 instrument_atomic_read_write(v, sizeof(*v));
2067 return raw_atomic64_inc_return_releas 2053 return raw_atomic64_inc_return_release(v);
2068 } 2054 }
2069 2055
2070 /** 2056 /**
2071 * atomic64_inc_return_relaxed() - atomic inc 2057 * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
2072 * @v: pointer to atomic64_t 2058 * @v: pointer to atomic64_t
2073 * 2059 *
2074 * Atomically updates @v to (@v + 1) with rel 2060 * Atomically updates @v to (@v + 1) with relaxed ordering.
2075 * 2061 *
2076 * Unsafe to use in noinstr code; use raw_ato 2062 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
2077 * 2063 *
2078 * Return: The updated value of @v. 2064 * Return: The updated value of @v.
2079 */ 2065 */
2080 static __always_inline s64 2066 static __always_inline s64
2081 atomic64_inc_return_relaxed(atomic64_t *v) 2067 atomic64_inc_return_relaxed(atomic64_t *v)
2082 { 2068 {
2083 instrument_atomic_read_write(v, sizeo 2069 instrument_atomic_read_write(v, sizeof(*v));
2084 return raw_atomic64_inc_return_relaxe 2070 return raw_atomic64_inc_return_relaxed(v);
2085 } 2071 }
2086 2072
2087 /** 2073 /**
2088 * atomic64_fetch_inc() - atomic increment wi 2074 * atomic64_fetch_inc() - atomic increment with full ordering
2089 * @v: pointer to atomic64_t 2075 * @v: pointer to atomic64_t
2090 * 2076 *
2091 * Atomically updates @v to (@v + 1) with ful 2077 * Atomically updates @v to (@v + 1) with full ordering.
2092 * 2078 *
2093 * Unsafe to use in noinstr code; use raw_ato 2079 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
2094 * 2080 *
2095 * Return: The original value of @v. 2081 * Return: The original value of @v.
2096 */ 2082 */
2097 static __always_inline s64 2083 static __always_inline s64
2098 atomic64_fetch_inc(atomic64_t *v) 2084 atomic64_fetch_inc(atomic64_t *v)
2099 { 2085 {
2100 kcsan_mb(); 2086 kcsan_mb();
2101 instrument_atomic_read_write(v, sizeo 2087 instrument_atomic_read_write(v, sizeof(*v));
2102 return raw_atomic64_fetch_inc(v); 2088 return raw_atomic64_fetch_inc(v);
2103 } 2089 }
2104 2090
2105 /** 2091 /**
2106 * atomic64_fetch_inc_acquire() - atomic incr 2092 * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
2107 * @v: pointer to atomic64_t 2093 * @v: pointer to atomic64_t
2108 * 2094 *
2109 * Atomically updates @v to (@v + 1) with acq 2095 * Atomically updates @v to (@v + 1) with acquire ordering.
2110 * 2096 *
2111 * Unsafe to use in noinstr code; use raw_ato 2097 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
2112 * 2098 *
2113 * Return: The original value of @v. 2099 * Return: The original value of @v.
2114 */ 2100 */
2115 static __always_inline s64 2101 static __always_inline s64
2116 atomic64_fetch_inc_acquire(atomic64_t *v) 2102 atomic64_fetch_inc_acquire(atomic64_t *v)
2117 { 2103 {
2118 instrument_atomic_read_write(v, sizeo 2104 instrument_atomic_read_write(v, sizeof(*v));
2119 return raw_atomic64_fetch_inc_acquire 2105 return raw_atomic64_fetch_inc_acquire(v);
2120 } 2106 }
2121 2107
2122 /** 2108 /**
2123 * atomic64_fetch_inc_release() - atomic incr 2109 * atomic64_fetch_inc_release() - atomic increment with release ordering
2124 * @v: pointer to atomic64_t 2110 * @v: pointer to atomic64_t
2125 * 2111 *
2126 * Atomically updates @v to (@v + 1) with rel 2112 * Atomically updates @v to (@v + 1) with release ordering.
2127 * 2113 *
2128 * Unsafe to use in noinstr code; use raw_ato 2114 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
2129 * 2115 *
2130 * Return: The original value of @v. 2116 * Return: The original value of @v.
2131 */ 2117 */
2132 static __always_inline s64 2118 static __always_inline s64
2133 atomic64_fetch_inc_release(atomic64_t *v) 2119 atomic64_fetch_inc_release(atomic64_t *v)
2134 { 2120 {
2135 kcsan_release(); 2121 kcsan_release();
2136 instrument_atomic_read_write(v, sizeo 2122 instrument_atomic_read_write(v, sizeof(*v));
2137 return raw_atomic64_fetch_inc_release 2123 return raw_atomic64_fetch_inc_release(v);
2138 } 2124 }
2139 2125
2140 /** 2126 /**
2141 * atomic64_fetch_inc_relaxed() - atomic incr 2127 * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
2142 * @v: pointer to atomic64_t 2128 * @v: pointer to atomic64_t
2143 * 2129 *
2144 * Atomically updates @v to (@v + 1) with rel 2130 * Atomically updates @v to (@v + 1) with relaxed ordering.
2145 * 2131 *
2146 * Unsafe to use in noinstr code; use raw_ato 2132 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
2147 * 2133 *
2148 * Return: The original value of @v. 2134 * Return: The original value of @v.
2149 */ 2135 */
2150 static __always_inline s64 2136 static __always_inline s64
2151 atomic64_fetch_inc_relaxed(atomic64_t *v) 2137 atomic64_fetch_inc_relaxed(atomic64_t *v)
2152 { 2138 {
2153 instrument_atomic_read_write(v, sizeo 2139 instrument_atomic_read_write(v, sizeof(*v));
2154 return raw_atomic64_fetch_inc_relaxed 2140 return raw_atomic64_fetch_inc_relaxed(v);
2155 } 2141 }
2156 2142
2157 /** 2143 /**
2158 * atomic64_dec() - atomic decrement with rel 2144 * atomic64_dec() - atomic decrement with relaxed ordering
2159 * @v: pointer to atomic64_t 2145 * @v: pointer to atomic64_t
2160 * 2146 *
2161 * Atomically updates @v to (@v - 1) with rel 2147 * Atomically updates @v to (@v - 1) with relaxed ordering.
2162 * 2148 *
2163 * Unsafe to use in noinstr code; use raw_ato 2149 * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
2164 * 2150 *
2165 * Return: Nothing. 2151 * Return: Nothing.
2166 */ 2152 */
2167 static __always_inline void 2153 static __always_inline void
2168 atomic64_dec(atomic64_t *v) 2154 atomic64_dec(atomic64_t *v)
2169 { 2155 {
2170 instrument_atomic_read_write(v, sizeo 2156 instrument_atomic_read_write(v, sizeof(*v));
2171 raw_atomic64_dec(v); 2157 raw_atomic64_dec(v);
2172 } 2158 }
2173 2159
2174 /** 2160 /**
2175 * atomic64_dec_return() - atomic decrement w 2161 * atomic64_dec_return() - atomic decrement with full ordering
2176 * @v: pointer to atomic64_t 2162 * @v: pointer to atomic64_t
2177 * 2163 *
2178 * Atomically updates @v to (@v - 1) with ful 2164 * Atomically updates @v to (@v - 1) with full ordering.
2179 * 2165 *
2180 * Unsafe to use in noinstr code; use raw_ato 2166 * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
2181 * 2167 *
2182 * Return: The updated value of @v. 2168 * Return: The updated value of @v.
2183 */ 2169 */
2184 static __always_inline s64 2170 static __always_inline s64
2185 atomic64_dec_return(atomic64_t *v) 2171 atomic64_dec_return(atomic64_t *v)
2186 { 2172 {
2187 kcsan_mb(); 2173 kcsan_mb();
2188 instrument_atomic_read_write(v, sizeo 2174 instrument_atomic_read_write(v, sizeof(*v));
2189 return raw_atomic64_dec_return(v); 2175 return raw_atomic64_dec_return(v);
2190 } 2176 }
2191 2177
2192 /** 2178 /**
2193 * atomic64_dec_return_acquire() - atomic dec 2179 * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
2194 * @v: pointer to atomic64_t 2180 * @v: pointer to atomic64_t
2195 * 2181 *
2196 * Atomically updates @v to (@v - 1) with acq 2182 * Atomically updates @v to (@v - 1) with acquire ordering.
2197 * 2183 *
2198 * Unsafe to use in noinstr code; use raw_ato 2184 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
2199 * 2185 *
2200 * Return: The updated value of @v. 2186 * Return: The updated value of @v.
2201 */ 2187 */
2202 static __always_inline s64 2188 static __always_inline s64
2203 atomic64_dec_return_acquire(atomic64_t *v) 2189 atomic64_dec_return_acquire(atomic64_t *v)
2204 { 2190 {
2205 instrument_atomic_read_write(v, sizeo 2191 instrument_atomic_read_write(v, sizeof(*v));
2206 return raw_atomic64_dec_return_acquir 2192 return raw_atomic64_dec_return_acquire(v);
2207 } 2193 }
2208 2194
2209 /** 2195 /**
2210 * atomic64_dec_return_release() - atomic dec 2196 * atomic64_dec_return_release() - atomic decrement with release ordering
2211 * @v: pointer to atomic64_t 2197 * @v: pointer to atomic64_t
2212 * 2198 *
2213 * Atomically updates @v to (@v - 1) with rel 2199 * Atomically updates @v to (@v - 1) with release ordering.
2214 * 2200 *
2215 * Unsafe to use in noinstr code; use raw_ato 2201 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
2216 * 2202 *
2217 * Return: The updated value of @v. 2203 * Return: The updated value of @v.
2218 */ 2204 */
2219 static __always_inline s64 2205 static __always_inline s64
2220 atomic64_dec_return_release(atomic64_t *v) 2206 atomic64_dec_return_release(atomic64_t *v)
2221 { 2207 {
2222 kcsan_release(); 2208 kcsan_release();
2223 instrument_atomic_read_write(v, sizeo 2209 instrument_atomic_read_write(v, sizeof(*v));
2224 return raw_atomic64_dec_return_releas 2210 return raw_atomic64_dec_return_release(v);
2225 } 2211 }
2226 2212
2227 /** 2213 /**
2228 * atomic64_dec_return_relaxed() - atomic dec 2214 * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
2229 * @v: pointer to atomic64_t 2215 * @v: pointer to atomic64_t
2230 * 2216 *
2231 * Atomically updates @v to (@v - 1) with rel 2217 * Atomically updates @v to (@v - 1) with relaxed ordering.
2232 * 2218 *
2233 * Unsafe to use in noinstr code; use raw_ato 2219 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
2234 * 2220 *
2235 * Return: The updated value of @v. 2221 * Return: The updated value of @v.
2236 */ 2222 */
2237 static __always_inline s64 2223 static __always_inline s64
2238 atomic64_dec_return_relaxed(atomic64_t *v) 2224 atomic64_dec_return_relaxed(atomic64_t *v)
2239 { 2225 {
2240 instrument_atomic_read_write(v, sizeo 2226 instrument_atomic_read_write(v, sizeof(*v));
2241 return raw_atomic64_dec_return_relaxe 2227 return raw_atomic64_dec_return_relaxed(v);
2242 } 2228 }
2243 2229
2244 /** 2230 /**
2245 * atomic64_fetch_dec() - atomic decrement wi 2231 * atomic64_fetch_dec() - atomic decrement with full ordering
2246 * @v: pointer to atomic64_t 2232 * @v: pointer to atomic64_t
2247 * 2233 *
2248 * Atomically updates @v to (@v - 1) with ful 2234 * Atomically updates @v to (@v - 1) with full ordering.
2249 * 2235 *
2250 * Unsafe to use in noinstr code; use raw_ato 2236 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
2251 * 2237 *
2252 * Return: The original value of @v. 2238 * Return: The original value of @v.
2253 */ 2239 */
2254 static __always_inline s64 2240 static __always_inline s64
2255 atomic64_fetch_dec(atomic64_t *v) 2241 atomic64_fetch_dec(atomic64_t *v)
2256 { 2242 {
2257 kcsan_mb(); 2243 kcsan_mb();
2258 instrument_atomic_read_write(v, sizeo 2244 instrument_atomic_read_write(v, sizeof(*v));
2259 return raw_atomic64_fetch_dec(v); 2245 return raw_atomic64_fetch_dec(v);
2260 } 2246 }
2261 2247
2262 /** 2248 /**
2263 * atomic64_fetch_dec_acquire() - atomic decr 2249 * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
2264 * @v: pointer to atomic64_t 2250 * @v: pointer to atomic64_t
2265 * 2251 *
2266 * Atomically updates @v to (@v - 1) with acq 2252 * Atomically updates @v to (@v - 1) with acquire ordering.
2267 * 2253 *
2268 * Unsafe to use in noinstr code; use raw_ato 2254 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
2269 * 2255 *
2270 * Return: The original value of @v. 2256 * Return: The original value of @v.
2271 */ 2257 */
2272 static __always_inline s64 2258 static __always_inline s64
2273 atomic64_fetch_dec_acquire(atomic64_t *v) 2259 atomic64_fetch_dec_acquire(atomic64_t *v)
2274 { 2260 {
2275 instrument_atomic_read_write(v, sizeo 2261 instrument_atomic_read_write(v, sizeof(*v));
2276 return raw_atomic64_fetch_dec_acquire 2262 return raw_atomic64_fetch_dec_acquire(v);
2277 } 2263 }
2278 2264
2279 /** 2265 /**
2280 * atomic64_fetch_dec_release() - atomic decr 2266 * atomic64_fetch_dec_release() - atomic decrement with release ordering
2281 * @v: pointer to atomic64_t 2267 * @v: pointer to atomic64_t
2282 * 2268 *
2283 * Atomically updates @v to (@v - 1) with rel 2269 * Atomically updates @v to (@v - 1) with release ordering.
2284 * 2270 *
2285 * Unsafe to use in noinstr code; use raw_ato 2271 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
2286 * 2272 *
2287 * Return: The original value of @v. 2273 * Return: The original value of @v.
2288 */ 2274 */
2289 static __always_inline s64 2275 static __always_inline s64
2290 atomic64_fetch_dec_release(atomic64_t *v) 2276 atomic64_fetch_dec_release(atomic64_t *v)
2291 { 2277 {
2292 kcsan_release(); 2278 kcsan_release();
2293 instrument_atomic_read_write(v, sizeo 2279 instrument_atomic_read_write(v, sizeof(*v));
2294 return raw_atomic64_fetch_dec_release 2280 return raw_atomic64_fetch_dec_release(v);
2295 } 2281 }
2296 2282
2297 /** 2283 /**
2298 * atomic64_fetch_dec_relaxed() - atomic decr 2284 * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
2299 * @v: pointer to atomic64_t 2285 * @v: pointer to atomic64_t
2300 * 2286 *
2301 * Atomically updates @v to (@v - 1) with rel 2287 * Atomically updates @v to (@v - 1) with relaxed ordering.
2302 * 2288 *
2303 * Unsafe to use in noinstr code; use raw_ato 2289 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
2304 * 2290 *
2305 * Return: The original value of @v. 2291 * Return: The original value of @v.
2306 */ 2292 */
2307 static __always_inline s64 2293 static __always_inline s64
2308 atomic64_fetch_dec_relaxed(atomic64_t *v) 2294 atomic64_fetch_dec_relaxed(atomic64_t *v)
2309 { 2295 {
2310 instrument_atomic_read_write(v, sizeo 2296 instrument_atomic_read_write(v, sizeof(*v));
2311 return raw_atomic64_fetch_dec_relaxed 2297 return raw_atomic64_fetch_dec_relaxed(v);
2312 } 2298 }
2313 2299
2314 /** 2300 /**
2315 * atomic64_and() - atomic bitwise AND with r 2301 * atomic64_and() - atomic bitwise AND with relaxed ordering
2316 * @i: s64 value 2302 * @i: s64 value
2317 * @v: pointer to atomic64_t 2303 * @v: pointer to atomic64_t
2318 * 2304 *
2319 * Atomically updates @v to (@v & @i) with re 2305 * Atomically updates @v to (@v & @i) with relaxed ordering.
2320 * 2306 *
2321 * Unsafe to use in noinstr code; use raw_ato 2307 * Unsafe to use in noinstr code; use raw_atomic64_and() there.
2322 * 2308 *
2323 * Return: Nothing. 2309 * Return: Nothing.
2324 */ 2310 */
2325 static __always_inline void 2311 static __always_inline void
2326 atomic64_and(s64 i, atomic64_t *v) 2312 atomic64_and(s64 i, atomic64_t *v)
2327 { 2313 {
2328 instrument_atomic_read_write(v, sizeo 2314 instrument_atomic_read_write(v, sizeof(*v));
2329 raw_atomic64_and(i, v); 2315 raw_atomic64_and(i, v);
2330 } 2316 }
2331 2317
2332 /** 2318 /**
2333 * atomic64_fetch_and() - atomic bitwise AND 2319 * atomic64_fetch_and() - atomic bitwise AND with full ordering
2334 * @i: s64 value 2320 * @i: s64 value
2335 * @v: pointer to atomic64_t 2321 * @v: pointer to atomic64_t
2336 * 2322 *
2337 * Atomically updates @v to (@v & @i) with fu 2323 * Atomically updates @v to (@v & @i) with full ordering.
2338 * 2324 *
2339 * Unsafe to use in noinstr code; use raw_ato 2325 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
2340 * 2326 *
2341 * Return: The original value of @v. 2327 * Return: The original value of @v.
2342 */ 2328 */
2343 static __always_inline s64 2329 static __always_inline s64
2344 atomic64_fetch_and(s64 i, atomic64_t *v) 2330 atomic64_fetch_and(s64 i, atomic64_t *v)
2345 { 2331 {
2346 kcsan_mb(); 2332 kcsan_mb();
2347 instrument_atomic_read_write(v, sizeo 2333 instrument_atomic_read_write(v, sizeof(*v));
2348 return raw_atomic64_fetch_and(i, v); 2334 return raw_atomic64_fetch_and(i, v);
2349 } 2335 }
2350 2336
2351 /** 2337 /**
2352 * atomic64_fetch_and_acquire() - atomic bitw 2338 * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
2353 * @i: s64 value 2339 * @i: s64 value
2354 * @v: pointer to atomic64_t 2340 * @v: pointer to atomic64_t
2355 * 2341 *
2356 * Atomically updates @v to (@v & @i) with ac 2342 * Atomically updates @v to (@v & @i) with acquire ordering.
2357 * 2343 *
2358 * Unsafe to use in noinstr code; use raw_ato 2344 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
2359 * 2345 *
2360 * Return: The original value of @v. 2346 * Return: The original value of @v.
2361 */ 2347 */
2362 static __always_inline s64 2348 static __always_inline s64
2363 atomic64_fetch_and_acquire(s64 i, atomic64_t 2349 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2364 { 2350 {
2365 instrument_atomic_read_write(v, sizeo 2351 instrument_atomic_read_write(v, sizeof(*v));
2366 return raw_atomic64_fetch_and_acquire 2352 return raw_atomic64_fetch_and_acquire(i, v);
2367 } 2353 }
2368 2354
2369 /** 2355 /**
2370 * atomic64_fetch_and_release() - atomic bitw 2356 * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
2371 * @i: s64 value 2357 * @i: s64 value
2372 * @v: pointer to atomic64_t 2358 * @v: pointer to atomic64_t
2373 * 2359 *
2374 * Atomically updates @v to (@v & @i) with re 2360 * Atomically updates @v to (@v & @i) with release ordering.
2375 * 2361 *
2376 * Unsafe to use in noinstr code; use raw_ato 2362 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
2377 * 2363 *
2378 * Return: The original value of @v. 2364 * Return: The original value of @v.
2379 */ 2365 */
2380 static __always_inline s64 2366 static __always_inline s64
2381 atomic64_fetch_and_release(s64 i, atomic64_t 2367 atomic64_fetch_and_release(s64 i, atomic64_t *v)
2382 { 2368 {
2383 kcsan_release(); 2369 kcsan_release();
2384 instrument_atomic_read_write(v, sizeo 2370 instrument_atomic_read_write(v, sizeof(*v));
2385 return raw_atomic64_fetch_and_release 2371 return raw_atomic64_fetch_and_release(i, v);
2386 } 2372 }
2387 2373
2388 /** 2374 /**
2389 * atomic64_fetch_and_relaxed() - atomic bitw 2375 * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
2390 * @i: s64 value 2376 * @i: s64 value
2391 * @v: pointer to atomic64_t 2377 * @v: pointer to atomic64_t
2392 * 2378 *
2393 * Atomically updates @v to (@v & @i) with re 2379 * Atomically updates @v to (@v & @i) with relaxed ordering.
2394 * 2380 *
2395 * Unsafe to use in noinstr code; use raw_ato 2381 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
2396 * 2382 *
2397 * Return: The original value of @v. 2383 * Return: The original value of @v.
2398 */ 2384 */
2399 static __always_inline s64 2385 static __always_inline s64
2400 atomic64_fetch_and_relaxed(s64 i, atomic64_t 2386 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
2401 { 2387 {
2402 instrument_atomic_read_write(v, sizeo 2388 instrument_atomic_read_write(v, sizeof(*v));
2403 return raw_atomic64_fetch_and_relaxed 2389 return raw_atomic64_fetch_and_relaxed(i, v);
2404 } 2390 }
2405 2391
2406 /** 2392 /**
2407 * atomic64_andnot() - atomic bitwise AND NOT 2393 * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
2408 * @i: s64 value 2394 * @i: s64 value
2409 * @v: pointer to atomic64_t 2395 * @v: pointer to atomic64_t
2410 * 2396 *
2411 * Atomically updates @v to (@v & ~@i) with r 2397 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2412 * 2398 *
2413 * Unsafe to use in noinstr code; use raw_ato 2399 * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
2414 * 2400 *
2415 * Return: Nothing. 2401 * Return: Nothing.
2416 */ 2402 */
2417 static __always_inline void 2403 static __always_inline void
2418 atomic64_andnot(s64 i, atomic64_t *v) 2404 atomic64_andnot(s64 i, atomic64_t *v)
2419 { 2405 {
2420 instrument_atomic_read_write(v, sizeo 2406 instrument_atomic_read_write(v, sizeof(*v));
2421 raw_atomic64_andnot(i, v); 2407 raw_atomic64_andnot(i, v);
2422 } 2408 }
2423 2409
2424 /** 2410 /**
2425 * atomic64_fetch_andnot() - atomic bitwise A 2411 * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
2426 * @i: s64 value 2412 * @i: s64 value
2427 * @v: pointer to atomic64_t 2413 * @v: pointer to atomic64_t
2428 * 2414 *
2429 * Atomically updates @v to (@v & ~@i) with f 2415 * Atomically updates @v to (@v & ~@i) with full ordering.
2430 * 2416 *
2431 * Unsafe to use in noinstr code; use raw_ato 2417 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
2432 * 2418 *
2433 * Return: The original value of @v. 2419 * Return: The original value of @v.
2434 */ 2420 */
2435 static __always_inline s64 2421 static __always_inline s64
2436 atomic64_fetch_andnot(s64 i, atomic64_t *v) 2422 atomic64_fetch_andnot(s64 i, atomic64_t *v)
2437 { 2423 {
2438 kcsan_mb(); 2424 kcsan_mb();
2439 instrument_atomic_read_write(v, sizeo 2425 instrument_atomic_read_write(v, sizeof(*v));
2440 return raw_atomic64_fetch_andnot(i, v 2426 return raw_atomic64_fetch_andnot(i, v);
2441 } 2427 }
2442 2428
2443 /** 2429 /**
2444 * atomic64_fetch_andnot_acquire() - atomic b 2430 * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
2445 * @i: s64 value 2431 * @i: s64 value
2446 * @v: pointer to atomic64_t 2432 * @v: pointer to atomic64_t
2447 * 2433 *
2448 * Atomically updates @v to (@v & ~@i) with a 2434 * Atomically updates @v to (@v & ~@i) with acquire ordering.
2449 * 2435 *
2450 * Unsafe to use in noinstr code; use raw_ato 2436 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
2451 * 2437 *
2452 * Return: The original value of @v. 2438 * Return: The original value of @v.
2453 */ 2439 */
2454 static __always_inline s64 2440 static __always_inline s64
2455 atomic64_fetch_andnot_acquire(s64 i, atomic64 2441 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2456 { 2442 {
2457 instrument_atomic_read_write(v, sizeo 2443 instrument_atomic_read_write(v, sizeof(*v));
2458 return raw_atomic64_fetch_andnot_acqu 2444 return raw_atomic64_fetch_andnot_acquire(i, v);
2459 } 2445 }
2460 2446
2461 /** 2447 /**
2462 * atomic64_fetch_andnot_release() - atomic b 2448 * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
2463 * @i: s64 value 2449 * @i: s64 value
2464 * @v: pointer to atomic64_t 2450 * @v: pointer to atomic64_t
2465 * 2451 *
2466 * Atomically updates @v to (@v & ~@i) with r 2452 * Atomically updates @v to (@v & ~@i) with release ordering.
2467 * 2453 *
2468 * Unsafe to use in noinstr code; use raw_ato 2454 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
2469 * 2455 *
2470 * Return: The original value of @v. 2456 * Return: The original value of @v.
2471 */ 2457 */
2472 static __always_inline s64 2458 static __always_inline s64
2473 atomic64_fetch_andnot_release(s64 i, atomic64 2459 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2474 { 2460 {
2475 kcsan_release(); 2461 kcsan_release();
2476 instrument_atomic_read_write(v, sizeo 2462 instrument_atomic_read_write(v, sizeof(*v));
2477 return raw_atomic64_fetch_andnot_rele 2463 return raw_atomic64_fetch_andnot_release(i, v);
2478 } 2464 }
2479 2465
2480 /** 2466 /**
2481 * atomic64_fetch_andnot_relaxed() - atomic b 2467 * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
2482 * @i: s64 value 2468 * @i: s64 value
2483 * @v: pointer to atomic64_t 2469 * @v: pointer to atomic64_t
2484 * 2470 *
2485 * Atomically updates @v to (@v & ~@i) with r 2471 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2486 * 2472 *
2487 * Unsafe to use in noinstr code; use raw_ato 2473 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
2488 * 2474 *
2489 * Return: The original value of @v. 2475 * Return: The original value of @v.
2490 */ 2476 */
2491 static __always_inline s64 2477 static __always_inline s64
2492 atomic64_fetch_andnot_relaxed(s64 i, atomic64 2478 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2493 { 2479 {
2494 instrument_atomic_read_write(v, sizeo 2480 instrument_atomic_read_write(v, sizeof(*v));
2495 return raw_atomic64_fetch_andnot_rela 2481 return raw_atomic64_fetch_andnot_relaxed(i, v);
2496 } 2482 }
2497 2483
2498 /** 2484 /**
2499 * atomic64_or() - atomic bitwise OR with rel 2485 * atomic64_or() - atomic bitwise OR with relaxed ordering
2500 * @i: s64 value 2486 * @i: s64 value
2501 * @v: pointer to atomic64_t 2487 * @v: pointer to atomic64_t
2502 * 2488 *
2503 * Atomically updates @v to (@v | @i) with re 2489 * Atomically updates @v to (@v | @i) with relaxed ordering.
2504 * 2490 *
2505 * Unsafe to use in noinstr code; use raw_ato 2491 * Unsafe to use in noinstr code; use raw_atomic64_or() there.
2506 * 2492 *
2507 * Return: Nothing. 2493 * Return: Nothing.
2508 */ 2494 */
2509 static __always_inline void 2495 static __always_inline void
2510 atomic64_or(s64 i, atomic64_t *v) 2496 atomic64_or(s64 i, atomic64_t *v)
2511 { 2497 {
2512 instrument_atomic_read_write(v, sizeo 2498 instrument_atomic_read_write(v, sizeof(*v));
2513 raw_atomic64_or(i, v); 2499 raw_atomic64_or(i, v);
2514 } 2500 }
2515 2501
2516 /** 2502 /**
2517 * atomic64_fetch_or() - atomic bitwise OR wi 2503 * atomic64_fetch_or() - atomic bitwise OR with full ordering
2518 * @i: s64 value 2504 * @i: s64 value
2519 * @v: pointer to atomic64_t 2505 * @v: pointer to atomic64_t
2520 * 2506 *
2521 * Atomically updates @v to (@v | @i) with fu 2507 * Atomically updates @v to (@v | @i) with full ordering.
2522 * 2508 *
2523 * Unsafe to use in noinstr code; use raw_ato 2509 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
2524 * 2510 *
2525 * Return: The original value of @v. 2511 * Return: The original value of @v.
2526 */ 2512 */
2527 static __always_inline s64 2513 static __always_inline s64
2528 atomic64_fetch_or(s64 i, atomic64_t *v) 2514 atomic64_fetch_or(s64 i, atomic64_t *v)
2529 { 2515 {
2530 kcsan_mb(); 2516 kcsan_mb();
2531 instrument_atomic_read_write(v, sizeo 2517 instrument_atomic_read_write(v, sizeof(*v));
2532 return raw_atomic64_fetch_or(i, v); 2518 return raw_atomic64_fetch_or(i, v);
2533 } 2519 }
2534 2520
2535 /** 2521 /**
2536 * atomic64_fetch_or_acquire() - atomic bitwi 2522 * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
2537 * @i: s64 value 2523 * @i: s64 value
2538 * @v: pointer to atomic64_t 2524 * @v: pointer to atomic64_t
2539 * 2525 *
2540 * Atomically updates @v to (@v | @i) with ac 2526 * Atomically updates @v to (@v | @i) with acquire ordering.
2541 * 2527 *
2542 * Unsafe to use in noinstr code; use raw_ato 2528 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
2543 * 2529 *
2544 * Return: The original value of @v. 2530 * Return: The original value of @v.
2545 */ 2531 */
2546 static __always_inline s64 2532 static __always_inline s64
2547 atomic64_fetch_or_acquire(s64 i, atomic64_t * 2533 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2548 { 2534 {
2549 instrument_atomic_read_write(v, sizeo 2535 instrument_atomic_read_write(v, sizeof(*v));
2550 return raw_atomic64_fetch_or_acquire( 2536 return raw_atomic64_fetch_or_acquire(i, v);
2551 } 2537 }
2552 2538
2553 /** 2539 /**
2554 * atomic64_fetch_or_release() - atomic bitwi 2540 * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
2555 * @i: s64 value 2541 * @i: s64 value
2556 * @v: pointer to atomic64_t 2542 * @v: pointer to atomic64_t
2557 * 2543 *
2558 * Atomically updates @v to (@v | @i) with re 2544 * Atomically updates @v to (@v | @i) with release ordering.
2559 * 2545 *
2560 * Unsafe to use in noinstr code; use raw_ato 2546 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
2561 * 2547 *
2562 * Return: The original value of @v. 2548 * Return: The original value of @v.
2563 */ 2549 */
2564 static __always_inline s64 2550 static __always_inline s64
2565 atomic64_fetch_or_release(s64 i, atomic64_t * 2551 atomic64_fetch_or_release(s64 i, atomic64_t *v)
2566 { 2552 {
2567 kcsan_release(); 2553 kcsan_release();
2568 instrument_atomic_read_write(v, sizeo 2554 instrument_atomic_read_write(v, sizeof(*v));
2569 return raw_atomic64_fetch_or_release( 2555 return raw_atomic64_fetch_or_release(i, v);
2570 } 2556 }
2571 2557
2572 /** 2558 /**
2573 * atomic64_fetch_or_relaxed() - atomic bitwi 2559 * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
2574 * @i: s64 value 2560 * @i: s64 value
2575 * @v: pointer to atomic64_t 2561 * @v: pointer to atomic64_t
2576 * 2562 *
2577 * Atomically updates @v to (@v | @i) with re 2563 * Atomically updates @v to (@v | @i) with relaxed ordering.
2578 * 2564 *
2579 * Unsafe to use in noinstr code; use raw_ato 2565 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
2580 * 2566 *
2581 * Return: The original value of @v. 2567 * Return: The original value of @v.
2582 */ 2568 */
2583 static __always_inline s64 2569 static __always_inline s64
2584 atomic64_fetch_or_relaxed(s64 i, atomic64_t * 2570 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
2585 { 2571 {
2586 instrument_atomic_read_write(v, sizeo 2572 instrument_atomic_read_write(v, sizeof(*v));
2587 return raw_atomic64_fetch_or_relaxed( 2573 return raw_atomic64_fetch_or_relaxed(i, v);
2588 } 2574 }
2589 2575
2590 /** 2576 /**
2591 * atomic64_xor() - atomic bitwise XOR with r 2577 * atomic64_xor() - atomic bitwise XOR with relaxed ordering
2592 * @i: s64 value 2578 * @i: s64 value
2593 * @v: pointer to atomic64_t 2579 * @v: pointer to atomic64_t
2594 * 2580 *
2595 * Atomically updates @v to (@v ^ @i) with re 2581 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2596 * 2582 *
2597 * Unsafe to use in noinstr code; use raw_ato 2583 * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
2598 * 2584 *
2599 * Return: Nothing. 2585 * Return: Nothing.
2600 */ 2586 */
2601 static __always_inline void 2587 static __always_inline void
2602 atomic64_xor(s64 i, atomic64_t *v) 2588 atomic64_xor(s64 i, atomic64_t *v)
2603 { 2589 {
2604 instrument_atomic_read_write(v, sizeo 2590 instrument_atomic_read_write(v, sizeof(*v));
2605 raw_atomic64_xor(i, v); 2591 raw_atomic64_xor(i, v);
2606 } 2592 }
2607 2593
2608 /** 2594 /**
2609 * atomic64_fetch_xor() - atomic bitwise XOR 2595 * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
2610 * @i: s64 value 2596 * @i: s64 value
2611 * @v: pointer to atomic64_t 2597 * @v: pointer to atomic64_t
2612 * 2598 *
2613 * Atomically updates @v to (@v ^ @i) with fu 2599 * Atomically updates @v to (@v ^ @i) with full ordering.
2614 * 2600 *
2615 * Unsafe to use in noinstr code; use raw_ato 2601 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
2616 * 2602 *
2617 * Return: The original value of @v. 2603 * Return: The original value of @v.
2618 */ 2604 */
2619 static __always_inline s64 2605 static __always_inline s64
2620 atomic64_fetch_xor(s64 i, atomic64_t *v) 2606 atomic64_fetch_xor(s64 i, atomic64_t *v)
2621 { 2607 {
2622 kcsan_mb(); 2608 kcsan_mb();
2623 instrument_atomic_read_write(v, sizeo 2609 instrument_atomic_read_write(v, sizeof(*v));
2624 return raw_atomic64_fetch_xor(i, v); 2610 return raw_atomic64_fetch_xor(i, v);
2625 } 2611 }
2626 2612
2627 /** 2613 /**
2628 * atomic64_fetch_xor_acquire() - atomic bitw 2614 * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
2629 * @i: s64 value 2615 * @i: s64 value
2630 * @v: pointer to atomic64_t 2616 * @v: pointer to atomic64_t
2631 * 2617 *
2632 * Atomically updates @v to (@v ^ @i) with ac 2618 * Atomically updates @v to (@v ^ @i) with acquire ordering.
2633 * 2619 *
2634 * Unsafe to use in noinstr code; use raw_ato 2620 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
2635 * 2621 *
2636 * Return: The original value of @v. 2622 * Return: The original value of @v.
2637 */ 2623 */
2638 static __always_inline s64 2624 static __always_inline s64
2639 atomic64_fetch_xor_acquire(s64 i, atomic64_t 2625 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2640 { 2626 {
2641 instrument_atomic_read_write(v, sizeo 2627 instrument_atomic_read_write(v, sizeof(*v));
2642 return raw_atomic64_fetch_xor_acquire 2628 return raw_atomic64_fetch_xor_acquire(i, v);
2643 } 2629 }
2644 2630
2645 /** 2631 /**
2646 * atomic64_fetch_xor_release() - atomic bitw 2632 * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
2647 * @i: s64 value 2633 * @i: s64 value
2648 * @v: pointer to atomic64_t 2634 * @v: pointer to atomic64_t
2649 * 2635 *
2650 * Atomically updates @v to (@v ^ @i) with re 2636 * Atomically updates @v to (@v ^ @i) with release ordering.
2651 * 2637 *
2652 * Unsafe to use in noinstr code; use raw_ato 2638 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
2653 * 2639 *
2654 * Return: The original value of @v. 2640 * Return: The original value of @v.
2655 */ 2641 */
2656 static __always_inline s64 2642 static __always_inline s64
2657 atomic64_fetch_xor_release(s64 i, atomic64_t 2643 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2658 { 2644 {
2659 kcsan_release(); 2645 kcsan_release();
2660 instrument_atomic_read_write(v, sizeo 2646 instrument_atomic_read_write(v, sizeof(*v));
2661 return raw_atomic64_fetch_xor_release 2647 return raw_atomic64_fetch_xor_release(i, v);
2662 } 2648 }
2663 2649
2664 /** 2650 /**
2665 * atomic64_fetch_xor_relaxed() - atomic bitw 2651 * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
2666 * @i: s64 value 2652 * @i: s64 value
2667 * @v: pointer to atomic64_t 2653 * @v: pointer to atomic64_t
2668 * 2654 *
2669 * Atomically updates @v to (@v ^ @i) with re 2655 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2670 * 2656 *
2671 * Unsafe to use in noinstr code; use raw_ato 2657 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
2672 * 2658 *
2673 * Return: The original value of @v. 2659 * Return: The original value of @v.
2674 */ 2660 */
2675 static __always_inline s64 2661 static __always_inline s64
2676 atomic64_fetch_xor_relaxed(s64 i, atomic64_t 2662 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
2677 { 2663 {
2678 instrument_atomic_read_write(v, sizeo 2664 instrument_atomic_read_write(v, sizeof(*v));
2679 return raw_atomic64_fetch_xor_relaxed 2665 return raw_atomic64_fetch_xor_relaxed(i, v);
2680 } 2666 }
2681 2667
2682 /** 2668 /**
2683 * atomic64_xchg() - atomic exchange with ful 2669 * atomic64_xchg() - atomic exchange with full ordering
2684 * @v: pointer to atomic64_t 2670 * @v: pointer to atomic64_t
2685 * @new: s64 value to assign 2671 * @new: s64 value to assign
2686 * 2672 *
2687 * Atomically updates @v to @new with full or 2673 * Atomically updates @v to @new with full ordering.
2688 * 2674 *
2689 * Unsafe to use in noinstr code; use raw_ato 2675 * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
2690 * 2676 *
2691 * Return: The original value of @v. 2677 * Return: The original value of @v.
2692 */ 2678 */
2693 static __always_inline s64 2679 static __always_inline s64
2694 atomic64_xchg(atomic64_t *v, s64 new) 2680 atomic64_xchg(atomic64_t *v, s64 new)
2695 { 2681 {
2696 kcsan_mb(); 2682 kcsan_mb();
2697 instrument_atomic_read_write(v, sizeo 2683 instrument_atomic_read_write(v, sizeof(*v));
2698 return raw_atomic64_xchg(v, new); 2684 return raw_atomic64_xchg(v, new);
2699 } 2685 }
2700 2686
2701 /** 2687 /**
2702 * atomic64_xchg_acquire() - atomic exchange 2688 * atomic64_xchg_acquire() - atomic exchange with acquire ordering
2703 * @v: pointer to atomic64_t 2689 * @v: pointer to atomic64_t
2704 * @new: s64 value to assign 2690 * @new: s64 value to assign
2705 * 2691 *
2706 * Atomically updates @v to @new with acquire 2692 * Atomically updates @v to @new with acquire ordering.
2707 * 2693 *
2708 * Unsafe to use in noinstr code; use raw_ato 2694 * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
2709 * 2695 *
2710 * Return: The original value of @v. 2696 * Return: The original value of @v.
2711 */ 2697 */
2712 static __always_inline s64 2698 static __always_inline s64
2713 atomic64_xchg_acquire(atomic64_t *v, s64 new) 2699 atomic64_xchg_acquire(atomic64_t *v, s64 new)
2714 { 2700 {
2715 instrument_atomic_read_write(v, sizeo 2701 instrument_atomic_read_write(v, sizeof(*v));
2716 return raw_atomic64_xchg_acquire(v, n 2702 return raw_atomic64_xchg_acquire(v, new);
2717 } 2703 }
2718 2704
2719 /** 2705 /**
2720 * atomic64_xchg_release() - atomic exchange 2706 * atomic64_xchg_release() - atomic exchange with release ordering
2721 * @v: pointer to atomic64_t 2707 * @v: pointer to atomic64_t
2722 * @new: s64 value to assign 2708 * @new: s64 value to assign
2723 * 2709 *
2724 * Atomically updates @v to @new with release 2710 * Atomically updates @v to @new with release ordering.
2725 * 2711 *
2726 * Unsafe to use in noinstr code; use raw_ato 2712 * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
2727 * 2713 *
2728 * Return: The original value of @v. 2714 * Return: The original value of @v.
2729 */ 2715 */
2730 static __always_inline s64 2716 static __always_inline s64
2731 atomic64_xchg_release(atomic64_t *v, s64 new) 2717 atomic64_xchg_release(atomic64_t *v, s64 new)
2732 { 2718 {
2733 kcsan_release(); 2719 kcsan_release();
2734 instrument_atomic_read_write(v, sizeo 2720 instrument_atomic_read_write(v, sizeof(*v));
2735 return raw_atomic64_xchg_release(v, n 2721 return raw_atomic64_xchg_release(v, new);
2736 } 2722 }
2737 2723
2738 /** 2724 /**
2739 * atomic64_xchg_relaxed() - atomic exchange 2725 * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
2740 * @v: pointer to atomic64_t 2726 * @v: pointer to atomic64_t
2741 * @new: s64 value to assign 2727 * @new: s64 value to assign
2742 * 2728 *
2743 * Atomically updates @v to @new with relaxed 2729 * Atomically updates @v to @new with relaxed ordering.
2744 * 2730 *
2745 * Unsafe to use in noinstr code; use raw_ato 2731 * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
2746 * 2732 *
2747 * Return: The original value of @v. 2733 * Return: The original value of @v.
2748 */ 2734 */
2749 static __always_inline s64 2735 static __always_inline s64
2750 atomic64_xchg_relaxed(atomic64_t *v, s64 new) 2736 atomic64_xchg_relaxed(atomic64_t *v, s64 new)
2751 { 2737 {
2752 instrument_atomic_read_write(v, sizeo 2738 instrument_atomic_read_write(v, sizeof(*v));
2753 return raw_atomic64_xchg_relaxed(v, n 2739 return raw_atomic64_xchg_relaxed(v, new);
2754 } 2740 }
2755 2741
2756 /** 2742 /**
2757 * atomic64_cmpxchg() - atomic compare and ex 2743 * atomic64_cmpxchg() - atomic compare and exchange with full ordering
2758 * @v: pointer to atomic64_t 2744 * @v: pointer to atomic64_t
2759 * @old: s64 value to compare with 2745 * @old: s64 value to compare with
2760 * @new: s64 value to assign 2746 * @new: s64 value to assign
2761 * 2747 *
2762 * If (@v == @old), atomically updates @v to 2748 * If (@v == @old), atomically updates @v to @new with full ordering.
2763 * Otherwise, @v is not modified and relaxed <<
2764 * 2749 *
2765 * Unsafe to use in noinstr code; use raw_ato 2750 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
2766 * 2751 *
2767 * Return: The original value of @v. 2752 * Return: The original value of @v.
2768 */ 2753 */
2769 static __always_inline s64 2754 static __always_inline s64
2770 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 2755 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2771 { 2756 {
2772 kcsan_mb(); 2757 kcsan_mb();
2773 instrument_atomic_read_write(v, sizeo 2758 instrument_atomic_read_write(v, sizeof(*v));
2774 return raw_atomic64_cmpxchg(v, old, n 2759 return raw_atomic64_cmpxchg(v, old, new);
2775 } 2760 }
2776 2761
2777 /** 2762 /**
2778 * atomic64_cmpxchg_acquire() - atomic compar 2763 * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2779 * @v: pointer to atomic64_t 2764 * @v: pointer to atomic64_t
2780 * @old: s64 value to compare with 2765 * @old: s64 value to compare with
2781 * @new: s64 value to assign 2766 * @new: s64 value to assign
2782 * 2767 *
2783 * If (@v == @old), atomically updates @v to 2768 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2784 * Otherwise, @v is not modified and relaxed <<
2785 * 2769 *
2786 * Unsafe to use in noinstr code; use raw_ato 2770 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
2787 * 2771 *
2788 * Return: The original value of @v. 2772 * Return: The original value of @v.
2789 */ 2773 */
2790 static __always_inline s64 2774 static __always_inline s64
2791 atomic64_cmpxchg_acquire(atomic64_t *v, s64 o 2775 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2792 { 2776 {
2793 instrument_atomic_read_write(v, sizeo 2777 instrument_atomic_read_write(v, sizeof(*v));
2794 return raw_atomic64_cmpxchg_acquire(v 2778 return raw_atomic64_cmpxchg_acquire(v, old, new);
2795 } 2779 }
2796 2780
2797 /** 2781 /**
2798 * atomic64_cmpxchg_release() - atomic compar 2782 * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
2799 * @v: pointer to atomic64_t 2783 * @v: pointer to atomic64_t
2800 * @old: s64 value to compare with 2784 * @old: s64 value to compare with
2801 * @new: s64 value to assign 2785 * @new: s64 value to assign
2802 * 2786 *
2803 * If (@v == @old), atomically updates @v to 2787 * If (@v == @old), atomically updates @v to @new with release ordering.
2804 * Otherwise, @v is not modified and relaxed <<
2805 * 2788 *
2806 * Unsafe to use in noinstr code; use raw_ato 2789 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
2807 * 2790 *
2808 * Return: The original value of @v. 2791 * Return: The original value of @v.
2809 */ 2792 */
2810 static __always_inline s64 2793 static __always_inline s64
2811 atomic64_cmpxchg_release(atomic64_t *v, s64 o 2794 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2812 { 2795 {
2813 kcsan_release(); 2796 kcsan_release();
2814 instrument_atomic_read_write(v, sizeo 2797 instrument_atomic_read_write(v, sizeof(*v));
2815 return raw_atomic64_cmpxchg_release(v 2798 return raw_atomic64_cmpxchg_release(v, old, new);
2816 } 2799 }
2817 2800
2818 /** 2801 /**
2819 * atomic64_cmpxchg_relaxed() - atomic compar 2802 * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2820 * @v: pointer to atomic64_t 2803 * @v: pointer to atomic64_t
2821 * @old: s64 value to compare with 2804 * @old: s64 value to compare with
2822 * @new: s64 value to assign 2805 * @new: s64 value to assign
2823 * 2806 *
2824 * If (@v == @old), atomically updates @v to 2807 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2825 * Otherwise, @v is not modified and relaxed <<
2826 * 2808 *
2827 * Unsafe to use in noinstr code; use raw_ato 2809 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
2828 * 2810 *
2829 * Return: The original value of @v. 2811 * Return: The original value of @v.
2830 */ 2812 */
2831 static __always_inline s64 2813 static __always_inline s64
2832 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 o 2814 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
2833 { 2815 {
2834 instrument_atomic_read_write(v, sizeo 2816 instrument_atomic_read_write(v, sizeof(*v));
2835 return raw_atomic64_cmpxchg_relaxed(v 2817 return raw_atomic64_cmpxchg_relaxed(v, old, new);
2836 } 2818 }
2837 2819
2838 /** 2820 /**
2839 * atomic64_try_cmpxchg() - atomic compare an 2821 * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
2840 * @v: pointer to atomic64_t 2822 * @v: pointer to atomic64_t
2841 * @old: pointer to s64 value to compare with 2823 * @old: pointer to s64 value to compare with
2842 * @new: s64 value to assign 2824 * @new: s64 value to assign
2843 * 2825 *
2844 * If (@v == @old), atomically updates @v to 2826 * If (@v == @old), atomically updates @v to @new with full ordering.
2845 * Otherwise, @v is not modified, @old is upd !! 2827 * Otherwise, updates @old to the current value of @v.
2846 * and relaxed ordering is provided. <<
2847 * 2828 *
2848 * Unsafe to use in noinstr code; use raw_ato 2829 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
2849 * 2830 *
2850 * Return: @true if the exchange occured, @fa 2831 * Return: @true if the exchange occured, @false otherwise.
2851 */ 2832 */
2852 static __always_inline bool 2833 static __always_inline bool
2853 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, 2834 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2854 { 2835 {
2855 kcsan_mb(); 2836 kcsan_mb();
2856 instrument_atomic_read_write(v, sizeo 2837 instrument_atomic_read_write(v, sizeof(*v));
2857 instrument_atomic_read_write(old, siz 2838 instrument_atomic_read_write(old, sizeof(*old));
2858 return raw_atomic64_try_cmpxchg(v, ol 2839 return raw_atomic64_try_cmpxchg(v, old, new);
2859 } 2840 }
2860 2841
2861 /** 2842 /**
2862 * atomic64_try_cmpxchg_acquire() - atomic co 2843 * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2863 * @v: pointer to atomic64_t 2844 * @v: pointer to atomic64_t
2864 * @old: pointer to s64 value to compare with 2845 * @old: pointer to s64 value to compare with
2865 * @new: s64 value to assign 2846 * @new: s64 value to assign
2866 * 2847 *
2867 * If (@v == @old), atomically updates @v to 2848 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2868 * Otherwise, @v is not modified, @old is upd !! 2849 * Otherwise, updates @old to the current value of @v.
2869 * and relaxed ordering is provided. <<
2870 * 2850 *
2871 * Unsafe to use in noinstr code; use raw_ato 2851 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
2872 * 2852 *
2873 * Return: @true if the exchange occured, @fa 2853 * Return: @true if the exchange occured, @false otherwise.
2874 */ 2854 */
2875 static __always_inline bool 2855 static __always_inline bool
2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s 2856 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2877 { 2857 {
2878 instrument_atomic_read_write(v, sizeo 2858 instrument_atomic_read_write(v, sizeof(*v));
2879 instrument_atomic_read_write(old, siz 2859 instrument_atomic_read_write(old, sizeof(*old));
2880 return raw_atomic64_try_cmpxchg_acqui 2860 return raw_atomic64_try_cmpxchg_acquire(v, old, new);
2881 } 2861 }
2882 2862
2883 /** 2863 /**
2884 * atomic64_try_cmpxchg_release() - atomic co 2864 * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
2885 * @v: pointer to atomic64_t 2865 * @v: pointer to atomic64_t
2886 * @old: pointer to s64 value to compare with 2866 * @old: pointer to s64 value to compare with
2887 * @new: s64 value to assign 2867 * @new: s64 value to assign
2888 * 2868 *
2889 * If (@v == @old), atomically updates @v to 2869 * If (@v == @old), atomically updates @v to @new with release ordering.
2890 * Otherwise, @v is not modified, @old is upd !! 2870 * Otherwise, updates @old to the current value of @v.
2891 * and relaxed ordering is provided. <<
2892 * 2871 *
2893 * Unsafe to use in noinstr code; use raw_ato 2872 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
2894 * 2873 *
2895 * Return: @true if the exchange occured, @fa 2874 * Return: @true if the exchange occured, @false otherwise.
2896 */ 2875 */
2897 static __always_inline bool 2876 static __always_inline bool
2898 atomic64_try_cmpxchg_release(atomic64_t *v, s 2877 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2899 { 2878 {
2900 kcsan_release(); 2879 kcsan_release();
2901 instrument_atomic_read_write(v, sizeo 2880 instrument_atomic_read_write(v, sizeof(*v));
2902 instrument_atomic_read_write(old, siz 2881 instrument_atomic_read_write(old, sizeof(*old));
2903 return raw_atomic64_try_cmpxchg_relea 2882 return raw_atomic64_try_cmpxchg_release(v, old, new);
2904 } 2883 }
2905 2884
2906 /** 2885 /**
2907 * atomic64_try_cmpxchg_relaxed() - atomic co 2886 * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2908 * @v: pointer to atomic64_t 2887 * @v: pointer to atomic64_t
2909 * @old: pointer to s64 value to compare with 2888 * @old: pointer to s64 value to compare with
2910 * @new: s64 value to assign 2889 * @new: s64 value to assign
2911 * 2890 *
2912 * If (@v == @old), atomically updates @v to 2891 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2913 * Otherwise, @v is not modified, @old is upd !! 2892 * Otherwise, updates @old to the current value of @v.
2914 * and relaxed ordering is provided. <<
2915 * 2893 *
2916 * Unsafe to use in noinstr code; use raw_ato 2894 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
2917 * 2895 *
2918 * Return: @true if the exchange occured, @fa 2896 * Return: @true if the exchange occured, @false otherwise.
2919 */ 2897 */
2920 static __always_inline bool 2898 static __always_inline bool
2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s 2899 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2922 { 2900 {
2923 instrument_atomic_read_write(v, sizeo 2901 instrument_atomic_read_write(v, sizeof(*v));
2924 instrument_atomic_read_write(old, siz 2902 instrument_atomic_read_write(old, sizeof(*old));
2925 return raw_atomic64_try_cmpxchg_relax 2903 return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
2926 } 2904 }
2927 2905
2928 /** 2906 /**
2929 * atomic64_sub_and_test() - atomic subtract 2907 * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
2930 * @i: s64 value to subtract !! 2908 * @i: s64 value to add
2931 * @v: pointer to atomic64_t 2909 * @v: pointer to atomic64_t
2932 * 2910 *
2933 * Atomically updates @v to (@v - @i) with fu 2911 * Atomically updates @v to (@v - @i) with full ordering.
2934 * 2912 *
2935 * Unsafe to use in noinstr code; use raw_ato 2913 * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
2936 * 2914 *
2937 * Return: @true if the resulting value of @v 2915 * Return: @true if the resulting value of @v is zero, @false otherwise.
2938 */ 2916 */
2939 static __always_inline bool 2917 static __always_inline bool
2940 atomic64_sub_and_test(s64 i, atomic64_t *v) 2918 atomic64_sub_and_test(s64 i, atomic64_t *v)
2941 { 2919 {
2942 kcsan_mb(); 2920 kcsan_mb();
2943 instrument_atomic_read_write(v, sizeo 2921 instrument_atomic_read_write(v, sizeof(*v));
2944 return raw_atomic64_sub_and_test(i, v 2922 return raw_atomic64_sub_and_test(i, v);
2945 } 2923 }
2946 2924
2947 /** 2925 /**
2948 * atomic64_dec_and_test() - atomic decrement 2926 * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
2949 * @v: pointer to atomic64_t 2927 * @v: pointer to atomic64_t
2950 * 2928 *
2951 * Atomically updates @v to (@v - 1) with ful 2929 * Atomically updates @v to (@v - 1) with full ordering.
2952 * 2930 *
2953 * Unsafe to use in noinstr code; use raw_ato 2931 * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
2954 * 2932 *
2955 * Return: @true if the resulting value of @v 2933 * Return: @true if the resulting value of @v is zero, @false otherwise.
2956 */ 2934 */
2957 static __always_inline bool 2935 static __always_inline bool
2958 atomic64_dec_and_test(atomic64_t *v) 2936 atomic64_dec_and_test(atomic64_t *v)
2959 { 2937 {
2960 kcsan_mb(); 2938 kcsan_mb();
2961 instrument_atomic_read_write(v, sizeo 2939 instrument_atomic_read_write(v, sizeof(*v));
2962 return raw_atomic64_dec_and_test(v); 2940 return raw_atomic64_dec_and_test(v);
2963 } 2941 }
2964 2942
2965 /** 2943 /**
2966 * atomic64_inc_and_test() - atomic increment 2944 * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
2967 * @v: pointer to atomic64_t 2945 * @v: pointer to atomic64_t
2968 * 2946 *
2969 * Atomically updates @v to (@v + 1) with ful 2947 * Atomically updates @v to (@v + 1) with full ordering.
2970 * 2948 *
2971 * Unsafe to use in noinstr code; use raw_ato 2949 * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
2972 * 2950 *
2973 * Return: @true if the resulting value of @v 2951 * Return: @true if the resulting value of @v is zero, @false otherwise.
2974 */ 2952 */
2975 static __always_inline bool 2953 static __always_inline bool
2976 atomic64_inc_and_test(atomic64_t *v) 2954 atomic64_inc_and_test(atomic64_t *v)
2977 { 2955 {
2978 kcsan_mb(); 2956 kcsan_mb();
2979 instrument_atomic_read_write(v, sizeo 2957 instrument_atomic_read_write(v, sizeof(*v));
2980 return raw_atomic64_inc_and_test(v); 2958 return raw_atomic64_inc_and_test(v);
2981 } 2959 }
2982 2960
2983 /** 2961 /**
2984 * atomic64_add_negative() - atomic add and t 2962 * atomic64_add_negative() - atomic add and test if negative with full ordering
2985 * @i: s64 value to add 2963 * @i: s64 value to add
2986 * @v: pointer to atomic64_t 2964 * @v: pointer to atomic64_t
2987 * 2965 *
2988 * Atomically updates @v to (@v + @i) with fu 2966 * Atomically updates @v to (@v + @i) with full ordering.
2989 * 2967 *
2990 * Unsafe to use in noinstr code; use raw_ato 2968 * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
2991 * 2969 *
2992 * Return: @true if the resulting value of @v 2970 * Return: @true if the resulting value of @v is negative, @false otherwise.
2993 */ 2971 */
2994 static __always_inline bool 2972 static __always_inline bool
2995 atomic64_add_negative(s64 i, atomic64_t *v) 2973 atomic64_add_negative(s64 i, atomic64_t *v)
2996 { 2974 {
2997 kcsan_mb(); 2975 kcsan_mb();
2998 instrument_atomic_read_write(v, sizeo 2976 instrument_atomic_read_write(v, sizeof(*v));
2999 return raw_atomic64_add_negative(i, v 2977 return raw_atomic64_add_negative(i, v);
3000 } 2978 }
3001 2979
3002 /** 2980 /**
3003 * atomic64_add_negative_acquire() - atomic a 2981 * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
3004 * @i: s64 value to add 2982 * @i: s64 value to add
3005 * @v: pointer to atomic64_t 2983 * @v: pointer to atomic64_t
3006 * 2984 *
3007 * Atomically updates @v to (@v + @i) with ac 2985 * Atomically updates @v to (@v + @i) with acquire ordering.
3008 * 2986 *
3009 * Unsafe to use in noinstr code; use raw_ato 2987 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
3010 * 2988 *
3011 * Return: @true if the resulting value of @v 2989 * Return: @true if the resulting value of @v is negative, @false otherwise.
3012 */ 2990 */
3013 static __always_inline bool 2991 static __always_inline bool
3014 atomic64_add_negative_acquire(s64 i, atomic64 2992 atomic64_add_negative_acquire(s64 i, atomic64_t *v)
3015 { 2993 {
3016 instrument_atomic_read_write(v, sizeo 2994 instrument_atomic_read_write(v, sizeof(*v));
3017 return raw_atomic64_add_negative_acqu 2995 return raw_atomic64_add_negative_acquire(i, v);
3018 } 2996 }
3019 2997
3020 /** 2998 /**
3021 * atomic64_add_negative_release() - atomic a 2999 * atomic64_add_negative_release() - atomic add and test if negative with release ordering
3022 * @i: s64 value to add 3000 * @i: s64 value to add
3023 * @v: pointer to atomic64_t 3001 * @v: pointer to atomic64_t
3024 * 3002 *
3025 * Atomically updates @v to (@v + @i) with re 3003 * Atomically updates @v to (@v + @i) with release ordering.
3026 * 3004 *
3027 * Unsafe to use in noinstr code; use raw_ato 3005 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
3028 * 3006 *
3029 * Return: @true if the resulting value of @v 3007 * Return: @true if the resulting value of @v is negative, @false otherwise.
3030 */ 3008 */
3031 static __always_inline bool 3009 static __always_inline bool
3032 atomic64_add_negative_release(s64 i, atomic64 3010 atomic64_add_negative_release(s64 i, atomic64_t *v)
3033 { 3011 {
3034 kcsan_release(); 3012 kcsan_release();
3035 instrument_atomic_read_write(v, sizeo 3013 instrument_atomic_read_write(v, sizeof(*v));
3036 return raw_atomic64_add_negative_rele 3014 return raw_atomic64_add_negative_release(i, v);
3037 } 3015 }
3038 3016
3039 /** 3017 /**
3040 * atomic64_add_negative_relaxed() - atomic a 3018 * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
3041 * @i: s64 value to add 3019 * @i: s64 value to add
3042 * @v: pointer to atomic64_t 3020 * @v: pointer to atomic64_t
3043 * 3021 *
3044 * Atomically updates @v to (@v + @i) with re 3022 * Atomically updates @v to (@v + @i) with relaxed ordering.
3045 * 3023 *
3046 * Unsafe to use in noinstr code; use raw_ato 3024 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
3047 * 3025 *
3048 * Return: @true if the resulting value of @v 3026 * Return: @true if the resulting value of @v is negative, @false otherwise.
3049 */ 3027 */
3050 static __always_inline bool 3028 static __always_inline bool
3051 atomic64_add_negative_relaxed(s64 i, atomic64 3029 atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
3052 { 3030 {
3053 instrument_atomic_read_write(v, sizeo 3031 instrument_atomic_read_write(v, sizeof(*v));
3054 return raw_atomic64_add_negative_rela 3032 return raw_atomic64_add_negative_relaxed(i, v);
3055 } 3033 }
3056 3034
3057 /** 3035 /**
3058 * atomic64_fetch_add_unless() - atomic add u 3036 * atomic64_fetch_add_unless() - atomic add unless value with full ordering
3059 * @v: pointer to atomic64_t 3037 * @v: pointer to atomic64_t
3060 * @a: s64 value to add 3038 * @a: s64 value to add
3061 * @u: s64 value to compare with 3039 * @u: s64 value to compare with
3062 * 3040 *
3063 * If (@v != @u), atomically updates @v to (@ 3041 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3064 * Otherwise, @v is not modified and relaxed <<
3065 * 3042 *
3066 * Unsafe to use in noinstr code; use raw_ato 3043 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
3067 * 3044 *
3068 * Return: The original value of @v. 3045 * Return: The original value of @v.
3069 */ 3046 */
3070 static __always_inline s64 3047 static __always_inline s64
3071 atomic64_fetch_add_unless(atomic64_t *v, s64 3048 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
3072 { 3049 {
3073 kcsan_mb(); 3050 kcsan_mb();
3074 instrument_atomic_read_write(v, sizeo 3051 instrument_atomic_read_write(v, sizeof(*v));
3075 return raw_atomic64_fetch_add_unless( 3052 return raw_atomic64_fetch_add_unless(v, a, u);
3076 } 3053 }
3077 3054
3078 /** 3055 /**
3079 * atomic64_add_unless() - atomic add unless 3056 * atomic64_add_unless() - atomic add unless value with full ordering
3080 * @v: pointer to atomic64_t 3057 * @v: pointer to atomic64_t
3081 * @a: s64 value to add 3058 * @a: s64 value to add
3082 * @u: s64 value to compare with 3059 * @u: s64 value to compare with
3083 * 3060 *
3084 * If (@v != @u), atomically updates @v to (@ 3061 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3085 * Otherwise, @v is not modified and relaxed <<
3086 * 3062 *
3087 * Unsafe to use in noinstr code; use raw_ato 3063 * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
3088 * 3064 *
3089 * Return: @true if @v was updated, @false ot 3065 * Return: @true if @v was updated, @false otherwise.
3090 */ 3066 */
3091 static __always_inline bool 3067 static __always_inline bool
3092 atomic64_add_unless(atomic64_t *v, s64 a, s64 3068 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
3093 { 3069 {
3094 kcsan_mb(); 3070 kcsan_mb();
3095 instrument_atomic_read_write(v, sizeo 3071 instrument_atomic_read_write(v, sizeof(*v));
3096 return raw_atomic64_add_unless(v, a, 3072 return raw_atomic64_add_unless(v, a, u);
3097 } 3073 }
3098 3074
3099 /** 3075 /**
3100 * atomic64_inc_not_zero() - atomic increment 3076 * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
3101 * @v: pointer to atomic64_t 3077 * @v: pointer to atomic64_t
3102 * 3078 *
3103 * If (@v != 0), atomically updates @v to (@v 3079 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
3104 * Otherwise, @v is not modified and relaxed <<
3105 * 3080 *
3106 * Unsafe to use in noinstr code; use raw_ato 3081 * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
3107 * 3082 *
3108 * Return: @true if @v was updated, @false ot 3083 * Return: @true if @v was updated, @false otherwise.
3109 */ 3084 */
3110 static __always_inline bool 3085 static __always_inline bool
3111 atomic64_inc_not_zero(atomic64_t *v) 3086 atomic64_inc_not_zero(atomic64_t *v)
3112 { 3087 {
3113 kcsan_mb(); 3088 kcsan_mb();
3114 instrument_atomic_read_write(v, sizeo 3089 instrument_atomic_read_write(v, sizeof(*v));
3115 return raw_atomic64_inc_not_zero(v); 3090 return raw_atomic64_inc_not_zero(v);
3116 } 3091 }
3117 3092
3118 /** 3093 /**
3119 * atomic64_inc_unless_negative() - atomic in 3094 * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
3120 * @v: pointer to atomic64_t 3095 * @v: pointer to atomic64_t
3121 * 3096 *
3122 * If (@v >= 0), atomically updates @v to (@v 3097 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
3123 * Otherwise, @v is not modified and relaxed <<
3124 * 3098 *
3125 * Unsafe to use in noinstr code; use raw_ato 3099 * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
3126 * 3100 *
3127 * Return: @true if @v was updated, @false ot 3101 * Return: @true if @v was updated, @false otherwise.
3128 */ 3102 */
3129 static __always_inline bool 3103 static __always_inline bool
3130 atomic64_inc_unless_negative(atomic64_t *v) 3104 atomic64_inc_unless_negative(atomic64_t *v)
3131 { 3105 {
3132 kcsan_mb(); 3106 kcsan_mb();
3133 instrument_atomic_read_write(v, sizeo 3107 instrument_atomic_read_write(v, sizeof(*v));
3134 return raw_atomic64_inc_unless_negati 3108 return raw_atomic64_inc_unless_negative(v);
3135 } 3109 }
3136 3110
3137 /** 3111 /**
3138 * atomic64_dec_unless_positive() - atomic de 3112 * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
3139 * @v: pointer to atomic64_t 3113 * @v: pointer to atomic64_t
3140 * 3114 *
3141 * If (@v <= 0), atomically updates @v to (@v 3115 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
3142 * Otherwise, @v is not modified and relaxed <<
3143 * 3116 *
3144 * Unsafe to use in noinstr code; use raw_ato 3117 * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
3145 * 3118 *
3146 * Return: @true if @v was updated, @false ot 3119 * Return: @true if @v was updated, @false otherwise.
3147 */ 3120 */
3148 static __always_inline bool 3121 static __always_inline bool
3149 atomic64_dec_unless_positive(atomic64_t *v) 3122 atomic64_dec_unless_positive(atomic64_t *v)
3150 { 3123 {
3151 kcsan_mb(); 3124 kcsan_mb();
3152 instrument_atomic_read_write(v, sizeo 3125 instrument_atomic_read_write(v, sizeof(*v));
3153 return raw_atomic64_dec_unless_positi 3126 return raw_atomic64_dec_unless_positive(v);
3154 } 3127 }
3155 3128
3156 /** 3129 /**
3157 * atomic64_dec_if_positive() - atomic decrem 3130 * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
3158 * @v: pointer to atomic64_t 3131 * @v: pointer to atomic64_t
3159 * 3132 *
3160 * If (@v > 0), atomically updates @v to (@v 3133 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
3161 * Otherwise, @v is not modified and relaxed <<
3162 * 3134 *
3163 * Unsafe to use in noinstr code; use raw_ato 3135 * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
3164 * 3136 *
3165 * Return: The old value of (@v - 1), regardl 3137 * Return: The old value of (@v - 1), regardless of whether @v was updated.
3166 */ 3138 */
3167 static __always_inline s64 3139 static __always_inline s64
3168 atomic64_dec_if_positive(atomic64_t *v) 3140 atomic64_dec_if_positive(atomic64_t *v)
3169 { 3141 {
3170 kcsan_mb(); 3142 kcsan_mb();
3171 instrument_atomic_read_write(v, sizeo 3143 instrument_atomic_read_write(v, sizeof(*v));
3172 return raw_atomic64_dec_if_positive(v 3144 return raw_atomic64_dec_if_positive(v);
3173 } 3145 }
3174 3146
3175 /** 3147 /**
3176 * atomic_long_read() - atomic load with rela 3148 * atomic_long_read() - atomic load with relaxed ordering
3177 * @v: pointer to atomic_long_t 3149 * @v: pointer to atomic_long_t
3178 * 3150 *
3179 * Atomically loads the value of @v with rela 3151 * Atomically loads the value of @v with relaxed ordering.
3180 * 3152 *
3181 * Unsafe to use in noinstr code; use raw_ato 3153 * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
3182 * 3154 *
3183 * Return: The value loaded from @v. 3155 * Return: The value loaded from @v.
3184 */ 3156 */
3185 static __always_inline long 3157 static __always_inline long
3186 atomic_long_read(const atomic_long_t *v) 3158 atomic_long_read(const atomic_long_t *v)
3187 { 3159 {
3188 instrument_atomic_read(v, sizeof(*v)) 3160 instrument_atomic_read(v, sizeof(*v));
3189 return raw_atomic_long_read(v); 3161 return raw_atomic_long_read(v);
3190 } 3162 }
3191 3163
3192 /** 3164 /**
3193 * atomic_long_read_acquire() - atomic load w 3165 * atomic_long_read_acquire() - atomic load with acquire ordering
3194 * @v: pointer to atomic_long_t 3166 * @v: pointer to atomic_long_t
3195 * 3167 *
3196 * Atomically loads the value of @v with acqu 3168 * Atomically loads the value of @v with acquire ordering.
3197 * 3169 *
3198 * Unsafe to use in noinstr code; use raw_ato 3170 * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
3199 * 3171 *
3200 * Return: The value loaded from @v. 3172 * Return: The value loaded from @v.
3201 */ 3173 */
3202 static __always_inline long 3174 static __always_inline long
3203 atomic_long_read_acquire(const atomic_long_t 3175 atomic_long_read_acquire(const atomic_long_t *v)
3204 { 3176 {
3205 instrument_atomic_read(v, sizeof(*v)) 3177 instrument_atomic_read(v, sizeof(*v));
3206 return raw_atomic_long_read_acquire(v 3178 return raw_atomic_long_read_acquire(v);
3207 } 3179 }
3208 3180
3209 /** 3181 /**
3210 * atomic_long_set() - atomic set with relaxe 3182 * atomic_long_set() - atomic set with relaxed ordering
3211 * @v: pointer to atomic_long_t 3183 * @v: pointer to atomic_long_t
3212 * @i: long value to assign 3184 * @i: long value to assign
3213 * 3185 *
3214 * Atomically sets @v to @i with relaxed orde 3186 * Atomically sets @v to @i with relaxed ordering.
3215 * 3187 *
3216 * Unsafe to use in noinstr code; use raw_ato 3188 * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
3217 * 3189 *
3218 * Return: Nothing. 3190 * Return: Nothing.
3219 */ 3191 */
3220 static __always_inline void 3192 static __always_inline void
3221 atomic_long_set(atomic_long_t *v, long i) 3193 atomic_long_set(atomic_long_t *v, long i)
3222 { 3194 {
3223 instrument_atomic_write(v, sizeof(*v) 3195 instrument_atomic_write(v, sizeof(*v));
3224 raw_atomic_long_set(v, i); 3196 raw_atomic_long_set(v, i);
3225 } 3197 }
3226 3198
3227 /** 3199 /**
3228 * atomic_long_set_release() - atomic set wit 3200 * atomic_long_set_release() - atomic set with release ordering
3229 * @v: pointer to atomic_long_t 3201 * @v: pointer to atomic_long_t
3230 * @i: long value to assign 3202 * @i: long value to assign
3231 * 3203 *
3232 * Atomically sets @v to @i with release orde 3204 * Atomically sets @v to @i with release ordering.
3233 * 3205 *
3234 * Unsafe to use in noinstr code; use raw_ato 3206 * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
3235 * 3207 *
3236 * Return: Nothing. 3208 * Return: Nothing.
3237 */ 3209 */
3238 static __always_inline void 3210 static __always_inline void
3239 atomic_long_set_release(atomic_long_t *v, lon 3211 atomic_long_set_release(atomic_long_t *v, long i)
3240 { 3212 {
3241 kcsan_release(); 3213 kcsan_release();
3242 instrument_atomic_write(v, sizeof(*v) 3214 instrument_atomic_write(v, sizeof(*v));
3243 raw_atomic_long_set_release(v, i); 3215 raw_atomic_long_set_release(v, i);
3244 } 3216 }
3245 3217
3246 /** 3218 /**
3247 * atomic_long_add() - atomic add with relaxe 3219 * atomic_long_add() - atomic add with relaxed ordering
3248 * @i: long value to add 3220 * @i: long value to add
3249 * @v: pointer to atomic_long_t 3221 * @v: pointer to atomic_long_t
3250 * 3222 *
3251 * Atomically updates @v to (@v + @i) with re 3223 * Atomically updates @v to (@v + @i) with relaxed ordering.
3252 * 3224 *
3253 * Unsafe to use in noinstr code; use raw_ato 3225 * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
3254 * 3226 *
3255 * Return: Nothing. 3227 * Return: Nothing.
3256 */ 3228 */
3257 static __always_inline void 3229 static __always_inline void
3258 atomic_long_add(long i, atomic_long_t *v) 3230 atomic_long_add(long i, atomic_long_t *v)
3259 { 3231 {
3260 instrument_atomic_read_write(v, sizeo 3232 instrument_atomic_read_write(v, sizeof(*v));
3261 raw_atomic_long_add(i, v); 3233 raw_atomic_long_add(i, v);
3262 } 3234 }
3263 3235
3264 /** 3236 /**
3265 * atomic_long_add_return() - atomic add with 3237 * atomic_long_add_return() - atomic add with full ordering
3266 * @i: long value to add 3238 * @i: long value to add
3267 * @v: pointer to atomic_long_t 3239 * @v: pointer to atomic_long_t
3268 * 3240 *
3269 * Atomically updates @v to (@v + @i) with fu 3241 * Atomically updates @v to (@v + @i) with full ordering.
3270 * 3242 *
3271 * Unsafe to use in noinstr code; use raw_ato 3243 * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
3272 * 3244 *
3273 * Return: The updated value of @v. 3245 * Return: The updated value of @v.
3274 */ 3246 */
3275 static __always_inline long 3247 static __always_inline long
3276 atomic_long_add_return(long i, atomic_long_t 3248 atomic_long_add_return(long i, atomic_long_t *v)
3277 { 3249 {
3278 kcsan_mb(); 3250 kcsan_mb();
3279 instrument_atomic_read_write(v, sizeo 3251 instrument_atomic_read_write(v, sizeof(*v));
3280 return raw_atomic_long_add_return(i, 3252 return raw_atomic_long_add_return(i, v);
3281 } 3253 }
3282 3254
3283 /** 3255 /**
3284 * atomic_long_add_return_acquire() - atomic 3256 * atomic_long_add_return_acquire() - atomic add with acquire ordering
3285 * @i: long value to add 3257 * @i: long value to add
3286 * @v: pointer to atomic_long_t 3258 * @v: pointer to atomic_long_t
3287 * 3259 *
3288 * Atomically updates @v to (@v + @i) with ac 3260 * Atomically updates @v to (@v + @i) with acquire ordering.
3289 * 3261 *
3290 * Unsafe to use in noinstr code; use raw_ato 3262 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
3291 * 3263 *
3292 * Return: The updated value of @v. 3264 * Return: The updated value of @v.
3293 */ 3265 */
3294 static __always_inline long 3266 static __always_inline long
3295 atomic_long_add_return_acquire(long i, atomic 3267 atomic_long_add_return_acquire(long i, atomic_long_t *v)
3296 { 3268 {
3297 instrument_atomic_read_write(v, sizeo 3269 instrument_atomic_read_write(v, sizeof(*v));
3298 return raw_atomic_long_add_return_acq 3270 return raw_atomic_long_add_return_acquire(i, v);
3299 } 3271 }
3300 3272
3301 /** 3273 /**
3302 * atomic_long_add_return_release() - atomic 3274 * atomic_long_add_return_release() - atomic add with release ordering
3303 * @i: long value to add 3275 * @i: long value to add
3304 * @v: pointer to atomic_long_t 3276 * @v: pointer to atomic_long_t
3305 * 3277 *
3306 * Atomically updates @v to (@v + @i) with re 3278 * Atomically updates @v to (@v + @i) with release ordering.
3307 * 3279 *
3308 * Unsafe to use in noinstr code; use raw_ato 3280 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
3309 * 3281 *
3310 * Return: The updated value of @v. 3282 * Return: The updated value of @v.
3311 */ 3283 */
3312 static __always_inline long 3284 static __always_inline long
3313 atomic_long_add_return_release(long i, atomic 3285 atomic_long_add_return_release(long i, atomic_long_t *v)
3314 { 3286 {
3315 kcsan_release(); 3287 kcsan_release();
3316 instrument_atomic_read_write(v, sizeo 3288 instrument_atomic_read_write(v, sizeof(*v));
3317 return raw_atomic_long_add_return_rel 3289 return raw_atomic_long_add_return_release(i, v);
3318 } 3290 }
3319 3291
3320 /** 3292 /**
3321 * atomic_long_add_return_relaxed() - atomic 3293 * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
3322 * @i: long value to add 3294 * @i: long value to add
3323 * @v: pointer to atomic_long_t 3295 * @v: pointer to atomic_long_t
3324 * 3296 *
3325 * Atomically updates @v to (@v + @i) with re 3297 * Atomically updates @v to (@v + @i) with relaxed ordering.
3326 * 3298 *
3327 * Unsafe to use in noinstr code; use raw_ato 3299 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
3328 * 3300 *
3329 * Return: The updated value of @v. 3301 * Return: The updated value of @v.
3330 */ 3302 */
3331 static __always_inline long 3303 static __always_inline long
3332 atomic_long_add_return_relaxed(long i, atomic 3304 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
3333 { 3305 {
3334 instrument_atomic_read_write(v, sizeo 3306 instrument_atomic_read_write(v, sizeof(*v));
3335 return raw_atomic_long_add_return_rel 3307 return raw_atomic_long_add_return_relaxed(i, v);
3336 } 3308 }
3337 3309
3338 /** 3310 /**
3339 * atomic_long_fetch_add() - atomic add with 3311 * atomic_long_fetch_add() - atomic add with full ordering
3340 * @i: long value to add 3312 * @i: long value to add
3341 * @v: pointer to atomic_long_t 3313 * @v: pointer to atomic_long_t
3342 * 3314 *
3343 * Atomically updates @v to (@v + @i) with fu 3315 * Atomically updates @v to (@v + @i) with full ordering.
3344 * 3316 *
3345 * Unsafe to use in noinstr code; use raw_ato 3317 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
3346 * 3318 *
3347 * Return: The original value of @v. 3319 * Return: The original value of @v.
3348 */ 3320 */
3349 static __always_inline long 3321 static __always_inline long
3350 atomic_long_fetch_add(long i, atomic_long_t * 3322 atomic_long_fetch_add(long i, atomic_long_t *v)
3351 { 3323 {
3352 kcsan_mb(); 3324 kcsan_mb();
3353 instrument_atomic_read_write(v, sizeo 3325 instrument_atomic_read_write(v, sizeof(*v));
3354 return raw_atomic_long_fetch_add(i, v 3326 return raw_atomic_long_fetch_add(i, v);
3355 } 3327 }
3356 3328
3357 /** 3329 /**
3358 * atomic_long_fetch_add_acquire() - atomic a 3330 * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
3359 * @i: long value to add 3331 * @i: long value to add
3360 * @v: pointer to atomic_long_t 3332 * @v: pointer to atomic_long_t
3361 * 3333 *
3362 * Atomically updates @v to (@v + @i) with ac 3334 * Atomically updates @v to (@v + @i) with acquire ordering.
3363 * 3335 *
3364 * Unsafe to use in noinstr code; use raw_ato 3336 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
3365 * 3337 *
3366 * Return: The original value of @v. 3338 * Return: The original value of @v.
3367 */ 3339 */
3368 static __always_inline long 3340 static __always_inline long
3369 atomic_long_fetch_add_acquire(long i, atomic_ 3341 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
3370 { 3342 {
3371 instrument_atomic_read_write(v, sizeo 3343 instrument_atomic_read_write(v, sizeof(*v));
3372 return raw_atomic_long_fetch_add_acqu 3344 return raw_atomic_long_fetch_add_acquire(i, v);
3373 } 3345 }
3374 3346
3375 /** 3347 /**
3376 * atomic_long_fetch_add_release() - atomic a 3348 * atomic_long_fetch_add_release() - atomic add with release ordering
3377 * @i: long value to add 3349 * @i: long value to add
3378 * @v: pointer to atomic_long_t 3350 * @v: pointer to atomic_long_t
3379 * 3351 *
3380 * Atomically updates @v to (@v + @i) with re 3352 * Atomically updates @v to (@v + @i) with release ordering.
3381 * 3353 *
3382 * Unsafe to use in noinstr code; use raw_ato 3354 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
3383 * 3355 *
3384 * Return: The original value of @v. 3356 * Return: The original value of @v.
3385 */ 3357 */
3386 static __always_inline long 3358 static __always_inline long
3387 atomic_long_fetch_add_release(long i, atomic_ 3359 atomic_long_fetch_add_release(long i, atomic_long_t *v)
3388 { 3360 {
3389 kcsan_release(); 3361 kcsan_release();
3390 instrument_atomic_read_write(v, sizeo 3362 instrument_atomic_read_write(v, sizeof(*v));
3391 return raw_atomic_long_fetch_add_rele 3363 return raw_atomic_long_fetch_add_release(i, v);
3392 } 3364 }
3393 3365
3394 /** 3366 /**
3395 * atomic_long_fetch_add_relaxed() - atomic a 3367 * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
3396 * @i: long value to add 3368 * @i: long value to add
3397 * @v: pointer to atomic_long_t 3369 * @v: pointer to atomic_long_t
3398 * 3370 *
3399 * Atomically updates @v to (@v + @i) with re 3371 * Atomically updates @v to (@v + @i) with relaxed ordering.
3400 * 3372 *
3401 * Unsafe to use in noinstr code; use raw_ato 3373 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
3402 * 3374 *
3403 * Return: The original value of @v. 3375 * Return: The original value of @v.
3404 */ 3376 */
3405 static __always_inline long 3377 static __always_inline long
3406 atomic_long_fetch_add_relaxed(long i, atomic_ 3378 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
3407 { 3379 {
3408 instrument_atomic_read_write(v, sizeo 3380 instrument_atomic_read_write(v, sizeof(*v));
3409 return raw_atomic_long_fetch_add_rela 3381 return raw_atomic_long_fetch_add_relaxed(i, v);
3410 } 3382 }
3411 3383
3412 /** 3384 /**
3413 * atomic_long_sub() - atomic subtract with r 3385 * atomic_long_sub() - atomic subtract with relaxed ordering
3414 * @i: long value to subtract 3386 * @i: long value to subtract
3415 * @v: pointer to atomic_long_t 3387 * @v: pointer to atomic_long_t
3416 * 3388 *
3417 * Atomically updates @v to (@v - @i) with re 3389 * Atomically updates @v to (@v - @i) with relaxed ordering.
3418 * 3390 *
3419 * Unsafe to use in noinstr code; use raw_ato 3391 * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
3420 * 3392 *
3421 * Return: Nothing. 3393 * Return: Nothing.
3422 */ 3394 */
3423 static __always_inline void 3395 static __always_inline void
3424 atomic_long_sub(long i, atomic_long_t *v) 3396 atomic_long_sub(long i, atomic_long_t *v)
3425 { 3397 {
3426 instrument_atomic_read_write(v, sizeo 3398 instrument_atomic_read_write(v, sizeof(*v));
3427 raw_atomic_long_sub(i, v); 3399 raw_atomic_long_sub(i, v);
3428 } 3400 }
3429 3401
3430 /** 3402 /**
3431 * atomic_long_sub_return() - atomic subtract 3403 * atomic_long_sub_return() - atomic subtract with full ordering
3432 * @i: long value to subtract 3404 * @i: long value to subtract
3433 * @v: pointer to atomic_long_t 3405 * @v: pointer to atomic_long_t
3434 * 3406 *
3435 * Atomically updates @v to (@v - @i) with fu 3407 * Atomically updates @v to (@v - @i) with full ordering.
3436 * 3408 *
3437 * Unsafe to use in noinstr code; use raw_ato 3409 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
3438 * 3410 *
3439 * Return: The updated value of @v. 3411 * Return: The updated value of @v.
3440 */ 3412 */
3441 static __always_inline long 3413 static __always_inline long
3442 atomic_long_sub_return(long i, atomic_long_t 3414 atomic_long_sub_return(long i, atomic_long_t *v)
3443 { 3415 {
3444 kcsan_mb(); 3416 kcsan_mb();
3445 instrument_atomic_read_write(v, sizeo 3417 instrument_atomic_read_write(v, sizeof(*v));
3446 return raw_atomic_long_sub_return(i, 3418 return raw_atomic_long_sub_return(i, v);
3447 } 3419 }
3448 3420
3449 /** 3421 /**
3450 * atomic_long_sub_return_acquire() - atomic 3422 * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
3451 * @i: long value to subtract 3423 * @i: long value to subtract
3452 * @v: pointer to atomic_long_t 3424 * @v: pointer to atomic_long_t
3453 * 3425 *
3454 * Atomically updates @v to (@v - @i) with ac 3426 * Atomically updates @v to (@v - @i) with acquire ordering.
3455 * 3427 *
3456 * Unsafe to use in noinstr code; use raw_ato 3428 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
3457 * 3429 *
3458 * Return: The updated value of @v. 3430 * Return: The updated value of @v.
3459 */ 3431 */
3460 static __always_inline long 3432 static __always_inline long
3461 atomic_long_sub_return_acquire(long i, atomic 3433 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
3462 { 3434 {
3463 instrument_atomic_read_write(v, sizeo 3435 instrument_atomic_read_write(v, sizeof(*v));
3464 return raw_atomic_long_sub_return_acq 3436 return raw_atomic_long_sub_return_acquire(i, v);
3465 } 3437 }
3466 3438
3467 /** 3439 /**
3468 * atomic_long_sub_return_release() - atomic 3440 * atomic_long_sub_return_release() - atomic subtract with release ordering
3469 * @i: long value to subtract 3441 * @i: long value to subtract
3470 * @v: pointer to atomic_long_t 3442 * @v: pointer to atomic_long_t
3471 * 3443 *
3472 * Atomically updates @v to (@v - @i) with re 3444 * Atomically updates @v to (@v - @i) with release ordering.
3473 * 3445 *
3474 * Unsafe to use in noinstr code; use raw_ato 3446 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
3475 * 3447 *
3476 * Return: The updated value of @v. 3448 * Return: The updated value of @v.
3477 */ 3449 */
3478 static __always_inline long 3450 static __always_inline long
3479 atomic_long_sub_return_release(long i, atomic 3451 atomic_long_sub_return_release(long i, atomic_long_t *v)
3480 { 3452 {
3481 kcsan_release(); 3453 kcsan_release();
3482 instrument_atomic_read_write(v, sizeo 3454 instrument_atomic_read_write(v, sizeof(*v));
3483 return raw_atomic_long_sub_return_rel 3455 return raw_atomic_long_sub_return_release(i, v);
3484 } 3456 }
3485 3457
3486 /** 3458 /**
3487 * atomic_long_sub_return_relaxed() - atomic 3459 * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
3488 * @i: long value to subtract 3460 * @i: long value to subtract
3489 * @v: pointer to atomic_long_t 3461 * @v: pointer to atomic_long_t
3490 * 3462 *
3491 * Atomically updates @v to (@v - @i) with re 3463 * Atomically updates @v to (@v - @i) with relaxed ordering.
3492 * 3464 *
3493 * Unsafe to use in noinstr code; use raw_ato 3465 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
3494 * 3466 *
3495 * Return: The updated value of @v. 3467 * Return: The updated value of @v.
3496 */ 3468 */
3497 static __always_inline long 3469 static __always_inline long
3498 atomic_long_sub_return_relaxed(long i, atomic 3470 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
3499 { 3471 {
3500 instrument_atomic_read_write(v, sizeo 3472 instrument_atomic_read_write(v, sizeof(*v));
3501 return raw_atomic_long_sub_return_rel 3473 return raw_atomic_long_sub_return_relaxed(i, v);
3502 } 3474 }
3503 3475
3504 /** 3476 /**
3505 * atomic_long_fetch_sub() - atomic subtract 3477 * atomic_long_fetch_sub() - atomic subtract with full ordering
3506 * @i: long value to subtract 3478 * @i: long value to subtract
3507 * @v: pointer to atomic_long_t 3479 * @v: pointer to atomic_long_t
3508 * 3480 *
3509 * Atomically updates @v to (@v - @i) with fu 3481 * Atomically updates @v to (@v - @i) with full ordering.
3510 * 3482 *
3511 * Unsafe to use in noinstr code; use raw_ato 3483 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
3512 * 3484 *
3513 * Return: The original value of @v. 3485 * Return: The original value of @v.
3514 */ 3486 */
3515 static __always_inline long 3487 static __always_inline long
3516 atomic_long_fetch_sub(long i, atomic_long_t * 3488 atomic_long_fetch_sub(long i, atomic_long_t *v)
3517 { 3489 {
3518 kcsan_mb(); 3490 kcsan_mb();
3519 instrument_atomic_read_write(v, sizeo 3491 instrument_atomic_read_write(v, sizeof(*v));
3520 return raw_atomic_long_fetch_sub(i, v 3492 return raw_atomic_long_fetch_sub(i, v);
3521 } 3493 }
3522 3494
3523 /** 3495 /**
3524 * atomic_long_fetch_sub_acquire() - atomic s 3496 * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
3525 * @i: long value to subtract 3497 * @i: long value to subtract
3526 * @v: pointer to atomic_long_t 3498 * @v: pointer to atomic_long_t
3527 * 3499 *
3528 * Atomically updates @v to (@v - @i) with ac 3500 * Atomically updates @v to (@v - @i) with acquire ordering.
3529 * 3501 *
3530 * Unsafe to use in noinstr code; use raw_ato 3502 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
3531 * 3503 *
3532 * Return: The original value of @v. 3504 * Return: The original value of @v.
3533 */ 3505 */
3534 static __always_inline long 3506 static __always_inline long
3535 atomic_long_fetch_sub_acquire(long i, atomic_ 3507 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
3536 { 3508 {
3537 instrument_atomic_read_write(v, sizeo 3509 instrument_atomic_read_write(v, sizeof(*v));
3538 return raw_atomic_long_fetch_sub_acqu 3510 return raw_atomic_long_fetch_sub_acquire(i, v);
3539 } 3511 }
3540 3512
3541 /** 3513 /**
3542 * atomic_long_fetch_sub_release() - atomic s 3514 * atomic_long_fetch_sub_release() - atomic subtract with release ordering
3543 * @i: long value to subtract 3515 * @i: long value to subtract
3544 * @v: pointer to atomic_long_t 3516 * @v: pointer to atomic_long_t
3545 * 3517 *
3546 * Atomically updates @v to (@v - @i) with re 3518 * Atomically updates @v to (@v - @i) with release ordering.
3547 * 3519 *
3548 * Unsafe to use in noinstr code; use raw_ato 3520 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
3549 * 3521 *
3550 * Return: The original value of @v. 3522 * Return: The original value of @v.
3551 */ 3523 */
3552 static __always_inline long 3524 static __always_inline long
3553 atomic_long_fetch_sub_release(long i, atomic_ 3525 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
3554 { 3526 {
3555 kcsan_release(); 3527 kcsan_release();
3556 instrument_atomic_read_write(v, sizeo 3528 instrument_atomic_read_write(v, sizeof(*v));
3557 return raw_atomic_long_fetch_sub_rele 3529 return raw_atomic_long_fetch_sub_release(i, v);
3558 } 3530 }
3559 3531
3560 /** 3532 /**
3561 * atomic_long_fetch_sub_relaxed() - atomic s 3533 * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3562 * @i: long value to subtract 3534 * @i: long value to subtract
3563 * @v: pointer to atomic_long_t 3535 * @v: pointer to atomic_long_t
3564 * 3536 *
3565 * Atomically updates @v to (@v - @i) with re 3537 * Atomically updates @v to (@v - @i) with relaxed ordering.
3566 * 3538 *
3567 * Unsafe to use in noinstr code; use raw_ato 3539 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
3568 * 3540 *
3569 * Return: The original value of @v. 3541 * Return: The original value of @v.
3570 */ 3542 */
3571 static __always_inline long 3543 static __always_inline long
3572 atomic_long_fetch_sub_relaxed(long i, atomic_ 3544 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
3573 { 3545 {
3574 instrument_atomic_read_write(v, sizeo 3546 instrument_atomic_read_write(v, sizeof(*v));
3575 return raw_atomic_long_fetch_sub_rela 3547 return raw_atomic_long_fetch_sub_relaxed(i, v);
3576 } 3548 }
3577 3549
3578 /** 3550 /**
3579 * atomic_long_inc() - atomic increment with 3551 * atomic_long_inc() - atomic increment with relaxed ordering
3580 * @v: pointer to atomic_long_t 3552 * @v: pointer to atomic_long_t
3581 * 3553 *
3582 * Atomically updates @v to (@v + 1) with rel 3554 * Atomically updates @v to (@v + 1) with relaxed ordering.
3583 * 3555 *
3584 * Unsafe to use in noinstr code; use raw_ato 3556 * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
3585 * 3557 *
3586 * Return: Nothing. 3558 * Return: Nothing.
3587 */ 3559 */
3588 static __always_inline void 3560 static __always_inline void
3589 atomic_long_inc(atomic_long_t *v) 3561 atomic_long_inc(atomic_long_t *v)
3590 { 3562 {
3591 instrument_atomic_read_write(v, sizeo 3563 instrument_atomic_read_write(v, sizeof(*v));
3592 raw_atomic_long_inc(v); 3564 raw_atomic_long_inc(v);
3593 } 3565 }
3594 3566
3595 /** 3567 /**
3596 * atomic_long_inc_return() - atomic incremen 3568 * atomic_long_inc_return() - atomic increment with full ordering
3597 * @v: pointer to atomic_long_t 3569 * @v: pointer to atomic_long_t
3598 * 3570 *
3599 * Atomically updates @v to (@v + 1) with ful 3571 * Atomically updates @v to (@v + 1) with full ordering.
3600 * 3572 *
3601 * Unsafe to use in noinstr code; use raw_ato 3573 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
3602 * 3574 *
3603 * Return: The updated value of @v. 3575 * Return: The updated value of @v.
3604 */ 3576 */
3605 static __always_inline long 3577 static __always_inline long
3606 atomic_long_inc_return(atomic_long_t *v) 3578 atomic_long_inc_return(atomic_long_t *v)
3607 { 3579 {
3608 kcsan_mb(); 3580 kcsan_mb();
3609 instrument_atomic_read_write(v, sizeo 3581 instrument_atomic_read_write(v, sizeof(*v));
3610 return raw_atomic_long_inc_return(v); 3582 return raw_atomic_long_inc_return(v);
3611 } 3583 }
3612 3584
3613 /** 3585 /**
3614 * atomic_long_inc_return_acquire() - atomic 3586 * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
3615 * @v: pointer to atomic_long_t 3587 * @v: pointer to atomic_long_t
3616 * 3588 *
3617 * Atomically updates @v to (@v + 1) with acq 3589 * Atomically updates @v to (@v + 1) with acquire ordering.
3618 * 3590 *
3619 * Unsafe to use in noinstr code; use raw_ato 3591 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
3620 * 3592 *
3621 * Return: The updated value of @v. 3593 * Return: The updated value of @v.
3622 */ 3594 */
3623 static __always_inline long 3595 static __always_inline long
3624 atomic_long_inc_return_acquire(atomic_long_t 3596 atomic_long_inc_return_acquire(atomic_long_t *v)
3625 { 3597 {
3626 instrument_atomic_read_write(v, sizeo 3598 instrument_atomic_read_write(v, sizeof(*v));
3627 return raw_atomic_long_inc_return_acq 3599 return raw_atomic_long_inc_return_acquire(v);
3628 } 3600 }
3629 3601
3630 /** 3602 /**
3631 * atomic_long_inc_return_release() - atomic 3603 * atomic_long_inc_return_release() - atomic increment with release ordering
3632 * @v: pointer to atomic_long_t 3604 * @v: pointer to atomic_long_t
3633 * 3605 *
3634 * Atomically updates @v to (@v + 1) with rel 3606 * Atomically updates @v to (@v + 1) with release ordering.
3635 * 3607 *
3636 * Unsafe to use in noinstr code; use raw_ato 3608 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
3637 * 3609 *
3638 * Return: The updated value of @v. 3610 * Return: The updated value of @v.
3639 */ 3611 */
3640 static __always_inline long 3612 static __always_inline long
3641 atomic_long_inc_return_release(atomic_long_t 3613 atomic_long_inc_return_release(atomic_long_t *v)
3642 { 3614 {
3643 kcsan_release(); 3615 kcsan_release();
3644 instrument_atomic_read_write(v, sizeo 3616 instrument_atomic_read_write(v, sizeof(*v));
3645 return raw_atomic_long_inc_return_rel 3617 return raw_atomic_long_inc_return_release(v);
3646 } 3618 }
3647 3619
3648 /** 3620 /**
3649 * atomic_long_inc_return_relaxed() - atomic 3621 * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
3650 * @v: pointer to atomic_long_t 3622 * @v: pointer to atomic_long_t
3651 * 3623 *
3652 * Atomically updates @v to (@v + 1) with rel 3624 * Atomically updates @v to (@v + 1) with relaxed ordering.
3653 * 3625 *
3654 * Unsafe to use in noinstr code; use raw_ato 3626 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
3655 * 3627 *
3656 * Return: The updated value of @v. 3628 * Return: The updated value of @v.
3657 */ 3629 */
3658 static __always_inline long 3630 static __always_inline long
3659 atomic_long_inc_return_relaxed(atomic_long_t 3631 atomic_long_inc_return_relaxed(atomic_long_t *v)
3660 { 3632 {
3661 instrument_atomic_read_write(v, sizeo 3633 instrument_atomic_read_write(v, sizeof(*v));
3662 return raw_atomic_long_inc_return_rel 3634 return raw_atomic_long_inc_return_relaxed(v);
3663 } 3635 }
3664 3636
3665 /** 3637 /**
3666 * atomic_long_fetch_inc() - atomic increment 3638 * atomic_long_fetch_inc() - atomic increment with full ordering
3667 * @v: pointer to atomic_long_t 3639 * @v: pointer to atomic_long_t
3668 * 3640 *
3669 * Atomically updates @v to (@v + 1) with ful 3641 * Atomically updates @v to (@v + 1) with full ordering.
3670 * 3642 *
3671 * Unsafe to use in noinstr code; use raw_ato 3643 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
3672 * 3644 *
3673 * Return: The original value of @v. 3645 * Return: The original value of @v.
3674 */ 3646 */
3675 static __always_inline long 3647 static __always_inline long
3676 atomic_long_fetch_inc(atomic_long_t *v) 3648 atomic_long_fetch_inc(atomic_long_t *v)
3677 { 3649 {
3678 kcsan_mb(); 3650 kcsan_mb();
3679 instrument_atomic_read_write(v, sizeo 3651 instrument_atomic_read_write(v, sizeof(*v));
3680 return raw_atomic_long_fetch_inc(v); 3652 return raw_atomic_long_fetch_inc(v);
3681 } 3653 }
3682 3654
3683 /** 3655 /**
3684 * atomic_long_fetch_inc_acquire() - atomic i 3656 * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
3685 * @v: pointer to atomic_long_t 3657 * @v: pointer to atomic_long_t
3686 * 3658 *
3687 * Atomically updates @v to (@v + 1) with acq 3659 * Atomically updates @v to (@v + 1) with acquire ordering.
3688 * 3660 *
3689 * Unsafe to use in noinstr code; use raw_ato 3661 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
3690 * 3662 *
3691 * Return: The original value of @v. 3663 * Return: The original value of @v.
3692 */ 3664 */
3693 static __always_inline long 3665 static __always_inline long
3694 atomic_long_fetch_inc_acquire(atomic_long_t * 3666 atomic_long_fetch_inc_acquire(atomic_long_t *v)
3695 { 3667 {
3696 instrument_atomic_read_write(v, sizeo 3668 instrument_atomic_read_write(v, sizeof(*v));
3697 return raw_atomic_long_fetch_inc_acqu 3669 return raw_atomic_long_fetch_inc_acquire(v);
3698 } 3670 }
3699 3671
3700 /** 3672 /**
3701 * atomic_long_fetch_inc_release() - atomic i 3673 * atomic_long_fetch_inc_release() - atomic increment with release ordering
3702 * @v: pointer to atomic_long_t 3674 * @v: pointer to atomic_long_t
3703 * 3675 *
3704 * Atomically updates @v to (@v + 1) with rel 3676 * Atomically updates @v to (@v + 1) with release ordering.
3705 * 3677 *
3706 * Unsafe to use in noinstr code; use raw_ato 3678 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
3707 * 3679 *
3708 * Return: The original value of @v. 3680 * Return: The original value of @v.
3709 */ 3681 */
3710 static __always_inline long 3682 static __always_inline long
3711 atomic_long_fetch_inc_release(atomic_long_t * 3683 atomic_long_fetch_inc_release(atomic_long_t *v)
3712 { 3684 {
3713 kcsan_release(); 3685 kcsan_release();
3714 instrument_atomic_read_write(v, sizeo 3686 instrument_atomic_read_write(v, sizeof(*v));
3715 return raw_atomic_long_fetch_inc_rele 3687 return raw_atomic_long_fetch_inc_release(v);
3716 } 3688 }
3717 3689
3718 /** 3690 /**
3719 * atomic_long_fetch_inc_relaxed() - atomic i 3691 * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
3720 * @v: pointer to atomic_long_t 3692 * @v: pointer to atomic_long_t
3721 * 3693 *
3722 * Atomically updates @v to (@v + 1) with rel 3694 * Atomically updates @v to (@v + 1) with relaxed ordering.
3723 * 3695 *
3724 * Unsafe to use in noinstr code; use raw_ato 3696 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
3725 * 3697 *
3726 * Return: The original value of @v. 3698 * Return: The original value of @v.
3727 */ 3699 */
3728 static __always_inline long 3700 static __always_inline long
3729 atomic_long_fetch_inc_relaxed(atomic_long_t * 3701 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
3730 { 3702 {
3731 instrument_atomic_read_write(v, sizeo 3703 instrument_atomic_read_write(v, sizeof(*v));
3732 return raw_atomic_long_fetch_inc_rela 3704 return raw_atomic_long_fetch_inc_relaxed(v);
3733 } 3705 }
3734 3706
3735 /** 3707 /**
3736 * atomic_long_dec() - atomic decrement with 3708 * atomic_long_dec() - atomic decrement with relaxed ordering
3737 * @v: pointer to atomic_long_t 3709 * @v: pointer to atomic_long_t
3738 * 3710 *
3739 * Atomically updates @v to (@v - 1) with rel 3711 * Atomically updates @v to (@v - 1) with relaxed ordering.
3740 * 3712 *
3741 * Unsafe to use in noinstr code; use raw_ato 3713 * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
3742 * 3714 *
3743 * Return: Nothing. 3715 * Return: Nothing.
3744 */ 3716 */
3745 static __always_inline void 3717 static __always_inline void
3746 atomic_long_dec(atomic_long_t *v) 3718 atomic_long_dec(atomic_long_t *v)
3747 { 3719 {
3748 instrument_atomic_read_write(v, sizeo 3720 instrument_atomic_read_write(v, sizeof(*v));
3749 raw_atomic_long_dec(v); 3721 raw_atomic_long_dec(v);
3750 } 3722 }
3751 3723
3752 /** 3724 /**
3753 * atomic_long_dec_return() - atomic decremen 3725 * atomic_long_dec_return() - atomic decrement with full ordering
3754 * @v: pointer to atomic_long_t 3726 * @v: pointer to atomic_long_t
3755 * 3727 *
3756 * Atomically updates @v to (@v - 1) with ful 3728 * Atomically updates @v to (@v - 1) with full ordering.
3757 * 3729 *
3758 * Unsafe to use in noinstr code; use raw_ato 3730 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
3759 * 3731 *
3760 * Return: The updated value of @v. 3732 * Return: The updated value of @v.
3761 */ 3733 */
3762 static __always_inline long 3734 static __always_inline long
3763 atomic_long_dec_return(atomic_long_t *v) 3735 atomic_long_dec_return(atomic_long_t *v)
3764 { 3736 {
3765 kcsan_mb(); 3737 kcsan_mb();
3766 instrument_atomic_read_write(v, sizeo 3738 instrument_atomic_read_write(v, sizeof(*v));
3767 return raw_atomic_long_dec_return(v); 3739 return raw_atomic_long_dec_return(v);
3768 } 3740 }
3769 3741
3770 /** 3742 /**
3771 * atomic_long_dec_return_acquire() - atomic 3743 * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
3772 * @v: pointer to atomic_long_t 3744 * @v: pointer to atomic_long_t
3773 * 3745 *
3774 * Atomically updates @v to (@v - 1) with acq 3746 * Atomically updates @v to (@v - 1) with acquire ordering.
3775 * 3747 *
3776 * Unsafe to use in noinstr code; use raw_ato 3748 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
3777 * 3749 *
3778 * Return: The updated value of @v. 3750 * Return: The updated value of @v.
3779 */ 3751 */
3780 static __always_inline long 3752 static __always_inline long
3781 atomic_long_dec_return_acquire(atomic_long_t 3753 atomic_long_dec_return_acquire(atomic_long_t *v)
3782 { 3754 {
3783 instrument_atomic_read_write(v, sizeo 3755 instrument_atomic_read_write(v, sizeof(*v));
3784 return raw_atomic_long_dec_return_acq 3756 return raw_atomic_long_dec_return_acquire(v);
3785 } 3757 }
3786 3758
3787 /** 3759 /**
3788 * atomic_long_dec_return_release() - atomic 3760 * atomic_long_dec_return_release() - atomic decrement with release ordering
3789 * @v: pointer to atomic_long_t 3761 * @v: pointer to atomic_long_t
3790 * 3762 *
3791 * Atomically updates @v to (@v - 1) with rel 3763 * Atomically updates @v to (@v - 1) with release ordering.
3792 * 3764 *
3793 * Unsafe to use in noinstr code; use raw_ato 3765 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
3794 * 3766 *
3795 * Return: The updated value of @v. 3767 * Return: The updated value of @v.
3796 */ 3768 */
3797 static __always_inline long 3769 static __always_inline long
3798 atomic_long_dec_return_release(atomic_long_t 3770 atomic_long_dec_return_release(atomic_long_t *v)
3799 { 3771 {
3800 kcsan_release(); 3772 kcsan_release();
3801 instrument_atomic_read_write(v, sizeo 3773 instrument_atomic_read_write(v, sizeof(*v));
3802 return raw_atomic_long_dec_return_rel 3774 return raw_atomic_long_dec_return_release(v);
3803 } 3775 }
3804 3776
3805 /** 3777 /**
3806 * atomic_long_dec_return_relaxed() - atomic 3778 * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
3807 * @v: pointer to atomic_long_t 3779 * @v: pointer to atomic_long_t
3808 * 3780 *
3809 * Atomically updates @v to (@v - 1) with rel 3781 * Atomically updates @v to (@v - 1) with relaxed ordering.
3810 * 3782 *
3811 * Unsafe to use in noinstr code; use raw_ato 3783 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
3812 * 3784 *
3813 * Return: The updated value of @v. 3785 * Return: The updated value of @v.
3814 */ 3786 */
3815 static __always_inline long 3787 static __always_inline long
3816 atomic_long_dec_return_relaxed(atomic_long_t 3788 atomic_long_dec_return_relaxed(atomic_long_t *v)
3817 { 3789 {
3818 instrument_atomic_read_write(v, sizeo 3790 instrument_atomic_read_write(v, sizeof(*v));
3819 return raw_atomic_long_dec_return_rel 3791 return raw_atomic_long_dec_return_relaxed(v);
3820 } 3792 }
3821 3793
3822 /** 3794 /**
3823 * atomic_long_fetch_dec() - atomic decrement 3795 * atomic_long_fetch_dec() - atomic decrement with full ordering
3824 * @v: pointer to atomic_long_t 3796 * @v: pointer to atomic_long_t
3825 * 3797 *
3826 * Atomically updates @v to (@v - 1) with ful 3798 * Atomically updates @v to (@v - 1) with full ordering.
3827 * 3799 *
3828 * Unsafe to use in noinstr code; use raw_ato 3800 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
3829 * 3801 *
3830 * Return: The original value of @v. 3802 * Return: The original value of @v.
3831 */ 3803 */
3832 static __always_inline long 3804 static __always_inline long
3833 atomic_long_fetch_dec(atomic_long_t *v) 3805 atomic_long_fetch_dec(atomic_long_t *v)
3834 { 3806 {
3835 kcsan_mb(); 3807 kcsan_mb();
3836 instrument_atomic_read_write(v, sizeo 3808 instrument_atomic_read_write(v, sizeof(*v));
3837 return raw_atomic_long_fetch_dec(v); 3809 return raw_atomic_long_fetch_dec(v);
3838 } 3810 }
3839 3811
3840 /** 3812 /**
3841 * atomic_long_fetch_dec_acquire() - atomic d 3813 * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
3842 * @v: pointer to atomic_long_t 3814 * @v: pointer to atomic_long_t
3843 * 3815 *
3844 * Atomically updates @v to (@v - 1) with acq 3816 * Atomically updates @v to (@v - 1) with acquire ordering.
3845 * 3817 *
3846 * Unsafe to use in noinstr code; use raw_ato 3818 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
3847 * 3819 *
3848 * Return: The original value of @v. 3820 * Return: The original value of @v.
3849 */ 3821 */
3850 static __always_inline long 3822 static __always_inline long
3851 atomic_long_fetch_dec_acquire(atomic_long_t * 3823 atomic_long_fetch_dec_acquire(atomic_long_t *v)
3852 { 3824 {
3853 instrument_atomic_read_write(v, sizeo 3825 instrument_atomic_read_write(v, sizeof(*v));
3854 return raw_atomic_long_fetch_dec_acqu 3826 return raw_atomic_long_fetch_dec_acquire(v);
3855 } 3827 }
3856 3828
3857 /** 3829 /**
3858 * atomic_long_fetch_dec_release() - atomic d 3830 * atomic_long_fetch_dec_release() - atomic decrement with release ordering
3859 * @v: pointer to atomic_long_t 3831 * @v: pointer to atomic_long_t
3860 * 3832 *
3861 * Atomically updates @v to (@v - 1) with rel 3833 * Atomically updates @v to (@v - 1) with release ordering.
3862 * 3834 *
3863 * Unsafe to use in noinstr code; use raw_ato 3835 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
3864 * 3836 *
3865 * Return: The original value of @v. 3837 * Return: The original value of @v.
3866 */ 3838 */
3867 static __always_inline long 3839 static __always_inline long
3868 atomic_long_fetch_dec_release(atomic_long_t * 3840 atomic_long_fetch_dec_release(atomic_long_t *v)
3869 { 3841 {
3870 kcsan_release(); 3842 kcsan_release();
3871 instrument_atomic_read_write(v, sizeo 3843 instrument_atomic_read_write(v, sizeof(*v));
3872 return raw_atomic_long_fetch_dec_rele 3844 return raw_atomic_long_fetch_dec_release(v);
3873 } 3845 }
3874 3846
3875 /** 3847 /**
3876 * atomic_long_fetch_dec_relaxed() - atomic d 3848 * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3877 * @v: pointer to atomic_long_t 3849 * @v: pointer to atomic_long_t
3878 * 3850 *
3879 * Atomically updates @v to (@v - 1) with rel 3851 * Atomically updates @v to (@v - 1) with relaxed ordering.
3880 * 3852 *
3881 * Unsafe to use in noinstr code; use raw_ato 3853 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
3882 * 3854 *
3883 * Return: The original value of @v. 3855 * Return: The original value of @v.
3884 */ 3856 */
3885 static __always_inline long 3857 static __always_inline long
3886 atomic_long_fetch_dec_relaxed(atomic_long_t * 3858 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
3887 { 3859 {
3888 instrument_atomic_read_write(v, sizeo 3860 instrument_atomic_read_write(v, sizeof(*v));
3889 return raw_atomic_long_fetch_dec_rela 3861 return raw_atomic_long_fetch_dec_relaxed(v);
3890 } 3862 }
3891 3863
3892 /** 3864 /**
3893 * atomic_long_and() - atomic bitwise AND wit 3865 * atomic_long_and() - atomic bitwise AND with relaxed ordering
3894 * @i: long value 3866 * @i: long value
3895 * @v: pointer to atomic_long_t 3867 * @v: pointer to atomic_long_t
3896 * 3868 *
3897 * Atomically updates @v to (@v & @i) with re 3869 * Atomically updates @v to (@v & @i) with relaxed ordering.
3898 * 3870 *
3899 * Unsafe to use in noinstr code; use raw_ato 3871 * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
3900 * 3872 *
3901 * Return: Nothing. 3873 * Return: Nothing.
3902 */ 3874 */
3903 static __always_inline void 3875 static __always_inline void
3904 atomic_long_and(long i, atomic_long_t *v) 3876 atomic_long_and(long i, atomic_long_t *v)
3905 { 3877 {
3906 instrument_atomic_read_write(v, sizeo 3878 instrument_atomic_read_write(v, sizeof(*v));
3907 raw_atomic_long_and(i, v); 3879 raw_atomic_long_and(i, v);
3908 } 3880 }
3909 3881
3910 /** 3882 /**
3911 * atomic_long_fetch_and() - atomic bitwise A 3883 * atomic_long_fetch_and() - atomic bitwise AND with full ordering
3912 * @i: long value 3884 * @i: long value
3913 * @v: pointer to atomic_long_t 3885 * @v: pointer to atomic_long_t
3914 * 3886 *
3915 * Atomically updates @v to (@v & @i) with fu 3887 * Atomically updates @v to (@v & @i) with full ordering.
3916 * 3888 *
3917 * Unsafe to use in noinstr code; use raw_ato 3889 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
3918 * 3890 *
3919 * Return: The original value of @v. 3891 * Return: The original value of @v.
3920 */ 3892 */
3921 static __always_inline long 3893 static __always_inline long
3922 atomic_long_fetch_and(long i, atomic_long_t * 3894 atomic_long_fetch_and(long i, atomic_long_t *v)
3923 { 3895 {
3924 kcsan_mb(); 3896 kcsan_mb();
3925 instrument_atomic_read_write(v, sizeo 3897 instrument_atomic_read_write(v, sizeof(*v));
3926 return raw_atomic_long_fetch_and(i, v 3898 return raw_atomic_long_fetch_and(i, v);
3927 } 3899 }
3928 3900
3929 /** 3901 /**
3930 * atomic_long_fetch_and_acquire() - atomic b 3902 * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3931 * @i: long value 3903 * @i: long value
3932 * @v: pointer to atomic_long_t 3904 * @v: pointer to atomic_long_t
3933 * 3905 *
3934 * Atomically updates @v to (@v & @i) with ac 3906 * Atomically updates @v to (@v & @i) with acquire ordering.
3935 * 3907 *
3936 * Unsafe to use in noinstr code; use raw_ato 3908 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
3937 * 3909 *
3938 * Return: The original value of @v. 3910 * Return: The original value of @v.
3939 */ 3911 */
3940 static __always_inline long 3912 static __always_inline long
3941 atomic_long_fetch_and_acquire(long i, atomic_ 3913 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
3942 { 3914 {
3943 instrument_atomic_read_write(v, sizeo 3915 instrument_atomic_read_write(v, sizeof(*v));
3944 return raw_atomic_long_fetch_and_acqu 3916 return raw_atomic_long_fetch_and_acquire(i, v);
3945 } 3917 }
3946 3918
3947 /** 3919 /**
3948 * atomic_long_fetch_and_release() - atomic b 3920 * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
3949 * @i: long value 3921 * @i: long value
3950 * @v: pointer to atomic_long_t 3922 * @v: pointer to atomic_long_t
3951 * 3923 *
3952 * Atomically updates @v to (@v & @i) with re 3924 * Atomically updates @v to (@v & @i) with release ordering.
3953 * 3925 *
3954 * Unsafe to use in noinstr code; use raw_ato 3926 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
3955 * 3927 *
3956 * Return: The original value of @v. 3928 * Return: The original value of @v.
3957 */ 3929 */
3958 static __always_inline long 3930 static __always_inline long
3959 atomic_long_fetch_and_release(long i, atomic_ 3931 atomic_long_fetch_and_release(long i, atomic_long_t *v)
3960 { 3932 {
3961 kcsan_release(); 3933 kcsan_release();
3962 instrument_atomic_read_write(v, sizeo 3934 instrument_atomic_read_write(v, sizeof(*v));
3963 return raw_atomic_long_fetch_and_rele 3935 return raw_atomic_long_fetch_and_release(i, v);
3964 } 3936 }
3965 3937
3966 /** 3938 /**
3967 * atomic_long_fetch_and_relaxed() - atomic b 3939 * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3968 * @i: long value 3940 * @i: long value
3969 * @v: pointer to atomic_long_t 3941 * @v: pointer to atomic_long_t
3970 * 3942 *
3971 * Atomically updates @v to (@v & @i) with re 3943 * Atomically updates @v to (@v & @i) with relaxed ordering.
3972 * 3944 *
3973 * Unsafe to use in noinstr code; use raw_ato 3945 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
3974 * 3946 *
3975 * Return: The original value of @v. 3947 * Return: The original value of @v.
3976 */ 3948 */
3977 static __always_inline long 3949 static __always_inline long
3978 atomic_long_fetch_and_relaxed(long i, atomic_ 3950 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
3979 { 3951 {
3980 instrument_atomic_read_write(v, sizeo 3952 instrument_atomic_read_write(v, sizeof(*v));
3981 return raw_atomic_long_fetch_and_rela 3953 return raw_atomic_long_fetch_and_relaxed(i, v);
3982 } 3954 }
3983 3955
3984 /** 3956 /**
3985 * atomic_long_andnot() - atomic bitwise AND 3957 * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
3986 * @i: long value 3958 * @i: long value
3987 * @v: pointer to atomic_long_t 3959 * @v: pointer to atomic_long_t
3988 * 3960 *
3989 * Atomically updates @v to (@v & ~@i) with r 3961 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3990 * 3962 *
3991 * Unsafe to use in noinstr code; use raw_ato 3963 * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
3992 * 3964 *
3993 * Return: Nothing. 3965 * Return: Nothing.
3994 */ 3966 */
3995 static __always_inline void 3967 static __always_inline void
3996 atomic_long_andnot(long i, atomic_long_t *v) 3968 atomic_long_andnot(long i, atomic_long_t *v)
3997 { 3969 {
3998 instrument_atomic_read_write(v, sizeo 3970 instrument_atomic_read_write(v, sizeof(*v));
3999 raw_atomic_long_andnot(i, v); 3971 raw_atomic_long_andnot(i, v);
4000 } 3972 }
4001 3973
4002 /** 3974 /**
4003 * atomic_long_fetch_andnot() - atomic bitwis 3975 * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
4004 * @i: long value 3976 * @i: long value
4005 * @v: pointer to atomic_long_t 3977 * @v: pointer to atomic_long_t
4006 * 3978 *
4007 * Atomically updates @v to (@v & ~@i) with f 3979 * Atomically updates @v to (@v & ~@i) with full ordering.
4008 * 3980 *
4009 * Unsafe to use in noinstr code; use raw_ato 3981 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
4010 * 3982 *
4011 * Return: The original value of @v. 3983 * Return: The original value of @v.
4012 */ 3984 */
4013 static __always_inline long 3985 static __always_inline long
4014 atomic_long_fetch_andnot(long i, atomic_long_ 3986 atomic_long_fetch_andnot(long i, atomic_long_t *v)
4015 { 3987 {
4016 kcsan_mb(); 3988 kcsan_mb();
4017 instrument_atomic_read_write(v, sizeo 3989 instrument_atomic_read_write(v, sizeof(*v));
4018 return raw_atomic_long_fetch_andnot(i 3990 return raw_atomic_long_fetch_andnot(i, v);
4019 } 3991 }
4020 3992
4021 /** 3993 /**
4022 * atomic_long_fetch_andnot_acquire() - atomi 3994 * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
4023 * @i: long value 3995 * @i: long value
4024 * @v: pointer to atomic_long_t 3996 * @v: pointer to atomic_long_t
4025 * 3997 *
4026 * Atomically updates @v to (@v & ~@i) with a 3998 * Atomically updates @v to (@v & ~@i) with acquire ordering.
4027 * 3999 *
4028 * Unsafe to use in noinstr code; use raw_ato 4000 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
4029 * 4001 *
4030 * Return: The original value of @v. 4002 * Return: The original value of @v.
4031 */ 4003 */
4032 static __always_inline long 4004 static __always_inline long
4033 atomic_long_fetch_andnot_acquire(long i, atom 4005 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
4034 { 4006 {
4035 instrument_atomic_read_write(v, sizeo 4007 instrument_atomic_read_write(v, sizeof(*v));
4036 return raw_atomic_long_fetch_andnot_a 4008 return raw_atomic_long_fetch_andnot_acquire(i, v);
4037 } 4009 }
4038 4010
4039 /** 4011 /**
4040 * atomic_long_fetch_andnot_release() - atomi 4012 * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
4041 * @i: long value 4013 * @i: long value
4042 * @v: pointer to atomic_long_t 4014 * @v: pointer to atomic_long_t
4043 * 4015 *
4044 * Atomically updates @v to (@v & ~@i) with r 4016 * Atomically updates @v to (@v & ~@i) with release ordering.
4045 * 4017 *
4046 * Unsafe to use in noinstr code; use raw_ato 4018 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
4047 * 4019 *
4048 * Return: The original value of @v. 4020 * Return: The original value of @v.
4049 */ 4021 */
4050 static __always_inline long 4022 static __always_inline long
4051 atomic_long_fetch_andnot_release(long i, atom 4023 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
4052 { 4024 {
4053 kcsan_release(); 4025 kcsan_release();
4054 instrument_atomic_read_write(v, sizeo 4026 instrument_atomic_read_write(v, sizeof(*v));
4055 return raw_atomic_long_fetch_andnot_r 4027 return raw_atomic_long_fetch_andnot_release(i, v);
4056 } 4028 }
4057 4029
4058 /** 4030 /**
4059 * atomic_long_fetch_andnot_relaxed() - atomi 4031 * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
4060 * @i: long value 4032 * @i: long value
4061 * @v: pointer to atomic_long_t 4033 * @v: pointer to atomic_long_t
4062 * 4034 *
4063 * Atomically updates @v to (@v & ~@i) with r 4035 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
4064 * 4036 *
4065 * Unsafe to use in noinstr code; use raw_ato 4037 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
4066 * 4038 *
4067 * Return: The original value of @v. 4039 * Return: The original value of @v.
4068 */ 4040 */
4069 static __always_inline long 4041 static __always_inline long
4070 atomic_long_fetch_andnot_relaxed(long i, atom 4042 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
4071 { 4043 {
4072 instrument_atomic_read_write(v, sizeo 4044 instrument_atomic_read_write(v, sizeof(*v));
4073 return raw_atomic_long_fetch_andnot_r 4045 return raw_atomic_long_fetch_andnot_relaxed(i, v);
4074 } 4046 }
4075 4047
4076 /** 4048 /**
4077 * atomic_long_or() - atomic bitwise OR with 4049 * atomic_long_or() - atomic bitwise OR with relaxed ordering
4078 * @i: long value 4050 * @i: long value
4079 * @v: pointer to atomic_long_t 4051 * @v: pointer to atomic_long_t
4080 * 4052 *
4081 * Atomically updates @v to (@v | @i) with re 4053 * Atomically updates @v to (@v | @i) with relaxed ordering.
4082 * 4054 *
4083 * Unsafe to use in noinstr code; use raw_ato 4055 * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
4084 * 4056 *
4085 * Return: Nothing. 4057 * Return: Nothing.
4086 */ 4058 */
4087 static __always_inline void 4059 static __always_inline void
4088 atomic_long_or(long i, atomic_long_t *v) 4060 atomic_long_or(long i, atomic_long_t *v)
4089 { 4061 {
4090 instrument_atomic_read_write(v, sizeo 4062 instrument_atomic_read_write(v, sizeof(*v));
4091 raw_atomic_long_or(i, v); 4063 raw_atomic_long_or(i, v);
4092 } 4064 }
4093 4065
4094 /** 4066 /**
4095 * atomic_long_fetch_or() - atomic bitwise OR 4067 * atomic_long_fetch_or() - atomic bitwise OR with full ordering
4096 * @i: long value 4068 * @i: long value
4097 * @v: pointer to atomic_long_t 4069 * @v: pointer to atomic_long_t
4098 * 4070 *
4099 * Atomically updates @v to (@v | @i) with fu 4071 * Atomically updates @v to (@v | @i) with full ordering.
4100 * 4072 *
4101 * Unsafe to use in noinstr code; use raw_ato 4073 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
4102 * 4074 *
4103 * Return: The original value of @v. 4075 * Return: The original value of @v.
4104 */ 4076 */
4105 static __always_inline long 4077 static __always_inline long
4106 atomic_long_fetch_or(long i, atomic_long_t *v 4078 atomic_long_fetch_or(long i, atomic_long_t *v)
4107 { 4079 {
4108 kcsan_mb(); 4080 kcsan_mb();
4109 instrument_atomic_read_write(v, sizeo 4081 instrument_atomic_read_write(v, sizeof(*v));
4110 return raw_atomic_long_fetch_or(i, v) 4082 return raw_atomic_long_fetch_or(i, v);
4111 } 4083 }
4112 4084
4113 /** 4085 /**
4114 * atomic_long_fetch_or_acquire() - atomic bi 4086 * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
4115 * @i: long value 4087 * @i: long value
4116 * @v: pointer to atomic_long_t 4088 * @v: pointer to atomic_long_t
4117 * 4089 *
4118 * Atomically updates @v to (@v | @i) with ac 4090 * Atomically updates @v to (@v | @i) with acquire ordering.
4119 * 4091 *
4120 * Unsafe to use in noinstr code; use raw_ato 4092 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
4121 * 4093 *
4122 * Return: The original value of @v. 4094 * Return: The original value of @v.
4123 */ 4095 */
4124 static __always_inline long 4096 static __always_inline long
4125 atomic_long_fetch_or_acquire(long i, atomic_l 4097 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
4126 { 4098 {
4127 instrument_atomic_read_write(v, sizeo 4099 instrument_atomic_read_write(v, sizeof(*v));
4128 return raw_atomic_long_fetch_or_acqui 4100 return raw_atomic_long_fetch_or_acquire(i, v);
4129 } 4101 }
4130 4102
4131 /** 4103 /**
4132 * atomic_long_fetch_or_release() - atomic bi 4104 * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
4133 * @i: long value 4105 * @i: long value
4134 * @v: pointer to atomic_long_t 4106 * @v: pointer to atomic_long_t
4135 * 4107 *
4136 * Atomically updates @v to (@v | @i) with re 4108 * Atomically updates @v to (@v | @i) with release ordering.
4137 * 4109 *
4138 * Unsafe to use in noinstr code; use raw_ato 4110 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
4139 * 4111 *
4140 * Return: The original value of @v. 4112 * Return: The original value of @v.
4141 */ 4113 */
4142 static __always_inline long 4114 static __always_inline long
4143 atomic_long_fetch_or_release(long i, atomic_l 4115 atomic_long_fetch_or_release(long i, atomic_long_t *v)
4144 { 4116 {
4145 kcsan_release(); 4117 kcsan_release();
4146 instrument_atomic_read_write(v, sizeo 4118 instrument_atomic_read_write(v, sizeof(*v));
4147 return raw_atomic_long_fetch_or_relea 4119 return raw_atomic_long_fetch_or_release(i, v);
4148 } 4120 }
4149 4121
4150 /** 4122 /**
4151 * atomic_long_fetch_or_relaxed() - atomic bi 4123 * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
4152 * @i: long value 4124 * @i: long value
4153 * @v: pointer to atomic_long_t 4125 * @v: pointer to atomic_long_t
4154 * 4126 *
4155 * Atomically updates @v to (@v | @i) with re 4127 * Atomically updates @v to (@v | @i) with relaxed ordering.
4156 * 4128 *
4157 * Unsafe to use in noinstr code; use raw_ato 4129 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
4158 * 4130 *
4159 * Return: The original value of @v. 4131 * Return: The original value of @v.
4160 */ 4132 */
4161 static __always_inline long 4133 static __always_inline long
4162 atomic_long_fetch_or_relaxed(long i, atomic_l 4134 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
4163 { 4135 {
4164 instrument_atomic_read_write(v, sizeo 4136 instrument_atomic_read_write(v, sizeof(*v));
4165 return raw_atomic_long_fetch_or_relax 4137 return raw_atomic_long_fetch_or_relaxed(i, v);
4166 } 4138 }
4167 4139
4168 /** 4140 /**
4169 * atomic_long_xor() - atomic bitwise XOR wit 4141 * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
4170 * @i: long value 4142 * @i: long value
4171 * @v: pointer to atomic_long_t 4143 * @v: pointer to atomic_long_t
4172 * 4144 *
4173 * Atomically updates @v to (@v ^ @i) with re 4145 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4174 * 4146 *
4175 * Unsafe to use in noinstr code; use raw_ato 4147 * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
4176 * 4148 *
4177 * Return: Nothing. 4149 * Return: Nothing.
4178 */ 4150 */
4179 static __always_inline void 4151 static __always_inline void
4180 atomic_long_xor(long i, atomic_long_t *v) 4152 atomic_long_xor(long i, atomic_long_t *v)
4181 { 4153 {
4182 instrument_atomic_read_write(v, sizeo 4154 instrument_atomic_read_write(v, sizeof(*v));
4183 raw_atomic_long_xor(i, v); 4155 raw_atomic_long_xor(i, v);
4184 } 4156 }
4185 4157
4186 /** 4158 /**
4187 * atomic_long_fetch_xor() - atomic bitwise X 4159 * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
4188 * @i: long value 4160 * @i: long value
4189 * @v: pointer to atomic_long_t 4161 * @v: pointer to atomic_long_t
4190 * 4162 *
4191 * Atomically updates @v to (@v ^ @i) with fu 4163 * Atomically updates @v to (@v ^ @i) with full ordering.
4192 * 4164 *
4193 * Unsafe to use in noinstr code; use raw_ato 4165 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
4194 * 4166 *
4195 * Return: The original value of @v. 4167 * Return: The original value of @v.
4196 */ 4168 */
4197 static __always_inline long 4169 static __always_inline long
4198 atomic_long_fetch_xor(long i, atomic_long_t * 4170 atomic_long_fetch_xor(long i, atomic_long_t *v)
4199 { 4171 {
4200 kcsan_mb(); 4172 kcsan_mb();
4201 instrument_atomic_read_write(v, sizeo 4173 instrument_atomic_read_write(v, sizeof(*v));
4202 return raw_atomic_long_fetch_xor(i, v 4174 return raw_atomic_long_fetch_xor(i, v);
4203 } 4175 }
4204 4176
4205 /** 4177 /**
4206 * atomic_long_fetch_xor_acquire() - atomic b 4178 * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
4207 * @i: long value 4179 * @i: long value
4208 * @v: pointer to atomic_long_t 4180 * @v: pointer to atomic_long_t
4209 * 4181 *
4210 * Atomically updates @v to (@v ^ @i) with ac 4182 * Atomically updates @v to (@v ^ @i) with acquire ordering.
4211 * 4183 *
4212 * Unsafe to use in noinstr code; use raw_ato 4184 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
4213 * 4185 *
4214 * Return: The original value of @v. 4186 * Return: The original value of @v.
4215 */ 4187 */
4216 static __always_inline long 4188 static __always_inline long
4217 atomic_long_fetch_xor_acquire(long i, atomic_ 4189 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
4218 { 4190 {
4219 instrument_atomic_read_write(v, sizeo 4191 instrument_atomic_read_write(v, sizeof(*v));
4220 return raw_atomic_long_fetch_xor_acqu 4192 return raw_atomic_long_fetch_xor_acquire(i, v);
4221 } 4193 }
4222 4194
4223 /** 4195 /**
4224 * atomic_long_fetch_xor_release() - atomic b 4196 * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
4225 * @i: long value 4197 * @i: long value
4226 * @v: pointer to atomic_long_t 4198 * @v: pointer to atomic_long_t
4227 * 4199 *
4228 * Atomically updates @v to (@v ^ @i) with re 4200 * Atomically updates @v to (@v ^ @i) with release ordering.
4229 * 4201 *
4230 * Unsafe to use in noinstr code; use raw_ato 4202 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
4231 * 4203 *
4232 * Return: The original value of @v. 4204 * Return: The original value of @v.
4233 */ 4205 */
4234 static __always_inline long 4206 static __always_inline long
4235 atomic_long_fetch_xor_release(long i, atomic_ 4207 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
4236 { 4208 {
4237 kcsan_release(); 4209 kcsan_release();
4238 instrument_atomic_read_write(v, sizeo 4210 instrument_atomic_read_write(v, sizeof(*v));
4239 return raw_atomic_long_fetch_xor_rele 4211 return raw_atomic_long_fetch_xor_release(i, v);
4240 } 4212 }
4241 4213
4242 /** 4214 /**
4243 * atomic_long_fetch_xor_relaxed() - atomic b 4215 * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
4244 * @i: long value 4216 * @i: long value
4245 * @v: pointer to atomic_long_t 4217 * @v: pointer to atomic_long_t
4246 * 4218 *
4247 * Atomically updates @v to (@v ^ @i) with re 4219 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4248 * 4220 *
4249 * Unsafe to use in noinstr code; use raw_ato 4221 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
4250 * 4222 *
4251 * Return: The original value of @v. 4223 * Return: The original value of @v.
4252 */ 4224 */
4253 static __always_inline long 4225 static __always_inline long
4254 atomic_long_fetch_xor_relaxed(long i, atomic_ 4226 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
4255 { 4227 {
4256 instrument_atomic_read_write(v, sizeo 4228 instrument_atomic_read_write(v, sizeof(*v));
4257 return raw_atomic_long_fetch_xor_rela 4229 return raw_atomic_long_fetch_xor_relaxed(i, v);
4258 } 4230 }
4259 4231
4260 /** 4232 /**
4261 * atomic_long_xchg() - atomic exchange with 4233 * atomic_long_xchg() - atomic exchange with full ordering
4262 * @v: pointer to atomic_long_t 4234 * @v: pointer to atomic_long_t
4263 * @new: long value to assign 4235 * @new: long value to assign
4264 * 4236 *
4265 * Atomically updates @v to @new with full or 4237 * Atomically updates @v to @new with full ordering.
4266 * 4238 *
4267 * Unsafe to use in noinstr code; use raw_ato 4239 * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
4268 * 4240 *
4269 * Return: The original value of @v. 4241 * Return: The original value of @v.
4270 */ 4242 */
4271 static __always_inline long 4243 static __always_inline long
4272 atomic_long_xchg(atomic_long_t *v, long new) 4244 atomic_long_xchg(atomic_long_t *v, long new)
4273 { 4245 {
4274 kcsan_mb(); 4246 kcsan_mb();
4275 instrument_atomic_read_write(v, sizeo 4247 instrument_atomic_read_write(v, sizeof(*v));
4276 return raw_atomic_long_xchg(v, new); 4248 return raw_atomic_long_xchg(v, new);
4277 } 4249 }
4278 4250
4279 /** 4251 /**
4280 * atomic_long_xchg_acquire() - atomic exchan 4252 * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
4281 * @v: pointer to atomic_long_t 4253 * @v: pointer to atomic_long_t
4282 * @new: long value to assign 4254 * @new: long value to assign
4283 * 4255 *
4284 * Atomically updates @v to @new with acquire 4256 * Atomically updates @v to @new with acquire ordering.
4285 * 4257 *
4286 * Unsafe to use in noinstr code; use raw_ato 4258 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
4287 * 4259 *
4288 * Return: The original value of @v. 4260 * Return: The original value of @v.
4289 */ 4261 */
4290 static __always_inline long 4262 static __always_inline long
4291 atomic_long_xchg_acquire(atomic_long_t *v, lo 4263 atomic_long_xchg_acquire(atomic_long_t *v, long new)
4292 { 4264 {
4293 instrument_atomic_read_write(v, sizeo 4265 instrument_atomic_read_write(v, sizeof(*v));
4294 return raw_atomic_long_xchg_acquire(v 4266 return raw_atomic_long_xchg_acquire(v, new);
4295 } 4267 }
4296 4268
4297 /** 4269 /**
4298 * atomic_long_xchg_release() - atomic exchan 4270 * atomic_long_xchg_release() - atomic exchange with release ordering
4299 * @v: pointer to atomic_long_t 4271 * @v: pointer to atomic_long_t
4300 * @new: long value to assign 4272 * @new: long value to assign
4301 * 4273 *
4302 * Atomically updates @v to @new with release 4274 * Atomically updates @v to @new with release ordering.
4303 * 4275 *
4304 * Unsafe to use in noinstr code; use raw_ato 4276 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
4305 * 4277 *
4306 * Return: The original value of @v. 4278 * Return: The original value of @v.
4307 */ 4279 */
4308 static __always_inline long 4280 static __always_inline long
4309 atomic_long_xchg_release(atomic_long_t *v, lo 4281 atomic_long_xchg_release(atomic_long_t *v, long new)
4310 { 4282 {
4311 kcsan_release(); 4283 kcsan_release();
4312 instrument_atomic_read_write(v, sizeo 4284 instrument_atomic_read_write(v, sizeof(*v));
4313 return raw_atomic_long_xchg_release(v 4285 return raw_atomic_long_xchg_release(v, new);
4314 } 4286 }
4315 4287
4316 /** 4288 /**
4317 * atomic_long_xchg_relaxed() - atomic exchan 4289 * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
4318 * @v: pointer to atomic_long_t 4290 * @v: pointer to atomic_long_t
4319 * @new: long value to assign 4291 * @new: long value to assign
4320 * 4292 *
4321 * Atomically updates @v to @new with relaxed 4293 * Atomically updates @v to @new with relaxed ordering.
4322 * 4294 *
4323 * Unsafe to use in noinstr code; use raw_ato 4295 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
4324 * 4296 *
4325 * Return: The original value of @v. 4297 * Return: The original value of @v.
4326 */ 4298 */
4327 static __always_inline long 4299 static __always_inline long
4328 atomic_long_xchg_relaxed(atomic_long_t *v, lo 4300 atomic_long_xchg_relaxed(atomic_long_t *v, long new)
4329 { 4301 {
4330 instrument_atomic_read_write(v, sizeo 4302 instrument_atomic_read_write(v, sizeof(*v));
4331 return raw_atomic_long_xchg_relaxed(v 4303 return raw_atomic_long_xchg_relaxed(v, new);
4332 } 4304 }
4333 4305
4334 /** 4306 /**
4335 * atomic_long_cmpxchg() - atomic compare and 4307 * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
4336 * @v: pointer to atomic_long_t 4308 * @v: pointer to atomic_long_t
4337 * @old: long value to compare with 4309 * @old: long value to compare with
4338 * @new: long value to assign 4310 * @new: long value to assign
4339 * 4311 *
4340 * If (@v == @old), atomically updates @v to 4312 * If (@v == @old), atomically updates @v to @new with full ordering.
4341 * Otherwise, @v is not modified and relaxed <<
4342 * 4313 *
4343 * Unsafe to use in noinstr code; use raw_ato 4314 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
4344 * 4315 *
4345 * Return: The original value of @v. 4316 * Return: The original value of @v.
4346 */ 4317 */
4347 static __always_inline long 4318 static __always_inline long
4348 atomic_long_cmpxchg(atomic_long_t *v, long ol 4319 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
4349 { 4320 {
4350 kcsan_mb(); 4321 kcsan_mb();
4351 instrument_atomic_read_write(v, sizeo 4322 instrument_atomic_read_write(v, sizeof(*v));
4352 return raw_atomic_long_cmpxchg(v, old 4323 return raw_atomic_long_cmpxchg(v, old, new);
4353 } 4324 }
4354 4325
4355 /** 4326 /**
4356 * atomic_long_cmpxchg_acquire() - atomic com 4327 * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4357 * @v: pointer to atomic_long_t 4328 * @v: pointer to atomic_long_t
4358 * @old: long value to compare with 4329 * @old: long value to compare with
4359 * @new: long value to assign 4330 * @new: long value to assign
4360 * 4331 *
4361 * If (@v == @old), atomically updates @v to 4332 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4362 * Otherwise, @v is not modified and relaxed <<
4363 * 4333 *
4364 * Unsafe to use in noinstr code; use raw_ato 4334 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
4365 * 4335 *
4366 * Return: The original value of @v. 4336 * Return: The original value of @v.
4367 */ 4337 */
4368 static __always_inline long 4338 static __always_inline long
4369 atomic_long_cmpxchg_acquire(atomic_long_t *v, 4339 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
4370 { 4340 {
4371 instrument_atomic_read_write(v, sizeo 4341 instrument_atomic_read_write(v, sizeof(*v));
4372 return raw_atomic_long_cmpxchg_acquir 4342 return raw_atomic_long_cmpxchg_acquire(v, old, new);
4373 } 4343 }
4374 4344
4375 /** 4345 /**
4376 * atomic_long_cmpxchg_release() - atomic com 4346 * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
4377 * @v: pointer to atomic_long_t 4347 * @v: pointer to atomic_long_t
4378 * @old: long value to compare with 4348 * @old: long value to compare with
4379 * @new: long value to assign 4349 * @new: long value to assign
4380 * 4350 *
4381 * If (@v == @old), atomically updates @v to 4351 * If (@v == @old), atomically updates @v to @new with release ordering.
4382 * Otherwise, @v is not modified and relaxed <<
4383 * 4352 *
4384 * Unsafe to use in noinstr code; use raw_ato 4353 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
4385 * 4354 *
4386 * Return: The original value of @v. 4355 * Return: The original value of @v.
4387 */ 4356 */
4388 static __always_inline long 4357 static __always_inline long
4389 atomic_long_cmpxchg_release(atomic_long_t *v, 4358 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
4390 { 4359 {
4391 kcsan_release(); 4360 kcsan_release();
4392 instrument_atomic_read_write(v, sizeo 4361 instrument_atomic_read_write(v, sizeof(*v));
4393 return raw_atomic_long_cmpxchg_releas 4362 return raw_atomic_long_cmpxchg_release(v, old, new);
4394 } 4363 }
4395 4364
4396 /** 4365 /**
4397 * atomic_long_cmpxchg_relaxed() - atomic com 4366 * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4398 * @v: pointer to atomic_long_t 4367 * @v: pointer to atomic_long_t
4399 * @old: long value to compare with 4368 * @old: long value to compare with
4400 * @new: long value to assign 4369 * @new: long value to assign
4401 * 4370 *
4402 * If (@v == @old), atomically updates @v to 4371 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4403 * Otherwise, @v is not modified and relaxed <<
4404 * 4372 *
4405 * Unsafe to use in noinstr code; use raw_ato 4373 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
4406 * 4374 *
4407 * Return: The original value of @v. 4375 * Return: The original value of @v.
4408 */ 4376 */
4409 static __always_inline long 4377 static __always_inline long
4410 atomic_long_cmpxchg_relaxed(atomic_long_t *v, 4378 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
4411 { 4379 {
4412 instrument_atomic_read_write(v, sizeo 4380 instrument_atomic_read_write(v, sizeof(*v));
4413 return raw_atomic_long_cmpxchg_relaxe 4381 return raw_atomic_long_cmpxchg_relaxed(v, old, new);
4414 } 4382 }
4415 4383
4416 /** 4384 /**
4417 * atomic_long_try_cmpxchg() - atomic compare 4385 * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
4418 * @v: pointer to atomic_long_t 4386 * @v: pointer to atomic_long_t
4419 * @old: pointer to long value to compare wit 4387 * @old: pointer to long value to compare with
4420 * @new: long value to assign 4388 * @new: long value to assign
4421 * 4389 *
4422 * If (@v == @old), atomically updates @v to 4390 * If (@v == @old), atomically updates @v to @new with full ordering.
4423 * Otherwise, @v is not modified, @old is upd !! 4391 * Otherwise, updates @old to the current value of @v.
4424 * and relaxed ordering is provided. <<
4425 * 4392 *
4426 * Unsafe to use in noinstr code; use raw_ato 4393 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
4427 * 4394 *
4428 * Return: @true if the exchange occured, @fa 4395 * Return: @true if the exchange occured, @false otherwise.
4429 */ 4396 */
4430 static __always_inline bool 4397 static __always_inline bool
4431 atomic_long_try_cmpxchg(atomic_long_t *v, lon 4398 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
4432 { 4399 {
4433 kcsan_mb(); 4400 kcsan_mb();
4434 instrument_atomic_read_write(v, sizeo 4401 instrument_atomic_read_write(v, sizeof(*v));
4435 instrument_atomic_read_write(old, siz 4402 instrument_atomic_read_write(old, sizeof(*old));
4436 return raw_atomic_long_try_cmpxchg(v, 4403 return raw_atomic_long_try_cmpxchg(v, old, new);
4437 } 4404 }
4438 4405
4439 /** 4406 /**
4440 * atomic_long_try_cmpxchg_acquire() - atomic 4407 * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4441 * @v: pointer to atomic_long_t 4408 * @v: pointer to atomic_long_t
4442 * @old: pointer to long value to compare wit 4409 * @old: pointer to long value to compare with
4443 * @new: long value to assign 4410 * @new: long value to assign
4444 * 4411 *
4445 * If (@v == @old), atomically updates @v to 4412 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4446 * Otherwise, @v is not modified, @old is upd !! 4413 * Otherwise, updates @old to the current value of @v.
4447 * and relaxed ordering is provided. <<
4448 * 4414 *
4449 * Unsafe to use in noinstr code; use raw_ato 4415 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
4450 * 4416 *
4451 * Return: @true if the exchange occured, @fa 4417 * Return: @true if the exchange occured, @false otherwise.
4452 */ 4418 */
4453 static __always_inline bool 4419 static __always_inline bool
4454 atomic_long_try_cmpxchg_acquire(atomic_long_t 4420 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
4455 { 4421 {
4456 instrument_atomic_read_write(v, sizeo 4422 instrument_atomic_read_write(v, sizeof(*v));
4457 instrument_atomic_read_write(old, siz 4423 instrument_atomic_read_write(old, sizeof(*old));
4458 return raw_atomic_long_try_cmpxchg_ac 4424 return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
4459 } 4425 }
4460 4426
4461 /** 4427 /**
4462 * atomic_long_try_cmpxchg_release() - atomic 4428 * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
4463 * @v: pointer to atomic_long_t 4429 * @v: pointer to atomic_long_t
4464 * @old: pointer to long value to compare wit 4430 * @old: pointer to long value to compare with
4465 * @new: long value to assign 4431 * @new: long value to assign
4466 * 4432 *
4467 * If (@v == @old), atomically updates @v to 4433 * If (@v == @old), atomically updates @v to @new with release ordering.
4468 * Otherwise, @v is not modified, @old is upd !! 4434 * Otherwise, updates @old to the current value of @v.
4469 * and relaxed ordering is provided. <<
4470 * 4435 *
4471 * Unsafe to use in noinstr code; use raw_ato 4436 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
4472 * 4437 *
4473 * Return: @true if the exchange occured, @fa 4438 * Return: @true if the exchange occured, @false otherwise.
4474 */ 4439 */
4475 static __always_inline bool 4440 static __always_inline bool
4476 atomic_long_try_cmpxchg_release(atomic_long_t 4441 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
4477 { 4442 {
4478 kcsan_release(); 4443 kcsan_release();
4479 instrument_atomic_read_write(v, sizeo 4444 instrument_atomic_read_write(v, sizeof(*v));
4480 instrument_atomic_read_write(old, siz 4445 instrument_atomic_read_write(old, sizeof(*old));
4481 return raw_atomic_long_try_cmpxchg_re 4446 return raw_atomic_long_try_cmpxchg_release(v, old, new);
4482 } 4447 }
4483 4448
4484 /** 4449 /**
4485 * atomic_long_try_cmpxchg_relaxed() - atomic 4450 * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4486 * @v: pointer to atomic_long_t 4451 * @v: pointer to atomic_long_t
4487 * @old: pointer to long value to compare wit 4452 * @old: pointer to long value to compare with
4488 * @new: long value to assign 4453 * @new: long value to assign
4489 * 4454 *
4490 * If (@v == @old), atomically updates @v to 4455 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4491 * Otherwise, @v is not modified, @old is upd !! 4456 * Otherwise, updates @old to the current value of @v.
4492 * and relaxed ordering is provided. <<
4493 * 4457 *
4494 * Unsafe to use in noinstr code; use raw_ato 4458 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
4495 * 4459 *
4496 * Return: @true if the exchange occured, @fa 4460 * Return: @true if the exchange occured, @false otherwise.
4497 */ 4461 */
4498 static __always_inline bool 4462 static __always_inline bool
4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t 4463 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
4500 { 4464 {
4501 instrument_atomic_read_write(v, sizeo 4465 instrument_atomic_read_write(v, sizeof(*v));
4502 instrument_atomic_read_write(old, siz 4466 instrument_atomic_read_write(old, sizeof(*old));
4503 return raw_atomic_long_try_cmpxchg_re 4467 return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
4504 } 4468 }
4505 4469
4506 /** 4470 /**
4507 * atomic_long_sub_and_test() - atomic subtra 4471 * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
4508 * @i: long value to subtract !! 4472 * @i: long value to add
4509 * @v: pointer to atomic_long_t 4473 * @v: pointer to atomic_long_t
4510 * 4474 *
4511 * Atomically updates @v to (@v - @i) with fu 4475 * Atomically updates @v to (@v - @i) with full ordering.
4512 * 4476 *
4513 * Unsafe to use in noinstr code; use raw_ato 4477 * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
4514 * 4478 *
4515 * Return: @true if the resulting value of @v 4479 * Return: @true if the resulting value of @v is zero, @false otherwise.
4516 */ 4480 */
4517 static __always_inline bool 4481 static __always_inline bool
4518 atomic_long_sub_and_test(long i, atomic_long_ 4482 atomic_long_sub_and_test(long i, atomic_long_t *v)
4519 { 4483 {
4520 kcsan_mb(); 4484 kcsan_mb();
4521 instrument_atomic_read_write(v, sizeo 4485 instrument_atomic_read_write(v, sizeof(*v));
4522 return raw_atomic_long_sub_and_test(i 4486 return raw_atomic_long_sub_and_test(i, v);
4523 } 4487 }
4524 4488
4525 /** 4489 /**
4526 * atomic_long_dec_and_test() - atomic decrem 4490 * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
4527 * @v: pointer to atomic_long_t 4491 * @v: pointer to atomic_long_t
4528 * 4492 *
4529 * Atomically updates @v to (@v - 1) with ful 4493 * Atomically updates @v to (@v - 1) with full ordering.
4530 * 4494 *
4531 * Unsafe to use in noinstr code; use raw_ato 4495 * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
4532 * 4496 *
4533 * Return: @true if the resulting value of @v 4497 * Return: @true if the resulting value of @v is zero, @false otherwise.
4534 */ 4498 */
4535 static __always_inline bool 4499 static __always_inline bool
4536 atomic_long_dec_and_test(atomic_long_t *v) 4500 atomic_long_dec_and_test(atomic_long_t *v)
4537 { 4501 {
4538 kcsan_mb(); 4502 kcsan_mb();
4539 instrument_atomic_read_write(v, sizeo 4503 instrument_atomic_read_write(v, sizeof(*v));
4540 return raw_atomic_long_dec_and_test(v 4504 return raw_atomic_long_dec_and_test(v);
4541 } 4505 }
4542 4506
4543 /** 4507 /**
4544 * atomic_long_inc_and_test() - atomic increm 4508 * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
4545 * @v: pointer to atomic_long_t 4509 * @v: pointer to atomic_long_t
4546 * 4510 *
4547 * Atomically updates @v to (@v + 1) with ful 4511 * Atomically updates @v to (@v + 1) with full ordering.
4548 * 4512 *
4549 * Unsafe to use in noinstr code; use raw_ato 4513 * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
4550 * 4514 *
4551 * Return: @true if the resulting value of @v 4515 * Return: @true if the resulting value of @v is zero, @false otherwise.
4552 */ 4516 */
4553 static __always_inline bool 4517 static __always_inline bool
4554 atomic_long_inc_and_test(atomic_long_t *v) 4518 atomic_long_inc_and_test(atomic_long_t *v)
4555 { 4519 {
4556 kcsan_mb(); 4520 kcsan_mb();
4557 instrument_atomic_read_write(v, sizeo 4521 instrument_atomic_read_write(v, sizeof(*v));
4558 return raw_atomic_long_inc_and_test(v 4522 return raw_atomic_long_inc_and_test(v);
4559 } 4523 }
4560 4524
4561 /** 4525 /**
4562 * atomic_long_add_negative() - atomic add an 4526 * atomic_long_add_negative() - atomic add and test if negative with full ordering
4563 * @i: long value to add 4527 * @i: long value to add
4564 * @v: pointer to atomic_long_t 4528 * @v: pointer to atomic_long_t
4565 * 4529 *
4566 * Atomically updates @v to (@v + @i) with fu 4530 * Atomically updates @v to (@v + @i) with full ordering.
4567 * 4531 *
4568 * Unsafe to use in noinstr code; use raw_ato 4532 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
4569 * 4533 *
4570 * Return: @true if the resulting value of @v 4534 * Return: @true if the resulting value of @v is negative, @false otherwise.
4571 */ 4535 */
4572 static __always_inline bool 4536 static __always_inline bool
4573 atomic_long_add_negative(long i, atomic_long_ 4537 atomic_long_add_negative(long i, atomic_long_t *v)
4574 { 4538 {
4575 kcsan_mb(); 4539 kcsan_mb();
4576 instrument_atomic_read_write(v, sizeo 4540 instrument_atomic_read_write(v, sizeof(*v));
4577 return raw_atomic_long_add_negative(i 4541 return raw_atomic_long_add_negative(i, v);
4578 } 4542 }
4579 4543
4580 /** 4544 /**
4581 * atomic_long_add_negative_acquire() - atomi 4545 * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
4582 * @i: long value to add 4546 * @i: long value to add
4583 * @v: pointer to atomic_long_t 4547 * @v: pointer to atomic_long_t
4584 * 4548 *
4585 * Atomically updates @v to (@v + @i) with ac 4549 * Atomically updates @v to (@v + @i) with acquire ordering.
4586 * 4550 *
4587 * Unsafe to use in noinstr code; use raw_ato 4551 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
4588 * 4552 *
4589 * Return: @true if the resulting value of @v 4553 * Return: @true if the resulting value of @v is negative, @false otherwise.
4590 */ 4554 */
4591 static __always_inline bool 4555 static __always_inline bool
4592 atomic_long_add_negative_acquire(long i, atom 4556 atomic_long_add_negative_acquire(long i, atomic_long_t *v)
4593 { 4557 {
4594 instrument_atomic_read_write(v, sizeo 4558 instrument_atomic_read_write(v, sizeof(*v));
4595 return raw_atomic_long_add_negative_a 4559 return raw_atomic_long_add_negative_acquire(i, v);
4596 } 4560 }
4597 4561
4598 /** 4562 /**
4599 * atomic_long_add_negative_release() - atomi 4563 * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
4600 * @i: long value to add 4564 * @i: long value to add
4601 * @v: pointer to atomic_long_t 4565 * @v: pointer to atomic_long_t
4602 * 4566 *
4603 * Atomically updates @v to (@v + @i) with re 4567 * Atomically updates @v to (@v + @i) with release ordering.
4604 * 4568 *
4605 * Unsafe to use in noinstr code; use raw_ato 4569 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
4606 * 4570 *
4607 * Return: @true if the resulting value of @v 4571 * Return: @true if the resulting value of @v is negative, @false otherwise.
4608 */ 4572 */
4609 static __always_inline bool 4573 static __always_inline bool
4610 atomic_long_add_negative_release(long i, atom 4574 atomic_long_add_negative_release(long i, atomic_long_t *v)
4611 { 4575 {
4612 kcsan_release(); 4576 kcsan_release();
4613 instrument_atomic_read_write(v, sizeo 4577 instrument_atomic_read_write(v, sizeof(*v));
4614 return raw_atomic_long_add_negative_r 4578 return raw_atomic_long_add_negative_release(i, v);
4615 } 4579 }
4616 4580
4617 /** 4581 /**
4618 * atomic_long_add_negative_relaxed() - atomi 4582 * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4619 * @i: long value to add 4583 * @i: long value to add
4620 * @v: pointer to atomic_long_t 4584 * @v: pointer to atomic_long_t
4621 * 4585 *
4622 * Atomically updates @v to (@v + @i) with re 4586 * Atomically updates @v to (@v + @i) with relaxed ordering.
4623 * 4587 *
4624 * Unsafe to use in noinstr code; use raw_ato 4588 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
4625 * 4589 *
4626 * Return: @true if the resulting value of @v 4590 * Return: @true if the resulting value of @v is negative, @false otherwise.
4627 */ 4591 */
4628 static __always_inline bool 4592 static __always_inline bool
4629 atomic_long_add_negative_relaxed(long i, atom 4593 atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
4630 { 4594 {
4631 instrument_atomic_read_write(v, sizeo 4595 instrument_atomic_read_write(v, sizeof(*v));
4632 return raw_atomic_long_add_negative_r 4596 return raw_atomic_long_add_negative_relaxed(i, v);
4633 } 4597 }
4634 4598
4635 /** 4599 /**
4636 * atomic_long_fetch_add_unless() - atomic ad 4600 * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
4637 * @v: pointer to atomic_long_t 4601 * @v: pointer to atomic_long_t
4638 * @a: long value to add 4602 * @a: long value to add
4639 * @u: long value to compare with 4603 * @u: long value to compare with
4640 * 4604 *
4641 * If (@v != @u), atomically updates @v to (@ 4605 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4642 * Otherwise, @v is not modified and relaxed <<
4643 * 4606 *
4644 * Unsafe to use in noinstr code; use raw_ato 4607 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
4645 * 4608 *
4646 * Return: The original value of @v. 4609 * Return: The original value of @v.
4647 */ 4610 */
4648 static __always_inline long 4611 static __always_inline long
4649 atomic_long_fetch_add_unless(atomic_long_t *v 4612 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
4650 { 4613 {
4651 kcsan_mb(); 4614 kcsan_mb();
4652 instrument_atomic_read_write(v, sizeo 4615 instrument_atomic_read_write(v, sizeof(*v));
4653 return raw_atomic_long_fetch_add_unle 4616 return raw_atomic_long_fetch_add_unless(v, a, u);
4654 } 4617 }
4655 4618
4656 /** 4619 /**
4657 * atomic_long_add_unless() - atomic add unle 4620 * atomic_long_add_unless() - atomic add unless value with full ordering
4658 * @v: pointer to atomic_long_t 4621 * @v: pointer to atomic_long_t
4659 * @a: long value to add 4622 * @a: long value to add
4660 * @u: long value to compare with 4623 * @u: long value to compare with
4661 * 4624 *
4662 * If (@v != @u), atomically updates @v to (@ 4625 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4663 * Otherwise, @v is not modified and relaxed <<
4664 * 4626 *
4665 * Unsafe to use in noinstr code; use raw_ato 4627 * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
4666 * 4628 *
4667 * Return: @true if @v was updated, @false ot 4629 * Return: @true if @v was updated, @false otherwise.
4668 */ 4630 */
4669 static __always_inline bool 4631 static __always_inline bool
4670 atomic_long_add_unless(atomic_long_t *v, long 4632 atomic_long_add_unless(atomic_long_t *v, long a, long u)
4671 { 4633 {
4672 kcsan_mb(); 4634 kcsan_mb();
4673 instrument_atomic_read_write(v, sizeo 4635 instrument_atomic_read_write(v, sizeof(*v));
4674 return raw_atomic_long_add_unless(v, 4636 return raw_atomic_long_add_unless(v, a, u);
4675 } 4637 }
4676 4638
4677 /** 4639 /**
4678 * atomic_long_inc_not_zero() - atomic increm 4640 * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
4679 * @v: pointer to atomic_long_t 4641 * @v: pointer to atomic_long_t
4680 * 4642 *
4681 * If (@v != 0), atomically updates @v to (@v 4643 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4682 * Otherwise, @v is not modified and relaxed <<
4683 * 4644 *
4684 * Unsafe to use in noinstr code; use raw_ato 4645 * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
4685 * 4646 *
4686 * Return: @true if @v was updated, @false ot 4647 * Return: @true if @v was updated, @false otherwise.
4687 */ 4648 */
4688 static __always_inline bool 4649 static __always_inline bool
4689 atomic_long_inc_not_zero(atomic_long_t *v) 4650 atomic_long_inc_not_zero(atomic_long_t *v)
4690 { 4651 {
4691 kcsan_mb(); 4652 kcsan_mb();
4692 instrument_atomic_read_write(v, sizeo 4653 instrument_atomic_read_write(v, sizeof(*v));
4693 return raw_atomic_long_inc_not_zero(v 4654 return raw_atomic_long_inc_not_zero(v);
4694 } 4655 }
4695 4656
4696 /** 4657 /**
4697 * atomic_long_inc_unless_negative() - atomic 4658 * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
4698 * @v: pointer to atomic_long_t 4659 * @v: pointer to atomic_long_t
4699 * 4660 *
4700 * If (@v >= 0), atomically updates @v to (@v 4661 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4701 * Otherwise, @v is not modified and relaxed <<
4702 * 4662 *
4703 * Unsafe to use in noinstr code; use raw_ato 4663 * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
4704 * 4664 *
4705 * Return: @true if @v was updated, @false ot 4665 * Return: @true if @v was updated, @false otherwise.
4706 */ 4666 */
4707 static __always_inline bool 4667 static __always_inline bool
4708 atomic_long_inc_unless_negative(atomic_long_t 4668 atomic_long_inc_unless_negative(atomic_long_t *v)
4709 { 4669 {
4710 kcsan_mb(); 4670 kcsan_mb();
4711 instrument_atomic_read_write(v, sizeo 4671 instrument_atomic_read_write(v, sizeof(*v));
4712 return raw_atomic_long_inc_unless_neg 4672 return raw_atomic_long_inc_unless_negative(v);
4713 } 4673 }
4714 4674
4715 /** 4675 /**
4716 * atomic_long_dec_unless_positive() - atomic 4676 * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
4717 * @v: pointer to atomic_long_t 4677 * @v: pointer to atomic_long_t
4718 * 4678 *
4719 * If (@v <= 0), atomically updates @v to (@v 4679 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4720 * Otherwise, @v is not modified and relaxed <<
4721 * 4680 *
4722 * Unsafe to use in noinstr code; use raw_ato 4681 * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
4723 * 4682 *
4724 * Return: @true if @v was updated, @false ot 4683 * Return: @true if @v was updated, @false otherwise.
4725 */ 4684 */
4726 static __always_inline bool 4685 static __always_inline bool
4727 atomic_long_dec_unless_positive(atomic_long_t 4686 atomic_long_dec_unless_positive(atomic_long_t *v)
4728 { 4687 {
4729 kcsan_mb(); 4688 kcsan_mb();
4730 instrument_atomic_read_write(v, sizeo 4689 instrument_atomic_read_write(v, sizeof(*v));
4731 return raw_atomic_long_dec_unless_pos 4690 return raw_atomic_long_dec_unless_positive(v);
4732 } 4691 }
4733 4692
4734 /** 4693 /**
4735 * atomic_long_dec_if_positive() - atomic dec 4694 * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
4736 * @v: pointer to atomic_long_t 4695 * @v: pointer to atomic_long_t
4737 * 4696 *
4738 * If (@v > 0), atomically updates @v to (@v 4697 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4739 * Otherwise, @v is not modified and relaxed <<
4740 * 4698 *
4741 * Unsafe to use in noinstr code; use raw_ato 4699 * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
4742 * 4700 *
4743 * Return: The old value of (@v - 1), regardl 4701 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4744 */ 4702 */
4745 static __always_inline long 4703 static __always_inline long
4746 atomic_long_dec_if_positive(atomic_long_t *v) 4704 atomic_long_dec_if_positive(atomic_long_t *v)
4747 { 4705 {
4748 kcsan_mb(); 4706 kcsan_mb();
4749 instrument_atomic_read_write(v, sizeo 4707 instrument_atomic_read_write(v, sizeof(*v));
4750 return raw_atomic_long_dec_if_positiv 4708 return raw_atomic_long_dec_if_positive(v);
4751 } 4709 }
4752 4710
4753 #define xchg(ptr, ...) \ 4711 #define xchg(ptr, ...) \
4754 ({ \ 4712 ({ \
4755 typeof(ptr) __ai_ptr = (ptr); \ 4713 typeof(ptr) __ai_ptr = (ptr); \
4756 kcsan_mb(); \ 4714 kcsan_mb(); \
4757 instrument_atomic_read_write(__ai_ptr 4715 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4758 raw_xchg(__ai_ptr, __VA_ARGS__); \ 4716 raw_xchg(__ai_ptr, __VA_ARGS__); \
4759 }) 4717 })
4760 4718
4761 #define xchg_acquire(ptr, ...) \ 4719 #define xchg_acquire(ptr, ...) \
4762 ({ \ 4720 ({ \
4763 typeof(ptr) __ai_ptr = (ptr); \ 4721 typeof(ptr) __ai_ptr = (ptr); \
4764 instrument_atomic_read_write(__ai_ptr 4722 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4765 raw_xchg_acquire(__ai_ptr, __VA_ARGS_ 4723 raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \
4766 }) 4724 })
4767 4725
4768 #define xchg_release(ptr, ...) \ 4726 #define xchg_release(ptr, ...) \
4769 ({ \ 4727 ({ \
4770 typeof(ptr) __ai_ptr = (ptr); \ 4728 typeof(ptr) __ai_ptr = (ptr); \
4771 kcsan_release(); \ 4729 kcsan_release(); \
4772 instrument_atomic_read_write(__ai_ptr 4730 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4773 raw_xchg_release(__ai_ptr, __VA_ARGS_ 4731 raw_xchg_release(__ai_ptr, __VA_ARGS__); \
4774 }) 4732 })
4775 4733
4776 #define xchg_relaxed(ptr, ...) \ 4734 #define xchg_relaxed(ptr, ...) \
4777 ({ \ 4735 ({ \
4778 typeof(ptr) __ai_ptr = (ptr); \ 4736 typeof(ptr) __ai_ptr = (ptr); \
4779 instrument_atomic_read_write(__ai_ptr 4737 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4780 raw_xchg_relaxed(__ai_ptr, __VA_ARGS_ 4738 raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
4781 }) 4739 })
4782 4740
4783 #define cmpxchg(ptr, ...) \ 4741 #define cmpxchg(ptr, ...) \
4784 ({ \ 4742 ({ \
4785 typeof(ptr) __ai_ptr = (ptr); \ 4743 typeof(ptr) __ai_ptr = (ptr); \
4786 kcsan_mb(); \ 4744 kcsan_mb(); \
4787 instrument_atomic_read_write(__ai_ptr 4745 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4788 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \ 4746 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
4789 }) 4747 })
4790 4748
4791 #define cmpxchg_acquire(ptr, ...) \ 4749 #define cmpxchg_acquire(ptr, ...) \
4792 ({ \ 4750 ({ \
4793 typeof(ptr) __ai_ptr = (ptr); \ 4751 typeof(ptr) __ai_ptr = (ptr); \
4794 instrument_atomic_read_write(__ai_ptr 4752 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4795 raw_cmpxchg_acquire(__ai_ptr, __VA_AR 4753 raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
4796 }) 4754 })
4797 4755
4798 #define cmpxchg_release(ptr, ...) \ 4756 #define cmpxchg_release(ptr, ...) \
4799 ({ \ 4757 ({ \
4800 typeof(ptr) __ai_ptr = (ptr); \ 4758 typeof(ptr) __ai_ptr = (ptr); \
4801 kcsan_release(); \ 4759 kcsan_release(); \
4802 instrument_atomic_read_write(__ai_ptr 4760 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4803 raw_cmpxchg_release(__ai_ptr, __VA_AR 4761 raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
4804 }) 4762 })
4805 4763
4806 #define cmpxchg_relaxed(ptr, ...) \ 4764 #define cmpxchg_relaxed(ptr, ...) \
4807 ({ \ 4765 ({ \
4808 typeof(ptr) __ai_ptr = (ptr); \ 4766 typeof(ptr) __ai_ptr = (ptr); \
4809 instrument_atomic_read_write(__ai_ptr 4767 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4810 raw_cmpxchg_relaxed(__ai_ptr, __VA_AR 4768 raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
4811 }) 4769 })
4812 4770
4813 #define cmpxchg64(ptr, ...) \ 4771 #define cmpxchg64(ptr, ...) \
4814 ({ \ 4772 ({ \
4815 typeof(ptr) __ai_ptr = (ptr); \ 4773 typeof(ptr) __ai_ptr = (ptr); \
4816 kcsan_mb(); \ 4774 kcsan_mb(); \
4817 instrument_atomic_read_write(__ai_ptr 4775 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4818 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); 4776 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \
4819 }) 4777 })
4820 4778
4821 #define cmpxchg64_acquire(ptr, ...) \ 4779 #define cmpxchg64_acquire(ptr, ...) \
4822 ({ \ 4780 ({ \
4823 typeof(ptr) __ai_ptr = (ptr); \ 4781 typeof(ptr) __ai_ptr = (ptr); \
4824 instrument_atomic_read_write(__ai_ptr 4782 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4825 raw_cmpxchg64_acquire(__ai_ptr, __VA_ 4783 raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
4826 }) 4784 })
4827 4785
4828 #define cmpxchg64_release(ptr, ...) \ 4786 #define cmpxchg64_release(ptr, ...) \
4829 ({ \ 4787 ({ \
4830 typeof(ptr) __ai_ptr = (ptr); \ 4788 typeof(ptr) __ai_ptr = (ptr); \
4831 kcsan_release(); \ 4789 kcsan_release(); \
4832 instrument_atomic_read_write(__ai_ptr 4790 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4833 raw_cmpxchg64_release(__ai_ptr, __VA_ 4791 raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
4834 }) 4792 })
4835 4793
4836 #define cmpxchg64_relaxed(ptr, ...) \ 4794 #define cmpxchg64_relaxed(ptr, ...) \
4837 ({ \ 4795 ({ \
4838 typeof(ptr) __ai_ptr = (ptr); \ 4796 typeof(ptr) __ai_ptr = (ptr); \
4839 instrument_atomic_read_write(__ai_ptr 4797 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4840 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ 4798 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
4841 }) 4799 })
4842 4800
4843 #define cmpxchg128(ptr, ...) \ 4801 #define cmpxchg128(ptr, ...) \
4844 ({ \ 4802 ({ \
4845 typeof(ptr) __ai_ptr = (ptr); \ 4803 typeof(ptr) __ai_ptr = (ptr); \
4846 kcsan_mb(); \ 4804 kcsan_mb(); \
4847 instrument_atomic_read_write(__ai_ptr 4805 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4848 raw_cmpxchg128(__ai_ptr, __VA_ARGS__) 4806 raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \
4849 }) 4807 })
4850 4808
4851 #define cmpxchg128_acquire(ptr, ...) \ 4809 #define cmpxchg128_acquire(ptr, ...) \
4852 ({ \ 4810 ({ \
4853 typeof(ptr) __ai_ptr = (ptr); \ 4811 typeof(ptr) __ai_ptr = (ptr); \
4854 instrument_atomic_read_write(__ai_ptr 4812 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4855 raw_cmpxchg128_acquire(__ai_ptr, __VA 4813 raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
4856 }) 4814 })
4857 4815
4858 #define cmpxchg128_release(ptr, ...) \ 4816 #define cmpxchg128_release(ptr, ...) \
4859 ({ \ 4817 ({ \
4860 typeof(ptr) __ai_ptr = (ptr); \ 4818 typeof(ptr) __ai_ptr = (ptr); \
4861 kcsan_release(); \ 4819 kcsan_release(); \
4862 instrument_atomic_read_write(__ai_ptr 4820 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4863 raw_cmpxchg128_release(__ai_ptr, __VA 4821 raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
4864 }) 4822 })
4865 4823
4866 #define cmpxchg128_relaxed(ptr, ...) \ 4824 #define cmpxchg128_relaxed(ptr, ...) \
4867 ({ \ 4825 ({ \
4868 typeof(ptr) __ai_ptr = (ptr); \ 4826 typeof(ptr) __ai_ptr = (ptr); \
4869 instrument_atomic_read_write(__ai_ptr 4827 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4870 raw_cmpxchg128_relaxed(__ai_ptr, __VA 4828 raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
4871 }) 4829 })
4872 4830
4873 #define try_cmpxchg(ptr, oldp, ...) \ 4831 #define try_cmpxchg(ptr, oldp, ...) \
4874 ({ \ 4832 ({ \
4875 typeof(ptr) __ai_ptr = (ptr); \ 4833 typeof(ptr) __ai_ptr = (ptr); \
4876 typeof(oldp) __ai_oldp = (oldp); \ 4834 typeof(oldp) __ai_oldp = (oldp); \
4877 kcsan_mb(); \ 4835 kcsan_mb(); \
4878 instrument_atomic_read_write(__ai_ptr 4836 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4879 instrument_read_write(__ai_oldp, size 4837 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4880 raw_try_cmpxchg(__ai_ptr, __ai_oldp, 4838 raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4881 }) 4839 })
4882 4840
4883 #define try_cmpxchg_acquire(ptr, oldp, ...) \ 4841 #define try_cmpxchg_acquire(ptr, oldp, ...) \
4884 ({ \ 4842 ({ \
4885 typeof(ptr) __ai_ptr = (ptr); \ 4843 typeof(ptr) __ai_ptr = (ptr); \
4886 typeof(oldp) __ai_oldp = (oldp); \ 4844 typeof(oldp) __ai_oldp = (oldp); \
4887 instrument_atomic_read_write(__ai_ptr 4845 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4888 instrument_read_write(__ai_oldp, size 4846 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4889 raw_try_cmpxchg_acquire(__ai_ptr, __a 4847 raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4890 }) 4848 })
4891 4849
4892 #define try_cmpxchg_release(ptr, oldp, ...) \ 4850 #define try_cmpxchg_release(ptr, oldp, ...) \
4893 ({ \ 4851 ({ \
4894 typeof(ptr) __ai_ptr = (ptr); \ 4852 typeof(ptr) __ai_ptr = (ptr); \
4895 typeof(oldp) __ai_oldp = (oldp); \ 4853 typeof(oldp) __ai_oldp = (oldp); \
4896 kcsan_release(); \ 4854 kcsan_release(); \
4897 instrument_atomic_read_write(__ai_ptr 4855 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4898 instrument_read_write(__ai_oldp, size 4856 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4899 raw_try_cmpxchg_release(__ai_ptr, __a 4857 raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4900 }) 4858 })
4901 4859
4902 #define try_cmpxchg_relaxed(ptr, oldp, ...) \ 4860 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
4903 ({ \ 4861 ({ \
4904 typeof(ptr) __ai_ptr = (ptr); \ 4862 typeof(ptr) __ai_ptr = (ptr); \
4905 typeof(oldp) __ai_oldp = (oldp); \ 4863 typeof(oldp) __ai_oldp = (oldp); \
4906 instrument_atomic_read_write(__ai_ptr 4864 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4907 instrument_read_write(__ai_oldp, size 4865 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4908 raw_try_cmpxchg_relaxed(__ai_ptr, __a 4866 raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4909 }) 4867 })
4910 4868
4911 #define try_cmpxchg64(ptr, oldp, ...) \ 4869 #define try_cmpxchg64(ptr, oldp, ...) \
4912 ({ \ 4870 ({ \
4913 typeof(ptr) __ai_ptr = (ptr); \ 4871 typeof(ptr) __ai_ptr = (ptr); \
4914 typeof(oldp) __ai_oldp = (oldp); \ 4872 typeof(oldp) __ai_oldp = (oldp); \
4915 kcsan_mb(); \ 4873 kcsan_mb(); \
4916 instrument_atomic_read_write(__ai_ptr 4874 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4917 instrument_read_write(__ai_oldp, size 4875 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4918 raw_try_cmpxchg64(__ai_ptr, __ai_oldp 4876 raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4919 }) 4877 })
4920 4878
4921 #define try_cmpxchg64_acquire(ptr, oldp, ...) 4879 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
4922 ({ \ 4880 ({ \
4923 typeof(ptr) __ai_ptr = (ptr); \ 4881 typeof(ptr) __ai_ptr = (ptr); \
4924 typeof(oldp) __ai_oldp = (oldp); \ 4882 typeof(oldp) __ai_oldp = (oldp); \
4925 instrument_atomic_read_write(__ai_ptr 4883 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4926 instrument_read_write(__ai_oldp, size 4884 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4927 raw_try_cmpxchg64_acquire(__ai_ptr, _ 4885 raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4928 }) 4886 })
4929 4887
4930 #define try_cmpxchg64_release(ptr, oldp, ...) 4888 #define try_cmpxchg64_release(ptr, oldp, ...) \
4931 ({ \ 4889 ({ \
4932 typeof(ptr) __ai_ptr = (ptr); \ 4890 typeof(ptr) __ai_ptr = (ptr); \
4933 typeof(oldp) __ai_oldp = (oldp); \ 4891 typeof(oldp) __ai_oldp = (oldp); \
4934 kcsan_release(); \ 4892 kcsan_release(); \
4935 instrument_atomic_read_write(__ai_ptr 4893 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4936 instrument_read_write(__ai_oldp, size 4894 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4937 raw_try_cmpxchg64_release(__ai_ptr, _ 4895 raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4938 }) 4896 })
4939 4897
4940 #define try_cmpxchg64_relaxed(ptr, oldp, ...) 4898 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
4941 ({ \ 4899 ({ \
4942 typeof(ptr) __ai_ptr = (ptr); \ 4900 typeof(ptr) __ai_ptr = (ptr); \
4943 typeof(oldp) __ai_oldp = (oldp); \ 4901 typeof(oldp) __ai_oldp = (oldp); \
4944 instrument_atomic_read_write(__ai_ptr 4902 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4945 instrument_read_write(__ai_oldp, size 4903 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4946 raw_try_cmpxchg64_relaxed(__ai_ptr, _ 4904 raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4947 }) 4905 })
4948 4906
4949 #define try_cmpxchg128(ptr, oldp, ...) \ 4907 #define try_cmpxchg128(ptr, oldp, ...) \
4950 ({ \ 4908 ({ \
4951 typeof(ptr) __ai_ptr = (ptr); \ 4909 typeof(ptr) __ai_ptr = (ptr); \
4952 typeof(oldp) __ai_oldp = (oldp); \ 4910 typeof(oldp) __ai_oldp = (oldp); \
4953 kcsan_mb(); \ 4911 kcsan_mb(); \
4954 instrument_atomic_read_write(__ai_ptr 4912 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4955 instrument_read_write(__ai_oldp, size 4913 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4956 raw_try_cmpxchg128(__ai_ptr, __ai_old 4914 raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4957 }) 4915 })
4958 4916
4959 #define try_cmpxchg128_acquire(ptr, oldp, ... 4917 #define try_cmpxchg128_acquire(ptr, oldp, ...) \
4960 ({ \ 4918 ({ \
4961 typeof(ptr) __ai_ptr = (ptr); \ 4919 typeof(ptr) __ai_ptr = (ptr); \
4962 typeof(oldp) __ai_oldp = (oldp); \ 4920 typeof(oldp) __ai_oldp = (oldp); \
4963 instrument_atomic_read_write(__ai_ptr 4921 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4964 instrument_read_write(__ai_oldp, size 4922 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4965 raw_try_cmpxchg128_acquire(__ai_ptr, 4923 raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4966 }) 4924 })
4967 4925
4968 #define try_cmpxchg128_release(ptr, oldp, ... 4926 #define try_cmpxchg128_release(ptr, oldp, ...) \
4969 ({ \ 4927 ({ \
4970 typeof(ptr) __ai_ptr = (ptr); \ 4928 typeof(ptr) __ai_ptr = (ptr); \
4971 typeof(oldp) __ai_oldp = (oldp); \ 4929 typeof(oldp) __ai_oldp = (oldp); \
4972 kcsan_release(); \ 4930 kcsan_release(); \
4973 instrument_atomic_read_write(__ai_ptr 4931 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4974 instrument_read_write(__ai_oldp, size 4932 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4975 raw_try_cmpxchg128_release(__ai_ptr, 4933 raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4976 }) 4934 })
4977 4935
4978 #define try_cmpxchg128_relaxed(ptr, oldp, ... 4936 #define try_cmpxchg128_relaxed(ptr, oldp, ...) \
4979 ({ \ 4937 ({ \
4980 typeof(ptr) __ai_ptr = (ptr); \ 4938 typeof(ptr) __ai_ptr = (ptr); \
4981 typeof(oldp) __ai_oldp = (oldp); \ 4939 typeof(oldp) __ai_oldp = (oldp); \
4982 instrument_atomic_read_write(__ai_ptr 4940 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4983 instrument_read_write(__ai_oldp, size 4941 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
4984 raw_try_cmpxchg128_relaxed(__ai_ptr, 4942 raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
4985 }) 4943 })
4986 4944
4987 #define cmpxchg_local(ptr, ...) \ 4945 #define cmpxchg_local(ptr, ...) \
4988 ({ \ 4946 ({ \
4989 typeof(ptr) __ai_ptr = (ptr); \ 4947 typeof(ptr) __ai_ptr = (ptr); \
4990 instrument_atomic_read_write(__ai_ptr 4948 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4991 raw_cmpxchg_local(__ai_ptr, __VA_ARGS 4949 raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
4992 }) 4950 })
4993 4951
4994 #define cmpxchg64_local(ptr, ...) \ 4952 #define cmpxchg64_local(ptr, ...) \
4995 ({ \ 4953 ({ \
4996 typeof(ptr) __ai_ptr = (ptr); \ 4954 typeof(ptr) __ai_ptr = (ptr); \
4997 instrument_atomic_read_write(__ai_ptr 4955 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
4998 raw_cmpxchg64_local(__ai_ptr, __VA_AR 4956 raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
4999 }) 4957 })
5000 4958
5001 #define cmpxchg128_local(ptr, ...) \ 4959 #define cmpxchg128_local(ptr, ...) \
5002 ({ \ 4960 ({ \
5003 typeof(ptr) __ai_ptr = (ptr); \ 4961 typeof(ptr) __ai_ptr = (ptr); \
5004 instrument_atomic_read_write(__ai_ptr 4962 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5005 raw_cmpxchg128_local(__ai_ptr, __VA_A 4963 raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
5006 }) 4964 })
5007 4965
5008 #define sync_cmpxchg(ptr, ...) \ 4966 #define sync_cmpxchg(ptr, ...) \
5009 ({ \ 4967 ({ \
5010 typeof(ptr) __ai_ptr = (ptr); \ 4968 typeof(ptr) __ai_ptr = (ptr); \
5011 kcsan_mb(); \ 4969 kcsan_mb(); \
5012 instrument_atomic_read_write(__ai_ptr 4970 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5013 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS_ 4971 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
5014 }) 4972 })
5015 4973
5016 #define try_cmpxchg_local(ptr, oldp, ...) \ 4974 #define try_cmpxchg_local(ptr, oldp, ...) \
5017 ({ \ 4975 ({ \
5018 typeof(ptr) __ai_ptr = (ptr); \ 4976 typeof(ptr) __ai_ptr = (ptr); \
5019 typeof(oldp) __ai_oldp = (oldp); \ 4977 typeof(oldp) __ai_oldp = (oldp); \
5020 instrument_atomic_read_write(__ai_ptr 4978 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5021 instrument_read_write(__ai_oldp, size 4979 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5022 raw_try_cmpxchg_local(__ai_ptr, __ai_ 4980 raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5023 }) 4981 })
5024 4982
5025 #define try_cmpxchg64_local(ptr, oldp, ...) \ 4983 #define try_cmpxchg64_local(ptr, oldp, ...) \
5026 ({ \ 4984 ({ \
5027 typeof(ptr) __ai_ptr = (ptr); \ 4985 typeof(ptr) __ai_ptr = (ptr); \
5028 typeof(oldp) __ai_oldp = (oldp); \ 4986 typeof(oldp) __ai_oldp = (oldp); \
5029 instrument_atomic_read_write(__ai_ptr 4987 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5030 instrument_read_write(__ai_oldp, size 4988 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5031 raw_try_cmpxchg64_local(__ai_ptr, __a 4989 raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5032 }) 4990 })
5033 4991
5034 #define try_cmpxchg128_local(ptr, oldp, ...) 4992 #define try_cmpxchg128_local(ptr, oldp, ...) \
5035 ({ \ 4993 ({ \
5036 typeof(ptr) __ai_ptr = (ptr); \ 4994 typeof(ptr) __ai_ptr = (ptr); \
5037 typeof(oldp) __ai_oldp = (oldp); \ 4995 typeof(oldp) __ai_oldp = (oldp); \
5038 instrument_atomic_read_write(__ai_ptr 4996 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5039 instrument_read_write(__ai_oldp, size 4997 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
5040 raw_try_cmpxchg128_local(__ai_ptr, __ 4998 raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
5041 }) 4999 })
5042 5000
5043 #define sync_try_cmpxchg(ptr, ...) \ 5001 #define sync_try_cmpxchg(ptr, ...) \
5044 ({ \ 5002 ({ \
5045 typeof(ptr) __ai_ptr = (ptr); \ 5003 typeof(ptr) __ai_ptr = (ptr); \
5046 kcsan_mb(); \ 5004 kcsan_mb(); \
5047 instrument_atomic_read_write(__ai_ptr 5005 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
5048 raw_sync_try_cmpxchg(__ai_ptr, __VA_A 5006 raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \
5049 }) 5007 })
5050 5008
5051 5009
5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ 5010 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
5053 // 8829b337928e9508259079d32581775ececd415b !! 5011 // 2cc4bc990fef44d3836ec108f11b610f3f438184
5054 5012
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.