1 // SPDX-License-Identifier: GPL-2.0 2 3 // Generated by scripts/atomic/gen-atomic-instrumented.sh 4 // DO NOT MODIFY THIS FILE DIRECTLY 5 6 /* 7 * This file provoides atomic operations with explicit instrumentation (e.g. 8 * KASAN, KCSAN), which should be used unless it is necessary to avoid 9 * instrumentation. Where it is necessary to aovid instrumenation, the 10 * raw_atomic*() operations should be used. 11 */ 12 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H 13 #define _LINUX_ATOMIC_INSTRUMENTED_H 14 15 #include <linux/build_bug.h> 16 #include <linux/compiler.h> 17 #include <linux/instrumented.h> 18 19 /** 20 * atomic_read() - atomic load with relaxed ordering 21 * @v: pointer to atomic_t 22 * 23 * Atomically loads the value of @v with relaxed ordering. 24 * 25 * Unsafe to use in noinstr code; use raw_atomic_read() there. 26 * 27 * Return: The value loaded from @v. 28 */ 29 static __always_inline int 30 atomic_read(const atomic_t *v) 31 { 32 instrument_atomic_read(v, sizeof(*v)); 33 return raw_atomic_read(v); 34 } 35 36 /** 37 * atomic_read_acquire() - atomic load with acquire ordering 38 * @v: pointer to atomic_t 39 * 40 * Atomically loads the value of @v with acquire ordering. 41 * 42 * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there. 43 * 44 * Return: The value loaded from @v. 45 */ 46 static __always_inline int 47 atomic_read_acquire(const atomic_t *v) 48 { 49 instrument_atomic_read(v, sizeof(*v)); 50 return raw_atomic_read_acquire(v); 51 } 52 53 /** 54 * atomic_set() - atomic set with relaxed ordering 55 * @v: pointer to atomic_t 56 * @i: int value to assign 57 * 58 * Atomically sets @v to @i with relaxed ordering. 59 * 60 * Unsafe to use in noinstr code; use raw_atomic_set() there. 61 * 62 * Return: Nothing. 63 */ 64 static __always_inline void 65 atomic_set(atomic_t *v, int i) 66 { 67 instrument_atomic_write(v, sizeof(*v)); 68 raw_atomic_set(v, i); 69 } 70 71 /** 72 * atomic_set_release() - atomic set with release ordering 73 * @v: pointer to atomic_t 74 * @i: int value to assign 75 * 76 * Atomically sets @v to @i with release ordering. 77 * 78 * Unsafe to use in noinstr code; use raw_atomic_set_release() there. 79 * 80 * Return: Nothing. 81 */ 82 static __always_inline void 83 atomic_set_release(atomic_t *v, int i) 84 { 85 kcsan_release(); 86 instrument_atomic_write(v, sizeof(*v)); 87 raw_atomic_set_release(v, i); 88 } 89 90 /** 91 * atomic_add() - atomic add with relaxed ordering 92 * @i: int value to add 93 * @v: pointer to atomic_t 94 * 95 * Atomically updates @v to (@v + @i) with relaxed ordering. 96 * 97 * Unsafe to use in noinstr code; use raw_atomic_add() there. 98 * 99 * Return: Nothing. 100 */ 101 static __always_inline void 102 atomic_add(int i, atomic_t *v) 103 { 104 instrument_atomic_read_write(v, sizeof(*v)); 105 raw_atomic_add(i, v); 106 } 107 108 /** 109 * atomic_add_return() - atomic add with full ordering 110 * @i: int value to add 111 * @v: pointer to atomic_t 112 * 113 * Atomically updates @v to (@v + @i) with full ordering. 114 * 115 * Unsafe to use in noinstr code; use raw_atomic_add_return() there. 116 * 117 * Return: The updated value of @v. 118 */ 119 static __always_inline int 120 atomic_add_return(int i, atomic_t *v) 121 { 122 kcsan_mb(); 123 instrument_atomic_read_write(v, sizeof(*v)); 124 return raw_atomic_add_return(i, v); 125 } 126 127 /** 128 * atomic_add_return_acquire() - atomic add with acquire ordering 129 * @i: int value to add 130 * @v: pointer to atomic_t 131 * 132 * Atomically updates @v to (@v + @i) with acquire ordering. 133 * 134 * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there. 135 * 136 * Return: The updated value of @v. 137 */ 138 static __always_inline int 139 atomic_add_return_acquire(int i, atomic_t *v) 140 { 141 instrument_atomic_read_write(v, sizeof(*v)); 142 return raw_atomic_add_return_acquire(i, v); 143 } 144 145 /** 146 * atomic_add_return_release() - atomic add with release ordering 147 * @i: int value to add 148 * @v: pointer to atomic_t 149 * 150 * Atomically updates @v to (@v + @i) with release ordering. 151 * 152 * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there. 153 * 154 * Return: The updated value of @v. 155 */ 156 static __always_inline int 157 atomic_add_return_release(int i, atomic_t *v) 158 { 159 kcsan_release(); 160 instrument_atomic_read_write(v, sizeof(*v)); 161 return raw_atomic_add_return_release(i, v); 162 } 163 164 /** 165 * atomic_add_return_relaxed() - atomic add with relaxed ordering 166 * @i: int value to add 167 * @v: pointer to atomic_t 168 * 169 * Atomically updates @v to (@v + @i) with relaxed ordering. 170 * 171 * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there. 172 * 173 * Return: The updated value of @v. 174 */ 175 static __always_inline int 176 atomic_add_return_relaxed(int i, atomic_t *v) 177 { 178 instrument_atomic_read_write(v, sizeof(*v)); 179 return raw_atomic_add_return_relaxed(i, v); 180 } 181 182 /** 183 * atomic_fetch_add() - atomic add with full ordering 184 * @i: int value to add 185 * @v: pointer to atomic_t 186 * 187 * Atomically updates @v to (@v + @i) with full ordering. 188 * 189 * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there. 190 * 191 * Return: The original value of @v. 192 */ 193 static __always_inline int 194 atomic_fetch_add(int i, atomic_t *v) 195 { 196 kcsan_mb(); 197 instrument_atomic_read_write(v, sizeof(*v)); 198 return raw_atomic_fetch_add(i, v); 199 } 200 201 /** 202 * atomic_fetch_add_acquire() - atomic add with acquire ordering 203 * @i: int value to add 204 * @v: pointer to atomic_t 205 * 206 * Atomically updates @v to (@v + @i) with acquire ordering. 207 * 208 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there. 209 * 210 * Return: The original value of @v. 211 */ 212 static __always_inline int 213 atomic_fetch_add_acquire(int i, atomic_t *v) 214 { 215 instrument_atomic_read_write(v, sizeof(*v)); 216 return raw_atomic_fetch_add_acquire(i, v); 217 } 218 219 /** 220 * atomic_fetch_add_release() - atomic add with release ordering 221 * @i: int value to add 222 * @v: pointer to atomic_t 223 * 224 * Atomically updates @v to (@v + @i) with release ordering. 225 * 226 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there. 227 * 228 * Return: The original value of @v. 229 */ 230 static __always_inline int 231 atomic_fetch_add_release(int i, atomic_t *v) 232 { 233 kcsan_release(); 234 instrument_atomic_read_write(v, sizeof(*v)); 235 return raw_atomic_fetch_add_release(i, v); 236 } 237 238 /** 239 * atomic_fetch_add_relaxed() - atomic add with relaxed ordering 240 * @i: int value to add 241 * @v: pointer to atomic_t 242 * 243 * Atomically updates @v to (@v + @i) with relaxed ordering. 244 * 245 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there. 246 * 247 * Return: The original value of @v. 248 */ 249 static __always_inline int 250 atomic_fetch_add_relaxed(int i, atomic_t *v) 251 { 252 instrument_atomic_read_write(v, sizeof(*v)); 253 return raw_atomic_fetch_add_relaxed(i, v); 254 } 255 256 /** 257 * atomic_sub() - atomic subtract with relaxed ordering 258 * @i: int value to subtract 259 * @v: pointer to atomic_t 260 * 261 * Atomically updates @v to (@v - @i) with relaxed ordering. 262 * 263 * Unsafe to use in noinstr code; use raw_atomic_sub() there. 264 * 265 * Return: Nothing. 266 */ 267 static __always_inline void 268 atomic_sub(int i, atomic_t *v) 269 { 270 instrument_atomic_read_write(v, sizeof(*v)); 271 raw_atomic_sub(i, v); 272 } 273 274 /** 275 * atomic_sub_return() - atomic subtract with full ordering 276 * @i: int value to subtract 277 * @v: pointer to atomic_t 278 * 279 * Atomically updates @v to (@v - @i) with full ordering. 280 * 281 * Unsafe to use in noinstr code; use raw_atomic_sub_return() there. 282 * 283 * Return: The updated value of @v. 284 */ 285 static __always_inline int 286 atomic_sub_return(int i, atomic_t *v) 287 { 288 kcsan_mb(); 289 instrument_atomic_read_write(v, sizeof(*v)); 290 return raw_atomic_sub_return(i, v); 291 } 292 293 /** 294 * atomic_sub_return_acquire() - atomic subtract with acquire ordering 295 * @i: int value to subtract 296 * @v: pointer to atomic_t 297 * 298 * Atomically updates @v to (@v - @i) with acquire ordering. 299 * 300 * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there. 301 * 302 * Return: The updated value of @v. 303 */ 304 static __always_inline int 305 atomic_sub_return_acquire(int i, atomic_t *v) 306 { 307 instrument_atomic_read_write(v, sizeof(*v)); 308 return raw_atomic_sub_return_acquire(i, v); 309 } 310 311 /** 312 * atomic_sub_return_release() - atomic subtract with release ordering 313 * @i: int value to subtract 314 * @v: pointer to atomic_t 315 * 316 * Atomically updates @v to (@v - @i) with release ordering. 317 * 318 * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there. 319 * 320 * Return: The updated value of @v. 321 */ 322 static __always_inline int 323 atomic_sub_return_release(int i, atomic_t *v) 324 { 325 kcsan_release(); 326 instrument_atomic_read_write(v, sizeof(*v)); 327 return raw_atomic_sub_return_release(i, v); 328 } 329 330 /** 331 * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering 332 * @i: int value to subtract 333 * @v: pointer to atomic_t 334 * 335 * Atomically updates @v to (@v - @i) with relaxed ordering. 336 * 337 * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there. 338 * 339 * Return: The updated value of @v. 340 */ 341 static __always_inline int 342 atomic_sub_return_relaxed(int i, atomic_t *v) 343 { 344 instrument_atomic_read_write(v, sizeof(*v)); 345 return raw_atomic_sub_return_relaxed(i, v); 346 } 347 348 /** 349 * atomic_fetch_sub() - atomic subtract with full ordering 350 * @i: int value to subtract 351 * @v: pointer to atomic_t 352 * 353 * Atomically updates @v to (@v - @i) with full ordering. 354 * 355 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there. 356 * 357 * Return: The original value of @v. 358 */ 359 static __always_inline int 360 atomic_fetch_sub(int i, atomic_t *v) 361 { 362 kcsan_mb(); 363 instrument_atomic_read_write(v, sizeof(*v)); 364 return raw_atomic_fetch_sub(i, v); 365 } 366 367 /** 368 * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering 369 * @i: int value to subtract 370 * @v: pointer to atomic_t 371 * 372 * Atomically updates @v to (@v - @i) with acquire ordering. 373 * 374 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there. 375 * 376 * Return: The original value of @v. 377 */ 378 static __always_inline int 379 atomic_fetch_sub_acquire(int i, atomic_t *v) 380 { 381 instrument_atomic_read_write(v, sizeof(*v)); 382 return raw_atomic_fetch_sub_acquire(i, v); 383 } 384 385 /** 386 * atomic_fetch_sub_release() - atomic subtract with release ordering 387 * @i: int value to subtract 388 * @v: pointer to atomic_t 389 * 390 * Atomically updates @v to (@v - @i) with release ordering. 391 * 392 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there. 393 * 394 * Return: The original value of @v. 395 */ 396 static __always_inline int 397 atomic_fetch_sub_release(int i, atomic_t *v) 398 { 399 kcsan_release(); 400 instrument_atomic_read_write(v, sizeof(*v)); 401 return raw_atomic_fetch_sub_release(i, v); 402 } 403 404 /** 405 * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering 406 * @i: int value to subtract 407 * @v: pointer to atomic_t 408 * 409 * Atomically updates @v to (@v - @i) with relaxed ordering. 410 * 411 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there. 412 * 413 * Return: The original value of @v. 414 */ 415 static __always_inline int 416 atomic_fetch_sub_relaxed(int i, atomic_t *v) 417 { 418 instrument_atomic_read_write(v, sizeof(*v)); 419 return raw_atomic_fetch_sub_relaxed(i, v); 420 } 421 422 /** 423 * atomic_inc() - atomic increment with relaxed ordering 424 * @v: pointer to atomic_t 425 * 426 * Atomically updates @v to (@v + 1) with relaxed ordering. 427 * 428 * Unsafe to use in noinstr code; use raw_atomic_inc() there. 429 * 430 * Return: Nothing. 431 */ 432 static __always_inline void 433 atomic_inc(atomic_t *v) 434 { 435 instrument_atomic_read_write(v, sizeof(*v)); 436 raw_atomic_inc(v); 437 } 438 439 /** 440 * atomic_inc_return() - atomic increment with full ordering 441 * @v: pointer to atomic_t 442 * 443 * Atomically updates @v to (@v + 1) with full ordering. 444 * 445 * Unsafe to use in noinstr code; use raw_atomic_inc_return() there. 446 * 447 * Return: The updated value of @v. 448 */ 449 static __always_inline int 450 atomic_inc_return(atomic_t *v) 451 { 452 kcsan_mb(); 453 instrument_atomic_read_write(v, sizeof(*v)); 454 return raw_atomic_inc_return(v); 455 } 456 457 /** 458 * atomic_inc_return_acquire() - atomic increment with acquire ordering 459 * @v: pointer to atomic_t 460 * 461 * Atomically updates @v to (@v + 1) with acquire ordering. 462 * 463 * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there. 464 * 465 * Return: The updated value of @v. 466 */ 467 static __always_inline int 468 atomic_inc_return_acquire(atomic_t *v) 469 { 470 instrument_atomic_read_write(v, sizeof(*v)); 471 return raw_atomic_inc_return_acquire(v); 472 } 473 474 /** 475 * atomic_inc_return_release() - atomic increment with release ordering 476 * @v: pointer to atomic_t 477 * 478 * Atomically updates @v to (@v + 1) with release ordering. 479 * 480 * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there. 481 * 482 * Return: The updated value of @v. 483 */ 484 static __always_inline int 485 atomic_inc_return_release(atomic_t *v) 486 { 487 kcsan_release(); 488 instrument_atomic_read_write(v, sizeof(*v)); 489 return raw_atomic_inc_return_release(v); 490 } 491 492 /** 493 * atomic_inc_return_relaxed() - atomic increment with relaxed ordering 494 * @v: pointer to atomic_t 495 * 496 * Atomically updates @v to (@v + 1) with relaxed ordering. 497 * 498 * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there. 499 * 500 * Return: The updated value of @v. 501 */ 502 static __always_inline int 503 atomic_inc_return_relaxed(atomic_t *v) 504 { 505 instrument_atomic_read_write(v, sizeof(*v)); 506 return raw_atomic_inc_return_relaxed(v); 507 } 508 509 /** 510 * atomic_fetch_inc() - atomic increment with full ordering 511 * @v: pointer to atomic_t 512 * 513 * Atomically updates @v to (@v + 1) with full ordering. 514 * 515 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there. 516 * 517 * Return: The original value of @v. 518 */ 519 static __always_inline int 520 atomic_fetch_inc(atomic_t *v) 521 { 522 kcsan_mb(); 523 instrument_atomic_read_write(v, sizeof(*v)); 524 return raw_atomic_fetch_inc(v); 525 } 526 527 /** 528 * atomic_fetch_inc_acquire() - atomic increment with acquire ordering 529 * @v: pointer to atomic_t 530 * 531 * Atomically updates @v to (@v + 1) with acquire ordering. 532 * 533 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there. 534 * 535 * Return: The original value of @v. 536 */ 537 static __always_inline int 538 atomic_fetch_inc_acquire(atomic_t *v) 539 { 540 instrument_atomic_read_write(v, sizeof(*v)); 541 return raw_atomic_fetch_inc_acquire(v); 542 } 543 544 /** 545 * atomic_fetch_inc_release() - atomic increment with release ordering 546 * @v: pointer to atomic_t 547 * 548 * Atomically updates @v to (@v + 1) with release ordering. 549 * 550 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there. 551 * 552 * Return: The original value of @v. 553 */ 554 static __always_inline int 555 atomic_fetch_inc_release(atomic_t *v) 556 { 557 kcsan_release(); 558 instrument_atomic_read_write(v, sizeof(*v)); 559 return raw_atomic_fetch_inc_release(v); 560 } 561 562 /** 563 * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering 564 * @v: pointer to atomic_t 565 * 566 * Atomically updates @v to (@v + 1) with relaxed ordering. 567 * 568 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there. 569 * 570 * Return: The original value of @v. 571 */ 572 static __always_inline int 573 atomic_fetch_inc_relaxed(atomic_t *v) 574 { 575 instrument_atomic_read_write(v, sizeof(*v)); 576 return raw_atomic_fetch_inc_relaxed(v); 577 } 578 579 /** 580 * atomic_dec() - atomic decrement with relaxed ordering 581 * @v: pointer to atomic_t 582 * 583 * Atomically updates @v to (@v - 1) with relaxed ordering. 584 * 585 * Unsafe to use in noinstr code; use raw_atomic_dec() there. 586 * 587 * Return: Nothing. 588 */ 589 static __always_inline void 590 atomic_dec(atomic_t *v) 591 { 592 instrument_atomic_read_write(v, sizeof(*v)); 593 raw_atomic_dec(v); 594 } 595 596 /** 597 * atomic_dec_return() - atomic decrement with full ordering 598 * @v: pointer to atomic_t 599 * 600 * Atomically updates @v to (@v - 1) with full ordering. 601 * 602 * Unsafe to use in noinstr code; use raw_atomic_dec_return() there. 603 * 604 * Return: The updated value of @v. 605 */ 606 static __always_inline int 607 atomic_dec_return(atomic_t *v) 608 { 609 kcsan_mb(); 610 instrument_atomic_read_write(v, sizeof(*v)); 611 return raw_atomic_dec_return(v); 612 } 613 614 /** 615 * atomic_dec_return_acquire() - atomic decrement with acquire ordering 616 * @v: pointer to atomic_t 617 * 618 * Atomically updates @v to (@v - 1) with acquire ordering. 619 * 620 * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there. 621 * 622 * Return: The updated value of @v. 623 */ 624 static __always_inline int 625 atomic_dec_return_acquire(atomic_t *v) 626 { 627 instrument_atomic_read_write(v, sizeof(*v)); 628 return raw_atomic_dec_return_acquire(v); 629 } 630 631 /** 632 * atomic_dec_return_release() - atomic decrement with release ordering 633 * @v: pointer to atomic_t 634 * 635 * Atomically updates @v to (@v - 1) with release ordering. 636 * 637 * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there. 638 * 639 * Return: The updated value of @v. 640 */ 641 static __always_inline int 642 atomic_dec_return_release(atomic_t *v) 643 { 644 kcsan_release(); 645 instrument_atomic_read_write(v, sizeof(*v)); 646 return raw_atomic_dec_return_release(v); 647 } 648 649 /** 650 * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering 651 * @v: pointer to atomic_t 652 * 653 * Atomically updates @v to (@v - 1) with relaxed ordering. 654 * 655 * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there. 656 * 657 * Return: The updated value of @v. 658 */ 659 static __always_inline int 660 atomic_dec_return_relaxed(atomic_t *v) 661 { 662 instrument_atomic_read_write(v, sizeof(*v)); 663 return raw_atomic_dec_return_relaxed(v); 664 } 665 666 /** 667 * atomic_fetch_dec() - atomic decrement with full ordering 668 * @v: pointer to atomic_t 669 * 670 * Atomically updates @v to (@v - 1) with full ordering. 671 * 672 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there. 673 * 674 * Return: The original value of @v. 675 */ 676 static __always_inline int 677 atomic_fetch_dec(atomic_t *v) 678 { 679 kcsan_mb(); 680 instrument_atomic_read_write(v, sizeof(*v)); 681 return raw_atomic_fetch_dec(v); 682 } 683 684 /** 685 * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering 686 * @v: pointer to atomic_t 687 * 688 * Atomically updates @v to (@v - 1) with acquire ordering. 689 * 690 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there. 691 * 692 * Return: The original value of @v. 693 */ 694 static __always_inline int 695 atomic_fetch_dec_acquire(atomic_t *v) 696 { 697 instrument_atomic_read_write(v, sizeof(*v)); 698 return raw_atomic_fetch_dec_acquire(v); 699 } 700 701 /** 702 * atomic_fetch_dec_release() - atomic decrement with release ordering 703 * @v: pointer to atomic_t 704 * 705 * Atomically updates @v to (@v - 1) with release ordering. 706 * 707 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there. 708 * 709 * Return: The original value of @v. 710 */ 711 static __always_inline int 712 atomic_fetch_dec_release(atomic_t *v) 713 { 714 kcsan_release(); 715 instrument_atomic_read_write(v, sizeof(*v)); 716 return raw_atomic_fetch_dec_release(v); 717 } 718 719 /** 720 * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering 721 * @v: pointer to atomic_t 722 * 723 * Atomically updates @v to (@v - 1) with relaxed ordering. 724 * 725 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there. 726 * 727 * Return: The original value of @v. 728 */ 729 static __always_inline int 730 atomic_fetch_dec_relaxed(atomic_t *v) 731 { 732 instrument_atomic_read_write(v, sizeof(*v)); 733 return raw_atomic_fetch_dec_relaxed(v); 734 } 735 736 /** 737 * atomic_and() - atomic bitwise AND with relaxed ordering 738 * @i: int value 739 * @v: pointer to atomic_t 740 * 741 * Atomically updates @v to (@v & @i) with relaxed ordering. 742 * 743 * Unsafe to use in noinstr code; use raw_atomic_and() there. 744 * 745 * Return: Nothing. 746 */ 747 static __always_inline void 748 atomic_and(int i, atomic_t *v) 749 { 750 instrument_atomic_read_write(v, sizeof(*v)); 751 raw_atomic_and(i, v); 752 } 753 754 /** 755 * atomic_fetch_and() - atomic bitwise AND with full ordering 756 * @i: int value 757 * @v: pointer to atomic_t 758 * 759 * Atomically updates @v to (@v & @i) with full ordering. 760 * 761 * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there. 762 * 763 * Return: The original value of @v. 764 */ 765 static __always_inline int 766 atomic_fetch_and(int i, atomic_t *v) 767 { 768 kcsan_mb(); 769 instrument_atomic_read_write(v, sizeof(*v)); 770 return raw_atomic_fetch_and(i, v); 771 } 772 773 /** 774 * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering 775 * @i: int value 776 * @v: pointer to atomic_t 777 * 778 * Atomically updates @v to (@v & @i) with acquire ordering. 779 * 780 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there. 781 * 782 * Return: The original value of @v. 783 */ 784 static __always_inline int 785 atomic_fetch_and_acquire(int i, atomic_t *v) 786 { 787 instrument_atomic_read_write(v, sizeof(*v)); 788 return raw_atomic_fetch_and_acquire(i, v); 789 } 790 791 /** 792 * atomic_fetch_and_release() - atomic bitwise AND with release ordering 793 * @i: int value 794 * @v: pointer to atomic_t 795 * 796 * Atomically updates @v to (@v & @i) with release ordering. 797 * 798 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there. 799 * 800 * Return: The original value of @v. 801 */ 802 static __always_inline int 803 atomic_fetch_and_release(int i, atomic_t *v) 804 { 805 kcsan_release(); 806 instrument_atomic_read_write(v, sizeof(*v)); 807 return raw_atomic_fetch_and_release(i, v); 808 } 809 810 /** 811 * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering 812 * @i: int value 813 * @v: pointer to atomic_t 814 * 815 * Atomically updates @v to (@v & @i) with relaxed ordering. 816 * 817 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there. 818 * 819 * Return: The original value of @v. 820 */ 821 static __always_inline int 822 atomic_fetch_and_relaxed(int i, atomic_t *v) 823 { 824 instrument_atomic_read_write(v, sizeof(*v)); 825 return raw_atomic_fetch_and_relaxed(i, v); 826 } 827 828 /** 829 * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering 830 * @i: int value 831 * @v: pointer to atomic_t 832 * 833 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 834 * 835 * Unsafe to use in noinstr code; use raw_atomic_andnot() there. 836 * 837 * Return: Nothing. 838 */ 839 static __always_inline void 840 atomic_andnot(int i, atomic_t *v) 841 { 842 instrument_atomic_read_write(v, sizeof(*v)); 843 raw_atomic_andnot(i, v); 844 } 845 846 /** 847 * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering 848 * @i: int value 849 * @v: pointer to atomic_t 850 * 851 * Atomically updates @v to (@v & ~@i) with full ordering. 852 * 853 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there. 854 * 855 * Return: The original value of @v. 856 */ 857 static __always_inline int 858 atomic_fetch_andnot(int i, atomic_t *v) 859 { 860 kcsan_mb(); 861 instrument_atomic_read_write(v, sizeof(*v)); 862 return raw_atomic_fetch_andnot(i, v); 863 } 864 865 /** 866 * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering 867 * @i: int value 868 * @v: pointer to atomic_t 869 * 870 * Atomically updates @v to (@v & ~@i) with acquire ordering. 871 * 872 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there. 873 * 874 * Return: The original value of @v. 875 */ 876 static __always_inline int 877 atomic_fetch_andnot_acquire(int i, atomic_t *v) 878 { 879 instrument_atomic_read_write(v, sizeof(*v)); 880 return raw_atomic_fetch_andnot_acquire(i, v); 881 } 882 883 /** 884 * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering 885 * @i: int value 886 * @v: pointer to atomic_t 887 * 888 * Atomically updates @v to (@v & ~@i) with release ordering. 889 * 890 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there. 891 * 892 * Return: The original value of @v. 893 */ 894 static __always_inline int 895 atomic_fetch_andnot_release(int i, atomic_t *v) 896 { 897 kcsan_release(); 898 instrument_atomic_read_write(v, sizeof(*v)); 899 return raw_atomic_fetch_andnot_release(i, v); 900 } 901 902 /** 903 * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering 904 * @i: int value 905 * @v: pointer to atomic_t 906 * 907 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 908 * 909 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there. 910 * 911 * Return: The original value of @v. 912 */ 913 static __always_inline int 914 atomic_fetch_andnot_relaxed(int i, atomic_t *v) 915 { 916 instrument_atomic_read_write(v, sizeof(*v)); 917 return raw_atomic_fetch_andnot_relaxed(i, v); 918 } 919 920 /** 921 * atomic_or() - atomic bitwise OR with relaxed ordering 922 * @i: int value 923 * @v: pointer to atomic_t 924 * 925 * Atomically updates @v to (@v | @i) with relaxed ordering. 926 * 927 * Unsafe to use in noinstr code; use raw_atomic_or() there. 928 * 929 * Return: Nothing. 930 */ 931 static __always_inline void 932 atomic_or(int i, atomic_t *v) 933 { 934 instrument_atomic_read_write(v, sizeof(*v)); 935 raw_atomic_or(i, v); 936 } 937 938 /** 939 * atomic_fetch_or() - atomic bitwise OR with full ordering 940 * @i: int value 941 * @v: pointer to atomic_t 942 * 943 * Atomically updates @v to (@v | @i) with full ordering. 944 * 945 * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there. 946 * 947 * Return: The original value of @v. 948 */ 949 static __always_inline int 950 atomic_fetch_or(int i, atomic_t *v) 951 { 952 kcsan_mb(); 953 instrument_atomic_read_write(v, sizeof(*v)); 954 return raw_atomic_fetch_or(i, v); 955 } 956 957 /** 958 * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering 959 * @i: int value 960 * @v: pointer to atomic_t 961 * 962 * Atomically updates @v to (@v | @i) with acquire ordering. 963 * 964 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there. 965 * 966 * Return: The original value of @v. 967 */ 968 static __always_inline int 969 atomic_fetch_or_acquire(int i, atomic_t *v) 970 { 971 instrument_atomic_read_write(v, sizeof(*v)); 972 return raw_atomic_fetch_or_acquire(i, v); 973 } 974 975 /** 976 * atomic_fetch_or_release() - atomic bitwise OR with release ordering 977 * @i: int value 978 * @v: pointer to atomic_t 979 * 980 * Atomically updates @v to (@v | @i) with release ordering. 981 * 982 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there. 983 * 984 * Return: The original value of @v. 985 */ 986 static __always_inline int 987 atomic_fetch_or_release(int i, atomic_t *v) 988 { 989 kcsan_release(); 990 instrument_atomic_read_write(v, sizeof(*v)); 991 return raw_atomic_fetch_or_release(i, v); 992 } 993 994 /** 995 * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering 996 * @i: int value 997 * @v: pointer to atomic_t 998 * 999 * Atomically updates @v to (@v | @i) with relaxed ordering. 1000 * 1001 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there. 1002 * 1003 * Return: The original value of @v. 1004 */ 1005 static __always_inline int 1006 atomic_fetch_or_relaxed(int i, atomic_t *v) 1007 { 1008 instrument_atomic_read_write(v, sizeof(*v)); 1009 return raw_atomic_fetch_or_relaxed(i, v); 1010 } 1011 1012 /** 1013 * atomic_xor() - atomic bitwise XOR with relaxed ordering 1014 * @i: int value 1015 * @v: pointer to atomic_t 1016 * 1017 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 1018 * 1019 * Unsafe to use in noinstr code; use raw_atomic_xor() there. 1020 * 1021 * Return: Nothing. 1022 */ 1023 static __always_inline void 1024 atomic_xor(int i, atomic_t *v) 1025 { 1026 instrument_atomic_read_write(v, sizeof(*v)); 1027 raw_atomic_xor(i, v); 1028 } 1029 1030 /** 1031 * atomic_fetch_xor() - atomic bitwise XOR with full ordering 1032 * @i: int value 1033 * @v: pointer to atomic_t 1034 * 1035 * Atomically updates @v to (@v ^ @i) with full ordering. 1036 * 1037 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there. 1038 * 1039 * Return: The original value of @v. 1040 */ 1041 static __always_inline int 1042 atomic_fetch_xor(int i, atomic_t *v) 1043 { 1044 kcsan_mb(); 1045 instrument_atomic_read_write(v, sizeof(*v)); 1046 return raw_atomic_fetch_xor(i, v); 1047 } 1048 1049 /** 1050 * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering 1051 * @i: int value 1052 * @v: pointer to atomic_t 1053 * 1054 * Atomically updates @v to (@v ^ @i) with acquire ordering. 1055 * 1056 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there. 1057 * 1058 * Return: The original value of @v. 1059 */ 1060 static __always_inline int 1061 atomic_fetch_xor_acquire(int i, atomic_t *v) 1062 { 1063 instrument_atomic_read_write(v, sizeof(*v)); 1064 return raw_atomic_fetch_xor_acquire(i, v); 1065 } 1066 1067 /** 1068 * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering 1069 * @i: int value 1070 * @v: pointer to atomic_t 1071 * 1072 * Atomically updates @v to (@v ^ @i) with release ordering. 1073 * 1074 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there. 1075 * 1076 * Return: The original value of @v. 1077 */ 1078 static __always_inline int 1079 atomic_fetch_xor_release(int i, atomic_t *v) 1080 { 1081 kcsan_release(); 1082 instrument_atomic_read_write(v, sizeof(*v)); 1083 return raw_atomic_fetch_xor_release(i, v); 1084 } 1085 1086 /** 1087 * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering 1088 * @i: int value 1089 * @v: pointer to atomic_t 1090 * 1091 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 1092 * 1093 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there. 1094 * 1095 * Return: The original value of @v. 1096 */ 1097 static __always_inline int 1098 atomic_fetch_xor_relaxed(int i, atomic_t *v) 1099 { 1100 instrument_atomic_read_write(v, sizeof(*v)); 1101 return raw_atomic_fetch_xor_relaxed(i, v); 1102 } 1103 1104 /** 1105 * atomic_xchg() - atomic exchange with full ordering 1106 * @v: pointer to atomic_t 1107 * @new: int value to assign 1108 * 1109 * Atomically updates @v to @new with full ordering. 1110 * 1111 * Unsafe to use in noinstr code; use raw_atomic_xchg() there. 1112 * 1113 * Return: The original value of @v. 1114 */ 1115 static __always_inline int 1116 atomic_xchg(atomic_t *v, int new) 1117 { 1118 kcsan_mb(); 1119 instrument_atomic_read_write(v, sizeof(*v)); 1120 return raw_atomic_xchg(v, new); 1121 } 1122 1123 /** 1124 * atomic_xchg_acquire() - atomic exchange with acquire ordering 1125 * @v: pointer to atomic_t 1126 * @new: int value to assign 1127 * 1128 * Atomically updates @v to @new with acquire ordering. 1129 * 1130 * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there. 1131 * 1132 * Return: The original value of @v. 1133 */ 1134 static __always_inline int 1135 atomic_xchg_acquire(atomic_t *v, int new) 1136 { 1137 instrument_atomic_read_write(v, sizeof(*v)); 1138 return raw_atomic_xchg_acquire(v, new); 1139 } 1140 1141 /** 1142 * atomic_xchg_release() - atomic exchange with release ordering 1143 * @v: pointer to atomic_t 1144 * @new: int value to assign 1145 * 1146 * Atomically updates @v to @new with release ordering. 1147 * 1148 * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there. 1149 * 1150 * Return: The original value of @v. 1151 */ 1152 static __always_inline int 1153 atomic_xchg_release(atomic_t *v, int new) 1154 { 1155 kcsan_release(); 1156 instrument_atomic_read_write(v, sizeof(*v)); 1157 return raw_atomic_xchg_release(v, new); 1158 } 1159 1160 /** 1161 * atomic_xchg_relaxed() - atomic exchange with relaxed ordering 1162 * @v: pointer to atomic_t 1163 * @new: int value to assign 1164 * 1165 * Atomically updates @v to @new with relaxed ordering. 1166 * 1167 * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there. 1168 * 1169 * Return: The original value of @v. 1170 */ 1171 static __always_inline int 1172 atomic_xchg_relaxed(atomic_t *v, int new) 1173 { 1174 instrument_atomic_read_write(v, sizeof(*v)); 1175 return raw_atomic_xchg_relaxed(v, new); 1176 } 1177 1178 /** 1179 * atomic_cmpxchg() - atomic compare and exchange with full ordering 1180 * @v: pointer to atomic_t 1181 * @old: int value to compare with 1182 * @new: int value to assign 1183 * 1184 * If (@v == @old), atomically updates @v to @new with full ordering. 1185 * Otherwise, @v is not modified and relaxed ordering is provided. 1186 * 1187 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there. 1188 * 1189 * Return: The original value of @v. 1190 */ 1191 static __always_inline int 1192 atomic_cmpxchg(atomic_t *v, int old, int new) 1193 { 1194 kcsan_mb(); 1195 instrument_atomic_read_write(v, sizeof(*v)); 1196 return raw_atomic_cmpxchg(v, old, new); 1197 } 1198 1199 /** 1200 * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 1201 * @v: pointer to atomic_t 1202 * @old: int value to compare with 1203 * @new: int value to assign 1204 * 1205 * If (@v == @old), atomically updates @v to @new with acquire ordering. 1206 * Otherwise, @v is not modified and relaxed ordering is provided. 1207 * 1208 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there. 1209 * 1210 * Return: The original value of @v. 1211 */ 1212 static __always_inline int 1213 atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 1214 { 1215 instrument_atomic_read_write(v, sizeof(*v)); 1216 return raw_atomic_cmpxchg_acquire(v, old, new); 1217 } 1218 1219 /** 1220 * atomic_cmpxchg_release() - atomic compare and exchange with release ordering 1221 * @v: pointer to atomic_t 1222 * @old: int value to compare with 1223 * @new: int value to assign 1224 * 1225 * If (@v == @old), atomically updates @v to @new with release ordering. 1226 * Otherwise, @v is not modified and relaxed ordering is provided. 1227 * 1228 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there. 1229 * 1230 * Return: The original value of @v. 1231 */ 1232 static __always_inline int 1233 atomic_cmpxchg_release(atomic_t *v, int old, int new) 1234 { 1235 kcsan_release(); 1236 instrument_atomic_read_write(v, sizeof(*v)); 1237 return raw_atomic_cmpxchg_release(v, old, new); 1238 } 1239 1240 /** 1241 * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 1242 * @v: pointer to atomic_t 1243 * @old: int value to compare with 1244 * @new: int value to assign 1245 * 1246 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 1247 * Otherwise, @v is not modified and relaxed ordering is provided. 1248 * 1249 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there. 1250 * 1251 * Return: The original value of @v. 1252 */ 1253 static __always_inline int 1254 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 1255 { 1256 instrument_atomic_read_write(v, sizeof(*v)); 1257 return raw_atomic_cmpxchg_relaxed(v, old, new); 1258 } 1259 1260 /** 1261 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering 1262 * @v: pointer to atomic_t 1263 * @old: pointer to int value to compare with 1264 * @new: int value to assign 1265 * 1266 * If (@v == @old), atomically updates @v to @new with full ordering. 1267 * Otherwise, @v is not modified, @old is updated to the current value of @v, 1268 * and relaxed ordering is provided. 1269 * 1270 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there. 1271 * 1272 * Return: @true if the exchange occured, @false otherwise. 1273 */ 1274 static __always_inline bool 1275 atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1276 { 1277 kcsan_mb(); 1278 instrument_atomic_read_write(v, sizeof(*v)); 1279 instrument_atomic_read_write(old, sizeof(*old)); 1280 return raw_atomic_try_cmpxchg(v, old, new); 1281 } 1282 1283 /** 1284 * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 1285 * @v: pointer to atomic_t 1286 * @old: pointer to int value to compare with 1287 * @new: int value to assign 1288 * 1289 * If (@v == @old), atomically updates @v to @new with acquire ordering. 1290 * Otherwise, @v is not modified, @old is updated to the current value of @v, 1291 * and relaxed ordering is provided. 1292 * 1293 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there. 1294 * 1295 * Return: @true if the exchange occured, @false otherwise. 1296 */ 1297 static __always_inline bool 1298 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1299 { 1300 instrument_atomic_read_write(v, sizeof(*v)); 1301 instrument_atomic_read_write(old, sizeof(*old)); 1302 return raw_atomic_try_cmpxchg_acquire(v, old, new); 1303 } 1304 1305 /** 1306 * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering 1307 * @v: pointer to atomic_t 1308 * @old: pointer to int value to compare with 1309 * @new: int value to assign 1310 * 1311 * If (@v == @old), atomically updates @v to @new with release ordering. 1312 * Otherwise, @v is not modified, @old is updated to the current value of @v, 1313 * and relaxed ordering is provided. 1314 * 1315 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there. 1316 * 1317 * Return: @true if the exchange occured, @false otherwise. 1318 */ 1319 static __always_inline bool 1320 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1321 { 1322 kcsan_release(); 1323 instrument_atomic_read_write(v, sizeof(*v)); 1324 instrument_atomic_read_write(old, sizeof(*old)); 1325 return raw_atomic_try_cmpxchg_release(v, old, new); 1326 } 1327 1328 /** 1329 * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 1330 * @v: pointer to atomic_t 1331 * @old: pointer to int value to compare with 1332 * @new: int value to assign 1333 * 1334 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 1335 * Otherwise, @v is not modified, @old is updated to the current value of @v, 1336 * and relaxed ordering is provided. 1337 * 1338 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there. 1339 * 1340 * Return: @true if the exchange occured, @false otherwise. 1341 */ 1342 static __always_inline bool 1343 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 1344 { 1345 instrument_atomic_read_write(v, sizeof(*v)); 1346 instrument_atomic_read_write(old, sizeof(*old)); 1347 return raw_atomic_try_cmpxchg_relaxed(v, old, new); 1348 } 1349 1350 /** 1351 * atomic_sub_and_test() - atomic subtract and test if zero with full ordering 1352 * @i: int value to subtract 1353 * @v: pointer to atomic_t 1354 * 1355 * Atomically updates @v to (@v - @i) with full ordering. 1356 * 1357 * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there. 1358 * 1359 * Return: @true if the resulting value of @v is zero, @false otherwise. 1360 */ 1361 static __always_inline bool 1362 atomic_sub_and_test(int i, atomic_t *v) 1363 { 1364 kcsan_mb(); 1365 instrument_atomic_read_write(v, sizeof(*v)); 1366 return raw_atomic_sub_and_test(i, v); 1367 } 1368 1369 /** 1370 * atomic_dec_and_test() - atomic decrement and test if zero with full ordering 1371 * @v: pointer to atomic_t 1372 * 1373 * Atomically updates @v to (@v - 1) with full ordering. 1374 * 1375 * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there. 1376 * 1377 * Return: @true if the resulting value of @v is zero, @false otherwise. 1378 */ 1379 static __always_inline bool 1380 atomic_dec_and_test(atomic_t *v) 1381 { 1382 kcsan_mb(); 1383 instrument_atomic_read_write(v, sizeof(*v)); 1384 return raw_atomic_dec_and_test(v); 1385 } 1386 1387 /** 1388 * atomic_inc_and_test() - atomic increment and test if zero with full ordering 1389 * @v: pointer to atomic_t 1390 * 1391 * Atomically updates @v to (@v + 1) with full ordering. 1392 * 1393 * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there. 1394 * 1395 * Return: @true if the resulting value of @v is zero, @false otherwise. 1396 */ 1397 static __always_inline bool 1398 atomic_inc_and_test(atomic_t *v) 1399 { 1400 kcsan_mb(); 1401 instrument_atomic_read_write(v, sizeof(*v)); 1402 return raw_atomic_inc_and_test(v); 1403 } 1404 1405 /** 1406 * atomic_add_negative() - atomic add and test if negative with full ordering 1407 * @i: int value to add 1408 * @v: pointer to atomic_t 1409 * 1410 * Atomically updates @v to (@v + @i) with full ordering. 1411 * 1412 * Unsafe to use in noinstr code; use raw_atomic_add_negative() there. 1413 * 1414 * Return: @true if the resulting value of @v is negative, @false otherwise. 1415 */ 1416 static __always_inline bool 1417 atomic_add_negative(int i, atomic_t *v) 1418 { 1419 kcsan_mb(); 1420 instrument_atomic_read_write(v, sizeof(*v)); 1421 return raw_atomic_add_negative(i, v); 1422 } 1423 1424 /** 1425 * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering 1426 * @i: int value to add 1427 * @v: pointer to atomic_t 1428 * 1429 * Atomically updates @v to (@v + @i) with acquire ordering. 1430 * 1431 * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there. 1432 * 1433 * Return: @true if the resulting value of @v is negative, @false otherwise. 1434 */ 1435 static __always_inline bool 1436 atomic_add_negative_acquire(int i, atomic_t *v) 1437 { 1438 instrument_atomic_read_write(v, sizeof(*v)); 1439 return raw_atomic_add_negative_acquire(i, v); 1440 } 1441 1442 /** 1443 * atomic_add_negative_release() - atomic add and test if negative with release ordering 1444 * @i: int value to add 1445 * @v: pointer to atomic_t 1446 * 1447 * Atomically updates @v to (@v + @i) with release ordering. 1448 * 1449 * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there. 1450 * 1451 * Return: @true if the resulting value of @v is negative, @false otherwise. 1452 */ 1453 static __always_inline bool 1454 atomic_add_negative_release(int i, atomic_t *v) 1455 { 1456 kcsan_release(); 1457 instrument_atomic_read_write(v, sizeof(*v)); 1458 return raw_atomic_add_negative_release(i, v); 1459 } 1460 1461 /** 1462 * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering 1463 * @i: int value to add 1464 * @v: pointer to atomic_t 1465 * 1466 * Atomically updates @v to (@v + @i) with relaxed ordering. 1467 * 1468 * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there. 1469 * 1470 * Return: @true if the resulting value of @v is negative, @false otherwise. 1471 */ 1472 static __always_inline bool 1473 atomic_add_negative_relaxed(int i, atomic_t *v) 1474 { 1475 instrument_atomic_read_write(v, sizeof(*v)); 1476 return raw_atomic_add_negative_relaxed(i, v); 1477 } 1478 1479 /** 1480 * atomic_fetch_add_unless() - atomic add unless value with full ordering 1481 * @v: pointer to atomic_t 1482 * @a: int value to add 1483 * @u: int value to compare with 1484 * 1485 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 1486 * Otherwise, @v is not modified and relaxed ordering is provided. 1487 * 1488 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there. 1489 * 1490 * Return: The original value of @v. 1491 */ 1492 static __always_inline int 1493 atomic_fetch_add_unless(atomic_t *v, int a, int u) 1494 { 1495 kcsan_mb(); 1496 instrument_atomic_read_write(v, sizeof(*v)); 1497 return raw_atomic_fetch_add_unless(v, a, u); 1498 } 1499 1500 /** 1501 * atomic_add_unless() - atomic add unless value with full ordering 1502 * @v: pointer to atomic_t 1503 * @a: int value to add 1504 * @u: int value to compare with 1505 * 1506 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 1507 * Otherwise, @v is not modified and relaxed ordering is provided. 1508 * 1509 * Unsafe to use in noinstr code; use raw_atomic_add_unless() there. 1510 * 1511 * Return: @true if @v was updated, @false otherwise. 1512 */ 1513 static __always_inline bool 1514 atomic_add_unless(atomic_t *v, int a, int u) 1515 { 1516 kcsan_mb(); 1517 instrument_atomic_read_write(v, sizeof(*v)); 1518 return raw_atomic_add_unless(v, a, u); 1519 } 1520 1521 /** 1522 * atomic_inc_not_zero() - atomic increment unless zero with full ordering 1523 * @v: pointer to atomic_t 1524 * 1525 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. 1526 * Otherwise, @v is not modified and relaxed ordering is provided. 1527 * 1528 * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there. 1529 * 1530 * Return: @true if @v was updated, @false otherwise. 1531 */ 1532 static __always_inline bool 1533 atomic_inc_not_zero(atomic_t *v) 1534 { 1535 kcsan_mb(); 1536 instrument_atomic_read_write(v, sizeof(*v)); 1537 return raw_atomic_inc_not_zero(v); 1538 } 1539 1540 /** 1541 * atomic_inc_unless_negative() - atomic increment unless negative with full ordering 1542 * @v: pointer to atomic_t 1543 * 1544 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. 1545 * Otherwise, @v is not modified and relaxed ordering is provided. 1546 * 1547 * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there. 1548 * 1549 * Return: @true if @v was updated, @false otherwise. 1550 */ 1551 static __always_inline bool 1552 atomic_inc_unless_negative(atomic_t *v) 1553 { 1554 kcsan_mb(); 1555 instrument_atomic_read_write(v, sizeof(*v)); 1556 return raw_atomic_inc_unless_negative(v); 1557 } 1558 1559 /** 1560 * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering 1561 * @v: pointer to atomic_t 1562 * 1563 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. 1564 * Otherwise, @v is not modified and relaxed ordering is provided. 1565 * 1566 * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there. 1567 * 1568 * Return: @true if @v was updated, @false otherwise. 1569 */ 1570 static __always_inline bool 1571 atomic_dec_unless_positive(atomic_t *v) 1572 { 1573 kcsan_mb(); 1574 instrument_atomic_read_write(v, sizeof(*v)); 1575 return raw_atomic_dec_unless_positive(v); 1576 } 1577 1578 /** 1579 * atomic_dec_if_positive() - atomic decrement if positive with full ordering 1580 * @v: pointer to atomic_t 1581 * 1582 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. 1583 * Otherwise, @v is not modified and relaxed ordering is provided. 1584 * 1585 * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there. 1586 * 1587 * Return: The old value of (@v - 1), regardless of whether @v was updated. 1588 */ 1589 static __always_inline int 1590 atomic_dec_if_positive(atomic_t *v) 1591 { 1592 kcsan_mb(); 1593 instrument_atomic_read_write(v, sizeof(*v)); 1594 return raw_atomic_dec_if_positive(v); 1595 } 1596 1597 /** 1598 * atomic64_read() - atomic load with relaxed ordering 1599 * @v: pointer to atomic64_t 1600 * 1601 * Atomically loads the value of @v with relaxed ordering. 1602 * 1603 * Unsafe to use in noinstr code; use raw_atomic64_read() there. 1604 * 1605 * Return: The value loaded from @v. 1606 */ 1607 static __always_inline s64 1608 atomic64_read(const atomic64_t *v) 1609 { 1610 instrument_atomic_read(v, sizeof(*v)); 1611 return raw_atomic64_read(v); 1612 } 1613 1614 /** 1615 * atomic64_read_acquire() - atomic load with acquire ordering 1616 * @v: pointer to atomic64_t 1617 * 1618 * Atomically loads the value of @v with acquire ordering. 1619 * 1620 * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there. 1621 * 1622 * Return: The value loaded from @v. 1623 */ 1624 static __always_inline s64 1625 atomic64_read_acquire(const atomic64_t *v) 1626 { 1627 instrument_atomic_read(v, sizeof(*v)); 1628 return raw_atomic64_read_acquire(v); 1629 } 1630 1631 /** 1632 * atomic64_set() - atomic set with relaxed ordering 1633 * @v: pointer to atomic64_t 1634 * @i: s64 value to assign 1635 * 1636 * Atomically sets @v to @i with relaxed ordering. 1637 * 1638 * Unsafe to use in noinstr code; use raw_atomic64_set() there. 1639 * 1640 * Return: Nothing. 1641 */ 1642 static __always_inline void 1643 atomic64_set(atomic64_t *v, s64 i) 1644 { 1645 instrument_atomic_write(v, sizeof(*v)); 1646 raw_atomic64_set(v, i); 1647 } 1648 1649 /** 1650 * atomic64_set_release() - atomic set with release ordering 1651 * @v: pointer to atomic64_t 1652 * @i: s64 value to assign 1653 * 1654 * Atomically sets @v to @i with release ordering. 1655 * 1656 * Unsafe to use in noinstr code; use raw_atomic64_set_release() there. 1657 * 1658 * Return: Nothing. 1659 */ 1660 static __always_inline void 1661 atomic64_set_release(atomic64_t *v, s64 i) 1662 { 1663 kcsan_release(); 1664 instrument_atomic_write(v, sizeof(*v)); 1665 raw_atomic64_set_release(v, i); 1666 } 1667 1668 /** 1669 * atomic64_add() - atomic add with relaxed ordering 1670 * @i: s64 value to add 1671 * @v: pointer to atomic64_t 1672 * 1673 * Atomically updates @v to (@v + @i) with relaxed ordering. 1674 * 1675 * Unsafe to use in noinstr code; use raw_atomic64_add() there. 1676 * 1677 * Return: Nothing. 1678 */ 1679 static __always_inline void 1680 atomic64_add(s64 i, atomic64_t *v) 1681 { 1682 instrument_atomic_read_write(v, sizeof(*v)); 1683 raw_atomic64_add(i, v); 1684 } 1685 1686 /** 1687 * atomic64_add_return() - atomic add with full ordering 1688 * @i: s64 value to add 1689 * @v: pointer to atomic64_t 1690 * 1691 * Atomically updates @v to (@v + @i) with full ordering. 1692 * 1693 * Unsafe to use in noinstr code; use raw_atomic64_add_return() there. 1694 * 1695 * Return: The updated value of @v. 1696 */ 1697 static __always_inline s64 1698 atomic64_add_return(s64 i, atomic64_t *v) 1699 { 1700 kcsan_mb(); 1701 instrument_atomic_read_write(v, sizeof(*v)); 1702 return raw_atomic64_add_return(i, v); 1703 } 1704 1705 /** 1706 * atomic64_add_return_acquire() - atomic add with acquire ordering 1707 * @i: s64 value to add 1708 * @v: pointer to atomic64_t 1709 * 1710 * Atomically updates @v to (@v + @i) with acquire ordering. 1711 * 1712 * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there. 1713 * 1714 * Return: The updated value of @v. 1715 */ 1716 static __always_inline s64 1717 atomic64_add_return_acquire(s64 i, atomic64_t *v) 1718 { 1719 instrument_atomic_read_write(v, sizeof(*v)); 1720 return raw_atomic64_add_return_acquire(i, v); 1721 } 1722 1723 /** 1724 * atomic64_add_return_release() - atomic add with release ordering 1725 * @i: s64 value to add 1726 * @v: pointer to atomic64_t 1727 * 1728 * Atomically updates @v to (@v + @i) with release ordering. 1729 * 1730 * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there. 1731 * 1732 * Return: The updated value of @v. 1733 */ 1734 static __always_inline s64 1735 atomic64_add_return_release(s64 i, atomic64_t *v) 1736 { 1737 kcsan_release(); 1738 instrument_atomic_read_write(v, sizeof(*v)); 1739 return raw_atomic64_add_return_release(i, v); 1740 } 1741 1742 /** 1743 * atomic64_add_return_relaxed() - atomic add with relaxed ordering 1744 * @i: s64 value to add 1745 * @v: pointer to atomic64_t 1746 * 1747 * Atomically updates @v to (@v + @i) with relaxed ordering. 1748 * 1749 * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there. 1750 * 1751 * Return: The updated value of @v. 1752 */ 1753 static __always_inline s64 1754 atomic64_add_return_relaxed(s64 i, atomic64_t *v) 1755 { 1756 instrument_atomic_read_write(v, sizeof(*v)); 1757 return raw_atomic64_add_return_relaxed(i, v); 1758 } 1759 1760 /** 1761 * atomic64_fetch_add() - atomic add with full ordering 1762 * @i: s64 value to add 1763 * @v: pointer to atomic64_t 1764 * 1765 * Atomically updates @v to (@v + @i) with full ordering. 1766 * 1767 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there. 1768 * 1769 * Return: The original value of @v. 1770 */ 1771 static __always_inline s64 1772 atomic64_fetch_add(s64 i, atomic64_t *v) 1773 { 1774 kcsan_mb(); 1775 instrument_atomic_read_write(v, sizeof(*v)); 1776 return raw_atomic64_fetch_add(i, v); 1777 } 1778 1779 /** 1780 * atomic64_fetch_add_acquire() - atomic add with acquire ordering 1781 * @i: s64 value to add 1782 * @v: pointer to atomic64_t 1783 * 1784 * Atomically updates @v to (@v + @i) with acquire ordering. 1785 * 1786 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there. 1787 * 1788 * Return: The original value of @v. 1789 */ 1790 static __always_inline s64 1791 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 1792 { 1793 instrument_atomic_read_write(v, sizeof(*v)); 1794 return raw_atomic64_fetch_add_acquire(i, v); 1795 } 1796 1797 /** 1798 * atomic64_fetch_add_release() - atomic add with release ordering 1799 * @i: s64 value to add 1800 * @v: pointer to atomic64_t 1801 * 1802 * Atomically updates @v to (@v + @i) with release ordering. 1803 * 1804 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there. 1805 * 1806 * Return: The original value of @v. 1807 */ 1808 static __always_inline s64 1809 atomic64_fetch_add_release(s64 i, atomic64_t *v) 1810 { 1811 kcsan_release(); 1812 instrument_atomic_read_write(v, sizeof(*v)); 1813 return raw_atomic64_fetch_add_release(i, v); 1814 } 1815 1816 /** 1817 * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering 1818 * @i: s64 value to add 1819 * @v: pointer to atomic64_t 1820 * 1821 * Atomically updates @v to (@v + @i) with relaxed ordering. 1822 * 1823 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there. 1824 * 1825 * Return: The original value of @v. 1826 */ 1827 static __always_inline s64 1828 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 1829 { 1830 instrument_atomic_read_write(v, sizeof(*v)); 1831 return raw_atomic64_fetch_add_relaxed(i, v); 1832 } 1833 1834 /** 1835 * atomic64_sub() - atomic subtract with relaxed ordering 1836 * @i: s64 value to subtract 1837 * @v: pointer to atomic64_t 1838 * 1839 * Atomically updates @v to (@v - @i) with relaxed ordering. 1840 * 1841 * Unsafe to use in noinstr code; use raw_atomic64_sub() there. 1842 * 1843 * Return: Nothing. 1844 */ 1845 static __always_inline void 1846 atomic64_sub(s64 i, atomic64_t *v) 1847 { 1848 instrument_atomic_read_write(v, sizeof(*v)); 1849 raw_atomic64_sub(i, v); 1850 } 1851 1852 /** 1853 * atomic64_sub_return() - atomic subtract with full ordering 1854 * @i: s64 value to subtract 1855 * @v: pointer to atomic64_t 1856 * 1857 * Atomically updates @v to (@v - @i) with full ordering. 1858 * 1859 * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there. 1860 * 1861 * Return: The updated value of @v. 1862 */ 1863 static __always_inline s64 1864 atomic64_sub_return(s64 i, atomic64_t *v) 1865 { 1866 kcsan_mb(); 1867 instrument_atomic_read_write(v, sizeof(*v)); 1868 return raw_atomic64_sub_return(i, v); 1869 } 1870 1871 /** 1872 * atomic64_sub_return_acquire() - atomic subtract with acquire ordering 1873 * @i: s64 value to subtract 1874 * @v: pointer to atomic64_t 1875 * 1876 * Atomically updates @v to (@v - @i) with acquire ordering. 1877 * 1878 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there. 1879 * 1880 * Return: The updated value of @v. 1881 */ 1882 static __always_inline s64 1883 atomic64_sub_return_acquire(s64 i, atomic64_t *v) 1884 { 1885 instrument_atomic_read_write(v, sizeof(*v)); 1886 return raw_atomic64_sub_return_acquire(i, v); 1887 } 1888 1889 /** 1890 * atomic64_sub_return_release() - atomic subtract with release ordering 1891 * @i: s64 value to subtract 1892 * @v: pointer to atomic64_t 1893 * 1894 * Atomically updates @v to (@v - @i) with release ordering. 1895 * 1896 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there. 1897 * 1898 * Return: The updated value of @v. 1899 */ 1900 static __always_inline s64 1901 atomic64_sub_return_release(s64 i, atomic64_t *v) 1902 { 1903 kcsan_release(); 1904 instrument_atomic_read_write(v, sizeof(*v)); 1905 return raw_atomic64_sub_return_release(i, v); 1906 } 1907 1908 /** 1909 * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering 1910 * @i: s64 value to subtract 1911 * @v: pointer to atomic64_t 1912 * 1913 * Atomically updates @v to (@v - @i) with relaxed ordering. 1914 * 1915 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there. 1916 * 1917 * Return: The updated value of @v. 1918 */ 1919 static __always_inline s64 1920 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 1921 { 1922 instrument_atomic_read_write(v, sizeof(*v)); 1923 return raw_atomic64_sub_return_relaxed(i, v); 1924 } 1925 1926 /** 1927 * atomic64_fetch_sub() - atomic subtract with full ordering 1928 * @i: s64 value to subtract 1929 * @v: pointer to atomic64_t 1930 * 1931 * Atomically updates @v to (@v - @i) with full ordering. 1932 * 1933 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there. 1934 * 1935 * Return: The original value of @v. 1936 */ 1937 static __always_inline s64 1938 atomic64_fetch_sub(s64 i, atomic64_t *v) 1939 { 1940 kcsan_mb(); 1941 instrument_atomic_read_write(v, sizeof(*v)); 1942 return raw_atomic64_fetch_sub(i, v); 1943 } 1944 1945 /** 1946 * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering 1947 * @i: s64 value to subtract 1948 * @v: pointer to atomic64_t 1949 * 1950 * Atomically updates @v to (@v - @i) with acquire ordering. 1951 * 1952 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there. 1953 * 1954 * Return: The original value of @v. 1955 */ 1956 static __always_inline s64 1957 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 1958 { 1959 instrument_atomic_read_write(v, sizeof(*v)); 1960 return raw_atomic64_fetch_sub_acquire(i, v); 1961 } 1962 1963 /** 1964 * atomic64_fetch_sub_release() - atomic subtract with release ordering 1965 * @i: s64 value to subtract 1966 * @v: pointer to atomic64_t 1967 * 1968 * Atomically updates @v to (@v - @i) with release ordering. 1969 * 1970 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there. 1971 * 1972 * Return: The original value of @v. 1973 */ 1974 static __always_inline s64 1975 atomic64_fetch_sub_release(s64 i, atomic64_t *v) 1976 { 1977 kcsan_release(); 1978 instrument_atomic_read_write(v, sizeof(*v)); 1979 return raw_atomic64_fetch_sub_release(i, v); 1980 } 1981 1982 /** 1983 * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering 1984 * @i: s64 value to subtract 1985 * @v: pointer to atomic64_t 1986 * 1987 * Atomically updates @v to (@v - @i) with relaxed ordering. 1988 * 1989 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there. 1990 * 1991 * Return: The original value of @v. 1992 */ 1993 static __always_inline s64 1994 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 1995 { 1996 instrument_atomic_read_write(v, sizeof(*v)); 1997 return raw_atomic64_fetch_sub_relaxed(i, v); 1998 } 1999 2000 /** 2001 * atomic64_inc() - atomic increment with relaxed ordering 2002 * @v: pointer to atomic64_t 2003 * 2004 * Atomically updates @v to (@v + 1) with relaxed ordering. 2005 * 2006 * Unsafe to use in noinstr code; use raw_atomic64_inc() there. 2007 * 2008 * Return: Nothing. 2009 */ 2010 static __always_inline void 2011 atomic64_inc(atomic64_t *v) 2012 { 2013 instrument_atomic_read_write(v, sizeof(*v)); 2014 raw_atomic64_inc(v); 2015 } 2016 2017 /** 2018 * atomic64_inc_return() - atomic increment with full ordering 2019 * @v: pointer to atomic64_t 2020 * 2021 * Atomically updates @v to (@v + 1) with full ordering. 2022 * 2023 * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there. 2024 * 2025 * Return: The updated value of @v. 2026 */ 2027 static __always_inline s64 2028 atomic64_inc_return(atomic64_t *v) 2029 { 2030 kcsan_mb(); 2031 instrument_atomic_read_write(v, sizeof(*v)); 2032 return raw_atomic64_inc_return(v); 2033 } 2034 2035 /** 2036 * atomic64_inc_return_acquire() - atomic increment with acquire ordering 2037 * @v: pointer to atomic64_t 2038 * 2039 * Atomically updates @v to (@v + 1) with acquire ordering. 2040 * 2041 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there. 2042 * 2043 * Return: The updated value of @v. 2044 */ 2045 static __always_inline s64 2046 atomic64_inc_return_acquire(atomic64_t *v) 2047 { 2048 instrument_atomic_read_write(v, sizeof(*v)); 2049 return raw_atomic64_inc_return_acquire(v); 2050 } 2051 2052 /** 2053 * atomic64_inc_return_release() - atomic increment with release ordering 2054 * @v: pointer to atomic64_t 2055 * 2056 * Atomically updates @v to (@v + 1) with release ordering. 2057 * 2058 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there. 2059 * 2060 * Return: The updated value of @v. 2061 */ 2062 static __always_inline s64 2063 atomic64_inc_return_release(atomic64_t *v) 2064 { 2065 kcsan_release(); 2066 instrument_atomic_read_write(v, sizeof(*v)); 2067 return raw_atomic64_inc_return_release(v); 2068 } 2069 2070 /** 2071 * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering 2072 * @v: pointer to atomic64_t 2073 * 2074 * Atomically updates @v to (@v + 1) with relaxed ordering. 2075 * 2076 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there. 2077 * 2078 * Return: The updated value of @v. 2079 */ 2080 static __always_inline s64 2081 atomic64_inc_return_relaxed(atomic64_t *v) 2082 { 2083 instrument_atomic_read_write(v, sizeof(*v)); 2084 return raw_atomic64_inc_return_relaxed(v); 2085 } 2086 2087 /** 2088 * atomic64_fetch_inc() - atomic increment with full ordering 2089 * @v: pointer to atomic64_t 2090 * 2091 * Atomically updates @v to (@v + 1) with full ordering. 2092 * 2093 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there. 2094 * 2095 * Return: The original value of @v. 2096 */ 2097 static __always_inline s64 2098 atomic64_fetch_inc(atomic64_t *v) 2099 { 2100 kcsan_mb(); 2101 instrument_atomic_read_write(v, sizeof(*v)); 2102 return raw_atomic64_fetch_inc(v); 2103 } 2104 2105 /** 2106 * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering 2107 * @v: pointer to atomic64_t 2108 * 2109 * Atomically updates @v to (@v + 1) with acquire ordering. 2110 * 2111 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there. 2112 * 2113 * Return: The original value of @v. 2114 */ 2115 static __always_inline s64 2116 atomic64_fetch_inc_acquire(atomic64_t *v) 2117 { 2118 instrument_atomic_read_write(v, sizeof(*v)); 2119 return raw_atomic64_fetch_inc_acquire(v); 2120 } 2121 2122 /** 2123 * atomic64_fetch_inc_release() - atomic increment with release ordering 2124 * @v: pointer to atomic64_t 2125 * 2126 * Atomically updates @v to (@v + 1) with release ordering. 2127 * 2128 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there. 2129 * 2130 * Return: The original value of @v. 2131 */ 2132 static __always_inline s64 2133 atomic64_fetch_inc_release(atomic64_t *v) 2134 { 2135 kcsan_release(); 2136 instrument_atomic_read_write(v, sizeof(*v)); 2137 return raw_atomic64_fetch_inc_release(v); 2138 } 2139 2140 /** 2141 * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering 2142 * @v: pointer to atomic64_t 2143 * 2144 * Atomically updates @v to (@v + 1) with relaxed ordering. 2145 * 2146 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there. 2147 * 2148 * Return: The original value of @v. 2149 */ 2150 static __always_inline s64 2151 atomic64_fetch_inc_relaxed(atomic64_t *v) 2152 { 2153 instrument_atomic_read_write(v, sizeof(*v)); 2154 return raw_atomic64_fetch_inc_relaxed(v); 2155 } 2156 2157 /** 2158 * atomic64_dec() - atomic decrement with relaxed ordering 2159 * @v: pointer to atomic64_t 2160 * 2161 * Atomically updates @v to (@v - 1) with relaxed ordering. 2162 * 2163 * Unsafe to use in noinstr code; use raw_atomic64_dec() there. 2164 * 2165 * Return: Nothing. 2166 */ 2167 static __always_inline void 2168 atomic64_dec(atomic64_t *v) 2169 { 2170 instrument_atomic_read_write(v, sizeof(*v)); 2171 raw_atomic64_dec(v); 2172 } 2173 2174 /** 2175 * atomic64_dec_return() - atomic decrement with full ordering 2176 * @v: pointer to atomic64_t 2177 * 2178 * Atomically updates @v to (@v - 1) with full ordering. 2179 * 2180 * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there. 2181 * 2182 * Return: The updated value of @v. 2183 */ 2184 static __always_inline s64 2185 atomic64_dec_return(atomic64_t *v) 2186 { 2187 kcsan_mb(); 2188 instrument_atomic_read_write(v, sizeof(*v)); 2189 return raw_atomic64_dec_return(v); 2190 } 2191 2192 /** 2193 * atomic64_dec_return_acquire() - atomic decrement with acquire ordering 2194 * @v: pointer to atomic64_t 2195 * 2196 * Atomically updates @v to (@v - 1) with acquire ordering. 2197 * 2198 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there. 2199 * 2200 * Return: The updated value of @v. 2201 */ 2202 static __always_inline s64 2203 atomic64_dec_return_acquire(atomic64_t *v) 2204 { 2205 instrument_atomic_read_write(v, sizeof(*v)); 2206 return raw_atomic64_dec_return_acquire(v); 2207 } 2208 2209 /** 2210 * atomic64_dec_return_release() - atomic decrement with release ordering 2211 * @v: pointer to atomic64_t 2212 * 2213 * Atomically updates @v to (@v - 1) with release ordering. 2214 * 2215 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there. 2216 * 2217 * Return: The updated value of @v. 2218 */ 2219 static __always_inline s64 2220 atomic64_dec_return_release(atomic64_t *v) 2221 { 2222 kcsan_release(); 2223 instrument_atomic_read_write(v, sizeof(*v)); 2224 return raw_atomic64_dec_return_release(v); 2225 } 2226 2227 /** 2228 * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering 2229 * @v: pointer to atomic64_t 2230 * 2231 * Atomically updates @v to (@v - 1) with relaxed ordering. 2232 * 2233 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there. 2234 * 2235 * Return: The updated value of @v. 2236 */ 2237 static __always_inline s64 2238 atomic64_dec_return_relaxed(atomic64_t *v) 2239 { 2240 instrument_atomic_read_write(v, sizeof(*v)); 2241 return raw_atomic64_dec_return_relaxed(v); 2242 } 2243 2244 /** 2245 * atomic64_fetch_dec() - atomic decrement with full ordering 2246 * @v: pointer to atomic64_t 2247 * 2248 * Atomically updates @v to (@v - 1) with full ordering. 2249 * 2250 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there. 2251 * 2252 * Return: The original value of @v. 2253 */ 2254 static __always_inline s64 2255 atomic64_fetch_dec(atomic64_t *v) 2256 { 2257 kcsan_mb(); 2258 instrument_atomic_read_write(v, sizeof(*v)); 2259 return raw_atomic64_fetch_dec(v); 2260 } 2261 2262 /** 2263 * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering 2264 * @v: pointer to atomic64_t 2265 * 2266 * Atomically updates @v to (@v - 1) with acquire ordering. 2267 * 2268 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there. 2269 * 2270 * Return: The original value of @v. 2271 */ 2272 static __always_inline s64 2273 atomic64_fetch_dec_acquire(atomic64_t *v) 2274 { 2275 instrument_atomic_read_write(v, sizeof(*v)); 2276 return raw_atomic64_fetch_dec_acquire(v); 2277 } 2278 2279 /** 2280 * atomic64_fetch_dec_release() - atomic decrement with release ordering 2281 * @v: pointer to atomic64_t 2282 * 2283 * Atomically updates @v to (@v - 1) with release ordering. 2284 * 2285 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there. 2286 * 2287 * Return: The original value of @v. 2288 */ 2289 static __always_inline s64 2290 atomic64_fetch_dec_release(atomic64_t *v) 2291 { 2292 kcsan_release(); 2293 instrument_atomic_read_write(v, sizeof(*v)); 2294 return raw_atomic64_fetch_dec_release(v); 2295 } 2296 2297 /** 2298 * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering 2299 * @v: pointer to atomic64_t 2300 * 2301 * Atomically updates @v to (@v - 1) with relaxed ordering. 2302 * 2303 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there. 2304 * 2305 * Return: The original value of @v. 2306 */ 2307 static __always_inline s64 2308 atomic64_fetch_dec_relaxed(atomic64_t *v) 2309 { 2310 instrument_atomic_read_write(v, sizeof(*v)); 2311 return raw_atomic64_fetch_dec_relaxed(v); 2312 } 2313 2314 /** 2315 * atomic64_and() - atomic bitwise AND with relaxed ordering 2316 * @i: s64 value 2317 * @v: pointer to atomic64_t 2318 * 2319 * Atomically updates @v to (@v & @i) with relaxed ordering. 2320 * 2321 * Unsafe to use in noinstr code; use raw_atomic64_and() there. 2322 * 2323 * Return: Nothing. 2324 */ 2325 static __always_inline void 2326 atomic64_and(s64 i, atomic64_t *v) 2327 { 2328 instrument_atomic_read_write(v, sizeof(*v)); 2329 raw_atomic64_and(i, v); 2330 } 2331 2332 /** 2333 * atomic64_fetch_and() - atomic bitwise AND with full ordering 2334 * @i: s64 value 2335 * @v: pointer to atomic64_t 2336 * 2337 * Atomically updates @v to (@v & @i) with full ordering. 2338 * 2339 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there. 2340 * 2341 * Return: The original value of @v. 2342 */ 2343 static __always_inline s64 2344 atomic64_fetch_and(s64 i, atomic64_t *v) 2345 { 2346 kcsan_mb(); 2347 instrument_atomic_read_write(v, sizeof(*v)); 2348 return raw_atomic64_fetch_and(i, v); 2349 } 2350 2351 /** 2352 * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering 2353 * @i: s64 value 2354 * @v: pointer to atomic64_t 2355 * 2356 * Atomically updates @v to (@v & @i) with acquire ordering. 2357 * 2358 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there. 2359 * 2360 * Return: The original value of @v. 2361 */ 2362 static __always_inline s64 2363 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 2364 { 2365 instrument_atomic_read_write(v, sizeof(*v)); 2366 return raw_atomic64_fetch_and_acquire(i, v); 2367 } 2368 2369 /** 2370 * atomic64_fetch_and_release() - atomic bitwise AND with release ordering 2371 * @i: s64 value 2372 * @v: pointer to atomic64_t 2373 * 2374 * Atomically updates @v to (@v & @i) with release ordering. 2375 * 2376 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there. 2377 * 2378 * Return: The original value of @v. 2379 */ 2380 static __always_inline s64 2381 atomic64_fetch_and_release(s64 i, atomic64_t *v) 2382 { 2383 kcsan_release(); 2384 instrument_atomic_read_write(v, sizeof(*v)); 2385 return raw_atomic64_fetch_and_release(i, v); 2386 } 2387 2388 /** 2389 * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering 2390 * @i: s64 value 2391 * @v: pointer to atomic64_t 2392 * 2393 * Atomically updates @v to (@v & @i) with relaxed ordering. 2394 * 2395 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there. 2396 * 2397 * Return: The original value of @v. 2398 */ 2399 static __always_inline s64 2400 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 2401 { 2402 instrument_atomic_read_write(v, sizeof(*v)); 2403 return raw_atomic64_fetch_and_relaxed(i, v); 2404 } 2405 2406 /** 2407 * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering 2408 * @i: s64 value 2409 * @v: pointer to atomic64_t 2410 * 2411 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 2412 * 2413 * Unsafe to use in noinstr code; use raw_atomic64_andnot() there. 2414 * 2415 * Return: Nothing. 2416 */ 2417 static __always_inline void 2418 atomic64_andnot(s64 i, atomic64_t *v) 2419 { 2420 instrument_atomic_read_write(v, sizeof(*v)); 2421 raw_atomic64_andnot(i, v); 2422 } 2423 2424 /** 2425 * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering 2426 * @i: s64 value 2427 * @v: pointer to atomic64_t 2428 * 2429 * Atomically updates @v to (@v & ~@i) with full ordering. 2430 * 2431 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there. 2432 * 2433 * Return: The original value of @v. 2434 */ 2435 static __always_inline s64 2436 atomic64_fetch_andnot(s64 i, atomic64_t *v) 2437 { 2438 kcsan_mb(); 2439 instrument_atomic_read_write(v, sizeof(*v)); 2440 return raw_atomic64_fetch_andnot(i, v); 2441 } 2442 2443 /** 2444 * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering 2445 * @i: s64 value 2446 * @v: pointer to atomic64_t 2447 * 2448 * Atomically updates @v to (@v & ~@i) with acquire ordering. 2449 * 2450 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there. 2451 * 2452 * Return: The original value of @v. 2453 */ 2454 static __always_inline s64 2455 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 2456 { 2457 instrument_atomic_read_write(v, sizeof(*v)); 2458 return raw_atomic64_fetch_andnot_acquire(i, v); 2459 } 2460 2461 /** 2462 * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering 2463 * @i: s64 value 2464 * @v: pointer to atomic64_t 2465 * 2466 * Atomically updates @v to (@v & ~@i) with release ordering. 2467 * 2468 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there. 2469 * 2470 * Return: The original value of @v. 2471 */ 2472 static __always_inline s64 2473 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 2474 { 2475 kcsan_release(); 2476 instrument_atomic_read_write(v, sizeof(*v)); 2477 return raw_atomic64_fetch_andnot_release(i, v); 2478 } 2479 2480 /** 2481 * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering 2482 * @i: s64 value 2483 * @v: pointer to atomic64_t 2484 * 2485 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 2486 * 2487 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there. 2488 * 2489 * Return: The original value of @v. 2490 */ 2491 static __always_inline s64 2492 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 2493 { 2494 instrument_atomic_read_write(v, sizeof(*v)); 2495 return raw_atomic64_fetch_andnot_relaxed(i, v); 2496 } 2497 2498 /** 2499 * atomic64_or() - atomic bitwise OR with relaxed ordering 2500 * @i: s64 value 2501 * @v: pointer to atomic64_t 2502 * 2503 * Atomically updates @v to (@v | @i) with relaxed ordering. 2504 * 2505 * Unsafe to use in noinstr code; use raw_atomic64_or() there. 2506 * 2507 * Return: Nothing. 2508 */ 2509 static __always_inline void 2510 atomic64_or(s64 i, atomic64_t *v) 2511 { 2512 instrument_atomic_read_write(v, sizeof(*v)); 2513 raw_atomic64_or(i, v); 2514 } 2515 2516 /** 2517 * atomic64_fetch_or() - atomic bitwise OR with full ordering 2518 * @i: s64 value 2519 * @v: pointer to atomic64_t 2520 * 2521 * Atomically updates @v to (@v | @i) with full ordering. 2522 * 2523 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there. 2524 * 2525 * Return: The original value of @v. 2526 */ 2527 static __always_inline s64 2528 atomic64_fetch_or(s64 i, atomic64_t *v) 2529 { 2530 kcsan_mb(); 2531 instrument_atomic_read_write(v, sizeof(*v)); 2532 return raw_atomic64_fetch_or(i, v); 2533 } 2534 2535 /** 2536 * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering 2537 * @i: s64 value 2538 * @v: pointer to atomic64_t 2539 * 2540 * Atomically updates @v to (@v | @i) with acquire ordering. 2541 * 2542 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there. 2543 * 2544 * Return: The original value of @v. 2545 */ 2546 static __always_inline s64 2547 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 2548 { 2549 instrument_atomic_read_write(v, sizeof(*v)); 2550 return raw_atomic64_fetch_or_acquire(i, v); 2551 } 2552 2553 /** 2554 * atomic64_fetch_or_release() - atomic bitwise OR with release ordering 2555 * @i: s64 value 2556 * @v: pointer to atomic64_t 2557 * 2558 * Atomically updates @v to (@v | @i) with release ordering. 2559 * 2560 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there. 2561 * 2562 * Return: The original value of @v. 2563 */ 2564 static __always_inline s64 2565 atomic64_fetch_or_release(s64 i, atomic64_t *v) 2566 { 2567 kcsan_release(); 2568 instrument_atomic_read_write(v, sizeof(*v)); 2569 return raw_atomic64_fetch_or_release(i, v); 2570 } 2571 2572 /** 2573 * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering 2574 * @i: s64 value 2575 * @v: pointer to atomic64_t 2576 * 2577 * Atomically updates @v to (@v | @i) with relaxed ordering. 2578 * 2579 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there. 2580 * 2581 * Return: The original value of @v. 2582 */ 2583 static __always_inline s64 2584 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 2585 { 2586 instrument_atomic_read_write(v, sizeof(*v)); 2587 return raw_atomic64_fetch_or_relaxed(i, v); 2588 } 2589 2590 /** 2591 * atomic64_xor() - atomic bitwise XOR with relaxed ordering 2592 * @i: s64 value 2593 * @v: pointer to atomic64_t 2594 * 2595 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 2596 * 2597 * Unsafe to use in noinstr code; use raw_atomic64_xor() there. 2598 * 2599 * Return: Nothing. 2600 */ 2601 static __always_inline void 2602 atomic64_xor(s64 i, atomic64_t *v) 2603 { 2604 instrument_atomic_read_write(v, sizeof(*v)); 2605 raw_atomic64_xor(i, v); 2606 } 2607 2608 /** 2609 * atomic64_fetch_xor() - atomic bitwise XOR with full ordering 2610 * @i: s64 value 2611 * @v: pointer to atomic64_t 2612 * 2613 * Atomically updates @v to (@v ^ @i) with full ordering. 2614 * 2615 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there. 2616 * 2617 * Return: The original value of @v. 2618 */ 2619 static __always_inline s64 2620 atomic64_fetch_xor(s64 i, atomic64_t *v) 2621 { 2622 kcsan_mb(); 2623 instrument_atomic_read_write(v, sizeof(*v)); 2624 return raw_atomic64_fetch_xor(i, v); 2625 } 2626 2627 /** 2628 * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering 2629 * @i: s64 value 2630 * @v: pointer to atomic64_t 2631 * 2632 * Atomically updates @v to (@v ^ @i) with acquire ordering. 2633 * 2634 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there. 2635 * 2636 * Return: The original value of @v. 2637 */ 2638 static __always_inline s64 2639 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 2640 { 2641 instrument_atomic_read_write(v, sizeof(*v)); 2642 return raw_atomic64_fetch_xor_acquire(i, v); 2643 } 2644 2645 /** 2646 * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering 2647 * @i: s64 value 2648 * @v: pointer to atomic64_t 2649 * 2650 * Atomically updates @v to (@v ^ @i) with release ordering. 2651 * 2652 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there. 2653 * 2654 * Return: The original value of @v. 2655 */ 2656 static __always_inline s64 2657 atomic64_fetch_xor_release(s64 i, atomic64_t *v) 2658 { 2659 kcsan_release(); 2660 instrument_atomic_read_write(v, sizeof(*v)); 2661 return raw_atomic64_fetch_xor_release(i, v); 2662 } 2663 2664 /** 2665 * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering 2666 * @i: s64 value 2667 * @v: pointer to atomic64_t 2668 * 2669 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 2670 * 2671 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there. 2672 * 2673 * Return: The original value of @v. 2674 */ 2675 static __always_inline s64 2676 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 2677 { 2678 instrument_atomic_read_write(v, sizeof(*v)); 2679 return raw_atomic64_fetch_xor_relaxed(i, v); 2680 } 2681 2682 /** 2683 * atomic64_xchg() - atomic exchange with full ordering 2684 * @v: pointer to atomic64_t 2685 * @new: s64 value to assign 2686 * 2687 * Atomically updates @v to @new with full ordering. 2688 * 2689 * Unsafe to use in noinstr code; use raw_atomic64_xchg() there. 2690 * 2691 * Return: The original value of @v. 2692 */ 2693 static __always_inline s64 2694 atomic64_xchg(atomic64_t *v, s64 new) 2695 { 2696 kcsan_mb(); 2697 instrument_atomic_read_write(v, sizeof(*v)); 2698 return raw_atomic64_xchg(v, new); 2699 } 2700 2701 /** 2702 * atomic64_xchg_acquire() - atomic exchange with acquire ordering 2703 * @v: pointer to atomic64_t 2704 * @new: s64 value to assign 2705 * 2706 * Atomically updates @v to @new with acquire ordering. 2707 * 2708 * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there. 2709 * 2710 * Return: The original value of @v. 2711 */ 2712 static __always_inline s64 2713 atomic64_xchg_acquire(atomic64_t *v, s64 new) 2714 { 2715 instrument_atomic_read_write(v, sizeof(*v)); 2716 return raw_atomic64_xchg_acquire(v, new); 2717 } 2718 2719 /** 2720 * atomic64_xchg_release() - atomic exchange with release ordering 2721 * @v: pointer to atomic64_t 2722 * @new: s64 value to assign 2723 * 2724 * Atomically updates @v to @new with release ordering. 2725 * 2726 * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there. 2727 * 2728 * Return: The original value of @v. 2729 */ 2730 static __always_inline s64 2731 atomic64_xchg_release(atomic64_t *v, s64 new) 2732 { 2733 kcsan_release(); 2734 instrument_atomic_read_write(v, sizeof(*v)); 2735 return raw_atomic64_xchg_release(v, new); 2736 } 2737 2738 /** 2739 * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering 2740 * @v: pointer to atomic64_t 2741 * @new: s64 value to assign 2742 * 2743 * Atomically updates @v to @new with relaxed ordering. 2744 * 2745 * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there. 2746 * 2747 * Return: The original value of @v. 2748 */ 2749 static __always_inline s64 2750 atomic64_xchg_relaxed(atomic64_t *v, s64 new) 2751 { 2752 instrument_atomic_read_write(v, sizeof(*v)); 2753 return raw_atomic64_xchg_relaxed(v, new); 2754 } 2755 2756 /** 2757 * atomic64_cmpxchg() - atomic compare and exchange with full ordering 2758 * @v: pointer to atomic64_t 2759 * @old: s64 value to compare with 2760 * @new: s64 value to assign 2761 * 2762 * If (@v == @old), atomically updates @v to @new with full ordering. 2763 * Otherwise, @v is not modified and relaxed ordering is provided. 2764 * 2765 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there. 2766 * 2767 * Return: The original value of @v. 2768 */ 2769 static __always_inline s64 2770 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2771 { 2772 kcsan_mb(); 2773 instrument_atomic_read_write(v, sizeof(*v)); 2774 return raw_atomic64_cmpxchg(v, old, new); 2775 } 2776 2777 /** 2778 * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 2779 * @v: pointer to atomic64_t 2780 * @old: s64 value to compare with 2781 * @new: s64 value to assign 2782 * 2783 * If (@v == @old), atomically updates @v to @new with acquire ordering. 2784 * Otherwise, @v is not modified and relaxed ordering is provided. 2785 * 2786 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there. 2787 * 2788 * Return: The original value of @v. 2789 */ 2790 static __always_inline s64 2791 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2792 { 2793 instrument_atomic_read_write(v, sizeof(*v)); 2794 return raw_atomic64_cmpxchg_acquire(v, old, new); 2795 } 2796 2797 /** 2798 * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering 2799 * @v: pointer to atomic64_t 2800 * @old: s64 value to compare with 2801 * @new: s64 value to assign 2802 * 2803 * If (@v == @old), atomically updates @v to @new with release ordering. 2804 * Otherwise, @v is not modified and relaxed ordering is provided. 2805 * 2806 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there. 2807 * 2808 * Return: The original value of @v. 2809 */ 2810 static __always_inline s64 2811 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2812 { 2813 kcsan_release(); 2814 instrument_atomic_read_write(v, sizeof(*v)); 2815 return raw_atomic64_cmpxchg_release(v, old, new); 2816 } 2817 2818 /** 2819 * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 2820 * @v: pointer to atomic64_t 2821 * @old: s64 value to compare with 2822 * @new: s64 value to assign 2823 * 2824 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 2825 * Otherwise, @v is not modified and relaxed ordering is provided. 2826 * 2827 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there. 2828 * 2829 * Return: The original value of @v. 2830 */ 2831 static __always_inline s64 2832 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 2833 { 2834 instrument_atomic_read_write(v, sizeof(*v)); 2835 return raw_atomic64_cmpxchg_relaxed(v, old, new); 2836 } 2837 2838 /** 2839 * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering 2840 * @v: pointer to atomic64_t 2841 * @old: pointer to s64 value to compare with 2842 * @new: s64 value to assign 2843 * 2844 * If (@v == @old), atomically updates @v to @new with full ordering. 2845 * Otherwise, @v is not modified, @old is updated to the current value of @v, 2846 * and relaxed ordering is provided. 2847 * 2848 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there. 2849 * 2850 * Return: @true if the exchange occured, @false otherwise. 2851 */ 2852 static __always_inline bool 2853 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2854 { 2855 kcsan_mb(); 2856 instrument_atomic_read_write(v, sizeof(*v)); 2857 instrument_atomic_read_write(old, sizeof(*old)); 2858 return raw_atomic64_try_cmpxchg(v, old, new); 2859 } 2860 2861 /** 2862 * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 2863 * @v: pointer to atomic64_t 2864 * @old: pointer to s64 value to compare with 2865 * @new: s64 value to assign 2866 * 2867 * If (@v == @old), atomically updates @v to @new with acquire ordering. 2868 * Otherwise, @v is not modified, @old is updated to the current value of @v, 2869 * and relaxed ordering is provided. 2870 * 2871 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there. 2872 * 2873 * Return: @true if the exchange occured, @false otherwise. 2874 */ 2875 static __always_inline bool 2876 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2877 { 2878 instrument_atomic_read_write(v, sizeof(*v)); 2879 instrument_atomic_read_write(old, sizeof(*old)); 2880 return raw_atomic64_try_cmpxchg_acquire(v, old, new); 2881 } 2882 2883 /** 2884 * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering 2885 * @v: pointer to atomic64_t 2886 * @old: pointer to s64 value to compare with 2887 * @new: s64 value to assign 2888 * 2889 * If (@v == @old), atomically updates @v to @new with release ordering. 2890 * Otherwise, @v is not modified, @old is updated to the current value of @v, 2891 * and relaxed ordering is provided. 2892 * 2893 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there. 2894 * 2895 * Return: @true if the exchange occured, @false otherwise. 2896 */ 2897 static __always_inline bool 2898 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2899 { 2900 kcsan_release(); 2901 instrument_atomic_read_write(v, sizeof(*v)); 2902 instrument_atomic_read_write(old, sizeof(*old)); 2903 return raw_atomic64_try_cmpxchg_release(v, old, new); 2904 } 2905 2906 /** 2907 * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 2908 * @v: pointer to atomic64_t 2909 * @old: pointer to s64 value to compare with 2910 * @new: s64 value to assign 2911 * 2912 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 2913 * Otherwise, @v is not modified, @old is updated to the current value of @v, 2914 * and relaxed ordering is provided. 2915 * 2916 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there. 2917 * 2918 * Return: @true if the exchange occured, @false otherwise. 2919 */ 2920 static __always_inline bool 2921 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 2922 { 2923 instrument_atomic_read_write(v, sizeof(*v)); 2924 instrument_atomic_read_write(old, sizeof(*old)); 2925 return raw_atomic64_try_cmpxchg_relaxed(v, old, new); 2926 } 2927 2928 /** 2929 * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering 2930 * @i: s64 value to subtract 2931 * @v: pointer to atomic64_t 2932 * 2933 * Atomically updates @v to (@v - @i) with full ordering. 2934 * 2935 * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there. 2936 * 2937 * Return: @true if the resulting value of @v is zero, @false otherwise. 2938 */ 2939 static __always_inline bool 2940 atomic64_sub_and_test(s64 i, atomic64_t *v) 2941 { 2942 kcsan_mb(); 2943 instrument_atomic_read_write(v, sizeof(*v)); 2944 return raw_atomic64_sub_and_test(i, v); 2945 } 2946 2947 /** 2948 * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering 2949 * @v: pointer to atomic64_t 2950 * 2951 * Atomically updates @v to (@v - 1) with full ordering. 2952 * 2953 * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there. 2954 * 2955 * Return: @true if the resulting value of @v is zero, @false otherwise. 2956 */ 2957 static __always_inline bool 2958 atomic64_dec_and_test(atomic64_t *v) 2959 { 2960 kcsan_mb(); 2961 instrument_atomic_read_write(v, sizeof(*v)); 2962 return raw_atomic64_dec_and_test(v); 2963 } 2964 2965 /** 2966 * atomic64_inc_and_test() - atomic increment and test if zero with full ordering 2967 * @v: pointer to atomic64_t 2968 * 2969 * Atomically updates @v to (@v + 1) with full ordering. 2970 * 2971 * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there. 2972 * 2973 * Return: @true if the resulting value of @v is zero, @false otherwise. 2974 */ 2975 static __always_inline bool 2976 atomic64_inc_and_test(atomic64_t *v) 2977 { 2978 kcsan_mb(); 2979 instrument_atomic_read_write(v, sizeof(*v)); 2980 return raw_atomic64_inc_and_test(v); 2981 } 2982 2983 /** 2984 * atomic64_add_negative() - atomic add and test if negative with full ordering 2985 * @i: s64 value to add 2986 * @v: pointer to atomic64_t 2987 * 2988 * Atomically updates @v to (@v + @i) with full ordering. 2989 * 2990 * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there. 2991 * 2992 * Return: @true if the resulting value of @v is negative, @false otherwise. 2993 */ 2994 static __always_inline bool 2995 atomic64_add_negative(s64 i, atomic64_t *v) 2996 { 2997 kcsan_mb(); 2998 instrument_atomic_read_write(v, sizeof(*v)); 2999 return raw_atomic64_add_negative(i, v); 3000 } 3001 3002 /** 3003 * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering 3004 * @i: s64 value to add 3005 * @v: pointer to atomic64_t 3006 * 3007 * Atomically updates @v to (@v + @i) with acquire ordering. 3008 * 3009 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there. 3010 * 3011 * Return: @true if the resulting value of @v is negative, @false otherwise. 3012 */ 3013 static __always_inline bool 3014 atomic64_add_negative_acquire(s64 i, atomic64_t *v) 3015 { 3016 instrument_atomic_read_write(v, sizeof(*v)); 3017 return raw_atomic64_add_negative_acquire(i, v); 3018 } 3019 3020 /** 3021 * atomic64_add_negative_release() - atomic add and test if negative with release ordering 3022 * @i: s64 value to add 3023 * @v: pointer to atomic64_t 3024 * 3025 * Atomically updates @v to (@v + @i) with release ordering. 3026 * 3027 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there. 3028 * 3029 * Return: @true if the resulting value of @v is negative, @false otherwise. 3030 */ 3031 static __always_inline bool 3032 atomic64_add_negative_release(s64 i, atomic64_t *v) 3033 { 3034 kcsan_release(); 3035 instrument_atomic_read_write(v, sizeof(*v)); 3036 return raw_atomic64_add_negative_release(i, v); 3037 } 3038 3039 /** 3040 * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering 3041 * @i: s64 value to add 3042 * @v: pointer to atomic64_t 3043 * 3044 * Atomically updates @v to (@v + @i) with relaxed ordering. 3045 * 3046 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there. 3047 * 3048 * Return: @true if the resulting value of @v is negative, @false otherwise. 3049 */ 3050 static __always_inline bool 3051 atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 3052 { 3053 instrument_atomic_read_write(v, sizeof(*v)); 3054 return raw_atomic64_add_negative_relaxed(i, v); 3055 } 3056 3057 /** 3058 * atomic64_fetch_add_unless() - atomic add unless value with full ordering 3059 * @v: pointer to atomic64_t 3060 * @a: s64 value to add 3061 * @u: s64 value to compare with 3062 * 3063 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 3064 * Otherwise, @v is not modified and relaxed ordering is provided. 3065 * 3066 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there. 3067 * 3068 * Return: The original value of @v. 3069 */ 3070 static __always_inline s64 3071 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 3072 { 3073 kcsan_mb(); 3074 instrument_atomic_read_write(v, sizeof(*v)); 3075 return raw_atomic64_fetch_add_unless(v, a, u); 3076 } 3077 3078 /** 3079 * atomic64_add_unless() - atomic add unless value with full ordering 3080 * @v: pointer to atomic64_t 3081 * @a: s64 value to add 3082 * @u: s64 value to compare with 3083 * 3084 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 3085 * Otherwise, @v is not modified and relaxed ordering is provided. 3086 * 3087 * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there. 3088 * 3089 * Return: @true if @v was updated, @false otherwise. 3090 */ 3091 static __always_inline bool 3092 atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 3093 { 3094 kcsan_mb(); 3095 instrument_atomic_read_write(v, sizeof(*v)); 3096 return raw_atomic64_add_unless(v, a, u); 3097 } 3098 3099 /** 3100 * atomic64_inc_not_zero() - atomic increment unless zero with full ordering 3101 * @v: pointer to atomic64_t 3102 * 3103 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. 3104 * Otherwise, @v is not modified and relaxed ordering is provided. 3105 * 3106 * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there. 3107 * 3108 * Return: @true if @v was updated, @false otherwise. 3109 */ 3110 static __always_inline bool 3111 atomic64_inc_not_zero(atomic64_t *v) 3112 { 3113 kcsan_mb(); 3114 instrument_atomic_read_write(v, sizeof(*v)); 3115 return raw_atomic64_inc_not_zero(v); 3116 } 3117 3118 /** 3119 * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering 3120 * @v: pointer to atomic64_t 3121 * 3122 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. 3123 * Otherwise, @v is not modified and relaxed ordering is provided. 3124 * 3125 * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there. 3126 * 3127 * Return: @true if @v was updated, @false otherwise. 3128 */ 3129 static __always_inline bool 3130 atomic64_inc_unless_negative(atomic64_t *v) 3131 { 3132 kcsan_mb(); 3133 instrument_atomic_read_write(v, sizeof(*v)); 3134 return raw_atomic64_inc_unless_negative(v); 3135 } 3136 3137 /** 3138 * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering 3139 * @v: pointer to atomic64_t 3140 * 3141 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. 3142 * Otherwise, @v is not modified and relaxed ordering is provided. 3143 * 3144 * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there. 3145 * 3146 * Return: @true if @v was updated, @false otherwise. 3147 */ 3148 static __always_inline bool 3149 atomic64_dec_unless_positive(atomic64_t *v) 3150 { 3151 kcsan_mb(); 3152 instrument_atomic_read_write(v, sizeof(*v)); 3153 return raw_atomic64_dec_unless_positive(v); 3154 } 3155 3156 /** 3157 * atomic64_dec_if_positive() - atomic decrement if positive with full ordering 3158 * @v: pointer to atomic64_t 3159 * 3160 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. 3161 * Otherwise, @v is not modified and relaxed ordering is provided. 3162 * 3163 * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there. 3164 * 3165 * Return: The old value of (@v - 1), regardless of whether @v was updated. 3166 */ 3167 static __always_inline s64 3168 atomic64_dec_if_positive(atomic64_t *v) 3169 { 3170 kcsan_mb(); 3171 instrument_atomic_read_write(v, sizeof(*v)); 3172 return raw_atomic64_dec_if_positive(v); 3173 } 3174 3175 /** 3176 * atomic_long_read() - atomic load with relaxed ordering 3177 * @v: pointer to atomic_long_t 3178 * 3179 * Atomically loads the value of @v with relaxed ordering. 3180 * 3181 * Unsafe to use in noinstr code; use raw_atomic_long_read() there. 3182 * 3183 * Return: The value loaded from @v. 3184 */ 3185 static __always_inline long 3186 atomic_long_read(const atomic_long_t *v) 3187 { 3188 instrument_atomic_read(v, sizeof(*v)); 3189 return raw_atomic_long_read(v); 3190 } 3191 3192 /** 3193 * atomic_long_read_acquire() - atomic load with acquire ordering 3194 * @v: pointer to atomic_long_t 3195 * 3196 * Atomically loads the value of @v with acquire ordering. 3197 * 3198 * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there. 3199 * 3200 * Return: The value loaded from @v. 3201 */ 3202 static __always_inline long 3203 atomic_long_read_acquire(const atomic_long_t *v) 3204 { 3205 instrument_atomic_read(v, sizeof(*v)); 3206 return raw_atomic_long_read_acquire(v); 3207 } 3208 3209 /** 3210 * atomic_long_set() - atomic set with relaxed ordering 3211 * @v: pointer to atomic_long_t 3212 * @i: long value to assign 3213 * 3214 * Atomically sets @v to @i with relaxed ordering. 3215 * 3216 * Unsafe to use in noinstr code; use raw_atomic_long_set() there. 3217 * 3218 * Return: Nothing. 3219 */ 3220 static __always_inline void 3221 atomic_long_set(atomic_long_t *v, long i) 3222 { 3223 instrument_atomic_write(v, sizeof(*v)); 3224 raw_atomic_long_set(v, i); 3225 } 3226 3227 /** 3228 * atomic_long_set_release() - atomic set with release ordering 3229 * @v: pointer to atomic_long_t 3230 * @i: long value to assign 3231 * 3232 * Atomically sets @v to @i with release ordering. 3233 * 3234 * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there. 3235 * 3236 * Return: Nothing. 3237 */ 3238 static __always_inline void 3239 atomic_long_set_release(atomic_long_t *v, long i) 3240 { 3241 kcsan_release(); 3242 instrument_atomic_write(v, sizeof(*v)); 3243 raw_atomic_long_set_release(v, i); 3244 } 3245 3246 /** 3247 * atomic_long_add() - atomic add with relaxed ordering 3248 * @i: long value to add 3249 * @v: pointer to atomic_long_t 3250 * 3251 * Atomically updates @v to (@v + @i) with relaxed ordering. 3252 * 3253 * Unsafe to use in noinstr code; use raw_atomic_long_add() there. 3254 * 3255 * Return: Nothing. 3256 */ 3257 static __always_inline void 3258 atomic_long_add(long i, atomic_long_t *v) 3259 { 3260 instrument_atomic_read_write(v, sizeof(*v)); 3261 raw_atomic_long_add(i, v); 3262 } 3263 3264 /** 3265 * atomic_long_add_return() - atomic add with full ordering 3266 * @i: long value to add 3267 * @v: pointer to atomic_long_t 3268 * 3269 * Atomically updates @v to (@v + @i) with full ordering. 3270 * 3271 * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there. 3272 * 3273 * Return: The updated value of @v. 3274 */ 3275 static __always_inline long 3276 atomic_long_add_return(long i, atomic_long_t *v) 3277 { 3278 kcsan_mb(); 3279 instrument_atomic_read_write(v, sizeof(*v)); 3280 return raw_atomic_long_add_return(i, v); 3281 } 3282 3283 /** 3284 * atomic_long_add_return_acquire() - atomic add with acquire ordering 3285 * @i: long value to add 3286 * @v: pointer to atomic_long_t 3287 * 3288 * Atomically updates @v to (@v + @i) with acquire ordering. 3289 * 3290 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there. 3291 * 3292 * Return: The updated value of @v. 3293 */ 3294 static __always_inline long 3295 atomic_long_add_return_acquire(long i, atomic_long_t *v) 3296 { 3297 instrument_atomic_read_write(v, sizeof(*v)); 3298 return raw_atomic_long_add_return_acquire(i, v); 3299 } 3300 3301 /** 3302 * atomic_long_add_return_release() - atomic add with release ordering 3303 * @i: long value to add 3304 * @v: pointer to atomic_long_t 3305 * 3306 * Atomically updates @v to (@v + @i) with release ordering. 3307 * 3308 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there. 3309 * 3310 * Return: The updated value of @v. 3311 */ 3312 static __always_inline long 3313 atomic_long_add_return_release(long i, atomic_long_t *v) 3314 { 3315 kcsan_release(); 3316 instrument_atomic_read_write(v, sizeof(*v)); 3317 return raw_atomic_long_add_return_release(i, v); 3318 } 3319 3320 /** 3321 * atomic_long_add_return_relaxed() - atomic add with relaxed ordering 3322 * @i: long value to add 3323 * @v: pointer to atomic_long_t 3324 * 3325 * Atomically updates @v to (@v + @i) with relaxed ordering. 3326 * 3327 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there. 3328 * 3329 * Return: The updated value of @v. 3330 */ 3331 static __always_inline long 3332 atomic_long_add_return_relaxed(long i, atomic_long_t *v) 3333 { 3334 instrument_atomic_read_write(v, sizeof(*v)); 3335 return raw_atomic_long_add_return_relaxed(i, v); 3336 } 3337 3338 /** 3339 * atomic_long_fetch_add() - atomic add with full ordering 3340 * @i: long value to add 3341 * @v: pointer to atomic_long_t 3342 * 3343 * Atomically updates @v to (@v + @i) with full ordering. 3344 * 3345 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there. 3346 * 3347 * Return: The original value of @v. 3348 */ 3349 static __always_inline long 3350 atomic_long_fetch_add(long i, atomic_long_t *v) 3351 { 3352 kcsan_mb(); 3353 instrument_atomic_read_write(v, sizeof(*v)); 3354 return raw_atomic_long_fetch_add(i, v); 3355 } 3356 3357 /** 3358 * atomic_long_fetch_add_acquire() - atomic add with acquire ordering 3359 * @i: long value to add 3360 * @v: pointer to atomic_long_t 3361 * 3362 * Atomically updates @v to (@v + @i) with acquire ordering. 3363 * 3364 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there. 3365 * 3366 * Return: The original value of @v. 3367 */ 3368 static __always_inline long 3369 atomic_long_fetch_add_acquire(long i, atomic_long_t *v) 3370 { 3371 instrument_atomic_read_write(v, sizeof(*v)); 3372 return raw_atomic_long_fetch_add_acquire(i, v); 3373 } 3374 3375 /** 3376 * atomic_long_fetch_add_release() - atomic add with release ordering 3377 * @i: long value to add 3378 * @v: pointer to atomic_long_t 3379 * 3380 * Atomically updates @v to (@v + @i) with release ordering. 3381 * 3382 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there. 3383 * 3384 * Return: The original value of @v. 3385 */ 3386 static __always_inline long 3387 atomic_long_fetch_add_release(long i, atomic_long_t *v) 3388 { 3389 kcsan_release(); 3390 instrument_atomic_read_write(v, sizeof(*v)); 3391 return raw_atomic_long_fetch_add_release(i, v); 3392 } 3393 3394 /** 3395 * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering 3396 * @i: long value to add 3397 * @v: pointer to atomic_long_t 3398 * 3399 * Atomically updates @v to (@v + @i) with relaxed ordering. 3400 * 3401 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there. 3402 * 3403 * Return: The original value of @v. 3404 */ 3405 static __always_inline long 3406 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) 3407 { 3408 instrument_atomic_read_write(v, sizeof(*v)); 3409 return raw_atomic_long_fetch_add_relaxed(i, v); 3410 } 3411 3412 /** 3413 * atomic_long_sub() - atomic subtract with relaxed ordering 3414 * @i: long value to subtract 3415 * @v: pointer to atomic_long_t 3416 * 3417 * Atomically updates @v to (@v - @i) with relaxed ordering. 3418 * 3419 * Unsafe to use in noinstr code; use raw_atomic_long_sub() there. 3420 * 3421 * Return: Nothing. 3422 */ 3423 static __always_inline void 3424 atomic_long_sub(long i, atomic_long_t *v) 3425 { 3426 instrument_atomic_read_write(v, sizeof(*v)); 3427 raw_atomic_long_sub(i, v); 3428 } 3429 3430 /** 3431 * atomic_long_sub_return() - atomic subtract with full ordering 3432 * @i: long value to subtract 3433 * @v: pointer to atomic_long_t 3434 * 3435 * Atomically updates @v to (@v - @i) with full ordering. 3436 * 3437 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there. 3438 * 3439 * Return: The updated value of @v. 3440 */ 3441 static __always_inline long 3442 atomic_long_sub_return(long i, atomic_long_t *v) 3443 { 3444 kcsan_mb(); 3445 instrument_atomic_read_write(v, sizeof(*v)); 3446 return raw_atomic_long_sub_return(i, v); 3447 } 3448 3449 /** 3450 * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering 3451 * @i: long value to subtract 3452 * @v: pointer to atomic_long_t 3453 * 3454 * Atomically updates @v to (@v - @i) with acquire ordering. 3455 * 3456 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there. 3457 * 3458 * Return: The updated value of @v. 3459 */ 3460 static __always_inline long 3461 atomic_long_sub_return_acquire(long i, atomic_long_t *v) 3462 { 3463 instrument_atomic_read_write(v, sizeof(*v)); 3464 return raw_atomic_long_sub_return_acquire(i, v); 3465 } 3466 3467 /** 3468 * atomic_long_sub_return_release() - atomic subtract with release ordering 3469 * @i: long value to subtract 3470 * @v: pointer to atomic_long_t 3471 * 3472 * Atomically updates @v to (@v - @i) with release ordering. 3473 * 3474 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there. 3475 * 3476 * Return: The updated value of @v. 3477 */ 3478 static __always_inline long 3479 atomic_long_sub_return_release(long i, atomic_long_t *v) 3480 { 3481 kcsan_release(); 3482 instrument_atomic_read_write(v, sizeof(*v)); 3483 return raw_atomic_long_sub_return_release(i, v); 3484 } 3485 3486 /** 3487 * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering 3488 * @i: long value to subtract 3489 * @v: pointer to atomic_long_t 3490 * 3491 * Atomically updates @v to (@v - @i) with relaxed ordering. 3492 * 3493 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there. 3494 * 3495 * Return: The updated value of @v. 3496 */ 3497 static __always_inline long 3498 atomic_long_sub_return_relaxed(long i, atomic_long_t *v) 3499 { 3500 instrument_atomic_read_write(v, sizeof(*v)); 3501 return raw_atomic_long_sub_return_relaxed(i, v); 3502 } 3503 3504 /** 3505 * atomic_long_fetch_sub() - atomic subtract with full ordering 3506 * @i: long value to subtract 3507 * @v: pointer to atomic_long_t 3508 * 3509 * Atomically updates @v to (@v - @i) with full ordering. 3510 * 3511 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there. 3512 * 3513 * Return: The original value of @v. 3514 */ 3515 static __always_inline long 3516 atomic_long_fetch_sub(long i, atomic_long_t *v) 3517 { 3518 kcsan_mb(); 3519 instrument_atomic_read_write(v, sizeof(*v)); 3520 return raw_atomic_long_fetch_sub(i, v); 3521 } 3522 3523 /** 3524 * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering 3525 * @i: long value to subtract 3526 * @v: pointer to atomic_long_t 3527 * 3528 * Atomically updates @v to (@v - @i) with acquire ordering. 3529 * 3530 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there. 3531 * 3532 * Return: The original value of @v. 3533 */ 3534 static __always_inline long 3535 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) 3536 { 3537 instrument_atomic_read_write(v, sizeof(*v)); 3538 return raw_atomic_long_fetch_sub_acquire(i, v); 3539 } 3540 3541 /** 3542 * atomic_long_fetch_sub_release() - atomic subtract with release ordering 3543 * @i: long value to subtract 3544 * @v: pointer to atomic_long_t 3545 * 3546 * Atomically updates @v to (@v - @i) with release ordering. 3547 * 3548 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there. 3549 * 3550 * Return: The original value of @v. 3551 */ 3552 static __always_inline long 3553 atomic_long_fetch_sub_release(long i, atomic_long_t *v) 3554 { 3555 kcsan_release(); 3556 instrument_atomic_read_write(v, sizeof(*v)); 3557 return raw_atomic_long_fetch_sub_release(i, v); 3558 } 3559 3560 /** 3561 * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering 3562 * @i: long value to subtract 3563 * @v: pointer to atomic_long_t 3564 * 3565 * Atomically updates @v to (@v - @i) with relaxed ordering. 3566 * 3567 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there. 3568 * 3569 * Return: The original value of @v. 3570 */ 3571 static __always_inline long 3572 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) 3573 { 3574 instrument_atomic_read_write(v, sizeof(*v)); 3575 return raw_atomic_long_fetch_sub_relaxed(i, v); 3576 } 3577 3578 /** 3579 * atomic_long_inc() - atomic increment with relaxed ordering 3580 * @v: pointer to atomic_long_t 3581 * 3582 * Atomically updates @v to (@v + 1) with relaxed ordering. 3583 * 3584 * Unsafe to use in noinstr code; use raw_atomic_long_inc() there. 3585 * 3586 * Return: Nothing. 3587 */ 3588 static __always_inline void 3589 atomic_long_inc(atomic_long_t *v) 3590 { 3591 instrument_atomic_read_write(v, sizeof(*v)); 3592 raw_atomic_long_inc(v); 3593 } 3594 3595 /** 3596 * atomic_long_inc_return() - atomic increment with full ordering 3597 * @v: pointer to atomic_long_t 3598 * 3599 * Atomically updates @v to (@v + 1) with full ordering. 3600 * 3601 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there. 3602 * 3603 * Return: The updated value of @v. 3604 */ 3605 static __always_inline long 3606 atomic_long_inc_return(atomic_long_t *v) 3607 { 3608 kcsan_mb(); 3609 instrument_atomic_read_write(v, sizeof(*v)); 3610 return raw_atomic_long_inc_return(v); 3611 } 3612 3613 /** 3614 * atomic_long_inc_return_acquire() - atomic increment with acquire ordering 3615 * @v: pointer to atomic_long_t 3616 * 3617 * Atomically updates @v to (@v + 1) with acquire ordering. 3618 * 3619 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there. 3620 * 3621 * Return: The updated value of @v. 3622 */ 3623 static __always_inline long 3624 atomic_long_inc_return_acquire(atomic_long_t *v) 3625 { 3626 instrument_atomic_read_write(v, sizeof(*v)); 3627 return raw_atomic_long_inc_return_acquire(v); 3628 } 3629 3630 /** 3631 * atomic_long_inc_return_release() - atomic increment with release ordering 3632 * @v: pointer to atomic_long_t 3633 * 3634 * Atomically updates @v to (@v + 1) with release ordering. 3635 * 3636 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there. 3637 * 3638 * Return: The updated value of @v. 3639 */ 3640 static __always_inline long 3641 atomic_long_inc_return_release(atomic_long_t *v) 3642 { 3643 kcsan_release(); 3644 instrument_atomic_read_write(v, sizeof(*v)); 3645 return raw_atomic_long_inc_return_release(v); 3646 } 3647 3648 /** 3649 * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering 3650 * @v: pointer to atomic_long_t 3651 * 3652 * Atomically updates @v to (@v + 1) with relaxed ordering. 3653 * 3654 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there. 3655 * 3656 * Return: The updated value of @v. 3657 */ 3658 static __always_inline long 3659 atomic_long_inc_return_relaxed(atomic_long_t *v) 3660 { 3661 instrument_atomic_read_write(v, sizeof(*v)); 3662 return raw_atomic_long_inc_return_relaxed(v); 3663 } 3664 3665 /** 3666 * atomic_long_fetch_inc() - atomic increment with full ordering 3667 * @v: pointer to atomic_long_t 3668 * 3669 * Atomically updates @v to (@v + 1) with full ordering. 3670 * 3671 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there. 3672 * 3673 * Return: The original value of @v. 3674 */ 3675 static __always_inline long 3676 atomic_long_fetch_inc(atomic_long_t *v) 3677 { 3678 kcsan_mb(); 3679 instrument_atomic_read_write(v, sizeof(*v)); 3680 return raw_atomic_long_fetch_inc(v); 3681 } 3682 3683 /** 3684 * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering 3685 * @v: pointer to atomic_long_t 3686 * 3687 * Atomically updates @v to (@v + 1) with acquire ordering. 3688 * 3689 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there. 3690 * 3691 * Return: The original value of @v. 3692 */ 3693 static __always_inline long 3694 atomic_long_fetch_inc_acquire(atomic_long_t *v) 3695 { 3696 instrument_atomic_read_write(v, sizeof(*v)); 3697 return raw_atomic_long_fetch_inc_acquire(v); 3698 } 3699 3700 /** 3701 * atomic_long_fetch_inc_release() - atomic increment with release ordering 3702 * @v: pointer to atomic_long_t 3703 * 3704 * Atomically updates @v to (@v + 1) with release ordering. 3705 * 3706 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there. 3707 * 3708 * Return: The original value of @v. 3709 */ 3710 static __always_inline long 3711 atomic_long_fetch_inc_release(atomic_long_t *v) 3712 { 3713 kcsan_release(); 3714 instrument_atomic_read_write(v, sizeof(*v)); 3715 return raw_atomic_long_fetch_inc_release(v); 3716 } 3717 3718 /** 3719 * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering 3720 * @v: pointer to atomic_long_t 3721 * 3722 * Atomically updates @v to (@v + 1) with relaxed ordering. 3723 * 3724 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there. 3725 * 3726 * Return: The original value of @v. 3727 */ 3728 static __always_inline long 3729 atomic_long_fetch_inc_relaxed(atomic_long_t *v) 3730 { 3731 instrument_atomic_read_write(v, sizeof(*v)); 3732 return raw_atomic_long_fetch_inc_relaxed(v); 3733 } 3734 3735 /** 3736 * atomic_long_dec() - atomic decrement with relaxed ordering 3737 * @v: pointer to atomic_long_t 3738 * 3739 * Atomically updates @v to (@v - 1) with relaxed ordering. 3740 * 3741 * Unsafe to use in noinstr code; use raw_atomic_long_dec() there. 3742 * 3743 * Return: Nothing. 3744 */ 3745 static __always_inline void 3746 atomic_long_dec(atomic_long_t *v) 3747 { 3748 instrument_atomic_read_write(v, sizeof(*v)); 3749 raw_atomic_long_dec(v); 3750 } 3751 3752 /** 3753 * atomic_long_dec_return() - atomic decrement with full ordering 3754 * @v: pointer to atomic_long_t 3755 * 3756 * Atomically updates @v to (@v - 1) with full ordering. 3757 * 3758 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there. 3759 * 3760 * Return: The updated value of @v. 3761 */ 3762 static __always_inline long 3763 atomic_long_dec_return(atomic_long_t *v) 3764 { 3765 kcsan_mb(); 3766 instrument_atomic_read_write(v, sizeof(*v)); 3767 return raw_atomic_long_dec_return(v); 3768 } 3769 3770 /** 3771 * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering 3772 * @v: pointer to atomic_long_t 3773 * 3774 * Atomically updates @v to (@v - 1) with acquire ordering. 3775 * 3776 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there. 3777 * 3778 * Return: The updated value of @v. 3779 */ 3780 static __always_inline long 3781 atomic_long_dec_return_acquire(atomic_long_t *v) 3782 { 3783 instrument_atomic_read_write(v, sizeof(*v)); 3784 return raw_atomic_long_dec_return_acquire(v); 3785 } 3786 3787 /** 3788 * atomic_long_dec_return_release() - atomic decrement with release ordering 3789 * @v: pointer to atomic_long_t 3790 * 3791 * Atomically updates @v to (@v - 1) with release ordering. 3792 * 3793 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there. 3794 * 3795 * Return: The updated value of @v. 3796 */ 3797 static __always_inline long 3798 atomic_long_dec_return_release(atomic_long_t *v) 3799 { 3800 kcsan_release(); 3801 instrument_atomic_read_write(v, sizeof(*v)); 3802 return raw_atomic_long_dec_return_release(v); 3803 } 3804 3805 /** 3806 * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering 3807 * @v: pointer to atomic_long_t 3808 * 3809 * Atomically updates @v to (@v - 1) with relaxed ordering. 3810 * 3811 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there. 3812 * 3813 * Return: The updated value of @v. 3814 */ 3815 static __always_inline long 3816 atomic_long_dec_return_relaxed(atomic_long_t *v) 3817 { 3818 instrument_atomic_read_write(v, sizeof(*v)); 3819 return raw_atomic_long_dec_return_relaxed(v); 3820 } 3821 3822 /** 3823 * atomic_long_fetch_dec() - atomic decrement with full ordering 3824 * @v: pointer to atomic_long_t 3825 * 3826 * Atomically updates @v to (@v - 1) with full ordering. 3827 * 3828 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there. 3829 * 3830 * Return: The original value of @v. 3831 */ 3832 static __always_inline long 3833 atomic_long_fetch_dec(atomic_long_t *v) 3834 { 3835 kcsan_mb(); 3836 instrument_atomic_read_write(v, sizeof(*v)); 3837 return raw_atomic_long_fetch_dec(v); 3838 } 3839 3840 /** 3841 * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering 3842 * @v: pointer to atomic_long_t 3843 * 3844 * Atomically updates @v to (@v - 1) with acquire ordering. 3845 * 3846 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there. 3847 * 3848 * Return: The original value of @v. 3849 */ 3850 static __always_inline long 3851 atomic_long_fetch_dec_acquire(atomic_long_t *v) 3852 { 3853 instrument_atomic_read_write(v, sizeof(*v)); 3854 return raw_atomic_long_fetch_dec_acquire(v); 3855 } 3856 3857 /** 3858 * atomic_long_fetch_dec_release() - atomic decrement with release ordering 3859 * @v: pointer to atomic_long_t 3860 * 3861 * Atomically updates @v to (@v - 1) with release ordering. 3862 * 3863 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there. 3864 * 3865 * Return: The original value of @v. 3866 */ 3867 static __always_inline long 3868 atomic_long_fetch_dec_release(atomic_long_t *v) 3869 { 3870 kcsan_release(); 3871 instrument_atomic_read_write(v, sizeof(*v)); 3872 return raw_atomic_long_fetch_dec_release(v); 3873 } 3874 3875 /** 3876 * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering 3877 * @v: pointer to atomic_long_t 3878 * 3879 * Atomically updates @v to (@v - 1) with relaxed ordering. 3880 * 3881 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there. 3882 * 3883 * Return: The original value of @v. 3884 */ 3885 static __always_inline long 3886 atomic_long_fetch_dec_relaxed(atomic_long_t *v) 3887 { 3888 instrument_atomic_read_write(v, sizeof(*v)); 3889 return raw_atomic_long_fetch_dec_relaxed(v); 3890 } 3891 3892 /** 3893 * atomic_long_and() - atomic bitwise AND with relaxed ordering 3894 * @i: long value 3895 * @v: pointer to atomic_long_t 3896 * 3897 * Atomically updates @v to (@v & @i) with relaxed ordering. 3898 * 3899 * Unsafe to use in noinstr code; use raw_atomic_long_and() there. 3900 * 3901 * Return: Nothing. 3902 */ 3903 static __always_inline void 3904 atomic_long_and(long i, atomic_long_t *v) 3905 { 3906 instrument_atomic_read_write(v, sizeof(*v)); 3907 raw_atomic_long_and(i, v); 3908 } 3909 3910 /** 3911 * atomic_long_fetch_and() - atomic bitwise AND with full ordering 3912 * @i: long value 3913 * @v: pointer to atomic_long_t 3914 * 3915 * Atomically updates @v to (@v & @i) with full ordering. 3916 * 3917 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there. 3918 * 3919 * Return: The original value of @v. 3920 */ 3921 static __always_inline long 3922 atomic_long_fetch_and(long i, atomic_long_t *v) 3923 { 3924 kcsan_mb(); 3925 instrument_atomic_read_write(v, sizeof(*v)); 3926 return raw_atomic_long_fetch_and(i, v); 3927 } 3928 3929 /** 3930 * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering 3931 * @i: long value 3932 * @v: pointer to atomic_long_t 3933 * 3934 * Atomically updates @v to (@v & @i) with acquire ordering. 3935 * 3936 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there. 3937 * 3938 * Return: The original value of @v. 3939 */ 3940 static __always_inline long 3941 atomic_long_fetch_and_acquire(long i, atomic_long_t *v) 3942 { 3943 instrument_atomic_read_write(v, sizeof(*v)); 3944 return raw_atomic_long_fetch_and_acquire(i, v); 3945 } 3946 3947 /** 3948 * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering 3949 * @i: long value 3950 * @v: pointer to atomic_long_t 3951 * 3952 * Atomically updates @v to (@v & @i) with release ordering. 3953 * 3954 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there. 3955 * 3956 * Return: The original value of @v. 3957 */ 3958 static __always_inline long 3959 atomic_long_fetch_and_release(long i, atomic_long_t *v) 3960 { 3961 kcsan_release(); 3962 instrument_atomic_read_write(v, sizeof(*v)); 3963 return raw_atomic_long_fetch_and_release(i, v); 3964 } 3965 3966 /** 3967 * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering 3968 * @i: long value 3969 * @v: pointer to atomic_long_t 3970 * 3971 * Atomically updates @v to (@v & @i) with relaxed ordering. 3972 * 3973 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there. 3974 * 3975 * Return: The original value of @v. 3976 */ 3977 static __always_inline long 3978 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) 3979 { 3980 instrument_atomic_read_write(v, sizeof(*v)); 3981 return raw_atomic_long_fetch_and_relaxed(i, v); 3982 } 3983 3984 /** 3985 * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering 3986 * @i: long value 3987 * @v: pointer to atomic_long_t 3988 * 3989 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 3990 * 3991 * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there. 3992 * 3993 * Return: Nothing. 3994 */ 3995 static __always_inline void 3996 atomic_long_andnot(long i, atomic_long_t *v) 3997 { 3998 instrument_atomic_read_write(v, sizeof(*v)); 3999 raw_atomic_long_andnot(i, v); 4000 } 4001 4002 /** 4003 * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering 4004 * @i: long value 4005 * @v: pointer to atomic_long_t 4006 * 4007 * Atomically updates @v to (@v & ~@i) with full ordering. 4008 * 4009 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there. 4010 * 4011 * Return: The original value of @v. 4012 */ 4013 static __always_inline long 4014 atomic_long_fetch_andnot(long i, atomic_long_t *v) 4015 { 4016 kcsan_mb(); 4017 instrument_atomic_read_write(v, sizeof(*v)); 4018 return raw_atomic_long_fetch_andnot(i, v); 4019 } 4020 4021 /** 4022 * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering 4023 * @i: long value 4024 * @v: pointer to atomic_long_t 4025 * 4026 * Atomically updates @v to (@v & ~@i) with acquire ordering. 4027 * 4028 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there. 4029 * 4030 * Return: The original value of @v. 4031 */ 4032 static __always_inline long 4033 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) 4034 { 4035 instrument_atomic_read_write(v, sizeof(*v)); 4036 return raw_atomic_long_fetch_andnot_acquire(i, v); 4037 } 4038 4039 /** 4040 * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering 4041 * @i: long value 4042 * @v: pointer to atomic_long_t 4043 * 4044 * Atomically updates @v to (@v & ~@i) with release ordering. 4045 * 4046 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there. 4047 * 4048 * Return: The original value of @v. 4049 */ 4050 static __always_inline long 4051 atomic_long_fetch_andnot_release(long i, atomic_long_t *v) 4052 { 4053 kcsan_release(); 4054 instrument_atomic_read_write(v, sizeof(*v)); 4055 return raw_atomic_long_fetch_andnot_release(i, v); 4056 } 4057 4058 /** 4059 * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering 4060 * @i: long value 4061 * @v: pointer to atomic_long_t 4062 * 4063 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 4064 * 4065 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there. 4066 * 4067 * Return: The original value of @v. 4068 */ 4069 static __always_inline long 4070 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) 4071 { 4072 instrument_atomic_read_write(v, sizeof(*v)); 4073 return raw_atomic_long_fetch_andnot_relaxed(i, v); 4074 } 4075 4076 /** 4077 * atomic_long_or() - atomic bitwise OR with relaxed ordering 4078 * @i: long value 4079 * @v: pointer to atomic_long_t 4080 * 4081 * Atomically updates @v to (@v | @i) with relaxed ordering. 4082 * 4083 * Unsafe to use in noinstr code; use raw_atomic_long_or() there. 4084 * 4085 * Return: Nothing. 4086 */ 4087 static __always_inline void 4088 atomic_long_or(long i, atomic_long_t *v) 4089 { 4090 instrument_atomic_read_write(v, sizeof(*v)); 4091 raw_atomic_long_or(i, v); 4092 } 4093 4094 /** 4095 * atomic_long_fetch_or() - atomic bitwise OR with full ordering 4096 * @i: long value 4097 * @v: pointer to atomic_long_t 4098 * 4099 * Atomically updates @v to (@v | @i) with full ordering. 4100 * 4101 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there. 4102 * 4103 * Return: The original value of @v. 4104 */ 4105 static __always_inline long 4106 atomic_long_fetch_or(long i, atomic_long_t *v) 4107 { 4108 kcsan_mb(); 4109 instrument_atomic_read_write(v, sizeof(*v)); 4110 return raw_atomic_long_fetch_or(i, v); 4111 } 4112 4113 /** 4114 * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering 4115 * @i: long value 4116 * @v: pointer to atomic_long_t 4117 * 4118 * Atomically updates @v to (@v | @i) with acquire ordering. 4119 * 4120 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there. 4121 * 4122 * Return: The original value of @v. 4123 */ 4124 static __always_inline long 4125 atomic_long_fetch_or_acquire(long i, atomic_long_t *v) 4126 { 4127 instrument_atomic_read_write(v, sizeof(*v)); 4128 return raw_atomic_long_fetch_or_acquire(i, v); 4129 } 4130 4131 /** 4132 * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering 4133 * @i: long value 4134 * @v: pointer to atomic_long_t 4135 * 4136 * Atomically updates @v to (@v | @i) with release ordering. 4137 * 4138 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there. 4139 * 4140 * Return: The original value of @v. 4141 */ 4142 static __always_inline long 4143 atomic_long_fetch_or_release(long i, atomic_long_t *v) 4144 { 4145 kcsan_release(); 4146 instrument_atomic_read_write(v, sizeof(*v)); 4147 return raw_atomic_long_fetch_or_release(i, v); 4148 } 4149 4150 /** 4151 * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering 4152 * @i: long value 4153 * @v: pointer to atomic_long_t 4154 * 4155 * Atomically updates @v to (@v | @i) with relaxed ordering. 4156 * 4157 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there. 4158 * 4159 * Return: The original value of @v. 4160 */ 4161 static __always_inline long 4162 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) 4163 { 4164 instrument_atomic_read_write(v, sizeof(*v)); 4165 return raw_atomic_long_fetch_or_relaxed(i, v); 4166 } 4167 4168 /** 4169 * atomic_long_xor() - atomic bitwise XOR with relaxed ordering 4170 * @i: long value 4171 * @v: pointer to atomic_long_t 4172 * 4173 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 4174 * 4175 * Unsafe to use in noinstr code; use raw_atomic_long_xor() there. 4176 * 4177 * Return: Nothing. 4178 */ 4179 static __always_inline void 4180 atomic_long_xor(long i, atomic_long_t *v) 4181 { 4182 instrument_atomic_read_write(v, sizeof(*v)); 4183 raw_atomic_long_xor(i, v); 4184 } 4185 4186 /** 4187 * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering 4188 * @i: long value 4189 * @v: pointer to atomic_long_t 4190 * 4191 * Atomically updates @v to (@v ^ @i) with full ordering. 4192 * 4193 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there. 4194 * 4195 * Return: The original value of @v. 4196 */ 4197 static __always_inline long 4198 atomic_long_fetch_xor(long i, atomic_long_t *v) 4199 { 4200 kcsan_mb(); 4201 instrument_atomic_read_write(v, sizeof(*v)); 4202 return raw_atomic_long_fetch_xor(i, v); 4203 } 4204 4205 /** 4206 * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering 4207 * @i: long value 4208 * @v: pointer to atomic_long_t 4209 * 4210 * Atomically updates @v to (@v ^ @i) with acquire ordering. 4211 * 4212 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there. 4213 * 4214 * Return: The original value of @v. 4215 */ 4216 static __always_inline long 4217 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) 4218 { 4219 instrument_atomic_read_write(v, sizeof(*v)); 4220 return raw_atomic_long_fetch_xor_acquire(i, v); 4221 } 4222 4223 /** 4224 * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering 4225 * @i: long value 4226 * @v: pointer to atomic_long_t 4227 * 4228 * Atomically updates @v to (@v ^ @i) with release ordering. 4229 * 4230 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there. 4231 * 4232 * Return: The original value of @v. 4233 */ 4234 static __always_inline long 4235 atomic_long_fetch_xor_release(long i, atomic_long_t *v) 4236 { 4237 kcsan_release(); 4238 instrument_atomic_read_write(v, sizeof(*v)); 4239 return raw_atomic_long_fetch_xor_release(i, v); 4240 } 4241 4242 /** 4243 * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering 4244 * @i: long value 4245 * @v: pointer to atomic_long_t 4246 * 4247 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 4248 * 4249 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there. 4250 * 4251 * Return: The original value of @v. 4252 */ 4253 static __always_inline long 4254 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) 4255 { 4256 instrument_atomic_read_write(v, sizeof(*v)); 4257 return raw_atomic_long_fetch_xor_relaxed(i, v); 4258 } 4259 4260 /** 4261 * atomic_long_xchg() - atomic exchange with full ordering 4262 * @v: pointer to atomic_long_t 4263 * @new: long value to assign 4264 * 4265 * Atomically updates @v to @new with full ordering. 4266 * 4267 * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there. 4268 * 4269 * Return: The original value of @v. 4270 */ 4271 static __always_inline long 4272 atomic_long_xchg(atomic_long_t *v, long new) 4273 { 4274 kcsan_mb(); 4275 instrument_atomic_read_write(v, sizeof(*v)); 4276 return raw_atomic_long_xchg(v, new); 4277 } 4278 4279 /** 4280 * atomic_long_xchg_acquire() - atomic exchange with acquire ordering 4281 * @v: pointer to atomic_long_t 4282 * @new: long value to assign 4283 * 4284 * Atomically updates @v to @new with acquire ordering. 4285 * 4286 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there. 4287 * 4288 * Return: The original value of @v. 4289 */ 4290 static __always_inline long 4291 atomic_long_xchg_acquire(atomic_long_t *v, long new) 4292 { 4293 instrument_atomic_read_write(v, sizeof(*v)); 4294 return raw_atomic_long_xchg_acquire(v, new); 4295 } 4296 4297 /** 4298 * atomic_long_xchg_release() - atomic exchange with release ordering 4299 * @v: pointer to atomic_long_t 4300 * @new: long value to assign 4301 * 4302 * Atomically updates @v to @new with release ordering. 4303 * 4304 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there. 4305 * 4306 * Return: The original value of @v. 4307 */ 4308 static __always_inline long 4309 atomic_long_xchg_release(atomic_long_t *v, long new) 4310 { 4311 kcsan_release(); 4312 instrument_atomic_read_write(v, sizeof(*v)); 4313 return raw_atomic_long_xchg_release(v, new); 4314 } 4315 4316 /** 4317 * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering 4318 * @v: pointer to atomic_long_t 4319 * @new: long value to assign 4320 * 4321 * Atomically updates @v to @new with relaxed ordering. 4322 * 4323 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there. 4324 * 4325 * Return: The original value of @v. 4326 */ 4327 static __always_inline long 4328 atomic_long_xchg_relaxed(atomic_long_t *v, long new) 4329 { 4330 instrument_atomic_read_write(v, sizeof(*v)); 4331 return raw_atomic_long_xchg_relaxed(v, new); 4332 } 4333 4334 /** 4335 * atomic_long_cmpxchg() - atomic compare and exchange with full ordering 4336 * @v: pointer to atomic_long_t 4337 * @old: long value to compare with 4338 * @new: long value to assign 4339 * 4340 * If (@v == @old), atomically updates @v to @new with full ordering. 4341 * Otherwise, @v is not modified and relaxed ordering is provided. 4342 * 4343 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there. 4344 * 4345 * Return: The original value of @v. 4346 */ 4347 static __always_inline long 4348 atomic_long_cmpxchg(atomic_long_t *v, long old, long new) 4349 { 4350 kcsan_mb(); 4351 instrument_atomic_read_write(v, sizeof(*v)); 4352 return raw_atomic_long_cmpxchg(v, old, new); 4353 } 4354 4355 /** 4356 * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 4357 * @v: pointer to atomic_long_t 4358 * @old: long value to compare with 4359 * @new: long value to assign 4360 * 4361 * If (@v == @old), atomically updates @v to @new with acquire ordering. 4362 * Otherwise, @v is not modified and relaxed ordering is provided. 4363 * 4364 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there. 4365 * 4366 * Return: The original value of @v. 4367 */ 4368 static __always_inline long 4369 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) 4370 { 4371 instrument_atomic_read_write(v, sizeof(*v)); 4372 return raw_atomic_long_cmpxchg_acquire(v, old, new); 4373 } 4374 4375 /** 4376 * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering 4377 * @v: pointer to atomic_long_t 4378 * @old: long value to compare with 4379 * @new: long value to assign 4380 * 4381 * If (@v == @old), atomically updates @v to @new with release ordering. 4382 * Otherwise, @v is not modified and relaxed ordering is provided. 4383 * 4384 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there. 4385 * 4386 * Return: The original value of @v. 4387 */ 4388 static __always_inline long 4389 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) 4390 { 4391 kcsan_release(); 4392 instrument_atomic_read_write(v, sizeof(*v)); 4393 return raw_atomic_long_cmpxchg_release(v, old, new); 4394 } 4395 4396 /** 4397 * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 4398 * @v: pointer to atomic_long_t 4399 * @old: long value to compare with 4400 * @new: long value to assign 4401 * 4402 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 4403 * Otherwise, @v is not modified and relaxed ordering is provided. 4404 * 4405 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there. 4406 * 4407 * Return: The original value of @v. 4408 */ 4409 static __always_inline long 4410 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) 4411 { 4412 instrument_atomic_read_write(v, sizeof(*v)); 4413 return raw_atomic_long_cmpxchg_relaxed(v, old, new); 4414 } 4415 4416 /** 4417 * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering 4418 * @v: pointer to atomic_long_t 4419 * @old: pointer to long value to compare with 4420 * @new: long value to assign 4421 * 4422 * If (@v == @old), atomically updates @v to @new with full ordering. 4423 * Otherwise, @v is not modified, @old is updated to the current value of @v, 4424 * and relaxed ordering is provided. 4425 * 4426 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there. 4427 * 4428 * Return: @true if the exchange occured, @false otherwise. 4429 */ 4430 static __always_inline bool 4431 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) 4432 { 4433 kcsan_mb(); 4434 instrument_atomic_read_write(v, sizeof(*v)); 4435 instrument_atomic_read_write(old, sizeof(*old)); 4436 return raw_atomic_long_try_cmpxchg(v, old, new); 4437 } 4438 4439 /** 4440 * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 4441 * @v: pointer to atomic_long_t 4442 * @old: pointer to long value to compare with 4443 * @new: long value to assign 4444 * 4445 * If (@v == @old), atomically updates @v to @new with acquire ordering. 4446 * Otherwise, @v is not modified, @old is updated to the current value of @v, 4447 * and relaxed ordering is provided. 4448 * 4449 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there. 4450 * 4451 * Return: @true if the exchange occured, @false otherwise. 4452 */ 4453 static __always_inline bool 4454 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) 4455 { 4456 instrument_atomic_read_write(v, sizeof(*v)); 4457 instrument_atomic_read_write(old, sizeof(*old)); 4458 return raw_atomic_long_try_cmpxchg_acquire(v, old, new); 4459 } 4460 4461 /** 4462 * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering 4463 * @v: pointer to atomic_long_t 4464 * @old: pointer to long value to compare with 4465 * @new: long value to assign 4466 * 4467 * If (@v == @old), atomically updates @v to @new with release ordering. 4468 * Otherwise, @v is not modified, @old is updated to the current value of @v, 4469 * and relaxed ordering is provided. 4470 * 4471 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there. 4472 * 4473 * Return: @true if the exchange occured, @false otherwise. 4474 */ 4475 static __always_inline bool 4476 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) 4477 { 4478 kcsan_release(); 4479 instrument_atomic_read_write(v, sizeof(*v)); 4480 instrument_atomic_read_write(old, sizeof(*old)); 4481 return raw_atomic_long_try_cmpxchg_release(v, old, new); 4482 } 4483 4484 /** 4485 * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 4486 * @v: pointer to atomic_long_t 4487 * @old: pointer to long value to compare with 4488 * @new: long value to assign 4489 * 4490 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 4491 * Otherwise, @v is not modified, @old is updated to the current value of @v, 4492 * and relaxed ordering is provided. 4493 * 4494 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there. 4495 * 4496 * Return: @true if the exchange occured, @false otherwise. 4497 */ 4498 static __always_inline bool 4499 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) 4500 { 4501 instrument_atomic_read_write(v, sizeof(*v)); 4502 instrument_atomic_read_write(old, sizeof(*old)); 4503 return raw_atomic_long_try_cmpxchg_relaxed(v, old, new); 4504 } 4505 4506 /** 4507 * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering 4508 * @i: long value to subtract 4509 * @v: pointer to atomic_long_t 4510 * 4511 * Atomically updates @v to (@v - @i) with full ordering. 4512 * 4513 * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there. 4514 * 4515 * Return: @true if the resulting value of @v is zero, @false otherwise. 4516 */ 4517 static __always_inline bool 4518 atomic_long_sub_and_test(long i, atomic_long_t *v) 4519 { 4520 kcsan_mb(); 4521 instrument_atomic_read_write(v, sizeof(*v)); 4522 return raw_atomic_long_sub_and_test(i, v); 4523 } 4524 4525 /** 4526 * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering 4527 * @v: pointer to atomic_long_t 4528 * 4529 * Atomically updates @v to (@v - 1) with full ordering. 4530 * 4531 * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there. 4532 * 4533 * Return: @true if the resulting value of @v is zero, @false otherwise. 4534 */ 4535 static __always_inline bool 4536 atomic_long_dec_and_test(atomic_long_t *v) 4537 { 4538 kcsan_mb(); 4539 instrument_atomic_read_write(v, sizeof(*v)); 4540 return raw_atomic_long_dec_and_test(v); 4541 } 4542 4543 /** 4544 * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering 4545 * @v: pointer to atomic_long_t 4546 * 4547 * Atomically updates @v to (@v + 1) with full ordering. 4548 * 4549 * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there. 4550 * 4551 * Return: @true if the resulting value of @v is zero, @false otherwise. 4552 */ 4553 static __always_inline bool 4554 atomic_long_inc_and_test(atomic_long_t *v) 4555 { 4556 kcsan_mb(); 4557 instrument_atomic_read_write(v, sizeof(*v)); 4558 return raw_atomic_long_inc_and_test(v); 4559 } 4560 4561 /** 4562 * atomic_long_add_negative() - atomic add and test if negative with full ordering 4563 * @i: long value to add 4564 * @v: pointer to atomic_long_t 4565 * 4566 * Atomically updates @v to (@v + @i) with full ordering. 4567 * 4568 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there. 4569 * 4570 * Return: @true if the resulting value of @v is negative, @false otherwise. 4571 */ 4572 static __always_inline bool 4573 atomic_long_add_negative(long i, atomic_long_t *v) 4574 { 4575 kcsan_mb(); 4576 instrument_atomic_read_write(v, sizeof(*v)); 4577 return raw_atomic_long_add_negative(i, v); 4578 } 4579 4580 /** 4581 * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering 4582 * @i: long value to add 4583 * @v: pointer to atomic_long_t 4584 * 4585 * Atomically updates @v to (@v + @i) with acquire ordering. 4586 * 4587 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there. 4588 * 4589 * Return: @true if the resulting value of @v is negative, @false otherwise. 4590 */ 4591 static __always_inline bool 4592 atomic_long_add_negative_acquire(long i, atomic_long_t *v) 4593 { 4594 instrument_atomic_read_write(v, sizeof(*v)); 4595 return raw_atomic_long_add_negative_acquire(i, v); 4596 } 4597 4598 /** 4599 * atomic_long_add_negative_release() - atomic add and test if negative with release ordering 4600 * @i: long value to add 4601 * @v: pointer to atomic_long_t 4602 * 4603 * Atomically updates @v to (@v + @i) with release ordering. 4604 * 4605 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there. 4606 * 4607 * Return: @true if the resulting value of @v is negative, @false otherwise. 4608 */ 4609 static __always_inline bool 4610 atomic_long_add_negative_release(long i, atomic_long_t *v) 4611 { 4612 kcsan_release(); 4613 instrument_atomic_read_write(v, sizeof(*v)); 4614 return raw_atomic_long_add_negative_release(i, v); 4615 } 4616 4617 /** 4618 * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering 4619 * @i: long value to add 4620 * @v: pointer to atomic_long_t 4621 * 4622 * Atomically updates @v to (@v + @i) with relaxed ordering. 4623 * 4624 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there. 4625 * 4626 * Return: @true if the resulting value of @v is negative, @false otherwise. 4627 */ 4628 static __always_inline bool 4629 atomic_long_add_negative_relaxed(long i, atomic_long_t *v) 4630 { 4631 instrument_atomic_read_write(v, sizeof(*v)); 4632 return raw_atomic_long_add_negative_relaxed(i, v); 4633 } 4634 4635 /** 4636 * atomic_long_fetch_add_unless() - atomic add unless value with full ordering 4637 * @v: pointer to atomic_long_t 4638 * @a: long value to add 4639 * @u: long value to compare with 4640 * 4641 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 4642 * Otherwise, @v is not modified and relaxed ordering is provided. 4643 * 4644 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there. 4645 * 4646 * Return: The original value of @v. 4647 */ 4648 static __always_inline long 4649 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) 4650 { 4651 kcsan_mb(); 4652 instrument_atomic_read_write(v, sizeof(*v)); 4653 return raw_atomic_long_fetch_add_unless(v, a, u); 4654 } 4655 4656 /** 4657 * atomic_long_add_unless() - atomic add unless value with full ordering 4658 * @v: pointer to atomic_long_t 4659 * @a: long value to add 4660 * @u: long value to compare with 4661 * 4662 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 4663 * Otherwise, @v is not modified and relaxed ordering is provided. 4664 * 4665 * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there. 4666 * 4667 * Return: @true if @v was updated, @false otherwise. 4668 */ 4669 static __always_inline bool 4670 atomic_long_add_unless(atomic_long_t *v, long a, long u) 4671 { 4672 kcsan_mb(); 4673 instrument_atomic_read_write(v, sizeof(*v)); 4674 return raw_atomic_long_add_unless(v, a, u); 4675 } 4676 4677 /** 4678 * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering 4679 * @v: pointer to atomic_long_t 4680 * 4681 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. 4682 * Otherwise, @v is not modified and relaxed ordering is provided. 4683 * 4684 * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there. 4685 * 4686 * Return: @true if @v was updated, @false otherwise. 4687 */ 4688 static __always_inline bool 4689 atomic_long_inc_not_zero(atomic_long_t *v) 4690 { 4691 kcsan_mb(); 4692 instrument_atomic_read_write(v, sizeof(*v)); 4693 return raw_atomic_long_inc_not_zero(v); 4694 } 4695 4696 /** 4697 * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering 4698 * @v: pointer to atomic_long_t 4699 * 4700 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. 4701 * Otherwise, @v is not modified and relaxed ordering is provided. 4702 * 4703 * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there. 4704 * 4705 * Return: @true if @v was updated, @false otherwise. 4706 */ 4707 static __always_inline bool 4708 atomic_long_inc_unless_negative(atomic_long_t *v) 4709 { 4710 kcsan_mb(); 4711 instrument_atomic_read_write(v, sizeof(*v)); 4712 return raw_atomic_long_inc_unless_negative(v); 4713 } 4714 4715 /** 4716 * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering 4717 * @v: pointer to atomic_long_t 4718 * 4719 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. 4720 * Otherwise, @v is not modified and relaxed ordering is provided. 4721 * 4722 * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there. 4723 * 4724 * Return: @true if @v was updated, @false otherwise. 4725 */ 4726 static __always_inline bool 4727 atomic_long_dec_unless_positive(atomic_long_t *v) 4728 { 4729 kcsan_mb(); 4730 instrument_atomic_read_write(v, sizeof(*v)); 4731 return raw_atomic_long_dec_unless_positive(v); 4732 } 4733 4734 /** 4735 * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering 4736 * @v: pointer to atomic_long_t 4737 * 4738 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. 4739 * Otherwise, @v is not modified and relaxed ordering is provided. 4740 * 4741 * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there. 4742 * 4743 * Return: The old value of (@v - 1), regardless of whether @v was updated. 4744 */ 4745 static __always_inline long 4746 atomic_long_dec_if_positive(atomic_long_t *v) 4747 { 4748 kcsan_mb(); 4749 instrument_atomic_read_write(v, sizeof(*v)); 4750 return raw_atomic_long_dec_if_positive(v); 4751 } 4752 4753 #define xchg(ptr, ...) \ 4754 ({ \ 4755 typeof(ptr) __ai_ptr = (ptr); \ 4756 kcsan_mb(); \ 4757 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4758 raw_xchg(__ai_ptr, __VA_ARGS__); \ 4759 }) 4760 4761 #define xchg_acquire(ptr, ...) \ 4762 ({ \ 4763 typeof(ptr) __ai_ptr = (ptr); \ 4764 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4765 raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \ 4766 }) 4767 4768 #define xchg_release(ptr, ...) \ 4769 ({ \ 4770 typeof(ptr) __ai_ptr = (ptr); \ 4771 kcsan_release(); \ 4772 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4773 raw_xchg_release(__ai_ptr, __VA_ARGS__); \ 4774 }) 4775 4776 #define xchg_relaxed(ptr, ...) \ 4777 ({ \ 4778 typeof(ptr) __ai_ptr = (ptr); \ 4779 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4780 raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ 4781 }) 4782 4783 #define cmpxchg(ptr, ...) \ 4784 ({ \ 4785 typeof(ptr) __ai_ptr = (ptr); \ 4786 kcsan_mb(); \ 4787 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4788 raw_cmpxchg(__ai_ptr, __VA_ARGS__); \ 4789 }) 4790 4791 #define cmpxchg_acquire(ptr, ...) \ 4792 ({ \ 4793 typeof(ptr) __ai_ptr = (ptr); \ 4794 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4795 raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ 4796 }) 4797 4798 #define cmpxchg_release(ptr, ...) \ 4799 ({ \ 4800 typeof(ptr) __ai_ptr = (ptr); \ 4801 kcsan_release(); \ 4802 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4803 raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ 4804 }) 4805 4806 #define cmpxchg_relaxed(ptr, ...) \ 4807 ({ \ 4808 typeof(ptr) __ai_ptr = (ptr); \ 4809 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4810 raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ 4811 }) 4812 4813 #define cmpxchg64(ptr, ...) \ 4814 ({ \ 4815 typeof(ptr) __ai_ptr = (ptr); \ 4816 kcsan_mb(); \ 4817 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4818 raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \ 4819 }) 4820 4821 #define cmpxchg64_acquire(ptr, ...) \ 4822 ({ \ 4823 typeof(ptr) __ai_ptr = (ptr); \ 4824 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4825 raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ 4826 }) 4827 4828 #define cmpxchg64_release(ptr, ...) \ 4829 ({ \ 4830 typeof(ptr) __ai_ptr = (ptr); \ 4831 kcsan_release(); \ 4832 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4833 raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ 4834 }) 4835 4836 #define cmpxchg64_relaxed(ptr, ...) \ 4837 ({ \ 4838 typeof(ptr) __ai_ptr = (ptr); \ 4839 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4840 raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ 4841 }) 4842 4843 #define cmpxchg128(ptr, ...) \ 4844 ({ \ 4845 typeof(ptr) __ai_ptr = (ptr); \ 4846 kcsan_mb(); \ 4847 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4848 raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \ 4849 }) 4850 4851 #define cmpxchg128_acquire(ptr, ...) \ 4852 ({ \ 4853 typeof(ptr) __ai_ptr = (ptr); \ 4854 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4855 raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \ 4856 }) 4857 4858 #define cmpxchg128_release(ptr, ...) \ 4859 ({ \ 4860 typeof(ptr) __ai_ptr = (ptr); \ 4861 kcsan_release(); \ 4862 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4863 raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \ 4864 }) 4865 4866 #define cmpxchg128_relaxed(ptr, ...) \ 4867 ({ \ 4868 typeof(ptr) __ai_ptr = (ptr); \ 4869 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4870 raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \ 4871 }) 4872 4873 #define try_cmpxchg(ptr, oldp, ...) \ 4874 ({ \ 4875 typeof(ptr) __ai_ptr = (ptr); \ 4876 typeof(oldp) __ai_oldp = (oldp); \ 4877 kcsan_mb(); \ 4878 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4879 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4880 raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4881 }) 4882 4883 #define try_cmpxchg_acquire(ptr, oldp, ...) \ 4884 ({ \ 4885 typeof(ptr) __ai_ptr = (ptr); \ 4886 typeof(oldp) __ai_oldp = (oldp); \ 4887 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4888 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4889 raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4890 }) 4891 4892 #define try_cmpxchg_release(ptr, oldp, ...) \ 4893 ({ \ 4894 typeof(ptr) __ai_ptr = (ptr); \ 4895 typeof(oldp) __ai_oldp = (oldp); \ 4896 kcsan_release(); \ 4897 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4898 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4899 raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4900 }) 4901 4902 #define try_cmpxchg_relaxed(ptr, oldp, ...) \ 4903 ({ \ 4904 typeof(ptr) __ai_ptr = (ptr); \ 4905 typeof(oldp) __ai_oldp = (oldp); \ 4906 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4907 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4908 raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4909 }) 4910 4911 #define try_cmpxchg64(ptr, oldp, ...) \ 4912 ({ \ 4913 typeof(ptr) __ai_ptr = (ptr); \ 4914 typeof(oldp) __ai_oldp = (oldp); \ 4915 kcsan_mb(); \ 4916 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4917 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4918 raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4919 }) 4920 4921 #define try_cmpxchg64_acquire(ptr, oldp, ...) \ 4922 ({ \ 4923 typeof(ptr) __ai_ptr = (ptr); \ 4924 typeof(oldp) __ai_oldp = (oldp); \ 4925 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4926 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4927 raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4928 }) 4929 4930 #define try_cmpxchg64_release(ptr, oldp, ...) \ 4931 ({ \ 4932 typeof(ptr) __ai_ptr = (ptr); \ 4933 typeof(oldp) __ai_oldp = (oldp); \ 4934 kcsan_release(); \ 4935 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4936 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4937 raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4938 }) 4939 4940 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \ 4941 ({ \ 4942 typeof(ptr) __ai_ptr = (ptr); \ 4943 typeof(oldp) __ai_oldp = (oldp); \ 4944 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4945 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4946 raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4947 }) 4948 4949 #define try_cmpxchg128(ptr, oldp, ...) \ 4950 ({ \ 4951 typeof(ptr) __ai_ptr = (ptr); \ 4952 typeof(oldp) __ai_oldp = (oldp); \ 4953 kcsan_mb(); \ 4954 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4955 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4956 raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4957 }) 4958 4959 #define try_cmpxchg128_acquire(ptr, oldp, ...) \ 4960 ({ \ 4961 typeof(ptr) __ai_ptr = (ptr); \ 4962 typeof(oldp) __ai_oldp = (oldp); \ 4963 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4964 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4965 raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4966 }) 4967 4968 #define try_cmpxchg128_release(ptr, oldp, ...) \ 4969 ({ \ 4970 typeof(ptr) __ai_ptr = (ptr); \ 4971 typeof(oldp) __ai_oldp = (oldp); \ 4972 kcsan_release(); \ 4973 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4974 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4975 raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4976 }) 4977 4978 #define try_cmpxchg128_relaxed(ptr, oldp, ...) \ 4979 ({ \ 4980 typeof(ptr) __ai_ptr = (ptr); \ 4981 typeof(oldp) __ai_oldp = (oldp); \ 4982 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4983 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 4984 raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 4985 }) 4986 4987 #define cmpxchg_local(ptr, ...) \ 4988 ({ \ 4989 typeof(ptr) __ai_ptr = (ptr); \ 4990 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4991 raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ 4992 }) 4993 4994 #define cmpxchg64_local(ptr, ...) \ 4995 ({ \ 4996 typeof(ptr) __ai_ptr = (ptr); \ 4997 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 4998 raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ 4999 }) 5000 5001 #define cmpxchg128_local(ptr, ...) \ 5002 ({ \ 5003 typeof(ptr) __ai_ptr = (ptr); \ 5004 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 5005 raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \ 5006 }) 5007 5008 #define sync_cmpxchg(ptr, ...) \ 5009 ({ \ 5010 typeof(ptr) __ai_ptr = (ptr); \ 5011 kcsan_mb(); \ 5012 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 5013 raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ 5014 }) 5015 5016 #define try_cmpxchg_local(ptr, oldp, ...) \ 5017 ({ \ 5018 typeof(ptr) __ai_ptr = (ptr); \ 5019 typeof(oldp) __ai_oldp = (oldp); \ 5020 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 5021 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 5022 raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 5023 }) 5024 5025 #define try_cmpxchg64_local(ptr, oldp, ...) \ 5026 ({ \ 5027 typeof(ptr) __ai_ptr = (ptr); \ 5028 typeof(oldp) __ai_oldp = (oldp); \ 5029 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 5030 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 5031 raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 5032 }) 5033 5034 #define try_cmpxchg128_local(ptr, oldp, ...) \ 5035 ({ \ 5036 typeof(ptr) __ai_ptr = (ptr); \ 5037 typeof(oldp) __ai_oldp = (oldp); \ 5038 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 5039 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 5040 raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 5041 }) 5042 5043 #define sync_try_cmpxchg(ptr, ...) \ 5044 ({ \ 5045 typeof(ptr) __ai_ptr = (ptr); \ 5046 kcsan_mb(); \ 5047 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 5048 raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \ 5049 }) 5050 5051 5052 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ 5053 // 8829b337928e9508259079d32581775ececd415b 5054
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.