1 // SPDX-License-Identifier: GPL-2.0+ 2 // 3 // An earlier version of this file appeared in the companion webpage for 4 // "Frightening small children and disconcerting grown-ups: Concurrency 5 // in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern, 6 // which appeared in ASPLOS 2018. 7 8 // ONCE 9 READ_ONCE(X) __load{once}(X) 10 WRITE_ONCE(X,V) { __store{once}(X,V); } 11 12 // Release Acquire and friends 13 smp_store_release(X,V) { __store{release}(*X,V); } 14 smp_load_acquire(X) __load{acquire}(*X) 15 rcu_assign_pointer(X,V) { __store{release}(X,V); } 16 rcu_dereference(X) __load{once}(X) 17 smp_store_mb(X,V) { __store{once}(X,V); __fence{mb}; } 18 19 // Fences 20 smp_mb() { __fence{mb}; } 21 smp_rmb() { __fence{rmb}; } 22 smp_wmb() { __fence{wmb}; } 23 smp_mb__before_atomic() { __fence{before-atomic}; } 24 smp_mb__after_atomic() { __fence{after-atomic}; } 25 smp_mb__after_spinlock() { __fence{after-spinlock}; } 26 smp_mb__after_unlock_lock() { __fence{after-unlock-lock}; } 27 smp_mb__after_srcu_read_unlock() { __fence{after-srcu-read-unlock}; } 28 barrier() { __fence{barrier}; } 29 30 // Exchange 31 xchg(X,V) __xchg{mb}(X,V) 32 xchg_relaxed(X,V) __xchg{once}(X,V) 33 xchg_release(X,V) __xchg{release}(X,V) 34 xchg_acquire(X,V) __xchg{acquire}(X,V) 35 cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W) 36 cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W) 37 cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W) 38 cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W) 39 40 // Spinlocks 41 spin_lock(X) { __lock(X); } 42 spin_unlock(X) { __unlock(X); } 43 spin_trylock(X) __trylock(X) 44 spin_is_locked(X) __islocked(X) 45 46 // RCU 47 rcu_read_lock() { __fence{rcu-lock}; } 48 rcu_read_unlock() { __fence{rcu-unlock}; } 49 synchronize_rcu() { __fence{sync-rcu}; } 50 synchronize_rcu_expedited() { __fence{sync-rcu}; } 51 52 // SRCU 53 srcu_read_lock(X) __load{srcu-lock}(*X) 54 srcu_read_unlock(X,Y) { __store{srcu-unlock}(*X,Y); } 55 srcu_down_read(X) __load{srcu-lock}(*X) 56 srcu_up_read(X,Y) { __store{srcu-unlock}(*X,Y); } 57 synchronize_srcu(X) { __srcu{sync-srcu}(X); } 58 synchronize_srcu_expedited(X) { __srcu{sync-srcu}(X); } 59 60 // Atomic 61 atomic_read(X) READ_ONCE(*X) 62 atomic_set(X,V) { WRITE_ONCE(*X,V); } 63 atomic_read_acquire(X) smp_load_acquire(X) 64 atomic_set_release(X,V) { smp_store_release(X,V); } 65 66 atomic_add(V,X) { __atomic_op(X,+,V); } 67 atomic_sub(V,X) { __atomic_op(X,-,V); } 68 atomic_inc(X) { __atomic_op(X,+,1); } 69 atomic_dec(X) { __atomic_op(X,-,1); } 70 71 atomic_add_return(V,X) __atomic_op_return{mb}(X,+,V) 72 atomic_add_return_relaxed(V,X) __atomic_op_return{once}(X,+,V) 73 atomic_add_return_acquire(V,X) __atomic_op_return{acquire}(X,+,V) 74 atomic_add_return_release(V,X) __atomic_op_return{release}(X,+,V) 75 atomic_fetch_add(V,X) __atomic_fetch_op{mb}(X,+,V) 76 atomic_fetch_add_relaxed(V,X) __atomic_fetch_op{once}(X,+,V) 77 atomic_fetch_add_acquire(V,X) __atomic_fetch_op{acquire}(X,+,V) 78 atomic_fetch_add_release(V,X) __atomic_fetch_op{release}(X,+,V) 79 80 atomic_inc_return(X) __atomic_op_return{mb}(X,+,1) 81 atomic_inc_return_relaxed(X) __atomic_op_return{once}(X,+,1) 82 atomic_inc_return_acquire(X) __atomic_op_return{acquire}(X,+,1) 83 atomic_inc_return_release(X) __atomic_op_return{release}(X,+,1) 84 atomic_fetch_inc(X) __atomic_fetch_op{mb}(X,+,1) 85 atomic_fetch_inc_relaxed(X) __atomic_fetch_op{once}(X,+,1) 86 atomic_fetch_inc_acquire(X) __atomic_fetch_op{acquire}(X,+,1) 87 atomic_fetch_inc_release(X) __atomic_fetch_op{release}(X,+,1) 88 89 atomic_sub_return(V,X) __atomic_op_return{mb}(X,-,V) 90 atomic_sub_return_relaxed(V,X) __atomic_op_return{once}(X,-,V) 91 atomic_sub_return_acquire(V,X) __atomic_op_return{acquire}(X,-,V) 92 atomic_sub_return_release(V,X) __atomic_op_return{release}(X,-,V) 93 atomic_fetch_sub(V,X) __atomic_fetch_op{mb}(X,-,V) 94 atomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{once}(X,-,V) 95 atomic_fetch_sub_acquire(V,X) __atomic_fetch_op{acquire}(X,-,V) 96 atomic_fetch_sub_release(V,X) __atomic_fetch_op{release}(X,-,V) 97 98 atomic_dec_return(X) __atomic_op_return{mb}(X,-,1) 99 atomic_dec_return_relaxed(X) __atomic_op_return{once}(X,-,1) 100 atomic_dec_return_acquire(X) __atomic_op_return{acquire}(X,-,1) 101 atomic_dec_return_release(X) __atomic_op_return{release}(X,-,1) 102 atomic_fetch_dec(X) __atomic_fetch_op{mb}(X,-,1) 103 atomic_fetch_dec_relaxed(X) __atomic_fetch_op{once}(X,-,1) 104 atomic_fetch_dec_acquire(X) __atomic_fetch_op{acquire}(X,-,1) 105 atomic_fetch_dec_release(X) __atomic_fetch_op{release}(X,-,1) 106 107 atomic_xchg(X,V) __xchg{mb}(X,V) 108 atomic_xchg_relaxed(X,V) __xchg{once}(X,V) 109 atomic_xchg_release(X,V) __xchg{release}(X,V) 110 atomic_xchg_acquire(X,V) __xchg{acquire}(X,V) 111 atomic_cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W) 112 atomic_cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W) 113 atomic_cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W) 114 atomic_cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W) 115 116 atomic_sub_and_test(V,X) __atomic_op_return{mb}(X,-,V) == 0 117 atomic_dec_and_test(X) __atomic_op_return{mb}(X,-,1) == 0 118 atomic_inc_and_test(X) __atomic_op_return{mb}(X,+,1) == 0 119 atomic_add_negative(V,X) __atomic_op_return{mb}(X,+,V) < 0
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.