~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/include/asm/atomic_lse.h

Version: ~ [ linux-6.11.5 ] ~ [ linux-6.10.14 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.58 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.114 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.169 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.228 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.284 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.322 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0-only */
  2 /*
  3  * Based on arch/arm/include/asm/atomic.h
  4  *
  5  * Copyright (C) 1996 Russell King.
  6  * Copyright (C) 2002 Deep Blue Solutions Ltd.
  7  * Copyright (C) 2012 ARM Ltd.
  8  */
  9 
 10 #ifndef __ASM_ATOMIC_LSE_H
 11 #define __ASM_ATOMIC_LSE_H
 12 
 13 #define ATOMIC_OP(op, asm_op)                                           \
 14 static __always_inline void                                             \
 15 __lse_atomic_##op(int i, atomic_t *v)                                   \
 16 {                                                                       \
 17         asm volatile(                                                   \
 18         __LSE_PREAMBLE                                                  \
 19         "       " #asm_op "     %w[i], %[v]\n"                          \
 20         : [v] "+Q" (v->counter)                                         \
 21         : [i] "r" (i));                                                 \
 22 }
 23 
 24 ATOMIC_OP(andnot, stclr)
 25 ATOMIC_OP(or, stset)
 26 ATOMIC_OP(xor, steor)
 27 ATOMIC_OP(add, stadd)
 28 
 29 static __always_inline void __lse_atomic_sub(int i, atomic_t *v)
 30 {
 31         __lse_atomic_add(-i, v);
 32 }
 33 
 34 #undef ATOMIC_OP
 35 
 36 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)                    \
 37 static __always_inline int                                              \
 38 __lse_atomic_fetch_##op##name(int i, atomic_t *v)                       \
 39 {                                                                       \
 40         int old;                                                        \
 41                                                                         \
 42         asm volatile(                                                   \
 43         __LSE_PREAMBLE                                                  \
 44         "       " #asm_op #mb " %w[i], %w[old], %[v]"                   \
 45         : [v] "+Q" (v->counter),                                        \
 46           [old] "=r" (old)                                              \
 47         : [i] "r" (i)                                                   \
 48         : cl);                                                          \
 49                                                                         \
 50         return old;                                                     \
 51 }
 52 
 53 #define ATOMIC_FETCH_OPS(op, asm_op)                                    \
 54         ATOMIC_FETCH_OP(_relaxed,   , op, asm_op)                       \
 55         ATOMIC_FETCH_OP(_acquire,  a, op, asm_op, "memory")             \
 56         ATOMIC_FETCH_OP(_release,  l, op, asm_op, "memory")             \
 57         ATOMIC_FETCH_OP(        , al, op, asm_op, "memory")
 58 
 59 ATOMIC_FETCH_OPS(andnot, ldclr)
 60 ATOMIC_FETCH_OPS(or, ldset)
 61 ATOMIC_FETCH_OPS(xor, ldeor)
 62 ATOMIC_FETCH_OPS(add, ldadd)
 63 
 64 #undef ATOMIC_FETCH_OP
 65 #undef ATOMIC_FETCH_OPS
 66 
 67 #define ATOMIC_FETCH_OP_SUB(name)                                       \
 68 static __always_inline int                                              \
 69 __lse_atomic_fetch_sub##name(int i, atomic_t *v)                        \
 70 {                                                                       \
 71         return __lse_atomic_fetch_add##name(-i, v);                     \
 72 }
 73 
 74 ATOMIC_FETCH_OP_SUB(_relaxed)
 75 ATOMIC_FETCH_OP_SUB(_acquire)
 76 ATOMIC_FETCH_OP_SUB(_release)
 77 ATOMIC_FETCH_OP_SUB(        )
 78 
 79 #undef ATOMIC_FETCH_OP_SUB
 80 
 81 #define ATOMIC_OP_ADD_SUB_RETURN(name)                                  \
 82 static __always_inline int                                              \
 83 __lse_atomic_add_return##name(int i, atomic_t *v)                       \
 84 {                                                                       \
 85         return __lse_atomic_fetch_add##name(i, v) + i;                  \
 86 }                                                                       \
 87                                                                         \
 88 static __always_inline int                                              \
 89 __lse_atomic_sub_return##name(int i, atomic_t *v)                       \
 90 {                                                                       \
 91         return __lse_atomic_fetch_sub(i, v) - i;                        \
 92 }
 93 
 94 ATOMIC_OP_ADD_SUB_RETURN(_relaxed)
 95 ATOMIC_OP_ADD_SUB_RETURN(_acquire)
 96 ATOMIC_OP_ADD_SUB_RETURN(_release)
 97 ATOMIC_OP_ADD_SUB_RETURN(        )
 98 
 99 #undef ATOMIC_OP_ADD_SUB_RETURN
100 
101 static __always_inline void __lse_atomic_and(int i, atomic_t *v)
102 {
103         return __lse_atomic_andnot(~i, v);
104 }
105 
106 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)                            \
107 static __always_inline int                                              \
108 __lse_atomic_fetch_and##name(int i, atomic_t *v)                        \
109 {                                                                       \
110         return __lse_atomic_fetch_andnot##name(~i, v);                  \
111 }
112 
113 ATOMIC_FETCH_OP_AND(_relaxed,   )
114 ATOMIC_FETCH_OP_AND(_acquire,  a, "memory")
115 ATOMIC_FETCH_OP_AND(_release,  l, "memory")
116 ATOMIC_FETCH_OP_AND(        , al, "memory")
117 
118 #undef ATOMIC_FETCH_OP_AND
119 
120 #define ATOMIC64_OP(op, asm_op)                                         \
121 static __always_inline void                                             \
122 __lse_atomic64_##op(s64 i, atomic64_t *v)                               \
123 {                                                                       \
124         asm volatile(                                                   \
125         __LSE_PREAMBLE                                                  \
126         "       " #asm_op "     %[i], %[v]\n"                           \
127         : [v] "+Q" (v->counter)                                         \
128         : [i] "r" (i));                                                 \
129 }
130 
131 ATOMIC64_OP(andnot, stclr)
132 ATOMIC64_OP(or, stset)
133 ATOMIC64_OP(xor, steor)
134 ATOMIC64_OP(add, stadd)
135 
136 static __always_inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
137 {
138         __lse_atomic64_add(-i, v);
139 }
140 
141 #undef ATOMIC64_OP
142 
143 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)                  \
144 static __always_inline long                                             \
145 __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)                   \
146 {                                                                       \
147         s64 old;                                                        \
148                                                                         \
149         asm volatile(                                                   \
150         __LSE_PREAMBLE                                                  \
151         "       " #asm_op #mb " %[i], %[old], %[v]"                     \
152         : [v] "+Q" (v->counter),                                        \
153           [old] "=r" (old)                                              \
154         : [i] "r" (i)                                                   \
155         : cl);                                                          \
156                                                                         \
157         return old;                                                     \
158 }
159 
160 #define ATOMIC64_FETCH_OPS(op, asm_op)                                  \
161         ATOMIC64_FETCH_OP(_relaxed,   , op, asm_op)                     \
162         ATOMIC64_FETCH_OP(_acquire,  a, op, asm_op, "memory")           \
163         ATOMIC64_FETCH_OP(_release,  l, op, asm_op, "memory")           \
164         ATOMIC64_FETCH_OP(        , al, op, asm_op, "memory")
165 
166 ATOMIC64_FETCH_OPS(andnot, ldclr)
167 ATOMIC64_FETCH_OPS(or, ldset)
168 ATOMIC64_FETCH_OPS(xor, ldeor)
169 ATOMIC64_FETCH_OPS(add, ldadd)
170 
171 #undef ATOMIC64_FETCH_OP
172 #undef ATOMIC64_FETCH_OPS
173 
174 #define ATOMIC64_FETCH_OP_SUB(name)                                     \
175 static __always_inline long                                             \
176 __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v)                    \
177 {                                                                       \
178         return __lse_atomic64_fetch_add##name(-i, v);                   \
179 }
180 
181 ATOMIC64_FETCH_OP_SUB(_relaxed)
182 ATOMIC64_FETCH_OP_SUB(_acquire)
183 ATOMIC64_FETCH_OP_SUB(_release)
184 ATOMIC64_FETCH_OP_SUB(        )
185 
186 #undef ATOMIC64_FETCH_OP_SUB
187 
188 #define ATOMIC64_OP_ADD_SUB_RETURN(name)                                \
189 static __always_inline long                                             \
190 __lse_atomic64_add_return##name(s64 i, atomic64_t *v)                   \
191 {                                                                       \
192         return __lse_atomic64_fetch_add##name(i, v) + i;                \
193 }                                                                       \
194                                                                         \
195 static __always_inline long                                             \
196 __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)                   \
197 {                                                                       \
198         return __lse_atomic64_fetch_sub##name(i, v) - i;                \
199 }
200 
201 ATOMIC64_OP_ADD_SUB_RETURN(_relaxed)
202 ATOMIC64_OP_ADD_SUB_RETURN(_acquire)
203 ATOMIC64_OP_ADD_SUB_RETURN(_release)
204 ATOMIC64_OP_ADD_SUB_RETURN(        )
205 
206 #undef ATOMIC64_OP_ADD_SUB_RETURN
207 
208 static __always_inline void __lse_atomic64_and(s64 i, atomic64_t *v)
209 {
210         return __lse_atomic64_andnot(~i, v);
211 }
212 
213 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)                          \
214 static __always_inline long                                             \
215 __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v)                    \
216 {                                                                       \
217         return __lse_atomic64_fetch_andnot##name(~i, v);                \
218 }
219 
220 ATOMIC64_FETCH_OP_AND(_relaxed,   )
221 ATOMIC64_FETCH_OP_AND(_acquire,  a, "memory")
222 ATOMIC64_FETCH_OP_AND(_release,  l, "memory")
223 ATOMIC64_FETCH_OP_AND(        , al, "memory")
224 
225 #undef ATOMIC64_FETCH_OP_AND
226 
227 static __always_inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
228 {
229         unsigned long tmp;
230 
231         asm volatile(
232         __LSE_PREAMBLE
233         "1:     ldr     %x[tmp], %[v]\n"
234         "       subs    %[ret], %x[tmp], #1\n"
235         "       b.lt    2f\n"
236         "       casal   %x[tmp], %[ret], %[v]\n"
237         "       sub     %x[tmp], %x[tmp], #1\n"
238         "       sub     %x[tmp], %x[tmp], %[ret]\n"
239         "       cbnz    %x[tmp], 1b\n"
240         "2:"
241         : [ret] "+&r" (v), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)
242         :
243         : "cc", "memory");
244 
245         return (long)v;
246 }
247 
248 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...)                     \
249 static __always_inline u##sz                                            \
250 __lse__cmpxchg_case_##name##sz(volatile void *ptr,                      \
251                                               u##sz old,                \
252                                               u##sz new)                \
253 {                                                                       \
254         asm volatile(                                                   \
255         __LSE_PREAMBLE                                                  \
256         "       cas" #mb #sfx " %" #w "[old], %" #w "[new], %[v]\n"     \
257         : [v] "+Q" (*(u##sz *)ptr),                                     \
258           [old] "+r" (old)                                              \
259         : [new] "rZ" (new)                                              \
260         : cl);                                                          \
261                                                                         \
262         return old;                                                     \
263 }
264 
265 __CMPXCHG_CASE(w, b,     ,  8,   )
266 __CMPXCHG_CASE(w, h,     , 16,   )
267 __CMPXCHG_CASE(w,  ,     , 32,   )
268 __CMPXCHG_CASE(x,  ,     , 64,   )
269 __CMPXCHG_CASE(w, b, acq_,  8,  a, "memory")
270 __CMPXCHG_CASE(w, h, acq_, 16,  a, "memory")
271 __CMPXCHG_CASE(w,  , acq_, 32,  a, "memory")
272 __CMPXCHG_CASE(x,  , acq_, 64,  a, "memory")
273 __CMPXCHG_CASE(w, b, rel_,  8,  l, "memory")
274 __CMPXCHG_CASE(w, h, rel_, 16,  l, "memory")
275 __CMPXCHG_CASE(w,  , rel_, 32,  l, "memory")
276 __CMPXCHG_CASE(x,  , rel_, 64,  l, "memory")
277 __CMPXCHG_CASE(w, b,  mb_,  8, al, "memory")
278 __CMPXCHG_CASE(w, h,  mb_, 16, al, "memory")
279 __CMPXCHG_CASE(w,  ,  mb_, 32, al, "memory")
280 __CMPXCHG_CASE(x,  ,  mb_, 64, al, "memory")
281 
282 #undef __CMPXCHG_CASE
283 
284 #define __CMPXCHG128(name, mb, cl...)                                   \
285 static __always_inline u128                                             \
286 __lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new)         \
287 {                                                                       \
288         union __u128_halves r, o = { .full = (old) },                   \
289                                n = { .full = (new) };                   \
290         register unsigned long x0 asm ("x0") = o.low;                   \
291         register unsigned long x1 asm ("x1") = o.high;                  \
292         register unsigned long x2 asm ("x2") = n.low;                   \
293         register unsigned long x3 asm ("x3") = n.high;                  \
294         register unsigned long x4 asm ("x4") = (unsigned long)ptr;      \
295                                                                         \
296         asm volatile(                                                   \
297         __LSE_PREAMBLE                                                  \
298         "       casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
299         : [old1] "+&r" (x0), [old2] "+&r" (x1),                         \
300           [v] "+Q" (*(u128 *)ptr)                                       \
301         : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),             \
302           [oldval1] "r" (o.low), [oldval2] "r" (o.high)                 \
303         : cl);                                                          \
304                                                                         \
305         r.low = x0; r.high = x1;                                        \
306                                                                         \
307         return r.full;                                                  \
308 }
309 
310 __CMPXCHG128(   ,   )
311 __CMPXCHG128(_mb, al, "memory")
312 
313 #undef __CMPXCHG128
314 
315 #endif  /* __ASM_ATOMIC_LSE_H */
316 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

sflogo.php