1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2014 Regents of the University of California
4 */
5
6 #ifndef _ASM_RISCV_CMPXCHG_H
7 #define _ASM_RISCV_CMPXCHG_H
8
9 #include <linux/bug.h>
10
11 #include <asm/alternative-macros.h>
12 #include <asm/fence.h>
13 #include <asm/hwcap.h>
14 #include <asm/insn-def.h>
15
16 #define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n) \
17 ({ \
18 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
19 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
20 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
21 << __s; \
22 ulong __newx = (ulong)(n) << __s; \
23 ulong __retx; \
24 ulong __rc; \
25 \
26 __asm__ __volatile__ ( \
27 prepend \
28 "0: lr.w %0, %2\n" \
29 " and %1, %0, %z4\n" \
30 " or %1, %1, %z3\n" \
31 " sc.w" sc_sfx " %1, %1, %2\n" \
32 " bnez %1, 0b\n" \
33 append \
34 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
35 : "rJ" (__newx), "rJ" (~__mask) \
36 : "memory"); \
37 \
38 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
39 })
40
41 #define __arch_xchg(sfx, prepend, append, r, p, n) \
42 ({ \
43 __asm__ __volatile__ ( \
44 prepend \
45 " amoswap" sfx " %0, %2, %1\n" \
46 append \
47 : "=r" (r), "+A" (*(p)) \
48 : "r" (n) \
49 : "memory"); \
50 })
51
52 #define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend, \
53 sc_append, swap_append) \
54 ({ \
55 __typeof__(ptr) __ptr = (ptr); \
56 __typeof__(*(__ptr)) __new = (new); \
57 __typeof__(*(__ptr)) __ret; \
58 \
59 switch (sizeof(*__ptr)) { \
60 case 1: \
61 case 2: \
62 __arch_xchg_masked(sc_sfx, prepend, sc_append, \
63 __ret, __ptr, __new); \
64 break; \
65 case 4: \
66 __arch_xchg(".w" swap_sfx, prepend, swap_append, \
67 __ret, __ptr, __new); \
68 break; \
69 case 8: \
70 __arch_xchg(".d" swap_sfx, prepend, swap_append, \
71 __ret, __ptr, __new); \
72 break; \
73 default: \
74 BUILD_BUG(); \
75 } \
76 (__typeof__(*(__ptr)))__ret; \
77 })
78
79 #define arch_xchg_relaxed(ptr, x) \
80 _arch_xchg(ptr, x, "", "", "", "", "")
81
82 #define arch_xchg_acquire(ptr, x) \
83 _arch_xchg(ptr, x, "", "", "", \
84 RISCV_ACQUIRE_BARRIER, RISCV_ACQUIRE_BARRIER)
85
86 #define arch_xchg_release(ptr, x) \
87 _arch_xchg(ptr, x, "", "", RISCV_RELEASE_BARRIER, "", "")
88
89 #define arch_xchg(ptr, x) \
90 _arch_xchg(ptr, x, ".rl", ".aqrl", "", RISCV_FULL_BARRIER, "")
91
92 #define xchg32(ptr, x) \
93 ({ \
94 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
95 arch_xchg((ptr), (x)); \
96 })
97
98 #define xchg64(ptr, x) \
99 ({ \
100 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
101 arch_xchg((ptr), (x)); \
102 })
103
104 /*
105 * Atomic compare and exchange. Compare OLD with MEM, if identical,
106 * store NEW in MEM. Return the initial value in MEM. Success is
107 * indicated by comparing RETURN with OLD.
108 */
109
110 #define __arch_cmpxchg_masked(sc_sfx, prepend, append, r, p, o, n) \
111 ({ \
112 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
113 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
114 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
115 << __s; \
116 ulong __newx = (ulong)(n) << __s; \
117 ulong __oldx = (ulong)(o) << __s; \
118 ulong __retx; \
119 ulong __rc; \
120 \
121 __asm__ __volatile__ ( \
122 prepend \
123 "0: lr.w %0, %2\n" \
124 " and %1, %0, %z5\n" \
125 " bne %1, %z3, 1f\n" \
126 " and %1, %0, %z6\n" \
127 " or %1, %1, %z4\n" \
128 " sc.w" sc_sfx " %1, %1, %2\n" \
129 " bnez %1, 0b\n" \
130 append \
131 "1:\n" \
132 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
133 : "rJ" ((long)__oldx), "rJ" (__newx), \
134 "rJ" (__mask), "rJ" (~__mask) \
135 : "memory"); \
136 \
137 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
138 })
139
140 #define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n) \
141 ({ \
142 register unsigned int __rc; \
143 \
144 __asm__ __volatile__ ( \
145 prepend \
146 "0: lr" lr_sfx " %0, %2\n" \
147 " bne %0, %z3, 1f\n" \
148 " sc" sc_sfx " %1, %z4, %2\n" \
149 " bnez %1, 0b\n" \
150 append \
151 "1:\n" \
152 : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
153 : "rJ" (co o), "rJ" (n) \
154 : "memory"); \
155 })
156
157 #define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append) \
158 ({ \
159 __typeof__(ptr) __ptr = (ptr); \
160 __typeof__(*(__ptr)) __old = (old); \
161 __typeof__(*(__ptr)) __new = (new); \
162 __typeof__(*(__ptr)) __ret; \
163 \
164 switch (sizeof(*__ptr)) { \
165 case 1: \
166 case 2: \
167 __arch_cmpxchg_masked(sc_sfx, prepend, append, \
168 __ret, __ptr, __old, __new); \
169 break; \
170 case 4: \
171 __arch_cmpxchg(".w", ".w" sc_sfx, prepend, append, \
172 __ret, __ptr, (long), __old, __new); \
173 break; \
174 case 8: \
175 __arch_cmpxchg(".d", ".d" sc_sfx, prepend, append, \
176 __ret, __ptr, /**/, __old, __new); \
177 break; \
178 default: \
179 BUILD_BUG(); \
180 } \
181 (__typeof__(*(__ptr)))__ret; \
182 })
183
184 #define arch_cmpxchg_relaxed(ptr, o, n) \
185 _arch_cmpxchg((ptr), (o), (n), "", "", "")
186
187 #define arch_cmpxchg_acquire(ptr, o, n) \
188 _arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)
189
190 #define arch_cmpxchg_release(ptr, o, n) \
191 _arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")
192
193 #define arch_cmpxchg(ptr, o, n) \
194 _arch_cmpxchg((ptr), (o), (n), ".rl", "", " fence rw, rw\n")
195
196 #define arch_cmpxchg_local(ptr, o, n) \
197 arch_cmpxchg_relaxed((ptr), (o), (n))
198
199 #define arch_cmpxchg64(ptr, o, n) \
200 ({ \
201 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
202 arch_cmpxchg((ptr), (o), (n)); \
203 })
204
205 #define arch_cmpxchg64_local(ptr, o, n) \
206 ({ \
207 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
208 arch_cmpxchg_relaxed((ptr), (o), (n)); \
209 })
210
211 #define arch_cmpxchg64_relaxed(ptr, o, n) \
212 ({ \
213 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
214 arch_cmpxchg_relaxed((ptr), (o), (n)); \
215 })
216
217 #define arch_cmpxchg64_acquire(ptr, o, n) \
218 ({ \
219 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
220 arch_cmpxchg_acquire((ptr), (o), (n)); \
221 })
222
223 #define arch_cmpxchg64_release(ptr, o, n) \
224 ({ \
225 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
226 arch_cmpxchg_release((ptr), (o), (n)); \
227 })
228
229 #ifdef CONFIG_RISCV_ISA_ZAWRS
230 /*
231 * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to
232 * @val we expect it to still terminate within a "reasonable" amount of time
233 * for an implementation-specific other reason, a pending, locally-enabled
234 * interrupt, or because it has been configured to raise an illegal
235 * instruction exception.
236 */
237 static __always_inline void __cmpwait(volatile void *ptr,
238 unsigned long val,
239 int size)
240 {
241 unsigned long tmp;
242
243 asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop",
244 0, RISCV_ISA_EXT_ZAWRS, 1)
245 : : : : no_zawrs);
246
247 switch (size) {
248 case 4:
249 asm volatile(
250 " lr.w %0, %1\n"
251 " xor %0, %0, %2\n"
252 " bnez %0, 1f\n"
253 ZAWRS_WRS_NTO "\n"
254 "1:"
255 : "=&r" (tmp), "+A" (*(u32 *)ptr)
256 : "r" (val));
257 break;
258 #if __riscv_xlen == 64
259 case 8:
260 asm volatile(
261 " lr.d %0, %1\n"
262 " xor %0, %0, %2\n"
263 " bnez %0, 1f\n"
264 ZAWRS_WRS_NTO "\n"
265 "1:"
266 : "=&r" (tmp), "+A" (*(u64 *)ptr)
267 : "r" (val));
268 break;
269 #endif
270 default:
271 BUILD_BUG();
272 }
273
274 return;
275
276 no_zawrs:
277 asm volatile(RISCV_PAUSE : : : "memory");
278 }
279
280 #define __cmpwait_relaxed(ptr, val) \
281 __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))
282 #endif
283
284 #endif /* _ASM_RISCV_CMPXCHG_H */
285
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.