1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * Generic barrier definitions.
4 *
5 * It should be possible to use these on really simple architectures,
6 * but it serves more as a starting point for new ports.
7 *
8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
9 * Written by David Howells (dhowells@redhat.com)
10 */
11 #ifndef __ASM_GENERIC_BARRIER_H
12 #define __ASM_GENERIC_BARRIER_H
13
14 #ifndef __ASSEMBLY__
15
16 #include <linux/compiler.h>
17 #include <linux/kcsan-checks.h>
18 #include <asm/rwonce.h>
19
20 #ifndef nop
21 #define nop() asm volatile ("nop")
22 #endif
23
24 /*
25 * Architectures that want generic instrumentation can define __ prefixed
26 * variants of all barriers.
27 */
28
29 #ifdef __mb
30 #define mb() do { kcsan_mb(); __mb(); } while (0)
31 #endif
32
33 #ifdef __rmb
34 #define rmb() do { kcsan_rmb(); __rmb(); } while (0)
35 #endif
36
37 #ifdef __wmb
38 #define wmb() do { kcsan_wmb(); __wmb(); } while (0)
39 #endif
40
41 #ifdef __dma_mb
42 #define dma_mb() do { kcsan_mb(); __dma_mb(); } while (0)
43 #endif
44
45 #ifdef __dma_rmb
46 #define dma_rmb() do { kcsan_rmb(); __dma_rmb(); } while (0)
47 #endif
48
49 #ifdef __dma_wmb
50 #define dma_wmb() do { kcsan_wmb(); __dma_wmb(); } while (0)
51 #endif
52
53 /*
54 * Force strict CPU ordering. And yes, this is required on UP too when we're
55 * talking to devices.
56 *
57 * Fall back to compiler barriers if nothing better is provided.
58 */
59
60 #ifndef mb
61 #define mb() barrier()
62 #endif
63
64 #ifndef rmb
65 #define rmb() mb()
66 #endif
67
68 #ifndef wmb
69 #define wmb() mb()
70 #endif
71
72 #ifndef dma_mb
73 #define dma_mb() mb()
74 #endif
75
76 #ifndef dma_rmb
77 #define dma_rmb() rmb()
78 #endif
79
80 #ifndef dma_wmb
81 #define dma_wmb() wmb()
82 #endif
83
84 #ifndef __smp_mb
85 #define __smp_mb() mb()
86 #endif
87
88 #ifndef __smp_rmb
89 #define __smp_rmb() rmb()
90 #endif
91
92 #ifndef __smp_wmb
93 #define __smp_wmb() wmb()
94 #endif
95
96 #ifdef CONFIG_SMP
97
98 #ifndef smp_mb
99 #define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0)
100 #endif
101
102 #ifndef smp_rmb
103 #define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)
104 #endif
105
106 #ifndef smp_wmb
107 #define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)
108 #endif
109
110 #else /* !CONFIG_SMP */
111
112 #ifndef smp_mb
113 #define smp_mb() barrier()
114 #endif
115
116 #ifndef smp_rmb
117 #define smp_rmb() barrier()
118 #endif
119
120 #ifndef smp_wmb
121 #define smp_wmb() barrier()
122 #endif
123
124 #endif /* CONFIG_SMP */
125
126 #ifndef __smp_store_mb
127 #define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
128 #endif
129
130 #ifndef __smp_mb__before_atomic
131 #define __smp_mb__before_atomic() __smp_mb()
132 #endif
133
134 #ifndef __smp_mb__after_atomic
135 #define __smp_mb__after_atomic() __smp_mb()
136 #endif
137
138 #ifndef __smp_store_release
139 #define __smp_store_release(p, v) \
140 do { \
141 compiletime_assert_atomic_type(*p); \
142 __smp_mb(); \
143 WRITE_ONCE(*p, v); \
144 } while (0)
145 #endif
146
147 #ifndef __smp_load_acquire
148 #define __smp_load_acquire(p) \
149 ({ \
150 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
151 compiletime_assert_atomic_type(*p); \
152 __smp_mb(); \
153 (typeof(*p))___p1; \
154 })
155 #endif
156
157 #ifdef CONFIG_SMP
158
159 #ifndef smp_store_mb
160 #define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)
161 #endif
162
163 #ifndef smp_mb__before_atomic
164 #define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)
165 #endif
166
167 #ifndef smp_mb__after_atomic
168 #define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)
169 #endif
170
171 #ifndef smp_store_release
172 #define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)
173 #endif
174
175 #ifndef smp_load_acquire
176 #define smp_load_acquire(p) __smp_load_acquire(p)
177 #endif
178
179 #else /* !CONFIG_SMP */
180
181 #ifndef smp_store_mb
182 #define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
183 #endif
184
185 #ifndef smp_mb__before_atomic
186 #define smp_mb__before_atomic() barrier()
187 #endif
188
189 #ifndef smp_mb__after_atomic
190 #define smp_mb__after_atomic() barrier()
191 #endif
192
193 #ifndef smp_store_release
194 #define smp_store_release(p, v) \
195 do { \
196 barrier(); \
197 WRITE_ONCE(*p, v); \
198 } while (0)
199 #endif
200
201 #ifndef smp_load_acquire
202 #define smp_load_acquire(p) \
203 ({ \
204 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
205 barrier(); \
206 (typeof(*p))___p1; \
207 })
208 #endif
209
210 #endif /* CONFIG_SMP */
211
212 /* Barriers for virtual machine guests when talking to an SMP host */
213 #define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0)
214 #define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)
215 #define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)
216 #define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)
217 #define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)
218 #define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)
219 #define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)
220 #define virt_load_acquire(p) __smp_load_acquire(p)
221
222 /**
223 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
224 *
225 * A control dependency provides a LOAD->STORE order, the additional RMB
226 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
227 * aka. (load)-ACQUIRE.
228 *
229 * Architectures that do not do load speculation can have this be barrier().
230 */
231 #ifndef smp_acquire__after_ctrl_dep
232 #define smp_acquire__after_ctrl_dep() smp_rmb()
233 #endif
234
235 /**
236 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
237 * @ptr: pointer to the variable to wait on
238 * @cond: boolean expression to wait for
239 *
240 * Equivalent to using READ_ONCE() on the condition variable.
241 *
242 * Due to C lacking lambda expressions we load the value of *ptr into a
243 * pre-named variable @VAL to be used in @cond.
244 */
245 #ifndef smp_cond_load_relaxed
246 #define smp_cond_load_relaxed(ptr, cond_expr) ({ \
247 typeof(ptr) __PTR = (ptr); \
248 __unqual_scalar_typeof(*ptr) VAL; \
249 for (;;) { \
250 VAL = READ_ONCE(*__PTR); \
251 if (cond_expr) \
252 break; \
253 cpu_relax(); \
254 } \
255 (typeof(*ptr))VAL; \
256 })
257 #endif
258
259 /**
260 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
261 * @ptr: pointer to the variable to wait on
262 * @cond: boolean expression to wait for
263 *
264 * Equivalent to using smp_load_acquire() on the condition variable but employs
265 * the control dependency of the wait to reduce the barrier on many platforms.
266 */
267 #ifndef smp_cond_load_acquire
268 #define smp_cond_load_acquire(ptr, cond_expr) ({ \
269 __unqual_scalar_typeof(*ptr) _val; \
270 _val = smp_cond_load_relaxed(ptr, cond_expr); \
271 smp_acquire__after_ctrl_dep(); \
272 (typeof(*ptr))_val; \
273 })
274 #endif
275
276 /*
277 * pmem_wmb() ensures that all stores for which the modification
278 * are written to persistent storage by preceding instructions have
279 * updated persistent storage before any data access or data transfer
280 * caused by subsequent instructions is initiated.
281 */
282 #ifndef pmem_wmb
283 #define pmem_wmb() wmb()
284 #endif
285
286 /*
287 * ioremap_wc() maps I/O memory as memory with write-combining attributes. For
288 * this kind of memory accesses, the CPU may wait for prior accesses to be
289 * merged with subsequent ones. In some situation, such wait is bad for the
290 * performance. io_stop_wc() can be used to prevent the merging of
291 * write-combining memory accesses before this macro with those after it.
292 */
293 #ifndef io_stop_wc
294 #define io_stop_wc() do { } while (0)
295 #endif
296
297 /*
298 * Architectures that guarantee an implicit smp_mb() in switch_mm()
299 * can override smp_mb__after_switch_mm.
300 */
301 #ifndef smp_mb__after_switch_mm
302 # define smp_mb__after_switch_mm() smp_mb()
303 #endif
304
305 #endif /* !__ASSEMBLY__ */
306 #endif /* __ASM_GENERIC_BARRIER_H */
307
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.