1 #!/bin/sh 1 #!/bin/sh
2 # SPDX-License-Identifier: GPL-2.0 2 # SPDX-License-Identifier: GPL-2.0
3 3
4 ATOMICDIR=$(dirname $0) 4 ATOMICDIR=$(dirname $0)
5 5
6 . ${ATOMICDIR}/atomic-tbl.sh 6 . ${ATOMICDIR}/atomic-tbl.sh
7 7
8 #gen_template_fallback(template, meta, pfx, na 8 #gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...)
9 gen_template_fallback() 9 gen_template_fallback()
10 { 10 {
11 local template="$1"; shift 11 local template="$1"; shift
12 local meta="$1"; shift 12 local meta="$1"; shift
13 local pfx="$1"; shift 13 local pfx="$1"; shift
14 local name="$1"; shift 14 local name="$1"; shift
15 local sfx="$1"; shift 15 local sfx="$1"; shift
16 local order="$1"; shift 16 local order="$1"; shift
17 local atomic="$1"; shift 17 local atomic="$1"; shift
18 local int="$1"; shift 18 local int="$1"; shift
19 19
20 local ret="$(gen_ret_type "${meta}" "$ 20 local ret="$(gen_ret_type "${meta}" "${int}")"
21 local retstmt="$(gen_ret_stmt "${meta} 21 local retstmt="$(gen_ret_stmt "${meta}")"
22 local params="$(gen_params "${int}" "$ 22 local params="$(gen_params "${int}" "${atomic}" "$@")"
23 local args="$(gen_args "$@")" 23 local args="$(gen_args "$@")"
24 24
25 . ${template} 25 . ${template}
26 } 26 }
27 27
28 #gen_order_fallback(meta, pfx, name, sfx, orde 28 #gen_order_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
29 gen_order_fallback() 29 gen_order_fallback()
30 { 30 {
31 local meta="$1"; shift 31 local meta="$1"; shift
32 local pfx="$1"; shift 32 local pfx="$1"; shift
33 local name="$1"; shift 33 local name="$1"; shift
34 local sfx="$1"; shift 34 local sfx="$1"; shift
35 local order="$1"; shift 35 local order="$1"; shift
36 36
37 local tmpl_order=${order#_} 37 local tmpl_order=${order#_}
38 local tmpl="${ATOMICDIR}/fallbacks/${t 38 local tmpl="${ATOMICDIR}/fallbacks/${tmpl_order:-fence}"
39 gen_template_fallback "${tmpl}" "${met 39 gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
40 } 40 }
41 41
42 #gen_proto_fallback(meta, pfx, name, sfx, orde 42 #gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
43 gen_proto_fallback() 43 gen_proto_fallback()
44 { 44 {
45 local meta="$1"; shift 45 local meta="$1"; shift
46 local pfx="$1"; shift 46 local pfx="$1"; shift
47 local name="$1"; shift 47 local name="$1"; shift
48 local sfx="$1"; shift 48 local sfx="$1"; shift
49 local order="$1"; shift 49 local order="$1"; shift
50 50
51 local tmpl="$(find_fallback_template " 51 local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
52 gen_template_fallback "${tmpl}" "${met 52 gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
53 } 53 }
54 54
55 #gen_proto_order_variant(meta, pfx, name, sfx, 55 #gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, args...)
56 gen_proto_order_variant() 56 gen_proto_order_variant()
57 { 57 {
58 local meta="$1"; shift 58 local meta="$1"; shift
59 local pfx="$1"; shift 59 local pfx="$1"; shift
60 local name="$1"; shift 60 local name="$1"; shift
61 local sfx="$1"; shift 61 local sfx="$1"; shift
62 local order="$1"; shift 62 local order="$1"; shift
63 local atomic="$1"; shift 63 local atomic="$1"; shift
64 local int="$1"; shift 64 local int="$1"; shift
65 65
66 local atomicname="${atomic}_${pfx}${na 66 local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
67 local basename="${atomic}_${pfx}${name 67 local basename="${atomic}_${pfx}${name}${sfx}"
68 68
69 local template="$(find_fallback_templa 69 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
70 70
71 local ret="$(gen_ret_type "${meta}" "$ 71 local ret="$(gen_ret_type "${meta}" "${int}")"
72 local retstmt="$(gen_ret_stmt "${meta} 72 local retstmt="$(gen_ret_stmt "${meta}")"
73 local params="$(gen_params "${int}" "$ 73 local params="$(gen_params "${int}" "${atomic}" "$@")"
74 local args="$(gen_args "$@")" 74 local args="$(gen_args "$@")"
75 75
76 gen_kerneldoc "raw_" "${meta}" "${pfx} 76 gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
77 77
78 printf "static __always_inline ${ret}\ 78 printf "static __always_inline ${ret}\n"
79 printf "raw_${atomicname}(${params})\n 79 printf "raw_${atomicname}(${params})\n"
80 printf "{\n" 80 printf "{\n"
81 81
82 # Where there is no possible fallback, 82 # Where there is no possible fallback, this order variant is mandatory
83 # and must be provided by arch code. A 83 # and must be provided by arch code. Add a comment to the header to
84 # make this obvious. 84 # make this obvious.
85 # 85 #
86 # Ideally we'd error on a missing defi 86 # Ideally we'd error on a missing definition, but arch code might
87 # define this order variant as a C fun 87 # define this order variant as a C function without a preprocessor
88 # symbol. 88 # symbol.
89 if [ -z ${template} ] && [ -z "${order 89 if [ -z ${template} ] && [ -z "${order}" ] && ! meta_has_relaxed "${meta}"; then
90 printf "\t${retstmt}arch_${ato 90 printf "\t${retstmt}arch_${atomicname}(${args});\n"
91 printf "}\n\n" 91 printf "}\n\n"
92 return 92 return
93 fi 93 fi
94 94
95 printf "#if defined(arch_${atomicname} 95 printf "#if defined(arch_${atomicname})\n"
96 printf "\t${retstmt}arch_${atomicname} 96 printf "\t${retstmt}arch_${atomicname}(${args});\n"
97 97
98 # Allow FULL/ACQUIRE/RELEASE ops to be 98 # Allow FULL/ACQUIRE/RELEASE ops to be defined in terms of RELAXED ops
99 if [ "${order}" != "_relaxed" ] && met 99 if [ "${order}" != "_relaxed" ] && meta_has_relaxed "${meta}"; then
100 printf "#elif defined(arch_${b 100 printf "#elif defined(arch_${basename}_relaxed)\n"
101 gen_order_fallback "${meta}" " 101 gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
102 fi 102 fi
103 103
104 # Allow ACQUIRE/RELEASE/RELAXED ops to 104 # Allow ACQUIRE/RELEASE/RELAXED ops to be defined in terms of FULL ops
105 if [ ! -z "${order}" ] && ! meta_is_im 105 if [ ! -z "${order}" ] && ! meta_is_implicitly_relaxed "${meta}"; then
106 printf "#elif defined(arch_${b 106 printf "#elif defined(arch_${basename})\n"
107 printf "\t${retstmt}arch_${bas 107 printf "\t${retstmt}arch_${basename}(${args});\n"
108 fi 108 fi
109 109
110 printf "#else\n" 110 printf "#else\n"
111 if [ ! -z "${template}" ]; then 111 if [ ! -z "${template}" ]; then
112 gen_proto_fallback "${meta}" " 112 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
113 else 113 else
114 printf "#error \"Unable to def 114 printf "#error \"Unable to define raw_${atomicname}\"\n"
115 fi 115 fi
116 116
117 printf "#endif\n" 117 printf "#endif\n"
118 printf "}\n\n" 118 printf "}\n\n"
119 } 119 }
120 120
121 121
122 #gen_proto_order_variants(meta, pfx, name, sfx 122 #gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
123 gen_proto_order_variants() 123 gen_proto_order_variants()
124 { 124 {
125 local meta="$1"; shift 125 local meta="$1"; shift
126 local pfx="$1"; shift 126 local pfx="$1"; shift
127 local name="$1"; shift 127 local name="$1"; shift
128 local sfx="$1"; shift 128 local sfx="$1"; shift
129 local atomic="$1" 129 local atomic="$1"
130 130
131 gen_proto_order_variant "${meta}" "${p 131 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
132 132
133 if meta_has_acquire "${meta}"; then 133 if meta_has_acquire "${meta}"; then
134 gen_proto_order_variant "${met 134 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
135 fi 135 fi
136 136
137 if meta_has_release "${meta}"; then 137 if meta_has_release "${meta}"; then
138 gen_proto_order_variant "${met 138 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
139 fi 139 fi
140 140
141 if meta_has_relaxed "${meta}"; then 141 if meta_has_relaxed "${meta}"; then
142 gen_proto_order_variant "${met 142 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
143 fi 143 fi
144 } 144 }
145 145
146 #gen_basic_fallbacks(basename) 146 #gen_basic_fallbacks(basename)
147 gen_basic_fallbacks() 147 gen_basic_fallbacks()
148 { 148 {
149 local basename="$1"; shift 149 local basename="$1"; shift
150 cat << EOF 150 cat << EOF
151 #define raw_${basename}_acquire arch_${basenam 151 #define raw_${basename}_acquire arch_${basename}
152 #define raw_${basename}_release arch_${basenam 152 #define raw_${basename}_release arch_${basename}
153 #define raw_${basename}_relaxed arch_${basenam 153 #define raw_${basename}_relaxed arch_${basename}
154 EOF 154 EOF
155 } 155 }
156 156
157 gen_order_fallbacks() 157 gen_order_fallbacks()
158 { 158 {
159 local xchg="$1"; shift 159 local xchg="$1"; shift
160 160
161 cat <<EOF 161 cat <<EOF
162 162
163 #define raw_${xchg}_relaxed arch_${xchg}_relax 163 #define raw_${xchg}_relaxed arch_${xchg}_relaxed
164 164
165 #ifdef arch_${xchg}_acquire 165 #ifdef arch_${xchg}_acquire
166 #define raw_${xchg}_acquire arch_${xchg}_acqui 166 #define raw_${xchg}_acquire arch_${xchg}_acquire
167 #else 167 #else
168 #define raw_${xchg}_acquire(...) \\ 168 #define raw_${xchg}_acquire(...) \\
169 __atomic_op_acquire(arch_${xchg}, __VA 169 __atomic_op_acquire(arch_${xchg}, __VA_ARGS__)
170 #endif 170 #endif
171 171
172 #ifdef arch_${xchg}_release 172 #ifdef arch_${xchg}_release
173 #define raw_${xchg}_release arch_${xchg}_relea 173 #define raw_${xchg}_release arch_${xchg}_release
174 #else 174 #else
175 #define raw_${xchg}_release(...) \\ 175 #define raw_${xchg}_release(...) \\
176 __atomic_op_release(arch_${xchg}, __VA 176 __atomic_op_release(arch_${xchg}, __VA_ARGS__)
177 #endif 177 #endif
178 178
179 #ifdef arch_${xchg} 179 #ifdef arch_${xchg}
180 #define raw_${xchg} arch_${xchg} 180 #define raw_${xchg} arch_${xchg}
181 #else 181 #else
182 #define raw_${xchg}(...) \\ 182 #define raw_${xchg}(...) \\
183 __atomic_op_fence(arch_${xchg}, __VA_A 183 __atomic_op_fence(arch_${xchg}, __VA_ARGS__)
184 #endif 184 #endif
185 185
186 EOF 186 EOF
187 } 187 }
188 188
189 gen_xchg_order_fallback() 189 gen_xchg_order_fallback()
190 { 190 {
191 local xchg="$1"; shift 191 local xchg="$1"; shift
192 local order="$1"; shift 192 local order="$1"; shift
193 local forder="${order:-_fence}" 193 local forder="${order:-_fence}"
194 194
195 printf "#if defined(arch_${xchg}${orde 195 printf "#if defined(arch_${xchg}${order})\n"
196 printf "#define raw_${xchg}${order} ar 196 printf "#define raw_${xchg}${order} arch_${xchg}${order}\n"
197 197
198 if [ "${order}" != "_relaxed" ]; then 198 if [ "${order}" != "_relaxed" ]; then
199 printf "#elif defined(arch_${x 199 printf "#elif defined(arch_${xchg}_relaxed)\n"
200 printf "#define raw_${xchg}${o 200 printf "#define raw_${xchg}${order}(...) \\\\\n"
201 printf " __atomic_op${f 201 printf " __atomic_op${forder}(arch_${xchg}, __VA_ARGS__)\n"
202 fi 202 fi
203 203
204 if [ ! -z "${order}" ]; then 204 if [ ! -z "${order}" ]; then
205 printf "#elif defined(arch_${x 205 printf "#elif defined(arch_${xchg})\n"
206 printf "#define raw_${xchg}${o 206 printf "#define raw_${xchg}${order} arch_${xchg}\n"
207 fi 207 fi
208 208
209 printf "#else\n" 209 printf "#else\n"
210 printf "extern void raw_${xchg}${order 210 printf "extern void raw_${xchg}${order}_not_implemented(void);\n"
211 printf "#define raw_${xchg}${order}(.. 211 printf "#define raw_${xchg}${order}(...) raw_${xchg}${order}_not_implemented()\n"
212 printf "#endif\n\n" 212 printf "#endif\n\n"
213 } 213 }
214 214
215 gen_xchg_fallbacks() 215 gen_xchg_fallbacks()
216 { 216 {
217 local xchg="$1"; shift 217 local xchg="$1"; shift
218 218
219 for order in "" "_acquire" "_release" 219 for order in "" "_acquire" "_release" "_relaxed"; do
220 gen_xchg_order_fallback "${xch 220 gen_xchg_order_fallback "${xchg}" "${order}"
221 done 221 done
222 } 222 }
223 223
224 gen_try_cmpxchg_fallback() 224 gen_try_cmpxchg_fallback()
225 { 225 {
226 local prefix="$1"; shift <<
227 local cmpxchg="$1"; shift; 226 local cmpxchg="$1"; shift;
228 local suffix="$1"; shift; !! 227 local order="$1"; shift;
229 228
230 cat <<EOF 229 cat <<EOF
231 #define raw_${prefix}try_${cmpxchg}${suffix}(_ !! 230 #define raw_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
232 ({ \\ 231 ({ \\
233 typeof(*(_ptr)) *___op = (_oldp), ___o 232 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
234 ___r = raw_${prefix}${cmpxchg}${suffix !! 233 ___r = raw_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
235 if (unlikely(___r != ___o)) \\ 234 if (unlikely(___r != ___o)) \\
236 *___op = ___r; \\ 235 *___op = ___r; \\
237 likely(___r == ___o); \\ 236 likely(___r == ___o); \\
238 }) 237 })
239 EOF 238 EOF
240 } 239 }
241 240
242 gen_try_cmpxchg_order_fallback() 241 gen_try_cmpxchg_order_fallback()
243 { 242 {
244 local cmpxchg="$1"; shift 243 local cmpxchg="$1"; shift
245 local order="$1"; shift 244 local order="$1"; shift
246 local forder="${order:-_fence}" 245 local forder="${order:-_fence}"
247 246
248 printf "#if defined(arch_try_${cmpxchg 247 printf "#if defined(arch_try_${cmpxchg}${order})\n"
249 printf "#define raw_try_${cmpxchg}${or 248 printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}${order}\n"
250 249
251 if [ "${order}" != "_relaxed" ]; then 250 if [ "${order}" != "_relaxed" ]; then
252 printf "#elif defined(arch_try 251 printf "#elif defined(arch_try_${cmpxchg}_relaxed)\n"
253 printf "#define raw_try_${cmpx 252 printf "#define raw_try_${cmpxchg}${order}(...) \\\\\n"
254 printf " __atomic_op${f 253 printf " __atomic_op${forder}(arch_try_${cmpxchg}, __VA_ARGS__)\n"
255 fi 254 fi
256 255
257 if [ ! -z "${order}" ]; then 256 if [ ! -z "${order}" ]; then
258 printf "#elif defined(arch_try 257 printf "#elif defined(arch_try_${cmpxchg})\n"
259 printf "#define raw_try_${cmpx 258 printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}\n"
260 fi 259 fi
261 260
262 printf "#else\n" 261 printf "#else\n"
263 gen_try_cmpxchg_fallback "" "${cmpxchg !! 262 gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
264 printf "#endif\n\n" 263 printf "#endif\n\n"
265 } 264 }
266 265
267 gen_try_cmpxchg_order_fallbacks() !! 266 gen_try_cmpxchg_fallbacks()
268 { 267 {
269 local cmpxchg="$1"; shift; 268 local cmpxchg="$1"; shift;
270 269
271 for order in "" "_acquire" "_release" 270 for order in "" "_acquire" "_release" "_relaxed"; do
272 gen_try_cmpxchg_order_fallback 271 gen_try_cmpxchg_order_fallback "${cmpxchg}" "${order}"
273 done 272 done
274 } 273 }
275 274
276 gen_def_and_try_cmpxchg_fallback() !! 275 gen_cmpxchg_local_fallbacks()
277 { 276 {
278 local prefix="$1"; shift <<
279 local cmpxchg="$1"; shift 277 local cmpxchg="$1"; shift
280 local suffix="$1"; shift <<
281 278
282 printf "#define raw_${prefix}${cmpxchg !! 279 printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
283 printf "#ifdef arch_${prefix}try_${cmp !! 280 printf "#ifdef arch_try_${cmpxchg}\n"
284 printf "#define raw_${prefix}try_${cmp !! 281 printf "#define raw_try_${cmpxchg} arch_try_${cmpxchg}\n"
285 printf "#else\n" 282 printf "#else\n"
286 gen_try_cmpxchg_fallback "${prefix}" " !! 283 gen_try_cmpxchg_fallback "${cmpxchg}" ""
287 printf "#endif\n\n" 284 printf "#endif\n\n"
288 } 285 }
289 286
290 cat << EOF 287 cat << EOF
291 // SPDX-License-Identifier: GPL-2.0 288 // SPDX-License-Identifier: GPL-2.0
292 289
293 // Generated by $0 290 // Generated by $0
294 // DO NOT MODIFY THIS FILE DIRECTLY 291 // DO NOT MODIFY THIS FILE DIRECTLY
295 292
296 #ifndef _LINUX_ATOMIC_FALLBACK_H 293 #ifndef _LINUX_ATOMIC_FALLBACK_H
297 #define _LINUX_ATOMIC_FALLBACK_H 294 #define _LINUX_ATOMIC_FALLBACK_H
298 295
299 #include <linux/compiler.h> 296 #include <linux/compiler.h>
300 297
301 EOF 298 EOF
302 299
303 for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpx 300 for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128"; do
304 gen_xchg_fallbacks "${xchg}" 301 gen_xchg_fallbacks "${xchg}"
305 done 302 done
306 303
307 for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg1 304 for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
308 gen_try_cmpxchg_order_fallbacks "${cmp !! 305 gen_try_cmpxchg_fallbacks "${cmpxchg}"
309 done 306 done
310 307
311 for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg1 !! 308 for cmpxchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local"; do
312 gen_def_and_try_cmpxchg_fallback "" "$ !! 309 gen_cmpxchg_local_fallbacks "${cmpxchg}" ""
313 done 310 done
314 311
315 for cmpxchg in "cmpxchg"; do !! 312 for cmpxchg in "sync_cmpxchg"; do
316 gen_def_and_try_cmpxchg_fallback "sync !! 313 printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
317 done 314 done
318 315
319 grep '^[a-z]' "$1" | while read name meta args 316 grep '^[a-z]' "$1" | while read name meta args; do
320 gen_proto "${meta}" "${name}" "atomic" 317 gen_proto "${meta}" "${name}" "atomic" "int" ${args}
321 done 318 done
322 319
323 cat <<EOF 320 cat <<EOF
324 #ifdef CONFIG_GENERIC_ATOMIC64 321 #ifdef CONFIG_GENERIC_ATOMIC64
325 #include <asm-generic/atomic64.h> 322 #include <asm-generic/atomic64.h>
326 #endif 323 #endif
327 324
328 EOF 325 EOF
329 326
330 grep '^[a-z]' "$1" | while read name meta args 327 grep '^[a-z]' "$1" | while read name meta args; do
331 gen_proto "${meta}" "${name}" "atomic6 328 gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
332 done 329 done
333 330
334 cat <<EOF 331 cat <<EOF
335 #endif /* _LINUX_ATOMIC_FALLBACK_H */ 332 #endif /* _LINUX_ATOMIC_FALLBACK_H */
336 EOF 333 EOF
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.