1 #!/bin/sh 1 #!/bin/sh
2 # SPDX-License-Identifier: GPL-2.0 2 # SPDX-License-Identifier: GPL-2.0
3 3
4 ATOMICDIR=$(dirname $0) 4 ATOMICDIR=$(dirname $0)
>> 5 ARCH=$2
5 6
6 . ${ATOMICDIR}/atomic-tbl.sh 7 . ${ATOMICDIR}/atomic-tbl.sh
7 8
8 #gen_template_fallback(template, meta, pfx, na !! 9 #gen_template_fallback(template, meta, pfx, name, sfx, order, arch, atomic, int, args...)
9 gen_template_fallback() 10 gen_template_fallback()
10 { 11 {
11 local template="$1"; shift 12 local template="$1"; shift
12 local meta="$1"; shift 13 local meta="$1"; shift
13 local pfx="$1"; shift 14 local pfx="$1"; shift
14 local name="$1"; shift 15 local name="$1"; shift
15 local sfx="$1"; shift 16 local sfx="$1"; shift
16 local order="$1"; shift 17 local order="$1"; shift
>> 18 local arch="$1"; shift
17 local atomic="$1"; shift 19 local atomic="$1"; shift
18 local int="$1"; shift 20 local int="$1"; shift
19 21
>> 22 local atomicname="${arch}${atomic}_${pfx}${name}${sfx}${order}"
>> 23
20 local ret="$(gen_ret_type "${meta}" "$ 24 local ret="$(gen_ret_type "${meta}" "${int}")"
21 local retstmt="$(gen_ret_stmt "${meta} 25 local retstmt="$(gen_ret_stmt "${meta}")"
22 local params="$(gen_params "${int}" "$ 26 local params="$(gen_params "${int}" "${atomic}" "$@")"
23 local args="$(gen_args "$@")" 27 local args="$(gen_args "$@")"
24 28
25 . ${template} !! 29 if [ ! -z "${template}" ]; then
>> 30 printf "#ifndef ${atomicname}\n"
>> 31 . ${template}
>> 32 printf "#define ${atomicname} ${atomicname}\n"
>> 33 printf "#endif\n\n"
>> 34 fi
26 } 35 }
27 36
28 #gen_order_fallback(meta, pfx, name, sfx, orde !! 37 #gen_proto_fallback(meta, pfx, name, sfx, order, arch, atomic, int, args...)
29 gen_order_fallback() !! 38 gen_proto_fallback()
30 { 39 {
31 local meta="$1"; shift 40 local meta="$1"; shift
32 local pfx="$1"; shift 41 local pfx="$1"; shift
33 local name="$1"; shift 42 local name="$1"; shift
34 local sfx="$1"; shift 43 local sfx="$1"; shift
35 local order="$1"; shift 44 local order="$1"; shift
36 45
37 local tmpl_order=${order#_} !! 46 local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
38 local tmpl="${ATOMICDIR}/fallbacks/${t <<
39 gen_template_fallback "${tmpl}" "${met 47 gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
40 } 48 }
41 49
42 #gen_proto_fallback(meta, pfx, name, sfx, orde !! 50 #gen_basic_fallbacks(basename)
43 gen_proto_fallback() !! 51 gen_basic_fallbacks()
>> 52 {
>> 53 local basename="$1"; shift
>> 54 cat << EOF
>> 55 #define ${basename}_acquire ${basename}
>> 56 #define ${basename}_release ${basename}
>> 57 #define ${basename}_relaxed ${basename}
>> 58 EOF
>> 59 }
>> 60
>> 61 gen_proto_order_variant()
44 { 62 {
45 local meta="$1"; shift 63 local meta="$1"; shift
46 local pfx="$1"; shift 64 local pfx="$1"; shift
47 local name="$1"; shift 65 local name="$1"; shift
48 local sfx="$1"; shift 66 local sfx="$1"; shift
49 local order="$1"; shift 67 local order="$1"; shift
>> 68 local arch="$1"
>> 69 local atomic="$2"
50 70
51 local tmpl="$(find_fallback_template " !! 71 local basename="${arch}${atomic}_${pfx}${name}${sfx}"
52 gen_template_fallback "${tmpl}" "${met !! 72
>> 73 printf "#define arch_${basename}${order} ${basename}${order}\n"
53 } 74 }
54 75
55 #gen_proto_order_variant(meta, pfx, name, sfx, !! 76 #gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...)
56 gen_proto_order_variant() !! 77 gen_proto_order_variants()
57 { 78 {
58 local meta="$1"; shift 79 local meta="$1"; shift
59 local pfx="$1"; shift 80 local pfx="$1"; shift
60 local name="$1"; shift 81 local name="$1"; shift
61 local sfx="$1"; shift 82 local sfx="$1"; shift
62 local order="$1"; shift !! 83 local arch="$1"
63 local atomic="$1"; shift !! 84 local atomic="$2"
64 local int="$1"; shift <<
65 85
66 local atomicname="${atomic}_${pfx}${na !! 86 local basename="${arch}${atomic}_${pfx}${name}${sfx}"
67 local basename="${atomic}_${pfx}${name <<
68 87
69 local template="$(find_fallback_templa 88 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
70 89
71 local ret="$(gen_ret_type "${meta}" "$ !! 90 if [ -z "$arch" ]; then
72 local retstmt="$(gen_ret_stmt "${meta} !! 91 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
73 local params="$(gen_params "${int}" "$ <<
74 local args="$(gen_args "$@")" <<
75 92
76 gen_kerneldoc "raw_" "${meta}" "${pfx} !! 93 if meta_has_acquire "${meta}"; then
>> 94 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
>> 95 fi
>> 96 if meta_has_release "${meta}"; then
>> 97 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
>> 98 fi
>> 99 if meta_has_relaxed "${meta}"; then
>> 100 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
>> 101 fi
>> 102
>> 103 echo ""
>> 104 fi
>> 105
>> 106 # If we don't have relaxed atomics, then we don't bother with ordering fallbacks
>> 107 # read_acquire and set_release need to be templated, though
>> 108 if ! meta_has_relaxed "${meta}"; then
>> 109 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
>> 110
>> 111 if meta_has_acquire "${meta}"; then
>> 112 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
>> 113 fi
>> 114
>> 115 if meta_has_release "${meta}"; then
>> 116 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
>> 117 fi
77 118
78 printf "static __always_inline ${ret}\ <<
79 printf "raw_${atomicname}(${params})\n <<
80 printf "{\n" <<
81 <<
82 # Where there is no possible fallback, <<
83 # and must be provided by arch code. A <<
84 # make this obvious. <<
85 # <<
86 # Ideally we'd error on a missing defi <<
87 # define this order variant as a C fun <<
88 # symbol. <<
89 if [ -z ${template} ] && [ -z "${order <<
90 printf "\t${retstmt}arch_${ato <<
91 printf "}\n\n" <<
92 return 119 return
93 fi 120 fi
94 121
95 printf "#if defined(arch_${atomicname} !! 122 printf "#ifndef ${basename}_relaxed\n"
96 printf "\t${retstmt}arch_${atomicname} <<
97 <<
98 # Allow FULL/ACQUIRE/RELEASE ops to be <<
99 if [ "${order}" != "_relaxed" ] && met <<
100 printf "#elif defined(arch_${b <<
101 gen_order_fallback "${meta}" " <<
102 fi <<
103 <<
104 # Allow ACQUIRE/RELEASE/RELAXED ops to <<
105 if [ ! -z "${order}" ] && ! meta_is_im <<
106 printf "#elif defined(arch_${b <<
107 printf "\t${retstmt}arch_${bas <<
108 fi <<
109 123
110 printf "#else\n" <<
111 if [ ! -z "${template}" ]; then 124 if [ ! -z "${template}" ]; then
112 gen_proto_fallback "${meta}" " !! 125 printf "#ifdef ${basename}\n"
113 else <<
114 printf "#error \"Unable to def <<
115 fi 126 fi
116 127
117 printf "#endif\n" !! 128 gen_basic_fallbacks "${basename}"
118 printf "}\n\n" <<
119 } <<
120 <<
121 <<
122 #gen_proto_order_variants(meta, pfx, name, sfx <<
123 gen_proto_order_variants() <<
124 { <<
125 local meta="$1"; shift <<
126 local pfx="$1"; shift <<
127 local name="$1"; shift <<
128 local sfx="$1"; shift <<
129 local atomic="$1" <<
130 <<
131 gen_proto_order_variant "${meta}" "${p <<
132 129
133 if meta_has_acquire "${meta}"; then !! 130 if [ ! -z "${template}" ]; then
134 gen_proto_order_variant "${met !! 131 printf "#endif /* ${arch}${atomic}_${pfx}${name}${sfx} */\n\n"
>> 132 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
>> 133 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
>> 134 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
>> 135 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
135 fi 136 fi
136 137
137 if meta_has_release "${meta}"; then !! 138 printf "#else /* ${basename}_relaxed */\n\n"
138 gen_proto_order_variant "${met <<
139 fi <<
140 139
141 if meta_has_relaxed "${meta}"; then !! 140 gen_template_fallback "${ATOMICDIR}/fallbacks/acquire" "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
142 gen_proto_order_variant "${met !! 141 gen_template_fallback "${ATOMICDIR}/fallbacks/release" "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
143 fi !! 142 gen_template_fallback "${ATOMICDIR}/fallbacks/fence" "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
144 } <<
145 143
146 #gen_basic_fallbacks(basename) !! 144 printf "#endif /* ${basename}_relaxed */\n\n"
147 gen_basic_fallbacks() <<
148 { <<
149 local basename="$1"; shift <<
150 cat << EOF <<
151 #define raw_${basename}_acquire arch_${basenam <<
152 #define raw_${basename}_release arch_${basenam <<
153 #define raw_${basename}_relaxed arch_${basenam <<
154 EOF <<
155 } 145 }
156 146
157 gen_order_fallbacks() 147 gen_order_fallbacks()
158 { 148 {
159 local xchg="$1"; shift 149 local xchg="$1"; shift
160 150
161 cat <<EOF 151 cat <<EOF
162 152
163 #define raw_${xchg}_relaxed arch_${xchg}_relax !! 153 #ifndef ${xchg}_acquire
164 !! 154 #define ${xchg}_acquire(...) \\
165 #ifdef arch_${xchg}_acquire !! 155 __atomic_op_acquire(${xchg}, __VA_ARGS__)
166 #define raw_${xchg}_acquire arch_${xchg}_acqui <<
167 #else <<
168 #define raw_${xchg}_acquire(...) \\ <<
169 __atomic_op_acquire(arch_${xchg}, __VA <<
170 #endif 156 #endif
171 157
172 #ifdef arch_${xchg}_release !! 158 #ifndef ${xchg}_release
173 #define raw_${xchg}_release arch_${xchg}_relea !! 159 #define ${xchg}_release(...) \\
174 #else !! 160 __atomic_op_release(${xchg}, __VA_ARGS__)
175 #define raw_${xchg}_release(...) \\ <<
176 __atomic_op_release(arch_${xchg}, __VA <<
177 #endif 161 #endif
178 162
179 #ifdef arch_${xchg} !! 163 #ifndef ${xchg}
180 #define raw_${xchg} arch_${xchg} !! 164 #define ${xchg}(...) \\
181 #else !! 165 __atomic_op_fence(${xchg}, __VA_ARGS__)
182 #define raw_${xchg}(...) \\ <<
183 __atomic_op_fence(arch_${xchg}, __VA_A <<
184 #endif 166 #endif
185 167
186 EOF 168 EOF
187 } 169 }
188 170
189 gen_xchg_order_fallback() !! 171 gen_xchg_fallbacks()
190 { 172 {
191 local xchg="$1"; shift 173 local xchg="$1"; shift
192 local order="$1"; shift !! 174 printf "#ifndef ${xchg}_relaxed\n"
193 local forder="${order:-_fence}" <<
194 175
195 printf "#if defined(arch_${xchg}${orde !! 176 gen_basic_fallbacks ${xchg}
196 printf "#define raw_${xchg}${order} ar <<
197 177
198 if [ "${order}" != "_relaxed" ]; then !! 178 printf "#else /* ${xchg}_relaxed */\n"
199 printf "#elif defined(arch_${x <<
200 printf "#define raw_${xchg}${o <<
201 printf " __atomic_op${f <<
202 fi <<
203 179
204 if [ ! -z "${order}" ]; then !! 180 gen_order_fallbacks ${xchg}
205 printf "#elif defined(arch_${x <<
206 printf "#define raw_${xchg}${o <<
207 fi <<
208 181
209 printf "#else\n" !! 182 printf "#endif /* ${xchg}_relaxed */\n\n"
210 printf "extern void raw_${xchg}${order <<
211 printf "#define raw_${xchg}${order}(.. <<
212 printf "#endif\n\n" <<
213 } <<
214 <<
215 gen_xchg_fallbacks() <<
216 { <<
217 local xchg="$1"; shift <<
218 <<
219 for order in "" "_acquire" "_release" <<
220 gen_xchg_order_fallback "${xch <<
221 done <<
222 } 183 }
223 184
224 gen_try_cmpxchg_fallback() 185 gen_try_cmpxchg_fallback()
225 { 186 {
226 local prefix="$1"; shift !! 187 local order="$1"; shift;
227 local cmpxchg="$1"; shift; <<
228 local suffix="$1"; shift; <<
229 188
230 cat <<EOF 189 cat <<EOF
231 #define raw_${prefix}try_${cmpxchg}${suffix}(_ !! 190 #ifndef ${ARCH}try_cmpxchg${order}
>> 191 #define ${ARCH}try_cmpxchg${order}(_ptr, _oldp, _new) \\
232 ({ \\ 192 ({ \\
233 typeof(*(_ptr)) *___op = (_oldp), ___o 193 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
234 ___r = raw_${prefix}${cmpxchg}${suffix !! 194 ___r = ${ARCH}cmpxchg${order}((_ptr), ___o, (_new)); \\
235 if (unlikely(___r != ___o)) \\ 195 if (unlikely(___r != ___o)) \\
236 *___op = ___r; \\ 196 *___op = ___r; \\
237 likely(___r == ___o); \\ 197 likely(___r == ___o); \\
238 }) 198 })
>> 199 #endif /* ${ARCH}try_cmpxchg${order} */
>> 200
239 EOF 201 EOF
240 } 202 }
241 203
242 gen_try_cmpxchg_order_fallback() !! 204 gen_try_cmpxchg_fallbacks()
243 { 205 {
244 local cmpxchg="$1"; shift !! 206 printf "#ifndef ${ARCH}try_cmpxchg_relaxed\n"
245 local order="$1"; shift !! 207 printf "#ifdef ${ARCH}try_cmpxchg\n"
246 local forder="${order:-_fence}" <<
247 <<
248 printf "#if defined(arch_try_${cmpxchg <<
249 printf "#define raw_try_${cmpxchg}${or <<
250 208
251 if [ "${order}" != "_relaxed" ]; then !! 209 gen_basic_fallbacks "${ARCH}try_cmpxchg"
252 printf "#elif defined(arch_try <<
253 printf "#define raw_try_${cmpx <<
254 printf " __atomic_op${f <<
255 fi <<
256 <<
257 if [ ! -z "${order}" ]; then <<
258 printf "#elif defined(arch_try <<
259 printf "#define raw_try_${cmpx <<
260 fi <<
261 <<
262 printf "#else\n" <<
263 gen_try_cmpxchg_fallback "" "${cmpxchg <<
264 printf "#endif\n\n" <<
265 } <<
266 210
267 gen_try_cmpxchg_order_fallbacks() !! 211 printf "#endif /* ${ARCH}try_cmpxchg */\n\n"
268 { <<
269 local cmpxchg="$1"; shift; <<
270 212
271 for order in "" "_acquire" "_release" 213 for order in "" "_acquire" "_release" "_relaxed"; do
272 gen_try_cmpxchg_order_fallback !! 214 gen_try_cmpxchg_fallback "${order}"
273 done 215 done
274 } <<
275 216
276 gen_def_and_try_cmpxchg_fallback() !! 217 printf "#else /* ${ARCH}try_cmpxchg_relaxed */\n"
277 { !! 218
278 local prefix="$1"; shift !! 219 gen_order_fallbacks "${ARCH}try_cmpxchg"
279 local cmpxchg="$1"; shift !! 220
280 local suffix="$1"; shift !! 221 printf "#endif /* ${ARCH}try_cmpxchg_relaxed */\n\n"
281 <<
282 printf "#define raw_${prefix}${cmpxchg <<
283 printf "#ifdef arch_${prefix}try_${cmp <<
284 printf "#define raw_${prefix}try_${cmp <<
285 printf "#else\n" <<
286 gen_try_cmpxchg_fallback "${prefix}" " <<
287 printf "#endif\n\n" <<
288 } 222 }
289 223
290 cat << EOF 224 cat << EOF
291 // SPDX-License-Identifier: GPL-2.0 225 // SPDX-License-Identifier: GPL-2.0
292 226
293 // Generated by $0 227 // Generated by $0
294 // DO NOT MODIFY THIS FILE DIRECTLY 228 // DO NOT MODIFY THIS FILE DIRECTLY
295 229
296 #ifndef _LINUX_ATOMIC_FALLBACK_H 230 #ifndef _LINUX_ATOMIC_FALLBACK_H
297 #define _LINUX_ATOMIC_FALLBACK_H 231 #define _LINUX_ATOMIC_FALLBACK_H
298 232
299 #include <linux/compiler.h> 233 #include <linux/compiler.h>
300 234
301 EOF 235 EOF
302 236
303 for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpx !! 237 for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do
304 gen_xchg_fallbacks "${xchg}" 238 gen_xchg_fallbacks "${xchg}"
305 done 239 done
306 240
307 for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg1 !! 241 gen_try_cmpxchg_fallbacks
308 gen_try_cmpxchg_order_fallbacks "${cmp <<
309 done <<
310 <<
311 for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg1 <<
312 gen_def_and_try_cmpxchg_fallback "" "$ <<
313 done <<
314 <<
315 for cmpxchg in "cmpxchg"; do <<
316 gen_def_and_try_cmpxchg_fallback "sync <<
317 done <<
318 242
319 grep '^[a-z]' "$1" | while read name meta args 243 grep '^[a-z]' "$1" | while read name meta args; do
320 gen_proto "${meta}" "${name}" "atomic" !! 244 gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args}
321 done 245 done
322 246
323 cat <<EOF 247 cat <<EOF
324 #ifdef CONFIG_GENERIC_ATOMIC64 248 #ifdef CONFIG_GENERIC_ATOMIC64
325 #include <asm-generic/atomic64.h> 249 #include <asm-generic/atomic64.h>
326 #endif 250 #endif
327 251
328 EOF 252 EOF
329 253
330 grep '^[a-z]' "$1" | while read name meta args 254 grep '^[a-z]' "$1" | while read name meta args; do
331 gen_proto "${meta}" "${name}" "atomic6 !! 255 gen_proto "${meta}" "${name}" "${ARCH}" "atomic64" "s64" ${args}
332 done 256 done
333 257
334 cat <<EOF 258 cat <<EOF
335 #endif /* _LINUX_ATOMIC_FALLBACK_H */ 259 #endif /* _LINUX_ATOMIC_FALLBACK_H */
336 EOF 260 EOF
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.