1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * Author: Paul Burton <paul.burton@mips.com>
4 * (C) Copyright 2018 MIPS Tech LLC
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include "rseq-bits-template.h"
9
10 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
11 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
12
13 static inline __attribute__((always_inline))
14 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
15 {
16 RSEQ_INJECT_C(9)
17
18 __asm__ __volatile__ goto (
19 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
20 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
21 #ifdef RSEQ_COMPARE_TWICE
22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
24 #endif
25 /* Start rseq by storing table entry pointer into rseq_cs. */
26 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
28 RSEQ_INJECT_ASM(3)
29 LONG_L " $4, %[v]\n\t"
30 "bne $4, %[expect], %l[cmpfail]\n\t"
31 RSEQ_INJECT_ASM(4)
32 #ifdef RSEQ_COMPARE_TWICE
33 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
34 LONG_L " $4, %[v]\n\t"
35 "bne $4, %[expect], %l[error2]\n\t"
36 #endif
37 /* final store */
38 LONG_S " %[newv], %[v]\n\t"
39 "2:\n\t"
40 RSEQ_INJECT_ASM(5)
41 "b 5f\n\t"
42 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
43 "5:\n\t"
44 : /* gcc asm goto does not allow outputs */
45 : [cpu_id] "r" (cpu),
46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
48 [v] "m" (*v),
49 [expect] "r" (expect),
50 [newv] "r" (newv)
51 RSEQ_INJECT_INPUT
52 : "$4", "memory"
53 RSEQ_INJECT_CLOBBER
54 : abort, cmpfail
55 #ifdef RSEQ_COMPARE_TWICE
56 , error1, error2
57 #endif
58 );
59 return 0;
60 abort:
61 RSEQ_INJECT_FAILED
62 return -1;
63 cmpfail:
64 return 1;
65 #ifdef RSEQ_COMPARE_TWICE
66 error1:
67 rseq_bug("cpu_id comparison failed");
68 error2:
69 rseq_bug("expected value comparison failed");
70 #endif
71 }
72
73 static inline __attribute__((always_inline))
74 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
75 long voffp, intptr_t *load, int cpu)
76 {
77 RSEQ_INJECT_C(9)
78
79 __asm__ __volatile__ goto (
80 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
81 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
82 #ifdef RSEQ_COMPARE_TWICE
83 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
84 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
85 #endif
86 /* Start rseq by storing table entry pointer into rseq_cs. */
87 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
89 RSEQ_INJECT_ASM(3)
90 LONG_L " $4, %[v]\n\t"
91 "beq $4, %[expectnot], %l[cmpfail]\n\t"
92 RSEQ_INJECT_ASM(4)
93 #ifdef RSEQ_COMPARE_TWICE
94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
95 LONG_L " $4, %[v]\n\t"
96 "beq $4, %[expectnot], %l[error2]\n\t"
97 #endif
98 LONG_S " $4, %[load]\n\t"
99 LONG_ADDI " $4, %[voffp]\n\t"
100 LONG_L " $4, 0($4)\n\t"
101 /* final store */
102 LONG_S " $4, %[v]\n\t"
103 "2:\n\t"
104 RSEQ_INJECT_ASM(5)
105 "b 5f\n\t"
106 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
107 "5:\n\t"
108 : /* gcc asm goto does not allow outputs */
109 : [cpu_id] "r" (cpu),
110 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
111 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
112 /* final store input */
113 [v] "m" (*v),
114 [expectnot] "r" (expectnot),
115 [voffp] "Ir" (voffp),
116 [load] "m" (*load)
117 RSEQ_INJECT_INPUT
118 : "$4", "memory"
119 RSEQ_INJECT_CLOBBER
120 : abort, cmpfail
121 #ifdef RSEQ_COMPARE_TWICE
122 , error1, error2
123 #endif
124 );
125 return 0;
126 abort:
127 RSEQ_INJECT_FAILED
128 return -1;
129 cmpfail:
130 return 1;
131 #ifdef RSEQ_COMPARE_TWICE
132 error1:
133 rseq_bug("cpu_id comparison failed");
134 error2:
135 rseq_bug("expected value comparison failed");
136 #endif
137 }
138
139 static inline __attribute__((always_inline))
140 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
141 {
142 RSEQ_INJECT_C(9)
143
144 __asm__ __volatile__ goto (
145 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
146 #ifdef RSEQ_COMPARE_TWICE
147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
148 #endif
149 /* Start rseq by storing table entry pointer into rseq_cs. */
150 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
152 RSEQ_INJECT_ASM(3)
153 #ifdef RSEQ_COMPARE_TWICE
154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
155 #endif
156 LONG_L " $4, %[v]\n\t"
157 LONG_ADDI " $4, %[count]\n\t"
158 /* final store */
159 LONG_S " $4, %[v]\n\t"
160 "2:\n\t"
161 RSEQ_INJECT_ASM(4)
162 "b 5f\n\t"
163 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
164 "5:\n\t"
165 : /* gcc asm goto does not allow outputs */
166 : [cpu_id] "r" (cpu),
167 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
168 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
169 [v] "m" (*v),
170 [count] "Ir" (count)
171 RSEQ_INJECT_INPUT
172 : "$4", "memory"
173 RSEQ_INJECT_CLOBBER
174 : abort
175 #ifdef RSEQ_COMPARE_TWICE
176 , error1
177 #endif
178 );
179 return 0;
180 abort:
181 RSEQ_INJECT_FAILED
182 return -1;
183 #ifdef RSEQ_COMPARE_TWICE
184 error1:
185 rseq_bug("cpu_id comparison failed");
186 #endif
187 }
188
189 static inline __attribute__((always_inline))
190 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
191 intptr_t *v2, intptr_t expect2,
192 intptr_t newv, int cpu)
193 {
194 RSEQ_INJECT_C(9)
195
196 __asm__ __volatile__ goto (
197 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
198 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
199 #ifdef RSEQ_COMPARE_TWICE
200 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
201 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
202 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
203 #endif
204 /* Start rseq by storing table entry pointer into rseq_cs. */
205 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
206 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
207 RSEQ_INJECT_ASM(3)
208 LONG_L " $4, %[v]\n\t"
209 "bne $4, %[expect], %l[cmpfail]\n\t"
210 RSEQ_INJECT_ASM(4)
211 LONG_L " $4, %[v2]\n\t"
212 "bne $4, %[expect2], %l[cmpfail]\n\t"
213 RSEQ_INJECT_ASM(5)
214 #ifdef RSEQ_COMPARE_TWICE
215 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
216 LONG_L " $4, %[v]\n\t"
217 "bne $4, %[expect], %l[error2]\n\t"
218 LONG_L " $4, %[v2]\n\t"
219 "bne $4, %[expect2], %l[error3]\n\t"
220 #endif
221 /* final store */
222 LONG_S " %[newv], %[v]\n\t"
223 "2:\n\t"
224 RSEQ_INJECT_ASM(6)
225 "b 5f\n\t"
226 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
227 "5:\n\t"
228 : /* gcc asm goto does not allow outputs */
229 : [cpu_id] "r" (cpu),
230 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
231 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
232 /* cmp2 input */
233 [v2] "m" (*v2),
234 [expect2] "r" (expect2),
235 /* final store input */
236 [v] "m" (*v),
237 [expect] "r" (expect),
238 [newv] "r" (newv)
239 RSEQ_INJECT_INPUT
240 : "$4", "memory"
241 RSEQ_INJECT_CLOBBER
242 : abort, cmpfail
243 #ifdef RSEQ_COMPARE_TWICE
244 , error1, error2, error3
245 #endif
246 );
247 return 0;
248 abort:
249 RSEQ_INJECT_FAILED
250 return -1;
251 cmpfail:
252 return 1;
253 #ifdef RSEQ_COMPARE_TWICE
254 error1:
255 rseq_bug("cpu_id comparison failed");
256 error2:
257 rseq_bug("1st expected value comparison failed");
258 error3:
259 rseq_bug("2nd expected value comparison failed");
260 #endif
261 }
262
263 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
264 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
265
266 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
267 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
268
269 static inline __attribute__((always_inline))
270 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
271 intptr_t *v2, intptr_t newv2,
272 intptr_t newv, int cpu)
273 {
274 RSEQ_INJECT_C(9)
275
276 __asm__ __volatile__ goto (
277 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
278 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
279 #ifdef RSEQ_COMPARE_TWICE
280 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
281 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
282 #endif
283 /* Start rseq by storing table entry pointer into rseq_cs. */
284 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
286 RSEQ_INJECT_ASM(3)
287 LONG_L " $4, %[v]\n\t"
288 "bne $4, %[expect], %l[cmpfail]\n\t"
289 RSEQ_INJECT_ASM(4)
290 #ifdef RSEQ_COMPARE_TWICE
291 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
292 LONG_L " $4, %[v]\n\t"
293 "bne $4, %[expect], %l[error2]\n\t"
294 #endif
295 /* try store */
296 LONG_S " %[newv2], %[v2]\n\t"
297 RSEQ_INJECT_ASM(5)
298 #ifdef RSEQ_TEMPLATE_MO_RELEASE
299 "sync\n\t" /* full sync provides store-release */
300 #endif
301 /* final store */
302 LONG_S " %[newv], %[v]\n\t"
303 "2:\n\t"
304 RSEQ_INJECT_ASM(6)
305 "b 5f\n\t"
306 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
307 "5:\n\t"
308 : /* gcc asm goto does not allow outputs */
309 : [cpu_id] "r" (cpu),
310 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
311 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
312 /* try store input */
313 [v2] "m" (*v2),
314 [newv2] "r" (newv2),
315 /* final store input */
316 [v] "m" (*v),
317 [expect] "r" (expect),
318 [newv] "r" (newv)
319 RSEQ_INJECT_INPUT
320 : "$4", "memory"
321 RSEQ_INJECT_CLOBBER
322 : abort, cmpfail
323 #ifdef RSEQ_COMPARE_TWICE
324 , error1, error2
325 #endif
326 );
327 return 0;
328 abort:
329 RSEQ_INJECT_FAILED
330 return -1;
331 cmpfail:
332 return 1;
333 #ifdef RSEQ_COMPARE_TWICE
334 error1:
335 rseq_bug("cpu_id comparison failed");
336 error2:
337 rseq_bug("expected value comparison failed");
338 #endif
339 }
340
341 static inline __attribute__((always_inline))
342 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
343 void *dst, void *src, size_t len,
344 intptr_t newv, int cpu)
345 {
346 uintptr_t rseq_scratch[3];
347
348 RSEQ_INJECT_C(9)
349
350 __asm__ __volatile__ goto (
351 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
352 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
353 #ifdef RSEQ_COMPARE_TWICE
354 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
356 #endif
357 LONG_S " %[src], %[rseq_scratch0]\n\t"
358 LONG_S " %[dst], %[rseq_scratch1]\n\t"
359 LONG_S " %[len], %[rseq_scratch2]\n\t"
360 /* Start rseq by storing table entry pointer into rseq_cs. */
361 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
363 RSEQ_INJECT_ASM(3)
364 LONG_L " $4, %[v]\n\t"
365 "bne $4, %[expect], 5f\n\t"
366 RSEQ_INJECT_ASM(4)
367 #ifdef RSEQ_COMPARE_TWICE
368 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
369 LONG_L " $4, %[v]\n\t"
370 "bne $4, %[expect], 7f\n\t"
371 #endif
372 /* try memcpy */
373 "beqz %[len], 333f\n\t" \
374 "222:\n\t" \
375 "lb $4, 0(%[src])\n\t" \
376 "sb $4, 0(%[dst])\n\t" \
377 LONG_ADDI " %[src], 1\n\t" \
378 LONG_ADDI " %[dst], 1\n\t" \
379 LONG_ADDI " %[len], -1\n\t" \
380 "bnez %[len], 222b\n\t" \
381 "333:\n\t" \
382 RSEQ_INJECT_ASM(5)
383 #ifdef RSEQ_TEMPLATE_MO_RELEASE
384 "sync\n\t" /* full sync provides store-release */
385 #endif
386 /* final store */
387 LONG_S " %[newv], %[v]\n\t"
388 "2:\n\t"
389 RSEQ_INJECT_ASM(6)
390 /* teardown */
391 LONG_L " %[len], %[rseq_scratch2]\n\t"
392 LONG_L " %[dst], %[rseq_scratch1]\n\t"
393 LONG_L " %[src], %[rseq_scratch0]\n\t"
394 "b 8f\n\t"
395 RSEQ_ASM_DEFINE_ABORT(3, 4,
396 /* teardown */
397 LONG_L " %[len], %[rseq_scratch2]\n\t"
398 LONG_L " %[dst], %[rseq_scratch1]\n\t"
399 LONG_L " %[src], %[rseq_scratch0]\n\t",
400 abort, 1b, 2b, 4f)
401 RSEQ_ASM_DEFINE_CMPFAIL(5,
402 /* teardown */
403 LONG_L " %[len], %[rseq_scratch2]\n\t"
404 LONG_L " %[dst], %[rseq_scratch1]\n\t"
405 LONG_L " %[src], %[rseq_scratch0]\n\t",
406 cmpfail)
407 #ifdef RSEQ_COMPARE_TWICE
408 RSEQ_ASM_DEFINE_CMPFAIL(6,
409 /* teardown */
410 LONG_L " %[len], %[rseq_scratch2]\n\t"
411 LONG_L " %[dst], %[rseq_scratch1]\n\t"
412 LONG_L " %[src], %[rseq_scratch0]\n\t",
413 error1)
414 RSEQ_ASM_DEFINE_CMPFAIL(7,
415 /* teardown */
416 LONG_L " %[len], %[rseq_scratch2]\n\t"
417 LONG_L " %[dst], %[rseq_scratch1]\n\t"
418 LONG_L " %[src], %[rseq_scratch0]\n\t",
419 error2)
420 #endif
421 "8:\n\t"
422 : /* gcc asm goto does not allow outputs */
423 : [cpu_id] "r" (cpu),
424 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
425 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
426 /* final store input */
427 [v] "m" (*v),
428 [expect] "r" (expect),
429 [newv] "r" (newv),
430 /* try memcpy input */
431 [dst] "r" (dst),
432 [src] "r" (src),
433 [len] "r" (len),
434 [rseq_scratch0] "m" (rseq_scratch[0]),
435 [rseq_scratch1] "m" (rseq_scratch[1]),
436 [rseq_scratch2] "m" (rseq_scratch[2])
437 RSEQ_INJECT_INPUT
438 : "$4", "memory"
439 RSEQ_INJECT_CLOBBER
440 : abort, cmpfail
441 #ifdef RSEQ_COMPARE_TWICE
442 , error1, error2
443 #endif
444 );
445 return 0;
446 abort:
447 RSEQ_INJECT_FAILED
448 return -1;
449 cmpfail:
450 return 1;
451 #ifdef RSEQ_COMPARE_TWICE
452 error1:
453 rseq_bug("cpu_id comparison failed");
454 error2:
455 rseq_bug("expected value comparison failed");
456 #endif
457 }
458
459 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
460 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
461
462 #include "rseq-bits-reset.h"
463
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.