1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-x86-bits.h
4 *
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include "rseq-bits-template.h"
9
10 #ifdef __x86_64__
11
12 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
13 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
14
15 static inline __attribute__((always_inline))
16 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
17 {
18 RSEQ_INJECT_C(9)
19
20 __asm__ __volatile__ goto (
21 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
23 #ifdef RSEQ_COMPARE_TWICE
24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
25 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
26 #endif
27 /* Start rseq by storing table entry pointer into rseq_cs. */
28 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
29 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
30 RSEQ_INJECT_ASM(3)
31 "cmpq %[v], %[expect]\n\t"
32 "jnz %l[cmpfail]\n\t"
33 RSEQ_INJECT_ASM(4)
34 #ifdef RSEQ_COMPARE_TWICE
35 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
36 "cmpq %[v], %[expect]\n\t"
37 "jnz %l[error2]\n\t"
38 #endif
39 /* final store */
40 "movq %[newv], %[v]\n\t"
41 "2:\n\t"
42 RSEQ_INJECT_ASM(5)
43 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
44 : /* gcc asm goto does not allow outputs */
45 : [cpu_id] "r" (cpu),
46 [rseq_offset] "r" (rseq_offset),
47 [v] "m" (*v),
48 [expect] "r" (expect),
49 [newv] "r" (newv)
50 : "memory", "cc", "rax"
51 RSEQ_INJECT_CLOBBER
52 : abort, cmpfail
53 #ifdef RSEQ_COMPARE_TWICE
54 , error1, error2
55 #endif
56 );
57 rseq_after_asm_goto();
58 return 0;
59 abort:
60 rseq_after_asm_goto();
61 RSEQ_INJECT_FAILED
62 return -1;
63 cmpfail:
64 rseq_after_asm_goto();
65 return 1;
66 #ifdef RSEQ_COMPARE_TWICE
67 error1:
68 rseq_after_asm_goto();
69 rseq_bug("cpu_id comparison failed");
70 error2:
71 rseq_after_asm_goto();
72 rseq_bug("expected value comparison failed");
73 #endif
74 }
75
76 /*
77 * Compare @v against @expectnot. When it does _not_ match, load @v
78 * into @load, and store the content of *@v + voffp into @v.
79 */
80 static inline __attribute__((always_inline))
81 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
82 long voffp, intptr_t *load, int cpu)
83 {
84 RSEQ_INJECT_C(9)
85
86 __asm__ __volatile__ goto (
87 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
89 #ifdef RSEQ_COMPARE_TWICE
90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
92 #endif
93 /* Start rseq by storing table entry pointer into rseq_cs. */
94 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
95 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
96 RSEQ_INJECT_ASM(3)
97 "movq %[v], %%rbx\n\t"
98 "cmpq %%rbx, %[expectnot]\n\t"
99 "je %l[cmpfail]\n\t"
100 RSEQ_INJECT_ASM(4)
101 #ifdef RSEQ_COMPARE_TWICE
102 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
103 "movq %[v], %%rbx\n\t"
104 "cmpq %%rbx, %[expectnot]\n\t"
105 "je %l[error2]\n\t"
106 #endif
107 "movq %%rbx, %[load]\n\t"
108 "addq %[voffp], %%rbx\n\t"
109 "movq (%%rbx), %%rbx\n\t"
110 /* final store */
111 "movq %%rbx, %[v]\n\t"
112 "2:\n\t"
113 RSEQ_INJECT_ASM(5)
114 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
115 : /* gcc asm goto does not allow outputs */
116 : [cpu_id] "r" (cpu),
117 [rseq_offset] "r" (rseq_offset),
118 /* final store input */
119 [v] "m" (*v),
120 [expectnot] "r" (expectnot),
121 [voffp] "er" (voffp),
122 [load] "m" (*load)
123 : "memory", "cc", "rax", "rbx"
124 RSEQ_INJECT_CLOBBER
125 : abort, cmpfail
126 #ifdef RSEQ_COMPARE_TWICE
127 , error1, error2
128 #endif
129 );
130 rseq_after_asm_goto();
131 return 0;
132 abort:
133 rseq_after_asm_goto();
134 RSEQ_INJECT_FAILED
135 return -1;
136 cmpfail:
137 rseq_after_asm_goto();
138 return 1;
139 #ifdef RSEQ_COMPARE_TWICE
140 error1:
141 rseq_after_asm_goto();
142 rseq_bug("cpu_id comparison failed");
143 error2:
144 rseq_after_asm_goto();
145 rseq_bug("expected value comparison failed");
146 #endif
147 }
148
149 static inline __attribute__((always_inline))
150 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
151 {
152 RSEQ_INJECT_C(9)
153
154 __asm__ __volatile__ goto (
155 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
156 #ifdef RSEQ_COMPARE_TWICE
157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
158 #endif
159 /* Start rseq by storing table entry pointer into rseq_cs. */
160 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
161 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
162 RSEQ_INJECT_ASM(3)
163 #ifdef RSEQ_COMPARE_TWICE
164 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
165 #endif
166 /* final store */
167 "addq %[count], %[v]\n\t"
168 "2:\n\t"
169 RSEQ_INJECT_ASM(4)
170 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
171 : /* gcc asm goto does not allow outputs */
172 : [cpu_id] "r" (cpu),
173 [rseq_offset] "r" (rseq_offset),
174 /* final store input */
175 [v] "m" (*v),
176 [count] "er" (count)
177 : "memory", "cc", "rax"
178 RSEQ_INJECT_CLOBBER
179 : abort
180 #ifdef RSEQ_COMPARE_TWICE
181 , error1
182 #endif
183 );
184 rseq_after_asm_goto();
185 return 0;
186 abort:
187 rseq_after_asm_goto();
188 RSEQ_INJECT_FAILED
189 return -1;
190 #ifdef RSEQ_COMPARE_TWICE
191 error1:
192 rseq_after_asm_goto();
193 rseq_bug("cpu_id comparison failed");
194 #endif
195 }
196
197 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
198
199 /*
200 * pval = *(ptr+off)
201 * *pval += inc;
202 */
203 static inline __attribute__((always_inline))
204 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu)
205 {
206 RSEQ_INJECT_C(9)
207
208 __asm__ __volatile__ goto (
209 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
210 #ifdef RSEQ_COMPARE_TWICE
211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
212 #endif
213 /* Start rseq by storing table entry pointer into rseq_cs. */
214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
215 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
216 RSEQ_INJECT_ASM(3)
217 #ifdef RSEQ_COMPARE_TWICE
218 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
219 #endif
220 /* get p+v */
221 "movq %[ptr], %%rbx\n\t"
222 "addq %[off], %%rbx\n\t"
223 /* get pv */
224 "movq (%%rbx), %%rcx\n\t"
225 /* *pv += inc */
226 "addq %[inc], (%%rcx)\n\t"
227 "2:\n\t"
228 RSEQ_INJECT_ASM(4)
229 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
230 : /* gcc asm goto does not allow outputs */
231 : [cpu_id] "r" (cpu),
232 [rseq_offset] "r" (rseq_offset),
233 /* final store input */
234 [ptr] "m" (*ptr),
235 [off] "er" (off),
236 [inc] "er" (inc)
237 : "memory", "cc", "rax", "rbx", "rcx"
238 RSEQ_INJECT_CLOBBER
239 : abort
240 #ifdef RSEQ_COMPARE_TWICE
241 , error1
242 #endif
243 );
244 return 0;
245 abort:
246 RSEQ_INJECT_FAILED
247 return -1;
248 #ifdef RSEQ_COMPARE_TWICE
249 error1:
250 rseq_bug("cpu_id comparison failed");
251 #endif
252 }
253
254 static inline __attribute__((always_inline))
255 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
256 intptr_t *v2, intptr_t expect2,
257 intptr_t newv, int cpu)
258 {
259 RSEQ_INJECT_C(9)
260
261 __asm__ __volatile__ goto (
262 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
263 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
264 #ifdef RSEQ_COMPARE_TWICE
265 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
266 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
267 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
268 #endif
269 /* Start rseq by storing table entry pointer into rseq_cs. */
270 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
271 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
272 RSEQ_INJECT_ASM(3)
273 "cmpq %[v], %[expect]\n\t"
274 "jnz %l[cmpfail]\n\t"
275 RSEQ_INJECT_ASM(4)
276 "cmpq %[v2], %[expect2]\n\t"
277 "jnz %l[cmpfail]\n\t"
278 RSEQ_INJECT_ASM(5)
279 #ifdef RSEQ_COMPARE_TWICE
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
281 "cmpq %[v], %[expect]\n\t"
282 "jnz %l[error2]\n\t"
283 "cmpq %[v2], %[expect2]\n\t"
284 "jnz %l[error3]\n\t"
285 #endif
286 /* final store */
287 "movq %[newv], %[v]\n\t"
288 "2:\n\t"
289 RSEQ_INJECT_ASM(6)
290 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
291 : /* gcc asm goto does not allow outputs */
292 : [cpu_id] "r" (cpu),
293 [rseq_offset] "r" (rseq_offset),
294 /* cmp2 input */
295 [v2] "m" (*v2),
296 [expect2] "r" (expect2),
297 /* final store input */
298 [v] "m" (*v),
299 [expect] "r" (expect),
300 [newv] "r" (newv)
301 : "memory", "cc", "rax"
302 RSEQ_INJECT_CLOBBER
303 : abort, cmpfail
304 #ifdef RSEQ_COMPARE_TWICE
305 , error1, error2, error3
306 #endif
307 );
308 rseq_after_asm_goto();
309 return 0;
310 abort:
311 rseq_after_asm_goto();
312 RSEQ_INJECT_FAILED
313 return -1;
314 cmpfail:
315 rseq_after_asm_goto();
316 return 1;
317 #ifdef RSEQ_COMPARE_TWICE
318 error1:
319 rseq_after_asm_goto();
320 rseq_bug("cpu_id comparison failed");
321 error2:
322 rseq_after_asm_goto();
323 rseq_bug("1st expected value comparison failed");
324 error3:
325 rseq_after_asm_goto();
326 rseq_bug("2nd expected value comparison failed");
327 #endif
328 }
329
330 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
331 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
332
333 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
334 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
335
336 static inline __attribute__((always_inline))
337 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
338 intptr_t *v2, intptr_t newv2,
339 intptr_t newv, int cpu)
340 {
341 RSEQ_INJECT_C(9)
342
343 __asm__ __volatile__ goto (
344 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
345 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
346 #ifdef RSEQ_COMPARE_TWICE
347 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
349 #endif
350 /* Start rseq by storing table entry pointer into rseq_cs. */
351 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
352 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
353 RSEQ_INJECT_ASM(3)
354 "cmpq %[v], %[expect]\n\t"
355 "jnz %l[cmpfail]\n\t"
356 RSEQ_INJECT_ASM(4)
357 #ifdef RSEQ_COMPARE_TWICE
358 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
359 "cmpq %[v], %[expect]\n\t"
360 "jnz %l[error2]\n\t"
361 #endif
362 /* try store */
363 "movq %[newv2], %[v2]\n\t"
364 RSEQ_INJECT_ASM(5)
365 /* final store */
366 "movq %[newv], %[v]\n\t"
367 "2:\n\t"
368 RSEQ_INJECT_ASM(6)
369 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
370 : /* gcc asm goto does not allow outputs */
371 : [cpu_id] "r" (cpu),
372 [rseq_offset] "r" (rseq_offset),
373 /* try store input */
374 [v2] "m" (*v2),
375 [newv2] "r" (newv2),
376 /* final store input */
377 [v] "m" (*v),
378 [expect] "r" (expect),
379 [newv] "r" (newv)
380 : "memory", "cc", "rax"
381 RSEQ_INJECT_CLOBBER
382 : abort, cmpfail
383 #ifdef RSEQ_COMPARE_TWICE
384 , error1, error2
385 #endif
386 );
387 rseq_after_asm_goto();
388 return 0;
389 abort:
390 rseq_after_asm_goto();
391 RSEQ_INJECT_FAILED
392 return -1;
393 cmpfail:
394 rseq_after_asm_goto();
395 return 1;
396 #ifdef RSEQ_COMPARE_TWICE
397 error1:
398 rseq_after_asm_goto();
399 rseq_bug("cpu_id comparison failed");
400 error2:
401 rseq_after_asm_goto();
402 rseq_bug("expected value comparison failed");
403 #endif
404 }
405
406 static inline __attribute__((always_inline))
407 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
408 void *dst, void *src, size_t len,
409 intptr_t newv, int cpu)
410 {
411 uint64_t rseq_scratch[3];
412
413 RSEQ_INJECT_C(9)
414
415 __asm__ __volatile__ goto (
416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
417 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
418 #ifdef RSEQ_COMPARE_TWICE
419 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
421 #endif
422 "movq %[src], %[rseq_scratch0]\n\t"
423 "movq %[dst], %[rseq_scratch1]\n\t"
424 "movq %[len], %[rseq_scratch2]\n\t"
425 /* Start rseq by storing table entry pointer into rseq_cs. */
426 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
427 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
428 RSEQ_INJECT_ASM(3)
429 "cmpq %[v], %[expect]\n\t"
430 "jnz 5f\n\t"
431 RSEQ_INJECT_ASM(4)
432 #ifdef RSEQ_COMPARE_TWICE
433 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
434 "cmpq %[v], %[expect]\n\t"
435 "jnz 7f\n\t"
436 #endif
437 /* try memcpy */
438 "test %[len], %[len]\n\t" \
439 "jz 333f\n\t" \
440 "222:\n\t" \
441 "movb (%[src]), %%al\n\t" \
442 "movb %%al, (%[dst])\n\t" \
443 "inc %[src]\n\t" \
444 "inc %[dst]\n\t" \
445 "dec %[len]\n\t" \
446 "jnz 222b\n\t" \
447 "333:\n\t" \
448 RSEQ_INJECT_ASM(5)
449 /* final store */
450 "movq %[newv], %[v]\n\t"
451 "2:\n\t"
452 RSEQ_INJECT_ASM(6)
453 /* teardown */
454 "movq %[rseq_scratch2], %[len]\n\t"
455 "movq %[rseq_scratch1], %[dst]\n\t"
456 "movq %[rseq_scratch0], %[src]\n\t"
457 RSEQ_ASM_DEFINE_ABORT(4,
458 "movq %[rseq_scratch2], %[len]\n\t"
459 "movq %[rseq_scratch1], %[dst]\n\t"
460 "movq %[rseq_scratch0], %[src]\n\t",
461 abort)
462 RSEQ_ASM_DEFINE_CMPFAIL(5,
463 "movq %[rseq_scratch2], %[len]\n\t"
464 "movq %[rseq_scratch1], %[dst]\n\t"
465 "movq %[rseq_scratch0], %[src]\n\t",
466 cmpfail)
467 #ifdef RSEQ_COMPARE_TWICE
468 RSEQ_ASM_DEFINE_CMPFAIL(6,
469 "movq %[rseq_scratch2], %[len]\n\t"
470 "movq %[rseq_scratch1], %[dst]\n\t"
471 "movq %[rseq_scratch0], %[src]\n\t",
472 error1)
473 RSEQ_ASM_DEFINE_CMPFAIL(7,
474 "movq %[rseq_scratch2], %[len]\n\t"
475 "movq %[rseq_scratch1], %[dst]\n\t"
476 "movq %[rseq_scratch0], %[src]\n\t",
477 error2)
478 #endif
479 : /* gcc asm goto does not allow outputs */
480 : [cpu_id] "r" (cpu),
481 [rseq_offset] "r" (rseq_offset),
482 /* final store input */
483 [v] "m" (*v),
484 [expect] "r" (expect),
485 [newv] "r" (newv),
486 /* try memcpy input */
487 [dst] "r" (dst),
488 [src] "r" (src),
489 [len] "r" (len),
490 [rseq_scratch0] "m" (rseq_scratch[0]),
491 [rseq_scratch1] "m" (rseq_scratch[1]),
492 [rseq_scratch2] "m" (rseq_scratch[2])
493 : "memory", "cc", "rax"
494 RSEQ_INJECT_CLOBBER
495 : abort, cmpfail
496 #ifdef RSEQ_COMPARE_TWICE
497 , error1, error2
498 #endif
499 );
500 rseq_after_asm_goto();
501 return 0;
502 abort:
503 rseq_after_asm_goto();
504 RSEQ_INJECT_FAILED
505 return -1;
506 cmpfail:
507 rseq_after_asm_goto();
508 return 1;
509 #ifdef RSEQ_COMPARE_TWICE
510 error1:
511 rseq_after_asm_goto();
512 rseq_bug("cpu_id comparison failed");
513 error2:
514 rseq_after_asm_goto();
515 rseq_bug("expected value comparison failed");
516 #endif
517 }
518
519 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
520 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
521
522 #elif defined(__i386__)
523
524 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
525 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
526
527 static inline __attribute__((always_inline))
528 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
529 {
530 RSEQ_INJECT_C(9)
531
532 __asm__ __volatile__ goto (
533 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
535 #ifdef RSEQ_COMPARE_TWICE
536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
537 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
538 #endif
539 /* Start rseq by storing table entry pointer into rseq_cs. */
540 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
541 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
542 RSEQ_INJECT_ASM(3)
543 "cmpl %[v], %[expect]\n\t"
544 "jnz %l[cmpfail]\n\t"
545 RSEQ_INJECT_ASM(4)
546 #ifdef RSEQ_COMPARE_TWICE
547 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
548 "cmpl %[v], %[expect]\n\t"
549 "jnz %l[error2]\n\t"
550 #endif
551 /* final store */
552 "movl %[newv], %[v]\n\t"
553 "2:\n\t"
554 RSEQ_INJECT_ASM(5)
555 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
556 : /* gcc asm goto does not allow outputs */
557 : [cpu_id] "r" (cpu),
558 [rseq_offset] "r" (rseq_offset),
559 [v] "m" (*v),
560 [expect] "r" (expect),
561 [newv] "r" (newv)
562 : "memory", "cc", "eax"
563 RSEQ_INJECT_CLOBBER
564 : abort, cmpfail
565 #ifdef RSEQ_COMPARE_TWICE
566 , error1, error2
567 #endif
568 );
569 rseq_after_asm_goto();
570 return 0;
571 abort:
572 rseq_after_asm_goto();
573 RSEQ_INJECT_FAILED
574 return -1;
575 cmpfail:
576 rseq_after_asm_goto();
577 return 1;
578 #ifdef RSEQ_COMPARE_TWICE
579 error1:
580 rseq_after_asm_goto();
581 rseq_bug("cpu_id comparison failed");
582 error2:
583 rseq_after_asm_goto();
584 rseq_bug("expected value comparison failed");
585 #endif
586 }
587
588 /*
589 * Compare @v against @expectnot. When it does _not_ match, load @v
590 * into @load, and store the content of *@v + voffp into @v.
591 */
592 static inline __attribute__((always_inline))
593 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
594 long voffp, intptr_t *load, int cpu)
595 {
596 RSEQ_INJECT_C(9)
597
598 __asm__ __volatile__ goto (
599 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
600 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
601 #ifdef RSEQ_COMPARE_TWICE
602 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
603 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
604 #endif
605 /* Start rseq by storing table entry pointer into rseq_cs. */
606 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
607 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
608 RSEQ_INJECT_ASM(3)
609 "movl %[v], %%ebx\n\t"
610 "cmpl %%ebx, %[expectnot]\n\t"
611 "je %l[cmpfail]\n\t"
612 RSEQ_INJECT_ASM(4)
613 #ifdef RSEQ_COMPARE_TWICE
614 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
615 "movl %[v], %%ebx\n\t"
616 "cmpl %%ebx, %[expectnot]\n\t"
617 "je %l[error2]\n\t"
618 #endif
619 "movl %%ebx, %[load]\n\t"
620 "addl %[voffp], %%ebx\n\t"
621 "movl (%%ebx), %%ebx\n\t"
622 /* final store */
623 "movl %%ebx, %[v]\n\t"
624 "2:\n\t"
625 RSEQ_INJECT_ASM(5)
626 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
627 : /* gcc asm goto does not allow outputs */
628 : [cpu_id] "r" (cpu),
629 [rseq_offset] "r" (rseq_offset),
630 /* final store input */
631 [v] "m" (*v),
632 [expectnot] "r" (expectnot),
633 [voffp] "ir" (voffp),
634 [load] "m" (*load)
635 : "memory", "cc", "eax", "ebx"
636 RSEQ_INJECT_CLOBBER
637 : abort, cmpfail
638 #ifdef RSEQ_COMPARE_TWICE
639 , error1, error2
640 #endif
641 );
642 rseq_after_asm_goto();
643 return 0;
644 abort:
645 rseq_after_asm_goto();
646 RSEQ_INJECT_FAILED
647 return -1;
648 cmpfail:
649 rseq_after_asm_goto();
650 return 1;
651 #ifdef RSEQ_COMPARE_TWICE
652 error1:
653 rseq_after_asm_goto();
654 rseq_bug("cpu_id comparison failed");
655 error2:
656 rseq_after_asm_goto();
657 rseq_bug("expected value comparison failed");
658 #endif
659 }
660
661 static inline __attribute__((always_inline))
662 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
663 {
664 RSEQ_INJECT_C(9)
665
666 __asm__ __volatile__ goto (
667 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
668 #ifdef RSEQ_COMPARE_TWICE
669 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
670 #endif
671 /* Start rseq by storing table entry pointer into rseq_cs. */
672 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
673 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
674 RSEQ_INJECT_ASM(3)
675 #ifdef RSEQ_COMPARE_TWICE
676 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
677 #endif
678 /* final store */
679 "addl %[count], %[v]\n\t"
680 "2:\n\t"
681 RSEQ_INJECT_ASM(4)
682 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
683 : /* gcc asm goto does not allow outputs */
684 : [cpu_id] "r" (cpu),
685 [rseq_offset] "r" (rseq_offset),
686 /* final store input */
687 [v] "m" (*v),
688 [count] "ir" (count)
689 : "memory", "cc", "eax"
690 RSEQ_INJECT_CLOBBER
691 : abort
692 #ifdef RSEQ_COMPARE_TWICE
693 , error1
694 #endif
695 );
696 rseq_after_asm_goto();
697 return 0;
698 abort:
699 rseq_after_asm_goto();
700 RSEQ_INJECT_FAILED
701 return -1;
702 #ifdef RSEQ_COMPARE_TWICE
703 error1:
704 rseq_after_asm_goto();
705 rseq_bug("cpu_id comparison failed");
706 #endif
707 }
708
709 static inline __attribute__((always_inline))
710 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
711 intptr_t *v2, intptr_t expect2,
712 intptr_t newv, int cpu)
713 {
714 RSEQ_INJECT_C(9)
715
716 __asm__ __volatile__ goto (
717 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
718 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
719 #ifdef RSEQ_COMPARE_TWICE
720 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
721 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
722 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
723 #endif
724 /* Start rseq by storing table entry pointer into rseq_cs. */
725 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
726 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
727 RSEQ_INJECT_ASM(3)
728 "cmpl %[v], %[expect]\n\t"
729 "jnz %l[cmpfail]\n\t"
730 RSEQ_INJECT_ASM(4)
731 "cmpl %[expect2], %[v2]\n\t"
732 "jnz %l[cmpfail]\n\t"
733 RSEQ_INJECT_ASM(5)
734 #ifdef RSEQ_COMPARE_TWICE
735 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
736 "cmpl %[v], %[expect]\n\t"
737 "jnz %l[error2]\n\t"
738 "cmpl %[expect2], %[v2]\n\t"
739 "jnz %l[error3]\n\t"
740 #endif
741 "movl %[newv], %%eax\n\t"
742 /* final store */
743 "movl %%eax, %[v]\n\t"
744 "2:\n\t"
745 RSEQ_INJECT_ASM(6)
746 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
747 : /* gcc asm goto does not allow outputs */
748 : [cpu_id] "r" (cpu),
749 [rseq_offset] "r" (rseq_offset),
750 /* cmp2 input */
751 [v2] "m" (*v2),
752 [expect2] "r" (expect2),
753 /* final store input */
754 [v] "m" (*v),
755 [expect] "r" (expect),
756 [newv] "m" (newv)
757 : "memory", "cc", "eax"
758 RSEQ_INJECT_CLOBBER
759 : abort, cmpfail
760 #ifdef RSEQ_COMPARE_TWICE
761 , error1, error2, error3
762 #endif
763 );
764 rseq_after_asm_goto();
765 return 0;
766 abort:
767 rseq_after_asm_goto();
768 RSEQ_INJECT_FAILED
769 return -1;
770 cmpfail:
771 rseq_after_asm_goto();
772 return 1;
773 #ifdef RSEQ_COMPARE_TWICE
774 error1:
775 rseq_after_asm_goto();
776 rseq_bug("cpu_id comparison failed");
777 error2:
778 rseq_after_asm_goto();
779 rseq_bug("1st expected value comparison failed");
780 error3:
781 rseq_after_asm_goto();
782 rseq_bug("2nd expected value comparison failed");
783 #endif
784 }
785
786 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
787 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
788
789 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
790 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
791
792 static inline __attribute__((always_inline))
793 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
794 intptr_t *v2, intptr_t newv2,
795 intptr_t newv, int cpu)
796 {
797 RSEQ_INJECT_C(9)
798
799 __asm__ __volatile__ goto (
800 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
801 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
802 #ifdef RSEQ_COMPARE_TWICE
803 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
804 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
805 #endif
806 /* Start rseq by storing table entry pointer into rseq_cs. */
807 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
808 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
809 RSEQ_INJECT_ASM(3)
810 "movl %[expect], %%eax\n\t"
811 "cmpl %[v], %%eax\n\t"
812 "jnz %l[cmpfail]\n\t"
813 RSEQ_INJECT_ASM(4)
814 #ifdef RSEQ_COMPARE_TWICE
815 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
816 "movl %[expect], %%eax\n\t"
817 "cmpl %[v], %%eax\n\t"
818 "jnz %l[error2]\n\t"
819 #endif
820 /* try store */
821 "movl %[newv2], %[v2]\n\t"
822 RSEQ_INJECT_ASM(5)
823 #ifdef RSEQ_TEMPLATE_MO_RELEASE
824 "lock; addl $0,-128(%%esp)\n\t"
825 #endif
826 /* final store */
827 "movl %[newv], %[v]\n\t"
828 "2:\n\t"
829 RSEQ_INJECT_ASM(6)
830 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
831 : /* gcc asm goto does not allow outputs */
832 : [cpu_id] "r" (cpu),
833 [rseq_offset] "r" (rseq_offset),
834 /* try store input */
835 [v2] "m" (*v2),
836 [newv2] "r" (newv2),
837 /* final store input */
838 [v] "m" (*v),
839 [expect] "m" (expect),
840 [newv] "r" (newv)
841 : "memory", "cc", "eax"
842 RSEQ_INJECT_CLOBBER
843 : abort, cmpfail
844 #ifdef RSEQ_COMPARE_TWICE
845 , error1, error2
846 #endif
847 );
848 rseq_after_asm_goto();
849 return 0;
850 abort:
851 rseq_after_asm_goto();
852 RSEQ_INJECT_FAILED
853 return -1;
854 cmpfail:
855 rseq_after_asm_goto();
856 return 1;
857 #ifdef RSEQ_COMPARE_TWICE
858 error1:
859 rseq_after_asm_goto();
860 rseq_bug("cpu_id comparison failed");
861 error2:
862 rseq_after_asm_goto();
863 rseq_bug("expected value comparison failed");
864 #endif
865
866 }
867
868 /* TODO: implement a faster memcpy. */
869 static inline __attribute__((always_inline))
870 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
871 void *dst, void *src, size_t len,
872 intptr_t newv, int cpu)
873 {
874 uint32_t rseq_scratch[3];
875
876 RSEQ_INJECT_C(9)
877
878 __asm__ __volatile__ goto (
879 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
880 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
881 #ifdef RSEQ_COMPARE_TWICE
882 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
883 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
884 #endif
885 "movl %[src], %[rseq_scratch0]\n\t"
886 "movl %[dst], %[rseq_scratch1]\n\t"
887 "movl %[len], %[rseq_scratch2]\n\t"
888 /* Start rseq by storing table entry pointer into rseq_cs. */
889 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
890 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
891 RSEQ_INJECT_ASM(3)
892 "movl %[expect], %%eax\n\t"
893 "cmpl %%eax, %[v]\n\t"
894 "jnz 5f\n\t"
895 RSEQ_INJECT_ASM(4)
896 #ifdef RSEQ_COMPARE_TWICE
897 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
898 "movl %[expect], %%eax\n\t"
899 "cmpl %%eax, %[v]\n\t"
900 "jnz 7f\n\t"
901 #endif
902 /* try memcpy */
903 "test %[len], %[len]\n\t" \
904 "jz 333f\n\t" \
905 "222:\n\t" \
906 "movb (%[src]), %%al\n\t" \
907 "movb %%al, (%[dst])\n\t" \
908 "inc %[src]\n\t" \
909 "inc %[dst]\n\t" \
910 "dec %[len]\n\t" \
911 "jnz 222b\n\t" \
912 "333:\n\t" \
913 RSEQ_INJECT_ASM(5)
914 #ifdef RSEQ_TEMPLATE_MO_RELEASE
915 "lock; addl $0,-128(%%esp)\n\t"
916 #endif
917 "movl %[newv], %%eax\n\t"
918 /* final store */
919 "movl %%eax, %[v]\n\t"
920 "2:\n\t"
921 RSEQ_INJECT_ASM(6)
922 /* teardown */
923 "movl %[rseq_scratch2], %[len]\n\t"
924 "movl %[rseq_scratch1], %[dst]\n\t"
925 "movl %[rseq_scratch0], %[src]\n\t"
926 RSEQ_ASM_DEFINE_ABORT(4,
927 "movl %[rseq_scratch2], %[len]\n\t"
928 "movl %[rseq_scratch1], %[dst]\n\t"
929 "movl %[rseq_scratch0], %[src]\n\t",
930 abort)
931 RSEQ_ASM_DEFINE_CMPFAIL(5,
932 "movl %[rseq_scratch2], %[len]\n\t"
933 "movl %[rseq_scratch1], %[dst]\n\t"
934 "movl %[rseq_scratch0], %[src]\n\t",
935 cmpfail)
936 #ifdef RSEQ_COMPARE_TWICE
937 RSEQ_ASM_DEFINE_CMPFAIL(6,
938 "movl %[rseq_scratch2], %[len]\n\t"
939 "movl %[rseq_scratch1], %[dst]\n\t"
940 "movl %[rseq_scratch0], %[src]\n\t",
941 error1)
942 RSEQ_ASM_DEFINE_CMPFAIL(7,
943 "movl %[rseq_scratch2], %[len]\n\t"
944 "movl %[rseq_scratch1], %[dst]\n\t"
945 "movl %[rseq_scratch0], %[src]\n\t",
946 error2)
947 #endif
948 : /* gcc asm goto does not allow outputs */
949 : [cpu_id] "r" (cpu),
950 [rseq_offset] "r" (rseq_offset),
951 /* final store input */
952 [v] "m" (*v),
953 [expect] "m" (expect),
954 [newv] "m" (newv),
955 /* try memcpy input */
956 [dst] "r" (dst),
957 [src] "r" (src),
958 [len] "r" (len),
959 [rseq_scratch0] "m" (rseq_scratch[0]),
960 [rseq_scratch1] "m" (rseq_scratch[1]),
961 [rseq_scratch2] "m" (rseq_scratch[2])
962 : "memory", "cc", "eax"
963 RSEQ_INJECT_CLOBBER
964 : abort, cmpfail
965 #ifdef RSEQ_COMPARE_TWICE
966 , error1, error2
967 #endif
968 );
969 rseq_after_asm_goto();
970 return 0;
971 abort:
972 rseq_after_asm_goto();
973 RSEQ_INJECT_FAILED
974 return -1;
975 cmpfail:
976 rseq_after_asm_goto();
977 return 1;
978 #ifdef RSEQ_COMPARE_TWICE
979 error1:
980 rseq_after_asm_goto();
981 rseq_bug("cpu_id comparison failed");
982 error2:
983 rseq_after_asm_goto();
984 rseq_bug("expected value comparison failed");
985 #endif
986 }
987
988 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
989 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
990
991 #endif
992
993 #include "rseq-bits-reset.h"
994
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.