1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #ifndef RSEQ_H
9 #error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
10 #endif
11
12 #include <stdint.h>
13
14 /*
15 * RSEQ_SIG is used with the following reserved undefined instructions, which
16 * trap in user-space:
17 *
18 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
19 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
20 */
21 #define RSEQ_SIG 0x53053053
22
23 /*
24 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
25 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
26 * address through a "r" input operand.
27 */
28
29 /* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */
30 #define RSEQ_CPU_ID_OFFSET 4
31 #define RSEQ_CS_OFFSET 8
32 #define RSEQ_MM_CID_OFFSET 24
33
34 #ifdef __x86_64__
35
36 #define RSEQ_ASM_TP_SEGMENT %%fs
37
38 #define rseq_smp_mb() \
39 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
40 #define rseq_smp_rmb() rseq_barrier()
41 #define rseq_smp_wmb() rseq_barrier()
42
43 #define rseq_smp_load_acquire(p) \
44 __extension__ ({ \
45 rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
46 rseq_barrier(); \
47 ____p1; \
48 })
49
50 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
51
52 #define rseq_smp_store_release(p, v) \
53 do { \
54 rseq_barrier(); \
55 RSEQ_WRITE_ONCE(*(p), v); \
56 } while (0)
57
58 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
59 start_ip, post_commit_offset, abort_ip) \
60 ".pushsection __rseq_cs, \"aw\"\n\t" \
61 ".balign 32\n\t" \
62 __rseq_str(label) ":\n\t" \
63 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
64 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
65 ".popsection\n\t" \
66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
67 ".quad " __rseq_str(label) "b\n\t" \
68 ".popsection\n\t"
69
70
71 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
72 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
73 (post_commit_ip - start_ip), abort_ip)
74
75 /*
76 * Exit points of a rseq critical section consist of all instructions outside
77 * of the critical section where a critical section can either branch to or
78 * reach through the normal course of its execution. The abort IP and the
79 * post-commit IP are already part of the __rseq_cs section and should not be
80 * explicitly defined as additional exit points. Knowing all exit points is
81 * useful to assist debuggers stepping over the critical section.
82 */
83 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
84 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
85 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
86 ".popsection\n\t"
87
88 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
89 RSEQ_INJECT_ASM(1) \
90 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
91 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
92 __rseq_str(label) ":\n\t"
93
94 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
95 RSEQ_INJECT_ASM(2) \
96 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
97 "jnz " __rseq_str(label) "\n\t"
98
99 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
100 ".pushsection __rseq_failure, \"ax\"\n\t" \
101 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
102 ".byte 0x0f, 0xb9, 0x3d\n\t" \
103 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
104 __rseq_str(label) ":\n\t" \
105 teardown \
106 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
107 ".popsection\n\t"
108
109 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
110 ".pushsection __rseq_failure, \"ax\"\n\t" \
111 __rseq_str(label) ":\n\t" \
112 teardown \
113 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
114 ".popsection\n\t"
115
116 #elif defined(__i386__)
117
118 #define RSEQ_ASM_TP_SEGMENT %%gs
119
120 #define rseq_smp_mb() \
121 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
122 #define rseq_smp_rmb() \
123 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
124 #define rseq_smp_wmb() \
125 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
126
127 #define rseq_smp_load_acquire(p) \
128 __extension__ ({ \
129 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
130 rseq_smp_mb(); \
131 ____p1; \
132 })
133
134 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
135
136 #define rseq_smp_store_release(p, v) \
137 do { \
138 rseq_smp_mb(); \
139 RSEQ_WRITE_ONCE(*p, v); \
140 } while (0)
141
142 /*
143 * Use eax as scratch register and take memory operands as input to
144 * lessen register pressure. Especially needed when compiling in O0.
145 */
146 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
147 start_ip, post_commit_offset, abort_ip) \
148 ".pushsection __rseq_cs, \"aw\"\n\t" \
149 ".balign 32\n\t" \
150 __rseq_str(label) ":\n\t" \
151 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
152 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
153 ".popsection\n\t" \
154 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
155 ".long " __rseq_str(label) "b, 0x0\n\t" \
156 ".popsection\n\t"
157
158 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
159 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
160 (post_commit_ip - start_ip), abort_ip)
161
162 /*
163 * Exit points of a rseq critical section consist of all instructions outside
164 * of the critical section where a critical section can either branch to or
165 * reach through the normal course of its execution. The abort IP and the
166 * post-commit IP are already part of the __rseq_cs section and should not be
167 * explicitly defined as additional exit points. Knowing all exit points is
168 * useful to assist debuggers stepping over the critical section.
169 */
170 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
171 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
172 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
173 ".popsection\n\t"
174
175 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
176 RSEQ_INJECT_ASM(1) \
177 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
178 __rseq_str(label) ":\n\t"
179
180 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
181 RSEQ_INJECT_ASM(2) \
182 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
183 "jnz " __rseq_str(label) "\n\t"
184
185 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
186 ".pushsection __rseq_failure, \"ax\"\n\t" \
187 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
188 ".byte 0x0f, 0xb9, 0x3d\n\t" \
189 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
190 __rseq_str(label) ":\n\t" \
191 teardown \
192 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
193 ".popsection\n\t"
194
195 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
196 ".pushsection __rseq_failure, \"ax\"\n\t" \
197 __rseq_str(label) ":\n\t" \
198 teardown \
199 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
200 ".popsection\n\t"
201
202 #endif
203
204 /* Per-cpu-id indexing. */
205
206 #define RSEQ_TEMPLATE_CPU_ID
207 #define RSEQ_TEMPLATE_MO_RELAXED
208 #include "rseq-x86-bits.h"
209 #undef RSEQ_TEMPLATE_MO_RELAXED
210
211 #define RSEQ_TEMPLATE_MO_RELEASE
212 #include "rseq-x86-bits.h"
213 #undef RSEQ_TEMPLATE_MO_RELEASE
214 #undef RSEQ_TEMPLATE_CPU_ID
215
216 /* Per-mm-cid indexing. */
217
218 #define RSEQ_TEMPLATE_MM_CID
219 #define RSEQ_TEMPLATE_MO_RELAXED
220 #include "rseq-x86-bits.h"
221 #undef RSEQ_TEMPLATE_MO_RELAXED
222
223 #define RSEQ_TEMPLATE_MO_RELEASE
224 #include "rseq-x86-bits.h"
225 #undef RSEQ_TEMPLATE_MO_RELEASE
226 #undef RSEQ_TEMPLATE_MM_CID
227
228 /* APIs which are not based on cpu ids. */
229
230 #define RSEQ_TEMPLATE_CPU_ID_NONE
231 #define RSEQ_TEMPLATE_MO_RELAXED
232 #include "rseq-x86-bits.h"
233 #undef RSEQ_TEMPLATE_MO_RELAXED
234 #undef RSEQ_TEMPLATE_CPU_ID_NONE
235
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.