1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * arch/arm64/kernel/entry-ftrace.S
4 *
5 * Copyright (C) 2013 Linaro Limited
6 * Author: AKASHI Takahiro <takahiro.akashi@lin
7 */
8
9 #include <linux/linkage.h>
10 #include <linux/cfi_types.h>
11 #include <asm/asm-offsets.h>
12 #include <asm/assembler.h>
13 #include <asm/ftrace.h>
14 #include <asm/insn.h>
15
16 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
17 /*
18 * Due to -fpatchable-function-entry=2, the co
19 * the regular function prologue. For an enabl
20 * ftrace_make_call() have patched those NOPs
21 *
22 * MOV X9, LR
23 * BL ftrace_caller
24 *
25 * Each instrumented function follows the AAPC
26 * live (x18 holds the Shadow Call Stack point
27 * clobber.
28 *
29 * We save the callsite's context into a struc
30 * ftrace callbacks. So that we can get a sens
31 * records for the callsite and the ftrace ent
32 * sufficient for reliable stacktrace: until w
33 * record, its caller is missing from the LR a
34 * records.
35 */
36 SYM_CODE_START(ftrace_caller)
37 bti c
38
39 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
40 /*
41 * The literal pointer to the ops is a
42 * which is either 12 or 16 bytes befo
43 * site. See ftrace_call_adjust() for
44 *
45 * Therefore here the LR points at `li
46 * and we can find the address of the
47 * aligning to an 8-byte boundary and
48 * alignment first as this allows us t
49 * LDR.
50 */
51 bic x11, x30, 0x7
52 ldr x11, [x11, #-(4 * AARCH64_INSN
53
54 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
55 /*
56 * If the op has a direct call, handle
57 * saving/restoring registers.
58 */
59 ldr x17, [x11, #FTRACE_OPS_DIRECT_
60 cbnz x17, ftrace_caller_direct
61 #endif
62 #endif
63
64 /* Save original SP */
65 mov x10, sp
66
67 /* Make room for ftrace regs, plus two
68 sub sp, sp, #(FREGS_SIZE + 32)
69
70 /* Save function arguments */
71 stp x0, x1, [sp, #FREGS_X0]
72 stp x2, x3, [sp, #FREGS_X2]
73 stp x4, x5, [sp, #FREGS_X4]
74 stp x6, x7, [sp, #FREGS_X6]
75 str x8, [sp, #FREGS_X8]
76
77 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
78 str xzr, [sp, #FREGS_DIRECT_TRAMP]
79 #endif
80
81 /* Save the callsite's FP, LR, SP */
82 str x29, [sp, #FREGS_FP]
83 str x9, [sp, #FREGS_LR]
84 str x10, [sp, #FREGS_SP]
85
86 /* Save the PC after the ftrace callsi
87 str x30, [sp, #FREGS_PC]
88
89 /* Create a frame record for the calls
90 stp x29, x9, [sp, #FREGS_SIZE + 16
91 add x29, sp, #FREGS_SIZE + 16
92
93 /* Create our frame record above the f
94 stp x29, x30, [sp, #FREGS_SIZE]
95 add x29, sp, #FREGS_SIZE
96
97 /* Prepare arguments for the the trace
98 sub x0, x30, #AARCH64_INSN_SIZE
99 mov x1, x9
100 mov x3, sp
101
102 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
103 mov x2, x11
104 ldr x4, [x2, #FTRACE_OPS_FUNC]
105 blr x4
106
107 #else
108 ldr_l x2, function_trace_op
109
110 SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
111 bl ftrace_stub
112 #endif
113
114 /*
115 * At the callsite x0-x8 and x19-x30 were live
116 * x19-x29 per the AAPCS, and we created frame
117 * to restore x0-x8, x29, and x30.
118 */
119 /* Restore function arguments */
120 ldp x0, x1, [sp, #FREGS_X0]
121 ldp x2, x3, [sp, #FREGS_X2]
122 ldp x4, x5, [sp, #FREGS_X4]
123 ldp x6, x7, [sp, #FREGS_X6]
124 ldr x8, [sp, #FREGS_X8]
125
126 /* Restore the callsite's FP */
127 ldr x29, [sp, #FREGS_FP]
128
129 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
130 ldr x17, [sp, #FREGS_DIRECT_TRAMP]
131 cbnz x17, ftrace_caller_direct_late
132 #endif
133
134 /* Restore the callsite's LR and PC */
135 ldr x30, [sp, #FREGS_LR]
136 ldr x9, [sp, #FREGS_PC]
137
138 /* Restore the callsite's SP */
139 add sp, sp, #FREGS_SIZE + 32
140
141 ret x9
142
143 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
144 SYM_INNER_LABEL(ftrace_caller_direct_late, SYM
145 /*
146 * Head to a direct trampoline in x17
147 * The ftrace_regs are live, and x0-x8
148 * LR, PC, and SP have not been restor
149 */
150
151 /*
152 * Restore the callsite's LR and PC ma
153 * convention.
154 */
155 ldr x9, [sp, #FREGS_LR]
156 ldr x30, [sp, #FREGS_PC]
157
158 /* Restore the callsite's SP */
159 add sp, sp, #FREGS_SIZE + 32
160
161 SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LO
162 /*
163 * Head to a direct trampoline in x17.
164 *
165 * We use `BR X17` as this can safely
166 * the trampoline, and will not unbala
167 */
168 br x17
169 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CA
170 SYM_CODE_END(ftrace_caller)
171
172 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
173 SYM_CODE_START(ftrace_stub_direct_tramp)
174 bti c
175 mov x10, x30
176 mov x30, x9
177 ret x10
178 SYM_CODE_END(ftrace_stub_direct_tramp)
179 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CA
180
181 #else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
182
183 /*
184 * Gcc with -pg will put the following code in
185 * mov x0, x30
186 * bl _mcount
187 * [function's body ...]
188 * "bl _mcount" may be replaced to "bl ftrace_
189 * ftrace is enabled.
190 *
191 * Please note that x0 as an argument will not
192 * get lr(x30) of instrumented function at any
193 * as long as the kernel is compiled without -
194 * (or CONFIG_FRAME_POINTER, this is forced on
195 *
196 * stack layout after mcount_enter in _mcount(
197 *
198 * current sp/fp => 0:+-----+
199 * in _mcount() | x29 | -> instrumented
200 * +-----+
201 * | x30 | -> _mcount()'s
202 * old sp => +16:+-----+
203 * when instrumented | |
204 * function calls | ... |
205 * _mcount() | |
206 * | |
207 * instrumented => +xx:+-----+
208 * function's fp | x29 | -> parent's fp
209 * +-----+
210 * | x30 | -> instrumented
211 * +-----+
212 * | ... |
213 */
214
215 .macro mcount_enter
216 stp x29, x30, [sp, #-16]!
217 mov x29, sp
218 .endm
219
220 .macro mcount_exit
221 ldp x29, x30, [sp], #16
222 ret
223 .endm
224
225 .macro mcount_adjust_addr rd, rn
226 sub \rd, \rn, #AARCH64_INSN_SIZE
227 .endm
228
229 /* for instrumented function's parent
230 .macro mcount_get_parent_fp reg
231 ldr \reg, [x29]
232 ldr \reg, [\reg]
233 .endm
234
235 /* for instrumented function */
236 .macro mcount_get_pc0 reg
237 mcount_adjust_addr \reg, x30
238 .endm
239
240 .macro mcount_get_pc reg
241 ldr \reg, [x29, #8]
242 mcount_adjust_addr \reg, \reg
243 .endm
244
245 .macro mcount_get_lr reg
246 ldr \reg, [x29]
247 ldr \reg, [\reg, #8]
248 .endm
249
250 .macro mcount_get_lr_addr reg
251 ldr \reg, [x29]
252 add \reg, \reg, #8
253 .endm
254
255 /*
256 * _mcount() is used to build the kernel with
257 * instructions to _mcount() are replaced to N
258 * and later on, NOP to branch to ftrace_calle
259 * NOP when disabled per-function base.
260 */
261 SYM_FUNC_START(_mcount)
262 ret
263 SYM_FUNC_END(_mcount)
264 EXPORT_SYMBOL(_mcount)
265 NOKPROBE(_mcount)
266
267 /*
268 * void ftrace_caller(unsigned long return_add
269 * @return_address: return address to instrume
270 *
271 * This function is a counterpart of _mcount()
272 * makes calls to:
273 * - tracer function to probe instrumented
274 * - ftrace_graph_caller to set up an exit
275 */
276 SYM_FUNC_START(ftrace_caller)
277 mcount_enter
278
279 mcount_get_pc0 x0 //
280 mcount_get_lr x1 //
281
282 SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
283 nop // Thi
284 // whe
285
286 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
287 SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBA
288 nop // If
289 // "b
290 #endif
291
292 mcount_exit
293 SYM_FUNC_END(ftrace_caller)
294
295 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
296 /*
297 * void ftrace_graph_caller(void)
298 *
299 * Called from _mcount() or ftrace_caller() wh
300 * selected.
301 * This function w/ prepare_ftrace_return() fa
302 * the call stack in order to intercept instru
303 * and run return_to_handler() later on its ex
304 */
305 SYM_FUNC_START(ftrace_graph_caller)
306 mcount_get_pc x0 //
307 mcount_get_lr_addr x1 //
308 mcount_get_parent_fp x2 //
309 bl prepare_ftrace_return // pre
310
311 mcount_exit
312 SYM_FUNC_END(ftrace_graph_caller)
313 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
314 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
315
316 SYM_TYPED_FUNC_START(ftrace_stub)
317 ret
318 SYM_FUNC_END(ftrace_stub)
319
320 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
321 SYM_TYPED_FUNC_START(ftrace_stub_graph)
322 ret
323 SYM_FUNC_END(ftrace_stub_graph)
324
325 /*
326 * void return_to_handler(void)
327 *
328 * Run ftrace_return_to_handler() before going
329 * @fp is checked against the value passed by
330 */
331 SYM_CODE_START(return_to_handler)
332 /* save return value regs */
333 sub sp, sp, #FGRET_REGS_SIZE
334 stp x0, x1, [sp, #FGRET_REGS_X0]
335 stp x2, x3, [sp, #FGRET_REGS_X2]
336 stp x4, x5, [sp, #FGRET_REGS_X4]
337 stp x6, x7, [sp, #FGRET_REGS_X6]
338 str x29, [sp, #FGRET_REGS_FP]
339
340 mov x0, sp
341 bl ftrace_return_to_handler
342 mov x30, x0
343
344 /* restore return value regs */
345 ldp x0, x1, [sp, #FGRET_REGS_X0]
346 ldp x2, x3, [sp, #FGRET_REGS_X2]
347 ldp x4, x5, [sp, #FGRET_REGS_X4]
348 ldp x6, x7, [sp, #FGRET_REGS_X6]
349 add sp, sp, #FGRET_REGS_SIZE
350
351 ret
352 SYM_CODE_END(return_to_handler)
353 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.