~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/kernel/entry-ftrace.S

Version: ~ [ linux-6.12-rc7 ] ~ [ linux-6.11.7 ] ~ [ linux-6.10.14 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.60 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.116 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.171 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.229 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.285 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.323 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.12 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

Differences between /arch/arm64/kernel/entry-ftrace.S (Version linux-6.12-rc7) and /arch/i386/kernel/entry-ftrace.S (Version linux-2.6.32.71)


  1 /* SPDX-License-Identifier: GPL-2.0-only */       
  2 /*                                                
  3  * arch/arm64/kernel/entry-ftrace.S               
  4  *                                                
  5  * Copyright (C) 2013 Linaro Limited              
  6  * Author: AKASHI Takahiro <takahiro.akashi@lin    
  7  */                                               
  8                                                   
  9 #include <linux/linkage.h>                        
 10 #include <linux/cfi_types.h>                      
 11 #include <asm/asm-offsets.h>                      
 12 #include <asm/assembler.h>                        
 13 #include <asm/ftrace.h>                           
 14 #include <asm/insn.h>                             
 15                                                   
 16 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS            
 17 /*                                                
 18  * Due to -fpatchable-function-entry=2, the co    
 19  * the regular function prologue. For an enabl    
 20  * ftrace_make_call() have patched those NOPs     
 21  *                                                
 22  *      MOV     X9, LR                            
 23  *      BL      ftrace_caller                     
 24  *                                                
 25  * Each instrumented function follows the AAPC    
 26  * live (x18 holds the Shadow Call Stack point    
 27  * clobber.                                       
 28  *                                                
 29  * We save the callsite's context into a struc    
 30  * ftrace callbacks. So that we can get a sens    
 31  * records for the callsite and the ftrace ent    
 32  * sufficient for reliable stacktrace: until w    
 33  * record, its caller is missing from the LR a    
 34  * records.                                       
 35  */                                               
 36 SYM_CODE_START(ftrace_caller)                     
 37         bti     c                                 
 38                                                   
 39 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS        
 40         /*                                        
 41          * The literal pointer to the ops is a    
 42          * which is either 12 or 16 bytes befo    
 43          * site. See ftrace_call_adjust() for     
 44          *                                        
 45          * Therefore here the LR points at `li    
 46          * and we can find the address of the     
 47          * aligning to an 8-byte boundary and     
 48          * alignment first as this allows us t    
 49          * LDR.                                   
 50          */                                       
 51         bic     x11, x30, 0x7                     
 52         ldr     x11, [x11, #-(4 * AARCH64_INSN    
 53                                                   
 54 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS    
 55         /*                                        
 56          * If the op has a direct call, handle    
 57          * saving/restoring registers.            
 58          */                                       
 59         ldr     x17, [x11, #FTRACE_OPS_DIRECT_    
 60         cbnz    x17, ftrace_caller_direct         
 61 #endif                                            
 62 #endif                                            
 63                                                   
 64         /* Save original SP */                    
 65         mov     x10, sp                           
 66                                                   
 67         /* Make room for ftrace regs, plus two    
 68         sub     sp, sp, #(FREGS_SIZE + 32)        
 69                                                   
 70         /* Save function arguments */             
 71         stp     x0, x1, [sp, #FREGS_X0]           
 72         stp     x2, x3, [sp, #FREGS_X2]           
 73         stp     x4, x5, [sp, #FREGS_X4]           
 74         stp     x6, x7, [sp, #FREGS_X6]           
 75         str     x8,     [sp, #FREGS_X8]           
 76                                                   
 77 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS    
 78         str     xzr, [sp, #FREGS_DIRECT_TRAMP]    
 79 #endif                                            
 80                                                   
 81         /* Save the callsite's FP, LR, SP */      
 82         str     x29, [sp, #FREGS_FP]              
 83         str     x9,  [sp, #FREGS_LR]              
 84         str     x10, [sp, #FREGS_SP]              
 85                                                   
 86         /* Save the PC after the ftrace callsi    
 87         str     x30, [sp, #FREGS_PC]              
 88                                                   
 89         /* Create a frame record for the calls    
 90         stp     x29, x9, [sp, #FREGS_SIZE + 16    
 91         add     x29, sp, #FREGS_SIZE + 16         
 92                                                   
 93         /* Create our frame record above the f    
 94         stp     x29, x30, [sp, #FREGS_SIZE]       
 95         add     x29, sp, #FREGS_SIZE              
 96                                                   
 97         /* Prepare arguments for the the trace    
 98         sub     x0, x30, #AARCH64_INSN_SIZE       
 99         mov     x1, x9                            
100         mov     x3, sp                            
101                                                   
102 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS        
103         mov     x2, x11                           
104         ldr     x4, [x2, #FTRACE_OPS_FUNC]        
105         blr     x4                                
106                                                   
107 #else                                             
108         ldr_l   x2, function_trace_op             
109                                                   
110 SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)        
111         bl      ftrace_stub                       
112 #endif                                            
113                                                   
114 /*                                                
115  * At the callsite x0-x8 and x19-x30 were live    
116  * x19-x29 per the AAPCS, and we created frame    
117  * to restore x0-x8, x29, and x30.                
118  */                                               
119         /* Restore function arguments */          
120         ldp     x0, x1, [sp, #FREGS_X0]           
121         ldp     x2, x3, [sp, #FREGS_X2]           
122         ldp     x4, x5, [sp, #FREGS_X4]           
123         ldp     x6, x7, [sp, #FREGS_X6]           
124         ldr     x8,     [sp, #FREGS_X8]           
125                                                   
126         /* Restore the callsite's FP */           
127         ldr     x29, [sp, #FREGS_FP]              
128                                                   
129 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS    
130         ldr     x17, [sp, #FREGS_DIRECT_TRAMP]    
131         cbnz    x17, ftrace_caller_direct_late    
132 #endif                                            
133                                                   
134         /* Restore the callsite's LR and PC */    
135         ldr     x30, [sp, #FREGS_LR]              
136         ldr     x9,  [sp, #FREGS_PC]              
137                                                   
138         /* Restore the callsite's SP */           
139         add     sp, sp, #FREGS_SIZE + 32          
140                                                   
141         ret     x9                                
142                                                   
143 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS    
144 SYM_INNER_LABEL(ftrace_caller_direct_late, SYM    
145         /*                                        
146          * Head to a direct trampoline in x17     
147          * The ftrace_regs are live, and x0-x8    
148          * LR, PC, and SP have not been restor    
149          */                                       
150                                                   
151         /*                                        
152          * Restore the callsite's LR and PC ma    
153          * convention.                            
154          */                                       
155         ldr     x9,  [sp, #FREGS_LR]              
156         ldr     x30, [sp, #FREGS_PC]              
157                                                   
158         /* Restore the callsite's SP */           
159         add     sp, sp, #FREGS_SIZE + 32          
160                                                   
161 SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LO    
162         /*                                        
163          * Head to a direct trampoline in x17.    
164          *                                        
165          * We use `BR X17` as this can safely     
166          * the trampoline, and will not unbala    
167          */                                       
168         br      x17                               
169 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CA    
170 SYM_CODE_END(ftrace_caller)                       
171                                                   
172 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS    
173 SYM_CODE_START(ftrace_stub_direct_tramp)          
174         bti     c                                 
175         mov     x10, x30                          
176         mov     x30, x9                           
177         ret     x10                               
178 SYM_CODE_END(ftrace_stub_direct_tramp)            
179 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CA    
180                                                   
181 #else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */       
182                                                   
183 /*                                                
184  * Gcc with -pg will put the following code in    
185  *      mov x0, x30                               
186  *      bl _mcount                                
187  *      [function's body ...]                     
188  * "bl _mcount" may be replaced to "bl ftrace_    
189  * ftrace is enabled.                             
190  *                                                
191  * Please note that x0 as an argument will not    
192  * get lr(x30) of instrumented function at any    
193  * as long as the kernel is compiled without -    
194  * (or CONFIG_FRAME_POINTER, this is forced on    
195  *                                                
196  * stack layout after mcount_enter in _mcount(    
197  *                                                
198  * current sp/fp =>  0:+-----+                    
199  * in _mcount()        | x29 | -> instrumented    
200  *                     +-----+                    
201  *                     | x30 | -> _mcount()'s     
202  * old sp       => +16:+-----+                    
203  * when instrumented   |     |                    
204  * function calls      | ... |                    
205  * _mcount()           |     |                    
206  *                     |     |                    
207  * instrumented => +xx:+-----+                    
208  * function's fp       | x29 | -> parent's fp     
209  *                     +-----+                    
210  *                     | x30 | -> instrumented    
211  *                     +-----+                    
212  *                     | ... |                    
213  */                                               
214                                                   
215         .macro mcount_enter                       
216         stp     x29, x30, [sp, #-16]!             
217         mov     x29, sp                           
218         .endm                                     
219                                                   
220         .macro mcount_exit                        
221         ldp     x29, x30, [sp], #16               
222         ret                                       
223         .endm                                     
224                                                   
225         .macro mcount_adjust_addr rd, rn          
226         sub     \rd, \rn, #AARCH64_INSN_SIZE      
227         .endm                                     
228                                                   
229         /* for instrumented function's parent     
230         .macro mcount_get_parent_fp reg           
231         ldr     \reg, [x29]                       
232         ldr     \reg, [\reg]                      
233         .endm                                     
234                                                   
235         /* for instrumented function */           
236         .macro mcount_get_pc0 reg                 
237         mcount_adjust_addr      \reg, x30         
238         .endm                                     
239                                                   
240         .macro mcount_get_pc reg                  
241         ldr     \reg, [x29, #8]                   
242         mcount_adjust_addr      \reg, \reg        
243         .endm                                     
244                                                   
245         .macro mcount_get_lr reg                  
246         ldr     \reg, [x29]                       
247         ldr     \reg, [\reg, #8]                  
248         .endm                                     
249                                                   
250         .macro mcount_get_lr_addr reg             
251         ldr     \reg, [x29]                       
252         add     \reg, \reg, #8                    
253         .endm                                     
254                                                   
255 /*                                                
256  * _mcount() is used to build the kernel with     
257  * instructions to _mcount() are replaced to N    
258  * and later on, NOP to branch to ftrace_calle    
259  * NOP when disabled per-function base.           
260  */                                               
261 SYM_FUNC_START(_mcount)                           
262         ret                                       
263 SYM_FUNC_END(_mcount)                             
264 EXPORT_SYMBOL(_mcount)                            
265 NOKPROBE(_mcount)                                 
266                                                   
267 /*                                                
268  * void ftrace_caller(unsigned long return_add    
269  * @return_address: return address to instrume    
270  *                                                
271  * This function is a counterpart of _mcount()    
272  * makes calls to:                                
273  *     - tracer function to probe instrumented    
274  *     - ftrace_graph_caller to set up an exit    
275  */                                               
276 SYM_FUNC_START(ftrace_caller)                     
277         mcount_enter                              
278                                                   
279         mcount_get_pc0  x0              //        
280         mcount_get_lr   x1              //        
281                                                   
282 SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)        
283         nop                             // Thi    
284                                         // whe    
285                                                   
286 #ifdef CONFIG_FUNCTION_GRAPH_TRACER               
287 SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBA    
288         nop                             // If     
289                                         // "b     
290 #endif                                            
291                                                   
292         mcount_exit                               
293 SYM_FUNC_END(ftrace_caller)                       
294                                                   
295 #ifdef CONFIG_FUNCTION_GRAPH_TRACER               
296 /*                                                
297  * void ftrace_graph_caller(void)                 
298  *                                                
299  * Called from _mcount() or ftrace_caller() wh    
300  * selected.                                      
301  * This function w/ prepare_ftrace_return() fa    
302  * the call stack in order to intercept instru    
303  * and run return_to_handler() later on its ex    
304  */                                               
305 SYM_FUNC_START(ftrace_graph_caller)               
306         mcount_get_pc             x0    //        
307         mcount_get_lr_addr        x1    //        
308         mcount_get_parent_fp      x2    //        
309         bl      prepare_ftrace_return   // pre    
310                                                   
311         mcount_exit                               
312 SYM_FUNC_END(ftrace_graph_caller)                 
313 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */         
314 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */      
315                                                   
316 SYM_TYPED_FUNC_START(ftrace_stub)                 
317         ret                                       
318 SYM_FUNC_END(ftrace_stub)                         
319                                                   
320 #ifdef CONFIG_FUNCTION_GRAPH_TRACER               
321 SYM_TYPED_FUNC_START(ftrace_stub_graph)           
322         ret                                       
323 SYM_FUNC_END(ftrace_stub_graph)                   
324                                                   
325 /*                                                
326  * void return_to_handler(void)                   
327  *                                                
328  * Run ftrace_return_to_handler() before going    
329  * @fp is checked against the value passed by     
330  */                                               
331 SYM_CODE_START(return_to_handler)                 
332         /* save return value regs */              
333         sub sp, sp, #FGRET_REGS_SIZE              
334         stp x0, x1, [sp, #FGRET_REGS_X0]          
335         stp x2, x3, [sp, #FGRET_REGS_X2]          
336         stp x4, x5, [sp, #FGRET_REGS_X4]          
337         stp x6, x7, [sp, #FGRET_REGS_X6]          
338         str x29,    [sp, #FGRET_REGS_FP]          
339                                                   
340         mov     x0, sp                            
341         bl      ftrace_return_to_handler          
342         mov     x30, x0                           
343                                                   
344         /* restore return value regs */           
345         ldp x0, x1, [sp, #FGRET_REGS_X0]          
346         ldp x2, x3, [sp, #FGRET_REGS_X2]          
347         ldp x4, x5, [sp, #FGRET_REGS_X4]          
348         ldp x6, x7, [sp, #FGRET_REGS_X6]          
349         add sp, sp, #FGRET_REGS_SIZE              
350                                                   
351         ret                                       
352 SYM_CODE_END(return_to_handler)                   
353 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */         
                                                      

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

sflogo.php