1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/spinlock.h> !! 2 /*
3 #include <linux/hardirq.h> !! 3 * Code for replacing ftrace calls with jumps.
4 #include <linux/ftrace.h> !! 4 *
5 #include <linux/percpu.h> !! 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
>> 6 * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China
>> 7 * Author: Wu Zhangjin <wuzhangjin@gmail.com>
>> 8 *
>> 9 * Thanks goes to Steven Rostedt for writing the original x86 version.
>> 10 */
>> 11
>> 12 #include <linux/uaccess.h>
6 #include <linux/init.h> 13 #include <linux/init.h>
7 #include <linux/list.h> !! 14 #include <linux/ftrace.h>
8 #include <trace/syscall.h> !! 15 #include <linux/syscalls.h>
9 16
10 #include <asm/ftrace.h> !! 17 #include <asm/asm.h>
>> 18 #include <asm/asm-offsets.h>
>> 19 #include <asm/cacheflush.h>
>> 20 #include <asm/syscall.h>
>> 21 #include <asm/uasm.h>
>> 22 #include <asm/unistd.h>
>> 23
>> 24 #include <asm-generic/sections.h>
>> 25
>> 26 #if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT)
>> 27 #define MCOUNT_OFFSET_INSNS 5
>> 28 #else
>> 29 #define MCOUNT_OFFSET_INSNS 4
>> 30 #endif
11 31
12 #ifdef CONFIG_DYNAMIC_FTRACE 32 #ifdef CONFIG_DYNAMIC_FTRACE
13 static const u32 ftrace_nop = 0x01000000; <<
14 33
15 static u32 ftrace_call_replace(unsigned long i !! 34 /* Arch override because MIPS doesn't need to run this from stop_machine() */
>> 35 void arch_ftrace_update_code(int command)
16 { 36 {
17 u32 call; !! 37 ftrace_modify_all_code(command);
18 s32 off; !! 38 }
19 39
20 off = ((s32)addr - (s32)ip); !! 40 #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */
21 call = 0x40000000 | ((u32)off >> 2); !! 41 #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */
>> 42 #define JUMP_RANGE_MASK ((1UL << 28) - 1)
>> 43
>> 44 #define INSN_NOP 0x00000000 /* nop */
>> 45 #define INSN_JAL(addr) \
>> 46 ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK)))
>> 47
>> 48 static unsigned int insn_jal_ftrace_caller __read_mostly;
>> 49 static unsigned int insn_la_mcount[2] __read_mostly;
>> 50 static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly;
>> 51
>> 52 static inline void ftrace_dyn_arch_init_insns(void)
>> 53 {
>> 54 u32 *buf;
>> 55 unsigned int v1;
22 56
23 return call; !! 57 /* la v1, _mcount */
>> 58 v1 = 3;
>> 59 buf = (u32 *)&insn_la_mcount[0];
>> 60 UASM_i_LA(&buf, v1, MCOUNT_ADDR);
>> 61
>> 62 /* jal (ftrace_caller + 8), jump over the first two instruction */
>> 63 buf = (u32 *)&insn_jal_ftrace_caller;
>> 64 uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK);
>> 65
>> 66 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
>> 67 /* j ftrace_graph_caller */
>> 68 buf = (u32 *)&insn_j_ftrace_graph_caller;
>> 69 uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK);
>> 70 #endif
24 } 71 }
25 72
26 static int ftrace_modify_code(unsigned long ip !! 73 static int ftrace_modify_code(unsigned long ip, unsigned int new_code)
27 { 74 {
28 u32 replaced; <<
29 int faulted; 75 int faulted;
30 76
31 __asm__ __volatile__( !! 77 /* *(unsigned int *)ip = new_code; */
32 "1: cas [%[ip]], %[old], %[new !! 78 safe_store_code(new_code, ip, faulted);
33 " flush %[ip]\n" !! 79
34 " mov 0, %[faulted]\n" !! 80 if (unlikely(faulted))
35 "2:\n" !! 81 return -EFAULT;
36 " .section .fixup,#alloc,#execin <<
37 " .align 4\n" <<
38 "3: sethi %%hi(2b), %[faulted]\n <<
39 " jmpl %[faulted] + %%lo(2b), <<
40 " mov 1, %[faulted]\n" <<
41 " .previous\n" <<
42 " .section __ex_table,\"a\"\n" <<
43 " .align 4\n" <<
44 " .word 1b, 3b\n" <<
45 " .previous\n" <<
46 : "=r" (replaced), [faulted] "=r" (fau <<
47 : [new] "" (new), [old] "r" (old), [ip <<
48 : "memory"); <<
49 82
50 if (replaced != old && replaced != new !! 83 flush_icache_range(ip, ip + 8);
51 faulted = 2; <<
52 84
53 return faulted; !! 85 return 0;
54 } 86 }
55 87
56 int ftrace_make_nop(struct module *mod, struct !! 88 #ifndef CONFIG_64BIT
>> 89 static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1,
>> 90 unsigned int new_code2)
57 { 91 {
>> 92 int faulted;
>> 93
>> 94 safe_store_code(new_code1, ip, faulted);
>> 95 if (unlikely(faulted))
>> 96 return -EFAULT;
>> 97
>> 98 ip += 4;
>> 99 safe_store_code(new_code2, ip, faulted);
>> 100 if (unlikely(faulted))
>> 101 return -EFAULT;
>> 102
>> 103 ip -= 4;
>> 104 flush_icache_range(ip, ip + 8);
>> 105
>> 106 return 0;
>> 107 }
>> 108
>> 109 static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1,
>> 110 unsigned int new_code2)
>> 111 {
>> 112 int faulted;
>> 113
>> 114 ip += 4;
>> 115 safe_store_code(new_code2, ip, faulted);
>> 116 if (unlikely(faulted))
>> 117 return -EFAULT;
>> 118
>> 119 ip -= 4;
>> 120 safe_store_code(new_code1, ip, faulted);
>> 121 if (unlikely(faulted))
>> 122 return -EFAULT;
>> 123
>> 124 flush_icache_range(ip, ip + 8);
>> 125
>> 126 return 0;
>> 127 }
>> 128 #endif
>> 129
>> 130 /*
>> 131 * The details about the calling site of mcount on MIPS
>> 132 *
>> 133 * 1. For kernel:
>> 134 *
>> 135 * move at, ra
>> 136 * jal _mcount --> nop
>> 137 * sub sp, sp, 8 --> nop (CONFIG_32BIT)
>> 138 *
>> 139 * 2. For modules:
>> 140 *
>> 141 * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT
>> 142 *
>> 143 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005)
>> 144 * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT)
>> 145 * move at, ra
>> 146 * move $12, ra_address
>> 147 * jalr v1
>> 148 * sub sp, sp, 8
>> 149 * 1: offset = 5 instructions
>> 150 * 2.2 For the Other situations
>> 151 *
>> 152 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004)
>> 153 * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT)
>> 154 * move at, ra
>> 155 * jalr v1
>> 156 * nop | move $12, ra_address | sub sp, sp, 8
>> 157 * 1: offset = 4 instructions
>> 158 */
>> 159
>> 160 #define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS)
>> 161
>> 162 int ftrace_make_nop(struct module *mod,
>> 163 struct dyn_ftrace *rec, unsigned long addr)
>> 164 {
>> 165 unsigned int new;
58 unsigned long ip = rec->ip; 166 unsigned long ip = rec->ip;
59 u32 old, new; <<
60 167
61 old = ftrace_call_replace(ip, addr); !! 168 /*
62 new = ftrace_nop; !! 169 * If ip is in kernel space, no long call, otherwise, long call is
63 return ftrace_modify_code(ip, old, new !! 170 * needed.
>> 171 */
>> 172 new = core_kernel_text(ip) ? INSN_NOP : INSN_B_1F;
>> 173 #ifdef CONFIG_64BIT
>> 174 return ftrace_modify_code(ip, new);
>> 175 #else
>> 176 /*
>> 177 * On 32 bit MIPS platforms, gcc adds a stack adjust
>> 178 * instruction in the delay slot after the branch to
>> 179 * mcount and expects mcount to restore the sp on return.
>> 180 * This is based on a legacy API and does nothing but
>> 181 * waste instructions so it's being removed at runtime.
>> 182 */
>> 183 return ftrace_modify_code_2(ip, new, INSN_NOP);
>> 184 #endif
64 } 185 }
65 186
66 int ftrace_make_call(struct dyn_ftrace *rec, u 187 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
67 { 188 {
>> 189 unsigned int new;
68 unsigned long ip = rec->ip; 190 unsigned long ip = rec->ip;
69 u32 old, new; <<
70 191
71 old = ftrace_nop; !! 192 new = core_kernel_text(ip) ? insn_jal_ftrace_caller : insn_la_mcount[0];
72 new = ftrace_call_replace(ip, addr); !! 193
73 return ftrace_modify_code(ip, old, new !! 194 #ifdef CONFIG_64BIT
>> 195 return ftrace_modify_code(ip, new);
>> 196 #else
>> 197 return ftrace_modify_code_2r(ip, new, core_kernel_text(ip) ?
>> 198 INSN_NOP : insn_la_mcount[1]);
>> 199 #endif
74 } 200 }
75 201
>> 202 #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call))
>> 203
76 int ftrace_update_ftrace_func(ftrace_func_t fu 204 int ftrace_update_ftrace_func(ftrace_func_t func)
77 { 205 {
78 unsigned long ip = (unsigned long)(&ft !! 206 unsigned int new;
79 u32 old, new; <<
80 207
81 old = *(u32 *) &ftrace_call; !! 208 new = INSN_JAL((unsigned long)func);
82 new = ftrace_call_replace(ip, (unsigne !! 209
83 return ftrace_modify_code(ip, old, new !! 210 return ftrace_modify_code(FTRACE_CALL_IP, new);
84 } 211 }
85 #endif !! 212
>> 213 int __init ftrace_dyn_arch_init(void)
>> 214 {
>> 215 /* Encode the instructions when booting */
>> 216 ftrace_dyn_arch_init_insns();
>> 217
>> 218 /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */
>> 219 ftrace_modify_code(MCOUNT_ADDR, INSN_NOP);
>> 220
>> 221 return 0;
>> 222 }
>> 223 #endif /* CONFIG_DYNAMIC_FTRACE */
86 224
87 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 225 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
88 226
89 #ifdef CONFIG_DYNAMIC_FTRACE 227 #ifdef CONFIG_DYNAMIC_FTRACE
>> 228
90 extern void ftrace_graph_call(void); 229 extern void ftrace_graph_call(void);
>> 230 #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call))
91 231
92 int ftrace_enable_ftrace_graph_caller(void) 232 int ftrace_enable_ftrace_graph_caller(void)
93 { 233 {
94 unsigned long ip = (unsigned long)(&ft !! 234 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP,
95 u32 old, new; !! 235 insn_j_ftrace_graph_caller);
96 <<
97 old = *(u32 *) &ftrace_graph_call; <<
98 new = ftrace_call_replace(ip, (unsigne <<
99 return ftrace_modify_code(ip, old, new <<
100 } 236 }
101 237
102 int ftrace_disable_ftrace_graph_caller(void) 238 int ftrace_disable_ftrace_graph_caller(void)
103 { 239 {
104 unsigned long ip = (unsigned long)(&ft !! 240 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP);
105 u32 old, new; !! 241 }
106 242
107 old = *(u32 *) &ftrace_graph_call; !! 243 #endif /* CONFIG_DYNAMIC_FTRACE */
108 new = ftrace_call_replace(ip, (unsigne <<
109 244
110 return ftrace_modify_code(ip, old, new !! 245 #ifndef KBUILD_MCOUNT_RA_ADDRESS
>> 246
>> 247 #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */
>> 248 #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */
>> 249 #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */
>> 250
>> 251 unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long
>> 252 old_parent_ra, unsigned long parent_ra_addr, unsigned long fp)
>> 253 {
>> 254 unsigned long sp, ip, tmp;
>> 255 unsigned int code;
>> 256 int faulted;
>> 257
>> 258 /*
>> 259 * For module, move the ip from the return address after the
>> 260 * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for
>> 261 * kernel, move after the instruction "move ra, at"(offset is 16)
>> 262 */
>> 263 ip = self_ra - (core_kernel_text(self_ra) ? 16 : 24);
>> 264
>> 265 /*
>> 266 * search the text until finding the non-store instruction or "s{d,w}
>> 267 * ra, offset(sp)" instruction
>> 268 */
>> 269 do {
>> 270 /* get the code at "ip": code = *(unsigned int *)ip; */
>> 271 safe_load_code(code, ip, faulted);
>> 272
>> 273 if (unlikely(faulted))
>> 274 return 0;
>> 275 /*
>> 276 * If we hit the non-store instruction before finding where the
>> 277 * ra is stored, then this is a leaf function and it does not
>> 278 * store the ra on the stack
>> 279 */
>> 280 if ((code & S_R_SP) != S_R_SP)
>> 281 return parent_ra_addr;
>> 282
>> 283 /* Move to the next instruction */
>> 284 ip -= 4;
>> 285 } while ((code & S_RA_SP) != S_RA_SP);
>> 286
>> 287 sp = fp + (code & OFFSET_MASK);
>> 288
>> 289 /* tmp = *(unsigned long *)sp; */
>> 290 safe_load_stack(tmp, sp, faulted);
>> 291 if (unlikely(faulted))
>> 292 return 0;
>> 293
>> 294 if (tmp == old_parent_ra)
>> 295 return sp;
>> 296 return 0;
111 } 297 }
112 298
113 #endif /* !CONFIG_DYNAMIC_FTRACE */ !! 299 #endif /* !KBUILD_MCOUNT_RA_ADDRESS */
114 300
115 /* 301 /*
116 * Hook the return address and push it in the 302 * Hook the return address and push it in the stack of return addrs
117 * in current thread info. 303 * in current thread info.
118 */ 304 */
119 unsigned long prepare_ftrace_return(unsigned l !! 305 void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra,
120 unsigned l !! 306 unsigned long fp)
121 unsigned l <<
122 { 307 {
123 unsigned long return_hooker = (unsigne !! 308 unsigned long old_parent_ra;
>> 309 unsigned long return_hooker = (unsigned long)
>> 310 &return_to_handler;
>> 311 int faulted, insns;
>> 312
>> 313 if (unlikely(ftrace_graph_is_dead()))
>> 314 return;
124 315
125 if (unlikely(atomic_read(¤t->tra 316 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
126 return parent + 8UL; !! 317 return;
>> 318
>> 319 /*
>> 320 * "parent_ra_addr" is the stack address where the return address of
>> 321 * the caller of _mcount is saved.
>> 322 *
>> 323 * If gcc < 4.5, a leaf function does not save the return address
>> 324 * in the stack address, so we "emulate" one in _mcount's stack space,
>> 325 * and hijack it directly.
>> 326 * For a non-leaf function, it does save the return address to its own
>> 327 * stack space, so we can not hijack it directly, but need to find the
>> 328 * real stack address, which is done by ftrace_get_parent_addr().
>> 329 *
>> 330 * If gcc >= 4.5, with the new -mmcount-ra-address option, for a
>> 331 * non-leaf function, the location of the return address will be saved
>> 332 * to $12 for us.
>> 333 * For a leaf function, it just puts a zero into $12, so we handle
>> 334 * it in ftrace_graph_caller() of mcount.S.
>> 335 */
>> 336
>> 337 /* old_parent_ra = *parent_ra_addr; */
>> 338 safe_load_stack(old_parent_ra, parent_ra_addr, faulted);
>> 339 if (unlikely(faulted))
>> 340 goto out;
>> 341 #ifndef KBUILD_MCOUNT_RA_ADDRESS
>> 342 parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra,
>> 343 old_parent_ra, (unsigned long)parent_ra_addr, fp);
>> 344 /*
>> 345 * If fails when getting the stack address of the non-leaf function's
>> 346 * ra, stop function graph tracer and return
>> 347 */
>> 348 if (parent_ra_addr == NULL)
>> 349 goto out;
>> 350 #endif
>> 351 /* *parent_ra_addr = return_hooker; */
>> 352 safe_store_stack(return_hooker, parent_ra_addr, faulted);
>> 353 if (unlikely(faulted))
>> 354 goto out;
>> 355
>> 356 /*
>> 357 * Get the recorded ip of the current mcount calling site in the
>> 358 * __mcount_loc section, which will be used to filter the function
>> 359 * entries configured through the tracing/set_graph_function interface.
>> 360 */
>> 361
>> 362 insns = core_kernel_text(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1;
>> 363 self_ra -= (MCOUNT_INSN_SIZE * insns);
>> 364
>> 365 if (function_graph_enter(old_parent_ra, self_ra, fp, NULL))
>> 366 *parent_ra_addr = old_parent_ra;
>> 367 return;
>> 368 out:
>> 369 ftrace_graph_stop();
>> 370 WARN_ON(1);
>> 371 }
>> 372 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
>> 373
>> 374 #ifdef CONFIG_FTRACE_SYSCALLS
>> 375
>> 376 #ifdef CONFIG_32BIT
>> 377 unsigned long __init arch_syscall_addr(int nr)
>> 378 {
>> 379 return (unsigned long)sys_call_table[nr - __NR_O32_Linux];
>> 380 }
>> 381 #endif
127 382
128 if (function_graph_enter(parent, self_ !! 383 #ifdef CONFIG_64BIT
129 return parent + 8UL; !! 384
>> 385 unsigned long __init arch_syscall_addr(int nr)
>> 386 {
>> 387 #ifdef CONFIG_MIPS32_N32
>> 388 if (nr >= __NR_N32_Linux && nr < __NR_N32_Linux + __NR_N32_Linux_syscalls)
>> 389 return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux];
>> 390 #endif
>> 391 if (nr >= __NR_64_Linux && nr < __NR_64_Linux + __NR_64_Linux_syscalls)
>> 392 return (unsigned long)sys_call_table[nr - __NR_64_Linux];
>> 393 #ifdef CONFIG_MIPS32_O32
>> 394 if (nr >= __NR_O32_Linux && nr < __NR_O32_Linux + __NR_O32_Linux_syscalls)
>> 395 return (unsigned long)sys32_call_table[nr - __NR_O32_Linux];
>> 396 #endif
130 397
131 return return_hooker; !! 398 return (unsigned long) &sys_ni_syscall;
132 } 399 }
133 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ !! 400 #endif
>> 401
>> 402 #endif /* CONFIG_FTRACE_SYSCALLS */
134 403
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.