1 #!/bin/bash 1 #!/bin/bash
2 # SPDX-License-Identifier: GPL-2.0 <<
3 # 2 #
4 # Here's how to use this: 3 # Here's how to use this:
5 # 4 #
6 # This script is used to help find functions t 5 # This script is used to help find functions that are being traced by function
7 # tracer or function graph tracing that causes 6 # tracer or function graph tracing that causes the machine to reboot, hang, or
8 # crash. Here's the steps to take. 7 # crash. Here's the steps to take.
9 # 8 #
10 # First, determine if function tracing is work 9 # First, determine if function tracing is working with a single function:
11 # 10 #
12 # (note, if this is a problem with function_ 11 # (note, if this is a problem with function_graph tracing, then simply
13 # replace "function" with "function_graph" 12 # replace "function" with "function_graph" in the following steps).
14 # 13 #
15 # # cd /sys/kernel/tracing !! 14 # # cd /sys/kernel/debug/tracing
16 # # echo schedule > set_ftrace_filter 15 # # echo schedule > set_ftrace_filter
17 # # echo function > current_tracer 16 # # echo function > current_tracer
18 # 17 #
19 # If this works, then we know that something i 18 # If this works, then we know that something is being traced that shouldn't be.
20 # 19 #
21 # # echo nop > current_tracer 20 # # echo nop > current_tracer
22 # 21 #
23 # Starting with v5.1 this can be done with num !! 22 # # cat available_filter_functions > ~/full-file
24 # <<
25 # The old (slow) way, for kernels before v5.1. <<
26 # <<
27 # [old-way] # cat available_filter_functions > <<
28 # <<
29 # [old-way] *** Note *** this process will ta <<
30 # [old-way] filters. Setting multiple function <<
31 # [old-way] are dealing with thousands of func <<
32 # [old-way] with your coworkers, read facebook <<
33 # [old-way] will end. <<
34 # <<
35 # The new way (using numbers) is an O(n) opera <<
36 # <<
37 # seq `wc -l available_filter_functions | cut <<
38 # <<
39 # This will create a sequence of numbers that <<
40 # available_filter_functions, and when echoing <<
41 # set_ftrace_filter file, it will enable the c <<
42 # O(1) time. Making enabling all functions O(n <<
43 # functions to enable. <<
44 # <<
45 # For either the new or old way, the rest of t <<
46 # <<
47 # # ftrace-bisect ~/full-file ~/test-file ~/n 23 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
48 # # cat ~/test-file > set_ftrace_filter 24 # # cat ~/test-file > set_ftrace_filter
49 # 25 #
>> 26 # *** Note *** this will take several minutes. Setting multiple functions is
>> 27 # an O(n^2) operation, and we are dealing with thousands of functions. So go
>> 28 # have coffee, talk with your coworkers, read facebook. And eventually, this
>> 29 # operation will end.
>> 30 #
50 # # echo function > current_tracer 31 # # echo function > current_tracer
51 # 32 #
52 # If it crashes, we know that ~/test-file has 33 # If it crashes, we know that ~/test-file has a bad function.
53 # 34 #
54 # Reboot back to test kernel. 35 # Reboot back to test kernel.
55 # 36 #
56 # # cd /sys/kernel/tracing !! 37 # # cd /sys/kernel/debug/tracing
57 # # mv ~/test-file ~/full-file 38 # # mv ~/test-file ~/full-file
58 # 39 #
59 # If it didn't crash. 40 # If it didn't crash.
60 # 41 #
61 # # echo nop > current_tracer 42 # # echo nop > current_tracer
62 # # mv ~/non-test-file ~/full-file 43 # # mv ~/non-test-file ~/full-file
63 # 44 #
64 # Get rid of the other test file from previous 45 # Get rid of the other test file from previous run (or save them off somewhere).
65 # # rm -f ~/test-file ~/non-test-file 46 # # rm -f ~/test-file ~/non-test-file
66 # 47 #
67 # And start again: 48 # And start again:
68 # 49 #
69 # # ftrace-bisect ~/full-file ~/test-file ~/n 50 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
70 # 51 #
71 # The good thing is, because this cuts the num 52 # The good thing is, because this cuts the number of functions in ~/test-file
72 # by half, the cat of it into set_ftrace_filte 53 # by half, the cat of it into set_ftrace_filter takes half as long each
73 # iteration, so don't talk so much at the wate 54 # iteration, so don't talk so much at the water cooler the second time.
74 # 55 #
75 # Eventually, if you did this correctly, you w 56 # Eventually, if you did this correctly, you will get down to the problem
76 # function, and all we need to do is to notrac 57 # function, and all we need to do is to notrace it.
77 # 58 #
78 # The way to figure out if the problem functio 59 # The way to figure out if the problem function is bad, just do:
79 # 60 #
80 # # echo <problem-function> > set_ftrace_notr 61 # # echo <problem-function> > set_ftrace_notrace
81 # # echo > set_ftrace_filter 62 # # echo > set_ftrace_filter
82 # # echo function > current_tracer 63 # # echo function > current_tracer
83 # 64 #
84 # And if it doesn't crash, we are done. 65 # And if it doesn't crash, we are done.
85 # 66 #
86 # If it does crash, do this again (there's mor 67 # If it does crash, do this again (there's more than one problem function)
87 # but you need to echo the problem function(s) 68 # but you need to echo the problem function(s) into set_ftrace_notrace before
88 # enabling function tracing in the above steps 69 # enabling function tracing in the above steps. Or if you can compile the
89 # kernel, annotate the problem functions with 70 # kernel, annotate the problem functions with "notrace" and start again.
90 # 71 #
91 72
92 73
93 if [ $# -ne 3 ]; then 74 if [ $# -ne 3 ]; then
94 echo 'usage: ftrace-bisect full-file test-fi 75 echo 'usage: ftrace-bisect full-file test-file non-test-file'
95 exit 76 exit
96 fi 77 fi
97 78
98 full=$1 79 full=$1
99 test=$2 80 test=$2
100 nontest=$3 81 nontest=$3
101 82
102 x=`cat $full | wc -l` 83 x=`cat $full | wc -l`
103 if [ $x -eq 1 ]; then 84 if [ $x -eq 1 ]; then
104 echo "There's only one function left, 85 echo "There's only one function left, must be the bad one"
105 cat $full 86 cat $full
106 exit 0 87 exit 0
107 fi 88 fi
108 89
109 let x=$x/2 90 let x=$x/2
110 let y=$x+1 91 let y=$x+1
111 92
112 if [ ! -f $full ]; then 93 if [ ! -f $full ]; then
113 echo "$full does not exist" 94 echo "$full does not exist"
114 exit 1 95 exit 1
115 fi 96 fi
116 97
117 if [ -f $test ]; then 98 if [ -f $test ]; then
118 echo -n "$test exists, delete it? [y/N 99 echo -n "$test exists, delete it? [y/N]"
119 read a 100 read a
120 if [ "$a" != "y" -a "$a" != "Y" ]; the 101 if [ "$a" != "y" -a "$a" != "Y" ]; then
121 exit 1 102 exit 1
122 fi 103 fi
123 fi 104 fi
124 105
125 if [ -f $nontest ]; then 106 if [ -f $nontest ]; then
126 echo -n "$nontest exists, delete it? [ 107 echo -n "$nontest exists, delete it? [y/N]"
127 read a 108 read a
128 if [ "$a" != "y" -a "$a" != "Y" ]; the 109 if [ "$a" != "y" -a "$a" != "Y" ]; then
129 exit 1 110 exit 1
130 fi 111 fi
131 fi 112 fi
132 113
133 sed -ne "1,${x}p" $full > $test 114 sed -ne "1,${x}p" $full > $test
134 sed -ne "$y,\$p" $full > $nontest 115 sed -ne "$y,\$p" $full > $nontest
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.