1 #!/bin/bash 1 #!/bin/bash
2 # SPDX-License-Identifier: GPL-2.0 2 # SPDX-License-Identifier: GPL-2.0
3 # 3 #
4 # Here's how to use this: 4 # Here's how to use this:
5 # 5 #
6 # This script is used to help find functions t 6 # This script is used to help find functions that are being traced by function
7 # tracer or function graph tracing that causes 7 # tracer or function graph tracing that causes the machine to reboot, hang, or
8 # crash. Here's the steps to take. 8 # crash. Here's the steps to take.
9 # 9 #
10 # First, determine if function tracing is work 10 # First, determine if function tracing is working with a single function:
11 # 11 #
12 # (note, if this is a problem with function_ 12 # (note, if this is a problem with function_graph tracing, then simply
13 # replace "function" with "function_graph" 13 # replace "function" with "function_graph" in the following steps).
14 # 14 #
15 # # cd /sys/kernel/tracing !! 15 # # cd /sys/kernel/debug/tracing
16 # # echo schedule > set_ftrace_filter 16 # # echo schedule > set_ftrace_filter
17 # # echo function > current_tracer 17 # # echo function > current_tracer
18 # 18 #
19 # If this works, then we know that something i 19 # If this works, then we know that something is being traced that shouldn't be.
20 # 20 #
21 # # echo nop > current_tracer 21 # # echo nop > current_tracer
22 # 22 #
23 # Starting with v5.1 this can be done with num !! 23 # # cat available_filter_functions > ~/full-file
24 # <<
25 # The old (slow) way, for kernels before v5.1. <<
26 # <<
27 # [old-way] # cat available_filter_functions > <<
28 # <<
29 # [old-way] *** Note *** this process will ta <<
30 # [old-way] filters. Setting multiple function <<
31 # [old-way] are dealing with thousands of func <<
32 # [old-way] with your coworkers, read facebook <<
33 # [old-way] will end. <<
34 # <<
35 # The new way (using numbers) is an O(n) opera <<
36 # <<
37 # seq `wc -l available_filter_functions | cut <<
38 # <<
39 # This will create a sequence of numbers that <<
40 # available_filter_functions, and when echoing <<
41 # set_ftrace_filter file, it will enable the c <<
42 # O(1) time. Making enabling all functions O(n <<
43 # functions to enable. <<
44 # <<
45 # For either the new or old way, the rest of t <<
46 # <<
47 # # ftrace-bisect ~/full-file ~/test-file ~/n 24 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
48 # # cat ~/test-file > set_ftrace_filter 25 # # cat ~/test-file > set_ftrace_filter
49 # 26 #
>> 27 # *** Note *** this will take several minutes. Setting multiple functions is
>> 28 # an O(n^2) operation, and we are dealing with thousands of functions. So go
>> 29 # have coffee, talk with your coworkers, read facebook. And eventually, this
>> 30 # operation will end.
>> 31 #
50 # # echo function > current_tracer 32 # # echo function > current_tracer
51 # 33 #
52 # If it crashes, we know that ~/test-file has 34 # If it crashes, we know that ~/test-file has a bad function.
53 # 35 #
54 # Reboot back to test kernel. 36 # Reboot back to test kernel.
55 # 37 #
56 # # cd /sys/kernel/tracing !! 38 # # cd /sys/kernel/debug/tracing
57 # # mv ~/test-file ~/full-file 39 # # mv ~/test-file ~/full-file
58 # 40 #
59 # If it didn't crash. 41 # If it didn't crash.
60 # 42 #
61 # # echo nop > current_tracer 43 # # echo nop > current_tracer
62 # # mv ~/non-test-file ~/full-file 44 # # mv ~/non-test-file ~/full-file
63 # 45 #
64 # Get rid of the other test file from previous 46 # Get rid of the other test file from previous run (or save them off somewhere).
65 # # rm -f ~/test-file ~/non-test-file 47 # # rm -f ~/test-file ~/non-test-file
66 # 48 #
67 # And start again: 49 # And start again:
68 # 50 #
69 # # ftrace-bisect ~/full-file ~/test-file ~/n 51 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
70 # 52 #
71 # The good thing is, because this cuts the num 53 # The good thing is, because this cuts the number of functions in ~/test-file
72 # by half, the cat of it into set_ftrace_filte 54 # by half, the cat of it into set_ftrace_filter takes half as long each
73 # iteration, so don't talk so much at the wate 55 # iteration, so don't talk so much at the water cooler the second time.
74 # 56 #
75 # Eventually, if you did this correctly, you w 57 # Eventually, if you did this correctly, you will get down to the problem
76 # function, and all we need to do is to notrac 58 # function, and all we need to do is to notrace it.
77 # 59 #
78 # The way to figure out if the problem functio 60 # The way to figure out if the problem function is bad, just do:
79 # 61 #
80 # # echo <problem-function> > set_ftrace_notr 62 # # echo <problem-function> > set_ftrace_notrace
81 # # echo > set_ftrace_filter 63 # # echo > set_ftrace_filter
82 # # echo function > current_tracer 64 # # echo function > current_tracer
83 # 65 #
84 # And if it doesn't crash, we are done. 66 # And if it doesn't crash, we are done.
85 # 67 #
86 # If it does crash, do this again (there's mor 68 # If it does crash, do this again (there's more than one problem function)
87 # but you need to echo the problem function(s) 69 # but you need to echo the problem function(s) into set_ftrace_notrace before
88 # enabling function tracing in the above steps 70 # enabling function tracing in the above steps. Or if you can compile the
89 # kernel, annotate the problem functions with 71 # kernel, annotate the problem functions with "notrace" and start again.
90 # 72 #
91 73
92 74
93 if [ $# -ne 3 ]; then 75 if [ $# -ne 3 ]; then
94 echo 'usage: ftrace-bisect full-file test-fi 76 echo 'usage: ftrace-bisect full-file test-file non-test-file'
95 exit 77 exit
96 fi 78 fi
97 79
98 full=$1 80 full=$1
99 test=$2 81 test=$2
100 nontest=$3 82 nontest=$3
101 83
102 x=`cat $full | wc -l` 84 x=`cat $full | wc -l`
103 if [ $x -eq 1 ]; then 85 if [ $x -eq 1 ]; then
104 echo "There's only one function left, 86 echo "There's only one function left, must be the bad one"
105 cat $full 87 cat $full
106 exit 0 88 exit 0
107 fi 89 fi
108 90
109 let x=$x/2 91 let x=$x/2
110 let y=$x+1 92 let y=$x+1
111 93
112 if [ ! -f $full ]; then 94 if [ ! -f $full ]; then
113 echo "$full does not exist" 95 echo "$full does not exist"
114 exit 1 96 exit 1
115 fi 97 fi
116 98
117 if [ -f $test ]; then 99 if [ -f $test ]; then
118 echo -n "$test exists, delete it? [y/N 100 echo -n "$test exists, delete it? [y/N]"
119 read a 101 read a
120 if [ "$a" != "y" -a "$a" != "Y" ]; the 102 if [ "$a" != "y" -a "$a" != "Y" ]; then
121 exit 1 103 exit 1
122 fi 104 fi
123 fi 105 fi
124 106
125 if [ -f $nontest ]; then 107 if [ -f $nontest ]; then
126 echo -n "$nontest exists, delete it? [ 108 echo -n "$nontest exists, delete it? [y/N]"
127 read a 109 read a
128 if [ "$a" != "y" -a "$a" != "Y" ]; the 110 if [ "$a" != "y" -a "$a" != "Y" ]; then
129 exit 1 111 exit 1
130 fi 112 fi
131 fi 113 fi
132 114
133 sed -ne "1,${x}p" $full > $test 115 sed -ne "1,${x}p" $full > $test
134 sed -ne "$y,\$p" $full > $nontest 116 sed -ne "$y,\$p" $full > $nontest
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.