1 #!/bin/bash 1 #!/bin/bash
2 # SPDX-License-Identifier: GPL-2.0 2 # SPDX-License-Identifier: GPL-2.0
3 # 3 #
4 # Here's how to use this: 4 # Here's how to use this:
5 # 5 #
6 # This script is used to help find functions t 6 # This script is used to help find functions that are being traced by function
7 # tracer or function graph tracing that causes 7 # tracer or function graph tracing that causes the machine to reboot, hang, or
8 # crash. Here's the steps to take. 8 # crash. Here's the steps to take.
9 # 9 #
10 # First, determine if function tracing is work 10 # First, determine if function tracing is working with a single function:
11 # 11 #
12 # (note, if this is a problem with function_ 12 # (note, if this is a problem with function_graph tracing, then simply
13 # replace "function" with "function_graph" 13 # replace "function" with "function_graph" in the following steps).
14 # 14 #
15 # # cd /sys/kernel/tracing 15 # # cd /sys/kernel/tracing
16 # # echo schedule > set_ftrace_filter 16 # # echo schedule > set_ftrace_filter
17 # # echo function > current_tracer 17 # # echo function > current_tracer
18 # 18 #
19 # If this works, then we know that something i 19 # If this works, then we know that something is being traced that shouldn't be.
20 # 20 #
21 # # echo nop > current_tracer 21 # # echo nop > current_tracer
22 # 22 #
23 # Starting with v5.1 this can be done with num 23 # Starting with v5.1 this can be done with numbers, making it much faster:
24 # 24 #
25 # The old (slow) way, for kernels before v5.1. 25 # The old (slow) way, for kernels before v5.1.
26 # 26 #
27 # [old-way] # cat available_filter_functions > 27 # [old-way] # cat available_filter_functions > ~/full-file
28 # 28 #
29 # [old-way] *** Note *** this process will ta 29 # [old-way] *** Note *** this process will take several minutes to update the
30 # [old-way] filters. Setting multiple function 30 # [old-way] filters. Setting multiple functions is an O(n^2) operation, and we
31 # [old-way] are dealing with thousands of func 31 # [old-way] are dealing with thousands of functions. So go have coffee, talk
32 # [old-way] with your coworkers, read facebook 32 # [old-way] with your coworkers, read facebook. And eventually, this operation
33 # [old-way] will end. 33 # [old-way] will end.
34 # 34 #
35 # The new way (using numbers) is an O(n) opera 35 # The new way (using numbers) is an O(n) operation, and usually takes less than a second.
36 # 36 #
37 # seq `wc -l available_filter_functions | cut 37 # seq `wc -l available_filter_functions | cut -d' ' -f1` > ~/full-file
38 # 38 #
39 # This will create a sequence of numbers that 39 # This will create a sequence of numbers that match the functions in
40 # available_filter_functions, and when echoing 40 # available_filter_functions, and when echoing in a number into the
41 # set_ftrace_filter file, it will enable the c 41 # set_ftrace_filter file, it will enable the corresponding function in
42 # O(1) time. Making enabling all functions O(n 42 # O(1) time. Making enabling all functions O(n) where n is the number of
43 # functions to enable. 43 # functions to enable.
44 # 44 #
45 # For either the new or old way, the rest of t 45 # For either the new or old way, the rest of the operations remain the same.
46 # 46 #
47 # # ftrace-bisect ~/full-file ~/test-file ~/n 47 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
48 # # cat ~/test-file > set_ftrace_filter 48 # # cat ~/test-file > set_ftrace_filter
49 # 49 #
50 # # echo function > current_tracer 50 # # echo function > current_tracer
51 # 51 #
52 # If it crashes, we know that ~/test-file has 52 # If it crashes, we know that ~/test-file has a bad function.
53 # 53 #
54 # Reboot back to test kernel. 54 # Reboot back to test kernel.
55 # 55 #
56 # # cd /sys/kernel/tracing 56 # # cd /sys/kernel/tracing
57 # # mv ~/test-file ~/full-file 57 # # mv ~/test-file ~/full-file
58 # 58 #
59 # If it didn't crash. 59 # If it didn't crash.
60 # 60 #
61 # # echo nop > current_tracer 61 # # echo nop > current_tracer
62 # # mv ~/non-test-file ~/full-file 62 # # mv ~/non-test-file ~/full-file
63 # 63 #
64 # Get rid of the other test file from previous 64 # Get rid of the other test file from previous run (or save them off somewhere).
65 # # rm -f ~/test-file ~/non-test-file 65 # # rm -f ~/test-file ~/non-test-file
66 # 66 #
67 # And start again: 67 # And start again:
68 # 68 #
69 # # ftrace-bisect ~/full-file ~/test-file ~/n 69 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
70 # 70 #
71 # The good thing is, because this cuts the num 71 # The good thing is, because this cuts the number of functions in ~/test-file
72 # by half, the cat of it into set_ftrace_filte 72 # by half, the cat of it into set_ftrace_filter takes half as long each
73 # iteration, so don't talk so much at the wate 73 # iteration, so don't talk so much at the water cooler the second time.
74 # 74 #
75 # Eventually, if you did this correctly, you w 75 # Eventually, if you did this correctly, you will get down to the problem
76 # function, and all we need to do is to notrac 76 # function, and all we need to do is to notrace it.
77 # 77 #
78 # The way to figure out if the problem functio 78 # The way to figure out if the problem function is bad, just do:
79 # 79 #
80 # # echo <problem-function> > set_ftrace_notr 80 # # echo <problem-function> > set_ftrace_notrace
81 # # echo > set_ftrace_filter 81 # # echo > set_ftrace_filter
82 # # echo function > current_tracer 82 # # echo function > current_tracer
83 # 83 #
84 # And if it doesn't crash, we are done. 84 # And if it doesn't crash, we are done.
85 # 85 #
86 # If it does crash, do this again (there's mor 86 # If it does crash, do this again (there's more than one problem function)
87 # but you need to echo the problem function(s) 87 # but you need to echo the problem function(s) into set_ftrace_notrace before
88 # enabling function tracing in the above steps 88 # enabling function tracing in the above steps. Or if you can compile the
89 # kernel, annotate the problem functions with 89 # kernel, annotate the problem functions with "notrace" and start again.
90 # 90 #
91 91
92 92
93 if [ $# -ne 3 ]; then 93 if [ $# -ne 3 ]; then
94 echo 'usage: ftrace-bisect full-file test-fi 94 echo 'usage: ftrace-bisect full-file test-file non-test-file'
95 exit 95 exit
96 fi 96 fi
97 97
98 full=$1 98 full=$1
99 test=$2 99 test=$2
100 nontest=$3 100 nontest=$3
101 101
102 x=`cat $full | wc -l` 102 x=`cat $full | wc -l`
103 if [ $x -eq 1 ]; then 103 if [ $x -eq 1 ]; then
104 echo "There's only one function left, 104 echo "There's only one function left, must be the bad one"
105 cat $full 105 cat $full
106 exit 0 106 exit 0
107 fi 107 fi
108 108
109 let x=$x/2 109 let x=$x/2
110 let y=$x+1 110 let y=$x+1
111 111
112 if [ ! -f $full ]; then 112 if [ ! -f $full ]; then
113 echo "$full does not exist" 113 echo "$full does not exist"
114 exit 1 114 exit 1
115 fi 115 fi
116 116
117 if [ -f $test ]; then 117 if [ -f $test ]; then
118 echo -n "$test exists, delete it? [y/N 118 echo -n "$test exists, delete it? [y/N]"
119 read a 119 read a
120 if [ "$a" != "y" -a "$a" != "Y" ]; the 120 if [ "$a" != "y" -a "$a" != "Y" ]; then
121 exit 1 121 exit 1
122 fi 122 fi
123 fi 123 fi
124 124
125 if [ -f $nontest ]; then 125 if [ -f $nontest ]; then
126 echo -n "$nontest exists, delete it? [ 126 echo -n "$nontest exists, delete it? [y/N]"
127 read a 127 read a
128 if [ "$a" != "y" -a "$a" != "Y" ]; the 128 if [ "$a" != "y" -a "$a" != "Y" ]; then
129 exit 1 129 exit 1
130 fi 130 fi
131 fi 131 fi
132 132
133 sed -ne "1,${x}p" $full > $test 133 sed -ne "1,${x}p" $full > $test
134 sed -ne "$y,\$p" $full > $nontest 134 sed -ne "$y,\$p" $full > $nontest
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.