1 #!/bin/bash
2 # SPDX-License-Identifier: GPL-2.0
3 #
4 # Here's how to use this:
5 #
6 # This script is used to help find functions that are being traced by function
7 # tracer or function graph tracing that causes the machine to reboot, hang, or
8 # crash. Here's the steps to take.
9 #
10 # First, determine if function tracing is working with a single function:
11 #
12 # (note, if this is a problem with function_graph tracing, then simply
13 # replace "function" with "function_graph" in the following steps).
14 #
15 # # cd /sys/kernel/tracing
16 # # echo schedule > set_ftrace_filter
17 # # echo function > current_tracer
18 #
19 # If this works, then we know that something is being traced that shouldn't be.
20 #
21 # # echo nop > current_tracer
22 #
23 # Starting with v5.1 this can be done with numbers, making it much faster:
24 #
25 # The old (slow) way, for kernels before v5.1.
26 #
27 # [old-way] # cat available_filter_functions > ~/full-file
28 #
29 # [old-way] *** Note *** this process will take several minutes to update the
30 # [old-way] filters. Setting multiple functions is an O(n^2) operation, and we
31 # [old-way] are dealing with thousands of functions. So go have coffee, talk
32 # [old-way] with your coworkers, read facebook. And eventually, this operation
33 # [old-way] will end.
34 #
35 # The new way (using numbers) is an O(n) operation, and usually takes less than a second.
36 #
37 # seq `wc -l available_filter_functions | cut -d' ' -f1` > ~/full-file
38 #
39 # This will create a sequence of numbers that match the functions in
40 # available_filter_functions, and when echoing in a number into the
41 # set_ftrace_filter file, it will enable the corresponding function in
42 # O(1) time. Making enabling all functions O(n) where n is the number of
43 # functions to enable.
44 #
45 # For either the new or old way, the rest of the operations remain the same.
46 #
47 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
48 # # cat ~/test-file > set_ftrace_filter
49 #
50 # # echo function > current_tracer
51 #
52 # If it crashes, we know that ~/test-file has a bad function.
53 #
54 # Reboot back to test kernel.
55 #
56 # # cd /sys/kernel/tracing
57 # # mv ~/test-file ~/full-file
58 #
59 # If it didn't crash.
60 #
61 # # echo nop > current_tracer
62 # # mv ~/non-test-file ~/full-file
63 #
64 # Get rid of the other test file from previous run (or save them off somewhere).
65 # # rm -f ~/test-file ~/non-test-file
66 #
67 # And start again:
68 #
69 # # ftrace-bisect ~/full-file ~/test-file ~/non-test-file
70 #
71 # The good thing is, because this cuts the number of functions in ~/test-file
72 # by half, the cat of it into set_ftrace_filter takes half as long each
73 # iteration, so don't talk so much at the water cooler the second time.
74 #
75 # Eventually, if you did this correctly, you will get down to the problem
76 # function, and all we need to do is to notrace it.
77 #
78 # The way to figure out if the problem function is bad, just do:
79 #
80 # # echo <problem-function> > set_ftrace_notrace
81 # # echo > set_ftrace_filter
82 # # echo function > current_tracer
83 #
84 # And if it doesn't crash, we are done.
85 #
86 # If it does crash, do this again (there's more than one problem function)
87 # but you need to echo the problem function(s) into set_ftrace_notrace before
88 # enabling function tracing in the above steps. Or if you can compile the
89 # kernel, annotate the problem functions with "notrace" and start again.
90 #
91
92
93 if [ $# -ne 3 ]; then
94 echo 'usage: ftrace-bisect full-file test-file non-test-file'
95 exit
96 fi
97
98 full=$1
99 test=$2
100 nontest=$3
101
102 x=`cat $full | wc -l`
103 if [ $x -eq 1 ]; then
104 echo "There's only one function left, must be the bad one"
105 cat $full
106 exit 0
107 fi
108
109 let x=$x/2
110 let y=$x+1
111
112 if [ ! -f $full ]; then
113 echo "$full does not exist"
114 exit 1
115 fi
116
117 if [ -f $test ]; then
118 echo -n "$test exists, delete it? [y/N]"
119 read a
120 if [ "$a" != "y" -a "$a" != "Y" ]; then
121 exit 1
122 fi
123 fi
124
125 if [ -f $nontest ]; then
126 echo -n "$nontest exists, delete it? [y/N]"
127 read a
128 if [ "$a" != "y" -a "$a" != "Y" ]; then
129 exit 1
130 fi
131 fi
132
133 sed -ne "1,${x}p" $full > $test
134 sed -ne "$y,\$p" $full > $nontest
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.