1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_TRACE_RECURSION_H 2 #ifndef _LINUX_TRACE_RECURSION_H
3 #define _LINUX_TRACE_RECURSION_H 3 #define _LINUX_TRACE_RECURSION_H
4 4
5 #include <linux/interrupt.h> 5 #include <linux/interrupt.h>
6 #include <linux/sched.h> 6 #include <linux/sched.h>
7 7
8 #ifdef CONFIG_TRACING 8 #ifdef CONFIG_TRACING
9 9
10 /* Only current can touch trace_recursion */ 10 /* Only current can touch trace_recursion */
11 11
12 /* 12 /*
13 * For function tracing recursion: 13 * For function tracing recursion:
14 * The order of these bits are important. 14 * The order of these bits are important.
15 * 15 *
16 * When function tracing occurs, the followin 16 * When function tracing occurs, the following steps are made:
17 * If arch does not support a ftrace feature 17 * If arch does not support a ftrace feature:
18 * call internal function (uses INTERNAL bi 18 * call internal function (uses INTERNAL bits) which calls...
19 * The function callback, which can use the 19 * The function callback, which can use the FTRACE bits to
20 * check for recursion. 20 * check for recursion.
21 */ 21 */
22 enum { 22 enum {
23 /* Function recursion bits */ 23 /* Function recursion bits */
24 TRACE_FTRACE_BIT, 24 TRACE_FTRACE_BIT,
25 TRACE_FTRACE_NMI_BIT, 25 TRACE_FTRACE_NMI_BIT,
26 TRACE_FTRACE_IRQ_BIT, 26 TRACE_FTRACE_IRQ_BIT,
27 TRACE_FTRACE_SIRQ_BIT, 27 TRACE_FTRACE_SIRQ_BIT,
28 TRACE_FTRACE_TRANSITION_BIT, 28 TRACE_FTRACE_TRANSITION_BIT,
29 29
30 /* Internal use recursion bits */ 30 /* Internal use recursion bits */
31 TRACE_INTERNAL_BIT, 31 TRACE_INTERNAL_BIT,
32 TRACE_INTERNAL_NMI_BIT, 32 TRACE_INTERNAL_NMI_BIT,
33 TRACE_INTERNAL_IRQ_BIT, 33 TRACE_INTERNAL_IRQ_BIT,
34 TRACE_INTERNAL_SIRQ_BIT, 34 TRACE_INTERNAL_SIRQ_BIT,
35 TRACE_INTERNAL_TRANSITION_BIT, 35 TRACE_INTERNAL_TRANSITION_BIT,
36 36
37 TRACE_BRANCH_BIT, 37 TRACE_BRANCH_BIT,
38 /* 38 /*
39 * Abuse of the trace_recursion. 39 * Abuse of the trace_recursion.
40 * As we need a way to maintain state if we ar 40 * As we need a way to maintain state if we are tracing the function
41 * graph in irq because we want to trace a par 41 * graph in irq because we want to trace a particular function that
42 * was called in irq context but we have irq t 42 * was called in irq context but we have irq tracing off. Since this
43 * can only be modified by current, we can reu 43 * can only be modified by current, we can reuse trace_recursion.
44 */ 44 */
45 TRACE_IRQ_BIT, 45 TRACE_IRQ_BIT,
46 46
>> 47 /* Set if the function is in the set_graph_function file */
>> 48 TRACE_GRAPH_BIT,
>> 49
>> 50 /*
>> 51 * In the very unlikely case that an interrupt came in
>> 52 * at a start of graph tracing, and we want to trace
>> 53 * the function in that interrupt, the depth can be greater
>> 54 * than zero, because of the preempted start of a previous
>> 55 * trace. In an even more unlikely case, depth could be 2
>> 56 * if a softirq interrupted the start of graph tracing,
>> 57 * followed by an interrupt preempting a start of graph
>> 58 * tracing in the softirq, and depth can even be 3
>> 59 * if an NMI came in at the start of an interrupt function
>> 60 * that preempted a softirq start of a function that
>> 61 * preempted normal context!!!! Luckily, it can't be
>> 62 * greater than 3, so the next two bits are a mask
>> 63 * of what the depth is when we set TRACE_GRAPH_BIT
>> 64 */
>> 65
>> 66 TRACE_GRAPH_DEPTH_START_BIT,
>> 67 TRACE_GRAPH_DEPTH_END_BIT,
>> 68
>> 69 /*
>> 70 * To implement set_graph_notrace, if this bit is set, we ignore
>> 71 * function graph tracing of called functions, until the return
>> 72 * function is called to clear it.
>> 73 */
>> 74 TRACE_GRAPH_NOTRACE_BIT,
>> 75
47 /* Used to prevent recursion recording 76 /* Used to prevent recursion recording from recursing. */
48 TRACE_RECORD_RECURSION_BIT, 77 TRACE_RECORD_RECURSION_BIT,
49 }; 78 };
50 79
51 #define trace_recursion_set(bit) do { ( 80 #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
52 #define trace_recursion_clear(bit) do { ( 81 #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
53 #define trace_recursion_test(bit) ((curr 82 #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
54 83
>> 84 #define trace_recursion_depth() \
>> 85 (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
>> 86 #define trace_recursion_set_depth(depth) \
>> 87 do { \
>> 88 current->trace_recursion &= \
>> 89 ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \
>> 90 current->trace_recursion |= \
>> 91 ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
>> 92 } while (0)
>> 93
55 #define TRACE_CONTEXT_BITS 4 94 #define TRACE_CONTEXT_BITS 4
56 95
57 #define TRACE_FTRACE_START TRACE_FTRACE_B 96 #define TRACE_FTRACE_START TRACE_FTRACE_BIT
58 97
59 #define TRACE_LIST_START TRACE_INTERNAL 98 #define TRACE_LIST_START TRACE_INTERNAL_BIT
60 99
61 #define TRACE_CONTEXT_MASK ((1 << (TRACE_ 100 #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
62 101
63 /* 102 /*
64 * Used for setting context 103 * Used for setting context
65 * NMI = 0 104 * NMI = 0
66 * IRQ = 1 105 * IRQ = 1
67 * SOFTIRQ = 2 106 * SOFTIRQ = 2
68 * NORMAL = 3 107 * NORMAL = 3
69 */ 108 */
70 enum { 109 enum {
71 TRACE_CTX_NMI, 110 TRACE_CTX_NMI,
72 TRACE_CTX_IRQ, 111 TRACE_CTX_IRQ,
73 TRACE_CTX_SOFTIRQ, 112 TRACE_CTX_SOFTIRQ,
74 TRACE_CTX_NORMAL, 113 TRACE_CTX_NORMAL,
75 TRACE_CTX_TRANSITION, 114 TRACE_CTX_TRANSITION,
76 }; 115 };
77 116
78 static __always_inline int trace_get_context_b 117 static __always_inline int trace_get_context_bit(void)
79 { 118 {
80 unsigned char bit = interrupt_context_ 119 unsigned char bit = interrupt_context_level();
81 120
82 return TRACE_CTX_NORMAL - bit; 121 return TRACE_CTX_NORMAL - bit;
83 } 122 }
84 123
85 #ifdef CONFIG_FTRACE_RECORD_RECURSION 124 #ifdef CONFIG_FTRACE_RECORD_RECURSION
86 extern void ftrace_record_recursion(unsigned l 125 extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
87 # define do_ftrace_record_recursion(ip, pip) 126 # define do_ftrace_record_recursion(ip, pip) \
88 do { 127 do { \
89 if (!trace_recursion_test(TRAC 128 if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
90 trace_recursion_set(TR 129 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
91 ftrace_record_recursio 130 ftrace_record_recursion(ip, pip); \
92 trace_recursion_clear( 131 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
93 } 132 } \
94 } while (0) 133 } while (0)
95 #else 134 #else
96 # define do_ftrace_record_recursion(ip, pip) 135 # define do_ftrace_record_recursion(ip, pip) do { } while (0)
97 #endif 136 #endif
98 137
99 #ifdef CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING !! 138 #ifdef CONFIG_ARCH_WANTS_NO_INSTR
100 # define trace_warn_on_no_rcu(ip) 139 # define trace_warn_on_no_rcu(ip) \
101 ({ 140 ({ \
102 bool __ret = !rcu_is_watching( 141 bool __ret = !rcu_is_watching(); \
103 if (__ret && !trace_recursion_ 142 if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
104 trace_recursion_set(TR 143 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
105 WARN_ONCE(true, "RCU n 144 WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \
106 trace_recursion_clear( 145 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
107 } 146 } \
108 __ret; 147 __ret; \
109 }) 148 })
110 #else 149 #else
111 # define trace_warn_on_no_rcu(ip) false 150 # define trace_warn_on_no_rcu(ip) false
112 #endif 151 #endif
113 152
114 /* 153 /*
115 * Preemption is promised to be disabled when 154 * Preemption is promised to be disabled when return bit >= 0.
116 */ 155 */
117 static __always_inline int trace_test_and_set_ 156 static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
118 157 int start)
119 { 158 {
120 unsigned int val = READ_ONCE(current-> 159 unsigned int val = READ_ONCE(current->trace_recursion);
121 int bit; 160 int bit;
122 161
123 if (trace_warn_on_no_rcu(ip)) 162 if (trace_warn_on_no_rcu(ip))
124 return -1; 163 return -1;
125 164
126 bit = trace_get_context_bit() + start; 165 bit = trace_get_context_bit() + start;
127 if (unlikely(val & (1 << bit))) { 166 if (unlikely(val & (1 << bit))) {
128 /* 167 /*
129 * If an interrupt occurs duri 168 * If an interrupt occurs during a trace, and another trace
130 * happens in that interrupt b 169 * happens in that interrupt but before the preempt_count is
131 * updated to reflect the new 170 * updated to reflect the new interrupt context, then this
132 * will think a recursion occu 171 * will think a recursion occurred, and the event will be dropped.
133 * Let a single instance happe 172 * Let a single instance happen via the TRANSITION_BIT to
134 * not drop those events. 173 * not drop those events.
135 */ 174 */
136 bit = TRACE_CTX_TRANSITION + s 175 bit = TRACE_CTX_TRANSITION + start;
137 if (val & (1 << bit)) { 176 if (val & (1 << bit)) {
138 do_ftrace_record_recur 177 do_ftrace_record_recursion(ip, pip);
139 return -1; 178 return -1;
140 } 179 }
141 } 180 }
142 181
143 val |= 1 << bit; 182 val |= 1 << bit;
144 current->trace_recursion = val; 183 current->trace_recursion = val;
145 barrier(); 184 barrier();
146 185
147 preempt_disable_notrace(); 186 preempt_disable_notrace();
148 187
149 return bit; 188 return bit;
150 } 189 }
151 190
152 /* 191 /*
153 * Preemption will be enabled (if it was previ 192 * Preemption will be enabled (if it was previously enabled).
154 */ 193 */
155 static __always_inline void trace_clear_recurs 194 static __always_inline void trace_clear_recursion(int bit)
156 { 195 {
157 preempt_enable_notrace(); 196 preempt_enable_notrace();
158 barrier(); 197 barrier();
159 trace_recursion_clear(bit); 198 trace_recursion_clear(bit);
160 } 199 }
161 200
162 /** 201 /**
163 * ftrace_test_recursion_trylock - tests for r 202 * ftrace_test_recursion_trylock - tests for recursion in same context
164 * 203 *
165 * Use this for ftrace callbacks. This will de 204 * Use this for ftrace callbacks. This will detect if the function
166 * tracing recursed in the same context (norma 205 * tracing recursed in the same context (normal vs interrupt),
167 * 206 *
168 * Returns: -1 if a recursion happened. 207 * Returns: -1 if a recursion happened.
169 * >= 0 if no recursion. 208 * >= 0 if no recursion.
170 */ 209 */
171 static __always_inline int ftrace_test_recursi 210 static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
172 211 unsigned long parent_ip)
173 { 212 {
174 return trace_test_and_set_recursion(ip 213 return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
175 } 214 }
176 215
177 /** 216 /**
178 * ftrace_test_recursion_unlock - called when 217 * ftrace_test_recursion_unlock - called when function callback is complete
179 * @bit: The return of a successful ftrace_tes 218 * @bit: The return of a successful ftrace_test_recursion_trylock()
180 * 219 *
181 * This is used at the end of a ftrace callbac 220 * This is used at the end of a ftrace callback.
182 */ 221 */
183 static __always_inline void ftrace_test_recurs 222 static __always_inline void ftrace_test_recursion_unlock(int bit)
184 { 223 {
185 trace_clear_recursion(bit); 224 trace_clear_recursion(bit);
186 } 225 }
187 226
188 #endif /* CONFIG_TRACING */ 227 #endif /* CONFIG_TRACING */
189 #endif /* _LINUX_TRACE_RECURSION_H */ 228 #endif /* _LINUX_TRACE_RECURSION_H */
190 229
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.