1 // SPDX-License-Identifier: GPL-2.0 <<
2 /* 1 /*
3 * trace irqs off critical timings 2 * trace irqs off critical timings
4 * 3 *
5 * Copyright (C) 2007-2008 Steven Rostedt <sro 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redha 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * 6 *
8 * From code in the latency_tracer, that is: 7 * From code in the latency_tracer, that is:
9 * 8 *
10 * Copyright (C) 2004-2006 Ingo Molnar 9 * Copyright (C) 2004-2006 Ingo Molnar
11 * Copyright (C) 2004 Nadia Yvette Chambers 10 * Copyright (C) 2004 Nadia Yvette Chambers
12 */ 11 */
13 #include <linux/kallsyms.h> 12 #include <linux/kallsyms.h>
14 #include <linux/uaccess.h> 13 #include <linux/uaccess.h>
15 #include <linux/module.h> 14 #include <linux/module.h>
16 #include <linux/ftrace.h> 15 #include <linux/ftrace.h>
17 #include <linux/kprobes.h> <<
18 16
19 #include "trace.h" 17 #include "trace.h"
20 18
21 #include <trace/events/preemptirq.h> <<
22 <<
23 #if defined(CONFIG_IRQSOFF_TRACER) || defined( <<
24 static struct trace_array *irqso 19 static struct trace_array *irqsoff_trace __read_mostly;
25 static int tracer 20 static int tracer_enabled __read_mostly;
26 21
27 static DEFINE_PER_CPU(int, tracing_cpu); 22 static DEFINE_PER_CPU(int, tracing_cpu);
28 23
29 static DEFINE_RAW_SPINLOCK(max_trace_lock); 24 static DEFINE_RAW_SPINLOCK(max_trace_lock);
30 25
31 enum { 26 enum {
32 TRACER_IRQS_OFF = (1 << 1), 27 TRACER_IRQS_OFF = (1 << 1),
33 TRACER_PREEMPT_OFF = (1 << 2), 28 TRACER_PREEMPT_OFF = (1 << 2),
34 }; 29 };
35 30
36 static int trace_type __read_mostly; 31 static int trace_type __read_mostly;
37 32
38 static int save_flags; 33 static int save_flags;
39 34
40 static void stop_irqsoff_tracer(struct trace_a 35 static void stop_irqsoff_tracer(struct trace_array *tr, int graph);
41 static int start_irqsoff_tracer(struct trace_a 36 static int start_irqsoff_tracer(struct trace_array *tr, int graph);
42 37
43 #ifdef CONFIG_PREEMPT_TRACER 38 #ifdef CONFIG_PREEMPT_TRACER
44 static inline int 39 static inline int
45 preempt_trace(int pc) !! 40 preempt_trace(void)
46 { 41 {
47 return ((trace_type & TRACER_PREEMPT_O !! 42 return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count());
48 } 43 }
49 #else 44 #else
50 # define preempt_trace(pc) (0) !! 45 # define preempt_trace() (0)
51 #endif 46 #endif
52 47
53 #ifdef CONFIG_IRQSOFF_TRACER 48 #ifdef CONFIG_IRQSOFF_TRACER
54 static inline int 49 static inline int
55 irq_trace(void) 50 irq_trace(void)
56 { 51 {
57 return ((trace_type & TRACER_IRQS_OFF) 52 return ((trace_type & TRACER_IRQS_OFF) &&
58 irqs_disabled()); 53 irqs_disabled());
59 } 54 }
60 #else 55 #else
61 # define irq_trace() (0) 56 # define irq_trace() (0)
62 #endif 57 #endif
63 58
64 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 59 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
65 static int irqsoff_display_graph(struct trace_ 60 static int irqsoff_display_graph(struct trace_array *tr, int set);
66 # define is_graph(tr) ((tr)->trace_flags & TRA 61 # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
67 #else 62 #else
68 static inline int irqsoff_display_graph(struct 63 static inline int irqsoff_display_graph(struct trace_array *tr, int set)
69 { 64 {
70 return -EINVAL; 65 return -EINVAL;
71 } 66 }
72 # define is_graph(tr) false 67 # define is_graph(tr) false
73 #endif 68 #endif
74 69
75 /* 70 /*
76 * Sequence count - we record it when starting 71 * Sequence count - we record it when starting a measurement and
77 * skip the latency if the sequence has change 72 * skip the latency if the sequence has changed - some other section
78 * did a maximum and could disturb our measure 73 * did a maximum and could disturb our measurement with serial console
79 * printouts, etc. Truly coinciding maximum la 74 * printouts, etc. Truly coinciding maximum latencies should be rare
80 * and what happens together happens separatel 75 * and what happens together happens separately as well, so this doesn't
81 * decrease the validity of the maximum found: 76 * decrease the validity of the maximum found:
82 */ 77 */
83 static __cacheline_aligned_in_smp unsign 78 static __cacheline_aligned_in_smp unsigned long max_sequence;
84 79
85 #ifdef CONFIG_FUNCTION_TRACER 80 #ifdef CONFIG_FUNCTION_TRACER
86 /* 81 /*
87 * Prologue for the preempt and irqs off funct 82 * Prologue for the preempt and irqs off function tracers.
88 * 83 *
89 * Returns 1 if it is OK to continue, and data 84 * Returns 1 if it is OK to continue, and data->disabled is
90 * incremented. 85 * incremented.
91 * 0 if the trace is to be ignored, an 86 * 0 if the trace is to be ignored, and data->disabled
92 * is kept the same. 87 * is kept the same.
93 * 88 *
94 * Note, this function is also used outside th 89 * Note, this function is also used outside this ifdef but
95 * inside the #ifdef of the function graph tr 90 * inside the #ifdef of the function graph tracer below.
96 * This is OK, since the function graph trace 91 * This is OK, since the function graph tracer is
97 * dependent on the function tracer. 92 * dependent on the function tracer.
98 */ 93 */
99 static int func_prolog_dec(struct trace_array 94 static int func_prolog_dec(struct trace_array *tr,
100 struct trace_array_ 95 struct trace_array_cpu **data,
101 unsigned long *flag 96 unsigned long *flags)
102 { 97 {
103 long disabled; 98 long disabled;
104 int cpu; 99 int cpu;
105 100
106 /* 101 /*
107 * Does not matter if we preempt. We t 102 * Does not matter if we preempt. We test the flags
108 * afterward, to see if irqs are disab 103 * afterward, to see if irqs are disabled or not.
109 * If we preempt and get a false posit 104 * If we preempt and get a false positive, the flags
110 * test will fail. 105 * test will fail.
111 */ 106 */
112 cpu = raw_smp_processor_id(); 107 cpu = raw_smp_processor_id();
113 if (likely(!per_cpu(tracing_cpu, cpu)) 108 if (likely(!per_cpu(tracing_cpu, cpu)))
114 return 0; 109 return 0;
115 110
116 local_save_flags(*flags); 111 local_save_flags(*flags);
117 /* 112 /*
118 * Slight chance to get a false positi 113 * Slight chance to get a false positive on tracing_cpu,
119 * although I'm starting to think ther 114 * although I'm starting to think there isn't a chance.
120 * Leave this for now just to be paran 115 * Leave this for now just to be paranoid.
121 */ 116 */
122 if (!irqs_disabled_flags(*flags) && !p 117 if (!irqs_disabled_flags(*flags) && !preempt_count())
123 return 0; 118 return 0;
124 119
125 *data = per_cpu_ptr(tr->array_buffer.d !! 120 *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
126 disabled = atomic_inc_return(&(*data)- 121 disabled = atomic_inc_return(&(*data)->disabled);
127 122
128 if (likely(disabled == 1)) 123 if (likely(disabled == 1))
129 return 1; 124 return 1;
130 125
131 atomic_dec(&(*data)->disabled); 126 atomic_dec(&(*data)->disabled);
132 127
133 return 0; 128 return 0;
134 } 129 }
135 130
136 /* 131 /*
137 * irqsoff uses its own tracer function to kee 132 * irqsoff uses its own tracer function to keep the overhead down:
138 */ 133 */
139 static void 134 static void
140 irqsoff_tracer_call(unsigned long ip, unsigned 135 irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip,
141 struct ftrace_ops *op, str !! 136 struct ftrace_ops *op, struct pt_regs *pt_regs)
142 { 137 {
143 struct trace_array *tr = irqsoff_trace 138 struct trace_array *tr = irqsoff_trace;
144 struct trace_array_cpu *data; 139 struct trace_array_cpu *data;
145 unsigned long flags; 140 unsigned long flags;
146 unsigned int trace_ctx; <<
147 141
148 if (!func_prolog_dec(tr, &data, &flags 142 if (!func_prolog_dec(tr, &data, &flags))
149 return; 143 return;
150 144
151 trace_ctx = tracing_gen_ctx_flags(flag !! 145 trace_function(tr, ip, parent_ip, flags, preempt_count());
152 <<
153 trace_function(tr, ip, parent_ip, trac <<
154 146
155 atomic_dec(&data->disabled); 147 atomic_dec(&data->disabled);
156 } 148 }
157 #endif /* CONFIG_FUNCTION_TRACER */ 149 #endif /* CONFIG_FUNCTION_TRACER */
158 150
159 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 151 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
160 static int irqsoff_display_graph(struct trace_ 152 static int irqsoff_display_graph(struct trace_array *tr, int set)
161 { 153 {
162 int cpu; 154 int cpu;
163 155
164 if (!(is_graph(tr) ^ set)) 156 if (!(is_graph(tr) ^ set))
165 return 0; 157 return 0;
166 158
167 stop_irqsoff_tracer(irqsoff_trace, !se 159 stop_irqsoff_tracer(irqsoff_trace, !set);
168 160
169 for_each_possible_cpu(cpu) 161 for_each_possible_cpu(cpu)
170 per_cpu(tracing_cpu, cpu) = 0; 162 per_cpu(tracing_cpu, cpu) = 0;
171 163
172 tr->max_latency = 0; 164 tr->max_latency = 0;
173 tracing_reset_online_cpus(&irqsoff_tra !! 165 tracing_reset_online_cpus(&irqsoff_trace->trace_buffer);
174 166
175 return start_irqsoff_tracer(irqsoff_tr 167 return start_irqsoff_tracer(irqsoff_trace, set);
176 } 168 }
177 169
178 static int irqsoff_graph_entry(struct ftrace_g !! 170 static int irqsoff_graph_entry(struct ftrace_graph_ent *trace)
179 struct fgraph_o <<
180 { 171 {
181 struct trace_array *tr = irqsoff_trace 172 struct trace_array *tr = irqsoff_trace;
182 struct trace_array_cpu *data; 173 struct trace_array_cpu *data;
183 unsigned long flags; 174 unsigned long flags;
184 unsigned int trace_ctx; <<
185 int ret; 175 int ret;
>> 176 int pc;
186 177
187 if (ftrace_graph_ignore_func(gops, tra !! 178 if (ftrace_graph_ignore_func(trace))
188 return 0; 179 return 0;
189 /* 180 /*
190 * Do not trace a function if it's fil 181 * Do not trace a function if it's filtered by set_graph_notrace.
191 * Make the index of ret stack negativ 182 * Make the index of ret stack negative to indicate that it should
192 * ignore further functions. But it n 183 * ignore further functions. But it needs its own ret stack entry
193 * to recover the original index in or 184 * to recover the original index in order to continue tracing after
194 * returning from the function. 185 * returning from the function.
195 */ 186 */
196 if (ftrace_graph_notrace_addr(trace->f 187 if (ftrace_graph_notrace_addr(trace->func))
197 return 1; 188 return 1;
198 189
199 if (!func_prolog_dec(tr, &data, &flags 190 if (!func_prolog_dec(tr, &data, &flags))
200 return 0; 191 return 0;
201 192
202 trace_ctx = tracing_gen_ctx_flags(flag !! 193 pc = preempt_count();
203 ret = __trace_graph_entry(tr, trace, t !! 194 ret = __trace_graph_entry(tr, trace, flags, pc);
204 atomic_dec(&data->disabled); 195 atomic_dec(&data->disabled);
205 196
206 return ret; 197 return ret;
207 } 198 }
208 199
209 static void irqsoff_graph_return(struct ftrace !! 200 static void irqsoff_graph_return(struct ftrace_graph_ret *trace)
210 struct fgraph <<
211 { 201 {
212 struct trace_array *tr = irqsoff_trace 202 struct trace_array *tr = irqsoff_trace;
213 struct trace_array_cpu *data; 203 struct trace_array_cpu *data;
214 unsigned long flags; 204 unsigned long flags;
215 unsigned int trace_ctx; !! 205 int pc;
216 206
217 ftrace_graph_addr_finish(gops, trace); !! 207 ftrace_graph_addr_finish(trace);
218 208
219 if (!func_prolog_dec(tr, &data, &flags 209 if (!func_prolog_dec(tr, &data, &flags))
220 return; 210 return;
221 211
222 trace_ctx = tracing_gen_ctx_flags(flag !! 212 pc = preempt_count();
223 __trace_graph_return(tr, trace, trace_ !! 213 __trace_graph_return(tr, trace, flags, pc);
224 atomic_dec(&data->disabled); 214 atomic_dec(&data->disabled);
225 } 215 }
226 216
227 static struct fgraph_ops fgraph_ops = { <<
228 .entryfunc = &irqsoff_gra <<
229 .retfunc = &irqsoff_gra <<
230 }; <<
231 <<
232 static void irqsoff_trace_open(struct trace_it 217 static void irqsoff_trace_open(struct trace_iterator *iter)
233 { 218 {
234 if (is_graph(iter->tr)) 219 if (is_graph(iter->tr))
235 graph_trace_open(iter); 220 graph_trace_open(iter);
236 else 221 else
237 iter->private = NULL; 222 iter->private = NULL;
238 } 223 }
239 224
240 static void irqsoff_trace_close(struct trace_i 225 static void irqsoff_trace_close(struct trace_iterator *iter)
241 { 226 {
242 if (iter->private) 227 if (iter->private)
243 graph_trace_close(iter); 228 graph_trace_close(iter);
244 } 229 }
245 230
246 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_ 231 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \
247 TRACE_GRAPH_PRINT_ 232 TRACE_GRAPH_PRINT_PROC | \
248 TRACE_GRAPH_PRINT_ !! 233 TRACE_GRAPH_PRINT_ABS_TIME | \
249 TRACE_GRAPH_PRINT_ 234 TRACE_GRAPH_PRINT_DURATION)
250 235
251 static enum print_line_t irqsoff_print_line(st 236 static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
252 { 237 {
253 /* 238 /*
254 * In graph mode call the graph tracer 239 * In graph mode call the graph tracer output function,
255 * otherwise go with the TRACE_FN even 240 * otherwise go with the TRACE_FN event handler
256 */ 241 */
257 if (is_graph(iter->tr)) 242 if (is_graph(iter->tr))
258 return print_graph_function_fl 243 return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
259 244
260 return TRACE_TYPE_UNHANDLED; 245 return TRACE_TYPE_UNHANDLED;
261 } 246 }
262 247
263 static void irqsoff_print_header(struct seq_fi 248 static void irqsoff_print_header(struct seq_file *s)
264 { 249 {
265 struct trace_array *tr = irqsoff_trace 250 struct trace_array *tr = irqsoff_trace;
266 251
267 if (is_graph(tr)) 252 if (is_graph(tr))
268 print_graph_headers_flags(s, G 253 print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
269 else 254 else
270 trace_default_header(s); 255 trace_default_header(s);
271 } 256 }
272 257
273 static void 258 static void
274 __trace_function(struct trace_array *tr, 259 __trace_function(struct trace_array *tr,
275 unsigned long ip, unsigned lo 260 unsigned long ip, unsigned long parent_ip,
276 unsigned int trace_ctx) !! 261 unsigned long flags, int pc)
277 { 262 {
278 if (is_graph(tr)) 263 if (is_graph(tr))
279 trace_graph_function(tr, ip, p !! 264 trace_graph_function(tr, ip, parent_ip, flags, pc);
280 else 265 else
281 trace_function(tr, ip, parent_ !! 266 trace_function(tr, ip, parent_ip, flags, pc);
282 } 267 }
283 268
284 #else 269 #else
285 #define __trace_function trace_function 270 #define __trace_function trace_function
286 271
>> 272 #ifdef CONFIG_FUNCTION_TRACER
>> 273 static int irqsoff_graph_entry(struct ftrace_graph_ent *trace)
>> 274 {
>> 275 return -1;
>> 276 }
>> 277 #endif
>> 278
287 static enum print_line_t irqsoff_print_line(st 279 static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
288 { 280 {
289 return TRACE_TYPE_UNHANDLED; 281 return TRACE_TYPE_UNHANDLED;
290 } 282 }
291 283
292 static void irqsoff_trace_open(struct trace_it 284 static void irqsoff_trace_open(struct trace_iterator *iter) { }
293 static void irqsoff_trace_close(struct trace_i 285 static void irqsoff_trace_close(struct trace_iterator *iter) { }
294 286
295 #ifdef CONFIG_FUNCTION_TRACER 287 #ifdef CONFIG_FUNCTION_TRACER
>> 288 static void irqsoff_graph_return(struct ftrace_graph_ret *trace) { }
296 static void irqsoff_print_header(struct seq_fi 289 static void irqsoff_print_header(struct seq_file *s)
297 { 290 {
298 trace_default_header(s); 291 trace_default_header(s);
299 } 292 }
300 #else 293 #else
301 static void irqsoff_print_header(struct seq_fi 294 static void irqsoff_print_header(struct seq_file *s)
302 { 295 {
303 trace_latency_header(s); 296 trace_latency_header(s);
304 } 297 }
305 #endif /* CONFIG_FUNCTION_TRACER */ 298 #endif /* CONFIG_FUNCTION_TRACER */
306 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 299 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
307 300
308 /* 301 /*
309 * Should this new latency be reported/recorde 302 * Should this new latency be reported/recorded?
310 */ 303 */
311 static bool report_latency(struct trace_array 304 static bool report_latency(struct trace_array *tr, u64 delta)
312 { 305 {
313 if (tracing_thresh) { 306 if (tracing_thresh) {
314 if (delta < tracing_thresh) 307 if (delta < tracing_thresh)
315 return false; 308 return false;
316 } else { 309 } else {
317 if (delta <= tr->max_latency) 310 if (delta <= tr->max_latency)
318 return false; 311 return false;
319 } 312 }
320 return true; 313 return true;
321 } 314 }
322 315
323 static void 316 static void
324 check_critical_timing(struct trace_array *tr, 317 check_critical_timing(struct trace_array *tr,
325 struct trace_array_cpu * 318 struct trace_array_cpu *data,
326 unsigned long parent_ip, 319 unsigned long parent_ip,
327 int cpu) 320 int cpu)
328 { 321 {
329 u64 T0, T1, delta; 322 u64 T0, T1, delta;
330 unsigned long flags; 323 unsigned long flags;
331 unsigned int trace_ctx; !! 324 int pc;
332 325
333 T0 = data->preempt_timestamp; 326 T0 = data->preempt_timestamp;
334 T1 = ftrace_now(cpu); 327 T1 = ftrace_now(cpu);
335 delta = T1-T0; 328 delta = T1-T0;
336 329
337 trace_ctx = tracing_gen_ctx(); !! 330 local_save_flags(flags);
>> 331
>> 332 pc = preempt_count();
338 333
339 if (!report_latency(tr, delta)) 334 if (!report_latency(tr, delta))
340 goto out; 335 goto out;
341 336
342 raw_spin_lock_irqsave(&max_trace_lock, 337 raw_spin_lock_irqsave(&max_trace_lock, flags);
343 338
344 /* check if we are still the max laten 339 /* check if we are still the max latency */
345 if (!report_latency(tr, delta)) 340 if (!report_latency(tr, delta))
346 goto out_unlock; 341 goto out_unlock;
347 342
348 __trace_function(tr, CALLER_ADDR0, par !! 343 __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
349 /* Skip 5 functions to get to the irq/ 344 /* Skip 5 functions to get to the irq/preempt enable function */
350 __trace_stack(tr, trace_ctx, 5); !! 345 __trace_stack(tr, flags, 5, pc);
351 346
352 if (data->critical_sequence != max_seq 347 if (data->critical_sequence != max_sequence)
353 goto out_unlock; 348 goto out_unlock;
354 349
355 data->critical_end = parent_ip; 350 data->critical_end = parent_ip;
356 351
357 if (likely(!is_tracing_stopped())) { 352 if (likely(!is_tracing_stopped())) {
358 tr->max_latency = delta; 353 tr->max_latency = delta;
359 update_max_tr_single(tr, curre 354 update_max_tr_single(tr, current, cpu);
360 } 355 }
361 356
362 max_sequence++; 357 max_sequence++;
363 358
364 out_unlock: 359 out_unlock:
365 raw_spin_unlock_irqrestore(&max_trace_ 360 raw_spin_unlock_irqrestore(&max_trace_lock, flags);
366 361
367 out: 362 out:
368 data->critical_sequence = max_sequence 363 data->critical_sequence = max_sequence;
369 data->preempt_timestamp = ftrace_now(c 364 data->preempt_timestamp = ftrace_now(cpu);
370 __trace_function(tr, CALLER_ADDR0, par !! 365 __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
371 } 366 }
372 367
373 static nokprobe_inline void !! 368 static inline void
374 start_critical_timing(unsigned long ip, unsign 369 start_critical_timing(unsigned long ip, unsigned long parent_ip)
375 { 370 {
376 int cpu; 371 int cpu;
377 struct trace_array *tr = irqsoff_trace 372 struct trace_array *tr = irqsoff_trace;
378 struct trace_array_cpu *data; 373 struct trace_array_cpu *data;
>> 374 unsigned long flags;
379 375
380 if (!tracer_enabled || !tracing_is_ena 376 if (!tracer_enabled || !tracing_is_enabled())
381 return; 377 return;
382 378
383 cpu = raw_smp_processor_id(); 379 cpu = raw_smp_processor_id();
384 380
385 if (per_cpu(tracing_cpu, cpu)) 381 if (per_cpu(tracing_cpu, cpu))
386 return; 382 return;
387 383
388 data = per_cpu_ptr(tr->array_buffer.da !! 384 data = per_cpu_ptr(tr->trace_buffer.data, cpu);
389 385
390 if (unlikely(!data) || atomic_read(&da 386 if (unlikely(!data) || atomic_read(&data->disabled))
391 return; 387 return;
392 388
393 atomic_inc(&data->disabled); 389 atomic_inc(&data->disabled);
394 390
395 data->critical_sequence = max_sequence 391 data->critical_sequence = max_sequence;
396 data->preempt_timestamp = ftrace_now(c 392 data->preempt_timestamp = ftrace_now(cpu);
397 data->critical_start = parent_ip ? : i 393 data->critical_start = parent_ip ? : ip;
398 394
399 __trace_function(tr, ip, parent_ip, tr !! 395 local_save_flags(flags);
>> 396
>> 397 __trace_function(tr, ip, parent_ip, flags, preempt_count());
400 398
401 per_cpu(tracing_cpu, cpu) = 1; 399 per_cpu(tracing_cpu, cpu) = 1;
402 400
403 atomic_dec(&data->disabled); 401 atomic_dec(&data->disabled);
404 } 402 }
405 403
406 static nokprobe_inline void !! 404 static inline void
407 stop_critical_timing(unsigned long ip, unsigne 405 stop_critical_timing(unsigned long ip, unsigned long parent_ip)
408 { 406 {
409 int cpu; 407 int cpu;
410 struct trace_array *tr = irqsoff_trace 408 struct trace_array *tr = irqsoff_trace;
411 struct trace_array_cpu *data; 409 struct trace_array_cpu *data;
412 unsigned int trace_ctx; !! 410 unsigned long flags;
413 411
414 cpu = raw_smp_processor_id(); 412 cpu = raw_smp_processor_id();
415 /* Always clear the tracing cpu on sto 413 /* Always clear the tracing cpu on stopping the trace */
416 if (unlikely(per_cpu(tracing_cpu, cpu) 414 if (unlikely(per_cpu(tracing_cpu, cpu)))
417 per_cpu(tracing_cpu, cpu) = 0; 415 per_cpu(tracing_cpu, cpu) = 0;
418 else 416 else
419 return; 417 return;
420 418
421 if (!tracer_enabled || !tracing_is_ena 419 if (!tracer_enabled || !tracing_is_enabled())
422 return; 420 return;
423 421
424 data = per_cpu_ptr(tr->array_buffer.da !! 422 data = per_cpu_ptr(tr->trace_buffer.data, cpu);
425 423
426 if (unlikely(!data) || 424 if (unlikely(!data) ||
427 !data->critical_start || atomic_re 425 !data->critical_start || atomic_read(&data->disabled))
428 return; 426 return;
429 427
430 atomic_inc(&data->disabled); 428 atomic_inc(&data->disabled);
431 429
432 trace_ctx = tracing_gen_ctx(); !! 430 local_save_flags(flags);
433 __trace_function(tr, ip, parent_ip, tr !! 431 __trace_function(tr, ip, parent_ip, flags, preempt_count());
434 check_critical_timing(tr, data, parent 432 check_critical_timing(tr, data, parent_ip ? : ip, cpu);
435 data->critical_start = 0; 433 data->critical_start = 0;
436 atomic_dec(&data->disabled); 434 atomic_dec(&data->disabled);
437 } 435 }
438 436
439 /* start and stop critical timings used to for 437 /* start and stop critical timings used to for stoppage (in idle) */
440 void start_critical_timings(void) 438 void start_critical_timings(void)
441 { 439 {
442 if (preempt_trace(preempt_count()) || !! 440 if (preempt_trace() || irq_trace())
443 start_critical_timing(CALLER_A 441 start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
444 } 442 }
445 EXPORT_SYMBOL_GPL(start_critical_timings); 443 EXPORT_SYMBOL_GPL(start_critical_timings);
446 NOKPROBE_SYMBOL(start_critical_timings); <<
447 444
448 void stop_critical_timings(void) 445 void stop_critical_timings(void)
449 { 446 {
450 if (preempt_trace(preempt_count()) || !! 447 if (preempt_trace() || irq_trace())
451 stop_critical_timing(CALLER_AD 448 stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
452 } 449 }
453 EXPORT_SYMBOL_GPL(stop_critical_timings); 450 EXPORT_SYMBOL_GPL(stop_critical_timings);
454 NOKPROBE_SYMBOL(stop_critical_timings); !! 451
>> 452 #ifdef CONFIG_IRQSOFF_TRACER
>> 453 #ifdef CONFIG_PROVE_LOCKING
>> 454 void time_hardirqs_on(unsigned long a0, unsigned long a1)
>> 455 {
>> 456 if (!preempt_trace() && irq_trace())
>> 457 stop_critical_timing(a0, a1);
>> 458 }
>> 459
>> 460 void time_hardirqs_off(unsigned long a0, unsigned long a1)
>> 461 {
>> 462 if (!preempt_trace() && irq_trace())
>> 463 start_critical_timing(a0, a1);
>> 464 }
>> 465
>> 466 #else /* !CONFIG_PROVE_LOCKING */
>> 467
>> 468 /*
>> 469 * Stubs:
>> 470 */
>> 471
>> 472 void trace_softirqs_on(unsigned long ip)
>> 473 {
>> 474 }
>> 475
>> 476 void trace_softirqs_off(unsigned long ip)
>> 477 {
>> 478 }
>> 479
>> 480 inline void print_irqtrace_events(struct task_struct *curr)
>> 481 {
>> 482 }
>> 483
>> 484 /*
>> 485 * We are only interested in hardirq on/off events:
>> 486 */
>> 487 void trace_hardirqs_on(void)
>> 488 {
>> 489 if (!preempt_trace() && irq_trace())
>> 490 stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
>> 491 }
>> 492 EXPORT_SYMBOL(trace_hardirqs_on);
>> 493
>> 494 void trace_hardirqs_off(void)
>> 495 {
>> 496 if (!preempt_trace() && irq_trace())
>> 497 start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
>> 498 }
>> 499 EXPORT_SYMBOL(trace_hardirqs_off);
>> 500
>> 501 __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
>> 502 {
>> 503 if (!preempt_trace() && irq_trace())
>> 504 stop_critical_timing(CALLER_ADDR0, caller_addr);
>> 505 }
>> 506 EXPORT_SYMBOL(trace_hardirqs_on_caller);
>> 507
>> 508 __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
>> 509 {
>> 510 if (!preempt_trace() && irq_trace())
>> 511 start_critical_timing(CALLER_ADDR0, caller_addr);
>> 512 }
>> 513 EXPORT_SYMBOL(trace_hardirqs_off_caller);
>> 514
>> 515 #endif /* CONFIG_PROVE_LOCKING */
>> 516 #endif /* CONFIG_IRQSOFF_TRACER */
>> 517
>> 518 #ifdef CONFIG_PREEMPT_TRACER
>> 519 void trace_preempt_on(unsigned long a0, unsigned long a1)
>> 520 {
>> 521 if (preempt_trace() && !irq_trace())
>> 522 stop_critical_timing(a0, a1);
>> 523 }
>> 524
>> 525 void trace_preempt_off(unsigned long a0, unsigned long a1)
>> 526 {
>> 527 if (preempt_trace() && !irq_trace())
>> 528 start_critical_timing(a0, a1);
>> 529 }
>> 530 #endif /* CONFIG_PREEMPT_TRACER */
455 531
456 #ifdef CONFIG_FUNCTION_TRACER 532 #ifdef CONFIG_FUNCTION_TRACER
457 static bool function_enabled; 533 static bool function_enabled;
458 534
459 static int register_irqsoff_function(struct tr 535 static int register_irqsoff_function(struct trace_array *tr, int graph, int set)
460 { 536 {
461 int ret; 537 int ret;
462 538
463 /* 'set' is set if TRACE_ITER_FUNCTION 539 /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
464 if (function_enabled || (!set && !(tr- 540 if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
465 return 0; 541 return 0;
466 542
467 if (graph) 543 if (graph)
468 ret = register_ftrace_graph(&f !! 544 ret = register_ftrace_graph(&irqsoff_graph_return,
>> 545 &irqsoff_graph_entry);
469 else 546 else
470 ret = register_ftrace_function 547 ret = register_ftrace_function(tr->ops);
471 548
472 if (!ret) 549 if (!ret)
473 function_enabled = true; 550 function_enabled = true;
474 551
475 return ret; 552 return ret;
476 } 553 }
477 554
478 static void unregister_irqsoff_function(struct 555 static void unregister_irqsoff_function(struct trace_array *tr, int graph)
479 { 556 {
480 if (!function_enabled) 557 if (!function_enabled)
481 return; 558 return;
482 559
483 if (graph) 560 if (graph)
484 unregister_ftrace_graph(&fgrap !! 561 unregister_ftrace_graph();
485 else 562 else
486 unregister_ftrace_function(tr- 563 unregister_ftrace_function(tr->ops);
487 564
488 function_enabled = false; 565 function_enabled = false;
489 } 566 }
490 567
491 static int irqsoff_function_set(struct trace_a 568 static int irqsoff_function_set(struct trace_array *tr, u32 mask, int set)
492 { 569 {
493 if (!(mask & TRACE_ITER_FUNCTION)) 570 if (!(mask & TRACE_ITER_FUNCTION))
494 return 0; 571 return 0;
495 572
496 if (set) 573 if (set)
497 register_irqsoff_function(tr, 574 register_irqsoff_function(tr, is_graph(tr), 1);
498 else 575 else
499 unregister_irqsoff_function(tr 576 unregister_irqsoff_function(tr, is_graph(tr));
500 return 1; 577 return 1;
501 } 578 }
502 #else 579 #else
503 static int register_irqsoff_function(struct tr 580 static int register_irqsoff_function(struct trace_array *tr, int graph, int set)
504 { 581 {
505 return 0; 582 return 0;
506 } 583 }
507 static void unregister_irqsoff_function(struct 584 static void unregister_irqsoff_function(struct trace_array *tr, int graph) { }
508 static inline int irqsoff_function_set(struct 585 static inline int irqsoff_function_set(struct trace_array *tr, u32 mask, int set)
509 { 586 {
510 return 0; 587 return 0;
511 } 588 }
512 #endif /* CONFIG_FUNCTION_TRACER */ 589 #endif /* CONFIG_FUNCTION_TRACER */
513 590
514 static int irqsoff_flag_changed(struct trace_a 591 static int irqsoff_flag_changed(struct trace_array *tr, u32 mask, int set)
515 { 592 {
516 struct tracer *tracer = tr->current_tr 593 struct tracer *tracer = tr->current_trace;
517 594
518 if (irqsoff_function_set(tr, mask, set 595 if (irqsoff_function_set(tr, mask, set))
519 return 0; 596 return 0;
520 597
521 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 598 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
522 if (mask & TRACE_ITER_DISPLAY_GRAPH) 599 if (mask & TRACE_ITER_DISPLAY_GRAPH)
523 return irqsoff_display_graph(t 600 return irqsoff_display_graph(tr, set);
524 #endif 601 #endif
525 602
526 return trace_keep_overwrite(tracer, ma 603 return trace_keep_overwrite(tracer, mask, set);
527 } 604 }
528 605
529 static int start_irqsoff_tracer(struct trace_a 606 static int start_irqsoff_tracer(struct trace_array *tr, int graph)
530 { 607 {
531 int ret; 608 int ret;
532 609
533 ret = register_irqsoff_function(tr, gr 610 ret = register_irqsoff_function(tr, graph, 0);
534 611
535 if (!ret && tracing_is_enabled()) 612 if (!ret && tracing_is_enabled())
536 tracer_enabled = 1; 613 tracer_enabled = 1;
537 else 614 else
538 tracer_enabled = 0; 615 tracer_enabled = 0;
539 616
540 return ret; 617 return ret;
541 } 618 }
542 619
543 static void stop_irqsoff_tracer(struct trace_a 620 static void stop_irqsoff_tracer(struct trace_array *tr, int graph)
544 { 621 {
545 tracer_enabled = 0; 622 tracer_enabled = 0;
546 623
547 unregister_irqsoff_function(tr, graph) 624 unregister_irqsoff_function(tr, graph);
548 } 625 }
549 626
550 static bool irqsoff_busy; 627 static bool irqsoff_busy;
551 628
552 static int __irqsoff_tracer_init(struct trace_ 629 static int __irqsoff_tracer_init(struct trace_array *tr)
553 { 630 {
554 if (irqsoff_busy) 631 if (irqsoff_busy)
555 return -EBUSY; 632 return -EBUSY;
556 633
557 save_flags = tr->trace_flags; 634 save_flags = tr->trace_flags;
558 635
559 /* non overwrite screws up the latency 636 /* non overwrite screws up the latency tracers */
560 set_tracer_flag(tr, TRACE_ITER_OVERWRI 637 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
561 set_tracer_flag(tr, TRACE_ITER_LATENCY 638 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
562 /* without pause, we will produce garb <<
563 set_tracer_flag(tr, TRACE_ITER_PAUSE_O <<
564 639
565 tr->max_latency = 0; 640 tr->max_latency = 0;
566 irqsoff_trace = tr; 641 irqsoff_trace = tr;
567 /* make sure that the tracer is visibl 642 /* make sure that the tracer is visible */
568 smp_wmb(); 643 smp_wmb();
569 644
570 ftrace_init_array_ops(tr, irqsoff_trac 645 ftrace_init_array_ops(tr, irqsoff_tracer_call);
571 646
572 /* Only toplevel instance supports gra 647 /* Only toplevel instance supports graph tracing */
573 if (start_irqsoff_tracer(tr, (tr->flag 648 if (start_irqsoff_tracer(tr, (tr->flags & TRACE_ARRAY_FL_GLOBAL &&
574 is_graph 649 is_graph(tr))))
575 printk(KERN_ERR "failed to sta 650 printk(KERN_ERR "failed to start irqsoff tracer\n");
576 651
577 irqsoff_busy = true; 652 irqsoff_busy = true;
578 return 0; 653 return 0;
579 } 654 }
580 655
581 static void __irqsoff_tracer_reset(struct trac !! 656 static void irqsoff_tracer_reset(struct trace_array *tr)
582 { 657 {
583 int lat_flag = save_flags & TRACE_ITER 658 int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
584 int overwrite_flag = save_flags & TRAC 659 int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
585 int pause_flag = save_flags & TRACE_IT <<
586 660
587 stop_irqsoff_tracer(tr, is_graph(tr)); 661 stop_irqsoff_tracer(tr, is_graph(tr));
588 662
589 set_tracer_flag(tr, TRACE_ITER_LATENCY 663 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
590 set_tracer_flag(tr, TRACE_ITER_OVERWRI 664 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
591 set_tracer_flag(tr, TRACE_ITER_PAUSE_O <<
592 ftrace_reset_array_ops(tr); 665 ftrace_reset_array_ops(tr);
593 666
594 irqsoff_busy = false; 667 irqsoff_busy = false;
595 } 668 }
596 669
597 static void irqsoff_tracer_start(struct trace_ 670 static void irqsoff_tracer_start(struct trace_array *tr)
598 { 671 {
599 tracer_enabled = 1; 672 tracer_enabled = 1;
600 } 673 }
601 674
602 static void irqsoff_tracer_stop(struct trace_a 675 static void irqsoff_tracer_stop(struct trace_array *tr)
603 { 676 {
604 tracer_enabled = 0; 677 tracer_enabled = 0;
605 } 678 }
606 679
607 #ifdef CONFIG_IRQSOFF_TRACER 680 #ifdef CONFIG_IRQSOFF_TRACER
608 /* <<
609 * We are only interested in hardirq on/off ev <<
610 */ <<
611 void tracer_hardirqs_on(unsigned long a0, unsi <<
612 { <<
613 if (!preempt_trace(preempt_count()) && <<
614 stop_critical_timing(a0, a1); <<
615 } <<
616 NOKPROBE_SYMBOL(tracer_hardirqs_on); <<
617 <<
618 void tracer_hardirqs_off(unsigned long a0, uns <<
619 { <<
620 if (!preempt_trace(preempt_count()) && <<
621 start_critical_timing(a0, a1); <<
622 } <<
623 NOKPROBE_SYMBOL(tracer_hardirqs_off); <<
624 <<
625 static int irqsoff_tracer_init(struct trace_ar 681 static int irqsoff_tracer_init(struct trace_array *tr)
626 { 682 {
627 trace_type = TRACER_IRQS_OFF; 683 trace_type = TRACER_IRQS_OFF;
628 684
629 return __irqsoff_tracer_init(tr); 685 return __irqsoff_tracer_init(tr);
630 } 686 }
631 <<
632 static void irqsoff_tracer_reset(struct trace_ <<
633 { <<
634 __irqsoff_tracer_reset(tr); <<
635 } <<
636 <<
637 static struct tracer irqsoff_tracer __read_mos 687 static struct tracer irqsoff_tracer __read_mostly =
638 { 688 {
639 .name = "irqsoff", 689 .name = "irqsoff",
640 .init = irqsoff_tracer_init, 690 .init = irqsoff_tracer_init,
641 .reset = irqsoff_tracer_reset 691 .reset = irqsoff_tracer_reset,
642 .start = irqsoff_tracer_start 692 .start = irqsoff_tracer_start,
643 .stop = irqsoff_tracer_stop, 693 .stop = irqsoff_tracer_stop,
644 .print_max = true, 694 .print_max = true,
645 .print_header = irqsoff_print_header 695 .print_header = irqsoff_print_header,
646 .print_line = irqsoff_print_line, 696 .print_line = irqsoff_print_line,
647 .flag_changed = irqsoff_flag_changed 697 .flag_changed = irqsoff_flag_changed,
648 #ifdef CONFIG_FTRACE_SELFTEST 698 #ifdef CONFIG_FTRACE_SELFTEST
649 .selftest = trace_selftest_startup_ 699 .selftest = trace_selftest_startup_irqsoff,
650 #endif 700 #endif
651 .open = irqsoff_trace_open, 701 .open = irqsoff_trace_open,
652 .close = irqsoff_trace_close, 702 .close = irqsoff_trace_close,
653 .allow_instances = true, 703 .allow_instances = true,
654 .use_max_tr = true, 704 .use_max_tr = true,
655 }; 705 };
656 #endif /* CONFIG_IRQSOFF_TRACER */ !! 706 # define register_irqsoff(trace) register_tracer(&trace)
>> 707 #else
>> 708 # define register_irqsoff(trace) do { } while (0)
>> 709 #endif
657 710
658 #ifdef CONFIG_PREEMPT_TRACER 711 #ifdef CONFIG_PREEMPT_TRACER
659 void tracer_preempt_on(unsigned long a0, unsig <<
660 { <<
661 if (preempt_trace(preempt_count()) && <<
662 stop_critical_timing(a0, a1); <<
663 } <<
664 <<
665 void tracer_preempt_off(unsigned long a0, unsi <<
666 { <<
667 if (preempt_trace(preempt_count()) && <<
668 start_critical_timing(a0, a1); <<
669 } <<
670 <<
671 static int preemptoff_tracer_init(struct trace 712 static int preemptoff_tracer_init(struct trace_array *tr)
672 { 713 {
673 trace_type = TRACER_PREEMPT_OFF; 714 trace_type = TRACER_PREEMPT_OFF;
674 715
675 return __irqsoff_tracer_init(tr); 716 return __irqsoff_tracer_init(tr);
676 } 717 }
677 718
678 static void preemptoff_tracer_reset(struct tra <<
679 { <<
680 __irqsoff_tracer_reset(tr); <<
681 } <<
682 <<
683 static struct tracer preemptoff_tracer __read_ 719 static struct tracer preemptoff_tracer __read_mostly =
684 { 720 {
685 .name = "preemptoff", 721 .name = "preemptoff",
686 .init = preemptoff_tracer_in 722 .init = preemptoff_tracer_init,
687 .reset = preemptoff_tracer_re !! 723 .reset = irqsoff_tracer_reset,
688 .start = irqsoff_tracer_start 724 .start = irqsoff_tracer_start,
689 .stop = irqsoff_tracer_stop, 725 .stop = irqsoff_tracer_stop,
690 .print_max = true, 726 .print_max = true,
691 .print_header = irqsoff_print_header 727 .print_header = irqsoff_print_header,
692 .print_line = irqsoff_print_line, 728 .print_line = irqsoff_print_line,
693 .flag_changed = irqsoff_flag_changed 729 .flag_changed = irqsoff_flag_changed,
694 #ifdef CONFIG_FTRACE_SELFTEST 730 #ifdef CONFIG_FTRACE_SELFTEST
695 .selftest = trace_selftest_startup_ 731 .selftest = trace_selftest_startup_preemptoff,
696 #endif 732 #endif
697 .open = irqsoff_trace_open, 733 .open = irqsoff_trace_open,
698 .close = irqsoff_trace_close, 734 .close = irqsoff_trace_close,
699 .allow_instances = true, 735 .allow_instances = true,
700 .use_max_tr = true, 736 .use_max_tr = true,
701 }; 737 };
702 #endif /* CONFIG_PREEMPT_TRACER */ !! 738 # define register_preemptoff(trace) register_tracer(&trace)
>> 739 #else
>> 740 # define register_preemptoff(trace) do { } while (0)
>> 741 #endif
703 742
704 #if defined(CONFIG_IRQSOFF_TRACER) && defined( !! 743 #if defined(CONFIG_IRQSOFF_TRACER) && \
>> 744 defined(CONFIG_PREEMPT_TRACER)
705 745
706 static int preemptirqsoff_tracer_init(struct t 746 static int preemptirqsoff_tracer_init(struct trace_array *tr)
707 { 747 {
708 trace_type = TRACER_IRQS_OFF | TRACER_ 748 trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
709 749
710 return __irqsoff_tracer_init(tr); 750 return __irqsoff_tracer_init(tr);
711 } 751 }
712 752
713 static void preemptirqsoff_tracer_reset(struct <<
714 { <<
715 __irqsoff_tracer_reset(tr); <<
716 } <<
717 <<
718 static struct tracer preemptirqsoff_tracer __r 753 static struct tracer preemptirqsoff_tracer __read_mostly =
719 { 754 {
720 .name = "preemptirqsoff", 755 .name = "preemptirqsoff",
721 .init = preemptirqsoff_trace 756 .init = preemptirqsoff_tracer_init,
722 .reset = preemptirqsoff_trace !! 757 .reset = irqsoff_tracer_reset,
723 .start = irqsoff_tracer_start 758 .start = irqsoff_tracer_start,
724 .stop = irqsoff_tracer_stop, 759 .stop = irqsoff_tracer_stop,
725 .print_max = true, 760 .print_max = true,
726 .print_header = irqsoff_print_header 761 .print_header = irqsoff_print_header,
727 .print_line = irqsoff_print_line, 762 .print_line = irqsoff_print_line,
728 .flag_changed = irqsoff_flag_changed 763 .flag_changed = irqsoff_flag_changed,
729 #ifdef CONFIG_FTRACE_SELFTEST 764 #ifdef CONFIG_FTRACE_SELFTEST
730 .selftest = trace_selftest_startup_ 765 .selftest = trace_selftest_startup_preemptirqsoff,
731 #endif 766 #endif
732 .open = irqsoff_trace_open, 767 .open = irqsoff_trace_open,
733 .close = irqsoff_trace_close, 768 .close = irqsoff_trace_close,
734 .allow_instances = true, 769 .allow_instances = true,
735 .use_max_tr = true, 770 .use_max_tr = true,
736 }; 771 };
>> 772
>> 773 # define register_preemptirqsoff(trace) register_tracer(&trace)
>> 774 #else
>> 775 # define register_preemptirqsoff(trace) do { } while (0)
737 #endif 776 #endif
738 777
739 __init static int init_irqsoff_tracer(void) 778 __init static int init_irqsoff_tracer(void)
740 { 779 {
741 #ifdef CONFIG_IRQSOFF_TRACER !! 780 register_irqsoff(irqsoff_tracer);
742 register_tracer(&irqsoff_tracer); !! 781 register_preemptoff(preemptoff_tracer);
743 #endif !! 782 register_preemptirqsoff(preemptirqsoff_tracer);
744 #ifdef CONFIG_PREEMPT_TRACER <<
745 register_tracer(&preemptoff_tracer); <<
746 #endif <<
747 #if defined(CONFIG_IRQSOFF_TRACER) && defined( <<
748 register_tracer(&preemptirqsoff_tracer <<
749 #endif <<
750 783
751 return 0; 784 return 0;
752 } 785 }
753 core_initcall(init_irqsoff_tracer); 786 core_initcall(init_irqsoff_tracer);
754 #endif /* IRQSOFF_TRACER || PREEMPTOFF_TRACER <<
755 787
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.