1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 /* 2 /*
3 * trace irqs off critical timings 3 * trace irqs off critical timings
4 * 4 *
5 * Copyright (C) 2007-2008 Steven Rostedt <sro 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redha 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * 7 *
8 * From code in the latency_tracer, that is: 8 * From code in the latency_tracer, that is:
9 * 9 *
10 * Copyright (C) 2004-2006 Ingo Molnar 10 * Copyright (C) 2004-2006 Ingo Molnar
11 * Copyright (C) 2004 Nadia Yvette Chambers 11 * Copyright (C) 2004 Nadia Yvette Chambers
12 */ 12 */
13 #include <linux/kallsyms.h> 13 #include <linux/kallsyms.h>
14 #include <linux/uaccess.h> 14 #include <linux/uaccess.h>
15 #include <linux/module.h> 15 #include <linux/module.h>
16 #include <linux/ftrace.h> 16 #include <linux/ftrace.h>
17 #include <linux/kprobes.h> 17 #include <linux/kprobes.h>
18 18
19 #include "trace.h" 19 #include "trace.h"
20 20
21 #include <trace/events/preemptirq.h> 21 #include <trace/events/preemptirq.h>
22 22
23 #if defined(CONFIG_IRQSOFF_TRACER) || defined( 23 #if defined(CONFIG_IRQSOFF_TRACER) || defined(CONFIG_PREEMPT_TRACER)
24 static struct trace_array *irqso 24 static struct trace_array *irqsoff_trace __read_mostly;
25 static int tracer 25 static int tracer_enabled __read_mostly;
26 26
27 static DEFINE_PER_CPU(int, tracing_cpu); 27 static DEFINE_PER_CPU(int, tracing_cpu);
28 28
29 static DEFINE_RAW_SPINLOCK(max_trace_lock); 29 static DEFINE_RAW_SPINLOCK(max_trace_lock);
30 30
31 enum { 31 enum {
32 TRACER_IRQS_OFF = (1 << 1), 32 TRACER_IRQS_OFF = (1 << 1),
33 TRACER_PREEMPT_OFF = (1 << 2), 33 TRACER_PREEMPT_OFF = (1 << 2),
34 }; 34 };
35 35
36 static int trace_type __read_mostly; 36 static int trace_type __read_mostly;
37 37
38 static int save_flags; 38 static int save_flags;
39 39
40 static void stop_irqsoff_tracer(struct trace_a 40 static void stop_irqsoff_tracer(struct trace_array *tr, int graph);
41 static int start_irqsoff_tracer(struct trace_a 41 static int start_irqsoff_tracer(struct trace_array *tr, int graph);
42 42
43 #ifdef CONFIG_PREEMPT_TRACER 43 #ifdef CONFIG_PREEMPT_TRACER
44 static inline int 44 static inline int
45 preempt_trace(int pc) 45 preempt_trace(int pc)
46 { 46 {
47 return ((trace_type & TRACER_PREEMPT_O 47 return ((trace_type & TRACER_PREEMPT_OFF) && pc);
48 } 48 }
49 #else 49 #else
50 # define preempt_trace(pc) (0) 50 # define preempt_trace(pc) (0)
51 #endif 51 #endif
52 52
53 #ifdef CONFIG_IRQSOFF_TRACER 53 #ifdef CONFIG_IRQSOFF_TRACER
54 static inline int 54 static inline int
55 irq_trace(void) 55 irq_trace(void)
56 { 56 {
57 return ((trace_type & TRACER_IRQS_OFF) 57 return ((trace_type & TRACER_IRQS_OFF) &&
58 irqs_disabled()); 58 irqs_disabled());
59 } 59 }
60 #else 60 #else
61 # define irq_trace() (0) 61 # define irq_trace() (0)
62 #endif 62 #endif
63 63
64 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 64 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
65 static int irqsoff_display_graph(struct trace_ 65 static int irqsoff_display_graph(struct trace_array *tr, int set);
66 # define is_graph(tr) ((tr)->trace_flags & TRA 66 # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
67 #else 67 #else
68 static inline int irqsoff_display_graph(struct 68 static inline int irqsoff_display_graph(struct trace_array *tr, int set)
69 { 69 {
70 return -EINVAL; 70 return -EINVAL;
71 } 71 }
72 # define is_graph(tr) false 72 # define is_graph(tr) false
73 #endif 73 #endif
74 74
75 /* 75 /*
76 * Sequence count - we record it when starting 76 * Sequence count - we record it when starting a measurement and
77 * skip the latency if the sequence has change 77 * skip the latency if the sequence has changed - some other section
78 * did a maximum and could disturb our measure 78 * did a maximum and could disturb our measurement with serial console
79 * printouts, etc. Truly coinciding maximum la 79 * printouts, etc. Truly coinciding maximum latencies should be rare
80 * and what happens together happens separatel 80 * and what happens together happens separately as well, so this doesn't
81 * decrease the validity of the maximum found: 81 * decrease the validity of the maximum found:
82 */ 82 */
83 static __cacheline_aligned_in_smp unsign 83 static __cacheline_aligned_in_smp unsigned long max_sequence;
84 84
85 #ifdef CONFIG_FUNCTION_TRACER 85 #ifdef CONFIG_FUNCTION_TRACER
86 /* 86 /*
87 * Prologue for the preempt and irqs off funct 87 * Prologue for the preempt and irqs off function tracers.
88 * 88 *
89 * Returns 1 if it is OK to continue, and data 89 * Returns 1 if it is OK to continue, and data->disabled is
90 * incremented. 90 * incremented.
91 * 0 if the trace is to be ignored, an 91 * 0 if the trace is to be ignored, and data->disabled
92 * is kept the same. 92 * is kept the same.
93 * 93 *
94 * Note, this function is also used outside th 94 * Note, this function is also used outside this ifdef but
95 * inside the #ifdef of the function graph tr 95 * inside the #ifdef of the function graph tracer below.
96 * This is OK, since the function graph trace 96 * This is OK, since the function graph tracer is
97 * dependent on the function tracer. 97 * dependent on the function tracer.
98 */ 98 */
99 static int func_prolog_dec(struct trace_array 99 static int func_prolog_dec(struct trace_array *tr,
100 struct trace_array_ 100 struct trace_array_cpu **data,
101 unsigned long *flag 101 unsigned long *flags)
102 { 102 {
103 long disabled; 103 long disabled;
104 int cpu; 104 int cpu;
105 105
106 /* 106 /*
107 * Does not matter if we preempt. We t 107 * Does not matter if we preempt. We test the flags
108 * afterward, to see if irqs are disab 108 * afterward, to see if irqs are disabled or not.
109 * If we preempt and get a false posit 109 * If we preempt and get a false positive, the flags
110 * test will fail. 110 * test will fail.
111 */ 111 */
112 cpu = raw_smp_processor_id(); 112 cpu = raw_smp_processor_id();
113 if (likely(!per_cpu(tracing_cpu, cpu)) 113 if (likely(!per_cpu(tracing_cpu, cpu)))
114 return 0; 114 return 0;
115 115
116 local_save_flags(*flags); 116 local_save_flags(*flags);
117 /* 117 /*
118 * Slight chance to get a false positi 118 * Slight chance to get a false positive on tracing_cpu,
119 * although I'm starting to think ther 119 * although I'm starting to think there isn't a chance.
120 * Leave this for now just to be paran 120 * Leave this for now just to be paranoid.
121 */ 121 */
122 if (!irqs_disabled_flags(*flags) && !p 122 if (!irqs_disabled_flags(*flags) && !preempt_count())
123 return 0; 123 return 0;
124 124
125 *data = per_cpu_ptr(tr->array_buffer.d 125 *data = per_cpu_ptr(tr->array_buffer.data, cpu);
126 disabled = atomic_inc_return(&(*data)- 126 disabled = atomic_inc_return(&(*data)->disabled);
127 127
128 if (likely(disabled == 1)) 128 if (likely(disabled == 1))
129 return 1; 129 return 1;
130 130
131 atomic_dec(&(*data)->disabled); 131 atomic_dec(&(*data)->disabled);
132 132
133 return 0; 133 return 0;
134 } 134 }
135 135
136 /* 136 /*
137 * irqsoff uses its own tracer function to kee 137 * irqsoff uses its own tracer function to keep the overhead down:
138 */ 138 */
139 static void 139 static void
140 irqsoff_tracer_call(unsigned long ip, unsigned 140 irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip,
141 struct ftrace_ops *op, str 141 struct ftrace_ops *op, struct ftrace_regs *fregs)
142 { 142 {
143 struct trace_array *tr = irqsoff_trace 143 struct trace_array *tr = irqsoff_trace;
144 struct trace_array_cpu *data; 144 struct trace_array_cpu *data;
145 unsigned long flags; 145 unsigned long flags;
146 unsigned int trace_ctx; 146 unsigned int trace_ctx;
147 147
148 if (!func_prolog_dec(tr, &data, &flags 148 if (!func_prolog_dec(tr, &data, &flags))
149 return; 149 return;
150 150
151 trace_ctx = tracing_gen_ctx_flags(flag 151 trace_ctx = tracing_gen_ctx_flags(flags);
152 152
153 trace_function(tr, ip, parent_ip, trac 153 trace_function(tr, ip, parent_ip, trace_ctx);
154 154
155 atomic_dec(&data->disabled); 155 atomic_dec(&data->disabled);
156 } 156 }
157 #endif /* CONFIG_FUNCTION_TRACER */ 157 #endif /* CONFIG_FUNCTION_TRACER */
158 158
159 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 159 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
160 static int irqsoff_display_graph(struct trace_ 160 static int irqsoff_display_graph(struct trace_array *tr, int set)
161 { 161 {
162 int cpu; 162 int cpu;
163 163
164 if (!(is_graph(tr) ^ set)) 164 if (!(is_graph(tr) ^ set))
165 return 0; 165 return 0;
166 166
167 stop_irqsoff_tracer(irqsoff_trace, !se 167 stop_irqsoff_tracer(irqsoff_trace, !set);
168 168
169 for_each_possible_cpu(cpu) 169 for_each_possible_cpu(cpu)
170 per_cpu(tracing_cpu, cpu) = 0; 170 per_cpu(tracing_cpu, cpu) = 0;
171 171
172 tr->max_latency = 0; 172 tr->max_latency = 0;
173 tracing_reset_online_cpus(&irqsoff_tra 173 tracing_reset_online_cpus(&irqsoff_trace->array_buffer);
174 174
175 return start_irqsoff_tracer(irqsoff_tr 175 return start_irqsoff_tracer(irqsoff_trace, set);
176 } 176 }
177 177
178 static int irqsoff_graph_entry(struct ftrace_g 178 static int irqsoff_graph_entry(struct ftrace_graph_ent *trace,
179 struct fgraph_o 179 struct fgraph_ops *gops)
180 { 180 {
181 struct trace_array *tr = irqsoff_trace 181 struct trace_array *tr = irqsoff_trace;
182 struct trace_array_cpu *data; 182 struct trace_array_cpu *data;
183 unsigned long flags; 183 unsigned long flags;
184 unsigned int trace_ctx; 184 unsigned int trace_ctx;
185 int ret; 185 int ret;
186 186
187 if (ftrace_graph_ignore_func(gops, tra 187 if (ftrace_graph_ignore_func(gops, trace))
188 return 0; 188 return 0;
189 /* 189 /*
190 * Do not trace a function if it's fil 190 * Do not trace a function if it's filtered by set_graph_notrace.
191 * Make the index of ret stack negativ 191 * Make the index of ret stack negative to indicate that it should
192 * ignore further functions. But it n 192 * ignore further functions. But it needs its own ret stack entry
193 * to recover the original index in or 193 * to recover the original index in order to continue tracing after
194 * returning from the function. 194 * returning from the function.
195 */ 195 */
196 if (ftrace_graph_notrace_addr(trace->f 196 if (ftrace_graph_notrace_addr(trace->func))
197 return 1; 197 return 1;
198 198
199 if (!func_prolog_dec(tr, &data, &flags 199 if (!func_prolog_dec(tr, &data, &flags))
200 return 0; 200 return 0;
201 201
202 trace_ctx = tracing_gen_ctx_flags(flag 202 trace_ctx = tracing_gen_ctx_flags(flags);
203 ret = __trace_graph_entry(tr, trace, t 203 ret = __trace_graph_entry(tr, trace, trace_ctx);
204 atomic_dec(&data->disabled); 204 atomic_dec(&data->disabled);
205 205
206 return ret; 206 return ret;
207 } 207 }
208 208
209 static void irqsoff_graph_return(struct ftrace 209 static void irqsoff_graph_return(struct ftrace_graph_ret *trace,
210 struct fgraph 210 struct fgraph_ops *gops)
211 { 211 {
212 struct trace_array *tr = irqsoff_trace 212 struct trace_array *tr = irqsoff_trace;
213 struct trace_array_cpu *data; 213 struct trace_array_cpu *data;
214 unsigned long flags; 214 unsigned long flags;
215 unsigned int trace_ctx; 215 unsigned int trace_ctx;
216 216
217 ftrace_graph_addr_finish(gops, trace); 217 ftrace_graph_addr_finish(gops, trace);
218 218
219 if (!func_prolog_dec(tr, &data, &flags 219 if (!func_prolog_dec(tr, &data, &flags))
220 return; 220 return;
221 221
222 trace_ctx = tracing_gen_ctx_flags(flag 222 trace_ctx = tracing_gen_ctx_flags(flags);
223 __trace_graph_return(tr, trace, trace_ 223 __trace_graph_return(tr, trace, trace_ctx);
224 atomic_dec(&data->disabled); 224 atomic_dec(&data->disabled);
225 } 225 }
226 226
227 static struct fgraph_ops fgraph_ops = { 227 static struct fgraph_ops fgraph_ops = {
228 .entryfunc = &irqsoff_gra 228 .entryfunc = &irqsoff_graph_entry,
229 .retfunc = &irqsoff_gra 229 .retfunc = &irqsoff_graph_return,
230 }; 230 };
231 231
232 static void irqsoff_trace_open(struct trace_it 232 static void irqsoff_trace_open(struct trace_iterator *iter)
233 { 233 {
234 if (is_graph(iter->tr)) 234 if (is_graph(iter->tr))
235 graph_trace_open(iter); 235 graph_trace_open(iter);
236 else 236 else
237 iter->private = NULL; 237 iter->private = NULL;
238 } 238 }
239 239
240 static void irqsoff_trace_close(struct trace_i 240 static void irqsoff_trace_close(struct trace_iterator *iter)
241 { 241 {
242 if (iter->private) 242 if (iter->private)
243 graph_trace_close(iter); 243 graph_trace_close(iter);
244 } 244 }
245 245
246 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_ 246 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \
247 TRACE_GRAPH_PRINT_ 247 TRACE_GRAPH_PRINT_PROC | \
248 TRACE_GRAPH_PRINT_ 248 TRACE_GRAPH_PRINT_REL_TIME | \
249 TRACE_GRAPH_PRINT_ 249 TRACE_GRAPH_PRINT_DURATION)
250 250
251 static enum print_line_t irqsoff_print_line(st 251 static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
252 { 252 {
253 /* 253 /*
254 * In graph mode call the graph tracer 254 * In graph mode call the graph tracer output function,
255 * otherwise go with the TRACE_FN even 255 * otherwise go with the TRACE_FN event handler
256 */ 256 */
257 if (is_graph(iter->tr)) 257 if (is_graph(iter->tr))
258 return print_graph_function_fl 258 return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
259 259
260 return TRACE_TYPE_UNHANDLED; 260 return TRACE_TYPE_UNHANDLED;
261 } 261 }
262 262
263 static void irqsoff_print_header(struct seq_fi 263 static void irqsoff_print_header(struct seq_file *s)
264 { 264 {
265 struct trace_array *tr = irqsoff_trace 265 struct trace_array *tr = irqsoff_trace;
266 266
267 if (is_graph(tr)) 267 if (is_graph(tr))
268 print_graph_headers_flags(s, G 268 print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
269 else 269 else
270 trace_default_header(s); 270 trace_default_header(s);
271 } 271 }
272 272
273 static void 273 static void
274 __trace_function(struct trace_array *tr, 274 __trace_function(struct trace_array *tr,
275 unsigned long ip, unsigned lo 275 unsigned long ip, unsigned long parent_ip,
276 unsigned int trace_ctx) 276 unsigned int trace_ctx)
277 { 277 {
278 if (is_graph(tr)) 278 if (is_graph(tr))
279 trace_graph_function(tr, ip, p 279 trace_graph_function(tr, ip, parent_ip, trace_ctx);
280 else 280 else
281 trace_function(tr, ip, parent_ 281 trace_function(tr, ip, parent_ip, trace_ctx);
282 } 282 }
283 283
284 #else 284 #else
285 #define __trace_function trace_function 285 #define __trace_function trace_function
286 286
287 static enum print_line_t irqsoff_print_line(st 287 static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
288 { 288 {
289 return TRACE_TYPE_UNHANDLED; 289 return TRACE_TYPE_UNHANDLED;
290 } 290 }
291 291
292 static void irqsoff_trace_open(struct trace_it 292 static void irqsoff_trace_open(struct trace_iterator *iter) { }
293 static void irqsoff_trace_close(struct trace_i 293 static void irqsoff_trace_close(struct trace_iterator *iter) { }
294 294
295 #ifdef CONFIG_FUNCTION_TRACER 295 #ifdef CONFIG_FUNCTION_TRACER
296 static void irqsoff_print_header(struct seq_fi 296 static void irqsoff_print_header(struct seq_file *s)
297 { 297 {
298 trace_default_header(s); 298 trace_default_header(s);
299 } 299 }
300 #else 300 #else
301 static void irqsoff_print_header(struct seq_fi 301 static void irqsoff_print_header(struct seq_file *s)
302 { 302 {
303 trace_latency_header(s); 303 trace_latency_header(s);
304 } 304 }
305 #endif /* CONFIG_FUNCTION_TRACER */ 305 #endif /* CONFIG_FUNCTION_TRACER */
306 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 306 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
307 307
308 /* 308 /*
309 * Should this new latency be reported/recorde 309 * Should this new latency be reported/recorded?
310 */ 310 */
311 static bool report_latency(struct trace_array 311 static bool report_latency(struct trace_array *tr, u64 delta)
312 { 312 {
313 if (tracing_thresh) { 313 if (tracing_thresh) {
314 if (delta < tracing_thresh) 314 if (delta < tracing_thresh)
315 return false; 315 return false;
316 } else { 316 } else {
317 if (delta <= tr->max_latency) 317 if (delta <= tr->max_latency)
318 return false; 318 return false;
319 } 319 }
320 return true; 320 return true;
321 } 321 }
322 322
323 static void 323 static void
324 check_critical_timing(struct trace_array *tr, 324 check_critical_timing(struct trace_array *tr,
325 struct trace_array_cpu * 325 struct trace_array_cpu *data,
326 unsigned long parent_ip, 326 unsigned long parent_ip,
327 int cpu) 327 int cpu)
328 { 328 {
329 u64 T0, T1, delta; 329 u64 T0, T1, delta;
330 unsigned long flags; 330 unsigned long flags;
331 unsigned int trace_ctx; 331 unsigned int trace_ctx;
332 332
333 T0 = data->preempt_timestamp; 333 T0 = data->preempt_timestamp;
334 T1 = ftrace_now(cpu); 334 T1 = ftrace_now(cpu);
335 delta = T1-T0; 335 delta = T1-T0;
336 336
337 trace_ctx = tracing_gen_ctx(); 337 trace_ctx = tracing_gen_ctx();
338 338
339 if (!report_latency(tr, delta)) 339 if (!report_latency(tr, delta))
340 goto out; 340 goto out;
341 341
342 raw_spin_lock_irqsave(&max_trace_lock, 342 raw_spin_lock_irqsave(&max_trace_lock, flags);
343 343
344 /* check if we are still the max laten 344 /* check if we are still the max latency */
345 if (!report_latency(tr, delta)) 345 if (!report_latency(tr, delta))
346 goto out_unlock; 346 goto out_unlock;
347 347
348 __trace_function(tr, CALLER_ADDR0, par 348 __trace_function(tr, CALLER_ADDR0, parent_ip, trace_ctx);
349 /* Skip 5 functions to get to the irq/ 349 /* Skip 5 functions to get to the irq/preempt enable function */
350 __trace_stack(tr, trace_ctx, 5); 350 __trace_stack(tr, trace_ctx, 5);
351 351
352 if (data->critical_sequence != max_seq 352 if (data->critical_sequence != max_sequence)
353 goto out_unlock; 353 goto out_unlock;
354 354
355 data->critical_end = parent_ip; 355 data->critical_end = parent_ip;
356 356
357 if (likely(!is_tracing_stopped())) { 357 if (likely(!is_tracing_stopped())) {
358 tr->max_latency = delta; 358 tr->max_latency = delta;
359 update_max_tr_single(tr, curre 359 update_max_tr_single(tr, current, cpu);
360 } 360 }
361 361
362 max_sequence++; 362 max_sequence++;
363 363
364 out_unlock: 364 out_unlock:
365 raw_spin_unlock_irqrestore(&max_trace_ 365 raw_spin_unlock_irqrestore(&max_trace_lock, flags);
366 366
367 out: 367 out:
368 data->critical_sequence = max_sequence 368 data->critical_sequence = max_sequence;
369 data->preempt_timestamp = ftrace_now(c 369 data->preempt_timestamp = ftrace_now(cpu);
370 __trace_function(tr, CALLER_ADDR0, par 370 __trace_function(tr, CALLER_ADDR0, parent_ip, trace_ctx);
371 } 371 }
372 372
373 static nokprobe_inline void 373 static nokprobe_inline void
374 start_critical_timing(unsigned long ip, unsign 374 start_critical_timing(unsigned long ip, unsigned long parent_ip)
375 { 375 {
376 int cpu; 376 int cpu;
377 struct trace_array *tr = irqsoff_trace 377 struct trace_array *tr = irqsoff_trace;
378 struct trace_array_cpu *data; 378 struct trace_array_cpu *data;
379 379
380 if (!tracer_enabled || !tracing_is_ena 380 if (!tracer_enabled || !tracing_is_enabled())
381 return; 381 return;
382 382
383 cpu = raw_smp_processor_id(); 383 cpu = raw_smp_processor_id();
384 384
385 if (per_cpu(tracing_cpu, cpu)) 385 if (per_cpu(tracing_cpu, cpu))
386 return; 386 return;
387 387
388 data = per_cpu_ptr(tr->array_buffer.da 388 data = per_cpu_ptr(tr->array_buffer.data, cpu);
389 389
390 if (unlikely(!data) || atomic_read(&da 390 if (unlikely(!data) || atomic_read(&data->disabled))
391 return; 391 return;
392 392
393 atomic_inc(&data->disabled); 393 atomic_inc(&data->disabled);
394 394
395 data->critical_sequence = max_sequence 395 data->critical_sequence = max_sequence;
396 data->preempt_timestamp = ftrace_now(c 396 data->preempt_timestamp = ftrace_now(cpu);
397 data->critical_start = parent_ip ? : i 397 data->critical_start = parent_ip ? : ip;
398 398
399 __trace_function(tr, ip, parent_ip, tr 399 __trace_function(tr, ip, parent_ip, tracing_gen_ctx());
400 400
401 per_cpu(tracing_cpu, cpu) = 1; 401 per_cpu(tracing_cpu, cpu) = 1;
402 402
403 atomic_dec(&data->disabled); 403 atomic_dec(&data->disabled);
404 } 404 }
405 405
406 static nokprobe_inline void 406 static nokprobe_inline void
407 stop_critical_timing(unsigned long ip, unsigne 407 stop_critical_timing(unsigned long ip, unsigned long parent_ip)
408 { 408 {
409 int cpu; 409 int cpu;
410 struct trace_array *tr = irqsoff_trace 410 struct trace_array *tr = irqsoff_trace;
411 struct trace_array_cpu *data; 411 struct trace_array_cpu *data;
412 unsigned int trace_ctx; 412 unsigned int trace_ctx;
413 413
414 cpu = raw_smp_processor_id(); 414 cpu = raw_smp_processor_id();
415 /* Always clear the tracing cpu on sto 415 /* Always clear the tracing cpu on stopping the trace */
416 if (unlikely(per_cpu(tracing_cpu, cpu) 416 if (unlikely(per_cpu(tracing_cpu, cpu)))
417 per_cpu(tracing_cpu, cpu) = 0; 417 per_cpu(tracing_cpu, cpu) = 0;
418 else 418 else
419 return; 419 return;
420 420
421 if (!tracer_enabled || !tracing_is_ena 421 if (!tracer_enabled || !tracing_is_enabled())
422 return; 422 return;
423 423
424 data = per_cpu_ptr(tr->array_buffer.da 424 data = per_cpu_ptr(tr->array_buffer.data, cpu);
425 425
426 if (unlikely(!data) || 426 if (unlikely(!data) ||
427 !data->critical_start || atomic_re 427 !data->critical_start || atomic_read(&data->disabled))
428 return; 428 return;
429 429
430 atomic_inc(&data->disabled); 430 atomic_inc(&data->disabled);
431 431
432 trace_ctx = tracing_gen_ctx(); 432 trace_ctx = tracing_gen_ctx();
433 __trace_function(tr, ip, parent_ip, tr 433 __trace_function(tr, ip, parent_ip, trace_ctx);
434 check_critical_timing(tr, data, parent 434 check_critical_timing(tr, data, parent_ip ? : ip, cpu);
435 data->critical_start = 0; 435 data->critical_start = 0;
436 atomic_dec(&data->disabled); 436 atomic_dec(&data->disabled);
437 } 437 }
438 438
439 /* start and stop critical timings used to for 439 /* start and stop critical timings used to for stoppage (in idle) */
440 void start_critical_timings(void) 440 void start_critical_timings(void)
441 { 441 {
442 if (preempt_trace(preempt_count()) || 442 if (preempt_trace(preempt_count()) || irq_trace())
443 start_critical_timing(CALLER_A 443 start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
444 } 444 }
445 EXPORT_SYMBOL_GPL(start_critical_timings); 445 EXPORT_SYMBOL_GPL(start_critical_timings);
446 NOKPROBE_SYMBOL(start_critical_timings); 446 NOKPROBE_SYMBOL(start_critical_timings);
447 447
448 void stop_critical_timings(void) 448 void stop_critical_timings(void)
449 { 449 {
450 if (preempt_trace(preempt_count()) || 450 if (preempt_trace(preempt_count()) || irq_trace())
451 stop_critical_timing(CALLER_AD 451 stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
452 } 452 }
453 EXPORT_SYMBOL_GPL(stop_critical_timings); 453 EXPORT_SYMBOL_GPL(stop_critical_timings);
454 NOKPROBE_SYMBOL(stop_critical_timings); 454 NOKPROBE_SYMBOL(stop_critical_timings);
455 455
456 #ifdef CONFIG_FUNCTION_TRACER 456 #ifdef CONFIG_FUNCTION_TRACER
457 static bool function_enabled; 457 static bool function_enabled;
458 458
459 static int register_irqsoff_function(struct tr 459 static int register_irqsoff_function(struct trace_array *tr, int graph, int set)
460 { 460 {
461 int ret; 461 int ret;
462 462
463 /* 'set' is set if TRACE_ITER_FUNCTION 463 /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
464 if (function_enabled || (!set && !(tr- 464 if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
465 return 0; 465 return 0;
466 466
467 if (graph) 467 if (graph)
468 ret = register_ftrace_graph(&f 468 ret = register_ftrace_graph(&fgraph_ops);
469 else 469 else
470 ret = register_ftrace_function 470 ret = register_ftrace_function(tr->ops);
471 471
472 if (!ret) 472 if (!ret)
473 function_enabled = true; 473 function_enabled = true;
474 474
475 return ret; 475 return ret;
476 } 476 }
477 477
478 static void unregister_irqsoff_function(struct 478 static void unregister_irqsoff_function(struct trace_array *tr, int graph)
479 { 479 {
480 if (!function_enabled) 480 if (!function_enabled)
481 return; 481 return;
482 482
483 if (graph) 483 if (graph)
484 unregister_ftrace_graph(&fgrap 484 unregister_ftrace_graph(&fgraph_ops);
485 else 485 else
486 unregister_ftrace_function(tr- 486 unregister_ftrace_function(tr->ops);
487 487
488 function_enabled = false; 488 function_enabled = false;
489 } 489 }
490 490
491 static int irqsoff_function_set(struct trace_a 491 static int irqsoff_function_set(struct trace_array *tr, u32 mask, int set)
492 { 492 {
493 if (!(mask & TRACE_ITER_FUNCTION)) 493 if (!(mask & TRACE_ITER_FUNCTION))
494 return 0; 494 return 0;
495 495
496 if (set) 496 if (set)
497 register_irqsoff_function(tr, 497 register_irqsoff_function(tr, is_graph(tr), 1);
498 else 498 else
499 unregister_irqsoff_function(tr 499 unregister_irqsoff_function(tr, is_graph(tr));
500 return 1; 500 return 1;
501 } 501 }
502 #else 502 #else
503 static int register_irqsoff_function(struct tr 503 static int register_irqsoff_function(struct trace_array *tr, int graph, int set)
504 { 504 {
505 return 0; 505 return 0;
506 } 506 }
507 static void unregister_irqsoff_function(struct 507 static void unregister_irqsoff_function(struct trace_array *tr, int graph) { }
508 static inline int irqsoff_function_set(struct 508 static inline int irqsoff_function_set(struct trace_array *tr, u32 mask, int set)
509 { 509 {
510 return 0; 510 return 0;
511 } 511 }
512 #endif /* CONFIG_FUNCTION_TRACER */ 512 #endif /* CONFIG_FUNCTION_TRACER */
513 513
514 static int irqsoff_flag_changed(struct trace_a 514 static int irqsoff_flag_changed(struct trace_array *tr, u32 mask, int set)
515 { 515 {
516 struct tracer *tracer = tr->current_tr 516 struct tracer *tracer = tr->current_trace;
517 517
518 if (irqsoff_function_set(tr, mask, set 518 if (irqsoff_function_set(tr, mask, set))
519 return 0; 519 return 0;
520 520
521 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 521 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
522 if (mask & TRACE_ITER_DISPLAY_GRAPH) 522 if (mask & TRACE_ITER_DISPLAY_GRAPH)
523 return irqsoff_display_graph(t 523 return irqsoff_display_graph(tr, set);
524 #endif 524 #endif
525 525
526 return trace_keep_overwrite(tracer, ma 526 return trace_keep_overwrite(tracer, mask, set);
527 } 527 }
528 528
529 static int start_irqsoff_tracer(struct trace_a 529 static int start_irqsoff_tracer(struct trace_array *tr, int graph)
530 { 530 {
531 int ret; 531 int ret;
532 532
533 ret = register_irqsoff_function(tr, gr 533 ret = register_irqsoff_function(tr, graph, 0);
534 534
535 if (!ret && tracing_is_enabled()) 535 if (!ret && tracing_is_enabled())
536 tracer_enabled = 1; 536 tracer_enabled = 1;
537 else 537 else
538 tracer_enabled = 0; 538 tracer_enabled = 0;
539 539
540 return ret; 540 return ret;
541 } 541 }
542 542
543 static void stop_irqsoff_tracer(struct trace_a 543 static void stop_irqsoff_tracer(struct trace_array *tr, int graph)
544 { 544 {
545 tracer_enabled = 0; 545 tracer_enabled = 0;
546 546
547 unregister_irqsoff_function(tr, graph) 547 unregister_irqsoff_function(tr, graph);
548 } 548 }
549 549
550 static bool irqsoff_busy; 550 static bool irqsoff_busy;
551 551
552 static int __irqsoff_tracer_init(struct trace_ 552 static int __irqsoff_tracer_init(struct trace_array *tr)
553 { 553 {
554 if (irqsoff_busy) 554 if (irqsoff_busy)
555 return -EBUSY; 555 return -EBUSY;
556 556
557 save_flags = tr->trace_flags; 557 save_flags = tr->trace_flags;
558 558
559 /* non overwrite screws up the latency 559 /* non overwrite screws up the latency tracers */
560 set_tracer_flag(tr, TRACE_ITER_OVERWRI 560 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
561 set_tracer_flag(tr, TRACE_ITER_LATENCY 561 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
562 /* without pause, we will produce garb 562 /* without pause, we will produce garbage if another latency occurs */
563 set_tracer_flag(tr, TRACE_ITER_PAUSE_O 563 set_tracer_flag(tr, TRACE_ITER_PAUSE_ON_TRACE, 1);
564 564
565 tr->max_latency = 0; 565 tr->max_latency = 0;
566 irqsoff_trace = tr; 566 irqsoff_trace = tr;
567 /* make sure that the tracer is visibl 567 /* make sure that the tracer is visible */
568 smp_wmb(); 568 smp_wmb();
569 569
570 ftrace_init_array_ops(tr, irqsoff_trac 570 ftrace_init_array_ops(tr, irqsoff_tracer_call);
571 571
572 /* Only toplevel instance supports gra 572 /* Only toplevel instance supports graph tracing */
573 if (start_irqsoff_tracer(tr, (tr->flag 573 if (start_irqsoff_tracer(tr, (tr->flags & TRACE_ARRAY_FL_GLOBAL &&
574 is_graph 574 is_graph(tr))))
575 printk(KERN_ERR "failed to sta 575 printk(KERN_ERR "failed to start irqsoff tracer\n");
576 576
577 irqsoff_busy = true; 577 irqsoff_busy = true;
578 return 0; 578 return 0;
579 } 579 }
580 580
581 static void __irqsoff_tracer_reset(struct trac 581 static void __irqsoff_tracer_reset(struct trace_array *tr)
582 { 582 {
583 int lat_flag = save_flags & TRACE_ITER 583 int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
584 int overwrite_flag = save_flags & TRAC 584 int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
585 int pause_flag = save_flags & TRACE_IT 585 int pause_flag = save_flags & TRACE_ITER_PAUSE_ON_TRACE;
586 586
587 stop_irqsoff_tracer(tr, is_graph(tr)); 587 stop_irqsoff_tracer(tr, is_graph(tr));
588 588
589 set_tracer_flag(tr, TRACE_ITER_LATENCY 589 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
590 set_tracer_flag(tr, TRACE_ITER_OVERWRI 590 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
591 set_tracer_flag(tr, TRACE_ITER_PAUSE_O 591 set_tracer_flag(tr, TRACE_ITER_PAUSE_ON_TRACE, pause_flag);
592 ftrace_reset_array_ops(tr); 592 ftrace_reset_array_ops(tr);
593 593
594 irqsoff_busy = false; 594 irqsoff_busy = false;
595 } 595 }
596 596
597 static void irqsoff_tracer_start(struct trace_ 597 static void irqsoff_tracer_start(struct trace_array *tr)
598 { 598 {
599 tracer_enabled = 1; 599 tracer_enabled = 1;
600 } 600 }
601 601
602 static void irqsoff_tracer_stop(struct trace_a 602 static void irqsoff_tracer_stop(struct trace_array *tr)
603 { 603 {
604 tracer_enabled = 0; 604 tracer_enabled = 0;
605 } 605 }
606 606
607 #ifdef CONFIG_IRQSOFF_TRACER 607 #ifdef CONFIG_IRQSOFF_TRACER
608 /* 608 /*
609 * We are only interested in hardirq on/off ev 609 * We are only interested in hardirq on/off events:
610 */ 610 */
611 void tracer_hardirqs_on(unsigned long a0, unsi 611 void tracer_hardirqs_on(unsigned long a0, unsigned long a1)
612 { 612 {
613 if (!preempt_trace(preempt_count()) && 613 if (!preempt_trace(preempt_count()) && irq_trace())
614 stop_critical_timing(a0, a1); 614 stop_critical_timing(a0, a1);
615 } 615 }
616 NOKPROBE_SYMBOL(tracer_hardirqs_on); 616 NOKPROBE_SYMBOL(tracer_hardirqs_on);
617 617
618 void tracer_hardirqs_off(unsigned long a0, uns 618 void tracer_hardirqs_off(unsigned long a0, unsigned long a1)
619 { 619 {
620 if (!preempt_trace(preempt_count()) && 620 if (!preempt_trace(preempt_count()) && irq_trace())
621 start_critical_timing(a0, a1); 621 start_critical_timing(a0, a1);
622 } 622 }
623 NOKPROBE_SYMBOL(tracer_hardirqs_off); 623 NOKPROBE_SYMBOL(tracer_hardirqs_off);
624 624
625 static int irqsoff_tracer_init(struct trace_ar 625 static int irqsoff_tracer_init(struct trace_array *tr)
626 { 626 {
627 trace_type = TRACER_IRQS_OFF; 627 trace_type = TRACER_IRQS_OFF;
628 628
629 return __irqsoff_tracer_init(tr); 629 return __irqsoff_tracer_init(tr);
630 } 630 }
631 631
632 static void irqsoff_tracer_reset(struct trace_ 632 static void irqsoff_tracer_reset(struct trace_array *tr)
633 { 633 {
634 __irqsoff_tracer_reset(tr); 634 __irqsoff_tracer_reset(tr);
635 } 635 }
636 636
637 static struct tracer irqsoff_tracer __read_mos 637 static struct tracer irqsoff_tracer __read_mostly =
638 { 638 {
639 .name = "irqsoff", 639 .name = "irqsoff",
640 .init = irqsoff_tracer_init, 640 .init = irqsoff_tracer_init,
641 .reset = irqsoff_tracer_reset 641 .reset = irqsoff_tracer_reset,
642 .start = irqsoff_tracer_start 642 .start = irqsoff_tracer_start,
643 .stop = irqsoff_tracer_stop, 643 .stop = irqsoff_tracer_stop,
644 .print_max = true, 644 .print_max = true,
645 .print_header = irqsoff_print_header 645 .print_header = irqsoff_print_header,
646 .print_line = irqsoff_print_line, 646 .print_line = irqsoff_print_line,
647 .flag_changed = irqsoff_flag_changed 647 .flag_changed = irqsoff_flag_changed,
648 #ifdef CONFIG_FTRACE_SELFTEST 648 #ifdef CONFIG_FTRACE_SELFTEST
649 .selftest = trace_selftest_startup_ 649 .selftest = trace_selftest_startup_irqsoff,
650 #endif 650 #endif
651 .open = irqsoff_trace_open, 651 .open = irqsoff_trace_open,
652 .close = irqsoff_trace_close, 652 .close = irqsoff_trace_close,
653 .allow_instances = true, 653 .allow_instances = true,
654 .use_max_tr = true, 654 .use_max_tr = true,
655 }; 655 };
656 #endif /* CONFIG_IRQSOFF_TRACER */ 656 #endif /* CONFIG_IRQSOFF_TRACER */
657 657
658 #ifdef CONFIG_PREEMPT_TRACER 658 #ifdef CONFIG_PREEMPT_TRACER
659 void tracer_preempt_on(unsigned long a0, unsig 659 void tracer_preempt_on(unsigned long a0, unsigned long a1)
660 { 660 {
661 if (preempt_trace(preempt_count()) && 661 if (preempt_trace(preempt_count()) && !irq_trace())
662 stop_critical_timing(a0, a1); 662 stop_critical_timing(a0, a1);
663 } 663 }
664 664
665 void tracer_preempt_off(unsigned long a0, unsi 665 void tracer_preempt_off(unsigned long a0, unsigned long a1)
666 { 666 {
667 if (preempt_trace(preempt_count()) && 667 if (preempt_trace(preempt_count()) && !irq_trace())
668 start_critical_timing(a0, a1); 668 start_critical_timing(a0, a1);
669 } 669 }
670 670
671 static int preemptoff_tracer_init(struct trace 671 static int preemptoff_tracer_init(struct trace_array *tr)
672 { 672 {
673 trace_type = TRACER_PREEMPT_OFF; 673 trace_type = TRACER_PREEMPT_OFF;
674 674
675 return __irqsoff_tracer_init(tr); 675 return __irqsoff_tracer_init(tr);
676 } 676 }
677 677
678 static void preemptoff_tracer_reset(struct tra 678 static void preemptoff_tracer_reset(struct trace_array *tr)
679 { 679 {
680 __irqsoff_tracer_reset(tr); 680 __irqsoff_tracer_reset(tr);
681 } 681 }
682 682
683 static struct tracer preemptoff_tracer __read_ 683 static struct tracer preemptoff_tracer __read_mostly =
684 { 684 {
685 .name = "preemptoff", 685 .name = "preemptoff",
686 .init = preemptoff_tracer_in 686 .init = preemptoff_tracer_init,
687 .reset = preemptoff_tracer_re 687 .reset = preemptoff_tracer_reset,
688 .start = irqsoff_tracer_start 688 .start = irqsoff_tracer_start,
689 .stop = irqsoff_tracer_stop, 689 .stop = irqsoff_tracer_stop,
690 .print_max = true, 690 .print_max = true,
691 .print_header = irqsoff_print_header 691 .print_header = irqsoff_print_header,
692 .print_line = irqsoff_print_line, 692 .print_line = irqsoff_print_line,
693 .flag_changed = irqsoff_flag_changed 693 .flag_changed = irqsoff_flag_changed,
694 #ifdef CONFIG_FTRACE_SELFTEST 694 #ifdef CONFIG_FTRACE_SELFTEST
695 .selftest = trace_selftest_startup_ 695 .selftest = trace_selftest_startup_preemptoff,
696 #endif 696 #endif
697 .open = irqsoff_trace_open, 697 .open = irqsoff_trace_open,
698 .close = irqsoff_trace_close, 698 .close = irqsoff_trace_close,
699 .allow_instances = true, 699 .allow_instances = true,
700 .use_max_tr = true, 700 .use_max_tr = true,
701 }; 701 };
702 #endif /* CONFIG_PREEMPT_TRACER */ 702 #endif /* CONFIG_PREEMPT_TRACER */
703 703
704 #if defined(CONFIG_IRQSOFF_TRACER) && defined( 704 #if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
705 705
706 static int preemptirqsoff_tracer_init(struct t 706 static int preemptirqsoff_tracer_init(struct trace_array *tr)
707 { 707 {
708 trace_type = TRACER_IRQS_OFF | TRACER_ 708 trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
709 709
710 return __irqsoff_tracer_init(tr); 710 return __irqsoff_tracer_init(tr);
711 } 711 }
712 712
713 static void preemptirqsoff_tracer_reset(struct 713 static void preemptirqsoff_tracer_reset(struct trace_array *tr)
714 { 714 {
715 __irqsoff_tracer_reset(tr); 715 __irqsoff_tracer_reset(tr);
716 } 716 }
717 717
718 static struct tracer preemptirqsoff_tracer __r 718 static struct tracer preemptirqsoff_tracer __read_mostly =
719 { 719 {
720 .name = "preemptirqsoff", 720 .name = "preemptirqsoff",
721 .init = preemptirqsoff_trace 721 .init = preemptirqsoff_tracer_init,
722 .reset = preemptirqsoff_trace 722 .reset = preemptirqsoff_tracer_reset,
723 .start = irqsoff_tracer_start 723 .start = irqsoff_tracer_start,
724 .stop = irqsoff_tracer_stop, 724 .stop = irqsoff_tracer_stop,
725 .print_max = true, 725 .print_max = true,
726 .print_header = irqsoff_print_header 726 .print_header = irqsoff_print_header,
727 .print_line = irqsoff_print_line, 727 .print_line = irqsoff_print_line,
728 .flag_changed = irqsoff_flag_changed 728 .flag_changed = irqsoff_flag_changed,
729 #ifdef CONFIG_FTRACE_SELFTEST 729 #ifdef CONFIG_FTRACE_SELFTEST
730 .selftest = trace_selftest_startup_ 730 .selftest = trace_selftest_startup_preemptirqsoff,
731 #endif 731 #endif
732 .open = irqsoff_trace_open, 732 .open = irqsoff_trace_open,
733 .close = irqsoff_trace_close, 733 .close = irqsoff_trace_close,
734 .allow_instances = true, 734 .allow_instances = true,
735 .use_max_tr = true, 735 .use_max_tr = true,
736 }; 736 };
737 #endif 737 #endif
738 738
739 __init static int init_irqsoff_tracer(void) 739 __init static int init_irqsoff_tracer(void)
740 { 740 {
741 #ifdef CONFIG_IRQSOFF_TRACER 741 #ifdef CONFIG_IRQSOFF_TRACER
742 register_tracer(&irqsoff_tracer); 742 register_tracer(&irqsoff_tracer);
743 #endif 743 #endif
744 #ifdef CONFIG_PREEMPT_TRACER 744 #ifdef CONFIG_PREEMPT_TRACER
745 register_tracer(&preemptoff_tracer); 745 register_tracer(&preemptoff_tracer);
746 #endif 746 #endif
747 #if defined(CONFIG_IRQSOFF_TRACER) && defined( 747 #if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
748 register_tracer(&preemptirqsoff_tracer 748 register_tracer(&preemptirqsoff_tracer);
749 #endif 749 #endif
750 750
751 return 0; 751 return 0;
752 } 752 }
753 core_initcall(init_irqsoff_tracer); 753 core_initcall(init_irqsoff_tracer);
754 #endif /* IRQSOFF_TRACER || PREEMPTOFF_TRACER 754 #endif /* IRQSOFF_TRACER || PREEMPTOFF_TRACER */
755 755
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.