1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 /* 2 /*
3 * trace irqs off critical timings 3 * trace irqs off critical timings
4 * 4 *
5 * Copyright (C) 2007-2008 Steven Rostedt <sro 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redha 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * 7 *
8 * From code in the latency_tracer, that is: 8 * From code in the latency_tracer, that is:
9 * 9 *
10 * Copyright (C) 2004-2006 Ingo Molnar 10 * Copyright (C) 2004-2006 Ingo Molnar
11 * Copyright (C) 2004 Nadia Yvette Chambers 11 * Copyright (C) 2004 Nadia Yvette Chambers
12 */ 12 */
13 #include <linux/kallsyms.h> 13 #include <linux/kallsyms.h>
14 #include <linux/uaccess.h> 14 #include <linux/uaccess.h>
15 #include <linux/module.h> 15 #include <linux/module.h>
16 #include <linux/ftrace.h> 16 #include <linux/ftrace.h>
17 #include <linux/kprobes.h> 17 #include <linux/kprobes.h>
18 18
19 #include "trace.h" 19 #include "trace.h"
20 20
21 #include <trace/events/preemptirq.h> 21 #include <trace/events/preemptirq.h>
22 22
23 #if defined(CONFIG_IRQSOFF_TRACER) || defined( 23 #if defined(CONFIG_IRQSOFF_TRACER) || defined(CONFIG_PREEMPT_TRACER)
24 static struct trace_array *irqso 24 static struct trace_array *irqsoff_trace __read_mostly;
25 static int tracer 25 static int tracer_enabled __read_mostly;
26 26
27 static DEFINE_PER_CPU(int, tracing_cpu); 27 static DEFINE_PER_CPU(int, tracing_cpu);
28 28
29 static DEFINE_RAW_SPINLOCK(max_trace_lock); 29 static DEFINE_RAW_SPINLOCK(max_trace_lock);
30 30
31 enum { 31 enum {
32 TRACER_IRQS_OFF = (1 << 1), 32 TRACER_IRQS_OFF = (1 << 1),
33 TRACER_PREEMPT_OFF = (1 << 2), 33 TRACER_PREEMPT_OFF = (1 << 2),
34 }; 34 };
35 35
36 static int trace_type __read_mostly; 36 static int trace_type __read_mostly;
37 37
38 static int save_flags; 38 static int save_flags;
39 39
40 static void stop_irqsoff_tracer(struct trace_a 40 static void stop_irqsoff_tracer(struct trace_array *tr, int graph);
41 static int start_irqsoff_tracer(struct trace_a 41 static int start_irqsoff_tracer(struct trace_array *tr, int graph);
42 42
43 #ifdef CONFIG_PREEMPT_TRACER 43 #ifdef CONFIG_PREEMPT_TRACER
44 static inline int 44 static inline int
45 preempt_trace(int pc) 45 preempt_trace(int pc)
46 { 46 {
47 return ((trace_type & TRACER_PREEMPT_O 47 return ((trace_type & TRACER_PREEMPT_OFF) && pc);
48 } 48 }
49 #else 49 #else
50 # define preempt_trace(pc) (0) 50 # define preempt_trace(pc) (0)
51 #endif 51 #endif
52 52
53 #ifdef CONFIG_IRQSOFF_TRACER 53 #ifdef CONFIG_IRQSOFF_TRACER
54 static inline int 54 static inline int
55 irq_trace(void) 55 irq_trace(void)
56 { 56 {
57 return ((trace_type & TRACER_IRQS_OFF) 57 return ((trace_type & TRACER_IRQS_OFF) &&
58 irqs_disabled()); 58 irqs_disabled());
59 } 59 }
60 #else 60 #else
61 # define irq_trace() (0) 61 # define irq_trace() (0)
62 #endif 62 #endif
63 63
64 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 64 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
65 static int irqsoff_display_graph(struct trace_ 65 static int irqsoff_display_graph(struct trace_array *tr, int set);
66 # define is_graph(tr) ((tr)->trace_flags & TRA 66 # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
67 #else 67 #else
68 static inline int irqsoff_display_graph(struct 68 static inline int irqsoff_display_graph(struct trace_array *tr, int set)
69 { 69 {
70 return -EINVAL; 70 return -EINVAL;
71 } 71 }
72 # define is_graph(tr) false 72 # define is_graph(tr) false
73 #endif 73 #endif
74 74
75 /* 75 /*
76 * Sequence count - we record it when starting 76 * Sequence count - we record it when starting a measurement and
77 * skip the latency if the sequence has change 77 * skip the latency if the sequence has changed - some other section
78 * did a maximum and could disturb our measure 78 * did a maximum and could disturb our measurement with serial console
79 * printouts, etc. Truly coinciding maximum la 79 * printouts, etc. Truly coinciding maximum latencies should be rare
80 * and what happens together happens separatel 80 * and what happens together happens separately as well, so this doesn't
81 * decrease the validity of the maximum found: 81 * decrease the validity of the maximum found:
82 */ 82 */
83 static __cacheline_aligned_in_smp unsign 83 static __cacheline_aligned_in_smp unsigned long max_sequence;
84 84
85 #ifdef CONFIG_FUNCTION_TRACER 85 #ifdef CONFIG_FUNCTION_TRACER
86 /* 86 /*
87 * Prologue for the preempt and irqs off funct 87 * Prologue for the preempt and irqs off function tracers.
88 * 88 *
89 * Returns 1 if it is OK to continue, and data 89 * Returns 1 if it is OK to continue, and data->disabled is
90 * incremented. 90 * incremented.
91 * 0 if the trace is to be ignored, an 91 * 0 if the trace is to be ignored, and data->disabled
92 * is kept the same. 92 * is kept the same.
93 * 93 *
94 * Note, this function is also used outside th 94 * Note, this function is also used outside this ifdef but
95 * inside the #ifdef of the function graph tr 95 * inside the #ifdef of the function graph tracer below.
96 * This is OK, since the function graph trace 96 * This is OK, since the function graph tracer is
97 * dependent on the function tracer. 97 * dependent on the function tracer.
98 */ 98 */
99 static int func_prolog_dec(struct trace_array 99 static int func_prolog_dec(struct trace_array *tr,
100 struct trace_array_ 100 struct trace_array_cpu **data,
101 unsigned long *flag 101 unsigned long *flags)
102 { 102 {
103 long disabled; 103 long disabled;
104 int cpu; 104 int cpu;
105 105
106 /* 106 /*
107 * Does not matter if we preempt. We t 107 * Does not matter if we preempt. We test the flags
108 * afterward, to see if irqs are disab 108 * afterward, to see if irqs are disabled or not.
109 * If we preempt and get a false posit 109 * If we preempt and get a false positive, the flags
110 * test will fail. 110 * test will fail.
111 */ 111 */
112 cpu = raw_smp_processor_id(); 112 cpu = raw_smp_processor_id();
113 if (likely(!per_cpu(tracing_cpu, cpu)) 113 if (likely(!per_cpu(tracing_cpu, cpu)))
114 return 0; 114 return 0;
115 115
116 local_save_flags(*flags); 116 local_save_flags(*flags);
117 /* 117 /*
118 * Slight chance to get a false positi 118 * Slight chance to get a false positive on tracing_cpu,
119 * although I'm starting to think ther 119 * although I'm starting to think there isn't a chance.
120 * Leave this for now just to be paran 120 * Leave this for now just to be paranoid.
121 */ 121 */
122 if (!irqs_disabled_flags(*flags) && !p 122 if (!irqs_disabled_flags(*flags) && !preempt_count())
123 return 0; 123 return 0;
124 124
125 *data = per_cpu_ptr(tr->array_buffer.d 125 *data = per_cpu_ptr(tr->array_buffer.data, cpu);
126 disabled = atomic_inc_return(&(*data)- 126 disabled = atomic_inc_return(&(*data)->disabled);
127 127
128 if (likely(disabled == 1)) 128 if (likely(disabled == 1))
129 return 1; 129 return 1;
130 130
131 atomic_dec(&(*data)->disabled); 131 atomic_dec(&(*data)->disabled);
132 132
133 return 0; 133 return 0;
134 } 134 }
135 135
136 /* 136 /*
137 * irqsoff uses its own tracer function to kee 137 * irqsoff uses its own tracer function to keep the overhead down:
138 */ 138 */
139 static void 139 static void
140 irqsoff_tracer_call(unsigned long ip, unsigned 140 irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip,
141 struct ftrace_ops *op, str !! 141 struct ftrace_ops *op, struct pt_regs *pt_regs)
142 { 142 {
143 struct trace_array *tr = irqsoff_trace 143 struct trace_array *tr = irqsoff_trace;
144 struct trace_array_cpu *data; 144 struct trace_array_cpu *data;
145 unsigned long flags; 145 unsigned long flags;
146 unsigned int trace_ctx; <<
147 146
148 if (!func_prolog_dec(tr, &data, &flags 147 if (!func_prolog_dec(tr, &data, &flags))
149 return; 148 return;
150 149
151 trace_ctx = tracing_gen_ctx_flags(flag !! 150 trace_function(tr, ip, parent_ip, flags, preempt_count());
152 <<
153 trace_function(tr, ip, parent_ip, trac <<
154 151
155 atomic_dec(&data->disabled); 152 atomic_dec(&data->disabled);
156 } 153 }
157 #endif /* CONFIG_FUNCTION_TRACER */ 154 #endif /* CONFIG_FUNCTION_TRACER */
158 155
159 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 156 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
160 static int irqsoff_display_graph(struct trace_ 157 static int irqsoff_display_graph(struct trace_array *tr, int set)
161 { 158 {
162 int cpu; 159 int cpu;
163 160
164 if (!(is_graph(tr) ^ set)) 161 if (!(is_graph(tr) ^ set))
165 return 0; 162 return 0;
166 163
167 stop_irqsoff_tracer(irqsoff_trace, !se 164 stop_irqsoff_tracer(irqsoff_trace, !set);
168 165
169 for_each_possible_cpu(cpu) 166 for_each_possible_cpu(cpu)
170 per_cpu(tracing_cpu, cpu) = 0; 167 per_cpu(tracing_cpu, cpu) = 0;
171 168
172 tr->max_latency = 0; 169 tr->max_latency = 0;
173 tracing_reset_online_cpus(&irqsoff_tra 170 tracing_reset_online_cpus(&irqsoff_trace->array_buffer);
174 171
175 return start_irqsoff_tracer(irqsoff_tr 172 return start_irqsoff_tracer(irqsoff_trace, set);
176 } 173 }
177 174
178 static int irqsoff_graph_entry(struct ftrace_g !! 175 static int irqsoff_graph_entry(struct ftrace_graph_ent *trace)
179 struct fgraph_o <<
180 { 176 {
181 struct trace_array *tr = irqsoff_trace 177 struct trace_array *tr = irqsoff_trace;
182 struct trace_array_cpu *data; 178 struct trace_array_cpu *data;
183 unsigned long flags; 179 unsigned long flags;
184 unsigned int trace_ctx; <<
185 int ret; 180 int ret;
>> 181 int pc;
186 182
187 if (ftrace_graph_ignore_func(gops, tra !! 183 if (ftrace_graph_ignore_func(trace))
188 return 0; 184 return 0;
189 /* 185 /*
190 * Do not trace a function if it's fil 186 * Do not trace a function if it's filtered by set_graph_notrace.
191 * Make the index of ret stack negativ 187 * Make the index of ret stack negative to indicate that it should
192 * ignore further functions. But it n 188 * ignore further functions. But it needs its own ret stack entry
193 * to recover the original index in or 189 * to recover the original index in order to continue tracing after
194 * returning from the function. 190 * returning from the function.
195 */ 191 */
196 if (ftrace_graph_notrace_addr(trace->f 192 if (ftrace_graph_notrace_addr(trace->func))
197 return 1; 193 return 1;
198 194
199 if (!func_prolog_dec(tr, &data, &flags 195 if (!func_prolog_dec(tr, &data, &flags))
200 return 0; 196 return 0;
201 197
202 trace_ctx = tracing_gen_ctx_flags(flag !! 198 pc = preempt_count();
203 ret = __trace_graph_entry(tr, trace, t !! 199 ret = __trace_graph_entry(tr, trace, flags, pc);
204 atomic_dec(&data->disabled); 200 atomic_dec(&data->disabled);
205 201
206 return ret; 202 return ret;
207 } 203 }
208 204
209 static void irqsoff_graph_return(struct ftrace !! 205 static void irqsoff_graph_return(struct ftrace_graph_ret *trace)
210 struct fgraph <<
211 { 206 {
212 struct trace_array *tr = irqsoff_trace 207 struct trace_array *tr = irqsoff_trace;
213 struct trace_array_cpu *data; 208 struct trace_array_cpu *data;
214 unsigned long flags; 209 unsigned long flags;
215 unsigned int trace_ctx; !! 210 int pc;
216 211
217 ftrace_graph_addr_finish(gops, trace); !! 212 ftrace_graph_addr_finish(trace);
218 213
219 if (!func_prolog_dec(tr, &data, &flags 214 if (!func_prolog_dec(tr, &data, &flags))
220 return; 215 return;
221 216
222 trace_ctx = tracing_gen_ctx_flags(flag !! 217 pc = preempt_count();
223 __trace_graph_return(tr, trace, trace_ !! 218 __trace_graph_return(tr, trace, flags, pc);
224 atomic_dec(&data->disabled); 219 atomic_dec(&data->disabled);
225 } 220 }
226 221
227 static struct fgraph_ops fgraph_ops = { 222 static struct fgraph_ops fgraph_ops = {
228 .entryfunc = &irqsoff_gra 223 .entryfunc = &irqsoff_graph_entry,
229 .retfunc = &irqsoff_gra 224 .retfunc = &irqsoff_graph_return,
230 }; 225 };
231 226
232 static void irqsoff_trace_open(struct trace_it 227 static void irqsoff_trace_open(struct trace_iterator *iter)
233 { 228 {
234 if (is_graph(iter->tr)) 229 if (is_graph(iter->tr))
235 graph_trace_open(iter); 230 graph_trace_open(iter);
236 else !! 231
237 iter->private = NULL; <<
238 } 232 }
239 233
240 static void irqsoff_trace_close(struct trace_i 234 static void irqsoff_trace_close(struct trace_iterator *iter)
241 { 235 {
242 if (iter->private) 236 if (iter->private)
243 graph_trace_close(iter); 237 graph_trace_close(iter);
244 } 238 }
245 239
246 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_ 240 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \
247 TRACE_GRAPH_PRINT_ 241 TRACE_GRAPH_PRINT_PROC | \
248 TRACE_GRAPH_PRINT_ 242 TRACE_GRAPH_PRINT_REL_TIME | \
249 TRACE_GRAPH_PRINT_ 243 TRACE_GRAPH_PRINT_DURATION)
250 244
251 static enum print_line_t irqsoff_print_line(st 245 static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
252 { 246 {
253 /* 247 /*
254 * In graph mode call the graph tracer 248 * In graph mode call the graph tracer output function,
255 * otherwise go with the TRACE_FN even 249 * otherwise go with the TRACE_FN event handler
256 */ 250 */
257 if (is_graph(iter->tr)) 251 if (is_graph(iter->tr))
258 return print_graph_function_fl 252 return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
259 253
260 return TRACE_TYPE_UNHANDLED; 254 return TRACE_TYPE_UNHANDLED;
261 } 255 }
262 256
263 static void irqsoff_print_header(struct seq_fi 257 static void irqsoff_print_header(struct seq_file *s)
264 { 258 {
265 struct trace_array *tr = irqsoff_trace 259 struct trace_array *tr = irqsoff_trace;
266 260
267 if (is_graph(tr)) 261 if (is_graph(tr))
268 print_graph_headers_flags(s, G 262 print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
269 else 263 else
270 trace_default_header(s); 264 trace_default_header(s);
271 } 265 }
272 266
273 static void 267 static void
274 __trace_function(struct trace_array *tr, 268 __trace_function(struct trace_array *tr,
275 unsigned long ip, unsigned lo 269 unsigned long ip, unsigned long parent_ip,
276 unsigned int trace_ctx) !! 270 unsigned long flags, int pc)
277 { 271 {
278 if (is_graph(tr)) 272 if (is_graph(tr))
279 trace_graph_function(tr, ip, p !! 273 trace_graph_function(tr, ip, parent_ip, flags, pc);
280 else 274 else
281 trace_function(tr, ip, parent_ !! 275 trace_function(tr, ip, parent_ip, flags, pc);
282 } 276 }
283 277
284 #else 278 #else
285 #define __trace_function trace_function 279 #define __trace_function trace_function
286 280
287 static enum print_line_t irqsoff_print_line(st 281 static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
288 { 282 {
289 return TRACE_TYPE_UNHANDLED; 283 return TRACE_TYPE_UNHANDLED;
290 } 284 }
291 285
292 static void irqsoff_trace_open(struct trace_it 286 static void irqsoff_trace_open(struct trace_iterator *iter) { }
293 static void irqsoff_trace_close(struct trace_i 287 static void irqsoff_trace_close(struct trace_iterator *iter) { }
294 288
295 #ifdef CONFIG_FUNCTION_TRACER 289 #ifdef CONFIG_FUNCTION_TRACER
296 static void irqsoff_print_header(struct seq_fi 290 static void irqsoff_print_header(struct seq_file *s)
297 { 291 {
298 trace_default_header(s); 292 trace_default_header(s);
299 } 293 }
300 #else 294 #else
301 static void irqsoff_print_header(struct seq_fi 295 static void irqsoff_print_header(struct seq_file *s)
302 { 296 {
303 trace_latency_header(s); 297 trace_latency_header(s);
304 } 298 }
305 #endif /* CONFIG_FUNCTION_TRACER */ 299 #endif /* CONFIG_FUNCTION_TRACER */
306 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 300 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
307 301
308 /* 302 /*
309 * Should this new latency be reported/recorde 303 * Should this new latency be reported/recorded?
310 */ 304 */
311 static bool report_latency(struct trace_array 305 static bool report_latency(struct trace_array *tr, u64 delta)
312 { 306 {
313 if (tracing_thresh) { 307 if (tracing_thresh) {
314 if (delta < tracing_thresh) 308 if (delta < tracing_thresh)
315 return false; 309 return false;
316 } else { 310 } else {
317 if (delta <= tr->max_latency) 311 if (delta <= tr->max_latency)
318 return false; 312 return false;
319 } 313 }
320 return true; 314 return true;
321 } 315 }
322 316
323 static void 317 static void
324 check_critical_timing(struct trace_array *tr, 318 check_critical_timing(struct trace_array *tr,
325 struct trace_array_cpu * 319 struct trace_array_cpu *data,
326 unsigned long parent_ip, 320 unsigned long parent_ip,
327 int cpu) 321 int cpu)
328 { 322 {
329 u64 T0, T1, delta; 323 u64 T0, T1, delta;
330 unsigned long flags; 324 unsigned long flags;
331 unsigned int trace_ctx; !! 325 int pc;
332 326
333 T0 = data->preempt_timestamp; 327 T0 = data->preempt_timestamp;
334 T1 = ftrace_now(cpu); 328 T1 = ftrace_now(cpu);
335 delta = T1-T0; 329 delta = T1-T0;
336 330
337 trace_ctx = tracing_gen_ctx(); !! 331 local_save_flags(flags);
>> 332
>> 333 pc = preempt_count();
338 334
339 if (!report_latency(tr, delta)) 335 if (!report_latency(tr, delta))
340 goto out; 336 goto out;
341 337
342 raw_spin_lock_irqsave(&max_trace_lock, 338 raw_spin_lock_irqsave(&max_trace_lock, flags);
343 339
344 /* check if we are still the max laten 340 /* check if we are still the max latency */
345 if (!report_latency(tr, delta)) 341 if (!report_latency(tr, delta))
346 goto out_unlock; 342 goto out_unlock;
347 343
348 __trace_function(tr, CALLER_ADDR0, par !! 344 __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
349 /* Skip 5 functions to get to the irq/ 345 /* Skip 5 functions to get to the irq/preempt enable function */
350 __trace_stack(tr, trace_ctx, 5); !! 346 __trace_stack(tr, flags, 5, pc);
351 347
352 if (data->critical_sequence != max_seq 348 if (data->critical_sequence != max_sequence)
353 goto out_unlock; 349 goto out_unlock;
354 350
355 data->critical_end = parent_ip; 351 data->critical_end = parent_ip;
356 352
357 if (likely(!is_tracing_stopped())) { 353 if (likely(!is_tracing_stopped())) {
358 tr->max_latency = delta; 354 tr->max_latency = delta;
359 update_max_tr_single(tr, curre 355 update_max_tr_single(tr, current, cpu);
360 } 356 }
361 357
362 max_sequence++; 358 max_sequence++;
363 359
364 out_unlock: 360 out_unlock:
365 raw_spin_unlock_irqrestore(&max_trace_ 361 raw_spin_unlock_irqrestore(&max_trace_lock, flags);
366 362
367 out: 363 out:
368 data->critical_sequence = max_sequence 364 data->critical_sequence = max_sequence;
369 data->preempt_timestamp = ftrace_now(c 365 data->preempt_timestamp = ftrace_now(cpu);
370 __trace_function(tr, CALLER_ADDR0, par !! 366 __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
371 } 367 }
372 368
373 static nokprobe_inline void 369 static nokprobe_inline void
374 start_critical_timing(unsigned long ip, unsign !! 370 start_critical_timing(unsigned long ip, unsigned long parent_ip, int pc)
375 { 371 {
376 int cpu; 372 int cpu;
377 struct trace_array *tr = irqsoff_trace 373 struct trace_array *tr = irqsoff_trace;
378 struct trace_array_cpu *data; 374 struct trace_array_cpu *data;
>> 375 unsigned long flags;
379 376
380 if (!tracer_enabled || !tracing_is_ena 377 if (!tracer_enabled || !tracing_is_enabled())
381 return; 378 return;
382 379
383 cpu = raw_smp_processor_id(); 380 cpu = raw_smp_processor_id();
384 381
385 if (per_cpu(tracing_cpu, cpu)) 382 if (per_cpu(tracing_cpu, cpu))
386 return; 383 return;
387 384
388 data = per_cpu_ptr(tr->array_buffer.da 385 data = per_cpu_ptr(tr->array_buffer.data, cpu);
389 386
390 if (unlikely(!data) || atomic_read(&da 387 if (unlikely(!data) || atomic_read(&data->disabled))
391 return; 388 return;
392 389
393 atomic_inc(&data->disabled); 390 atomic_inc(&data->disabled);
394 391
395 data->critical_sequence = max_sequence 392 data->critical_sequence = max_sequence;
396 data->preempt_timestamp = ftrace_now(c 393 data->preempt_timestamp = ftrace_now(cpu);
397 data->critical_start = parent_ip ? : i 394 data->critical_start = parent_ip ? : ip;
398 395
399 __trace_function(tr, ip, parent_ip, tr !! 396 local_save_flags(flags);
>> 397
>> 398 __trace_function(tr, ip, parent_ip, flags, pc);
400 399
401 per_cpu(tracing_cpu, cpu) = 1; 400 per_cpu(tracing_cpu, cpu) = 1;
402 401
403 atomic_dec(&data->disabled); 402 atomic_dec(&data->disabled);
404 } 403 }
405 404
406 static nokprobe_inline void 405 static nokprobe_inline void
407 stop_critical_timing(unsigned long ip, unsigne !! 406 stop_critical_timing(unsigned long ip, unsigned long parent_ip, int pc)
408 { 407 {
409 int cpu; 408 int cpu;
410 struct trace_array *tr = irqsoff_trace 409 struct trace_array *tr = irqsoff_trace;
411 struct trace_array_cpu *data; 410 struct trace_array_cpu *data;
412 unsigned int trace_ctx; !! 411 unsigned long flags;
413 412
414 cpu = raw_smp_processor_id(); 413 cpu = raw_smp_processor_id();
415 /* Always clear the tracing cpu on sto 414 /* Always clear the tracing cpu on stopping the trace */
416 if (unlikely(per_cpu(tracing_cpu, cpu) 415 if (unlikely(per_cpu(tracing_cpu, cpu)))
417 per_cpu(tracing_cpu, cpu) = 0; 416 per_cpu(tracing_cpu, cpu) = 0;
418 else 417 else
419 return; 418 return;
420 419
421 if (!tracer_enabled || !tracing_is_ena 420 if (!tracer_enabled || !tracing_is_enabled())
422 return; 421 return;
423 422
424 data = per_cpu_ptr(tr->array_buffer.da 423 data = per_cpu_ptr(tr->array_buffer.data, cpu);
425 424
426 if (unlikely(!data) || 425 if (unlikely(!data) ||
427 !data->critical_start || atomic_re 426 !data->critical_start || atomic_read(&data->disabled))
428 return; 427 return;
429 428
430 atomic_inc(&data->disabled); 429 atomic_inc(&data->disabled);
431 430
432 trace_ctx = tracing_gen_ctx(); !! 431 local_save_flags(flags);
433 __trace_function(tr, ip, parent_ip, tr !! 432 __trace_function(tr, ip, parent_ip, flags, pc);
434 check_critical_timing(tr, data, parent 433 check_critical_timing(tr, data, parent_ip ? : ip, cpu);
435 data->critical_start = 0; 434 data->critical_start = 0;
436 atomic_dec(&data->disabled); 435 atomic_dec(&data->disabled);
437 } 436 }
438 437
439 /* start and stop critical timings used to for 438 /* start and stop critical timings used to for stoppage (in idle) */
440 void start_critical_timings(void) 439 void start_critical_timings(void)
441 { 440 {
442 if (preempt_trace(preempt_count()) || !! 441 int pc = preempt_count();
443 start_critical_timing(CALLER_A !! 442
>> 443 if (preempt_trace(pc) || irq_trace())
>> 444 start_critical_timing(CALLER_ADDR0, CALLER_ADDR1, pc);
444 } 445 }
445 EXPORT_SYMBOL_GPL(start_critical_timings); 446 EXPORT_SYMBOL_GPL(start_critical_timings);
446 NOKPROBE_SYMBOL(start_critical_timings); 447 NOKPROBE_SYMBOL(start_critical_timings);
447 448
448 void stop_critical_timings(void) 449 void stop_critical_timings(void)
449 { 450 {
450 if (preempt_trace(preempt_count()) || !! 451 int pc = preempt_count();
451 stop_critical_timing(CALLER_AD !! 452
>> 453 if (preempt_trace(pc) || irq_trace())
>> 454 stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1, pc);
452 } 455 }
453 EXPORT_SYMBOL_GPL(stop_critical_timings); 456 EXPORT_SYMBOL_GPL(stop_critical_timings);
454 NOKPROBE_SYMBOL(stop_critical_timings); 457 NOKPROBE_SYMBOL(stop_critical_timings);
455 458
456 #ifdef CONFIG_FUNCTION_TRACER 459 #ifdef CONFIG_FUNCTION_TRACER
457 static bool function_enabled; 460 static bool function_enabled;
458 461
459 static int register_irqsoff_function(struct tr 462 static int register_irqsoff_function(struct trace_array *tr, int graph, int set)
460 { 463 {
461 int ret; 464 int ret;
462 465
463 /* 'set' is set if TRACE_ITER_FUNCTION 466 /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
464 if (function_enabled || (!set && !(tr- 467 if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
465 return 0; 468 return 0;
466 469
467 if (graph) 470 if (graph)
468 ret = register_ftrace_graph(&f 471 ret = register_ftrace_graph(&fgraph_ops);
469 else 472 else
470 ret = register_ftrace_function 473 ret = register_ftrace_function(tr->ops);
471 474
472 if (!ret) 475 if (!ret)
473 function_enabled = true; 476 function_enabled = true;
474 477
475 return ret; 478 return ret;
476 } 479 }
477 480
478 static void unregister_irqsoff_function(struct 481 static void unregister_irqsoff_function(struct trace_array *tr, int graph)
479 { 482 {
480 if (!function_enabled) 483 if (!function_enabled)
481 return; 484 return;
482 485
483 if (graph) 486 if (graph)
484 unregister_ftrace_graph(&fgrap 487 unregister_ftrace_graph(&fgraph_ops);
485 else 488 else
486 unregister_ftrace_function(tr- 489 unregister_ftrace_function(tr->ops);
487 490
488 function_enabled = false; 491 function_enabled = false;
489 } 492 }
490 493
491 static int irqsoff_function_set(struct trace_a 494 static int irqsoff_function_set(struct trace_array *tr, u32 mask, int set)
492 { 495 {
493 if (!(mask & TRACE_ITER_FUNCTION)) 496 if (!(mask & TRACE_ITER_FUNCTION))
494 return 0; 497 return 0;
495 498
496 if (set) 499 if (set)
497 register_irqsoff_function(tr, 500 register_irqsoff_function(tr, is_graph(tr), 1);
498 else 501 else
499 unregister_irqsoff_function(tr 502 unregister_irqsoff_function(tr, is_graph(tr));
500 return 1; 503 return 1;
501 } 504 }
502 #else 505 #else
503 static int register_irqsoff_function(struct tr 506 static int register_irqsoff_function(struct trace_array *tr, int graph, int set)
504 { 507 {
505 return 0; 508 return 0;
506 } 509 }
507 static void unregister_irqsoff_function(struct 510 static void unregister_irqsoff_function(struct trace_array *tr, int graph) { }
508 static inline int irqsoff_function_set(struct 511 static inline int irqsoff_function_set(struct trace_array *tr, u32 mask, int set)
509 { 512 {
510 return 0; 513 return 0;
511 } 514 }
512 #endif /* CONFIG_FUNCTION_TRACER */ 515 #endif /* CONFIG_FUNCTION_TRACER */
513 516
514 static int irqsoff_flag_changed(struct trace_a 517 static int irqsoff_flag_changed(struct trace_array *tr, u32 mask, int set)
515 { 518 {
516 struct tracer *tracer = tr->current_tr 519 struct tracer *tracer = tr->current_trace;
517 520
518 if (irqsoff_function_set(tr, mask, set 521 if (irqsoff_function_set(tr, mask, set))
519 return 0; 522 return 0;
520 523
521 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 524 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
522 if (mask & TRACE_ITER_DISPLAY_GRAPH) 525 if (mask & TRACE_ITER_DISPLAY_GRAPH)
523 return irqsoff_display_graph(t 526 return irqsoff_display_graph(tr, set);
524 #endif 527 #endif
525 528
526 return trace_keep_overwrite(tracer, ma 529 return trace_keep_overwrite(tracer, mask, set);
527 } 530 }
528 531
529 static int start_irqsoff_tracer(struct trace_a 532 static int start_irqsoff_tracer(struct trace_array *tr, int graph)
530 { 533 {
531 int ret; 534 int ret;
532 535
533 ret = register_irqsoff_function(tr, gr 536 ret = register_irqsoff_function(tr, graph, 0);
534 537
535 if (!ret && tracing_is_enabled()) 538 if (!ret && tracing_is_enabled())
536 tracer_enabled = 1; 539 tracer_enabled = 1;
537 else 540 else
538 tracer_enabled = 0; 541 tracer_enabled = 0;
539 542
540 return ret; 543 return ret;
541 } 544 }
542 545
543 static void stop_irqsoff_tracer(struct trace_a 546 static void stop_irqsoff_tracer(struct trace_array *tr, int graph)
544 { 547 {
545 tracer_enabled = 0; 548 tracer_enabled = 0;
546 549
547 unregister_irqsoff_function(tr, graph) 550 unregister_irqsoff_function(tr, graph);
548 } 551 }
549 552
550 static bool irqsoff_busy; 553 static bool irqsoff_busy;
551 554
552 static int __irqsoff_tracer_init(struct trace_ 555 static int __irqsoff_tracer_init(struct trace_array *tr)
553 { 556 {
554 if (irqsoff_busy) 557 if (irqsoff_busy)
555 return -EBUSY; 558 return -EBUSY;
556 559
557 save_flags = tr->trace_flags; 560 save_flags = tr->trace_flags;
558 561
559 /* non overwrite screws up the latency 562 /* non overwrite screws up the latency tracers */
560 set_tracer_flag(tr, TRACE_ITER_OVERWRI 563 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
561 set_tracer_flag(tr, TRACE_ITER_LATENCY 564 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
562 /* without pause, we will produce garb <<
563 set_tracer_flag(tr, TRACE_ITER_PAUSE_O <<
564 565
565 tr->max_latency = 0; 566 tr->max_latency = 0;
566 irqsoff_trace = tr; 567 irqsoff_trace = tr;
567 /* make sure that the tracer is visibl 568 /* make sure that the tracer is visible */
568 smp_wmb(); 569 smp_wmb();
569 570
570 ftrace_init_array_ops(tr, irqsoff_trac 571 ftrace_init_array_ops(tr, irqsoff_tracer_call);
571 572
572 /* Only toplevel instance supports gra 573 /* Only toplevel instance supports graph tracing */
573 if (start_irqsoff_tracer(tr, (tr->flag 574 if (start_irqsoff_tracer(tr, (tr->flags & TRACE_ARRAY_FL_GLOBAL &&
574 is_graph 575 is_graph(tr))))
575 printk(KERN_ERR "failed to sta 576 printk(KERN_ERR "failed to start irqsoff tracer\n");
576 577
577 irqsoff_busy = true; 578 irqsoff_busy = true;
578 return 0; 579 return 0;
579 } 580 }
580 581
581 static void __irqsoff_tracer_reset(struct trac 582 static void __irqsoff_tracer_reset(struct trace_array *tr)
582 { 583 {
583 int lat_flag = save_flags & TRACE_ITER 584 int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
584 int overwrite_flag = save_flags & TRAC 585 int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
585 int pause_flag = save_flags & TRACE_IT <<
586 586
587 stop_irqsoff_tracer(tr, is_graph(tr)); 587 stop_irqsoff_tracer(tr, is_graph(tr));
588 588
589 set_tracer_flag(tr, TRACE_ITER_LATENCY 589 set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
590 set_tracer_flag(tr, TRACE_ITER_OVERWRI 590 set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
591 set_tracer_flag(tr, TRACE_ITER_PAUSE_O <<
592 ftrace_reset_array_ops(tr); 591 ftrace_reset_array_ops(tr);
593 592
594 irqsoff_busy = false; 593 irqsoff_busy = false;
595 } 594 }
596 595
597 static void irqsoff_tracer_start(struct trace_ 596 static void irqsoff_tracer_start(struct trace_array *tr)
598 { 597 {
599 tracer_enabled = 1; 598 tracer_enabled = 1;
600 } 599 }
601 600
602 static void irqsoff_tracer_stop(struct trace_a 601 static void irqsoff_tracer_stop(struct trace_array *tr)
603 { 602 {
604 tracer_enabled = 0; 603 tracer_enabled = 0;
605 } 604 }
606 605
607 #ifdef CONFIG_IRQSOFF_TRACER 606 #ifdef CONFIG_IRQSOFF_TRACER
608 /* 607 /*
609 * We are only interested in hardirq on/off ev 608 * We are only interested in hardirq on/off events:
610 */ 609 */
611 void tracer_hardirqs_on(unsigned long a0, unsi 610 void tracer_hardirqs_on(unsigned long a0, unsigned long a1)
612 { 611 {
613 if (!preempt_trace(preempt_count()) && !! 612 unsigned int pc = preempt_count();
614 stop_critical_timing(a0, a1); !! 613
>> 614 if (!preempt_trace(pc) && irq_trace())
>> 615 stop_critical_timing(a0, a1, pc);
615 } 616 }
616 NOKPROBE_SYMBOL(tracer_hardirqs_on); 617 NOKPROBE_SYMBOL(tracer_hardirqs_on);
617 618
618 void tracer_hardirqs_off(unsigned long a0, uns 619 void tracer_hardirqs_off(unsigned long a0, unsigned long a1)
619 { 620 {
620 if (!preempt_trace(preempt_count()) && !! 621 unsigned int pc = preempt_count();
621 start_critical_timing(a0, a1); !! 622
>> 623 if (!preempt_trace(pc) && irq_trace())
>> 624 start_critical_timing(a0, a1, pc);
622 } 625 }
623 NOKPROBE_SYMBOL(tracer_hardirqs_off); 626 NOKPROBE_SYMBOL(tracer_hardirqs_off);
624 627
625 static int irqsoff_tracer_init(struct trace_ar 628 static int irqsoff_tracer_init(struct trace_array *tr)
626 { 629 {
627 trace_type = TRACER_IRQS_OFF; 630 trace_type = TRACER_IRQS_OFF;
628 631
629 return __irqsoff_tracer_init(tr); 632 return __irqsoff_tracer_init(tr);
630 } 633 }
631 634
632 static void irqsoff_tracer_reset(struct trace_ 635 static void irqsoff_tracer_reset(struct trace_array *tr)
633 { 636 {
634 __irqsoff_tracer_reset(tr); 637 __irqsoff_tracer_reset(tr);
635 } 638 }
636 639
637 static struct tracer irqsoff_tracer __read_mos 640 static struct tracer irqsoff_tracer __read_mostly =
638 { 641 {
639 .name = "irqsoff", 642 .name = "irqsoff",
640 .init = irqsoff_tracer_init, 643 .init = irqsoff_tracer_init,
641 .reset = irqsoff_tracer_reset 644 .reset = irqsoff_tracer_reset,
642 .start = irqsoff_tracer_start 645 .start = irqsoff_tracer_start,
643 .stop = irqsoff_tracer_stop, 646 .stop = irqsoff_tracer_stop,
644 .print_max = true, 647 .print_max = true,
645 .print_header = irqsoff_print_header 648 .print_header = irqsoff_print_header,
646 .print_line = irqsoff_print_line, 649 .print_line = irqsoff_print_line,
647 .flag_changed = irqsoff_flag_changed 650 .flag_changed = irqsoff_flag_changed,
648 #ifdef CONFIG_FTRACE_SELFTEST 651 #ifdef CONFIG_FTRACE_SELFTEST
649 .selftest = trace_selftest_startup_ 652 .selftest = trace_selftest_startup_irqsoff,
650 #endif 653 #endif
651 .open = irqsoff_trace_open, 654 .open = irqsoff_trace_open,
652 .close = irqsoff_trace_close, 655 .close = irqsoff_trace_close,
653 .allow_instances = true, 656 .allow_instances = true,
654 .use_max_tr = true, 657 .use_max_tr = true,
655 }; 658 };
656 #endif /* CONFIG_IRQSOFF_TRACER */ 659 #endif /* CONFIG_IRQSOFF_TRACER */
657 660
658 #ifdef CONFIG_PREEMPT_TRACER 661 #ifdef CONFIG_PREEMPT_TRACER
659 void tracer_preempt_on(unsigned long a0, unsig 662 void tracer_preempt_on(unsigned long a0, unsigned long a1)
660 { 663 {
661 if (preempt_trace(preempt_count()) && !! 664 int pc = preempt_count();
662 stop_critical_timing(a0, a1); !! 665
>> 666 if (preempt_trace(pc) && !irq_trace())
>> 667 stop_critical_timing(a0, a1, pc);
663 } 668 }
664 669
665 void tracer_preempt_off(unsigned long a0, unsi 670 void tracer_preempt_off(unsigned long a0, unsigned long a1)
666 { 671 {
667 if (preempt_trace(preempt_count()) && !! 672 int pc = preempt_count();
668 start_critical_timing(a0, a1); !! 673
>> 674 if (preempt_trace(pc) && !irq_trace())
>> 675 start_critical_timing(a0, a1, pc);
669 } 676 }
670 677
671 static int preemptoff_tracer_init(struct trace 678 static int preemptoff_tracer_init(struct trace_array *tr)
672 { 679 {
673 trace_type = TRACER_PREEMPT_OFF; 680 trace_type = TRACER_PREEMPT_OFF;
674 681
675 return __irqsoff_tracer_init(tr); 682 return __irqsoff_tracer_init(tr);
676 } 683 }
677 684
678 static void preemptoff_tracer_reset(struct tra 685 static void preemptoff_tracer_reset(struct trace_array *tr)
679 { 686 {
680 __irqsoff_tracer_reset(tr); 687 __irqsoff_tracer_reset(tr);
681 } 688 }
682 689
683 static struct tracer preemptoff_tracer __read_ 690 static struct tracer preemptoff_tracer __read_mostly =
684 { 691 {
685 .name = "preemptoff", 692 .name = "preemptoff",
686 .init = preemptoff_tracer_in 693 .init = preemptoff_tracer_init,
687 .reset = preemptoff_tracer_re 694 .reset = preemptoff_tracer_reset,
688 .start = irqsoff_tracer_start 695 .start = irqsoff_tracer_start,
689 .stop = irqsoff_tracer_stop, 696 .stop = irqsoff_tracer_stop,
690 .print_max = true, 697 .print_max = true,
691 .print_header = irqsoff_print_header 698 .print_header = irqsoff_print_header,
692 .print_line = irqsoff_print_line, 699 .print_line = irqsoff_print_line,
693 .flag_changed = irqsoff_flag_changed 700 .flag_changed = irqsoff_flag_changed,
694 #ifdef CONFIG_FTRACE_SELFTEST 701 #ifdef CONFIG_FTRACE_SELFTEST
695 .selftest = trace_selftest_startup_ 702 .selftest = trace_selftest_startup_preemptoff,
696 #endif 703 #endif
697 .open = irqsoff_trace_open, 704 .open = irqsoff_trace_open,
698 .close = irqsoff_trace_close, 705 .close = irqsoff_trace_close,
699 .allow_instances = true, 706 .allow_instances = true,
700 .use_max_tr = true, 707 .use_max_tr = true,
701 }; 708 };
702 #endif /* CONFIG_PREEMPT_TRACER */ 709 #endif /* CONFIG_PREEMPT_TRACER */
703 710
704 #if defined(CONFIG_IRQSOFF_TRACER) && defined( 711 #if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
705 712
706 static int preemptirqsoff_tracer_init(struct t 713 static int preemptirqsoff_tracer_init(struct trace_array *tr)
707 { 714 {
708 trace_type = TRACER_IRQS_OFF | TRACER_ 715 trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
709 716
710 return __irqsoff_tracer_init(tr); 717 return __irqsoff_tracer_init(tr);
711 } 718 }
712 719
713 static void preemptirqsoff_tracer_reset(struct 720 static void preemptirqsoff_tracer_reset(struct trace_array *tr)
714 { 721 {
715 __irqsoff_tracer_reset(tr); 722 __irqsoff_tracer_reset(tr);
716 } 723 }
717 724
718 static struct tracer preemptirqsoff_tracer __r 725 static struct tracer preemptirqsoff_tracer __read_mostly =
719 { 726 {
720 .name = "preemptirqsoff", 727 .name = "preemptirqsoff",
721 .init = preemptirqsoff_trace 728 .init = preemptirqsoff_tracer_init,
722 .reset = preemptirqsoff_trace 729 .reset = preemptirqsoff_tracer_reset,
723 .start = irqsoff_tracer_start 730 .start = irqsoff_tracer_start,
724 .stop = irqsoff_tracer_stop, 731 .stop = irqsoff_tracer_stop,
725 .print_max = true, 732 .print_max = true,
726 .print_header = irqsoff_print_header 733 .print_header = irqsoff_print_header,
727 .print_line = irqsoff_print_line, 734 .print_line = irqsoff_print_line,
728 .flag_changed = irqsoff_flag_changed 735 .flag_changed = irqsoff_flag_changed,
729 #ifdef CONFIG_FTRACE_SELFTEST 736 #ifdef CONFIG_FTRACE_SELFTEST
730 .selftest = trace_selftest_startup_ 737 .selftest = trace_selftest_startup_preemptirqsoff,
731 #endif 738 #endif
732 .open = irqsoff_trace_open, 739 .open = irqsoff_trace_open,
733 .close = irqsoff_trace_close, 740 .close = irqsoff_trace_close,
734 .allow_instances = true, 741 .allow_instances = true,
735 .use_max_tr = true, 742 .use_max_tr = true,
736 }; 743 };
737 #endif 744 #endif
738 745
739 __init static int init_irqsoff_tracer(void) 746 __init static int init_irqsoff_tracer(void)
740 { 747 {
741 #ifdef CONFIG_IRQSOFF_TRACER 748 #ifdef CONFIG_IRQSOFF_TRACER
742 register_tracer(&irqsoff_tracer); 749 register_tracer(&irqsoff_tracer);
743 #endif 750 #endif
744 #ifdef CONFIG_PREEMPT_TRACER 751 #ifdef CONFIG_PREEMPT_TRACER
745 register_tracer(&preemptoff_tracer); 752 register_tracer(&preemptoff_tracer);
746 #endif 753 #endif
747 #if defined(CONFIG_IRQSOFF_TRACER) && defined( 754 #if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
748 register_tracer(&preemptirqsoff_tracer 755 register_tracer(&preemptirqsoff_tracer);
749 #endif 756 #endif
750 757
751 return 0; 758 return 0;
752 } 759 }
753 core_initcall(init_irqsoff_tracer); 760 core_initcall(init_irqsoff_tracer);
754 #endif /* IRQSOFF_TRACER || PREEMPTOFF_TRACER 761 #endif /* IRQSOFF_TRACER || PREEMPTOFF_TRACER */
755 762
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.