1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * trace context switch
4 *
5 * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
6 *
7 */
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/uaccess.h>
11 #include <linux/kmemleak.h>
12 #include <linux/ftrace.h>
13 #include <trace/events/sched.h>
14
15 #include "trace.h"
16
17 #define RECORD_CMDLINE 1
18 #define RECORD_TGID 2
19
20 static int sched_cmdline_ref;
21 static int sched_tgid_ref;
22 static DEFINE_MUTEX(sched_register_mutex);
23
24 static void
25 probe_sched_switch(void *ignore, bool preempt,
26 struct task_struct *prev, struct task_struct *next,
27 unsigned int prev_state)
28 {
29 int flags;
30
31 flags = (RECORD_TGID * !!sched_tgid_ref) +
32 (RECORD_CMDLINE * !!sched_cmdline_ref);
33
34 if (!flags)
35 return;
36 tracing_record_taskinfo_sched_switch(prev, next, flags);
37 }
38
39 static void
40 probe_sched_wakeup(void *ignore, struct task_struct *wakee)
41 {
42 int flags;
43
44 flags = (RECORD_TGID * !!sched_tgid_ref) +
45 (RECORD_CMDLINE * !!sched_cmdline_ref);
46
47 if (!flags)
48 return;
49 tracing_record_taskinfo_sched_switch(current, wakee, flags);
50 }
51
52 static int tracing_sched_register(void)
53 {
54 int ret;
55
56 ret = register_trace_sched_wakeup(probe_sched_wakeup, NULL);
57 if (ret) {
58 pr_info("wakeup trace: Couldn't activate tracepoint"
59 " probe to kernel_sched_wakeup\n");
60 return ret;
61 }
62
63 ret = register_trace_sched_wakeup_new(probe_sched_wakeup, NULL);
64 if (ret) {
65 pr_info("wakeup trace: Couldn't activate tracepoint"
66 " probe to kernel_sched_wakeup_new\n");
67 goto fail_deprobe;
68 }
69
70 ret = register_trace_sched_switch(probe_sched_switch, NULL);
71 if (ret) {
72 pr_info("sched trace: Couldn't activate tracepoint"
73 " probe to kernel_sched_switch\n");
74 goto fail_deprobe_wake_new;
75 }
76
77 return ret;
78 fail_deprobe_wake_new:
79 unregister_trace_sched_wakeup_new(probe_sched_wakeup, NULL);
80 fail_deprobe:
81 unregister_trace_sched_wakeup(probe_sched_wakeup, NULL);
82 return ret;
83 }
84
85 static void tracing_sched_unregister(void)
86 {
87 unregister_trace_sched_switch(probe_sched_switch, NULL);
88 unregister_trace_sched_wakeup_new(probe_sched_wakeup, NULL);
89 unregister_trace_sched_wakeup(probe_sched_wakeup, NULL);
90 }
91
92 static void tracing_start_sched_switch(int ops)
93 {
94 bool sched_register;
95
96 mutex_lock(&sched_register_mutex);
97 sched_register = (!sched_cmdline_ref && !sched_tgid_ref);
98
99 switch (ops) {
100 case RECORD_CMDLINE:
101 sched_cmdline_ref++;
102 break;
103
104 case RECORD_TGID:
105 sched_tgid_ref++;
106 break;
107 }
108
109 if (sched_register && (sched_cmdline_ref || sched_tgid_ref))
110 tracing_sched_register();
111 mutex_unlock(&sched_register_mutex);
112 }
113
114 static void tracing_stop_sched_switch(int ops)
115 {
116 mutex_lock(&sched_register_mutex);
117
118 switch (ops) {
119 case RECORD_CMDLINE:
120 sched_cmdline_ref--;
121 break;
122
123 case RECORD_TGID:
124 sched_tgid_ref--;
125 break;
126 }
127
128 if (!sched_cmdline_ref && !sched_tgid_ref)
129 tracing_sched_unregister();
130 mutex_unlock(&sched_register_mutex);
131 }
132
133 void tracing_start_cmdline_record(void)
134 {
135 tracing_start_sched_switch(RECORD_CMDLINE);
136 }
137
138 void tracing_stop_cmdline_record(void)
139 {
140 tracing_stop_sched_switch(RECORD_CMDLINE);
141 }
142
143 void tracing_start_tgid_record(void)
144 {
145 tracing_start_sched_switch(RECORD_TGID);
146 }
147
148 void tracing_stop_tgid_record(void)
149 {
150 tracing_stop_sched_switch(RECORD_TGID);
151 }
152
153 /*
154 * The tgid_map array maps from pid to tgid; i.e. the value stored at index i
155 * is the tgid last observed corresponding to pid=i.
156 */
157 static int *tgid_map;
158
159 /* The maximum valid index into tgid_map. */
160 static size_t tgid_map_max;
161
162 #define SAVED_CMDLINES_DEFAULT 128
163 #define NO_CMDLINE_MAP UINT_MAX
164 /*
165 * Preemption must be disabled before acquiring trace_cmdline_lock.
166 * The various trace_arrays' max_lock must be acquired in a context
167 * where interrupt is disabled.
168 */
169 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
170 struct saved_cmdlines_buffer {
171 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
172 unsigned *map_cmdline_to_pid;
173 unsigned cmdline_num;
174 int cmdline_idx;
175 char saved_cmdlines[];
176 };
177 static struct saved_cmdlines_buffer *savedcmd;
178
179 /* Holds the size of a cmdline and pid element */
180 #define SAVED_CMDLINE_MAP_ELEMENT_SIZE(s) \
181 (TASK_COMM_LEN + sizeof((s)->map_cmdline_to_pid[0]))
182
183 static inline char *get_saved_cmdlines(int idx)
184 {
185 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
186 }
187
188 static inline void set_cmdline(int idx, const char *cmdline)
189 {
190 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
191 }
192
193 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
194 {
195 int order = get_order(sizeof(*s) + s->cmdline_num * TASK_COMM_LEN);
196
197 kmemleak_free(s);
198 free_pages((unsigned long)s, order);
199 }
200
201 static struct saved_cmdlines_buffer *allocate_cmdlines_buffer(unsigned int val)
202 {
203 struct saved_cmdlines_buffer *s;
204 struct page *page;
205 int orig_size, size;
206 int order;
207
208 /* Figure out how much is needed to hold the given number of cmdlines */
209 orig_size = sizeof(*s) + val * SAVED_CMDLINE_MAP_ELEMENT_SIZE(s);
210 order = get_order(orig_size);
211 size = 1 << (order + PAGE_SHIFT);
212 page = alloc_pages(GFP_KERNEL, order);
213 if (!page)
214 return NULL;
215
216 s = page_address(page);
217 kmemleak_alloc(s, size, 1, GFP_KERNEL);
218 memset(s, 0, sizeof(*s));
219
220 /* Round up to actual allocation */
221 val = (size - sizeof(*s)) / SAVED_CMDLINE_MAP_ELEMENT_SIZE(s);
222 s->cmdline_num = val;
223
224 /* Place map_cmdline_to_pid array right after saved_cmdlines */
225 s->map_cmdline_to_pid = (unsigned *)&s->saved_cmdlines[val * TASK_COMM_LEN];
226
227 s->cmdline_idx = 0;
228 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
229 sizeof(s->map_pid_to_cmdline));
230 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
231 val * sizeof(*s->map_cmdline_to_pid));
232
233 return s;
234 }
235
236 int trace_create_savedcmd(void)
237 {
238 savedcmd = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT);
239
240 return savedcmd ? 0 : -ENOMEM;
241 }
242
243 int trace_save_cmdline(struct task_struct *tsk)
244 {
245 unsigned tpid, idx;
246
247 /* treat recording of idle task as a success */
248 if (!tsk->pid)
249 return 1;
250
251 tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
252
253 /*
254 * It's not the end of the world if we don't get
255 * the lock, but we also don't want to spin
256 * nor do we want to disable interrupts,
257 * so if we miss here, then better luck next time.
258 *
259 * This is called within the scheduler and wake up, so interrupts
260 * had better been disabled and run queue lock been held.
261 */
262 lockdep_assert_preemption_disabled();
263 if (!arch_spin_trylock(&trace_cmdline_lock))
264 return 0;
265
266 idx = savedcmd->map_pid_to_cmdline[tpid];
267 if (idx == NO_CMDLINE_MAP) {
268 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
269
270 savedcmd->map_pid_to_cmdline[tpid] = idx;
271 savedcmd->cmdline_idx = idx;
272 }
273
274 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
275 set_cmdline(idx, tsk->comm);
276
277 arch_spin_unlock(&trace_cmdline_lock);
278
279 return 1;
280 }
281
282 static void __trace_find_cmdline(int pid, char comm[])
283 {
284 unsigned map;
285 int tpid;
286
287 if (!pid) {
288 strcpy(comm, "<idle>");
289 return;
290 }
291
292 if (WARN_ON_ONCE(pid < 0)) {
293 strcpy(comm, "<XXX>");
294 return;
295 }
296
297 tpid = pid & (PID_MAX_DEFAULT - 1);
298 map = savedcmd->map_pid_to_cmdline[tpid];
299 if (map != NO_CMDLINE_MAP) {
300 tpid = savedcmd->map_cmdline_to_pid[map];
301 if (tpid == pid) {
302 strscpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
303 return;
304 }
305 }
306 strcpy(comm, "<...>");
307 }
308
309 void trace_find_cmdline(int pid, char comm[])
310 {
311 preempt_disable();
312 arch_spin_lock(&trace_cmdline_lock);
313
314 __trace_find_cmdline(pid, comm);
315
316 arch_spin_unlock(&trace_cmdline_lock);
317 preempt_enable();
318 }
319
320 static int *trace_find_tgid_ptr(int pid)
321 {
322 /*
323 * Pairs with the smp_store_release in set_tracer_flag() to ensure that
324 * if we observe a non-NULL tgid_map then we also observe the correct
325 * tgid_map_max.
326 */
327 int *map = smp_load_acquire(&tgid_map);
328
329 if (unlikely(!map || pid > tgid_map_max))
330 return NULL;
331
332 return &map[pid];
333 }
334
335 int trace_find_tgid(int pid)
336 {
337 int *ptr = trace_find_tgid_ptr(pid);
338
339 return ptr ? *ptr : 0;
340 }
341
342 static int trace_save_tgid(struct task_struct *tsk)
343 {
344 int *ptr;
345
346 /* treat recording of idle task as a success */
347 if (!tsk->pid)
348 return 1;
349
350 ptr = trace_find_tgid_ptr(tsk->pid);
351 if (!ptr)
352 return 0;
353
354 *ptr = tsk->tgid;
355 return 1;
356 }
357
358 static bool tracing_record_taskinfo_skip(int flags)
359 {
360 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
361 return true;
362 if (!__this_cpu_read(trace_taskinfo_save))
363 return true;
364 return false;
365 }
366
367 /**
368 * tracing_record_taskinfo - record the task info of a task
369 *
370 * @task: task to record
371 * @flags: TRACE_RECORD_CMDLINE for recording comm
372 * TRACE_RECORD_TGID for recording tgid
373 */
374 void tracing_record_taskinfo(struct task_struct *task, int flags)
375 {
376 bool done;
377
378 if (tracing_record_taskinfo_skip(flags))
379 return;
380
381 /*
382 * Record as much task information as possible. If some fail, continue
383 * to try to record the others.
384 */
385 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
386 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
387
388 /* If recording any information failed, retry again soon. */
389 if (!done)
390 return;
391
392 __this_cpu_write(trace_taskinfo_save, false);
393 }
394
395 /**
396 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
397 *
398 * @prev: previous task during sched_switch
399 * @next: next task during sched_switch
400 * @flags: TRACE_RECORD_CMDLINE for recording comm
401 * TRACE_RECORD_TGID for recording tgid
402 */
403 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
404 struct task_struct *next, int flags)
405 {
406 bool done;
407
408 if (tracing_record_taskinfo_skip(flags))
409 return;
410
411 /*
412 * Record as much task information as possible. If some fail, continue
413 * to try to record the others.
414 */
415 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
416 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
417 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
418 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
419
420 /* If recording any information failed, retry again soon. */
421 if (!done)
422 return;
423
424 __this_cpu_write(trace_taskinfo_save, false);
425 }
426
427 /* Helpers to record a specific task information */
428 void tracing_record_cmdline(struct task_struct *task)
429 {
430 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
431 }
432
433 void tracing_record_tgid(struct task_struct *task)
434 {
435 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
436 }
437
438 int trace_alloc_tgid_map(void)
439 {
440 int *map;
441
442 if (tgid_map)
443 return 0;
444
445 tgid_map_max = pid_max;
446 map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map),
447 GFP_KERNEL);
448 if (!map)
449 return -ENOMEM;
450
451 /*
452 * Pairs with smp_load_acquire() in
453 * trace_find_tgid_ptr() to ensure that if it observes
454 * the tgid_map we just allocated then it also observes
455 * the corresponding tgid_map_max value.
456 */
457 smp_store_release(&tgid_map, map);
458 return 0;
459 }
460
461 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
462 {
463 int pid = ++(*pos);
464
465 return trace_find_tgid_ptr(pid);
466 }
467
468 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
469 {
470 int pid = *pos;
471
472 return trace_find_tgid_ptr(pid);
473 }
474
475 static void saved_tgids_stop(struct seq_file *m, void *v)
476 {
477 }
478
479 static int saved_tgids_show(struct seq_file *m, void *v)
480 {
481 int *entry = (int *)v;
482 int pid = entry - tgid_map;
483 int tgid = *entry;
484
485 if (tgid == 0)
486 return SEQ_SKIP;
487
488 seq_printf(m, "%d %d\n", pid, tgid);
489 return 0;
490 }
491
492 static const struct seq_operations tracing_saved_tgids_seq_ops = {
493 .start = saved_tgids_start,
494 .stop = saved_tgids_stop,
495 .next = saved_tgids_next,
496 .show = saved_tgids_show,
497 };
498
499 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
500 {
501 int ret;
502
503 ret = tracing_check_open_get_tr(NULL);
504 if (ret)
505 return ret;
506
507 return seq_open(filp, &tracing_saved_tgids_seq_ops);
508 }
509
510
511 const struct file_operations tracing_saved_tgids_fops = {
512 .open = tracing_saved_tgids_open,
513 .read = seq_read,
514 .llseek = seq_lseek,
515 .release = seq_release,
516 };
517
518 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
519 {
520 unsigned int *ptr = v;
521
522 if (*pos || m->count)
523 ptr++;
524
525 (*pos)++;
526
527 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
528 ptr++) {
529 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
530 continue;
531
532 return ptr;
533 }
534
535 return NULL;
536 }
537
538 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
539 {
540 void *v;
541 loff_t l = 0;
542
543 preempt_disable();
544 arch_spin_lock(&trace_cmdline_lock);
545
546 v = &savedcmd->map_cmdline_to_pid[0];
547 while (l <= *pos) {
548 v = saved_cmdlines_next(m, v, &l);
549 if (!v)
550 return NULL;
551 }
552
553 return v;
554 }
555
556 static void saved_cmdlines_stop(struct seq_file *m, void *v)
557 {
558 arch_spin_unlock(&trace_cmdline_lock);
559 preempt_enable();
560 }
561
562 static int saved_cmdlines_show(struct seq_file *m, void *v)
563 {
564 char buf[TASK_COMM_LEN];
565 unsigned int *pid = v;
566
567 __trace_find_cmdline(*pid, buf);
568 seq_printf(m, "%d %s\n", *pid, buf);
569 return 0;
570 }
571
572 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
573 .start = saved_cmdlines_start,
574 .next = saved_cmdlines_next,
575 .stop = saved_cmdlines_stop,
576 .show = saved_cmdlines_show,
577 };
578
579 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
580 {
581 int ret;
582
583 ret = tracing_check_open_get_tr(NULL);
584 if (ret)
585 return ret;
586
587 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
588 }
589
590 const struct file_operations tracing_saved_cmdlines_fops = {
591 .open = tracing_saved_cmdlines_open,
592 .read = seq_read,
593 .llseek = seq_lseek,
594 .release = seq_release,
595 };
596
597 static ssize_t
598 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
599 size_t cnt, loff_t *ppos)
600 {
601 char buf[64];
602 int r;
603
604 preempt_disable();
605 arch_spin_lock(&trace_cmdline_lock);
606 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
607 arch_spin_unlock(&trace_cmdline_lock);
608 preempt_enable();
609
610 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
611 }
612
613 void trace_free_saved_cmdlines_buffer(void)
614 {
615 free_saved_cmdlines_buffer(savedcmd);
616 }
617
618 static int tracing_resize_saved_cmdlines(unsigned int val)
619 {
620 struct saved_cmdlines_buffer *s, *savedcmd_temp;
621
622 s = allocate_cmdlines_buffer(val);
623 if (!s)
624 return -ENOMEM;
625
626 preempt_disable();
627 arch_spin_lock(&trace_cmdline_lock);
628 savedcmd_temp = savedcmd;
629 savedcmd = s;
630 arch_spin_unlock(&trace_cmdline_lock);
631 preempt_enable();
632 free_saved_cmdlines_buffer(savedcmd_temp);
633
634 return 0;
635 }
636
637 static ssize_t
638 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
639 size_t cnt, loff_t *ppos)
640 {
641 unsigned long val;
642 int ret;
643
644 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
645 if (ret)
646 return ret;
647
648 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
649 if (!val || val > PID_MAX_DEFAULT)
650 return -EINVAL;
651
652 ret = tracing_resize_saved_cmdlines((unsigned int)val);
653 if (ret < 0)
654 return ret;
655
656 *ppos += cnt;
657
658 return cnt;
659 }
660
661 const struct file_operations tracing_saved_cmdlines_size_fops = {
662 .open = tracing_open_generic,
663 .read = tracing_saved_cmdlines_size_read,
664 .write = tracing_saved_cmdlines_size_write,
665 };
666
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.