1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * builtin-inject.c
4 *
5 * Builtin inject command: Examine the live mode (stdin) event stream
6 * and repipe it to stdout while optionally injecting additional
7 * events into it.
8 */
9 #include "builtin.h"
10
11 #include "util/color.h"
12 #include "util/dso.h"
13 #include "util/vdso.h"
14 #include "util/evlist.h"
15 #include "util/evsel.h"
16 #include "util/map.h"
17 #include "util/session.h"
18 #include "util/tool.h"
19 #include "util/debug.h"
20 #include "util/build-id.h"
21 #include "util/data.h"
22 #include "util/auxtrace.h"
23 #include "util/jit.h"
24 #include "util/string2.h"
25 #include "util/symbol.h"
26 #include "util/synthetic-events.h"
27 #include "util/thread.h"
28 #include "util/namespaces.h"
29 #include "util/util.h"
30 #include "util/tsc.h"
31
32 #include <internal/lib.h>
33
34 #include <linux/err.h>
35 #include <subcmd/parse-options.h>
36 #include <uapi/linux/mman.h> /* To get things like MAP_HUGETLB even on older libc headers */
37
38 #include <linux/list.h>
39 #include <linux/string.h>
40 #include <linux/zalloc.h>
41 #include <linux/hash.h>
42 #include <ctype.h>
43 #include <errno.h>
44 #include <signal.h>
45 #include <inttypes.h>
46
47 struct guest_event {
48 struct perf_sample sample;
49 union perf_event *event;
50 char *event_buf;
51 };
52
53 struct guest_id {
54 /* hlist_node must be first, see free_hlist() */
55 struct hlist_node node;
56 u64 id;
57 u64 host_id;
58 u32 vcpu;
59 };
60
61 struct guest_tid {
62 /* hlist_node must be first, see free_hlist() */
63 struct hlist_node node;
64 /* Thread ID of QEMU thread */
65 u32 tid;
66 u32 vcpu;
67 };
68
69 struct guest_vcpu {
70 /* Current host CPU */
71 u32 cpu;
72 /* Thread ID of QEMU thread */
73 u32 tid;
74 };
75
76 struct guest_session {
77 char *perf_data_file;
78 u32 machine_pid;
79 u64 time_offset;
80 double time_scale;
81 struct perf_tool tool;
82 struct perf_data data;
83 struct perf_session *session;
84 char *tmp_file_name;
85 int tmp_fd;
86 struct perf_tsc_conversion host_tc;
87 struct perf_tsc_conversion guest_tc;
88 bool copy_kcore_dir;
89 bool have_tc;
90 bool fetched;
91 bool ready;
92 u16 dflt_id_hdr_size;
93 u64 dflt_id;
94 u64 highest_id;
95 /* Array of guest_vcpu */
96 struct guest_vcpu *vcpu;
97 size_t vcpu_cnt;
98 /* Hash table for guest_id */
99 struct hlist_head heads[PERF_EVLIST__HLIST_SIZE];
100 /* Hash table for guest_tid */
101 struct hlist_head tids[PERF_EVLIST__HLIST_SIZE];
102 /* Place to stash next guest event */
103 struct guest_event ev;
104 };
105
106 struct perf_inject {
107 struct perf_tool tool;
108 struct perf_session *session;
109 bool build_ids;
110 bool build_id_all;
111 bool sched_stat;
112 bool have_auxtrace;
113 bool strip;
114 bool jit_mode;
115 bool in_place_update;
116 bool in_place_update_dry_run;
117 bool is_pipe;
118 bool copy_kcore_dir;
119 const char *input_name;
120 struct perf_data output;
121 u64 bytes_written;
122 u64 aux_id;
123 struct list_head samples;
124 struct itrace_synth_opts itrace_synth_opts;
125 char *event_copy;
126 struct perf_file_section secs[HEADER_FEAT_BITS];
127 struct guest_session guest_session;
128 struct strlist *known_build_ids;
129 };
130
131 struct event_entry {
132 struct list_head node;
133 u32 tid;
134 union perf_event event[];
135 };
136
137 static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool,
138 struct machine *machine, u8 cpumode, u32 flags);
139
140 static int output_bytes(struct perf_inject *inject, void *buf, size_t sz)
141 {
142 ssize_t size;
143
144 size = perf_data__write(&inject->output, buf, sz);
145 if (size < 0)
146 return -errno;
147
148 inject->bytes_written += size;
149 return 0;
150 }
151
152 static int perf_event__repipe_synth(struct perf_tool *tool,
153 union perf_event *event)
154 {
155 struct perf_inject *inject = container_of(tool, struct perf_inject,
156 tool);
157
158 return output_bytes(inject, event, event->header.size);
159 }
160
161 static int perf_event__repipe_oe_synth(struct perf_tool *tool,
162 union perf_event *event,
163 struct ordered_events *oe __maybe_unused)
164 {
165 return perf_event__repipe_synth(tool, event);
166 }
167
168 #ifdef HAVE_JITDUMP
169 static int perf_event__drop_oe(struct perf_tool *tool __maybe_unused,
170 union perf_event *event __maybe_unused,
171 struct ordered_events *oe __maybe_unused)
172 {
173 return 0;
174 }
175 #endif
176
177 static int perf_event__repipe_op2_synth(struct perf_session *session,
178 union perf_event *event)
179 {
180 return perf_event__repipe_synth(session->tool, event);
181 }
182
183 static int perf_event__repipe_op4_synth(struct perf_session *session,
184 union perf_event *event,
185 u64 data __maybe_unused,
186 const char *str __maybe_unused)
187 {
188 return perf_event__repipe_synth(session->tool, event);
189 }
190
191 static int perf_event__repipe_attr(struct perf_tool *tool,
192 union perf_event *event,
193 struct evlist **pevlist)
194 {
195 struct perf_inject *inject = container_of(tool, struct perf_inject,
196 tool);
197 int ret;
198
199 ret = perf_event__process_attr(tool, event, pevlist);
200 if (ret)
201 return ret;
202
203 if (!inject->is_pipe)
204 return 0;
205
206 return perf_event__repipe_synth(tool, event);
207 }
208
209 static int perf_event__repipe_event_update(struct perf_tool *tool,
210 union perf_event *event,
211 struct evlist **pevlist __maybe_unused)
212 {
213 return perf_event__repipe_synth(tool, event);
214 }
215
216 #ifdef HAVE_AUXTRACE_SUPPORT
217
218 static int copy_bytes(struct perf_inject *inject, struct perf_data *data, off_t size)
219 {
220 char buf[4096];
221 ssize_t ssz;
222 int ret;
223
224 while (size > 0) {
225 ssz = perf_data__read(data, buf, min(size, (off_t)sizeof(buf)));
226 if (ssz < 0)
227 return -errno;
228 ret = output_bytes(inject, buf, ssz);
229 if (ret)
230 return ret;
231 size -= ssz;
232 }
233
234 return 0;
235 }
236
237 static s64 perf_event__repipe_auxtrace(struct perf_session *session,
238 union perf_event *event)
239 {
240 struct perf_tool *tool = session->tool;
241 struct perf_inject *inject = container_of(tool, struct perf_inject,
242 tool);
243 int ret;
244
245 inject->have_auxtrace = true;
246
247 if (!inject->output.is_pipe) {
248 off_t offset;
249
250 offset = lseek(inject->output.file.fd, 0, SEEK_CUR);
251 if (offset == -1)
252 return -errno;
253 ret = auxtrace_index__auxtrace_event(&session->auxtrace_index,
254 event, offset);
255 if (ret < 0)
256 return ret;
257 }
258
259 if (perf_data__is_pipe(session->data) || !session->one_mmap) {
260 ret = output_bytes(inject, event, event->header.size);
261 if (ret < 0)
262 return ret;
263 ret = copy_bytes(inject, session->data,
264 event->auxtrace.size);
265 } else {
266 ret = output_bytes(inject, event,
267 event->header.size + event->auxtrace.size);
268 }
269 if (ret < 0)
270 return ret;
271
272 return event->auxtrace.size;
273 }
274
275 #else
276
277 static s64
278 perf_event__repipe_auxtrace(struct perf_session *session __maybe_unused,
279 union perf_event *event __maybe_unused)
280 {
281 pr_err("AUX area tracing not supported\n");
282 return -EINVAL;
283 }
284
285 #endif
286
287 static int perf_event__repipe(struct perf_tool *tool,
288 union perf_event *event,
289 struct perf_sample *sample __maybe_unused,
290 struct machine *machine __maybe_unused)
291 {
292 return perf_event__repipe_synth(tool, event);
293 }
294
295 static int perf_event__drop(struct perf_tool *tool __maybe_unused,
296 union perf_event *event __maybe_unused,
297 struct perf_sample *sample __maybe_unused,
298 struct machine *machine __maybe_unused)
299 {
300 return 0;
301 }
302
303 static int perf_event__drop_aux(struct perf_tool *tool,
304 union perf_event *event __maybe_unused,
305 struct perf_sample *sample,
306 struct machine *machine __maybe_unused)
307 {
308 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
309
310 if (!inject->aux_id)
311 inject->aux_id = sample->id;
312
313 return 0;
314 }
315
316 static union perf_event *
317 perf_inject__cut_auxtrace_sample(struct perf_inject *inject,
318 union perf_event *event,
319 struct perf_sample *sample)
320 {
321 size_t sz1 = sample->aux_sample.data - (void *)event;
322 size_t sz2 = event->header.size - sample->aux_sample.size - sz1;
323 union perf_event *ev;
324
325 if (inject->event_copy == NULL) {
326 inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE);
327 if (!inject->event_copy)
328 return ERR_PTR(-ENOMEM);
329 }
330 ev = (union perf_event *)inject->event_copy;
331 if (sz1 > event->header.size || sz2 > event->header.size ||
332 sz1 + sz2 > event->header.size ||
333 sz1 < sizeof(struct perf_event_header) + sizeof(u64))
334 return event;
335
336 memcpy(ev, event, sz1);
337 memcpy((void *)ev + sz1, (void *)event + event->header.size - sz2, sz2);
338 ev->header.size = sz1 + sz2;
339 ((u64 *)((void *)ev + sz1))[-1] = 0;
340
341 return ev;
342 }
343
344 typedef int (*inject_handler)(struct perf_tool *tool,
345 union perf_event *event,
346 struct perf_sample *sample,
347 struct evsel *evsel,
348 struct machine *machine);
349
350 static int perf_event__repipe_sample(struct perf_tool *tool,
351 union perf_event *event,
352 struct perf_sample *sample,
353 struct evsel *evsel,
354 struct machine *machine)
355 {
356 struct perf_inject *inject = container_of(tool, struct perf_inject,
357 tool);
358
359 if (evsel && evsel->handler) {
360 inject_handler f = evsel->handler;
361 return f(tool, event, sample, evsel, machine);
362 }
363
364 build_id__mark_dso_hit(tool, event, sample, evsel, machine);
365
366 if (inject->itrace_synth_opts.set && sample->aux_sample.size) {
367 event = perf_inject__cut_auxtrace_sample(inject, event, sample);
368 if (IS_ERR(event))
369 return PTR_ERR(event);
370 }
371
372 return perf_event__repipe_synth(tool, event);
373 }
374
375 static int perf_event__repipe_mmap(struct perf_tool *tool,
376 union perf_event *event,
377 struct perf_sample *sample,
378 struct machine *machine)
379 {
380 int err;
381
382 err = perf_event__process_mmap(tool, event, sample, machine);
383 perf_event__repipe(tool, event, sample, machine);
384
385 return err;
386 }
387
388 #ifdef HAVE_JITDUMP
389 static int perf_event__jit_repipe_mmap(struct perf_tool *tool,
390 union perf_event *event,
391 struct perf_sample *sample,
392 struct machine *machine)
393 {
394 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
395 u64 n = 0;
396 int ret;
397
398 /*
399 * if jit marker, then inject jit mmaps and generate ELF images
400 */
401 ret = jit_process(inject->session, &inject->output, machine,
402 event->mmap.filename, event->mmap.pid, event->mmap.tid, &n);
403 if (ret < 0)
404 return ret;
405 if (ret) {
406 inject->bytes_written += n;
407 return 0;
408 }
409 return perf_event__repipe_mmap(tool, event, sample, machine);
410 }
411 #endif
412
413 static struct dso *findnew_dso(int pid, int tid, const char *filename,
414 struct dso_id *id, struct machine *machine)
415 {
416 struct thread *thread;
417 struct nsinfo *nsi = NULL;
418 struct nsinfo *nnsi;
419 struct dso *dso;
420 bool vdso;
421
422 thread = machine__findnew_thread(machine, pid, tid);
423 if (thread == NULL) {
424 pr_err("cannot find or create a task %d/%d.\n", tid, pid);
425 return NULL;
426 }
427
428 vdso = is_vdso_map(filename);
429 nsi = nsinfo__get(thread__nsinfo(thread));
430
431 if (vdso) {
432 /* The vdso maps are always on the host and not the
433 * container. Ensure that we don't use setns to look
434 * them up.
435 */
436 nnsi = nsinfo__copy(nsi);
437 if (nnsi) {
438 nsinfo__put(nsi);
439 nsinfo__clear_need_setns(nnsi);
440 nsi = nnsi;
441 }
442 dso = machine__findnew_vdso(machine, thread);
443 } else {
444 dso = machine__findnew_dso_id(machine, filename, id);
445 }
446
447 if (dso) {
448 mutex_lock(dso__lock(dso));
449 dso__set_nsinfo(dso, nsi);
450 mutex_unlock(dso__lock(dso));
451 } else
452 nsinfo__put(nsi);
453
454 thread__put(thread);
455 return dso;
456 }
457
458 static int perf_event__repipe_buildid_mmap(struct perf_tool *tool,
459 union perf_event *event,
460 struct perf_sample *sample,
461 struct machine *machine)
462 {
463 struct dso *dso;
464
465 dso = findnew_dso(event->mmap.pid, event->mmap.tid,
466 event->mmap.filename, NULL, machine);
467
468 if (dso && !dso__hit(dso)) {
469 dso__set_hit(dso);
470 dso__inject_build_id(dso, tool, machine, sample->cpumode, 0);
471 }
472 dso__put(dso);
473
474 return perf_event__repipe(tool, event, sample, machine);
475 }
476
477 static int perf_event__repipe_mmap2(struct perf_tool *tool,
478 union perf_event *event,
479 struct perf_sample *sample,
480 struct machine *machine)
481 {
482 int err;
483
484 err = perf_event__process_mmap2(tool, event, sample, machine);
485 perf_event__repipe(tool, event, sample, machine);
486
487 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
488 struct dso *dso;
489
490 dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
491 event->mmap2.filename, NULL, machine);
492 if (dso) {
493 /* mark it not to inject build-id */
494 dso__set_hit(dso);
495 }
496 dso__put(dso);
497 }
498
499 return err;
500 }
501
502 #ifdef HAVE_JITDUMP
503 static int perf_event__jit_repipe_mmap2(struct perf_tool *tool,
504 union perf_event *event,
505 struct perf_sample *sample,
506 struct machine *machine)
507 {
508 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
509 u64 n = 0;
510 int ret;
511
512 /*
513 * if jit marker, then inject jit mmaps and generate ELF images
514 */
515 ret = jit_process(inject->session, &inject->output, machine,
516 event->mmap2.filename, event->mmap2.pid, event->mmap2.tid, &n);
517 if (ret < 0)
518 return ret;
519 if (ret) {
520 inject->bytes_written += n;
521 return 0;
522 }
523 return perf_event__repipe_mmap2(tool, event, sample, machine);
524 }
525 #endif
526
527 static int perf_event__repipe_buildid_mmap2(struct perf_tool *tool,
528 union perf_event *event,
529 struct perf_sample *sample,
530 struct machine *machine)
531 {
532 struct dso_id dso_id = {
533 .maj = event->mmap2.maj,
534 .min = event->mmap2.min,
535 .ino = event->mmap2.ino,
536 .ino_generation = event->mmap2.ino_generation,
537 };
538 struct dso *dso;
539
540 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
541 /* cannot use dso_id since it'd have invalid info */
542 dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
543 event->mmap2.filename, NULL, machine);
544 if (dso) {
545 /* mark it not to inject build-id */
546 dso__set_hit(dso);
547 }
548 dso__put(dso);
549 perf_event__repipe(tool, event, sample, machine);
550 return 0;
551 }
552
553 dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
554 event->mmap2.filename, &dso_id, machine);
555
556 if (dso && !dso__hit(dso)) {
557 dso__set_hit(dso);
558 dso__inject_build_id(dso, tool, machine, sample->cpumode,
559 event->mmap2.flags);
560 }
561 dso__put(dso);
562
563 perf_event__repipe(tool, event, sample, machine);
564
565 return 0;
566 }
567
568 static int perf_event__repipe_fork(struct perf_tool *tool,
569 union perf_event *event,
570 struct perf_sample *sample,
571 struct machine *machine)
572 {
573 int err;
574
575 err = perf_event__process_fork(tool, event, sample, machine);
576 perf_event__repipe(tool, event, sample, machine);
577
578 return err;
579 }
580
581 static int perf_event__repipe_comm(struct perf_tool *tool,
582 union perf_event *event,
583 struct perf_sample *sample,
584 struct machine *machine)
585 {
586 int err;
587
588 err = perf_event__process_comm(tool, event, sample, machine);
589 perf_event__repipe(tool, event, sample, machine);
590
591 return err;
592 }
593
594 static int perf_event__repipe_namespaces(struct perf_tool *tool,
595 union perf_event *event,
596 struct perf_sample *sample,
597 struct machine *machine)
598 {
599 int err = perf_event__process_namespaces(tool, event, sample, machine);
600
601 perf_event__repipe(tool, event, sample, machine);
602
603 return err;
604 }
605
606 static int perf_event__repipe_exit(struct perf_tool *tool,
607 union perf_event *event,
608 struct perf_sample *sample,
609 struct machine *machine)
610 {
611 int err;
612
613 err = perf_event__process_exit(tool, event, sample, machine);
614 perf_event__repipe(tool, event, sample, machine);
615
616 return err;
617 }
618
619 #ifdef HAVE_LIBTRACEEVENT
620 static int perf_event__repipe_tracing_data(struct perf_session *session,
621 union perf_event *event)
622 {
623 perf_event__repipe_synth(session->tool, event);
624
625 return perf_event__process_tracing_data(session, event);
626 }
627 #endif
628
629 static int dso__read_build_id(struct dso *dso)
630 {
631 struct nscookie nsc;
632
633 if (dso__has_build_id(dso))
634 return 0;
635
636 mutex_lock(dso__lock(dso));
637 nsinfo__mountns_enter(dso__nsinfo(dso), &nsc);
638 if (filename__read_build_id(dso__long_name(dso), dso__bid(dso)) > 0)
639 dso__set_has_build_id(dso);
640 else if (dso__nsinfo(dso)) {
641 char *new_name = dso__filename_with_chroot(dso, dso__long_name(dso));
642
643 if (new_name && filename__read_build_id(new_name, dso__bid(dso)) > 0)
644 dso__set_has_build_id(dso);
645 free(new_name);
646 }
647 nsinfo__mountns_exit(&nsc);
648 mutex_unlock(dso__lock(dso));
649
650 return dso__has_build_id(dso) ? 0 : -1;
651 }
652
653 static struct strlist *perf_inject__parse_known_build_ids(
654 const char *known_build_ids_string)
655 {
656 struct str_node *pos, *tmp;
657 struct strlist *known_build_ids;
658 int bid_len;
659
660 known_build_ids = strlist__new(known_build_ids_string, NULL);
661 if (known_build_ids == NULL)
662 return NULL;
663 strlist__for_each_entry_safe(pos, tmp, known_build_ids) {
664 const char *build_id, *dso_name;
665
666 build_id = skip_spaces(pos->s);
667 dso_name = strchr(build_id, ' ');
668 if (dso_name == NULL) {
669 strlist__remove(known_build_ids, pos);
670 continue;
671 }
672 bid_len = dso_name - pos->s;
673 dso_name = skip_spaces(dso_name);
674 if (bid_len % 2 != 0 || bid_len >= SBUILD_ID_SIZE) {
675 strlist__remove(known_build_ids, pos);
676 continue;
677 }
678 for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
679 if (!isxdigit(build_id[2 * ix]) ||
680 !isxdigit(build_id[2 * ix + 1])) {
681 strlist__remove(known_build_ids, pos);
682 break;
683 }
684 }
685 }
686 return known_build_ids;
687 }
688
689 static bool perf_inject__lookup_known_build_id(struct perf_inject *inject,
690 struct dso *dso)
691 {
692 struct str_node *pos;
693 int bid_len;
694
695 strlist__for_each_entry(pos, inject->known_build_ids) {
696 const char *build_id, *dso_name;
697
698 build_id = skip_spaces(pos->s);
699 dso_name = strchr(build_id, ' ');
700 bid_len = dso_name - pos->s;
701 dso_name = skip_spaces(dso_name);
702 if (strcmp(dso__long_name(dso), dso_name))
703 continue;
704 for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
705 dso__bid(dso)->data[ix] = (hex(build_id[2 * ix]) << 4 |
706 hex(build_id[2 * ix + 1]));
707 }
708 dso__bid(dso)->size = bid_len / 2;
709 dso__set_has_build_id(dso);
710 return true;
711 }
712 return false;
713 }
714
715 static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool,
716 struct machine *machine, u8 cpumode, u32 flags)
717 {
718 struct perf_inject *inject = container_of(tool, struct perf_inject,
719 tool);
720 int err;
721
722 if (is_anon_memory(dso__long_name(dso)) || flags & MAP_HUGETLB)
723 return 0;
724 if (is_no_dso_memory(dso__long_name(dso)))
725 return 0;
726
727 if (inject->known_build_ids != NULL &&
728 perf_inject__lookup_known_build_id(inject, dso))
729 return 1;
730
731 if (dso__read_build_id(dso) < 0) {
732 pr_debug("no build_id found for %s\n", dso__long_name(dso));
733 return -1;
734 }
735
736 err = perf_event__synthesize_build_id(tool, dso, cpumode,
737 perf_event__repipe, machine);
738 if (err) {
739 pr_err("Can't synthesize build_id event for %s\n", dso__long_name(dso));
740 return -1;
741 }
742
743 return 0;
744 }
745
746 int perf_event__inject_buildid(struct perf_tool *tool, union perf_event *event,
747 struct perf_sample *sample,
748 struct evsel *evsel __maybe_unused,
749 struct machine *machine)
750 {
751 struct addr_location al;
752 struct thread *thread;
753
754 addr_location__init(&al);
755 thread = machine__findnew_thread(machine, sample->pid, sample->tid);
756 if (thread == NULL) {
757 pr_err("problem processing %d event, skipping it.\n",
758 event->header.type);
759 goto repipe;
760 }
761
762 if (thread__find_map(thread, sample->cpumode, sample->ip, &al)) {
763 struct dso *dso = map__dso(al.map);
764
765 if (!dso__hit(dso)) {
766 dso__set_hit(dso);
767 dso__inject_build_id(dso, tool, machine,
768 sample->cpumode, map__flags(al.map));
769 }
770 }
771
772 thread__put(thread);
773 repipe:
774 perf_event__repipe(tool, event, sample, machine);
775 addr_location__exit(&al);
776 return 0;
777 }
778
779 static int perf_inject__sched_process_exit(struct perf_tool *tool,
780 union perf_event *event __maybe_unused,
781 struct perf_sample *sample,
782 struct evsel *evsel __maybe_unused,
783 struct machine *machine __maybe_unused)
784 {
785 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
786 struct event_entry *ent;
787
788 list_for_each_entry(ent, &inject->samples, node) {
789 if (sample->tid == ent->tid) {
790 list_del_init(&ent->node);
791 free(ent);
792 break;
793 }
794 }
795
796 return 0;
797 }
798
799 static int perf_inject__sched_switch(struct perf_tool *tool,
800 union perf_event *event,
801 struct perf_sample *sample,
802 struct evsel *evsel,
803 struct machine *machine)
804 {
805 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
806 struct event_entry *ent;
807
808 perf_inject__sched_process_exit(tool, event, sample, evsel, machine);
809
810 ent = malloc(event->header.size + sizeof(struct event_entry));
811 if (ent == NULL) {
812 color_fprintf(stderr, PERF_COLOR_RED,
813 "Not enough memory to process sched switch event!");
814 return -1;
815 }
816
817 ent->tid = sample->tid;
818 memcpy(&ent->event, event, event->header.size);
819 list_add(&ent->node, &inject->samples);
820 return 0;
821 }
822
823 #ifdef HAVE_LIBTRACEEVENT
824 static int perf_inject__sched_stat(struct perf_tool *tool,
825 union perf_event *event __maybe_unused,
826 struct perf_sample *sample,
827 struct evsel *evsel,
828 struct machine *machine)
829 {
830 struct event_entry *ent;
831 union perf_event *event_sw;
832 struct perf_sample sample_sw;
833 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
834 u32 pid = evsel__intval(evsel, sample, "pid");
835
836 list_for_each_entry(ent, &inject->samples, node) {
837 if (pid == ent->tid)
838 goto found;
839 }
840
841 return 0;
842 found:
843 event_sw = &ent->event[0];
844 evsel__parse_sample(evsel, event_sw, &sample_sw);
845
846 sample_sw.period = sample->period;
847 sample_sw.time = sample->time;
848 perf_event__synthesize_sample(event_sw, evsel->core.attr.sample_type,
849 evsel->core.attr.read_format, &sample_sw);
850 build_id__mark_dso_hit(tool, event_sw, &sample_sw, evsel, machine);
851 return perf_event__repipe(tool, event_sw, &sample_sw, machine);
852 }
853 #endif
854
855 static struct guest_vcpu *guest_session__vcpu(struct guest_session *gs, u32 vcpu)
856 {
857 if (realloc_array_as_needed(gs->vcpu, gs->vcpu_cnt, vcpu, NULL))
858 return NULL;
859 return &gs->vcpu[vcpu];
860 }
861
862 static int guest_session__output_bytes(struct guest_session *gs, void *buf, size_t sz)
863 {
864 ssize_t ret = writen(gs->tmp_fd, buf, sz);
865
866 return ret < 0 ? ret : 0;
867 }
868
869 static int guest_session__repipe(struct perf_tool *tool,
870 union perf_event *event,
871 struct perf_sample *sample __maybe_unused,
872 struct machine *machine __maybe_unused)
873 {
874 struct guest_session *gs = container_of(tool, struct guest_session, tool);
875
876 return guest_session__output_bytes(gs, event, event->header.size);
877 }
878
879 static int guest_session__map_tid(struct guest_session *gs, u32 tid, u32 vcpu)
880 {
881 struct guest_tid *guest_tid = zalloc(sizeof(*guest_tid));
882 int hash;
883
884 if (!guest_tid)
885 return -ENOMEM;
886
887 guest_tid->tid = tid;
888 guest_tid->vcpu = vcpu;
889 hash = hash_32(guest_tid->tid, PERF_EVLIST__HLIST_BITS);
890 hlist_add_head(&guest_tid->node, &gs->tids[hash]);
891
892 return 0;
893 }
894
895 static int host_peek_vm_comms_cb(struct perf_session *session __maybe_unused,
896 union perf_event *event,
897 u64 offset __maybe_unused, void *data)
898 {
899 struct guest_session *gs = data;
900 unsigned int vcpu;
901 struct guest_vcpu *guest_vcpu;
902 int ret;
903
904 if (event->header.type != PERF_RECORD_COMM ||
905 event->comm.pid != gs->machine_pid)
906 return 0;
907
908 /*
909 * QEMU option -name debug-threads=on, causes thread names formatted as
910 * below, although it is not an ABI. Also libvirt seems to use this by
911 * default. Here we rely on it to tell us which thread is which VCPU.
912 */
913 ret = sscanf(event->comm.comm, "CPU %u/KVM", &vcpu);
914 if (ret <= 0)
915 return ret;
916 pr_debug("Found VCPU: tid %u comm %s vcpu %u\n",
917 event->comm.tid, event->comm.comm, vcpu);
918 if (vcpu > INT_MAX) {
919 pr_err("Invalid VCPU %u\n", vcpu);
920 return -EINVAL;
921 }
922 guest_vcpu = guest_session__vcpu(gs, vcpu);
923 if (!guest_vcpu)
924 return -ENOMEM;
925 if (guest_vcpu->tid && guest_vcpu->tid != event->comm.tid) {
926 pr_err("Fatal error: Two threads found with the same VCPU\n");
927 return -EINVAL;
928 }
929 guest_vcpu->tid = event->comm.tid;
930
931 return guest_session__map_tid(gs, event->comm.tid, vcpu);
932 }
933
934 static int host_peek_vm_comms(struct perf_session *session, struct guest_session *gs)
935 {
936 return perf_session__peek_events(session, session->header.data_offset,
937 session->header.data_size,
938 host_peek_vm_comms_cb, gs);
939 }
940
941 static bool evlist__is_id_used(struct evlist *evlist, u64 id)
942 {
943 return evlist__id2sid(evlist, id);
944 }
945
946 static u64 guest_session__allocate_new_id(struct guest_session *gs, struct evlist *host_evlist)
947 {
948 do {
949 gs->highest_id += 1;
950 } while (!gs->highest_id || evlist__is_id_used(host_evlist, gs->highest_id));
951
952 return gs->highest_id;
953 }
954
955 static int guest_session__map_id(struct guest_session *gs, u64 id, u64 host_id, u32 vcpu)
956 {
957 struct guest_id *guest_id = zalloc(sizeof(*guest_id));
958 int hash;
959
960 if (!guest_id)
961 return -ENOMEM;
962
963 guest_id->id = id;
964 guest_id->host_id = host_id;
965 guest_id->vcpu = vcpu;
966 hash = hash_64(guest_id->id, PERF_EVLIST__HLIST_BITS);
967 hlist_add_head(&guest_id->node, &gs->heads[hash]);
968
969 return 0;
970 }
971
972 static u64 evlist__find_highest_id(struct evlist *evlist)
973 {
974 struct evsel *evsel;
975 u64 highest_id = 1;
976
977 evlist__for_each_entry(evlist, evsel) {
978 u32 j;
979
980 for (j = 0; j < evsel->core.ids; j++) {
981 u64 id = evsel->core.id[j];
982
983 if (id > highest_id)
984 highest_id = id;
985 }
986 }
987
988 return highest_id;
989 }
990
991 static int guest_session__map_ids(struct guest_session *gs, struct evlist *host_evlist)
992 {
993 struct evlist *evlist = gs->session->evlist;
994 struct evsel *evsel;
995 int ret;
996
997 evlist__for_each_entry(evlist, evsel) {
998 u32 j;
999
1000 for (j = 0; j < evsel->core.ids; j++) {
1001 struct perf_sample_id *sid;
1002 u64 host_id;
1003 u64 id;
1004
1005 id = evsel->core.id[j];
1006 sid = evlist__id2sid(evlist, id);
1007 if (!sid || sid->cpu.cpu == -1)
1008 continue;
1009 host_id = guest_session__allocate_new_id(gs, host_evlist);
1010 ret = guest_session__map_id(gs, id, host_id, sid->cpu.cpu);
1011 if (ret)
1012 return ret;
1013 }
1014 }
1015
1016 return 0;
1017 }
1018
1019 static struct guest_id *guest_session__lookup_id(struct guest_session *gs, u64 id)
1020 {
1021 struct hlist_head *head;
1022 struct guest_id *guest_id;
1023 int hash;
1024
1025 hash = hash_64(id, PERF_EVLIST__HLIST_BITS);
1026 head = &gs->heads[hash];
1027
1028 hlist_for_each_entry(guest_id, head, node)
1029 if (guest_id->id == id)
1030 return guest_id;
1031
1032 return NULL;
1033 }
1034
1035 static int process_attr(struct perf_tool *tool, union perf_event *event,
1036 struct perf_sample *sample __maybe_unused,
1037 struct machine *machine __maybe_unused)
1038 {
1039 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1040
1041 return perf_event__process_attr(tool, event, &inject->session->evlist);
1042 }
1043
1044 static int guest_session__add_attr(struct guest_session *gs, struct evsel *evsel)
1045 {
1046 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1047 struct perf_event_attr attr = evsel->core.attr;
1048 u64 *id_array;
1049 u32 *vcpu_array;
1050 int ret = -ENOMEM;
1051 u32 i;
1052
1053 id_array = calloc(evsel->core.ids, sizeof(*id_array));
1054 if (!id_array)
1055 return -ENOMEM;
1056
1057 vcpu_array = calloc(evsel->core.ids, sizeof(*vcpu_array));
1058 if (!vcpu_array)
1059 goto out;
1060
1061 for (i = 0; i < evsel->core.ids; i++) {
1062 u64 id = evsel->core.id[i];
1063 struct guest_id *guest_id = guest_session__lookup_id(gs, id);
1064
1065 if (!guest_id) {
1066 pr_err("Failed to find guest id %"PRIu64"\n", id);
1067 ret = -EINVAL;
1068 goto out;
1069 }
1070 id_array[i] = guest_id->host_id;
1071 vcpu_array[i] = guest_id->vcpu;
1072 }
1073
1074 attr.sample_type |= PERF_SAMPLE_IDENTIFIER;
1075 attr.exclude_host = 1;
1076 attr.exclude_guest = 0;
1077
1078 ret = perf_event__synthesize_attr(&inject->tool, &attr, evsel->core.ids,
1079 id_array, process_attr);
1080 if (ret)
1081 pr_err("Failed to add guest attr.\n");
1082
1083 for (i = 0; i < evsel->core.ids; i++) {
1084 struct perf_sample_id *sid;
1085 u32 vcpu = vcpu_array[i];
1086
1087 sid = evlist__id2sid(inject->session->evlist, id_array[i]);
1088 /* Guest event is per-thread from the host point of view */
1089 sid->cpu.cpu = -1;
1090 sid->tid = gs->vcpu[vcpu].tid;
1091 sid->machine_pid = gs->machine_pid;
1092 sid->vcpu.cpu = vcpu;
1093 }
1094 out:
1095 free(vcpu_array);
1096 free(id_array);
1097 return ret;
1098 }
1099
1100 static int guest_session__add_attrs(struct guest_session *gs)
1101 {
1102 struct evlist *evlist = gs->session->evlist;
1103 struct evsel *evsel;
1104 int ret;
1105
1106 evlist__for_each_entry(evlist, evsel) {
1107 ret = guest_session__add_attr(gs, evsel);
1108 if (ret)
1109 return ret;
1110 }
1111
1112 return 0;
1113 }
1114
1115 static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt)
1116 {
1117 struct perf_session *session = inject->session;
1118 struct evlist *evlist = session->evlist;
1119 struct machine *machine = &session->machines.host;
1120 size_t from = evlist->core.nr_entries - new_cnt;
1121
1122 return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe,
1123 evlist, machine, from);
1124 }
1125
1126 static struct guest_tid *guest_session__lookup_tid(struct guest_session *gs, u32 tid)
1127 {
1128 struct hlist_head *head;
1129 struct guest_tid *guest_tid;
1130 int hash;
1131
1132 hash = hash_32(tid, PERF_EVLIST__HLIST_BITS);
1133 head = &gs->tids[hash];
1134
1135 hlist_for_each_entry(guest_tid, head, node)
1136 if (guest_tid->tid == tid)
1137 return guest_tid;
1138
1139 return NULL;
1140 }
1141
1142 static bool dso__is_in_kernel_space(struct dso *dso)
1143 {
1144 if (dso__is_vdso(dso))
1145 return false;
1146
1147 return dso__is_kcore(dso) ||
1148 dso__kernel(dso) ||
1149 is_kernel_module(dso__long_name(dso), PERF_RECORD_MISC_CPUMODE_UNKNOWN);
1150 }
1151
1152 static u64 evlist__first_id(struct evlist *evlist)
1153 {
1154 struct evsel *evsel;
1155
1156 evlist__for_each_entry(evlist, evsel) {
1157 if (evsel->core.ids)
1158 return evsel->core.id[0];
1159 }
1160 return 0;
1161 }
1162
1163 static int process_build_id(struct perf_tool *tool,
1164 union perf_event *event,
1165 struct perf_sample *sample __maybe_unused,
1166 struct machine *machine __maybe_unused)
1167 {
1168 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1169
1170 return perf_event__process_build_id(inject->session, event);
1171 }
1172
1173 static int synthesize_build_id(struct perf_inject *inject, struct dso *dso, pid_t machine_pid)
1174 {
1175 struct machine *machine = perf_session__findnew_machine(inject->session, machine_pid);
1176 u8 cpumode = dso__is_in_kernel_space(dso) ?
1177 PERF_RECORD_MISC_GUEST_KERNEL :
1178 PERF_RECORD_MISC_GUEST_USER;
1179
1180 if (!machine)
1181 return -ENOMEM;
1182
1183 dso__set_hit(dso);
1184
1185 return perf_event__synthesize_build_id(&inject->tool, dso, cpumode,
1186 process_build_id, machine);
1187 }
1188
1189 static int guest_session__add_build_ids_cb(struct dso *dso, void *data)
1190 {
1191 struct guest_session *gs = data;
1192 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1193
1194 if (!dso__has_build_id(dso))
1195 return 0;
1196
1197 return synthesize_build_id(inject, dso, gs->machine_pid);
1198
1199 }
1200
1201 static int guest_session__add_build_ids(struct guest_session *gs)
1202 {
1203 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1204
1205 /* Build IDs will be put in the Build ID feature section */
1206 perf_header__set_feat(&inject->session->header, HEADER_BUILD_ID);
1207
1208 return dsos__for_each_dso(&gs->session->machines.host.dsos,
1209 guest_session__add_build_ids_cb,
1210 gs);
1211 }
1212
1213 static int guest_session__ksymbol_event(struct perf_tool *tool,
1214 union perf_event *event,
1215 struct perf_sample *sample __maybe_unused,
1216 struct machine *machine __maybe_unused)
1217 {
1218 struct guest_session *gs = container_of(tool, struct guest_session, tool);
1219
1220 /* Only support out-of-line i.e. no BPF support */
1221 if (event->ksymbol.ksym_type != PERF_RECORD_KSYMBOL_TYPE_OOL)
1222 return 0;
1223
1224 return guest_session__output_bytes(gs, event, event->header.size);
1225 }
1226
1227 static int guest_session__start(struct guest_session *gs, const char *name, bool force)
1228 {
1229 char tmp_file_name[] = "/tmp/perf-inject-guest_session-XXXXXX";
1230 struct perf_session *session;
1231 int ret;
1232
1233 /* Only these events will be injected */
1234 gs->tool.mmap = guest_session__repipe;
1235 gs->tool.mmap2 = guest_session__repipe;
1236 gs->tool.comm = guest_session__repipe;
1237 gs->tool.fork = guest_session__repipe;
1238 gs->tool.exit = guest_session__repipe;
1239 gs->tool.lost = guest_session__repipe;
1240 gs->tool.context_switch = guest_session__repipe;
1241 gs->tool.ksymbol = guest_session__ksymbol_event;
1242 gs->tool.text_poke = guest_session__repipe;
1243 /*
1244 * Processing a build ID creates a struct dso with that build ID. Later,
1245 * all guest dsos are iterated and the build IDs processed into the host
1246 * session where they will be output to the Build ID feature section
1247 * when the perf.data file header is written.
1248 */
1249 gs->tool.build_id = perf_event__process_build_id;
1250 /* Process the id index to know what VCPU an ID belongs to */
1251 gs->tool.id_index = perf_event__process_id_index;
1252
1253 gs->tool.ordered_events = true;
1254 gs->tool.ordering_requires_timestamps = true;
1255
1256 gs->data.path = name;
1257 gs->data.force = force;
1258 gs->data.mode = PERF_DATA_MODE_READ;
1259
1260 session = perf_session__new(&gs->data, &gs->tool);
1261 if (IS_ERR(session))
1262 return PTR_ERR(session);
1263 gs->session = session;
1264
1265 /*
1266 * Initial events have zero'd ID samples. Get default ID sample size
1267 * used for removing them.
1268 */
1269 gs->dflt_id_hdr_size = session->machines.host.id_hdr_size;
1270 /* And default ID for adding back a host-compatible ID sample */
1271 gs->dflt_id = evlist__first_id(session->evlist);
1272 if (!gs->dflt_id) {
1273 pr_err("Guest data has no sample IDs");
1274 return -EINVAL;
1275 }
1276
1277 /* Temporary file for guest events */
1278 gs->tmp_file_name = strdup(tmp_file_name);
1279 if (!gs->tmp_file_name)
1280 return -ENOMEM;
1281 gs->tmp_fd = mkstemp(gs->tmp_file_name);
1282 if (gs->tmp_fd < 0)
1283 return -errno;
1284
1285 if (zstd_init(&gs->session->zstd_data, 0) < 0)
1286 pr_warning("Guest session decompression initialization failed.\n");
1287
1288 /*
1289 * perf does not support processing 2 sessions simultaneously, so output
1290 * guest events to a temporary file.
1291 */
1292 ret = perf_session__process_events(gs->session);
1293 if (ret)
1294 return ret;
1295
1296 if (lseek(gs->tmp_fd, 0, SEEK_SET))
1297 return -errno;
1298
1299 return 0;
1300 }
1301
1302 /* Free hlist nodes assuming hlist_node is the first member of hlist entries */
1303 static void free_hlist(struct hlist_head *heads, size_t hlist_sz)
1304 {
1305 struct hlist_node *pos, *n;
1306 size_t i;
1307
1308 for (i = 0; i < hlist_sz; ++i) {
1309 hlist_for_each_safe(pos, n, &heads[i]) {
1310 hlist_del(pos);
1311 free(pos);
1312 }
1313 }
1314 }
1315
1316 static void guest_session__exit(struct guest_session *gs)
1317 {
1318 if (gs->session) {
1319 perf_session__delete(gs->session);
1320 free_hlist(gs->heads, PERF_EVLIST__HLIST_SIZE);
1321 free_hlist(gs->tids, PERF_EVLIST__HLIST_SIZE);
1322 }
1323 if (gs->tmp_file_name) {
1324 if (gs->tmp_fd >= 0)
1325 close(gs->tmp_fd);
1326 unlink(gs->tmp_file_name);
1327 zfree(&gs->tmp_file_name);
1328 }
1329 zfree(&gs->vcpu);
1330 zfree(&gs->perf_data_file);
1331 }
1332
1333 static void get_tsc_conv(struct perf_tsc_conversion *tc, struct perf_record_time_conv *time_conv)
1334 {
1335 tc->time_shift = time_conv->time_shift;
1336 tc->time_mult = time_conv->time_mult;
1337 tc->time_zero = time_conv->time_zero;
1338 tc->time_cycles = time_conv->time_cycles;
1339 tc->time_mask = time_conv->time_mask;
1340 tc->cap_user_time_zero = time_conv->cap_user_time_zero;
1341 tc->cap_user_time_short = time_conv->cap_user_time_short;
1342 }
1343
1344 static void guest_session__get_tc(struct guest_session *gs)
1345 {
1346 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1347
1348 get_tsc_conv(&gs->host_tc, &inject->session->time_conv);
1349 get_tsc_conv(&gs->guest_tc, &gs->session->time_conv);
1350 }
1351
1352 static void guest_session__convert_time(struct guest_session *gs, u64 guest_time, u64 *host_time)
1353 {
1354 u64 tsc;
1355
1356 if (!guest_time) {
1357 *host_time = 0;
1358 return;
1359 }
1360
1361 if (gs->guest_tc.cap_user_time_zero)
1362 tsc = perf_time_to_tsc(guest_time, &gs->guest_tc);
1363 else
1364 tsc = guest_time;
1365
1366 /*
1367 * This is the correct order of operations for x86 if the TSC Offset and
1368 * Multiplier values are used.
1369 */
1370 tsc -= gs->time_offset;
1371 tsc /= gs->time_scale;
1372
1373 if (gs->host_tc.cap_user_time_zero)
1374 *host_time = tsc_to_perf_time(tsc, &gs->host_tc);
1375 else
1376 *host_time = tsc;
1377 }
1378
1379 static int guest_session__fetch(struct guest_session *gs)
1380 {
1381 void *buf;
1382 struct perf_event_header *hdr;
1383 size_t hdr_sz = sizeof(*hdr);
1384 ssize_t ret;
1385
1386 buf = gs->ev.event_buf;
1387 if (!buf) {
1388 buf = malloc(PERF_SAMPLE_MAX_SIZE);
1389 if (!buf)
1390 return -ENOMEM;
1391 gs->ev.event_buf = buf;
1392 }
1393 hdr = buf;
1394 ret = readn(gs->tmp_fd, buf, hdr_sz);
1395 if (ret < 0)
1396 return ret;
1397
1398 if (!ret) {
1399 /* Zero size means EOF */
1400 hdr->size = 0;
1401 return 0;
1402 }
1403
1404 buf += hdr_sz;
1405
1406 ret = readn(gs->tmp_fd, buf, hdr->size - hdr_sz);
1407 if (ret < 0)
1408 return ret;
1409
1410 gs->ev.event = (union perf_event *)gs->ev.event_buf;
1411 gs->ev.sample.time = 0;
1412
1413 if (hdr->type >= PERF_RECORD_USER_TYPE_START) {
1414 pr_err("Unexpected type fetching guest event");
1415 return 0;
1416 }
1417
1418 ret = evlist__parse_sample(gs->session->evlist, gs->ev.event, &gs->ev.sample);
1419 if (ret) {
1420 pr_err("Parse failed fetching guest event");
1421 return ret;
1422 }
1423
1424 if (!gs->have_tc) {
1425 guest_session__get_tc(gs);
1426 gs->have_tc = true;
1427 }
1428
1429 guest_session__convert_time(gs, gs->ev.sample.time, &gs->ev.sample.time);
1430
1431 return 0;
1432 }
1433
1434 static int evlist__append_id_sample(struct evlist *evlist, union perf_event *ev,
1435 const struct perf_sample *sample)
1436 {
1437 struct evsel *evsel;
1438 void *array;
1439 int ret;
1440
1441 evsel = evlist__id2evsel(evlist, sample->id);
1442 array = ev;
1443
1444 if (!evsel) {
1445 pr_err("No evsel for id %"PRIu64"\n", sample->id);
1446 return -EINVAL;
1447 }
1448
1449 array += ev->header.size;
1450 ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample);
1451 if (ret < 0)
1452 return ret;
1453
1454 if (ret & 7) {
1455 pr_err("Bad id sample size %d\n", ret);
1456 return -EINVAL;
1457 }
1458
1459 ev->header.size += ret;
1460
1461 return 0;
1462 }
1463
1464 static int guest_session__inject_events(struct guest_session *gs, u64 timestamp)
1465 {
1466 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1467 int ret;
1468
1469 if (!gs->ready)
1470 return 0;
1471
1472 while (1) {
1473 struct perf_sample *sample;
1474 struct guest_id *guest_id;
1475 union perf_event *ev;
1476 u16 id_hdr_size;
1477 u8 cpumode;
1478 u64 id;
1479
1480 if (!gs->fetched) {
1481 ret = guest_session__fetch(gs);
1482 if (ret)
1483 return ret;
1484 gs->fetched = true;
1485 }
1486
1487 ev = gs->ev.event;
1488 sample = &gs->ev.sample;
1489
1490 if (!ev->header.size)
1491 return 0; /* EOF */
1492
1493 if (sample->time > timestamp)
1494 return 0;
1495
1496 /* Change cpumode to guest */
1497 cpumode = ev->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
1498 if (cpumode & PERF_RECORD_MISC_USER)
1499 cpumode = PERF_RECORD_MISC_GUEST_USER;
1500 else
1501 cpumode = PERF_RECORD_MISC_GUEST_KERNEL;
1502 ev->header.misc &= ~PERF_RECORD_MISC_CPUMODE_MASK;
1503 ev->header.misc |= cpumode;
1504
1505 id = sample->id;
1506 if (!id) {
1507 id = gs->dflt_id;
1508 id_hdr_size = gs->dflt_id_hdr_size;
1509 } else {
1510 struct evsel *evsel = evlist__id2evsel(gs->session->evlist, id);
1511
1512 id_hdr_size = evsel__id_hdr_size(evsel);
1513 }
1514
1515 if (id_hdr_size & 7) {
1516 pr_err("Bad id_hdr_size %u\n", id_hdr_size);
1517 return -EINVAL;
1518 }
1519
1520 if (ev->header.size & 7) {
1521 pr_err("Bad event size %u\n", ev->header.size);
1522 return -EINVAL;
1523 }
1524
1525 /* Remove guest id sample */
1526 ev->header.size -= id_hdr_size;
1527
1528 if (ev->header.size & 7) {
1529 pr_err("Bad raw event size %u\n", ev->header.size);
1530 return -EINVAL;
1531 }
1532
1533 guest_id = guest_session__lookup_id(gs, id);
1534 if (!guest_id) {
1535 pr_err("Guest event with unknown id %llu\n",
1536 (unsigned long long)id);
1537 return -EINVAL;
1538 }
1539
1540 /* Change to host ID to avoid conflicting ID values */
1541 sample->id = guest_id->host_id;
1542 sample->stream_id = guest_id->host_id;
1543
1544 if (sample->cpu != (u32)-1) {
1545 if (sample->cpu >= gs->vcpu_cnt) {
1546 pr_err("Guest event with unknown VCPU %u\n",
1547 sample->cpu);
1548 return -EINVAL;
1549 }
1550 /* Change to host CPU instead of guest VCPU */
1551 sample->cpu = gs->vcpu[sample->cpu].cpu;
1552 }
1553
1554 /* New id sample with new ID and CPU */
1555 ret = evlist__append_id_sample(inject->session->evlist, ev, sample);
1556 if (ret)
1557 return ret;
1558
1559 if (ev->header.size & 7) {
1560 pr_err("Bad new event size %u\n", ev->header.size);
1561 return -EINVAL;
1562 }
1563
1564 gs->fetched = false;
1565
1566 ret = output_bytes(inject, ev, ev->header.size);
1567 if (ret)
1568 return ret;
1569 }
1570 }
1571
1572 static int guest_session__flush_events(struct guest_session *gs)
1573 {
1574 return guest_session__inject_events(gs, -1);
1575 }
1576
1577 static int host__repipe(struct perf_tool *tool,
1578 union perf_event *event,
1579 struct perf_sample *sample,
1580 struct machine *machine)
1581 {
1582 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1583 int ret;
1584
1585 ret = guest_session__inject_events(&inject->guest_session, sample->time);
1586 if (ret)
1587 return ret;
1588
1589 return perf_event__repipe(tool, event, sample, machine);
1590 }
1591
1592 static int host__finished_init(struct perf_session *session, union perf_event *event)
1593 {
1594 struct perf_inject *inject = container_of(session->tool, struct perf_inject, tool);
1595 struct guest_session *gs = &inject->guest_session;
1596 int ret;
1597
1598 /*
1599 * Peek through host COMM events to find QEMU threads and the VCPU they
1600 * are running.
1601 */
1602 ret = host_peek_vm_comms(session, gs);
1603 if (ret)
1604 return ret;
1605
1606 if (!gs->vcpu_cnt) {
1607 pr_err("No VCPU threads found for pid %u\n", gs->machine_pid);
1608 return -EINVAL;
1609 }
1610
1611 /*
1612 * Allocate new (unused) host sample IDs and map them to the guest IDs.
1613 */
1614 gs->highest_id = evlist__find_highest_id(session->evlist);
1615 ret = guest_session__map_ids(gs, session->evlist);
1616 if (ret)
1617 return ret;
1618
1619 ret = guest_session__add_attrs(gs);
1620 if (ret)
1621 return ret;
1622
1623 ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries);
1624 if (ret) {
1625 pr_err("Failed to synthesize id_index\n");
1626 return ret;
1627 }
1628
1629 ret = guest_session__add_build_ids(gs);
1630 if (ret) {
1631 pr_err("Failed to add guest build IDs\n");
1632 return ret;
1633 }
1634
1635 gs->ready = true;
1636
1637 ret = guest_session__inject_events(gs, 0);
1638 if (ret)
1639 return ret;
1640
1641 return perf_event__repipe_op2_synth(session, event);
1642 }
1643
1644 /*
1645 * Obey finished-round ordering. The FINISHED_ROUND event is first processed
1646 * which flushes host events to file up until the last flush time. Then inject
1647 * guest events up to the same time. Finally write out the FINISHED_ROUND event
1648 * itself.
1649 */
1650 static int host__finished_round(struct perf_tool *tool,
1651 union perf_event *event,
1652 struct ordered_events *oe)
1653 {
1654 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1655 int ret = perf_event__process_finished_round(tool, event, oe);
1656 u64 timestamp = ordered_events__last_flush_time(oe);
1657
1658 if (ret)
1659 return ret;
1660
1661 ret = guest_session__inject_events(&inject->guest_session, timestamp);
1662 if (ret)
1663 return ret;
1664
1665 return perf_event__repipe_oe_synth(tool, event, oe);
1666 }
1667
1668 static int host__context_switch(struct perf_tool *tool,
1669 union perf_event *event,
1670 struct perf_sample *sample,
1671 struct machine *machine)
1672 {
1673 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1674 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1675 struct guest_session *gs = &inject->guest_session;
1676 u32 pid = event->context_switch.next_prev_pid;
1677 u32 tid = event->context_switch.next_prev_tid;
1678 struct guest_tid *guest_tid;
1679 u32 vcpu;
1680
1681 if (out || pid != gs->machine_pid)
1682 goto out;
1683
1684 guest_tid = guest_session__lookup_tid(gs, tid);
1685 if (!guest_tid)
1686 goto out;
1687
1688 if (sample->cpu == (u32)-1) {
1689 pr_err("Switch event does not have CPU\n");
1690 return -EINVAL;
1691 }
1692
1693 vcpu = guest_tid->vcpu;
1694 if (vcpu >= gs->vcpu_cnt)
1695 return -EINVAL;
1696
1697 /* Guest is switching in, record which CPU the VCPU is now running on */
1698 gs->vcpu[vcpu].cpu = sample->cpu;
1699 out:
1700 return host__repipe(tool, event, sample, machine);
1701 }
1702
1703 static void sig_handler(int sig __maybe_unused)
1704 {
1705 session_done = 1;
1706 }
1707
1708 static int evsel__check_stype(struct evsel *evsel, u64 sample_type, const char *sample_msg)
1709 {
1710 struct perf_event_attr *attr = &evsel->core.attr;
1711 const char *name = evsel__name(evsel);
1712
1713 if (!(attr->sample_type & sample_type)) {
1714 pr_err("Samples for %s event do not have %s attribute set.",
1715 name, sample_msg);
1716 return -EINVAL;
1717 }
1718
1719 return 0;
1720 }
1721
1722 static int drop_sample(struct perf_tool *tool __maybe_unused,
1723 union perf_event *event __maybe_unused,
1724 struct perf_sample *sample __maybe_unused,
1725 struct evsel *evsel __maybe_unused,
1726 struct machine *machine __maybe_unused)
1727 {
1728 return 0;
1729 }
1730
1731 static void strip_init(struct perf_inject *inject)
1732 {
1733 struct evlist *evlist = inject->session->evlist;
1734 struct evsel *evsel;
1735
1736 inject->tool.context_switch = perf_event__drop;
1737
1738 evlist__for_each_entry(evlist, evsel)
1739 evsel->handler = drop_sample;
1740 }
1741
1742 static int parse_vm_time_correlation(const struct option *opt, const char *str, int unset)
1743 {
1744 struct perf_inject *inject = opt->value;
1745 const char *args;
1746 char *dry_run;
1747
1748 if (unset)
1749 return 0;
1750
1751 inject->itrace_synth_opts.set = true;
1752 inject->itrace_synth_opts.vm_time_correlation = true;
1753 inject->in_place_update = true;
1754
1755 if (!str)
1756 return 0;
1757
1758 dry_run = skip_spaces(str);
1759 if (!strncmp(dry_run, "dry-run", strlen("dry-run"))) {
1760 inject->itrace_synth_opts.vm_tm_corr_dry_run = true;
1761 inject->in_place_update_dry_run = true;
1762 args = dry_run + strlen("dry-run");
1763 } else {
1764 args = str;
1765 }
1766
1767 inject->itrace_synth_opts.vm_tm_corr_args = strdup(args);
1768
1769 return inject->itrace_synth_opts.vm_tm_corr_args ? 0 : -ENOMEM;
1770 }
1771
1772 static int parse_guest_data(const struct option *opt, const char *str, int unset)
1773 {
1774 struct perf_inject *inject = opt->value;
1775 struct guest_session *gs = &inject->guest_session;
1776 char *tok;
1777 char *s;
1778
1779 if (unset)
1780 return 0;
1781
1782 if (!str)
1783 goto bad_args;
1784
1785 s = strdup(str);
1786 if (!s)
1787 return -ENOMEM;
1788
1789 gs->perf_data_file = strsep(&s, ",");
1790 if (!gs->perf_data_file)
1791 goto bad_args;
1792
1793 gs->copy_kcore_dir = has_kcore_dir(gs->perf_data_file);
1794 if (gs->copy_kcore_dir)
1795 inject->output.is_dir = true;
1796
1797 tok = strsep(&s, ",");
1798 if (!tok)
1799 goto bad_args;
1800 gs->machine_pid = strtoul(tok, NULL, 0);
1801 if (!inject->guest_session.machine_pid)
1802 goto bad_args;
1803
1804 gs->time_scale = 1;
1805
1806 tok = strsep(&s, ",");
1807 if (!tok)
1808 goto out;
1809 gs->time_offset = strtoull(tok, NULL, 0);
1810
1811 tok = strsep(&s, ",");
1812 if (!tok)
1813 goto out;
1814 gs->time_scale = strtod(tok, NULL);
1815 if (!gs->time_scale)
1816 goto bad_args;
1817 out:
1818 return 0;
1819
1820 bad_args:
1821 pr_err("--guest-data option requires guest perf.data file name, "
1822 "guest machine PID, and optionally guest timestamp offset, "
1823 "and guest timestamp scale factor, separated by commas.\n");
1824 return -1;
1825 }
1826
1827 static int save_section_info_cb(struct perf_file_section *section,
1828 struct perf_header *ph __maybe_unused,
1829 int feat, int fd __maybe_unused, void *data)
1830 {
1831 struct perf_inject *inject = data;
1832
1833 inject->secs[feat] = *section;
1834 return 0;
1835 }
1836
1837 static int save_section_info(struct perf_inject *inject)
1838 {
1839 struct perf_header *header = &inject->session->header;
1840 int fd = perf_data__fd(inject->session->data);
1841
1842 return perf_header__process_sections(header, fd, inject, save_section_info_cb);
1843 }
1844
1845 static bool keep_feat(int feat)
1846 {
1847 switch (feat) {
1848 /* Keep original information that describes the machine or software */
1849 case HEADER_TRACING_DATA:
1850 case HEADER_HOSTNAME:
1851 case HEADER_OSRELEASE:
1852 case HEADER_VERSION:
1853 case HEADER_ARCH:
1854 case HEADER_NRCPUS:
1855 case HEADER_CPUDESC:
1856 case HEADER_CPUID:
1857 case HEADER_TOTAL_MEM:
1858 case HEADER_CPU_TOPOLOGY:
1859 case HEADER_NUMA_TOPOLOGY:
1860 case HEADER_PMU_MAPPINGS:
1861 case HEADER_CACHE:
1862 case HEADER_MEM_TOPOLOGY:
1863 case HEADER_CLOCKID:
1864 case HEADER_BPF_PROG_INFO:
1865 case HEADER_BPF_BTF:
1866 case HEADER_CPU_PMU_CAPS:
1867 case HEADER_CLOCK_DATA:
1868 case HEADER_HYBRID_TOPOLOGY:
1869 case HEADER_PMU_CAPS:
1870 return true;
1871 /* Information that can be updated */
1872 case HEADER_BUILD_ID:
1873 case HEADER_CMDLINE:
1874 case HEADER_EVENT_DESC:
1875 case HEADER_BRANCH_STACK:
1876 case HEADER_GROUP_DESC:
1877 case HEADER_AUXTRACE:
1878 case HEADER_STAT:
1879 case HEADER_SAMPLE_TIME:
1880 case HEADER_DIR_FORMAT:
1881 case HEADER_COMPRESSED:
1882 default:
1883 return false;
1884 };
1885 }
1886
1887 static int read_file(int fd, u64 offs, void *buf, size_t sz)
1888 {
1889 ssize_t ret = preadn(fd, buf, sz, offs);
1890
1891 if (ret < 0)
1892 return -errno;
1893 if ((size_t)ret != sz)
1894 return -EINVAL;
1895 return 0;
1896 }
1897
1898 static int feat_copy(struct perf_inject *inject, int feat, struct feat_writer *fw)
1899 {
1900 int fd = perf_data__fd(inject->session->data);
1901 u64 offs = inject->secs[feat].offset;
1902 size_t sz = inject->secs[feat].size;
1903 void *buf = malloc(sz);
1904 int ret;
1905
1906 if (!buf)
1907 return -ENOMEM;
1908
1909 ret = read_file(fd, offs, buf, sz);
1910 if (ret)
1911 goto out_free;
1912
1913 ret = fw->write(fw, buf, sz);
1914 out_free:
1915 free(buf);
1916 return ret;
1917 }
1918
1919 struct inject_fc {
1920 struct feat_copier fc;
1921 struct perf_inject *inject;
1922 };
1923
1924 static int feat_copy_cb(struct feat_copier *fc, int feat, struct feat_writer *fw)
1925 {
1926 struct inject_fc *inj_fc = container_of(fc, struct inject_fc, fc);
1927 struct perf_inject *inject = inj_fc->inject;
1928 int ret;
1929
1930 if (!inject->secs[feat].offset ||
1931 !keep_feat(feat))
1932 return 0;
1933
1934 ret = feat_copy(inject, feat, fw);
1935 if (ret < 0)
1936 return ret;
1937
1938 return 1; /* Feature section copied */
1939 }
1940
1941 static int copy_kcore_dir(struct perf_inject *inject)
1942 {
1943 char *cmd;
1944 int ret;
1945
1946 ret = asprintf(&cmd, "cp -r -n %s/kcore_dir* %s >/dev/null 2>&1",
1947 inject->input_name, inject->output.path);
1948 if (ret < 0)
1949 return ret;
1950 pr_debug("%s\n", cmd);
1951 ret = system(cmd);
1952 free(cmd);
1953 return ret;
1954 }
1955
1956 static int guest_session__copy_kcore_dir(struct guest_session *gs)
1957 {
1958 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1959 char *cmd;
1960 int ret;
1961
1962 ret = asprintf(&cmd, "cp -r -n %s/kcore_dir %s/kcore_dir__%u >/dev/null 2>&1",
1963 gs->perf_data_file, inject->output.path, gs->machine_pid);
1964 if (ret < 0)
1965 return ret;
1966 pr_debug("%s\n", cmd);
1967 ret = system(cmd);
1968 free(cmd);
1969 return ret;
1970 }
1971
1972 static int output_fd(struct perf_inject *inject)
1973 {
1974 return inject->in_place_update ? -1 : perf_data__fd(&inject->output);
1975 }
1976
1977 static int __cmd_inject(struct perf_inject *inject)
1978 {
1979 int ret = -EINVAL;
1980 struct guest_session *gs = &inject->guest_session;
1981 struct perf_session *session = inject->session;
1982 int fd = output_fd(inject);
1983 u64 output_data_offset;
1984
1985 signal(SIGINT, sig_handler);
1986
1987 if (inject->build_ids || inject->sched_stat ||
1988 inject->itrace_synth_opts.set || inject->build_id_all) {
1989 inject->tool.mmap = perf_event__repipe_mmap;
1990 inject->tool.mmap2 = perf_event__repipe_mmap2;
1991 inject->tool.fork = perf_event__repipe_fork;
1992 #ifdef HAVE_LIBTRACEEVENT
1993 inject->tool.tracing_data = perf_event__repipe_tracing_data;
1994 #endif
1995 }
1996
1997 output_data_offset = perf_session__data_offset(session->evlist);
1998
1999 if (inject->build_id_all) {
2000 inject->tool.mmap = perf_event__repipe_buildid_mmap;
2001 inject->tool.mmap2 = perf_event__repipe_buildid_mmap2;
2002 } else if (inject->build_ids) {
2003 inject->tool.sample = perf_event__inject_buildid;
2004 } else if (inject->sched_stat) {
2005 struct evsel *evsel;
2006
2007 evlist__for_each_entry(session->evlist, evsel) {
2008 const char *name = evsel__name(evsel);
2009
2010 if (!strcmp(name, "sched:sched_switch")) {
2011 if (evsel__check_stype(evsel, PERF_SAMPLE_TID, "TID"))
2012 return -EINVAL;
2013
2014 evsel->handler = perf_inject__sched_switch;
2015 } else if (!strcmp(name, "sched:sched_process_exit"))
2016 evsel->handler = perf_inject__sched_process_exit;
2017 #ifdef HAVE_LIBTRACEEVENT
2018 else if (!strncmp(name, "sched:sched_stat_", 17))
2019 evsel->handler = perf_inject__sched_stat;
2020 #endif
2021 }
2022 } else if (inject->itrace_synth_opts.vm_time_correlation) {
2023 session->itrace_synth_opts = &inject->itrace_synth_opts;
2024 memset(&inject->tool, 0, sizeof(inject->tool));
2025 inject->tool.id_index = perf_event__process_id_index;
2026 inject->tool.auxtrace_info = perf_event__process_auxtrace_info;
2027 inject->tool.auxtrace = perf_event__process_auxtrace;
2028 inject->tool.auxtrace_error = perf_event__process_auxtrace_error;
2029 inject->tool.ordered_events = true;
2030 inject->tool.ordering_requires_timestamps = true;
2031 } else if (inject->itrace_synth_opts.set) {
2032 session->itrace_synth_opts = &inject->itrace_synth_opts;
2033 inject->itrace_synth_opts.inject = true;
2034 inject->tool.comm = perf_event__repipe_comm;
2035 inject->tool.namespaces = perf_event__repipe_namespaces;
2036 inject->tool.exit = perf_event__repipe_exit;
2037 inject->tool.id_index = perf_event__process_id_index;
2038 inject->tool.auxtrace_info = perf_event__process_auxtrace_info;
2039 inject->tool.auxtrace = perf_event__process_auxtrace;
2040 inject->tool.aux = perf_event__drop_aux;
2041 inject->tool.itrace_start = perf_event__drop_aux;
2042 inject->tool.aux_output_hw_id = perf_event__drop_aux;
2043 inject->tool.ordered_events = true;
2044 inject->tool.ordering_requires_timestamps = true;
2045 /* Allow space in the header for new attributes */
2046 output_data_offset = roundup(8192 + session->header.data_offset, 4096);
2047 if (inject->strip)
2048 strip_init(inject);
2049 } else if (gs->perf_data_file) {
2050 char *name = gs->perf_data_file;
2051
2052 /*
2053 * Not strictly necessary, but keep these events in order wrt
2054 * guest events.
2055 */
2056 inject->tool.mmap = host__repipe;
2057 inject->tool.mmap2 = host__repipe;
2058 inject->tool.comm = host__repipe;
2059 inject->tool.fork = host__repipe;
2060 inject->tool.exit = host__repipe;
2061 inject->tool.lost = host__repipe;
2062 inject->tool.context_switch = host__repipe;
2063 inject->tool.ksymbol = host__repipe;
2064 inject->tool.text_poke = host__repipe;
2065 /*
2066 * Once the host session has initialized, set up sample ID
2067 * mapping and feed in guest attrs, build IDs and initial
2068 * events.
2069 */
2070 inject->tool.finished_init = host__finished_init;
2071 /* Obey finished round ordering */
2072 inject->tool.finished_round = host__finished_round,
2073 /* Keep track of which CPU a VCPU is runnng on */
2074 inject->tool.context_switch = host__context_switch;
2075 /*
2076 * Must order events to be able to obey finished round
2077 * ordering.
2078 */
2079 inject->tool.ordered_events = true;
2080 inject->tool.ordering_requires_timestamps = true;
2081 /* Set up a separate session to process guest perf.data file */
2082 ret = guest_session__start(gs, name, session->data->force);
2083 if (ret) {
2084 pr_err("Failed to process %s, error %d\n", name, ret);
2085 return ret;
2086 }
2087 /* Allow space in the header for guest attributes */
2088 output_data_offset += gs->session->header.data_offset;
2089 output_data_offset = roundup(output_data_offset, 4096);
2090 }
2091
2092 if (!inject->itrace_synth_opts.set)
2093 auxtrace_index__free(&session->auxtrace_index);
2094
2095 if (!inject->is_pipe && !inject->in_place_update)
2096 lseek(fd, output_data_offset, SEEK_SET);
2097
2098 ret = perf_session__process_events(session);
2099 if (ret)
2100 return ret;
2101
2102 if (gs->session) {
2103 /*
2104 * Remaining guest events have later timestamps. Flush them
2105 * out to file.
2106 */
2107 ret = guest_session__flush_events(gs);
2108 if (ret) {
2109 pr_err("Failed to flush guest events\n");
2110 return ret;
2111 }
2112 }
2113
2114 if (!inject->is_pipe && !inject->in_place_update) {
2115 struct inject_fc inj_fc = {
2116 .fc.copy = feat_copy_cb,
2117 .inject = inject,
2118 };
2119
2120 if (inject->build_ids)
2121 perf_header__set_feat(&session->header,
2122 HEADER_BUILD_ID);
2123 /*
2124 * Keep all buildids when there is unprocessed AUX data because
2125 * it is not known which ones the AUX trace hits.
2126 */
2127 if (perf_header__has_feat(&session->header, HEADER_BUILD_ID) &&
2128 inject->have_auxtrace && !inject->itrace_synth_opts.set)
2129 perf_session__dsos_hit_all(session);
2130 /*
2131 * The AUX areas have been removed and replaced with
2132 * synthesized hardware events, so clear the feature flag.
2133 */
2134 if (inject->itrace_synth_opts.set) {
2135 perf_header__clear_feat(&session->header,
2136 HEADER_AUXTRACE);
2137 if (inject->itrace_synth_opts.last_branch ||
2138 inject->itrace_synth_opts.add_last_branch)
2139 perf_header__set_feat(&session->header,
2140 HEADER_BRANCH_STACK);
2141 }
2142 session->header.data_offset = output_data_offset;
2143 session->header.data_size = inject->bytes_written;
2144 perf_session__inject_header(session, session->evlist, fd, &inj_fc.fc);
2145
2146 if (inject->copy_kcore_dir) {
2147 ret = copy_kcore_dir(inject);
2148 if (ret) {
2149 pr_err("Failed to copy kcore\n");
2150 return ret;
2151 }
2152 }
2153 if (gs->copy_kcore_dir) {
2154 ret = guest_session__copy_kcore_dir(gs);
2155 if (ret) {
2156 pr_err("Failed to copy guest kcore\n");
2157 return ret;
2158 }
2159 }
2160 }
2161
2162 return ret;
2163 }
2164
2165 int cmd_inject(int argc, const char **argv)
2166 {
2167 struct perf_inject inject = {
2168 .tool = {
2169 .sample = perf_event__repipe_sample,
2170 .read = perf_event__repipe_sample,
2171 .mmap = perf_event__repipe,
2172 .mmap2 = perf_event__repipe,
2173 .comm = perf_event__repipe,
2174 .namespaces = perf_event__repipe,
2175 .cgroup = perf_event__repipe,
2176 .fork = perf_event__repipe,
2177 .exit = perf_event__repipe,
2178 .lost = perf_event__repipe,
2179 .lost_samples = perf_event__repipe,
2180 .aux = perf_event__repipe,
2181 .itrace_start = perf_event__repipe,
2182 .aux_output_hw_id = perf_event__repipe,
2183 .context_switch = perf_event__repipe,
2184 .throttle = perf_event__repipe,
2185 .unthrottle = perf_event__repipe,
2186 .ksymbol = perf_event__repipe,
2187 .bpf = perf_event__repipe,
2188 .text_poke = perf_event__repipe,
2189 .attr = perf_event__repipe_attr,
2190 .event_update = perf_event__repipe_event_update,
2191 .tracing_data = perf_event__repipe_op2_synth,
2192 .finished_round = perf_event__repipe_oe_synth,
2193 .build_id = perf_event__repipe_op2_synth,
2194 .id_index = perf_event__repipe_op2_synth,
2195 .auxtrace_info = perf_event__repipe_op2_synth,
2196 .auxtrace_error = perf_event__repipe_op2_synth,
2197 .time_conv = perf_event__repipe_op2_synth,
2198 .thread_map = perf_event__repipe_op2_synth,
2199 .cpu_map = perf_event__repipe_op2_synth,
2200 .stat_config = perf_event__repipe_op2_synth,
2201 .stat = perf_event__repipe_op2_synth,
2202 .stat_round = perf_event__repipe_op2_synth,
2203 .feature = perf_event__repipe_op2_synth,
2204 .finished_init = perf_event__repipe_op2_synth,
2205 .compressed = perf_event__repipe_op4_synth,
2206 .auxtrace = perf_event__repipe_auxtrace,
2207 .dont_split_sample_group = true,
2208 },
2209 .input_name = "-",
2210 .samples = LIST_HEAD_INIT(inject.samples),
2211 .output = {
2212 .path = "-",
2213 .mode = PERF_DATA_MODE_WRITE,
2214 .use_stdio = true,
2215 },
2216 };
2217 struct perf_data data = {
2218 .mode = PERF_DATA_MODE_READ,
2219 .use_stdio = true,
2220 };
2221 int ret;
2222 bool repipe = true;
2223 const char *known_build_ids = NULL;
2224
2225 struct option options[] = {
2226 OPT_BOOLEAN('b', "build-ids", &inject.build_ids,
2227 "Inject build-ids into the output stream"),
2228 OPT_BOOLEAN(0, "buildid-all", &inject.build_id_all,
2229 "Inject build-ids of all DSOs into the output stream"),
2230 OPT_STRING(0, "known-build-ids", &known_build_ids,
2231 "buildid path [,buildid path...]",
2232 "build-ids to use for given paths"),
2233 OPT_STRING('i', "input", &inject.input_name, "file",
2234 "input file name"),
2235 OPT_STRING('o', "output", &inject.output.path, "file",
2236 "output file name"),
2237 OPT_BOOLEAN('s', "sched-stat", &inject.sched_stat,
2238 "Merge sched-stat and sched-switch for getting events "
2239 "where and how long tasks slept"),
2240 #ifdef HAVE_JITDUMP
2241 OPT_BOOLEAN('j', "jit", &inject.jit_mode, "merge jitdump files into perf.data file"),
2242 #endif
2243 OPT_INCR('v', "verbose", &verbose,
2244 "be more verbose (show build ids, etc)"),
2245 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
2246 "file", "vmlinux pathname"),
2247 OPT_BOOLEAN(0, "ignore-vmlinux", &symbol_conf.ignore_vmlinux,
2248 "don't load vmlinux even if found"),
2249 OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name, "file",
2250 "kallsyms pathname"),
2251 OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"),
2252 OPT_CALLBACK_OPTARG(0, "itrace", &inject.itrace_synth_opts,
2253 NULL, "opts", "Instruction Tracing options\n"
2254 ITRACE_HELP,
2255 itrace_parse_synth_opts),
2256 OPT_BOOLEAN(0, "strip", &inject.strip,
2257 "strip non-synthesized events (use with --itrace)"),
2258 OPT_CALLBACK_OPTARG(0, "vm-time-correlation", &inject, NULL, "opts",
2259 "correlate time between VM guests and the host",
2260 parse_vm_time_correlation),
2261 OPT_CALLBACK_OPTARG(0, "guest-data", &inject, NULL, "opts",
2262 "inject events from a guest perf.data file",
2263 parse_guest_data),
2264 OPT_STRING(0, "guestmount", &symbol_conf.guestmount, "directory",
2265 "guest mount directory under which every guest os"
2266 " instance has a subdir"),
2267 OPT_END()
2268 };
2269 const char * const inject_usage[] = {
2270 "perf inject [<options>]",
2271 NULL
2272 };
2273
2274 if (!inject.itrace_synth_opts.set) {
2275 /* Disable eager loading of kernel symbols that adds overhead to perf inject. */
2276 symbol_conf.lazy_load_kernel_maps = true;
2277 }
2278
2279 #ifndef HAVE_JITDUMP
2280 set_option_nobuild(options, 'j', "jit", "NO_LIBELF=1", true);
2281 #endif
2282 argc = parse_options(argc, argv, options, inject_usage, 0);
2283
2284 /*
2285 * Any (unrecognized) arguments left?
2286 */
2287 if (argc)
2288 usage_with_options(inject_usage, options);
2289
2290 if (inject.strip && !inject.itrace_synth_opts.set) {
2291 pr_err("--strip option requires --itrace option\n");
2292 return -1;
2293 }
2294
2295 if (symbol__validate_sym_arguments())
2296 return -1;
2297
2298 if (inject.in_place_update) {
2299 if (!strcmp(inject.input_name, "-")) {
2300 pr_err("Input file name required for in-place updating\n");
2301 return -1;
2302 }
2303 if (strcmp(inject.output.path, "-")) {
2304 pr_err("Output file name must not be specified for in-place updating\n");
2305 return -1;
2306 }
2307 if (!data.force && !inject.in_place_update_dry_run) {
2308 pr_err("The input file would be updated in place, "
2309 "the --force option is required.\n");
2310 return -1;
2311 }
2312 if (!inject.in_place_update_dry_run)
2313 data.in_place_update = true;
2314 } else {
2315 if (strcmp(inject.output.path, "-") && !inject.strip &&
2316 has_kcore_dir(inject.input_name)) {
2317 inject.output.is_dir = true;
2318 inject.copy_kcore_dir = true;
2319 }
2320 if (perf_data__open(&inject.output)) {
2321 perror("failed to create output file");
2322 return -1;
2323 }
2324 }
2325
2326 data.path = inject.input_name;
2327 if (!strcmp(inject.input_name, "-") || inject.output.is_pipe) {
2328 inject.is_pipe = true;
2329 /*
2330 * Do not repipe header when input is a regular file
2331 * since either it can rewrite the header at the end
2332 * or write a new pipe header.
2333 */
2334 if (strcmp(inject.input_name, "-"))
2335 repipe = false;
2336 }
2337
2338 inject.session = __perf_session__new(&data, repipe,
2339 output_fd(&inject),
2340 &inject.tool);
2341 if (IS_ERR(inject.session)) {
2342 ret = PTR_ERR(inject.session);
2343 goto out_close_output;
2344 }
2345
2346 if (zstd_init(&(inject.session->zstd_data), 0) < 0)
2347 pr_warning("Decompression initialization failed.\n");
2348
2349 /* Save original section info before feature bits change */
2350 ret = save_section_info(&inject);
2351 if (ret)
2352 goto out_delete;
2353
2354 if (!data.is_pipe && inject.output.is_pipe) {
2355 ret = perf_header__write_pipe(perf_data__fd(&inject.output));
2356 if (ret < 0) {
2357 pr_err("Couldn't write a new pipe header.\n");
2358 goto out_delete;
2359 }
2360
2361 ret = perf_event__synthesize_for_pipe(&inject.tool,
2362 inject.session,
2363 &inject.output,
2364 perf_event__repipe);
2365 if (ret < 0)
2366 goto out_delete;
2367 }
2368
2369 if (inject.build_ids && !inject.build_id_all) {
2370 /*
2371 * to make sure the mmap records are ordered correctly
2372 * and so that the correct especially due to jitted code
2373 * mmaps. We cannot generate the buildid hit list and
2374 * inject the jit mmaps at the same time for now.
2375 */
2376 inject.tool.ordered_events = true;
2377 inject.tool.ordering_requires_timestamps = true;
2378 if (known_build_ids != NULL) {
2379 inject.known_build_ids =
2380 perf_inject__parse_known_build_ids(known_build_ids);
2381
2382 if (inject.known_build_ids == NULL) {
2383 pr_err("Couldn't parse known build ids.\n");
2384 goto out_delete;
2385 }
2386 }
2387 }
2388
2389 if (inject.sched_stat) {
2390 inject.tool.ordered_events = true;
2391 }
2392
2393 #ifdef HAVE_JITDUMP
2394 if (inject.jit_mode) {
2395 inject.tool.mmap2 = perf_event__jit_repipe_mmap2;
2396 inject.tool.mmap = perf_event__jit_repipe_mmap;
2397 inject.tool.ordered_events = true;
2398 inject.tool.ordering_requires_timestamps = true;
2399 /*
2400 * JIT MMAP injection injects all MMAP events in one go, so it
2401 * does not obey finished_round semantics.
2402 */
2403 inject.tool.finished_round = perf_event__drop_oe;
2404 }
2405 #endif
2406 ret = symbol__init(&inject.session->header.env);
2407 if (ret < 0)
2408 goto out_delete;
2409
2410 ret = __cmd_inject(&inject);
2411
2412 guest_session__exit(&inject.guest_session);
2413
2414 out_delete:
2415 strlist__delete(inject.known_build_ids);
2416 zstd_fini(&(inject.session->zstd_data));
2417 perf_session__delete(inject.session);
2418 out_close_output:
2419 if (!inject.in_place_update)
2420 perf_data__close(&inject.output);
2421 free(inject.itrace_synth_opts.vm_tm_corr_args);
2422 free(inject.event_copy);
2423 free(inject.guest_session.ev.event_buf);
2424 return ret;
2425 }
2426
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.