1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * intel_tpebs.c: Intel TPEBS support 3 * intel_tpebs.c: Intel TPEBS support
4 */ 4 */
5 5
6 6
7 #include <sys/param.h> 7 #include <sys/param.h>
8 #include <subcmd/run-command.h> 8 #include <subcmd/run-command.h>
9 #include <thread.h> 9 #include <thread.h>
10 #include "intel-tpebs.h" 10 #include "intel-tpebs.h"
11 #include <linux/list.h> 11 #include <linux/list.h>
12 #include <linux/zalloc.h> 12 #include <linux/zalloc.h>
13 #include <linux/err.h> 13 #include <linux/err.h>
14 #include "sample.h" 14 #include "sample.h"
15 #include "debug.h" 15 #include "debug.h"
16 #include "evlist.h" 16 #include "evlist.h"
17 #include "evsel.h" 17 #include "evsel.h"
18 #include "session.h" 18 #include "session.h"
19 #include "tool.h" 19 #include "tool.h"
20 #include "cpumap.h" 20 #include "cpumap.h"
21 #include "metricgroup.h" 21 #include "metricgroup.h"
22 #include <sys/stat.h> 22 #include <sys/stat.h>
23 #include <sys/file.h> 23 #include <sys/file.h>
24 #include <poll.h> 24 #include <poll.h>
25 #include <math.h> 25 #include <math.h>
26 26
27 #define PERF_DATA "-" 27 #define PERF_DATA "-"
28 28
29 bool tpebs_recording; 29 bool tpebs_recording;
30 static pid_t tpebs_pid = -1; 30 static pid_t tpebs_pid = -1;
31 static size_t tpebs_event_size; 31 static size_t tpebs_event_size;
32 static LIST_HEAD(tpebs_results); 32 static LIST_HEAD(tpebs_results);
33 static pthread_t tpebs_reader_thread; 33 static pthread_t tpebs_reader_thread;
34 static struct child_process *tpebs_cmd; 34 static struct child_process *tpebs_cmd;
35 35
36 struct tpebs_retire_lat { 36 struct tpebs_retire_lat {
37 struct list_head nd; 37 struct list_head nd;
38 /* Event name */ 38 /* Event name */
39 const char *name; 39 const char *name;
40 /* Event name with the TPEBS modifier 40 /* Event name with the TPEBS modifier R */
41 const char *tpebs_name; 41 const char *tpebs_name;
42 /* Count of retire_latency values foun 42 /* Count of retire_latency values found in sample data */
43 size_t count; 43 size_t count;
44 /* Sum of all the retire_latency value 44 /* Sum of all the retire_latency values in sample data */
45 int sum; 45 int sum;
46 /* Average of retire_latency, val = su 46 /* Average of retire_latency, val = sum / count */
47 double val; 47 double val;
48 }; 48 };
49 49
50 static int get_perf_record_args(const char **r 50 static int get_perf_record_args(const char **record_argv, char buf[],
51 const char *cp 51 const char *cpumap_buf)
52 { 52 {
53 struct tpebs_retire_lat *e; 53 struct tpebs_retire_lat *e;
54 int i = 0; 54 int i = 0;
55 55
56 pr_debug("tpebs: Prepare perf record f 56 pr_debug("tpebs: Prepare perf record for retire_latency\n");
57 57
58 record_argv[i++] = "perf"; 58 record_argv[i++] = "perf";
59 record_argv[i++] = "record"; 59 record_argv[i++] = "record";
60 record_argv[i++] = "-W"; 60 record_argv[i++] = "-W";
61 record_argv[i++] = "--synth=no"; 61 record_argv[i++] = "--synth=no";
62 record_argv[i++] = buf; 62 record_argv[i++] = buf;
63 63
64 if (!cpumap_buf) { 64 if (!cpumap_buf) {
65 pr_err("tpebs: Require cpumap 65 pr_err("tpebs: Require cpumap list to run sampling\n");
66 return -ECANCELED; 66 return -ECANCELED;
67 } 67 }
68 /* Use -C when cpumap_buf is not "-1" 68 /* Use -C when cpumap_buf is not "-1" */
69 if (strcmp(cpumap_buf, "-1")) { 69 if (strcmp(cpumap_buf, "-1")) {
70 record_argv[i++] = "-C"; 70 record_argv[i++] = "-C";
71 record_argv[i++] = cpumap_buf; 71 record_argv[i++] = cpumap_buf;
72 } 72 }
73 73
74 list_for_each_entry(e, &tpebs_results, 74 list_for_each_entry(e, &tpebs_results, nd) {
75 record_argv[i++] = "-e"; 75 record_argv[i++] = "-e";
76 record_argv[i++] = e->name; 76 record_argv[i++] = e->name;
77 } 77 }
78 78
79 record_argv[i++] = "-o"; 79 record_argv[i++] = "-o";
80 record_argv[i++] = PERF_DATA; 80 record_argv[i++] = PERF_DATA;
81 81
82 return 0; 82 return 0;
83 } 83 }
84 84
85 static int prepare_run_command(const char **ar 85 static int prepare_run_command(const char **argv)
86 { 86 {
87 tpebs_cmd = zalloc(sizeof(struct child 87 tpebs_cmd = zalloc(sizeof(struct child_process));
88 if (!tpebs_cmd) 88 if (!tpebs_cmd)
89 return -ENOMEM; 89 return -ENOMEM;
90 tpebs_cmd->argv = argv; 90 tpebs_cmd->argv = argv;
91 tpebs_cmd->out = -1; 91 tpebs_cmd->out = -1;
92 return 0; 92 return 0;
93 } 93 }
94 94
95 static int start_perf_record(int control_fd[], 95 static int start_perf_record(int control_fd[], int ack_fd[],
96 const char *cp 96 const char *cpumap_buf)
97 { 97 {
98 const char **record_argv; 98 const char **record_argv;
99 int ret; 99 int ret;
100 char buf[32]; 100 char buf[32];
101 101
102 scnprintf(buf, sizeof(buf), "--control 102 scnprintf(buf, sizeof(buf), "--control=fd:%d,%d", control_fd[0], ack_fd[1]);
103 103
104 record_argv = calloc(12 + 2 * tpebs_ev 104 record_argv = calloc(12 + 2 * tpebs_event_size, sizeof(char *));
105 if (!record_argv) 105 if (!record_argv)
106 return -ENOMEM; 106 return -ENOMEM;
107 107
108 ret = get_perf_record_args(record_argv 108 ret = get_perf_record_args(record_argv, buf, cpumap_buf);
109 if (ret) 109 if (ret)
110 goto out; 110 goto out;
111 111
112 ret = prepare_run_command(record_argv) 112 ret = prepare_run_command(record_argv);
113 if (ret) 113 if (ret)
114 goto out; 114 goto out;
115 ret = start_command(tpebs_cmd); 115 ret = start_command(tpebs_cmd);
116 out: 116 out:
117 free(record_argv); 117 free(record_argv);
118 return ret; 118 return ret;
119 } 119 }
120 120
121 static int process_sample_event(const struct p 121 static int process_sample_event(const struct perf_tool *tool __maybe_unused,
122 union perf_eve 122 union perf_event *event __maybe_unused,
123 struct perf_sa 123 struct perf_sample *sample,
124 struct evsel * 124 struct evsel *evsel,
125 struct machine 125 struct machine *machine __maybe_unused)
126 { 126 {
127 int ret = 0; 127 int ret = 0;
128 const char *evname; 128 const char *evname;
129 struct tpebs_retire_lat *t; 129 struct tpebs_retire_lat *t;
130 130
131 evname = evsel__name(evsel); 131 evname = evsel__name(evsel);
132 132
133 /* 133 /*
134 * Need to handle per core results? We 134 * Need to handle per core results? We are assuming average retire
135 * latency value will be used. Save th 135 * latency value will be used. Save the number of samples and the sum of
136 * retire latency value for each event 136 * retire latency value for each event.
137 */ 137 */
138 list_for_each_entry(t, &tpebs_results, 138 list_for_each_entry(t, &tpebs_results, nd) {
139 if (!strcmp(evname, t->name)) 139 if (!strcmp(evname, t->name)) {
140 t->count += 1; 140 t->count += 1;
141 t->sum += sample->reti 141 t->sum += sample->retire_lat;
142 t->val = (double) t->s 142 t->val = (double) t->sum / t->count;
143 break; 143 break;
144 } 144 }
145 } 145 }
146 146
147 return ret; 147 return ret;
148 } 148 }
149 149
150 static int process_feature_event(struct perf_s 150 static int process_feature_event(struct perf_session *session,
151 union perf_ev 151 union perf_event *event)
152 { 152 {
153 if (event->feat.feat_id < HEADER_LAST_ 153 if (event->feat.feat_id < HEADER_LAST_FEATURE)
154 return perf_event__process_fea 154 return perf_event__process_feature(session, event);
155 return 0; 155 return 0;
156 } 156 }
157 157
158 static void *__sample_reader(void *arg) 158 static void *__sample_reader(void *arg)
159 { 159 {
160 struct child_process *child = arg; 160 struct child_process *child = arg;
161 struct perf_session *session; 161 struct perf_session *session;
162 struct perf_data data = { 162 struct perf_data data = {
163 .mode = PERF_DATA_MODE_READ, 163 .mode = PERF_DATA_MODE_READ,
164 .path = PERF_DATA, 164 .path = PERF_DATA,
165 .file.fd = child->out, 165 .file.fd = child->out,
166 }; 166 };
167 struct perf_tool tool; 167 struct perf_tool tool;
168 168
169 perf_tool__init(&tool, /*ordered_event 169 perf_tool__init(&tool, /*ordered_events=*/false);
170 tool.sample = process_sample_event; 170 tool.sample = process_sample_event;
171 tool.feature = process_feature_event; 171 tool.feature = process_feature_event;
172 tool.attr = perf_event__process_attr; 172 tool.attr = perf_event__process_attr;
173 173
174 session = perf_session__new(&data, &to 174 session = perf_session__new(&data, &tool);
175 if (IS_ERR(session)) 175 if (IS_ERR(session))
176 return NULL; 176 return NULL;
177 perf_session__process_events(session); 177 perf_session__process_events(session);
178 perf_session__delete(session); 178 perf_session__delete(session);
179 179
180 return NULL; 180 return NULL;
181 } 181 }
182 182
183 /* 183 /*
184 * tpebs_stop - stop the sample data read thre 184 * tpebs_stop - stop the sample data read thread and the perf record process.
185 */ 185 */
186 static int tpebs_stop(void) 186 static int tpebs_stop(void)
187 { 187 {
188 int ret = 0; 188 int ret = 0;
189 189
190 /* Like tpebs_start, we should only ru 190 /* Like tpebs_start, we should only run tpebs_end once. */
191 if (tpebs_pid != -1) { 191 if (tpebs_pid != -1) {
192 kill(tpebs_cmd->pid, SIGTERM); 192 kill(tpebs_cmd->pid, SIGTERM);
193 tpebs_pid = -1; 193 tpebs_pid = -1;
194 pthread_join(tpebs_reader_thre 194 pthread_join(tpebs_reader_thread, NULL);
195 close(tpebs_cmd->out); 195 close(tpebs_cmd->out);
196 ret = finish_command(tpebs_cmd 196 ret = finish_command(tpebs_cmd);
197 if (ret == -ERR_RUN_COMMAND_WA 197 if (ret == -ERR_RUN_COMMAND_WAITPID_SIGNAL)
198 ret = 0; 198 ret = 0;
199 } 199 }
200 return ret; 200 return ret;
201 } 201 }
202 202
203 /* 203 /*
204 * tpebs_start - start tpebs execution. 204 * tpebs_start - start tpebs execution.
205 * @evsel_list: retire_latency evsels in this 205 * @evsel_list: retire_latency evsels in this list will be selected and sampled
206 * to get the average retire_latency value. 206 * to get the average retire_latency value.
207 * 207 *
208 * This function will be called from evlist le 208 * This function will be called from evlist level later when evlist__open() is
209 * called consistently. 209 * called consistently.
210 */ 210 */
211 int tpebs_start(struct evlist *evsel_list) 211 int tpebs_start(struct evlist *evsel_list)
212 { 212 {
213 int ret = 0; 213 int ret = 0;
214 struct evsel *evsel; 214 struct evsel *evsel;
215 char cpumap_buf[50]; 215 char cpumap_buf[50];
216 216
217 /* 217 /*
218 * We should only run tpebs_start when 218 * We should only run tpebs_start when tpebs_recording is enabled.
219 * And we should only run it once with 219 * And we should only run it once with all the required events.
220 */ 220 */
221 if (tpebs_pid != -1 || !tpebs_recordin 221 if (tpebs_pid != -1 || !tpebs_recording)
222 return 0; 222 return 0;
223 223
224 cpu_map__snprint(evsel_list->core.user 224 cpu_map__snprint(evsel_list->core.user_requested_cpus, cpumap_buf, sizeof(cpumap_buf));
225 /* 225 /*
226 * Prepare perf record for sampling ev 226 * Prepare perf record for sampling event retire_latency before fork and
227 * prepare workload 227 * prepare workload
228 */ 228 */
229 evlist__for_each_entry(evsel_list, evs 229 evlist__for_each_entry(evsel_list, evsel) {
230 int i; 230 int i;
231 char *name; 231 char *name;
232 struct tpebs_retire_lat *new; 232 struct tpebs_retire_lat *new;
233 233
234 if (!evsel->retire_lat) 234 if (!evsel->retire_lat)
235 continue; 235 continue;
236 236
237 pr_debug("tpebs: Retire_latenc 237 pr_debug("tpebs: Retire_latency of event %s is required\n", evsel->name);
238 for (i = strlen(evsel->name) - 238 for (i = strlen(evsel->name) - 1; i > 0; i--) {
239 if (evsel->name[i] == 239 if (evsel->name[i] == 'R')
240 break; 240 break;
241 } 241 }
242 if (i <= 0 || evsel->name[i] ! 242 if (i <= 0 || evsel->name[i] != 'R') {
243 ret = -1; 243 ret = -1;
244 goto err; 244 goto err;
245 } 245 }
246 246
247 name = strdup(evsel->name); 247 name = strdup(evsel->name);
248 if (!name) { 248 if (!name) {
249 ret = -ENOMEM; 249 ret = -ENOMEM;
250 goto err; 250 goto err;
251 } 251 }
252 name[i] = 'p'; 252 name[i] = 'p';
253 253
254 new = zalloc(sizeof(*new)); 254 new = zalloc(sizeof(*new));
255 if (!new) { 255 if (!new) {
256 ret = -1; 256 ret = -1;
257 zfree(name); 257 zfree(name);
258 goto err; 258 goto err;
259 } 259 }
260 new->name = name; 260 new->name = name;
261 new->tpebs_name = evsel->name; 261 new->tpebs_name = evsel->name;
262 list_add_tail(&new->nd, &tpebs 262 list_add_tail(&new->nd, &tpebs_results);
263 tpebs_event_size += 1; 263 tpebs_event_size += 1;
264 } 264 }
265 265
266 if (tpebs_event_size > 0) { 266 if (tpebs_event_size > 0) {
267 struct pollfd pollfd = { .even 267 struct pollfd pollfd = { .events = POLLIN, };
268 int control_fd[2], ack_fd[2], 268 int control_fd[2], ack_fd[2], len;
269 char ack_buf[8]; 269 char ack_buf[8];
270 270
271 /*Create control and ack fd fo 271 /*Create control and ack fd for --control*/
272 if (pipe(control_fd) < 0) { 272 if (pipe(control_fd) < 0) {
273 pr_err("tpebs: Failed 273 pr_err("tpebs: Failed to create control fifo");
274 ret = -1; 274 ret = -1;
275 goto out; 275 goto out;
276 } 276 }
277 if (pipe(ack_fd) < 0) { 277 if (pipe(ack_fd) < 0) {
278 pr_err("tpebs: Failed 278 pr_err("tpebs: Failed to create control fifo");
279 ret = -1; 279 ret = -1;
280 goto out; 280 goto out;
281 } 281 }
282 282
283 ret = start_perf_record(contro 283 ret = start_perf_record(control_fd, ack_fd, cpumap_buf);
284 if (ret) 284 if (ret)
285 goto out; 285 goto out;
286 tpebs_pid = tpebs_cmd->pid; 286 tpebs_pid = tpebs_cmd->pid;
287 if (pthread_create(&tpebs_read 287 if (pthread_create(&tpebs_reader_thread, NULL, __sample_reader, tpebs_cmd)) {
288 kill(tpebs_cmd->pid, S 288 kill(tpebs_cmd->pid, SIGTERM);
289 close(tpebs_cmd->out); 289 close(tpebs_cmd->out);
290 pr_err("Could not crea 290 pr_err("Could not create thread to process sample data.\n");
291 ret = -1; 291 ret = -1;
292 goto out; 292 goto out;
293 } 293 }
294 /* Wait for perf record initia 294 /* Wait for perf record initialization.*/
295 len = strlen(EVLIST_CTL_CMD_EN 295 len = strlen(EVLIST_CTL_CMD_ENABLE_TAG);
296 ret = write(control_fd[1], EVL 296 ret = write(control_fd[1], EVLIST_CTL_CMD_ENABLE_TAG, len);
297 if (ret != len) { 297 if (ret != len) {
298 pr_err("perf record co 298 pr_err("perf record control write control message failed\n");
299 goto out; 299 goto out;
300 } 300 }
301 301
302 /* wait for an ack */ 302 /* wait for an ack */
303 pollfd.fd = ack_fd[0]; 303 pollfd.fd = ack_fd[0];
304 304
305 /* 305 /*
306 * We need this poll to ensure 306 * We need this poll to ensure the ack_fd PIPE will not hang
307 * when perf record failed for 307 * when perf record failed for any reason. The timeout value
308 * 3000ms is an empirical sele 308 * 3000ms is an empirical selection.
309 */ 309 */
310 if (!poll(&pollfd, 1, 3000)) { 310 if (!poll(&pollfd, 1, 3000)) {
311 pr_err("tpebs failed: 311 pr_err("tpebs failed: perf record ack timeout\n");
312 ret = -1; 312 ret = -1;
313 goto out; 313 goto out;
314 } 314 }
315 315
316 if (!(pollfd.revents & POLLIN) 316 if (!(pollfd.revents & POLLIN)) {
317 pr_err("tpebs failed: 317 pr_err("tpebs failed: did not received an ack\n");
318 ret = -1; 318 ret = -1;
319 goto out; 319 goto out;
320 } 320 }
321 321
322 ret = read(ack_fd[0], ack_buf, 322 ret = read(ack_fd[0], ack_buf, sizeof(ack_buf));
323 if (ret > 0) 323 if (ret > 0)
324 ret = strcmp(ack_buf, 324 ret = strcmp(ack_buf, EVLIST_CTL_CMD_ACK_TAG);
325 else { 325 else {
326 pr_err("tpebs: perf re 326 pr_err("tpebs: perf record control ack failed\n");
327 goto out; 327 goto out;
328 } 328 }
329 out: 329 out:
330 close(control_fd[0]); 330 close(control_fd[0]);
331 close(control_fd[1]); 331 close(control_fd[1]);
332 close(ack_fd[0]); 332 close(ack_fd[0]);
333 close(ack_fd[1]); 333 close(ack_fd[1]);
334 } 334 }
335 err: 335 err:
336 if (ret) 336 if (ret)
337 tpebs_delete(); 337 tpebs_delete();
338 return ret; 338 return ret;
339 } 339 }
340 340
341 341
342 int tpebs_set_evsel(struct evsel *evsel, int c 342 int tpebs_set_evsel(struct evsel *evsel, int cpu_map_idx, int thread)
343 { 343 {
344 __u64 val; 344 __u64 val;
345 bool found = false; 345 bool found = false;
346 struct tpebs_retire_lat *t; 346 struct tpebs_retire_lat *t;
347 struct perf_counts_values *count; 347 struct perf_counts_values *count;
348 348
349 /* Non reitre_latency evsel should nev 349 /* Non reitre_latency evsel should never enter this function. */
350 if (!evsel__is_retire_lat(evsel)) 350 if (!evsel__is_retire_lat(evsel))
351 return -1; 351 return -1;
352 352
353 /* 353 /*
354 * Need to stop the forked record to e 354 * Need to stop the forked record to ensure get sampled data from the
355 * PIPE to process and get non-zero re 355 * PIPE to process and get non-zero retire_lat value for hybrid.
356 */ 356 */
357 tpebs_stop(); 357 tpebs_stop();
358 count = perf_counts(evsel->counts, cpu 358 count = perf_counts(evsel->counts, cpu_map_idx, thread);
359 359
360 list_for_each_entry(t, &tpebs_results, 360 list_for_each_entry(t, &tpebs_results, nd) {
361 if (t->tpebs_name == evsel->na 361 if (t->tpebs_name == evsel->name ||
362 (evsel->metric_id && !strc 362 (evsel->metric_id && !strcmp(t->tpebs_name, evsel->metric_id))) {
363 found = true; 363 found = true;
364 break; 364 break;
365 } 365 }
366 } 366 }
367 367
368 /* Set ena and run to non-zero */ 368 /* Set ena and run to non-zero */
369 count->ena = count->run = 1; 369 count->ena = count->run = 1;
370 count->lost = 0; 370 count->lost = 0;
371 371
372 if (!found) { 372 if (!found) {
373 /* 373 /*
374 * Set default value or 0 when 374 * Set default value or 0 when retire_latency for this event is
375 * not found from sampling dat 375 * not found from sampling data (record_tpebs not set or 0
376 * sample recorded). 376 * sample recorded).
377 */ 377 */
378 count->val = 0; 378 count->val = 0;
379 return 0; 379 return 0;
380 } 380 }
381 381
382 /* 382 /*
383 * Only set retire_latency value to th 383 * Only set retire_latency value to the first CPU and thread.
384 */ 384 */
385 if (cpu_map_idx == 0 && thread == 0) 385 if (cpu_map_idx == 0 && thread == 0)
386 val = rint(t->val); 386 val = rint(t->val);
387 else 387 else
388 val = 0; 388 val = 0;
389 389
390 count->val = val; 390 count->val = val;
391 return 0; 391 return 0;
392 } 392 }
393 393
394 static void tpebs_retire_lat__delete(struct tp 394 static void tpebs_retire_lat__delete(struct tpebs_retire_lat *r)
395 { 395 {
396 zfree(&r->name); 396 zfree(&r->name);
397 free(r); 397 free(r);
398 } 398 }
399 399
400 400
401 /* 401 /*
402 * tpebs_delete - delete tpebs related data an 402 * tpebs_delete - delete tpebs related data and stop the created thread and
403 * process by calling tpebs_stop(). 403 * process by calling tpebs_stop().
404 * 404 *
405 * This function is called from evlist_delete( 405 * This function is called from evlist_delete() and also from builtin-stat
406 * stat_handle_error(). If tpebs_start() is ca 406 * stat_handle_error(). If tpebs_start() is called from places other then perf
407 * stat, need to ensure tpebs_delete() is also 407 * stat, need to ensure tpebs_delete() is also called to safely free mem and
408 * close the data read thread and the forked p 408 * close the data read thread and the forked perf record process.
409 * 409 *
410 * This function is also called in evsel__clos 410 * This function is also called in evsel__close() to be symmetric with
411 * tpebs_start() being called in evsel__open() 411 * tpebs_start() being called in evsel__open(). We will update this call site
412 * when move tpebs_start() to evlist level. 412 * when move tpebs_start() to evlist level.
413 */ 413 */
414 void tpebs_delete(void) 414 void tpebs_delete(void)
415 { 415 {
416 struct tpebs_retire_lat *r, *rtmp; 416 struct tpebs_retire_lat *r, *rtmp;
417 417
418 if (tpebs_pid == -1) 418 if (tpebs_pid == -1)
419 return; 419 return;
420 420
421 tpebs_stop(); 421 tpebs_stop();
422 422
423 list_for_each_entry_safe(r, rtmp, &tpe 423 list_for_each_entry_safe(r, rtmp, &tpebs_results, nd) {
424 list_del_init(&r->nd); 424 list_del_init(&r->nd);
425 tpebs_retire_lat__delete(r); 425 tpebs_retire_lat__delete(r);
426 } 426 }
427 427
428 if (tpebs_cmd) { 428 if (tpebs_cmd) {
429 free(tpebs_cmd); 429 free(tpebs_cmd);
430 tpebs_cmd = NULL; 430 tpebs_cmd = NULL;
431 } 431 }
432 } 432 }
433 433
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.