1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Hypervisor supplied "24x7" performance counter support
4 *
5 * Author: Cody P Schafer <cody@linux.vnet.ibm.com>
6 * Copyright 2014 IBM Corporation.
7 */
8
9 #define pr_fmt(fmt) "hv-24x7: " fmt
10
11 #include <linux/perf_event.h>
12 #include <linux/rbtree.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16
17 #include <asm/cputhreads.h>
18 #include <asm/firmware.h>
19 #include <asm/hvcall.h>
20 #include <asm/io.h>
21 #include <asm/papr-sysparm.h>
22 #include <linux/byteorder/generic.h>
23
24 #include <asm/rtas.h>
25 #include "hv-24x7.h"
26 #include "hv-24x7-catalog.h"
27 #include "hv-common.h"
28
29 /* Version of the 24x7 hypervisor API that we should use in this machine. */
30 static int interface_version;
31
32 /* Whether we have to aggregate result data for some domains. */
33 static bool aggregate_result_elements;
34
35 static cpumask_t hv_24x7_cpumask;
36
37 static bool domain_is_valid(unsigned int domain)
38 {
39 switch (domain) {
40 #define DOMAIN(n, v, x, c) \
41 case HV_PERF_DOMAIN_##n: \
42 /* fall through */
43 #include "hv-24x7-domains.h"
44 #undef DOMAIN
45 return true;
46 default:
47 return false;
48 }
49 }
50
51 static bool is_physical_domain(unsigned int domain)
52 {
53 switch (domain) {
54 #define DOMAIN(n, v, x, c) \
55 case HV_PERF_DOMAIN_##n: \
56 return c;
57 #include "hv-24x7-domains.h"
58 #undef DOMAIN
59 default:
60 return false;
61 }
62 }
63
64 /*
65 * The Processor Module Information system parameter allows transferring
66 * of certain processor module information from the platform to the OS.
67 * Refer PAPR+ document to get parameter token value as '43'.
68 */
69
70 static u32 phys_sockets; /* Physical sockets */
71 static u32 phys_chipspersocket; /* Physical chips per socket*/
72 static u32 phys_coresperchip; /* Physical cores per chip */
73
74 /*
75 * read_24x7_sys_info()
76 * Retrieve the number of sockets and chips per socket and cores per
77 * chip details through the get-system-parameter rtas call.
78 */
79 void read_24x7_sys_info(void)
80 {
81 struct papr_sysparm_buf *buf;
82
83 /*
84 * Making system parameter: chips and sockets and cores per chip
85 * default to 1.
86 */
87 phys_sockets = 1;
88 phys_chipspersocket = 1;
89 phys_coresperchip = 1;
90
91 buf = papr_sysparm_buf_alloc();
92 if (!buf)
93 return;
94
95 if (!papr_sysparm_get(PAPR_SYSPARM_PROC_MODULE_INFO, buf)) {
96 int ntypes = be16_to_cpup((__be16 *)&buf->val[0]);
97 int len = be16_to_cpu(buf->len);
98
99 if (len >= 8 && ntypes != 0) {
100 phys_sockets = be16_to_cpup((__be16 *)&buf->val[2]);
101 phys_chipspersocket = be16_to_cpup((__be16 *)&buf->val[4]);
102 phys_coresperchip = be16_to_cpup((__be16 *)&buf->val[6]);
103 }
104 }
105
106 papr_sysparm_buf_free(buf);
107 }
108
109 /* Domains for which more than one result element are returned for each event. */
110 static bool domain_needs_aggregation(unsigned int domain)
111 {
112 return aggregate_result_elements &&
113 (domain == HV_PERF_DOMAIN_PHYS_CORE ||
114 (domain >= HV_PERF_DOMAIN_VCPU_HOME_CORE &&
115 domain <= HV_PERF_DOMAIN_VCPU_REMOTE_NODE));
116 }
117
118 static const char *domain_name(unsigned int domain)
119 {
120 if (!domain_is_valid(domain))
121 return NULL;
122
123 switch (domain) {
124 case HV_PERF_DOMAIN_PHYS_CHIP: return "Physical Chip";
125 case HV_PERF_DOMAIN_PHYS_CORE: return "Physical Core";
126 case HV_PERF_DOMAIN_VCPU_HOME_CORE: return "VCPU Home Core";
127 case HV_PERF_DOMAIN_VCPU_HOME_CHIP: return "VCPU Home Chip";
128 case HV_PERF_DOMAIN_VCPU_HOME_NODE: return "VCPU Home Node";
129 case HV_PERF_DOMAIN_VCPU_REMOTE_NODE: return "VCPU Remote Node";
130 }
131
132 WARN_ON_ONCE(domain);
133 return NULL;
134 }
135
136 static bool catalog_entry_domain_is_valid(unsigned int domain)
137 {
138 /* POWER8 doesn't support virtual domains. */
139 if (interface_version == 1)
140 return is_physical_domain(domain);
141 else
142 return domain_is_valid(domain);
143 }
144
145 /*
146 * TODO: Merging events:
147 * - Think of the hcall as an interface to a 4d array of counters:
148 * - x = domains
149 * - y = indexes in the domain (core, chip, vcpu, node, etc)
150 * - z = offset into the counter space
151 * - w = lpars (guest vms, "logical partitions")
152 * - A single request is: x,y,y_last,z,z_last,w,w_last
153 * - this means we can retrieve a rectangle of counters in y,z for a single x.
154 *
155 * - Things to consider (ignoring w):
156 * - input cost_per_request = 16
157 * - output cost_per_result(ys,zs) = 8 + 8 * ys + ys * zs
158 * - limited number of requests per hcall (must fit into 4K bytes)
159 * - 4k = 16 [buffer header] - 16 [request size] * request_count
160 * - 255 requests per hcall
161 * - sometimes it will be more efficient to read extra data and discard
162 */
163
164 /*
165 * Example usage:
166 * perf stat -e 'hv_24x7/domain=2,offset=8,vcpu=0,lpar=0xffffffff/'
167 */
168
169 /* u3 0-6, one of HV_24X7_PERF_DOMAIN */
170 EVENT_DEFINE_RANGE_FORMAT(domain, config, 0, 3);
171 /* u16 */
172 EVENT_DEFINE_RANGE_FORMAT(core, config, 16, 31);
173 EVENT_DEFINE_RANGE_FORMAT(chip, config, 16, 31);
174 EVENT_DEFINE_RANGE_FORMAT(vcpu, config, 16, 31);
175 /* u32, see "data_offset" */
176 EVENT_DEFINE_RANGE_FORMAT(offset, config, 32, 63);
177 /* u16 */
178 EVENT_DEFINE_RANGE_FORMAT(lpar, config1, 0, 15);
179
180 EVENT_DEFINE_RANGE(reserved1, config, 4, 15);
181 EVENT_DEFINE_RANGE(reserved2, config1, 16, 63);
182 EVENT_DEFINE_RANGE(reserved3, config2, 0, 63);
183
184 static struct attribute *format_attrs[] = {
185 &format_attr_domain.attr,
186 &format_attr_offset.attr,
187 &format_attr_core.attr,
188 &format_attr_chip.attr,
189 &format_attr_vcpu.attr,
190 &format_attr_lpar.attr,
191 NULL,
192 };
193
194 static const struct attribute_group format_group = {
195 .name = "format",
196 .attrs = format_attrs,
197 };
198
199 static struct attribute_group event_group = {
200 .name = "events",
201 /* .attrs is set in init */
202 };
203
204 static struct attribute_group event_desc_group = {
205 .name = "event_descs",
206 /* .attrs is set in init */
207 };
208
209 static struct attribute_group event_long_desc_group = {
210 .name = "event_long_descs",
211 /* .attrs is set in init */
212 };
213
214 static struct kmem_cache *hv_page_cache;
215
216 static DEFINE_PER_CPU(int, hv_24x7_txn_flags);
217 static DEFINE_PER_CPU(int, hv_24x7_txn_err);
218
219 struct hv_24x7_hw {
220 struct perf_event *events[255];
221 };
222
223 static DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);
224
225 /*
226 * request_buffer and result_buffer are not required to be 4k aligned,
227 * but are not allowed to cross any 4k boundary. Aligning them to 4k is
228 * the simplest way to ensure that.
229 */
230 #define H24x7_DATA_BUFFER_SIZE 4096
231 static DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
232 static DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
233
234 static unsigned int max_num_requests(int interface_version)
235 {
236 return (H24x7_DATA_BUFFER_SIZE - sizeof(struct hv_24x7_request_buffer))
237 / H24x7_REQUEST_SIZE(interface_version);
238 }
239
240 static char *event_name(struct hv_24x7_event_data *ev, int *len)
241 {
242 *len = be16_to_cpu(ev->event_name_len) - 2;
243 return (char *)ev->remainder;
244 }
245
246 static char *event_desc(struct hv_24x7_event_data *ev, int *len)
247 {
248 unsigned int nl = be16_to_cpu(ev->event_name_len);
249 __be16 *desc_len = (__be16 *)(ev->remainder + nl - 2);
250
251 *len = be16_to_cpu(*desc_len) - 2;
252 return (char *)ev->remainder + nl;
253 }
254
255 static char *event_long_desc(struct hv_24x7_event_data *ev, int *len)
256 {
257 unsigned int nl = be16_to_cpu(ev->event_name_len);
258 __be16 *desc_len_ = (__be16 *)(ev->remainder + nl - 2);
259 unsigned int desc_len = be16_to_cpu(*desc_len_);
260 __be16 *long_desc_len = (__be16 *)(ev->remainder + nl + desc_len - 2);
261
262 *len = be16_to_cpu(*long_desc_len) - 2;
263 return (char *)ev->remainder + nl + desc_len;
264 }
265
266 static bool event_fixed_portion_is_within(struct hv_24x7_event_data *ev,
267 void *end)
268 {
269 void *start = ev;
270
271 return (start + offsetof(struct hv_24x7_event_data, remainder)) < end;
272 }
273
274 /*
275 * Things we don't check:
276 * - padding for desc, name, and long/detailed desc is required to be '\0'
277 * bytes.
278 *
279 * Return NULL if we pass end,
280 * Otherwise return the address of the byte just following the event.
281 */
282 static void *event_end(struct hv_24x7_event_data *ev, void *end)
283 {
284 void *start = ev;
285 __be16 *dl_, *ldl_;
286 unsigned int dl, ldl;
287 unsigned int nl = be16_to_cpu(ev->event_name_len);
288
289 if (nl < 2) {
290 pr_debug("%s: name length too short: %d", __func__, nl);
291 return NULL;
292 }
293
294 if (start + nl > end) {
295 pr_debug("%s: start=%p + nl=%u > end=%p",
296 __func__, start, nl, end);
297 return NULL;
298 }
299
300 dl_ = (__be16 *)(ev->remainder + nl - 2);
301 if (!IS_ALIGNED((uintptr_t)dl_, 2))
302 pr_warn("desc len not aligned %p", dl_);
303 dl = be16_to_cpu(*dl_);
304 if (dl < 2) {
305 pr_debug("%s: desc len too short: %d", __func__, dl);
306 return NULL;
307 }
308
309 if (start + nl + dl > end) {
310 pr_debug("%s: (start=%p + nl=%u + dl=%u)=%p > end=%p",
311 __func__, start, nl, dl, start + nl + dl, end);
312 return NULL;
313 }
314
315 ldl_ = (__be16 *)(ev->remainder + nl + dl - 2);
316 if (!IS_ALIGNED((uintptr_t)ldl_, 2))
317 pr_warn("long desc len not aligned %p", ldl_);
318 ldl = be16_to_cpu(*ldl_);
319 if (ldl < 2) {
320 pr_debug("%s: long desc len too short (ldl=%u)",
321 __func__, ldl);
322 return NULL;
323 }
324
325 if (start + nl + dl + ldl > end) {
326 pr_debug("%s: start=%p + nl=%u + dl=%u + ldl=%u > end=%p",
327 __func__, start, nl, dl, ldl, end);
328 return NULL;
329 }
330
331 return start + nl + dl + ldl;
332 }
333
334 static long h_get_24x7_catalog_page_(unsigned long phys_4096,
335 unsigned long version, unsigned long index)
336 {
337 pr_devel("h_get_24x7_catalog_page(0x%lx, %lu, %lu)",
338 phys_4096, version, index);
339
340 WARN_ON(!IS_ALIGNED(phys_4096, 4096));
341
342 return plpar_hcall_norets(H_GET_24X7_CATALOG_PAGE,
343 phys_4096, version, index);
344 }
345
346 static long h_get_24x7_catalog_page(char page[], u64 version, u32 index)
347 {
348 return h_get_24x7_catalog_page_(virt_to_phys(page),
349 version, index);
350 }
351
352 /*
353 * Each event we find in the catalog, will have a sysfs entry. Format the
354 * data for this sysfs entry based on the event's domain.
355 *
356 * Events belonging to the Chip domain can only be monitored in that domain.
357 * i.e the domain for these events is a fixed/knwon value.
358 *
359 * Events belonging to the Core domain can be monitored either in the physical
360 * core or in one of the virtual CPU domains. So the domain value for these
361 * events must be specified by the user (i.e is a required parameter). Format
362 * the Core events with 'domain=?' so the perf-tool can error check required
363 * parameters.
364 *
365 * NOTE: For the Core domain events, rather than making domain a required
366 * parameter we could default it to PHYS_CORE and allowe users to
367 * override the domain to one of the VCPU domains.
368 *
369 * However, this can make the interface a little inconsistent.
370 *
371 * If we set domain=2 (PHYS_CHIP) and allow user to override this field
372 * the user may be tempted to also modify the "offset=x" field in which
373 * can lead to confusing usage. Consider the HPM_PCYC (offset=0x18) and
374 * HPM_INST (offset=0x20) events. With:
375 *
376 * perf stat -e hv_24x7/HPM_PCYC,offset=0x20/
377 *
378 * we end up monitoring HPM_INST, while the command line has HPM_PCYC.
379 *
380 * By not assigning a default value to the domain for the Core events,
381 * we can have simple guidelines:
382 *
383 * - Specifying values for parameters with "=?" is required.
384 *
385 * - Specifying (i.e overriding) values for other parameters
386 * is undefined.
387 */
388 static char *event_fmt(struct hv_24x7_event_data *event, unsigned int domain)
389 {
390 const char *sindex;
391 const char *lpar;
392 const char *domain_str;
393 char buf[8];
394
395 switch (domain) {
396 case HV_PERF_DOMAIN_PHYS_CHIP:
397 snprintf(buf, sizeof(buf), "%d", domain);
398 domain_str = buf;
399 lpar = "0x0";
400 sindex = "chip";
401 break;
402 case HV_PERF_DOMAIN_PHYS_CORE:
403 domain_str = "?";
404 lpar = "0x0";
405 sindex = "core";
406 break;
407 default:
408 domain_str = "?";
409 lpar = "?";
410 sindex = "vcpu";
411 }
412
413 return kasprintf(GFP_KERNEL,
414 "domain=%s,offset=0x%x,%s=?,lpar=%s",
415 domain_str,
416 be16_to_cpu(event->event_counter_offs) +
417 be16_to_cpu(event->event_group_record_offs),
418 sindex,
419 lpar);
420 }
421
422 /* Avoid trusting fw to NUL terminate strings */
423 static char *memdup_to_str(char *maybe_str, int max_len, gfp_t gfp)
424 {
425 return kasprintf(gfp, "%.*s", max_len, maybe_str);
426 }
427
428 static ssize_t cpumask_show(struct device *dev,
429 struct device_attribute *attr, char *buf)
430 {
431 return cpumap_print_to_pagebuf(true, buf, &hv_24x7_cpumask);
432 }
433
434 static ssize_t sockets_show(struct device *dev,
435 struct device_attribute *attr, char *buf)
436 {
437 return sprintf(buf, "%d\n", phys_sockets);
438 }
439
440 static ssize_t chipspersocket_show(struct device *dev,
441 struct device_attribute *attr, char *buf)
442 {
443 return sprintf(buf, "%d\n", phys_chipspersocket);
444 }
445
446 static ssize_t coresperchip_show(struct device *dev,
447 struct device_attribute *attr, char *buf)
448 {
449 return sprintf(buf, "%d\n", phys_coresperchip);
450 }
451
452 static struct attribute *device_str_attr_create_(char *name, char *str)
453 {
454 struct dev_ext_attribute *attr = kzalloc(sizeof(*attr), GFP_KERNEL);
455
456 if (!attr)
457 return NULL;
458
459 sysfs_attr_init(&attr->attr.attr);
460
461 attr->var = str;
462 attr->attr.attr.name = name;
463 attr->attr.attr.mode = 0444;
464 attr->attr.show = device_show_string;
465
466 return &attr->attr.attr;
467 }
468
469 /*
470 * Allocate and initialize strings representing event attributes.
471 *
472 * NOTE: The strings allocated here are never destroyed and continue to
473 * exist till shutdown. This is to allow us to create as many events
474 * from the catalog as possible, even if we encounter errors with some.
475 * In case of changes to error paths in future, these may need to be
476 * freed by the caller.
477 */
478 static struct attribute *device_str_attr_create(char *name, int name_max,
479 int name_nonce,
480 char *str, size_t str_max)
481 {
482 char *n;
483 char *s = memdup_to_str(str, str_max, GFP_KERNEL);
484 struct attribute *a;
485
486 if (!s)
487 return NULL;
488
489 if (!name_nonce)
490 n = kasprintf(GFP_KERNEL, "%.*s", name_max, name);
491 else
492 n = kasprintf(GFP_KERNEL, "%.*s__%d", name_max, name,
493 name_nonce);
494 if (!n)
495 goto out_s;
496
497 a = device_str_attr_create_(n, s);
498 if (!a)
499 goto out_n;
500
501 return a;
502 out_n:
503 kfree(n);
504 out_s:
505 kfree(s);
506 return NULL;
507 }
508
509 static struct attribute *event_to_attr(unsigned int ix,
510 struct hv_24x7_event_data *event,
511 unsigned int domain,
512 int nonce)
513 {
514 int event_name_len;
515 char *ev_name, *a_ev_name, *val;
516 struct attribute *attr;
517
518 if (!domain_is_valid(domain)) {
519 pr_warn("catalog event %u has invalid domain %u\n",
520 ix, domain);
521 return NULL;
522 }
523
524 val = event_fmt(event, domain);
525 if (!val)
526 return NULL;
527
528 ev_name = event_name(event, &event_name_len);
529 if (!nonce)
530 a_ev_name = kasprintf(GFP_KERNEL, "%.*s",
531 (int)event_name_len, ev_name);
532 else
533 a_ev_name = kasprintf(GFP_KERNEL, "%.*s__%d",
534 (int)event_name_len, ev_name, nonce);
535
536 if (!a_ev_name)
537 goto out_val;
538
539 attr = device_str_attr_create_(a_ev_name, val);
540 if (!attr)
541 goto out_name;
542
543 return attr;
544 out_name:
545 kfree(a_ev_name);
546 out_val:
547 kfree(val);
548 return NULL;
549 }
550
551 static struct attribute *event_to_desc_attr(struct hv_24x7_event_data *event,
552 int nonce)
553 {
554 int nl, dl;
555 char *name = event_name(event, &nl);
556 char *desc = event_desc(event, &dl);
557
558 /* If there isn't a description, don't create the sysfs file */
559 if (!dl)
560 return NULL;
561
562 return device_str_attr_create(name, nl, nonce, desc, dl);
563 }
564
565 static struct attribute *
566 event_to_long_desc_attr(struct hv_24x7_event_data *event, int nonce)
567 {
568 int nl, dl;
569 char *name = event_name(event, &nl);
570 char *desc = event_long_desc(event, &dl);
571
572 /* If there isn't a description, don't create the sysfs file */
573 if (!dl)
574 return NULL;
575
576 return device_str_attr_create(name, nl, nonce, desc, dl);
577 }
578
579 static int event_data_to_attrs(unsigned int ix, struct attribute **attrs,
580 struct hv_24x7_event_data *event, int nonce)
581 {
582 *attrs = event_to_attr(ix, event, event->domain, nonce);
583 if (!*attrs)
584 return -1;
585
586 return 0;
587 }
588
589 /* */
590 struct event_uniq {
591 struct rb_node node;
592 const char *name;
593 int nl;
594 unsigned int ct;
595 unsigned int domain;
596 };
597
598 static int memord(const void *d1, size_t s1, const void *d2, size_t s2)
599 {
600 if (s1 < s2)
601 return 1;
602 if (s1 > s2)
603 return -1;
604
605 return memcmp(d1, d2, s1);
606 }
607
608 static int ev_uniq_ord(const void *v1, size_t s1, unsigned int d1,
609 const void *v2, size_t s2, unsigned int d2)
610 {
611 int r = memord(v1, s1, v2, s2);
612
613 if (r)
614 return r;
615 if (d1 > d2)
616 return 1;
617 if (d2 > d1)
618 return -1;
619 return 0;
620 }
621
622 static int event_uniq_add(struct rb_root *root, const char *name, int nl,
623 unsigned int domain)
624 {
625 struct rb_node **new = &(root->rb_node), *parent = NULL;
626 struct event_uniq *data;
627
628 /* Figure out where to put new node */
629 while (*new) {
630 struct event_uniq *it;
631 int result;
632
633 it = rb_entry(*new, struct event_uniq, node);
634 result = ev_uniq_ord(name, nl, domain, it->name, it->nl,
635 it->domain);
636
637 parent = *new;
638 if (result < 0)
639 new = &((*new)->rb_left);
640 else if (result > 0)
641 new = &((*new)->rb_right);
642 else {
643 it->ct++;
644 pr_info("found a duplicate event %.*s, ct=%u\n", nl,
645 name, it->ct);
646 return it->ct;
647 }
648 }
649
650 data = kmalloc(sizeof(*data), GFP_KERNEL);
651 if (!data)
652 return -ENOMEM;
653
654 *data = (struct event_uniq) {
655 .name = name,
656 .nl = nl,
657 .ct = 0,
658 .domain = domain,
659 };
660
661 /* Add new node and rebalance tree. */
662 rb_link_node(&data->node, parent, new);
663 rb_insert_color(&data->node, root);
664
665 /* data->ct */
666 return 0;
667 }
668
669 static void event_uniq_destroy(struct rb_root *root)
670 {
671 /*
672 * the strings we point to are in the giant block of memory filled by
673 * the catalog, and are freed separately.
674 */
675 struct event_uniq *pos, *n;
676
677 rbtree_postorder_for_each_entry_safe(pos, n, root, node)
678 kfree(pos);
679 }
680
681
682 /*
683 * ensure the event structure's sizes are self consistent and don't cause us to
684 * read outside of the event
685 *
686 * On success, return the event length in bytes.
687 * Otherwise, return -1 (and print as appropriate).
688 */
689 static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event,
690 size_t event_idx,
691 size_t event_data_bytes,
692 size_t event_entry_count,
693 size_t offset, void *end)
694 {
695 ssize_t ev_len;
696 void *ev_end, *calc_ev_end;
697
698 if (offset >= event_data_bytes)
699 return -1;
700
701 if (event_idx >= event_entry_count) {
702 pr_devel("catalog event data has %zu bytes of padding after last event\n",
703 event_data_bytes - offset);
704 return -1;
705 }
706
707 if (!event_fixed_portion_is_within(event, end)) {
708 pr_warn("event %zu fixed portion is not within range\n",
709 event_idx);
710 return -1;
711 }
712
713 ev_len = be16_to_cpu(event->length);
714
715 if (ev_len % 16)
716 pr_info("event %zu has length %zu not divisible by 16: event=%pK\n",
717 event_idx, ev_len, event);
718
719 ev_end = (__u8 *)event + ev_len;
720 if (ev_end > end) {
721 pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%pK > end=%pK, offset=%zu\n",
722 event_idx, ev_len, ev_end, end,
723 offset);
724 return -1;
725 }
726
727 calc_ev_end = event_end(event, end);
728 if (!calc_ev_end) {
729 pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%pK end=%pK, offset=%zu\n",
730 event_idx, event_data_bytes, event, end,
731 offset);
732 return -1;
733 }
734
735 if (calc_ev_end > ev_end) {
736 pr_warn("event %zu exceeds its own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n",
737 event_idx, event, ev_end, offset, calc_ev_end);
738 return -1;
739 }
740
741 return ev_len;
742 }
743
744 /*
745 * Return true incase of invalid or dummy events with names like RESERVED*
746 */
747 static bool ignore_event(const char *name)
748 {
749 return strncmp(name, "RESERVED", 8) == 0;
750 }
751
752 #define MAX_4K (SIZE_MAX / 4096)
753
754 static int create_events_from_catalog(struct attribute ***events_,
755 struct attribute ***event_descs_,
756 struct attribute ***event_long_descs_)
757 {
758 long hret;
759 size_t catalog_len, catalog_page_len, event_entry_count,
760 event_data_len, event_data_offs,
761 event_data_bytes, junk_events, event_idx, event_attr_ct, i,
762 attr_max, event_idx_last, desc_ct, long_desc_ct;
763 ssize_t ct, ev_len;
764 uint64_t catalog_version_num;
765 struct attribute **events, **event_descs, **event_long_descs;
766 struct hv_24x7_catalog_page_0 *page_0 =
767 kmem_cache_alloc(hv_page_cache, GFP_KERNEL);
768 void *page = page_0;
769 void *event_data, *end;
770 struct hv_24x7_event_data *event;
771 struct rb_root ev_uniq = RB_ROOT;
772 int ret = 0;
773
774 if (!page) {
775 ret = -ENOMEM;
776 goto e_out;
777 }
778
779 hret = h_get_24x7_catalog_page(page, 0, 0);
780 if (hret) {
781 ret = -EIO;
782 goto e_free;
783 }
784
785 catalog_version_num = be64_to_cpu(page_0->version);
786 catalog_page_len = be32_to_cpu(page_0->length);
787
788 if (MAX_4K < catalog_page_len) {
789 pr_err("invalid page count: %zu\n", catalog_page_len);
790 ret = -EIO;
791 goto e_free;
792 }
793
794 catalog_len = catalog_page_len * 4096;
795
796 event_entry_count = be16_to_cpu(page_0->event_entry_count);
797 event_data_offs = be16_to_cpu(page_0->event_data_offs);
798 event_data_len = be16_to_cpu(page_0->event_data_len);
799
800 pr_devel("cv %llu cl %zu eec %zu edo %zu edl %zu\n",
801 catalog_version_num, catalog_len,
802 event_entry_count, event_data_offs, event_data_len);
803
804 if ((MAX_4K < event_data_len)
805 || (MAX_4K < event_data_offs)
806 || (MAX_4K - event_data_offs < event_data_len)) {
807 pr_err("invalid event data offs %zu and/or len %zu\n",
808 event_data_offs, event_data_len);
809 ret = -EIO;
810 goto e_free;
811 }
812
813 if ((event_data_offs + event_data_len) > catalog_page_len) {
814 pr_err("event data %zu-%zu does not fit inside catalog 0-%zu\n",
815 event_data_offs,
816 event_data_offs + event_data_len,
817 catalog_page_len);
818 ret = -EIO;
819 goto e_free;
820 }
821
822 if (SIZE_MAX - 1 < event_entry_count) {
823 pr_err("event_entry_count %zu is invalid\n", event_entry_count);
824 ret = -EIO;
825 goto e_free;
826 }
827
828 event_data_bytes = event_data_len * 4096;
829
830 /*
831 * event data can span several pages, events can cross between these
832 * pages. Use vmalloc to make this easier.
833 */
834 event_data = vmalloc(event_data_bytes);
835 if (!event_data) {
836 pr_err("could not allocate event data\n");
837 ret = -ENOMEM;
838 goto e_free;
839 }
840
841 end = event_data + event_data_bytes;
842
843 /*
844 * using vmalloc_to_phys() like this only works if PAGE_SIZE is
845 * divisible by 4096
846 */
847 BUILD_BUG_ON(PAGE_SIZE % 4096);
848
849 for (i = 0; i < event_data_len; i++) {
850 hret = h_get_24x7_catalog_page_(
851 vmalloc_to_phys(event_data + i * 4096),
852 catalog_version_num,
853 i + event_data_offs);
854 if (hret) {
855 pr_err("Failed to get event data in page %zu: rc=%ld\n",
856 i + event_data_offs, hret);
857 ret = -EIO;
858 goto e_event_data;
859 }
860 }
861
862 /*
863 * scan the catalog to determine the number of attributes we need, and
864 * verify it at the same time.
865 */
866 for (junk_events = 0, event = event_data, event_idx = 0, attr_max = 0;
867 ;
868 event_idx++, event = (void *)event + ev_len) {
869 size_t offset = (void *)event - (void *)event_data;
870 char *name;
871 int nl;
872
873 ev_len = catalog_event_len_validate(event, event_idx,
874 event_data_bytes,
875 event_entry_count,
876 offset, end);
877 if (ev_len < 0)
878 break;
879
880 name = event_name(event, &nl);
881
882 if (ignore_event(name)) {
883 junk_events++;
884 continue;
885 }
886 if (event->event_group_record_len == 0) {
887 pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n",
888 event_idx, nl, name);
889 junk_events++;
890 continue;
891 }
892
893 if (!catalog_entry_domain_is_valid(event->domain)) {
894 pr_info("event %zu (%.*s) has invalid domain %d\n",
895 event_idx, nl, name, event->domain);
896 junk_events++;
897 continue;
898 }
899
900 attr_max++;
901 }
902
903 event_idx_last = event_idx;
904 if (event_idx_last != event_entry_count)
905 pr_warn("event buffer ended before listed # of events were parsed (got %zu, wanted %zu, junk %zu)\n",
906 event_idx_last, event_entry_count, junk_events);
907
908 events = kmalloc_array(attr_max + 1, sizeof(*events), GFP_KERNEL);
909 if (!events) {
910 ret = -ENOMEM;
911 goto e_event_data;
912 }
913
914 event_descs = kmalloc_array(event_idx + 1, sizeof(*event_descs),
915 GFP_KERNEL);
916 if (!event_descs) {
917 ret = -ENOMEM;
918 goto e_event_attrs;
919 }
920
921 event_long_descs = kmalloc_array(event_idx + 1,
922 sizeof(*event_long_descs), GFP_KERNEL);
923 if (!event_long_descs) {
924 ret = -ENOMEM;
925 goto e_event_descs;
926 }
927
928 /* Iterate over the catalog filling in the attribute vector */
929 for (junk_events = 0, event_attr_ct = 0, desc_ct = 0, long_desc_ct = 0,
930 event = event_data, event_idx = 0;
931 event_idx < event_idx_last;
932 event_idx++, ev_len = be16_to_cpu(event->length),
933 event = (void *)event + ev_len) {
934 char *name;
935 int nl;
936 int nonce;
937 /*
938 * these are the only "bad" events that are intermixed and that
939 * we can ignore without issue. make sure to skip them here
940 */
941 if (event->event_group_record_len == 0)
942 continue;
943 if (!catalog_entry_domain_is_valid(event->domain))
944 continue;
945
946 name = event_name(event, &nl);
947 if (ignore_event(name))
948 continue;
949
950 nonce = event_uniq_add(&ev_uniq, name, nl, event->domain);
951 ct = event_data_to_attrs(event_idx, events + event_attr_ct,
952 event, nonce);
953 if (ct < 0) {
954 pr_warn("event %zu (%.*s) creation failure, skipping\n",
955 event_idx, nl, name);
956 junk_events++;
957 } else {
958 event_attr_ct++;
959 event_descs[desc_ct] = event_to_desc_attr(event, nonce);
960 if (event_descs[desc_ct])
961 desc_ct++;
962 event_long_descs[long_desc_ct] =
963 event_to_long_desc_attr(event, nonce);
964 if (event_long_descs[long_desc_ct])
965 long_desc_ct++;
966 }
967 }
968
969 pr_info("read %zu catalog entries, created %zu event attrs (%zu failures), %zu descs\n",
970 event_idx, event_attr_ct, junk_events, desc_ct);
971
972 events[event_attr_ct] = NULL;
973 event_descs[desc_ct] = NULL;
974 event_long_descs[long_desc_ct] = NULL;
975
976 event_uniq_destroy(&ev_uniq);
977 vfree(event_data);
978 kmem_cache_free(hv_page_cache, page);
979
980 *events_ = events;
981 *event_descs_ = event_descs;
982 *event_long_descs_ = event_long_descs;
983 return 0;
984
985 e_event_descs:
986 kfree(event_descs);
987 e_event_attrs:
988 kfree(events);
989 e_event_data:
990 vfree(event_data);
991 e_free:
992 kmem_cache_free(hv_page_cache, page);
993 e_out:
994 *events_ = NULL;
995 *event_descs_ = NULL;
996 *event_long_descs_ = NULL;
997 return ret;
998 }
999
1000 static ssize_t catalog_read(struct file *filp, struct kobject *kobj,
1001 struct bin_attribute *bin_attr, char *buf,
1002 loff_t offset, size_t count)
1003 {
1004 long hret;
1005 ssize_t ret = 0;
1006 size_t catalog_len = 0, catalog_page_len = 0;
1007 loff_t page_offset = 0;
1008 loff_t offset_in_page;
1009 size_t copy_len;
1010 uint64_t catalog_version_num = 0;
1011 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER);
1012 struct hv_24x7_catalog_page_0 *page_0 = page;
1013
1014 if (!page)
1015 return -ENOMEM;
1016
1017 hret = h_get_24x7_catalog_page(page, 0, 0);
1018 if (hret) {
1019 ret = -EIO;
1020 goto e_free;
1021 }
1022
1023 catalog_version_num = be64_to_cpu(page_0->version);
1024 catalog_page_len = be32_to_cpu(page_0->length);
1025 catalog_len = catalog_page_len * 4096;
1026
1027 page_offset = offset / 4096;
1028 offset_in_page = offset % 4096;
1029
1030 if (page_offset >= catalog_page_len)
1031 goto e_free;
1032
1033 if (page_offset != 0) {
1034 hret = h_get_24x7_catalog_page(page, catalog_version_num,
1035 page_offset);
1036 if (hret) {
1037 ret = -EIO;
1038 goto e_free;
1039 }
1040 }
1041
1042 copy_len = 4096 - offset_in_page;
1043 if (copy_len > count)
1044 copy_len = count;
1045
1046 memcpy(buf, page+offset_in_page, copy_len);
1047 ret = copy_len;
1048
1049 e_free:
1050 if (hret)
1051 pr_err("h_get_24x7_catalog_page(ver=%lld, page=%lld) failed:"
1052 " rc=%ld\n",
1053 catalog_version_num, page_offset, hret);
1054 kmem_cache_free(hv_page_cache, page);
1055
1056 pr_devel("catalog_read: offset=%lld(%lld) count=%zu "
1057 "catalog_len=%zu(%zu) => %zd\n", offset, page_offset,
1058 count, catalog_len, catalog_page_len, ret);
1059
1060 return ret;
1061 }
1062
1063 static ssize_t domains_show(struct device *dev, struct device_attribute *attr,
1064 char *page)
1065 {
1066 int d, n, count = 0;
1067 const char *str;
1068
1069 for (d = 0; d < HV_PERF_DOMAIN_MAX; d++) {
1070 str = domain_name(d);
1071 if (!str)
1072 continue;
1073
1074 n = sprintf(page, "%d: %s\n", d, str);
1075 if (n < 0)
1076 break;
1077
1078 count += n;
1079 page += n;
1080 }
1081 return count;
1082 }
1083
1084 #define PAGE_0_ATTR(_name, _fmt, _expr) \
1085 static ssize_t _name##_show(struct device *dev, \
1086 struct device_attribute *dev_attr, \
1087 char *buf) \
1088 { \
1089 long hret; \
1090 ssize_t ret = 0; \
1091 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); \
1092 struct hv_24x7_catalog_page_0 *page_0 = page; \
1093 if (!page) \
1094 return -ENOMEM; \
1095 hret = h_get_24x7_catalog_page(page, 0, 0); \
1096 if (hret) { \
1097 ret = -EIO; \
1098 goto e_free; \
1099 } \
1100 ret = sprintf(buf, _fmt, _expr); \
1101 e_free: \
1102 kmem_cache_free(hv_page_cache, page); \
1103 return ret; \
1104 } \
1105 static DEVICE_ATTR_RO(_name)
1106
1107 PAGE_0_ATTR(catalog_version, "%lld\n",
1108 (unsigned long long)be64_to_cpu(page_0->version));
1109 PAGE_0_ATTR(catalog_len, "%lld\n",
1110 (unsigned long long)be32_to_cpu(page_0->length) * 4096);
1111 static BIN_ATTR_RO(catalog, 0/* real length varies */);
1112 static DEVICE_ATTR_RO(domains);
1113 static DEVICE_ATTR_RO(sockets);
1114 static DEVICE_ATTR_RO(chipspersocket);
1115 static DEVICE_ATTR_RO(coresperchip);
1116 static DEVICE_ATTR_RO(cpumask);
1117
1118 static struct bin_attribute *if_bin_attrs[] = {
1119 &bin_attr_catalog,
1120 NULL,
1121 };
1122
1123 static struct attribute *cpumask_attrs[] = {
1124 &dev_attr_cpumask.attr,
1125 NULL,
1126 };
1127
1128 static const struct attribute_group cpumask_attr_group = {
1129 .attrs = cpumask_attrs,
1130 };
1131
1132 static struct attribute *if_attrs[] = {
1133 &dev_attr_catalog_len.attr,
1134 &dev_attr_catalog_version.attr,
1135 &dev_attr_domains.attr,
1136 &dev_attr_sockets.attr,
1137 &dev_attr_chipspersocket.attr,
1138 &dev_attr_coresperchip.attr,
1139 NULL,
1140 };
1141
1142 static const struct attribute_group if_group = {
1143 .name = "interface",
1144 .bin_attrs = if_bin_attrs,
1145 .attrs = if_attrs,
1146 };
1147
1148 static const struct attribute_group *attr_groups[] = {
1149 &format_group,
1150 &event_group,
1151 &event_desc_group,
1152 &event_long_desc_group,
1153 &if_group,
1154 &cpumask_attr_group,
1155 NULL,
1156 };
1157
1158 /*
1159 * Start the process for a new H_GET_24x7_DATA hcall.
1160 */
1161 static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1162 struct hv_24x7_data_result_buffer *result_buffer)
1163 {
1164
1165 memset(request_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1166 memset(result_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1167
1168 request_buffer->interface_version = interface_version;
1169 /* memset above set request_buffer->num_requests to 0 */
1170 }
1171
1172 /*
1173 * Commit (i.e perform) the H_GET_24x7_DATA hcall using the data collected
1174 * by 'init_24x7_request()' and 'add_event_to_24x7_request()'.
1175 */
1176 static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1177 struct hv_24x7_data_result_buffer *result_buffer)
1178 {
1179 long ret;
1180
1181 /*
1182 * NOTE: Due to variable number of array elements in request and
1183 * result buffer(s), sizeof() is not reliable. Use the actual
1184 * allocated buffer size, H24x7_DATA_BUFFER_SIZE.
1185 */
1186 ret = plpar_hcall_norets(H_GET_24X7_DATA,
1187 virt_to_phys(request_buffer), H24x7_DATA_BUFFER_SIZE,
1188 virt_to_phys(result_buffer), H24x7_DATA_BUFFER_SIZE);
1189
1190 if (ret) {
1191 struct hv_24x7_request *req;
1192
1193 req = request_buffer->requests;
1194 pr_notice_ratelimited("hcall failed: [%d %#x %#x %d] => ret 0x%lx (%ld) detail=0x%x failing ix=%x\n",
1195 req->performance_domain, req->data_offset,
1196 req->starting_ix, req->starting_lpar_ix,
1197 ret, ret, result_buffer->detailed_rc,
1198 result_buffer->failing_request_ix);
1199 return -EIO;
1200 }
1201
1202 return 0;
1203 }
1204
1205 /*
1206 * Add the given @event to the next slot in the 24x7 request_buffer.
1207 *
1208 * Note that H_GET_24X7_DATA hcall allows reading several counters'
1209 * values in a single HCALL. We expect the caller to add events to the
1210 * request buffer one by one, make the HCALL and process the results.
1211 */
1212 static int add_event_to_24x7_request(struct perf_event *event,
1213 struct hv_24x7_request_buffer *request_buffer)
1214 {
1215 u16 idx;
1216 int i;
1217 size_t req_size;
1218 struct hv_24x7_request *req;
1219
1220 if (request_buffer->num_requests >=
1221 max_num_requests(request_buffer->interface_version)) {
1222 pr_devel("Too many requests for 24x7 HCALL %d\n",
1223 request_buffer->num_requests);
1224 return -EINVAL;
1225 }
1226
1227 switch (event_get_domain(event)) {
1228 case HV_PERF_DOMAIN_PHYS_CHIP:
1229 idx = event_get_chip(event);
1230 break;
1231 case HV_PERF_DOMAIN_PHYS_CORE:
1232 idx = event_get_core(event);
1233 break;
1234 default:
1235 idx = event_get_vcpu(event);
1236 }
1237
1238 req_size = H24x7_REQUEST_SIZE(request_buffer->interface_version);
1239
1240 i = request_buffer->num_requests++;
1241 req = (void *) request_buffer->requests + i * req_size;
1242
1243 req->performance_domain = event_get_domain(event);
1244 req->data_size = cpu_to_be16(8);
1245 req->data_offset = cpu_to_be32(event_get_offset(event));
1246 req->starting_lpar_ix = cpu_to_be16(event_get_lpar(event));
1247 req->max_num_lpars = cpu_to_be16(1);
1248 req->starting_ix = cpu_to_be16(idx);
1249 req->max_ix = cpu_to_be16(1);
1250
1251 if (request_buffer->interface_version > 1) {
1252 if (domain_needs_aggregation(req->performance_domain))
1253 req->max_num_thread_groups = -1;
1254 else if (req->performance_domain != HV_PERF_DOMAIN_PHYS_CHIP) {
1255 req->starting_thread_group_ix = idx % 2;
1256 req->max_num_thread_groups = 1;
1257 }
1258 }
1259
1260 return 0;
1261 }
1262
1263 /**
1264 * get_count_from_result - get event count from all result elements in result
1265 *
1266 * If the event corresponding to this result needs aggregation of the result
1267 * element values, then this function does that.
1268 *
1269 * @event: Event associated with @res.
1270 * @resb: Result buffer containing @res.
1271 * @res: Result to work on.
1272 * @countp: Output variable containing the event count.
1273 * @next: Optional output variable pointing to the next result in @resb.
1274 */
1275 static int get_count_from_result(struct perf_event *event,
1276 struct hv_24x7_data_result_buffer *resb,
1277 struct hv_24x7_result *res, u64 *countp,
1278 struct hv_24x7_result **next)
1279 {
1280 u16 num_elements = be16_to_cpu(res->num_elements_returned);
1281 u16 data_size = be16_to_cpu(res->result_element_data_size);
1282 unsigned int data_offset;
1283 void *element_data;
1284 int i;
1285 u64 count;
1286
1287 /*
1288 * We can bail out early if the result is empty.
1289 */
1290 if (!num_elements) {
1291 pr_debug("Result of request %hhu is empty, nothing to do\n",
1292 res->result_ix);
1293
1294 if (next)
1295 *next = (struct hv_24x7_result *) res->elements;
1296
1297 return -ENODATA;
1298 }
1299
1300 /*
1301 * Since we always specify 1 as the maximum for the smallest resource
1302 * we're requesting, there should to be only one element per result.
1303 * Except when an event needs aggregation, in which case there are more.
1304 */
1305 if (num_elements != 1 &&
1306 !domain_needs_aggregation(event_get_domain(event))) {
1307 pr_err("Error: result of request %hhu has %hu elements\n",
1308 res->result_ix, num_elements);
1309
1310 return -EIO;
1311 }
1312
1313 if (data_size != sizeof(u64)) {
1314 pr_debug("Error: result of request %hhu has data of %hu bytes\n",
1315 res->result_ix, data_size);
1316
1317 return -ENOTSUPP;
1318 }
1319
1320 if (resb->interface_version == 1)
1321 data_offset = offsetof(struct hv_24x7_result_element_v1,
1322 element_data);
1323 else
1324 data_offset = offsetof(struct hv_24x7_result_element_v2,
1325 element_data);
1326
1327 /* Go through the result elements in the result. */
1328 for (i = count = 0, element_data = res->elements + data_offset;
1329 i < num_elements;
1330 i++, element_data += data_size + data_offset)
1331 count += be64_to_cpu(*((__be64 *)element_data));
1332
1333 *countp = count;
1334
1335 /* The next result is after the last result element. */
1336 if (next)
1337 *next = element_data - data_offset;
1338
1339 return 0;
1340 }
1341
1342 static int single_24x7_request(struct perf_event *event, u64 *count)
1343 {
1344 int ret;
1345 struct hv_24x7_request_buffer *request_buffer;
1346 struct hv_24x7_data_result_buffer *result_buffer;
1347
1348 BUILD_BUG_ON(sizeof(*request_buffer) > 4096);
1349 BUILD_BUG_ON(sizeof(*result_buffer) > 4096);
1350
1351 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1352 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1353
1354 init_24x7_request(request_buffer, result_buffer);
1355
1356 ret = add_event_to_24x7_request(event, request_buffer);
1357 if (ret)
1358 goto out;
1359
1360 ret = make_24x7_request(request_buffer, result_buffer);
1361 if (ret)
1362 goto out;
1363
1364 /* process result from hcall */
1365 ret = get_count_from_result(event, result_buffer,
1366 result_buffer->results, count, NULL);
1367
1368 out:
1369 put_cpu_var(hv_24x7_reqb);
1370 put_cpu_var(hv_24x7_resb);
1371 return ret;
1372 }
1373
1374
1375 static int h_24x7_event_init(struct perf_event *event)
1376 {
1377 struct hv_perf_caps caps;
1378 unsigned int domain;
1379 unsigned long hret;
1380 u64 ct;
1381
1382 /* Not our event */
1383 if (event->attr.type != event->pmu->type)
1384 return -ENOENT;
1385
1386 /* Unused areas must be 0 */
1387 if (event_get_reserved1(event) ||
1388 event_get_reserved2(event) ||
1389 event_get_reserved3(event)) {
1390 pr_devel("reserved set when forbidden 0x%llx(0x%llx) 0x%llx(0x%llx) 0x%llx(0x%llx)\n",
1391 event->attr.config,
1392 event_get_reserved1(event),
1393 event->attr.config1,
1394 event_get_reserved2(event),
1395 event->attr.config2,
1396 event_get_reserved3(event));
1397 return -EINVAL;
1398 }
1399
1400 /* no branch sampling */
1401 if (has_branch_stack(event))
1402 return -EOPNOTSUPP;
1403
1404 /* offset must be 8 byte aligned */
1405 if (event_get_offset(event) % 8) {
1406 pr_devel("bad alignment\n");
1407 return -EINVAL;
1408 }
1409
1410 domain = event_get_domain(event);
1411 if (domain == 0 || domain >= HV_PERF_DOMAIN_MAX) {
1412 pr_devel("invalid domain %d\n", domain);
1413 return -EINVAL;
1414 }
1415
1416 hret = hv_perf_caps_get(&caps);
1417 if (hret) {
1418 pr_devel("could not get capabilities: rc=%ld\n", hret);
1419 return -EIO;
1420 }
1421
1422 /* Physical domains & other lpars require extra capabilities */
1423 if (!caps.collect_privileged && (is_physical_domain(domain) ||
1424 (event_get_lpar(event) != event_get_lpar_max()))) {
1425 pr_devel("hv permissions disallow: is_physical_domain:%d, lpar=0x%llx\n",
1426 is_physical_domain(domain),
1427 event_get_lpar(event));
1428 return -EACCES;
1429 }
1430
1431 /* Get the initial value of the counter for this event */
1432 if (single_24x7_request(event, &ct)) {
1433 pr_devel("test hcall failed\n");
1434 return -EIO;
1435 }
1436 (void)local64_xchg(&event->hw.prev_count, ct);
1437
1438 return 0;
1439 }
1440
1441 static u64 h_24x7_get_value(struct perf_event *event)
1442 {
1443 u64 ct;
1444
1445 if (single_24x7_request(event, &ct))
1446 /* We checked this in event init, shouldn't fail here... */
1447 return 0;
1448
1449 return ct;
1450 }
1451
1452 static void update_event_count(struct perf_event *event, u64 now)
1453 {
1454 s64 prev;
1455
1456 prev = local64_xchg(&event->hw.prev_count, now);
1457 local64_add(now - prev, &event->count);
1458 }
1459
1460 static void h_24x7_event_read(struct perf_event *event)
1461 {
1462 u64 now;
1463 struct hv_24x7_request_buffer *request_buffer;
1464 struct hv_24x7_hw *h24x7hw;
1465 int txn_flags;
1466
1467 txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1468
1469 /*
1470 * If in a READ transaction, add this counter to the list of
1471 * counters to read during the next HCALL (i.e commit_txn()).
1472 * If not in a READ transaction, go ahead and make the HCALL
1473 * to read this counter by itself.
1474 */
1475
1476 if (txn_flags & PERF_PMU_TXN_READ) {
1477 int i;
1478 int ret;
1479
1480 if (__this_cpu_read(hv_24x7_txn_err))
1481 return;
1482
1483 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1484
1485 ret = add_event_to_24x7_request(event, request_buffer);
1486 if (ret) {
1487 __this_cpu_write(hv_24x7_txn_err, ret);
1488 } else {
1489 /*
1490 * Associate the event with the HCALL request index,
1491 * so ->commit_txn() can quickly find/update count.
1492 */
1493 i = request_buffer->num_requests - 1;
1494
1495 h24x7hw = &get_cpu_var(hv_24x7_hw);
1496 h24x7hw->events[i] = event;
1497 put_cpu_var(h24x7hw);
1498 }
1499
1500 put_cpu_var(hv_24x7_reqb);
1501 } else {
1502 now = h_24x7_get_value(event);
1503 update_event_count(event, now);
1504 }
1505 }
1506
1507 static void h_24x7_event_start(struct perf_event *event, int flags)
1508 {
1509 if (flags & PERF_EF_RELOAD)
1510 local64_set(&event->hw.prev_count, h_24x7_get_value(event));
1511 }
1512
1513 static void h_24x7_event_stop(struct perf_event *event, int flags)
1514 {
1515 h_24x7_event_read(event);
1516 }
1517
1518 static int h_24x7_event_add(struct perf_event *event, int flags)
1519 {
1520 if (flags & PERF_EF_START)
1521 h_24x7_event_start(event, flags);
1522
1523 return 0;
1524 }
1525
1526 /*
1527 * 24x7 counters only support READ transactions. They are
1528 * always counting and dont need/support ADD transactions.
1529 * Cache the flags, but otherwise ignore transactions that
1530 * are not PERF_PMU_TXN_READ.
1531 */
1532 static void h_24x7_event_start_txn(struct pmu *pmu, unsigned int flags)
1533 {
1534 struct hv_24x7_request_buffer *request_buffer;
1535 struct hv_24x7_data_result_buffer *result_buffer;
1536
1537 /* We should not be called if we are already in a txn */
1538 WARN_ON_ONCE(__this_cpu_read(hv_24x7_txn_flags));
1539
1540 __this_cpu_write(hv_24x7_txn_flags, flags);
1541 if (flags & ~PERF_PMU_TXN_READ)
1542 return;
1543
1544 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1545 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1546
1547 init_24x7_request(request_buffer, result_buffer);
1548
1549 put_cpu_var(hv_24x7_resb);
1550 put_cpu_var(hv_24x7_reqb);
1551 }
1552
1553 /*
1554 * Clean up transaction state.
1555 *
1556 * NOTE: Ignore state of request and result buffers for now.
1557 * We will initialize them during the next read/txn.
1558 */
1559 static void reset_txn(void)
1560 {
1561 __this_cpu_write(hv_24x7_txn_flags, 0);
1562 __this_cpu_write(hv_24x7_txn_err, 0);
1563 }
1564
1565 /*
1566 * 24x7 counters only support READ transactions. They are always counting
1567 * and dont need/support ADD transactions. Clear ->txn_flags but otherwise
1568 * ignore transactions that are not of type PERF_PMU_TXN_READ.
1569 *
1570 * For READ transactions, submit all pending 24x7 requests (i.e requests
1571 * that were queued by h_24x7_event_read()), to the hypervisor and update
1572 * the event counts.
1573 */
1574 static int h_24x7_event_commit_txn(struct pmu *pmu)
1575 {
1576 struct hv_24x7_request_buffer *request_buffer;
1577 struct hv_24x7_data_result_buffer *result_buffer;
1578 struct hv_24x7_result *res, *next_res;
1579 u64 count;
1580 int i, ret, txn_flags;
1581 struct hv_24x7_hw *h24x7hw;
1582
1583 txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1584 WARN_ON_ONCE(!txn_flags);
1585
1586 ret = 0;
1587 if (txn_flags & ~PERF_PMU_TXN_READ)
1588 goto out;
1589
1590 ret = __this_cpu_read(hv_24x7_txn_err);
1591 if (ret)
1592 goto out;
1593
1594 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1595 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1596
1597 ret = make_24x7_request(request_buffer, result_buffer);
1598 if (ret)
1599 goto put_reqb;
1600
1601 h24x7hw = &get_cpu_var(hv_24x7_hw);
1602
1603 /* Go through results in the result buffer to update event counts. */
1604 for (i = 0, res = result_buffer->results;
1605 i < result_buffer->num_results; i++, res = next_res) {
1606 struct perf_event *event = h24x7hw->events[res->result_ix];
1607
1608 ret = get_count_from_result(event, result_buffer, res, &count,
1609 &next_res);
1610 if (ret)
1611 break;
1612
1613 update_event_count(event, count);
1614 }
1615
1616 put_cpu_var(hv_24x7_hw);
1617
1618 put_reqb:
1619 put_cpu_var(hv_24x7_resb);
1620 put_cpu_var(hv_24x7_reqb);
1621 out:
1622 reset_txn();
1623 return ret;
1624 }
1625
1626 /*
1627 * 24x7 counters only support READ transactions. They are always counting
1628 * and dont need/support ADD transactions. However, regardless of type
1629 * of transaction, all we need to do is cleanup, so we don't have to check
1630 * the type of transaction.
1631 */
1632 static void h_24x7_event_cancel_txn(struct pmu *pmu)
1633 {
1634 WARN_ON_ONCE(!__this_cpu_read(hv_24x7_txn_flags));
1635 reset_txn();
1636 }
1637
1638 static struct pmu h_24x7_pmu = {
1639 .task_ctx_nr = perf_invalid_context,
1640
1641 .name = "hv_24x7",
1642 .attr_groups = attr_groups,
1643 .event_init = h_24x7_event_init,
1644 .add = h_24x7_event_add,
1645 .del = h_24x7_event_stop,
1646 .start = h_24x7_event_start,
1647 .stop = h_24x7_event_stop,
1648 .read = h_24x7_event_read,
1649 .start_txn = h_24x7_event_start_txn,
1650 .commit_txn = h_24x7_event_commit_txn,
1651 .cancel_txn = h_24x7_event_cancel_txn,
1652 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1653 };
1654
1655 static int ppc_hv_24x7_cpu_online(unsigned int cpu)
1656 {
1657 if (cpumask_empty(&hv_24x7_cpumask))
1658 cpumask_set_cpu(cpu, &hv_24x7_cpumask);
1659
1660 return 0;
1661 }
1662
1663 static int ppc_hv_24x7_cpu_offline(unsigned int cpu)
1664 {
1665 int target;
1666
1667 /* Check if exiting cpu is used for collecting 24x7 events */
1668 if (!cpumask_test_and_clear_cpu(cpu, &hv_24x7_cpumask))
1669 return 0;
1670
1671 /* Find a new cpu to collect 24x7 events */
1672 target = cpumask_last(cpu_active_mask);
1673
1674 if (target < 0 || target >= nr_cpu_ids) {
1675 pr_err("hv_24x7: CPU hotplug init failed\n");
1676 return -1;
1677 }
1678
1679 /* Migrate 24x7 events to the new target */
1680 cpumask_set_cpu(target, &hv_24x7_cpumask);
1681 perf_pmu_migrate_context(&h_24x7_pmu, cpu, target);
1682
1683 return 0;
1684 }
1685
1686 static int hv_24x7_cpu_hotplug_init(void)
1687 {
1688 return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE,
1689 "perf/powerpc/hv_24x7:online",
1690 ppc_hv_24x7_cpu_online,
1691 ppc_hv_24x7_cpu_offline);
1692 }
1693
1694 static int hv_24x7_init(void)
1695 {
1696 int r;
1697 unsigned long hret;
1698 unsigned int pvr = mfspr(SPRN_PVR);
1699 struct hv_perf_caps caps;
1700
1701 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
1702 pr_debug("not a virtualized system, not enabling\n");
1703 return -ENODEV;
1704 }
1705
1706 /* POWER8 only supports v1, while POWER9 only supports v2. */
1707 if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E ||
1708 PVR_VER(pvr) == PVR_POWER8NVL)
1709 interface_version = 1;
1710 else {
1711 interface_version = 2;
1712
1713 /* SMT8 in POWER9 needs to aggregate result elements. */
1714 if (threads_per_core == 8)
1715 aggregate_result_elements = true;
1716 }
1717
1718 hret = hv_perf_caps_get(&caps);
1719 if (hret) {
1720 pr_debug("could not obtain capabilities, not enabling, rc=%ld\n",
1721 hret);
1722 return -ENODEV;
1723 }
1724
1725 hv_page_cache = kmem_cache_create("hv-page-4096", 4096, 4096, 0, NULL);
1726 if (!hv_page_cache)
1727 return -ENOMEM;
1728
1729 /* sampling not supported */
1730 h_24x7_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1731
1732 r = create_events_from_catalog(&event_group.attrs,
1733 &event_desc_group.attrs,
1734 &event_long_desc_group.attrs);
1735
1736 if (r)
1737 return r;
1738
1739 /* init cpuhotplug */
1740 r = hv_24x7_cpu_hotplug_init();
1741 if (r)
1742 return r;
1743
1744 r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1);
1745 if (r)
1746 return r;
1747
1748 read_24x7_sys_info();
1749
1750 return 0;
1751 }
1752
1753 device_initcall(hv_24x7_init);
1754
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.