1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance events ring-buffer code: 4 * 5 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 9 */ 10 11 #include <linux/perf_event.h> 12 #include <linux/vmalloc.h> 13 #include <linux/slab.h> 14 #include <linux/circ_buf.h> 15 #include <linux/poll.h> 16 #include <linux/nospec.h> 17 18 #include "internal.h" 19 20 static void perf_output_wakeup(struct perf_output_handle *handle) 21 { 22 atomic_set(&handle->rb->poll, EPOLLIN); 23 24 handle->event->pending_wakeup = 1; 25 26 if (*perf_event_fasync(handle->event) && !handle->event->pending_kill) 27 handle->event->pending_kill = POLL_IN; 28 29 irq_work_queue(&handle->event->pending_irq); 30 } 31 32 /* 33 * We need to ensure a later event_id doesn't publish a head when a former 34 * event isn't done writing. However since we need to deal with NMIs we 35 * cannot fully serialize things. 36 * 37 * We only publish the head (and generate a wakeup) when the outer-most 38 * event completes. 39 */ 40 static void perf_output_get_handle(struct perf_output_handle *handle) 41 { 42 struct perf_buffer *rb = handle->rb; 43 44 preempt_disable(); 45 46 /* 47 * Avoid an explicit LOAD/STORE such that architectures with memops 48 * can use them. 49 */ 50 (*(volatile unsigned int *)&rb->nest)++; 51 handle->wakeup = local_read(&rb->wakeup); 52 } 53 54 static void perf_output_put_handle(struct perf_output_handle *handle) 55 { 56 struct perf_buffer *rb = handle->rb; 57 unsigned long head; 58 unsigned int nest; 59 60 /* 61 * If this isn't the outermost nesting, we don't have to update 62 * @rb->user_page->data_head. 63 */ 64 nest = READ_ONCE(rb->nest); 65 if (nest > 1) { 66 WRITE_ONCE(rb->nest, nest - 1); 67 goto out; 68 } 69 70 again: 71 /* 72 * In order to avoid publishing a head value that goes backwards, 73 * we must ensure the load of @rb->head happens after we've 74 * incremented @rb->nest. 75 * 76 * Otherwise we can observe a @rb->head value before one published 77 * by an IRQ/NMI happening between the load and the increment. 78 */ 79 barrier(); 80 head = local_read(&rb->head); 81 82 /* 83 * IRQ/NMI can happen here and advance @rb->head, causing our 84 * load above to be stale. 85 */ 86 87 /* 88 * Since the mmap() consumer (userspace) can run on a different CPU: 89 * 90 * kernel user 91 * 92 * if (LOAD ->data_tail) { LOAD ->data_head 93 * (A) smp_rmb() (C) 94 * STORE $data LOAD $data 95 * smp_wmb() (B) smp_mb() (D) 96 * STORE ->data_head STORE ->data_tail 97 * } 98 * 99 * Where A pairs with D, and B pairs with C. 100 * 101 * In our case (A) is a control dependency that separates the load of 102 * the ->data_tail and the stores of $data. In case ->data_tail 103 * indicates there is no room in the buffer to store $data we do not. 104 * 105 * D needs to be a full barrier since it separates the data READ 106 * from the tail WRITE. 107 * 108 * For B a WMB is sufficient since it separates two WRITEs, and for C 109 * an RMB is sufficient since it separates two READs. 110 * 111 * See perf_output_begin(). 112 */ 113 smp_wmb(); /* B, matches C */ 114 WRITE_ONCE(rb->user_page->data_head, head); 115 116 /* 117 * We must publish the head before decrementing the nest count, 118 * otherwise an IRQ/NMI can publish a more recent head value and our 119 * write will (temporarily) publish a stale value. 120 */ 121 barrier(); 122 WRITE_ONCE(rb->nest, 0); 123 124 /* 125 * Ensure we decrement @rb->nest before we validate the @rb->head. 126 * Otherwise we cannot be sure we caught the 'last' nested update. 127 */ 128 barrier(); 129 if (unlikely(head != local_read(&rb->head))) { 130 WRITE_ONCE(rb->nest, 1); 131 goto again; 132 } 133 134 if (handle->wakeup != local_read(&rb->wakeup)) 135 perf_output_wakeup(handle); 136 137 out: 138 preempt_enable(); 139 } 140 141 static __always_inline bool 142 ring_buffer_has_space(unsigned long head, unsigned long tail, 143 unsigned long data_size, unsigned int size, 144 bool backward) 145 { 146 if (!backward) 147 return CIRC_SPACE(head, tail, data_size) >= size; 148 else 149 return CIRC_SPACE(tail, head, data_size) >= size; 150 } 151 152 static __always_inline int 153 __perf_output_begin(struct perf_output_handle *handle, 154 struct perf_sample_data *data, 155 struct perf_event *event, unsigned int size, 156 bool backward) 157 { 158 struct perf_buffer *rb; 159 unsigned long tail, offset, head; 160 int have_lost, page_shift; 161 struct { 162 struct perf_event_header header; 163 u64 id; 164 u64 lost; 165 } lost_event; 166 167 rcu_read_lock(); 168 /* 169 * For inherited events we send all the output towards the parent. 170 */ 171 if (event->parent) 172 event = event->parent; 173 174 rb = rcu_dereference(event->rb); 175 if (unlikely(!rb)) 176 goto out; 177 178 if (unlikely(rb->paused)) { 179 if (rb->nr_pages) { 180 local_inc(&rb->lost); 181 atomic64_inc(&event->lost_samples); 182 } 183 goto out; 184 } 185 186 handle->rb = rb; 187 handle->event = event; 188 189 have_lost = local_read(&rb->lost); 190 if (unlikely(have_lost)) { 191 size += sizeof(lost_event); 192 if (event->attr.sample_id_all) 193 size += event->id_header_size; 194 } 195 196 perf_output_get_handle(handle); 197 198 offset = local_read(&rb->head); 199 do { 200 head = offset; 201 tail = READ_ONCE(rb->user_page->data_tail); 202 if (!rb->overwrite) { 203 if (unlikely(!ring_buffer_has_space(head, tail, 204 perf_data_size(rb), 205 size, backward))) 206 goto fail; 207 } 208 209 /* 210 * The above forms a control dependency barrier separating the 211 * @tail load above from the data stores below. Since the @tail 212 * load is required to compute the branch to fail below. 213 * 214 * A, matches D; the full memory barrier userspace SHOULD issue 215 * after reading the data and before storing the new tail 216 * position. 217 * 218 * See perf_output_put_handle(). 219 */ 220 221 if (!backward) 222 head += size; 223 else 224 head -= size; 225 } while (!local_try_cmpxchg(&rb->head, &offset, head)); 226 227 if (backward) { 228 offset = head; 229 head = (u64)(-head); 230 } 231 232 /* 233 * We rely on the implied barrier() by local_cmpxchg() to ensure 234 * none of the data stores below can be lifted up by the compiler. 235 */ 236 237 if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) 238 local_add(rb->watermark, &rb->wakeup); 239 240 page_shift = PAGE_SHIFT + page_order(rb); 241 242 handle->page = (offset >> page_shift) & (rb->nr_pages - 1); 243 offset &= (1UL << page_shift) - 1; 244 handle->addr = rb->data_pages[handle->page] + offset; 245 handle->size = (1UL << page_shift) - offset; 246 247 if (unlikely(have_lost)) { 248 lost_event.header.size = sizeof(lost_event); 249 lost_event.header.type = PERF_RECORD_LOST; 250 lost_event.header.misc = 0; 251 lost_event.id = event->id; 252 lost_event.lost = local_xchg(&rb->lost, 0); 253 254 /* XXX mostly redundant; @data is already fully initializes */ 255 perf_event_header__init_id(&lost_event.header, data, event); 256 perf_output_put(handle, lost_event); 257 perf_event__output_id_sample(event, handle, data); 258 } 259 260 return 0; 261 262 fail: 263 local_inc(&rb->lost); 264 atomic64_inc(&event->lost_samples); 265 perf_output_put_handle(handle); 266 out: 267 rcu_read_unlock(); 268 269 return -ENOSPC; 270 } 271 272 int perf_output_begin_forward(struct perf_output_handle *handle, 273 struct perf_sample_data *data, 274 struct perf_event *event, unsigned int size) 275 { 276 return __perf_output_begin(handle, data, event, size, false); 277 } 278 279 int perf_output_begin_backward(struct perf_output_handle *handle, 280 struct perf_sample_data *data, 281 struct perf_event *event, unsigned int size) 282 { 283 return __perf_output_begin(handle, data, event, size, true); 284 } 285 286 int perf_output_begin(struct perf_output_handle *handle, 287 struct perf_sample_data *data, 288 struct perf_event *event, unsigned int size) 289 { 290 291 return __perf_output_begin(handle, data, event, size, 292 unlikely(is_write_backward(event))); 293 } 294 295 unsigned int perf_output_copy(struct perf_output_handle *handle, 296 const void *buf, unsigned int len) 297 { 298 return __output_copy(handle, buf, len); 299 } 300 301 unsigned int perf_output_skip(struct perf_output_handle *handle, 302 unsigned int len) 303 { 304 return __output_skip(handle, NULL, len); 305 } 306 307 void perf_output_end(struct perf_output_handle *handle) 308 { 309 perf_output_put_handle(handle); 310 rcu_read_unlock(); 311 } 312 313 static void 314 ring_buffer_init(struct perf_buffer *rb, long watermark, int flags) 315 { 316 long max_size = perf_data_size(rb); 317 318 if (watermark) 319 rb->watermark = min(max_size, watermark); 320 321 if (!rb->watermark) 322 rb->watermark = max_size / 2; 323 324 if (flags & RING_BUFFER_WRITABLE) 325 rb->overwrite = 0; 326 else 327 rb->overwrite = 1; 328 329 refcount_set(&rb->refcount, 1); 330 331 INIT_LIST_HEAD(&rb->event_list); 332 spin_lock_init(&rb->event_lock); 333 334 /* 335 * perf_output_begin() only checks rb->paused, therefore 336 * rb->paused must be true if we have no pages for output. 337 */ 338 if (!rb->nr_pages) 339 rb->paused = 1; 340 341 mutex_init(&rb->aux_mutex); 342 } 343 344 void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags) 345 { 346 /* 347 * OVERWRITE is determined by perf_aux_output_end() and can't 348 * be passed in directly. 349 */ 350 if (WARN_ON_ONCE(flags & PERF_AUX_FLAG_OVERWRITE)) 351 return; 352 353 handle->aux_flags |= flags; 354 } 355 EXPORT_SYMBOL_GPL(perf_aux_output_flag); 356 357 /* 358 * This is called before hardware starts writing to the AUX area to 359 * obtain an output handle and make sure there's room in the buffer. 360 * When the capture completes, call perf_aux_output_end() to commit 361 * the recorded data to the buffer. 362 * 363 * The ordering is similar to that of perf_output_{begin,end}, with 364 * the exception of (B), which should be taken care of by the pmu 365 * driver, since ordering rules will differ depending on hardware. 366 * 367 * Call this from pmu::start(); see the comment in perf_aux_output_end() 368 * about its use in pmu callbacks. Both can also be called from the PMI 369 * handler if needed. 370 */ 371 void *perf_aux_output_begin(struct perf_output_handle *handle, 372 struct perf_event *event) 373 { 374 struct perf_event *output_event = event; 375 unsigned long aux_head, aux_tail; 376 struct perf_buffer *rb; 377 unsigned int nest; 378 379 if (output_event->parent) 380 output_event = output_event->parent; 381 382 /* 383 * Since this will typically be open across pmu::add/pmu::del, we 384 * grab ring_buffer's refcount instead of holding rcu read lock 385 * to make sure it doesn't disappear under us. 386 */ 387 rb = ring_buffer_get(output_event); 388 if (!rb) 389 return NULL; 390 391 if (!rb_has_aux(rb)) 392 goto err; 393 394 /* 395 * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(), 396 * about to get freed, so we leave immediately. 397 * 398 * Checking rb::aux_mmap_count and rb::refcount has to be done in 399 * the same order, see perf_mmap_close. Otherwise we end up freeing 400 * aux pages in this path, which is a bug, because in_atomic(). 401 */ 402 if (!atomic_read(&rb->aux_mmap_count)) 403 goto err; 404 405 if (!refcount_inc_not_zero(&rb->aux_refcount)) 406 goto err; 407 408 nest = READ_ONCE(rb->aux_nest); 409 /* 410 * Nesting is not supported for AUX area, make sure nested 411 * writers are caught early 412 */ 413 if (WARN_ON_ONCE(nest)) 414 goto err_put; 415 416 WRITE_ONCE(rb->aux_nest, nest + 1); 417 418 aux_head = rb->aux_head; 419 420 handle->rb = rb; 421 handle->event = event; 422 handle->head = aux_head; 423 handle->size = 0; 424 handle->aux_flags = 0; 425 426 /* 427 * In overwrite mode, AUX data stores do not depend on aux_tail, 428 * therefore (A) control dependency barrier does not exist. The 429 * (B) <-> (C) ordering is still observed by the pmu driver. 430 */ 431 if (!rb->aux_overwrite) { 432 aux_tail = READ_ONCE(rb->user_page->aux_tail); 433 handle->wakeup = rb->aux_wakeup + rb->aux_watermark; 434 if (aux_head - aux_tail < perf_aux_size(rb)) 435 handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb)); 436 437 /* 438 * handle->size computation depends on aux_tail load; this forms a 439 * control dependency barrier separating aux_tail load from aux data 440 * store that will be enabled on successful return 441 */ 442 if (!handle->size) { /* A, matches D */ 443 event->pending_disable = smp_processor_id(); 444 perf_output_wakeup(handle); 445 WRITE_ONCE(rb->aux_nest, 0); 446 goto err_put; 447 } 448 } 449 450 return handle->rb->aux_priv; 451 452 err_put: 453 /* can't be last */ 454 rb_free_aux(rb); 455 456 err: 457 ring_buffer_put(rb); 458 handle->event = NULL; 459 460 return NULL; 461 } 462 EXPORT_SYMBOL_GPL(perf_aux_output_begin); 463 464 static __always_inline bool rb_need_aux_wakeup(struct perf_buffer *rb) 465 { 466 if (rb->aux_overwrite) 467 return false; 468 469 if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) { 470 rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark); 471 return true; 472 } 473 474 return false; 475 } 476 477 /* 478 * Commit the data written by hardware into the ring buffer by adjusting 479 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the 480 * pmu driver's responsibility to observe ordering rules of the hardware, 481 * so that all the data is externally visible before this is called. 482 * 483 * Note: this has to be called from pmu::stop() callback, as the assumption 484 * of the AUX buffer management code is that after pmu::stop(), the AUX 485 * transaction must be stopped and therefore drop the AUX reference count. 486 */ 487 void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) 488 { 489 bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED); 490 struct perf_buffer *rb = handle->rb; 491 unsigned long aux_head; 492 493 /* in overwrite mode, driver provides aux_head via handle */ 494 if (rb->aux_overwrite) { 495 handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE; 496 497 aux_head = handle->head; 498 rb->aux_head = aux_head; 499 } else { 500 handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE; 501 502 aux_head = rb->aux_head; 503 rb->aux_head += size; 504 } 505 506 /* 507 * Only send RECORD_AUX if we have something useful to communicate 508 * 509 * Note: the OVERWRITE records by themselves are not considered 510 * useful, as they don't communicate any *new* information, 511 * aside from the short-lived offset, that becomes history at 512 * the next event sched-in and therefore isn't useful. 513 * The userspace that needs to copy out AUX data in overwrite 514 * mode should know to use user_page::aux_head for the actual 515 * offset. So, from now on we don't output AUX records that 516 * have *only* OVERWRITE flag set. 517 */ 518 if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE)) 519 perf_event_aux_event(handle->event, aux_head, size, 520 handle->aux_flags); 521 522 WRITE_ONCE(rb->user_page->aux_head, rb->aux_head); 523 if (rb_need_aux_wakeup(rb)) 524 wakeup = true; 525 526 if (wakeup) { 527 if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED) 528 handle->event->pending_disable = smp_processor_id(); 529 perf_output_wakeup(handle); 530 } 531 532 handle->event = NULL; 533 534 WRITE_ONCE(rb->aux_nest, 0); 535 /* can't be last */ 536 rb_free_aux(rb); 537 ring_buffer_put(rb); 538 } 539 EXPORT_SYMBOL_GPL(perf_aux_output_end); 540 541 /* 542 * Skip over a given number of bytes in the AUX buffer, due to, for example, 543 * hardware's alignment constraints. 544 */ 545 int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size) 546 { 547 struct perf_buffer *rb = handle->rb; 548 549 if (size > handle->size) 550 return -ENOSPC; 551 552 rb->aux_head += size; 553 554 WRITE_ONCE(rb->user_page->aux_head, rb->aux_head); 555 if (rb_need_aux_wakeup(rb)) { 556 perf_output_wakeup(handle); 557 handle->wakeup = rb->aux_wakeup + rb->aux_watermark; 558 } 559 560 handle->head = rb->aux_head; 561 handle->size -= size; 562 563 return 0; 564 } 565 EXPORT_SYMBOL_GPL(perf_aux_output_skip); 566 567 void *perf_get_aux(struct perf_output_handle *handle) 568 { 569 /* this is only valid between perf_aux_output_begin and *_end */ 570 if (!handle->event) 571 return NULL; 572 573 return handle->rb->aux_priv; 574 } 575 EXPORT_SYMBOL_GPL(perf_get_aux); 576 577 /* 578 * Copy out AUX data from an AUX handle. 579 */ 580 long perf_output_copy_aux(struct perf_output_handle *aux_handle, 581 struct perf_output_handle *handle, 582 unsigned long from, unsigned long to) 583 { 584 struct perf_buffer *rb = aux_handle->rb; 585 unsigned long tocopy, remainder, len = 0; 586 void *addr; 587 588 from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1; 589 to &= (rb->aux_nr_pages << PAGE_SHIFT) - 1; 590 591 do { 592 tocopy = PAGE_SIZE - offset_in_page(from); 593 if (to > from) 594 tocopy = min(tocopy, to - from); 595 if (!tocopy) 596 break; 597 598 addr = rb->aux_pages[from >> PAGE_SHIFT]; 599 addr += offset_in_page(from); 600 601 remainder = perf_output_copy(handle, addr, tocopy); 602 if (remainder) 603 return -EFAULT; 604 605 len += tocopy; 606 from += tocopy; 607 from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1; 608 } while (to != from); 609 610 return len; 611 } 612 613 #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY) 614 615 static struct page *rb_alloc_aux_page(int node, int order) 616 { 617 struct page *page; 618 619 if (order > MAX_PAGE_ORDER) 620 order = MAX_PAGE_ORDER; 621 622 do { 623 page = alloc_pages_node(node, PERF_AUX_GFP, order); 624 } while (!page && order--); 625 626 if (page && order) { 627 /* 628 * Communicate the allocation size to the driver: 629 * if we managed to secure a high-order allocation, 630 * set its first page's private to this order; 631 * !PagePrivate(page) means it's just a normal page. 632 */ 633 split_page(page, order); 634 SetPagePrivate(page); 635 set_page_private(page, order); 636 } 637 638 return page; 639 } 640 641 static void rb_free_aux_page(struct perf_buffer *rb, int idx) 642 { 643 struct page *page = virt_to_page(rb->aux_pages[idx]); 644 645 ClearPagePrivate(page); 646 page->mapping = NULL; 647 __free_page(page); 648 } 649 650 static void __rb_free_aux(struct perf_buffer *rb) 651 { 652 int pg; 653 654 /* 655 * Should never happen, the last reference should be dropped from 656 * perf_mmap_close() path, which first stops aux transactions (which 657 * in turn are the atomic holders of aux_refcount) and then does the 658 * last rb_free_aux(). 659 */ 660 WARN_ON_ONCE(in_atomic()); 661 662 if (rb->aux_priv) { 663 rb->free_aux(rb->aux_priv); 664 rb->free_aux = NULL; 665 rb->aux_priv = NULL; 666 } 667 668 if (rb->aux_nr_pages) { 669 for (pg = 0; pg < rb->aux_nr_pages; pg++) 670 rb_free_aux_page(rb, pg); 671 672 kfree(rb->aux_pages); 673 rb->aux_nr_pages = 0; 674 } 675 } 676 677 int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, 678 pgoff_t pgoff, int nr_pages, long watermark, int flags) 679 { 680 bool overwrite = !(flags & RING_BUFFER_WRITABLE); 681 int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); 682 int ret = -ENOMEM, max_order; 683 684 if (!has_aux(event)) 685 return -EOPNOTSUPP; 686 687 if (nr_pages <= 0) 688 return -EINVAL; 689 690 if (!overwrite) { 691 /* 692 * Watermark defaults to half the buffer, and so does the 693 * max_order, to aid PMU drivers in double buffering. 694 */ 695 if (!watermark) 696 watermark = min_t(unsigned long, 697 U32_MAX, 698 (unsigned long)nr_pages << (PAGE_SHIFT - 1)); 699 700 /* 701 * Use aux_watermark as the basis for chunking to 702 * help PMU drivers honor the watermark. 703 */ 704 max_order = get_order(watermark); 705 } else { 706 /* 707 * We need to start with the max_order that fits in nr_pages, 708 * not the other way around, hence ilog2() and not get_order. 709 */ 710 max_order = ilog2(nr_pages); 711 watermark = 0; 712 } 713 714 /* 715 * kcalloc_node() is unable to allocate buffer if the size is larger 716 * than: PAGE_SIZE << MAX_PAGE_ORDER; directly bail out in this case. 717 */ 718 if (get_order((unsigned long)nr_pages * sizeof(void *)) > MAX_PAGE_ORDER) 719 return -ENOMEM; 720 rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL, 721 node); 722 if (!rb->aux_pages) 723 return -ENOMEM; 724 725 rb->free_aux = event->pmu->free_aux; 726 for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) { 727 struct page *page; 728 int last, order; 729 730 order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages)); 731 page = rb_alloc_aux_page(node, order); 732 if (!page) 733 goto out; 734 735 for (last = rb->aux_nr_pages + (1 << page_private(page)); 736 last > rb->aux_nr_pages; rb->aux_nr_pages++) 737 rb->aux_pages[rb->aux_nr_pages] = page_address(page++); 738 } 739 740 /* 741 * In overwrite mode, PMUs that don't support SG may not handle more 742 * than one contiguous allocation, since they rely on PMI to do double 743 * buffering. In this case, the entire buffer has to be one contiguous 744 * chunk. 745 */ 746 if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) && 747 overwrite) { 748 struct page *page = virt_to_page(rb->aux_pages[0]); 749 750 if (page_private(page) != max_order) 751 goto out; 752 } 753 754 rb->aux_priv = event->pmu->setup_aux(event, rb->aux_pages, nr_pages, 755 overwrite); 756 if (!rb->aux_priv) 757 goto out; 758 759 ret = 0; 760 761 /* 762 * aux_pages (and pmu driver's private data, aux_priv) will be 763 * referenced in both producer's and consumer's contexts, thus 764 * we keep a refcount here to make sure either of the two can 765 * reference them safely. 766 */ 767 refcount_set(&rb->aux_refcount, 1); 768 769 rb->aux_overwrite = overwrite; 770 rb->aux_watermark = watermark; 771 772 out: 773 if (!ret) 774 rb->aux_pgoff = pgoff; 775 else 776 __rb_free_aux(rb); 777 778 return ret; 779 } 780 781 void rb_free_aux(struct perf_buffer *rb) 782 { 783 if (refcount_dec_and_test(&rb->aux_refcount)) 784 __rb_free_aux(rb); 785 } 786 787 #ifndef CONFIG_PERF_USE_VMALLOC 788 789 /* 790 * Back perf_mmap() with regular GFP_KERNEL-0 pages. 791 */ 792 793 static struct page * 794 __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) 795 { 796 if (pgoff > rb->nr_pages) 797 return NULL; 798 799 if (pgoff == 0) 800 return virt_to_page(rb->user_page); 801 802 return virt_to_page(rb->data_pages[pgoff - 1]); 803 } 804 805 static void *perf_mmap_alloc_page(int cpu) 806 { 807 struct page *page; 808 int node; 809 810 node = (cpu == -1) ? cpu : cpu_to_node(cpu); 811 page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); 812 if (!page) 813 return NULL; 814 815 return page_address(page); 816 } 817 818 static void perf_mmap_free_page(void *addr) 819 { 820 struct page *page = virt_to_page(addr); 821 822 page->mapping = NULL; 823 __free_page(page); 824 } 825 826 struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) 827 { 828 struct perf_buffer *rb; 829 unsigned long size; 830 int i, node; 831 832 size = sizeof(struct perf_buffer); 833 size += nr_pages * sizeof(void *); 834 835 if (order_base_2(size) > PAGE_SHIFT+MAX_PAGE_ORDER) 836 goto fail; 837 838 node = (cpu == -1) ? cpu : cpu_to_node(cpu); 839 rb = kzalloc_node(size, GFP_KERNEL, node); 840 if (!rb) 841 goto fail; 842 843 rb->user_page = perf_mmap_alloc_page(cpu); 844 if (!rb->user_page) 845 goto fail_user_page; 846 847 for (i = 0; i < nr_pages; i++) { 848 rb->data_pages[i] = perf_mmap_alloc_page(cpu); 849 if (!rb->data_pages[i]) 850 goto fail_data_pages; 851 } 852 853 rb->nr_pages = nr_pages; 854 855 ring_buffer_init(rb, watermark, flags); 856 857 return rb; 858 859 fail_data_pages: 860 for (i--; i >= 0; i--) 861 perf_mmap_free_page(rb->data_pages[i]); 862 863 perf_mmap_free_page(rb->user_page); 864 865 fail_user_page: 866 kfree(rb); 867 868 fail: 869 return NULL; 870 } 871 872 void rb_free(struct perf_buffer *rb) 873 { 874 int i; 875 876 perf_mmap_free_page(rb->user_page); 877 for (i = 0; i < rb->nr_pages; i++) 878 perf_mmap_free_page(rb->data_pages[i]); 879 kfree(rb); 880 } 881 882 #else 883 static struct page * 884 __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) 885 { 886 /* The '>' counts in the user page. */ 887 if (pgoff > data_page_nr(rb)) 888 return NULL; 889 890 return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE); 891 } 892 893 static void perf_mmap_unmark_page(void *addr) 894 { 895 struct page *page = vmalloc_to_page(addr); 896 897 page->mapping = NULL; 898 } 899 900 static void rb_free_work(struct work_struct *work) 901 { 902 struct perf_buffer *rb; 903 void *base; 904 int i, nr; 905 906 rb = container_of(work, struct perf_buffer, work); 907 nr = data_page_nr(rb); 908 909 base = rb->user_page; 910 /* The '<=' counts in the user page. */ 911 for (i = 0; i <= nr; i++) 912 perf_mmap_unmark_page(base + (i * PAGE_SIZE)); 913 914 vfree(base); 915 kfree(rb); 916 } 917 918 void rb_free(struct perf_buffer *rb) 919 { 920 schedule_work(&rb->work); 921 } 922 923 struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) 924 { 925 struct perf_buffer *rb; 926 unsigned long size; 927 void *all_buf; 928 int node; 929 930 size = sizeof(struct perf_buffer); 931 size += sizeof(void *); 932 933 node = (cpu == -1) ? cpu : cpu_to_node(cpu); 934 rb = kzalloc_node(size, GFP_KERNEL, node); 935 if (!rb) 936 goto fail; 937 938 INIT_WORK(&rb->work, rb_free_work); 939 940 all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE); 941 if (!all_buf) 942 goto fail_all_buf; 943 944 rb->user_page = all_buf; 945 rb->data_pages[0] = all_buf + PAGE_SIZE; 946 if (nr_pages) { 947 rb->nr_pages = 1; 948 rb->page_order = ilog2(nr_pages); 949 } 950 951 ring_buffer_init(rb, watermark, flags); 952 953 return rb; 954 955 fail_all_buf: 956 kfree(rb); 957 958 fail: 959 return NULL; 960 } 961 962 #endif 963 964 struct page * 965 perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) 966 { 967 if (rb->aux_nr_pages) { 968 /* above AUX space */ 969 if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) 970 return NULL; 971 972 /* AUX space */ 973 if (pgoff >= rb->aux_pgoff) { 974 int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages); 975 return virt_to_page(rb->aux_pages[aux_pgoff]); 976 } 977 } 978 979 return __perf_mmap_to_page(rb, pgoff); 980 } 981
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.