1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
4 * using the CPU's debug registers.
5 *
6 * Copyright (C) 2012 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
8 */
9
10 #define pr_fmt(fmt) "hw-breakpoint: " fmt
11
12 #include <linux/compat.h>
13 #include <linux/cpu_pm.h>
14 #include <linux/errno.h>
15 #include <linux/hw_breakpoint.h>
16 #include <linux/kprobes.h>
17 #include <linux/perf_event.h>
18 #include <linux/ptrace.h>
19 #include <linux/smp.h>
20 #include <linux/uaccess.h>
21
22 #include <asm/current.h>
23 #include <asm/debug-monitors.h>
24 #include <asm/esr.h>
25 #include <asm/hw_breakpoint.h>
26 #include <asm/traps.h>
27 #include <asm/cputype.h>
28 #include <asm/system_misc.h>
29
30 /* Breakpoint currently in use for each BRP. */
31 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
32
33 /* Watchpoint currently in use for each WRP. */
34 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
35
36 /* Currently stepping a per-CPU kernel breakpoint. */
37 static DEFINE_PER_CPU(int, stepping_kernel_bp);
38
39 /* Number of BRP/WRP registers on this CPU. */
40 static int core_num_brps;
41 static int core_num_wrps;
42
43 int hw_breakpoint_slots(int type)
44 {
45 /*
46 * We can be called early, so don't rely on
47 * our static variables being initialised.
48 */
49 switch (type) {
50 case TYPE_INST:
51 return get_num_brps();
52 case TYPE_DATA:
53 return get_num_wrps();
54 default:
55 pr_warn("unknown slot type: %d\n", type);
56 return 0;
57 }
58 }
59
60 #define READ_WB_REG_CASE(OFF, N, REG, VAL) \
61 case (OFF + N): \
62 AARCH64_DBG_READ(N, REG, VAL); \
63 break
64
65 #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
66 case (OFF + N): \
67 AARCH64_DBG_WRITE(N, REG, VAL); \
68 break
69
70 #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \
71 READ_WB_REG_CASE(OFF, 0, REG, VAL); \
72 READ_WB_REG_CASE(OFF, 1, REG, VAL); \
73 READ_WB_REG_CASE(OFF, 2, REG, VAL); \
74 READ_WB_REG_CASE(OFF, 3, REG, VAL); \
75 READ_WB_REG_CASE(OFF, 4, REG, VAL); \
76 READ_WB_REG_CASE(OFF, 5, REG, VAL); \
77 READ_WB_REG_CASE(OFF, 6, REG, VAL); \
78 READ_WB_REG_CASE(OFF, 7, REG, VAL); \
79 READ_WB_REG_CASE(OFF, 8, REG, VAL); \
80 READ_WB_REG_CASE(OFF, 9, REG, VAL); \
81 READ_WB_REG_CASE(OFF, 10, REG, VAL); \
82 READ_WB_REG_CASE(OFF, 11, REG, VAL); \
83 READ_WB_REG_CASE(OFF, 12, REG, VAL); \
84 READ_WB_REG_CASE(OFF, 13, REG, VAL); \
85 READ_WB_REG_CASE(OFF, 14, REG, VAL); \
86 READ_WB_REG_CASE(OFF, 15, REG, VAL)
87
88 #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \
89 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
90 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
91 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
92 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
93 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
94 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
95 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
96 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
97 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
98 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
99 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
100 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
101 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
102 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
103 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
104 WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
105
106 static u64 read_wb_reg(int reg, int n)
107 {
108 u64 val = 0;
109
110 switch (reg + n) {
111 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
112 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
113 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
114 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
115 default:
116 pr_warn("attempt to read from unknown breakpoint register %d\n", n);
117 }
118
119 return val;
120 }
121 NOKPROBE_SYMBOL(read_wb_reg);
122
123 static void write_wb_reg(int reg, int n, u64 val)
124 {
125 switch (reg + n) {
126 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
127 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
128 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
129 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
130 default:
131 pr_warn("attempt to write to unknown breakpoint register %d\n", n);
132 }
133 isb();
134 }
135 NOKPROBE_SYMBOL(write_wb_reg);
136
137 /*
138 * Convert a breakpoint privilege level to the corresponding exception
139 * level.
140 */
141 static enum dbg_active_el debug_exception_level(int privilege)
142 {
143 switch (privilege) {
144 case AARCH64_BREAKPOINT_EL0:
145 return DBG_ACTIVE_EL0;
146 case AARCH64_BREAKPOINT_EL1:
147 return DBG_ACTIVE_EL1;
148 default:
149 pr_warn("invalid breakpoint privilege level %d\n", privilege);
150 return -EINVAL;
151 }
152 }
153 NOKPROBE_SYMBOL(debug_exception_level);
154
155 enum hw_breakpoint_ops {
156 HW_BREAKPOINT_INSTALL,
157 HW_BREAKPOINT_UNINSTALL,
158 HW_BREAKPOINT_RESTORE
159 };
160
161 static int is_compat_bp(struct perf_event *bp)
162 {
163 struct task_struct *tsk = bp->hw.target;
164
165 /*
166 * tsk can be NULL for per-cpu (non-ptrace) breakpoints.
167 * In this case, use the native interface, since we don't have
168 * the notion of a "compat CPU" and could end up relying on
169 * deprecated behaviour if we use unaligned watchpoints in
170 * AArch64 state.
171 */
172 return tsk && is_compat_thread(task_thread_info(tsk));
173 }
174
175 /**
176 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
177 * operations
178 *
179 * @slots: pointer to array of slots
180 * @max_slots: max number of slots
181 * @bp: perf_event to setup
182 * @ops: operation to be carried out on the slot
183 *
184 * Return:
185 * slot index on success
186 * -ENOSPC if no slot is available/matches
187 * -EINVAL on wrong operations parameter
188 */
189 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
190 struct perf_event *bp,
191 enum hw_breakpoint_ops ops)
192 {
193 int i;
194 struct perf_event **slot;
195
196 for (i = 0; i < max_slots; ++i) {
197 slot = &slots[i];
198 switch (ops) {
199 case HW_BREAKPOINT_INSTALL:
200 if (!*slot) {
201 *slot = bp;
202 return i;
203 }
204 break;
205 case HW_BREAKPOINT_UNINSTALL:
206 if (*slot == bp) {
207 *slot = NULL;
208 return i;
209 }
210 break;
211 case HW_BREAKPOINT_RESTORE:
212 if (*slot == bp)
213 return i;
214 break;
215 default:
216 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
217 return -EINVAL;
218 }
219 }
220 return -ENOSPC;
221 }
222
223 static int hw_breakpoint_control(struct perf_event *bp,
224 enum hw_breakpoint_ops ops)
225 {
226 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
227 struct perf_event **slots;
228 struct debug_info *debug_info = ¤t->thread.debug;
229 int i, max_slots, ctrl_reg, val_reg, reg_enable;
230 enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege);
231 u32 ctrl;
232
233 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
234 /* Breakpoint */
235 ctrl_reg = AARCH64_DBG_REG_BCR;
236 val_reg = AARCH64_DBG_REG_BVR;
237 slots = this_cpu_ptr(bp_on_reg);
238 max_slots = core_num_brps;
239 reg_enable = !debug_info->bps_disabled;
240 } else {
241 /* Watchpoint */
242 ctrl_reg = AARCH64_DBG_REG_WCR;
243 val_reg = AARCH64_DBG_REG_WVR;
244 slots = this_cpu_ptr(wp_on_reg);
245 max_slots = core_num_wrps;
246 reg_enable = !debug_info->wps_disabled;
247 }
248
249 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
250
251 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
252 return i;
253
254 switch (ops) {
255 case HW_BREAKPOINT_INSTALL:
256 /*
257 * Ensure debug monitors are enabled at the correct exception
258 * level.
259 */
260 enable_debug_monitors(dbg_el);
261 fallthrough;
262 case HW_BREAKPOINT_RESTORE:
263 /* Setup the address register. */
264 write_wb_reg(val_reg, i, info->address);
265
266 /* Setup the control register. */
267 ctrl = encode_ctrl_reg(info->ctrl);
268 write_wb_reg(ctrl_reg, i,
269 reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
270 break;
271 case HW_BREAKPOINT_UNINSTALL:
272 /* Reset the control register. */
273 write_wb_reg(ctrl_reg, i, 0);
274
275 /*
276 * Release the debug monitors for the correct exception
277 * level.
278 */
279 disable_debug_monitors(dbg_el);
280 break;
281 }
282
283 return 0;
284 }
285
286 /*
287 * Install a perf counter breakpoint.
288 */
289 int arch_install_hw_breakpoint(struct perf_event *bp)
290 {
291 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
292 }
293
294 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
295 {
296 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
297 }
298
299 static int get_hbp_len(u8 hbp_len)
300 {
301 unsigned int len_in_bytes = 0;
302
303 switch (hbp_len) {
304 case ARM_BREAKPOINT_LEN_1:
305 len_in_bytes = 1;
306 break;
307 case ARM_BREAKPOINT_LEN_2:
308 len_in_bytes = 2;
309 break;
310 case ARM_BREAKPOINT_LEN_3:
311 len_in_bytes = 3;
312 break;
313 case ARM_BREAKPOINT_LEN_4:
314 len_in_bytes = 4;
315 break;
316 case ARM_BREAKPOINT_LEN_5:
317 len_in_bytes = 5;
318 break;
319 case ARM_BREAKPOINT_LEN_6:
320 len_in_bytes = 6;
321 break;
322 case ARM_BREAKPOINT_LEN_7:
323 len_in_bytes = 7;
324 break;
325 case ARM_BREAKPOINT_LEN_8:
326 len_in_bytes = 8;
327 break;
328 }
329
330 return len_in_bytes;
331 }
332
333 /*
334 * Check whether bp virtual address is in kernel space.
335 */
336 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
337 {
338 unsigned int len;
339 unsigned long va;
340
341 va = hw->address;
342 len = get_hbp_len(hw->ctrl.len);
343
344 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
345 }
346
347 /*
348 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
349 * Hopefully this will disappear when ptrace can bypass the conversion
350 * to generic breakpoint descriptions.
351 */
352 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
353 int *gen_len, int *gen_type, int *offset)
354 {
355 /* Type */
356 switch (ctrl.type) {
357 case ARM_BREAKPOINT_EXECUTE:
358 *gen_type = HW_BREAKPOINT_X;
359 break;
360 case ARM_BREAKPOINT_LOAD:
361 *gen_type = HW_BREAKPOINT_R;
362 break;
363 case ARM_BREAKPOINT_STORE:
364 *gen_type = HW_BREAKPOINT_W;
365 break;
366 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
367 *gen_type = HW_BREAKPOINT_RW;
368 break;
369 default:
370 return -EINVAL;
371 }
372
373 if (!ctrl.len)
374 return -EINVAL;
375 *offset = __ffs(ctrl.len);
376
377 /* Len */
378 switch (ctrl.len >> *offset) {
379 case ARM_BREAKPOINT_LEN_1:
380 *gen_len = HW_BREAKPOINT_LEN_1;
381 break;
382 case ARM_BREAKPOINT_LEN_2:
383 *gen_len = HW_BREAKPOINT_LEN_2;
384 break;
385 case ARM_BREAKPOINT_LEN_3:
386 *gen_len = HW_BREAKPOINT_LEN_3;
387 break;
388 case ARM_BREAKPOINT_LEN_4:
389 *gen_len = HW_BREAKPOINT_LEN_4;
390 break;
391 case ARM_BREAKPOINT_LEN_5:
392 *gen_len = HW_BREAKPOINT_LEN_5;
393 break;
394 case ARM_BREAKPOINT_LEN_6:
395 *gen_len = HW_BREAKPOINT_LEN_6;
396 break;
397 case ARM_BREAKPOINT_LEN_7:
398 *gen_len = HW_BREAKPOINT_LEN_7;
399 break;
400 case ARM_BREAKPOINT_LEN_8:
401 *gen_len = HW_BREAKPOINT_LEN_8;
402 break;
403 default:
404 return -EINVAL;
405 }
406
407 return 0;
408 }
409
410 /*
411 * Construct an arch_hw_breakpoint from a perf_event.
412 */
413 static int arch_build_bp_info(struct perf_event *bp,
414 const struct perf_event_attr *attr,
415 struct arch_hw_breakpoint *hw)
416 {
417 /* Type */
418 switch (attr->bp_type) {
419 case HW_BREAKPOINT_X:
420 hw->ctrl.type = ARM_BREAKPOINT_EXECUTE;
421 break;
422 case HW_BREAKPOINT_R:
423 hw->ctrl.type = ARM_BREAKPOINT_LOAD;
424 break;
425 case HW_BREAKPOINT_W:
426 hw->ctrl.type = ARM_BREAKPOINT_STORE;
427 break;
428 case HW_BREAKPOINT_RW:
429 hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
430 break;
431 default:
432 return -EINVAL;
433 }
434
435 /* Len */
436 switch (attr->bp_len) {
437 case HW_BREAKPOINT_LEN_1:
438 hw->ctrl.len = ARM_BREAKPOINT_LEN_1;
439 break;
440 case HW_BREAKPOINT_LEN_2:
441 hw->ctrl.len = ARM_BREAKPOINT_LEN_2;
442 break;
443 case HW_BREAKPOINT_LEN_3:
444 hw->ctrl.len = ARM_BREAKPOINT_LEN_3;
445 break;
446 case HW_BREAKPOINT_LEN_4:
447 hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
448 break;
449 case HW_BREAKPOINT_LEN_5:
450 hw->ctrl.len = ARM_BREAKPOINT_LEN_5;
451 break;
452 case HW_BREAKPOINT_LEN_6:
453 hw->ctrl.len = ARM_BREAKPOINT_LEN_6;
454 break;
455 case HW_BREAKPOINT_LEN_7:
456 hw->ctrl.len = ARM_BREAKPOINT_LEN_7;
457 break;
458 case HW_BREAKPOINT_LEN_8:
459 hw->ctrl.len = ARM_BREAKPOINT_LEN_8;
460 break;
461 default:
462 return -EINVAL;
463 }
464
465 /*
466 * On AArch64, we only permit breakpoints of length 4, whereas
467 * AArch32 also requires breakpoints of length 2 for Thumb.
468 * Watchpoints can be of length 1, 2, 4 or 8 bytes.
469 */
470 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
471 if (is_compat_bp(bp)) {
472 if (hw->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
473 hw->ctrl.len != ARM_BREAKPOINT_LEN_4)
474 return -EINVAL;
475 } else if (hw->ctrl.len != ARM_BREAKPOINT_LEN_4) {
476 /*
477 * FIXME: Some tools (I'm looking at you perf) assume
478 * that breakpoints should be sizeof(long). This
479 * is nonsense. For now, we fix up the parameter
480 * but we should probably return -EINVAL instead.
481 */
482 hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
483 }
484 }
485
486 /* Address */
487 hw->address = attr->bp_addr;
488
489 /*
490 * Privilege
491 * Note that we disallow combined EL0/EL1 breakpoints because
492 * that would complicate the stepping code.
493 */
494 if (arch_check_bp_in_kernelspace(hw))
495 hw->ctrl.privilege = AARCH64_BREAKPOINT_EL1;
496 else
497 hw->ctrl.privilege = AARCH64_BREAKPOINT_EL0;
498
499 /* Enabled? */
500 hw->ctrl.enabled = !attr->disabled;
501
502 return 0;
503 }
504
505 /*
506 * Validate the arch-specific HW Breakpoint register settings.
507 */
508 int hw_breakpoint_arch_parse(struct perf_event *bp,
509 const struct perf_event_attr *attr,
510 struct arch_hw_breakpoint *hw)
511 {
512 int ret;
513 u64 alignment_mask, offset;
514
515 /* Build the arch_hw_breakpoint. */
516 ret = arch_build_bp_info(bp, attr, hw);
517 if (ret)
518 return ret;
519
520 /*
521 * Check address alignment.
522 * We don't do any clever alignment correction for watchpoints
523 * because using 64-bit unaligned addresses is deprecated for
524 * AArch64.
525 *
526 * AArch32 tasks expect some simple alignment fixups, so emulate
527 * that here.
528 */
529 if (is_compat_bp(bp)) {
530 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8)
531 alignment_mask = 0x7;
532 else
533 alignment_mask = 0x3;
534 offset = hw->address & alignment_mask;
535 switch (offset) {
536 case 0:
537 /* Aligned */
538 break;
539 case 1:
540 case 2:
541 /* Allow halfword watchpoints and breakpoints. */
542 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2)
543 break;
544
545 fallthrough;
546 case 3:
547 /* Allow single byte watchpoint. */
548 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1)
549 break;
550
551 fallthrough;
552 default:
553 return -EINVAL;
554 }
555 } else {
556 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE)
557 alignment_mask = 0x3;
558 else
559 alignment_mask = 0x7;
560 offset = hw->address & alignment_mask;
561 }
562
563 hw->address &= ~alignment_mask;
564 hw->ctrl.len <<= offset;
565
566 /*
567 * Disallow per-task kernel breakpoints since these would
568 * complicate the stepping code.
569 */
570 if (hw->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target)
571 return -EINVAL;
572
573 return 0;
574 }
575
576 /*
577 * Enable/disable all of the breakpoints active at the specified
578 * exception level at the register level.
579 * This is used when single-stepping after a breakpoint exception.
580 */
581 static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable)
582 {
583 int i, max_slots, privilege;
584 u32 ctrl;
585 struct perf_event **slots;
586
587 switch (reg) {
588 case AARCH64_DBG_REG_BCR:
589 slots = this_cpu_ptr(bp_on_reg);
590 max_slots = core_num_brps;
591 break;
592 case AARCH64_DBG_REG_WCR:
593 slots = this_cpu_ptr(wp_on_reg);
594 max_slots = core_num_wrps;
595 break;
596 default:
597 return;
598 }
599
600 for (i = 0; i < max_slots; ++i) {
601 if (!slots[i])
602 continue;
603
604 privilege = counter_arch_bp(slots[i])->ctrl.privilege;
605 if (debug_exception_level(privilege) != el)
606 continue;
607
608 ctrl = read_wb_reg(reg, i);
609 if (enable)
610 ctrl |= 0x1;
611 else
612 ctrl &= ~0x1;
613 write_wb_reg(reg, i, ctrl);
614 }
615 }
616 NOKPROBE_SYMBOL(toggle_bp_registers);
617
618 /*
619 * Debug exception handlers.
620 */
621 static int breakpoint_handler(unsigned long unused, unsigned long esr,
622 struct pt_regs *regs)
623 {
624 int i, step = 0, *kernel_step;
625 u32 ctrl_reg;
626 u64 addr, val;
627 struct perf_event *bp, **slots;
628 struct debug_info *debug_info;
629 struct arch_hw_breakpoint_ctrl ctrl;
630
631 slots = this_cpu_ptr(bp_on_reg);
632 addr = instruction_pointer(regs);
633 debug_info = ¤t->thread.debug;
634
635 for (i = 0; i < core_num_brps; ++i) {
636 rcu_read_lock();
637
638 bp = slots[i];
639
640 if (bp == NULL)
641 goto unlock;
642
643 /* Check if the breakpoint value matches. */
644 val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
645 if (val != (addr & ~0x3))
646 goto unlock;
647
648 /* Possible match, check the byte address select to confirm. */
649 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
650 decode_ctrl_reg(ctrl_reg, &ctrl);
651 if (!((1 << (addr & 0x3)) & ctrl.len))
652 goto unlock;
653
654 counter_arch_bp(bp)->trigger = addr;
655 perf_bp_event(bp, regs);
656
657 /* Do we need to handle the stepping? */
658 if (is_default_overflow_handler(bp))
659 step = 1;
660 unlock:
661 rcu_read_unlock();
662 }
663
664 if (!step)
665 return 0;
666
667 if (user_mode(regs)) {
668 debug_info->bps_disabled = 1;
669 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
670
671 /* If we're already stepping a watchpoint, just return. */
672 if (debug_info->wps_disabled)
673 return 0;
674
675 if (test_thread_flag(TIF_SINGLESTEP))
676 debug_info->suspended_step = 1;
677 else
678 user_enable_single_step(current);
679 } else {
680 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
681 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
682
683 if (*kernel_step != ARM_KERNEL_STEP_NONE)
684 return 0;
685
686 if (kernel_active_single_step()) {
687 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
688 } else {
689 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
690 kernel_enable_single_step(regs);
691 }
692 }
693
694 return 0;
695 }
696 NOKPROBE_SYMBOL(breakpoint_handler);
697
698 /*
699 * Arm64 hardware does not always report a watchpoint hit address that matches
700 * one of the watchpoints set. It can also report an address "near" the
701 * watchpoint if a single instruction access both watched and unwatched
702 * addresses. There is no straight-forward way, short of disassembling the
703 * offending instruction, to map that address back to the watchpoint. This
704 * function computes the distance of the memory access from the watchpoint as a
705 * heuristic for the likelihood that a given access triggered the watchpoint.
706 *
707 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
708 * exception" of ARMv8 Architecture Reference Manual for details.
709 *
710 * The function returns the distance of the address from the bytes watched by
711 * the watchpoint. In case of an exact match, it returns 0.
712 */
713 static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
714 struct arch_hw_breakpoint_ctrl *ctrl)
715 {
716 u64 wp_low, wp_high;
717 u32 lens, lene;
718
719 addr = untagged_addr(addr);
720
721 lens = __ffs(ctrl->len);
722 lene = __fls(ctrl->len);
723
724 wp_low = val + lens;
725 wp_high = val + lene;
726 if (addr < wp_low)
727 return wp_low - addr;
728 else if (addr > wp_high)
729 return addr - wp_high;
730 else
731 return 0;
732 }
733
734 static int watchpoint_report(struct perf_event *wp, unsigned long addr,
735 struct pt_regs *regs)
736 {
737 int step = is_default_overflow_handler(wp);
738 struct arch_hw_breakpoint *info = counter_arch_bp(wp);
739
740 info->trigger = addr;
741
742 /*
743 * If we triggered a user watchpoint from a uaccess routine, then
744 * handle the stepping ourselves since userspace really can't help
745 * us with this.
746 */
747 if (!user_mode(regs) && info->ctrl.privilege == AARCH64_BREAKPOINT_EL0)
748 step = 1;
749 else
750 perf_bp_event(wp, regs);
751
752 return step;
753 }
754
755 static int watchpoint_handler(unsigned long addr, unsigned long esr,
756 struct pt_regs *regs)
757 {
758 int i, step = 0, *kernel_step, access, closest_match = 0;
759 u64 min_dist = -1, dist;
760 u32 ctrl_reg;
761 u64 val;
762 struct perf_event *wp, **slots;
763 struct debug_info *debug_info;
764 struct arch_hw_breakpoint_ctrl ctrl;
765
766 slots = this_cpu_ptr(wp_on_reg);
767 debug_info = ¤t->thread.debug;
768
769 /*
770 * Find all watchpoints that match the reported address. If no exact
771 * match is found. Attribute the hit to the closest watchpoint.
772 */
773 rcu_read_lock();
774 for (i = 0; i < core_num_wrps; ++i) {
775 wp = slots[i];
776 if (wp == NULL)
777 continue;
778
779 /*
780 * Check that the access type matches.
781 * 0 => load, otherwise => store
782 */
783 access = (esr & ESR_ELx_WNR) ? HW_BREAKPOINT_W :
784 HW_BREAKPOINT_R;
785 if (!(access & hw_breakpoint_type(wp)))
786 continue;
787
788 /* Check if the watchpoint value and byte select match. */
789 val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
790 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
791 decode_ctrl_reg(ctrl_reg, &ctrl);
792 dist = get_distance_from_watchpoint(addr, val, &ctrl);
793 if (dist < min_dist) {
794 min_dist = dist;
795 closest_match = i;
796 }
797 /* Is this an exact match? */
798 if (dist != 0)
799 continue;
800
801 step = watchpoint_report(wp, addr, regs);
802 }
803
804 /* No exact match found? */
805 if (min_dist > 0 && min_dist != -1)
806 step = watchpoint_report(slots[closest_match], addr, regs);
807
808 rcu_read_unlock();
809
810 if (!step)
811 return 0;
812
813 /*
814 * We always disable EL0 watchpoints because the kernel can
815 * cause these to fire via an unprivileged access.
816 */
817 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
818
819 if (user_mode(regs)) {
820 debug_info->wps_disabled = 1;
821
822 /* If we're already stepping a breakpoint, just return. */
823 if (debug_info->bps_disabled)
824 return 0;
825
826 if (test_thread_flag(TIF_SINGLESTEP))
827 debug_info->suspended_step = 1;
828 else
829 user_enable_single_step(current);
830 } else {
831 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
832 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
833
834 if (*kernel_step != ARM_KERNEL_STEP_NONE)
835 return 0;
836
837 if (kernel_active_single_step()) {
838 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
839 } else {
840 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
841 kernel_enable_single_step(regs);
842 }
843 }
844
845 return 0;
846 }
847 NOKPROBE_SYMBOL(watchpoint_handler);
848
849 /*
850 * Handle single-step exception.
851 */
852 int reinstall_suspended_bps(struct pt_regs *regs)
853 {
854 struct debug_info *debug_info = ¤t->thread.debug;
855 int handled_exception = 0, *kernel_step;
856
857 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
858
859 /*
860 * Called from single-step exception handler.
861 * Return 0 if execution can resume, 1 if a SIGTRAP should be
862 * reported.
863 */
864 if (user_mode(regs)) {
865 if (debug_info->bps_disabled) {
866 debug_info->bps_disabled = 0;
867 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1);
868 handled_exception = 1;
869 }
870
871 if (debug_info->wps_disabled) {
872 debug_info->wps_disabled = 0;
873 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
874 handled_exception = 1;
875 }
876
877 if (handled_exception) {
878 if (debug_info->suspended_step) {
879 debug_info->suspended_step = 0;
880 /* Allow exception handling to fall-through. */
881 handled_exception = 0;
882 } else {
883 user_disable_single_step(current);
884 }
885 }
886 } else if (*kernel_step != ARM_KERNEL_STEP_NONE) {
887 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1);
888 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1);
889
890 if (!debug_info->wps_disabled)
891 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
892
893 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) {
894 kernel_disable_single_step();
895 handled_exception = 1;
896 } else {
897 handled_exception = 0;
898 }
899
900 *kernel_step = ARM_KERNEL_STEP_NONE;
901 }
902
903 return !handled_exception;
904 }
905 NOKPROBE_SYMBOL(reinstall_suspended_bps);
906
907 /*
908 * Context-switcher for restoring suspended breakpoints.
909 */
910 void hw_breakpoint_thread_switch(struct task_struct *next)
911 {
912 /*
913 * current next
914 * disabled: 0 0 => The usual case, NOTIFY_DONE
915 * 0 1 => Disable the registers
916 * 1 0 => Enable the registers
917 * 1 1 => NOTIFY_DONE. per-task bps will
918 * get taken care of by perf.
919 */
920
921 struct debug_info *current_debug_info, *next_debug_info;
922
923 current_debug_info = ¤t->thread.debug;
924 next_debug_info = &next->thread.debug;
925
926 /* Update breakpoints. */
927 if (current_debug_info->bps_disabled != next_debug_info->bps_disabled)
928 toggle_bp_registers(AARCH64_DBG_REG_BCR,
929 DBG_ACTIVE_EL0,
930 !next_debug_info->bps_disabled);
931
932 /* Update watchpoints. */
933 if (current_debug_info->wps_disabled != next_debug_info->wps_disabled)
934 toggle_bp_registers(AARCH64_DBG_REG_WCR,
935 DBG_ACTIVE_EL0,
936 !next_debug_info->wps_disabled);
937 }
938
939 /*
940 * CPU initialisation.
941 */
942 static int hw_breakpoint_reset(unsigned int cpu)
943 {
944 int i;
945 struct perf_event **slots;
946 /*
947 * When a CPU goes through cold-boot, it does not have any installed
948 * slot, so it is safe to share the same function for restoring and
949 * resetting breakpoints; when a CPU is hotplugged in, it goes
950 * through the slots, which are all empty, hence it just resets control
951 * and value for debug registers.
952 * When this function is triggered on warm-boot through a CPU PM
953 * notifier some slots might be initialized; if so they are
954 * reprogrammed according to the debug slots content.
955 */
956 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
957 if (slots[i]) {
958 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
959 } else {
960 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
961 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
962 }
963 }
964
965 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
966 if (slots[i]) {
967 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
968 } else {
969 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
970 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
971 }
972 }
973
974 return 0;
975 }
976
977 /*
978 * One-time initialisation.
979 */
980 static int __init arch_hw_breakpoint_init(void)
981 {
982 int ret;
983
984 core_num_brps = get_num_brps();
985 core_num_wrps = get_num_wrps();
986
987 pr_info("found %d breakpoint and %d watchpoint registers.\n",
988 core_num_brps, core_num_wrps);
989
990 /* Register debug fault handlers. */
991 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
992 TRAP_HWBKPT, "hw-breakpoint handler");
993 hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP,
994 TRAP_HWBKPT, "hw-watchpoint handler");
995
996 /*
997 * Reset the breakpoint resources. We assume that a halting
998 * debugger will leave the world in a nice state for us.
999 */
1000 ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING,
1001 "perf/arm64/hw_breakpoint:starting",
1002 hw_breakpoint_reset, NULL);
1003 if (ret)
1004 pr_err("failed to register CPU hotplug notifier: %d\n", ret);
1005
1006 /* Register cpu_suspend hw breakpoint restore hook */
1007 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
1008
1009 return ret;
1010 }
1011 arch_initcall(arch_hw_breakpoint_init);
1012
1013 void hw_breakpoint_pmu_read(struct perf_event *bp)
1014 {
1015 }
1016
1017 /*
1018 * Dummy function to register with die_notifier.
1019 */
1020 int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1021 unsigned long val, void *data)
1022 {
1023 return NOTIFY_DONE;
1024 }
1025
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.