1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * KVM nVHE hypervisor stack tracing support.
4 *
5 * The unwinder implementation depends on the nVHE mode:
6 *
7 * 1) Non-protected nVHE mode - the host can directly access the
8 * HYP stack pages and unwind the HYP stack in EL1. This saves having
9 * to allocate shared buffers for the host to read the unwinded
10 * stacktrace.
11 *
12 * 2) pKVM (protected nVHE) mode - the host cannot directly access
13 * the HYP memory. The stack is unwinded in EL2 and dumped to a shared
14 * buffer where the host can read and print the stacktrace.
15 *
16 * Copyright (C) 2022 Google LLC
17 */
18
19 #include <linux/kvm.h>
20 #include <linux/kvm_host.h>
21
22 #include <asm/stacktrace/nvhe.h>
23
24 static struct stack_info stackinfo_get_overflow(void)
25 {
26 struct kvm_nvhe_stacktrace_info *stacktrace_info
27 = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
28 unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
29 unsigned long high = low + OVERFLOW_STACK_SIZE;
30
31 return (struct stack_info) {
32 .low = low,
33 .high = high,
34 };
35 }
36
37 static struct stack_info stackinfo_get_overflow_kern_va(void)
38 {
39 unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
40 unsigned long high = low + OVERFLOW_STACK_SIZE;
41
42 return (struct stack_info) {
43 .low = low,
44 .high = high,
45 };
46 }
47
48 static struct stack_info stackinfo_get_hyp(void)
49 {
50 struct kvm_nvhe_stacktrace_info *stacktrace_info
51 = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
52 unsigned long low = (unsigned long)stacktrace_info->stack_base;
53 unsigned long high = low + PAGE_SIZE;
54
55 return (struct stack_info) {
56 .low = low,
57 .high = high,
58 };
59 }
60
61 static struct stack_info stackinfo_get_hyp_kern_va(void)
62 {
63 unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page);
64 unsigned long high = low + PAGE_SIZE;
65
66 return (struct stack_info) {
67 .low = low,
68 .high = high,
69 };
70 }
71
72 /*
73 * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs
74 *
75 * The nVHE hypervisor stack is mapped in the flexible 'private' VA range, to
76 * allow for guard pages below the stack. Consequently, the fixed offset address
77 * translation macros won't work here.
78 *
79 * The kernel VA is calculated as an offset from the kernel VA of the hypervisor
80 * stack base.
81 *
82 * Returns true on success and updates @addr to its corresponding kernel VA;
83 * otherwise returns false.
84 */
85 static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size)
86 {
87 struct stack_info stack_hyp, stack_kern;
88
89 stack_hyp = stackinfo_get_hyp();
90 stack_kern = stackinfo_get_hyp_kern_va();
91 if (stackinfo_on_stack(&stack_hyp, *addr, size))
92 goto found;
93
94 stack_hyp = stackinfo_get_overflow();
95 stack_kern = stackinfo_get_overflow_kern_va();
96 if (stackinfo_on_stack(&stack_hyp, *addr, size))
97 goto found;
98
99 return false;
100
101 found:
102 *addr = *addr - stack_hyp.low + stack_kern.low;
103 return true;
104 }
105
106 /*
107 * Convert a KVN nVHE HYP frame record address to a kernel VA
108 */
109 static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr)
110 {
111 return kvm_nvhe_stack_kern_va(addr, 16);
112 }
113
114 static int unwind_next(struct unwind_state *state)
115 {
116 /*
117 * The FP is in the hypervisor VA space. Convert it to the kernel VA
118 * space so it can be unwound by the regular unwind functions.
119 */
120 if (!kvm_nvhe_stack_kern_record_va(&state->fp))
121 return -EINVAL;
122
123 return unwind_next_frame_record(state);
124 }
125
126 static void unwind(struct unwind_state *state,
127 stack_trace_consume_fn consume_entry, void *cookie)
128 {
129 while (1) {
130 int ret;
131
132 if (!consume_entry(cookie, state->pc))
133 break;
134 ret = unwind_next(state);
135 if (ret < 0)
136 break;
137 }
138 }
139
140 /*
141 * kvm_nvhe_dump_backtrace_entry - Symbolize and print an nVHE backtrace entry
142 *
143 * @arg : the hypervisor offset, used for address translation
144 * @where : the program counter corresponding to the stack frame
145 */
146 static bool kvm_nvhe_dump_backtrace_entry(void *arg, unsigned long where)
147 {
148 unsigned long va_mask = GENMASK_ULL(vabits_actual - 1, 0);
149 unsigned long hyp_offset = (unsigned long)arg;
150
151 /* Mask tags and convert to kern addr */
152 where = (where & va_mask) + hyp_offset;
153 kvm_err(" [<%016lx>] %pB\n", where, (void *)(where + kaslr_offset()));
154
155 return true;
156 }
157
158 static void kvm_nvhe_dump_backtrace_start(void)
159 {
160 kvm_err("nVHE call trace:\n");
161 }
162
163 static void kvm_nvhe_dump_backtrace_end(void)
164 {
165 kvm_err("---[ end nVHE call trace ]---\n");
166 }
167
168 /*
169 * hyp_dump_backtrace - Dump the non-protected nVHE backtrace.
170 *
171 * @hyp_offset: hypervisor offset, used for address translation.
172 *
173 * The host can directly access HYP stack pages in non-protected
174 * mode, so the unwinding is done directly from EL1. This removes
175 * the need for shared buffers between host and hypervisor for
176 * the stacktrace.
177 */
178 static void hyp_dump_backtrace(unsigned long hyp_offset)
179 {
180 struct kvm_nvhe_stacktrace_info *stacktrace_info;
181 struct stack_info stacks[] = {
182 stackinfo_get_overflow_kern_va(),
183 stackinfo_get_hyp_kern_va(),
184 };
185 struct unwind_state state = {
186 .stacks = stacks,
187 .nr_stacks = ARRAY_SIZE(stacks),
188 };
189
190 stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
191
192 kvm_nvhe_unwind_init(&state, stacktrace_info->fp, stacktrace_info->pc);
193
194 kvm_nvhe_dump_backtrace_start();
195 unwind(&state, kvm_nvhe_dump_backtrace_entry, (void *)hyp_offset);
196 kvm_nvhe_dump_backtrace_end();
197 }
198
199 #ifdef CONFIG_PROTECTED_NVHE_STACKTRACE
200 DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
201 pkvm_stacktrace);
202
203 /*
204 * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
205 *
206 * @hyp_offset: hypervisor offset, used for address translation.
207 *
208 * Dumping of the pKVM HYP backtrace is done by reading the
209 * stack addresses from the shared stacktrace buffer, since the
210 * host cannot directly access hypervisor memory in protected
211 * mode.
212 */
213 static void pkvm_dump_backtrace(unsigned long hyp_offset)
214 {
215 unsigned long *stacktrace
216 = (unsigned long *) this_cpu_ptr_nvhe_sym(pkvm_stacktrace);
217 int i;
218
219 kvm_nvhe_dump_backtrace_start();
220 /* The saved stacktrace is terminated by a null entry */
221 for (i = 0;
222 i < ARRAY_SIZE(kvm_nvhe_sym(pkvm_stacktrace)) && stacktrace[i];
223 i++)
224 kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]);
225 kvm_nvhe_dump_backtrace_end();
226 }
227 #else /* !CONFIG_PROTECTED_NVHE_STACKTRACE */
228 static void pkvm_dump_backtrace(unsigned long hyp_offset)
229 {
230 kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PROTECTED_NVHE_STACKTRACE\n");
231 }
232 #endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */
233
234 /*
235 * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace.
236 *
237 * @hyp_offset: hypervisor offset, used for address translation.
238 */
239 void kvm_nvhe_dump_backtrace(unsigned long hyp_offset)
240 {
241 if (is_protected_kvm_enabled())
242 pkvm_dump_backtrace(hyp_offset);
243 else
244 hyp_dump_backtrace(hyp_offset);
245 }
246
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.