1 // SPDX-License-Identifier: GPL-2.0-only 1
2 /*
3 * Copyright (C) 2020 - Google LLC
4 * Author: David Brazdil <dbrazdil@google.com>
5 */
6
7 #include <asm/kvm_asm.h>
8 #include <asm/kvm_hyp.h>
9 #include <asm/kvm_mmu.h>
10 #include <linux/arm-smccc.h>
11 #include <linux/kvm_host.h>
12 #include <uapi/linux/psci.h>
13
14 #include <nvhe/memory.h>
15 #include <nvhe/trap_handler.h>
16
17 void kvm_hyp_cpu_entry(unsigned long r0);
18 void kvm_hyp_cpu_resume(unsigned long r0);
19
20 void __noreturn __host_enter(struct kvm_cpu_co
21
22 /* Config options set by the host. */
23 struct kvm_host_psci_config __ro_after_init kv
24
25 #define INVALID_CPU_ID UINT_MAX
26
27 struct psci_boot_args {
28 atomic_t lock;
29 unsigned long pc;
30 unsigned long r0;
31 };
32
33 #define PSCI_BOOT_ARGS_UNLOCKED 0
34 #define PSCI_BOOT_ARGS_LOCKED 1
35
36 #define PSCI_BOOT_ARGS_INIT
37 ((struct psci_boot_args){
38 .lock = ATOMIC_INIT(PSCI_BOOT_
39 })
40
41 static DEFINE_PER_CPU(struct psci_boot_args, c
42 static DEFINE_PER_CPU(struct psci_boot_args, s
43
44 #define is_psci_0_1(what, func_id)
45 (kvm_host_psci_config.psci_0_1_ ## wha
46 (func_id) == kvm_host_psci_config.fun
47
48 static bool is_psci_0_1_call(u64 func_id)
49 {
50 return (is_psci_0_1(cpu_suspend, func_
51 is_psci_0_1(cpu_on, func_id) |
52 is_psci_0_1(cpu_off, func_id)
53 is_psci_0_1(migrate, func_id))
54 }
55
56 static bool is_psci_0_2_call(u64 func_id)
57 {
58 /* SMCCC reserves IDs 0x00-1F with the
59 return (PSCI_0_2_FN(0) <= func_id && f
60 (PSCI_0_2_FN64(0) <= func_id &&
61 }
62
63 static unsigned long psci_call(unsigned long f
64 unsigned long a
65 {
66 struct arm_smccc_res res;
67
68 arm_smccc_1_1_smc(fn, arg0, arg1, arg2
69 return res.a0;
70 }
71
72 static unsigned long psci_forward(struct kvm_c
73 {
74 return psci_call(cpu_reg(host_ctxt, 0)
75 cpu_reg(host_ctxt, 2)
76 }
77
78 static unsigned int find_cpu_id(u64 mpidr)
79 {
80 unsigned int i;
81
82 /* Reject invalid MPIDRs */
83 if (mpidr & ~MPIDR_HWID_BITMASK)
84 return INVALID_CPU_ID;
85
86 for (i = 0; i < NR_CPUS; i++) {
87 if (cpu_logical_map(i) == mpid
88 return i;
89 }
90
91 return INVALID_CPU_ID;
92 }
93
94 static __always_inline bool try_acquire_boot_a
95 {
96 return atomic_cmpxchg_acquire(&args->l
97 PSCI_BOO
98 PSCI_BOO
99 PSCI_BOOT_ARGS_UNLOCKED;
100 }
101
102 static __always_inline void release_boot_args(
103 {
104 atomic_set_release(&args->lock, PSCI_B
105 }
106
107 static int psci_cpu_on(u64 func_id, struct kvm
108 {
109 DECLARE_REG(u64, mpidr, host_ctxt, 1);
110 DECLARE_REG(unsigned long, pc, host_ct
111 DECLARE_REG(unsigned long, r0, host_ct
112
113 unsigned int cpu_id;
114 struct psci_boot_args *boot_args;
115 struct kvm_nvhe_init_params *init_para
116 int ret;
117
118 /*
119 * Find the logical CPU ID for the giv
120 * the set of CPUs that were online at
121 * Booting other CPUs is rejected beca
122 * checked against the finalized capab
123 * by doing the feature checks in hyp.
124 */
125 cpu_id = find_cpu_id(mpidr);
126 if (cpu_id == INVALID_CPU_ID)
127 return PSCI_RET_INVALID_PARAMS
128
129 boot_args = per_cpu_ptr(&cpu_on_args,
130 init_params = per_cpu_ptr(&kvm_init_pa
131
132 /* Check if the target CPU is already
133 if (!try_acquire_boot_args(boot_args))
134 return PSCI_RET_ALREADY_ON;
135
136 boot_args->pc = pc;
137 boot_args->r0 = r0;
138 wmb();
139
140 ret = psci_call(func_id, mpidr,
141 __hyp_pa(&kvm_hyp_cpu_
142 __hyp_pa(init_params))
143
144 /* If successful, the lock will be rel
145 if (ret != PSCI_RET_SUCCESS)
146 release_boot_args(boot_args);
147
148 return ret;
149 }
150
151 static int psci_cpu_suspend(u64 func_id, struc
152 {
153 DECLARE_REG(u64, power_state, host_ctx
154 DECLARE_REG(unsigned long, pc, host_ct
155 DECLARE_REG(unsigned long, r0, host_ct
156
157 struct psci_boot_args *boot_args;
158 struct kvm_nvhe_init_params *init_para
159
160 boot_args = this_cpu_ptr(&suspend_args
161 init_params = this_cpu_ptr(&kvm_init_p
162
163 /*
164 * No need to acquire a lock before wr
165 * can only suspend itself. Racy CPU_O
166 */
167 boot_args->pc = pc;
168 boot_args->r0 = r0;
169
170 /*
171 * Will either return if shallow sleep
172 * point if it is a deep sleep state.
173 */
174 return psci_call(func_id, power_state,
175 __hyp_pa(&kvm_hyp_cpu
176 __hyp_pa(init_params)
177 }
178
179 static int psci_system_suspend(u64 func_id, st
180 {
181 DECLARE_REG(unsigned long, pc, host_ct
182 DECLARE_REG(unsigned long, r0, host_ct
183
184 struct psci_boot_args *boot_args;
185 struct kvm_nvhe_init_params *init_para
186
187 boot_args = this_cpu_ptr(&suspend_args
188 init_params = this_cpu_ptr(&kvm_init_p
189
190 /*
191 * No need to acquire a lock before wr
192 * can only suspend itself. Racy CPU_O
193 */
194 boot_args->pc = pc;
195 boot_args->r0 = r0;
196
197 /* Will only return on error. */
198 return psci_call(func_id,
199 __hyp_pa(&kvm_hyp_cpu
200 __hyp_pa(init_params)
201 }
202
203 asmlinkage void __noreturn __kvm_host_psci_cpu
204 {
205 struct psci_boot_args *boot_args;
206 struct kvm_cpu_context *host_ctxt;
207
208 host_ctxt = host_data_ptr(host_ctxt);
209
210 if (is_cpu_on)
211 boot_args = this_cpu_ptr(&cpu_
212 else
213 boot_args = this_cpu_ptr(&susp
214
215 cpu_reg(host_ctxt, 0) = boot_args->r0;
216 write_sysreg_el2(boot_args->pc, SYS_EL
217
218 if (is_cpu_on)
219 release_boot_args(boot_args);
220
221 __host_enter(host_ctxt);
222 }
223
224 static unsigned long psci_0_1_handler(u64 func
225 {
226 if (is_psci_0_1(cpu_off, func_id) || i
227 return psci_forward(host_ctxt)
228 if (is_psci_0_1(cpu_on, func_id))
229 return psci_cpu_on(func_id, ho
230 if (is_psci_0_1(cpu_suspend, func_id))
231 return psci_cpu_suspend(func_i
232
233 return PSCI_RET_NOT_SUPPORTED;
234 }
235
236 static unsigned long psci_0_2_handler(u64 func
237 {
238 switch (func_id) {
239 case PSCI_0_2_FN_PSCI_VERSION:
240 case PSCI_0_2_FN_CPU_OFF:
241 case PSCI_0_2_FN64_AFFINITY_INFO:
242 case PSCI_0_2_FN64_MIGRATE:
243 case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
244 case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU
245 return psci_forward(host_ctxt)
246 /*
247 * SYSTEM_OFF/RESET should not return
248 * Allow it so as to stay robust to br
249 */
250 case PSCI_0_2_FN_SYSTEM_OFF:
251 case PSCI_0_2_FN_SYSTEM_RESET:
252 return psci_forward(host_ctxt)
253 case PSCI_0_2_FN64_CPU_SUSPEND:
254 return psci_cpu_suspend(func_i
255 case PSCI_0_2_FN64_CPU_ON:
256 return psci_cpu_on(func_id, ho
257 default:
258 return PSCI_RET_NOT_SUPPORTED;
259 }
260 }
261
262 static unsigned long psci_1_0_handler(u64 func
263 {
264 switch (func_id) {
265 case PSCI_1_0_FN_PSCI_FEATURES:
266 case PSCI_1_0_FN_SET_SUSPEND_MODE:
267 case PSCI_1_1_FN64_SYSTEM_RESET2:
268 return psci_forward(host_ctxt)
269 case PSCI_1_0_FN64_SYSTEM_SUSPEND:
270 return psci_system_suspend(fun
271 default:
272 return psci_0_2_handler(func_i
273 }
274 }
275
276 bool kvm_host_psci_handler(struct kvm_cpu_cont
277 {
278 unsigned long ret;
279
280 switch (kvm_host_psci_config.version)
281 case PSCI_VERSION(0, 1):
282 if (!is_psci_0_1_call(func_id)
283 return false;
284 ret = psci_0_1_handler(func_id
285 break;
286 case PSCI_VERSION(0, 2):
287 if (!is_psci_0_2_call(func_id)
288 return false;
289 ret = psci_0_2_handler(func_id
290 break;
291 default:
292 if (!is_psci_0_2_call(func_id)
293 return false;
294 ret = psci_1_0_handler(func_id
295 break;
296 }
297
298 cpu_reg(host_ctxt, 0) = ret;
299 cpu_reg(host_ctxt, 1) = 0;
300 cpu_reg(host_ctxt, 2) = 0;
301 cpu_reg(host_ctxt, 3) = 0;
302 return true;
303 }
304
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.