1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Memory Encryption Support
4 *
5 * Copyright (C) 2019 SUSE
6 *
7 * Author: Joerg Roedel <jroedel@suse.de>
8 */
9
10 #define pr_fmt(fmt) "SEV: " fmt
11
12 #include <linux/sched/debug.h> /* For show_regs() */
13 #include <linux/percpu-defs.h>
14 #include <linux/cc_platform.h>
15 #include <linux/printk.h>
16 #include <linux/mm_types.h>
17 #include <linux/set_memory.h>
18 #include <linux/memblock.h>
19 #include <linux/kernel.h>
20 #include <linux/mm.h>
21 #include <linux/cpumask.h>
22 #include <linux/efi.h>
23 #include <linux/platform_device.h>
24 #include <linux/io.h>
25 #include <linux/psp-sev.h>
26 #include <linux/dmi.h>
27 #include <uapi/linux/sev-guest.h>
28
29 #include <asm/init.h>
30 #include <asm/cpu_entry_area.h>
31 #include <asm/stacktrace.h>
32 #include <asm/sev.h>
33 #include <asm/insn-eval.h>
34 #include <asm/fpu/xcr.h>
35 #include <asm/processor.h>
36 #include <asm/realmode.h>
37 #include <asm/setup.h>
38 #include <asm/traps.h>
39 #include <asm/svm.h>
40 #include <asm/smp.h>
41 #include <asm/cpu.h>
42 #include <asm/apic.h>
43 #include <asm/cpuid.h>
44 #include <asm/cmdline.h>
45
46 #define DR7_RESET_VALUE 0x400
47
48 /* AP INIT values as documented in the APM2 section "Processor Initialization State" */
49 #define AP_INIT_CS_LIMIT 0xffff
50 #define AP_INIT_DS_LIMIT 0xffff
51 #define AP_INIT_LDTR_LIMIT 0xffff
52 #define AP_INIT_GDTR_LIMIT 0xffff
53 #define AP_INIT_IDTR_LIMIT 0xffff
54 #define AP_INIT_TR_LIMIT 0xffff
55 #define AP_INIT_RFLAGS_DEFAULT 0x2
56 #define AP_INIT_DR6_DEFAULT 0xffff0ff0
57 #define AP_INIT_GPAT_DEFAULT 0x0007040600070406ULL
58 #define AP_INIT_XCR0_DEFAULT 0x1
59 #define AP_INIT_X87_FTW_DEFAULT 0x5555
60 #define AP_INIT_X87_FCW_DEFAULT 0x0040
61 #define AP_INIT_CR0_DEFAULT 0x60000010
62 #define AP_INIT_MXCSR_DEFAULT 0x1f80
63
64 static const char * const sev_status_feat_names[] = {
65 [MSR_AMD64_SEV_ENABLED_BIT] = "SEV",
66 [MSR_AMD64_SEV_ES_ENABLED_BIT] = "SEV-ES",
67 [MSR_AMD64_SEV_SNP_ENABLED_BIT] = "SEV-SNP",
68 [MSR_AMD64_SNP_VTOM_BIT] = "vTom",
69 [MSR_AMD64_SNP_REFLECT_VC_BIT] = "ReflectVC",
70 [MSR_AMD64_SNP_RESTRICTED_INJ_BIT] = "RI",
71 [MSR_AMD64_SNP_ALT_INJ_BIT] = "AI",
72 [MSR_AMD64_SNP_DEBUG_SWAP_BIT] = "DebugSwap",
73 [MSR_AMD64_SNP_PREVENT_HOST_IBS_BIT] = "NoHostIBS",
74 [MSR_AMD64_SNP_BTB_ISOLATION_BIT] = "BTBIsol",
75 [MSR_AMD64_SNP_VMPL_SSS_BIT] = "VmplSSS",
76 [MSR_AMD64_SNP_SECURE_TSC_BIT] = "SecureTSC",
77 [MSR_AMD64_SNP_VMGEXIT_PARAM_BIT] = "VMGExitParam",
78 [MSR_AMD64_SNP_IBS_VIRT_BIT] = "IBSVirt",
79 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt",
80 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt",
81 };
82
83 /* For early boot hypervisor communication in SEV-ES enabled guests */
84 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
85
86 /*
87 * Needs to be in the .data section because we need it NULL before bss is
88 * cleared
89 */
90 static struct ghcb *boot_ghcb __section(".data");
91
92 /* Bitmap of SEV features supported by the hypervisor */
93 static u64 sev_hv_features __ro_after_init;
94
95 /* #VC handler runtime per-CPU data */
96 struct sev_es_runtime_data {
97 struct ghcb ghcb_page;
98
99 /*
100 * Reserve one page per CPU as backup storage for the unencrypted GHCB.
101 * It is needed when an NMI happens while the #VC handler uses the real
102 * GHCB, and the NMI handler itself is causing another #VC exception. In
103 * that case the GHCB content of the first handler needs to be backed up
104 * and restored.
105 */
106 struct ghcb backup_ghcb;
107
108 /*
109 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
110 * There is no need for it to be atomic, because nothing is written to
111 * the GHCB between the read and the write of ghcb_active. So it is safe
112 * to use it when a nested #VC exception happens before the write.
113 *
114 * This is necessary for example in the #VC->NMI->#VC case when the NMI
115 * happens while the first #VC handler uses the GHCB. When the NMI code
116 * raises a second #VC handler it might overwrite the contents of the
117 * GHCB written by the first handler. To avoid this the content of the
118 * GHCB is saved and restored when the GHCB is detected to be in use
119 * already.
120 */
121 bool ghcb_active;
122 bool backup_ghcb_active;
123
124 /*
125 * Cached DR7 value - write it on DR7 writes and return it on reads.
126 * That value will never make it to the real hardware DR7 as debugging
127 * is currently unsupported in SEV-ES guests.
128 */
129 unsigned long dr7;
130 };
131
132 struct ghcb_state {
133 struct ghcb *ghcb;
134 };
135
136 /* For early boot SVSM communication */
137 static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
138
139 static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
140 static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
141 static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
142 static DEFINE_PER_CPU(u64, svsm_caa_pa);
143
144 struct sev_config {
145 __u64 debug : 1,
146
147 /*
148 * Indicates when the per-CPU GHCB has been created and registered
149 * and thus can be used by the BSP instead of the early boot GHCB.
150 *
151 * For APs, the per-CPU GHCB is created before they are started
152 * and registered upon startup, so this flag can be used globally
153 * for the BSP and APs.
154 */
155 ghcbs_initialized : 1,
156
157 /*
158 * Indicates when the per-CPU SVSM CA is to be used instead of the
159 * boot SVSM CA.
160 *
161 * For APs, the per-CPU SVSM CA is created as part of the AP
162 * bringup, so this flag can be used globally for the BSP and APs.
163 */
164 use_cas : 1,
165
166 __reserved : 61;
167 };
168
169 static struct sev_config sev_cfg __read_mostly;
170
171 static __always_inline bool on_vc_stack(struct pt_regs *regs)
172 {
173 unsigned long sp = regs->sp;
174
175 /* User-mode RSP is not trusted */
176 if (user_mode(regs))
177 return false;
178
179 /* SYSCALL gap still has user-mode RSP */
180 if (ip_within_syscall_gap(regs))
181 return false;
182
183 return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
184 }
185
186 /*
187 * This function handles the case when an NMI is raised in the #VC
188 * exception handler entry code, before the #VC handler has switched off
189 * its IST stack. In this case, the IST entry for #VC must be adjusted,
190 * so that any nested #VC exception will not overwrite the stack
191 * contents of the interrupted #VC handler.
192 *
193 * The IST entry is adjusted unconditionally so that it can be also be
194 * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a
195 * nested sev_es_ist_exit() call may adjust back the IST entry too
196 * early.
197 *
198 * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run
199 * on the NMI IST stack, as they are only called from NMI handling code
200 * right now.
201 */
202 void noinstr __sev_es_ist_enter(struct pt_regs *regs)
203 {
204 unsigned long old_ist, new_ist;
205
206 /* Read old IST entry */
207 new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
208
209 /*
210 * If NMI happened while on the #VC IST stack, set the new IST
211 * value below regs->sp, so that the interrupted stack frame is
212 * not overwritten by subsequent #VC exceptions.
213 */
214 if (on_vc_stack(regs))
215 new_ist = regs->sp;
216
217 /*
218 * Reserve additional 8 bytes and store old IST value so this
219 * adjustment can be unrolled in __sev_es_ist_exit().
220 */
221 new_ist -= sizeof(old_ist);
222 *(unsigned long *)new_ist = old_ist;
223
224 /* Set new IST entry */
225 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist);
226 }
227
228 void noinstr __sev_es_ist_exit(void)
229 {
230 unsigned long ist;
231
232 /* Read IST entry */
233 ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
234
235 if (WARN_ON(ist == __this_cpu_ist_top_va(VC)))
236 return;
237
238 /* Read back old IST entry and write it to the TSS */
239 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
240 }
241
242 /*
243 * Nothing shall interrupt this code path while holding the per-CPU
244 * GHCB. The backup GHCB is only for NMIs interrupting this path.
245 *
246 * Callers must disable local interrupts around it.
247 */
248 static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
249 {
250 struct sev_es_runtime_data *data;
251 struct ghcb *ghcb;
252
253 WARN_ON(!irqs_disabled());
254
255 data = this_cpu_read(runtime_data);
256 ghcb = &data->ghcb_page;
257
258 if (unlikely(data->ghcb_active)) {
259 /* GHCB is already in use - save its contents */
260
261 if (unlikely(data->backup_ghcb_active)) {
262 /*
263 * Backup-GHCB is also already in use. There is no way
264 * to continue here so just kill the machine. To make
265 * panic() work, mark GHCBs inactive so that messages
266 * can be printed out.
267 */
268 data->ghcb_active = false;
269 data->backup_ghcb_active = false;
270
271 instrumentation_begin();
272 panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
273 instrumentation_end();
274 }
275
276 /* Mark backup_ghcb active before writing to it */
277 data->backup_ghcb_active = true;
278
279 state->ghcb = &data->backup_ghcb;
280
281 /* Backup GHCB content */
282 *state->ghcb = *ghcb;
283 } else {
284 state->ghcb = NULL;
285 data->ghcb_active = true;
286 }
287
288 return ghcb;
289 }
290
291 static inline u64 sev_es_rd_ghcb_msr(void)
292 {
293 return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
294 }
295
296 static __always_inline void sev_es_wr_ghcb_msr(u64 val)
297 {
298 u32 low, high;
299
300 low = (u32)(val);
301 high = (u32)(val >> 32);
302
303 native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
304 }
305
306 static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
307 unsigned char *buffer)
308 {
309 return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
310 }
311
312 static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
313 {
314 char buffer[MAX_INSN_SIZE];
315 int insn_bytes;
316
317 insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
318 if (insn_bytes == 0) {
319 /* Nothing could be copied */
320 ctxt->fi.vector = X86_TRAP_PF;
321 ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
322 ctxt->fi.cr2 = ctxt->regs->ip;
323 return ES_EXCEPTION;
324 } else if (insn_bytes == -EINVAL) {
325 /* Effective RIP could not be calculated */
326 ctxt->fi.vector = X86_TRAP_GP;
327 ctxt->fi.error_code = 0;
328 ctxt->fi.cr2 = 0;
329 return ES_EXCEPTION;
330 }
331
332 if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
333 return ES_DECODE_FAILED;
334
335 if (ctxt->insn.immediate.got)
336 return ES_OK;
337 else
338 return ES_DECODE_FAILED;
339 }
340
341 static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
342 {
343 char buffer[MAX_INSN_SIZE];
344 int res, ret;
345
346 res = vc_fetch_insn_kernel(ctxt, buffer);
347 if (res) {
348 ctxt->fi.vector = X86_TRAP_PF;
349 ctxt->fi.error_code = X86_PF_INSTR;
350 ctxt->fi.cr2 = ctxt->regs->ip;
351 return ES_EXCEPTION;
352 }
353
354 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
355 if (ret < 0)
356 return ES_DECODE_FAILED;
357 else
358 return ES_OK;
359 }
360
361 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
362 {
363 if (user_mode(ctxt->regs))
364 return __vc_decode_user_insn(ctxt);
365 else
366 return __vc_decode_kern_insn(ctxt);
367 }
368
369 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
370 char *dst, char *buf, size_t size)
371 {
372 unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
373
374 /*
375 * This function uses __put_user() independent of whether kernel or user
376 * memory is accessed. This works fine because __put_user() does no
377 * sanity checks of the pointer being accessed. All that it does is
378 * to report when the access failed.
379 *
380 * Also, this function runs in atomic context, so __put_user() is not
381 * allowed to sleep. The page-fault handler detects that it is running
382 * in atomic context and will not try to take mmap_sem and handle the
383 * fault, so additional pagefault_enable()/disable() calls are not
384 * needed.
385 *
386 * The access can't be done via copy_to_user() here because
387 * vc_write_mem() must not use string instructions to access unsafe
388 * memory. The reason is that MOVS is emulated by the #VC handler by
389 * splitting the move up into a read and a write and taking a nested #VC
390 * exception on whatever of them is the MMIO access. Using string
391 * instructions here would cause infinite nesting.
392 */
393 switch (size) {
394 case 1: {
395 u8 d1;
396 u8 __user *target = (u8 __user *)dst;
397
398 memcpy(&d1, buf, 1);
399 if (__put_user(d1, target))
400 goto fault;
401 break;
402 }
403 case 2: {
404 u16 d2;
405 u16 __user *target = (u16 __user *)dst;
406
407 memcpy(&d2, buf, 2);
408 if (__put_user(d2, target))
409 goto fault;
410 break;
411 }
412 case 4: {
413 u32 d4;
414 u32 __user *target = (u32 __user *)dst;
415
416 memcpy(&d4, buf, 4);
417 if (__put_user(d4, target))
418 goto fault;
419 break;
420 }
421 case 8: {
422 u64 d8;
423 u64 __user *target = (u64 __user *)dst;
424
425 memcpy(&d8, buf, 8);
426 if (__put_user(d8, target))
427 goto fault;
428 break;
429 }
430 default:
431 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
432 return ES_UNSUPPORTED;
433 }
434
435 return ES_OK;
436
437 fault:
438 if (user_mode(ctxt->regs))
439 error_code |= X86_PF_USER;
440
441 ctxt->fi.vector = X86_TRAP_PF;
442 ctxt->fi.error_code = error_code;
443 ctxt->fi.cr2 = (unsigned long)dst;
444
445 return ES_EXCEPTION;
446 }
447
448 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
449 char *src, char *buf, size_t size)
450 {
451 unsigned long error_code = X86_PF_PROT;
452
453 /*
454 * This function uses __get_user() independent of whether kernel or user
455 * memory is accessed. This works fine because __get_user() does no
456 * sanity checks of the pointer being accessed. All that it does is
457 * to report when the access failed.
458 *
459 * Also, this function runs in atomic context, so __get_user() is not
460 * allowed to sleep. The page-fault handler detects that it is running
461 * in atomic context and will not try to take mmap_sem and handle the
462 * fault, so additional pagefault_enable()/disable() calls are not
463 * needed.
464 *
465 * The access can't be done via copy_from_user() here because
466 * vc_read_mem() must not use string instructions to access unsafe
467 * memory. The reason is that MOVS is emulated by the #VC handler by
468 * splitting the move up into a read and a write and taking a nested #VC
469 * exception on whatever of them is the MMIO access. Using string
470 * instructions here would cause infinite nesting.
471 */
472 switch (size) {
473 case 1: {
474 u8 d1;
475 u8 __user *s = (u8 __user *)src;
476
477 if (__get_user(d1, s))
478 goto fault;
479 memcpy(buf, &d1, 1);
480 break;
481 }
482 case 2: {
483 u16 d2;
484 u16 __user *s = (u16 __user *)src;
485
486 if (__get_user(d2, s))
487 goto fault;
488 memcpy(buf, &d2, 2);
489 break;
490 }
491 case 4: {
492 u32 d4;
493 u32 __user *s = (u32 __user *)src;
494
495 if (__get_user(d4, s))
496 goto fault;
497 memcpy(buf, &d4, 4);
498 break;
499 }
500 case 8: {
501 u64 d8;
502 u64 __user *s = (u64 __user *)src;
503 if (__get_user(d8, s))
504 goto fault;
505 memcpy(buf, &d8, 8);
506 break;
507 }
508 default:
509 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
510 return ES_UNSUPPORTED;
511 }
512
513 return ES_OK;
514
515 fault:
516 if (user_mode(ctxt->regs))
517 error_code |= X86_PF_USER;
518
519 ctxt->fi.vector = X86_TRAP_PF;
520 ctxt->fi.error_code = error_code;
521 ctxt->fi.cr2 = (unsigned long)src;
522
523 return ES_EXCEPTION;
524 }
525
526 static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
527 unsigned long vaddr, phys_addr_t *paddr)
528 {
529 unsigned long va = (unsigned long)vaddr;
530 unsigned int level;
531 phys_addr_t pa;
532 pgd_t *pgd;
533 pte_t *pte;
534
535 pgd = __va(read_cr3_pa());
536 pgd = &pgd[pgd_index(va)];
537 pte = lookup_address_in_pgd(pgd, va, &level);
538 if (!pte) {
539 ctxt->fi.vector = X86_TRAP_PF;
540 ctxt->fi.cr2 = vaddr;
541 ctxt->fi.error_code = 0;
542
543 if (user_mode(ctxt->regs))
544 ctxt->fi.error_code |= X86_PF_USER;
545
546 return ES_EXCEPTION;
547 }
548
549 if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
550 /* Emulated MMIO to/from encrypted memory not supported */
551 return ES_UNSUPPORTED;
552
553 pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
554 pa |= va & ~page_level_mask(level);
555
556 *paddr = pa;
557
558 return ES_OK;
559 }
560
561 static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
562 {
563 BUG_ON(size > 4);
564
565 if (user_mode(ctxt->regs)) {
566 struct thread_struct *t = ¤t->thread;
567 struct io_bitmap *iobm = t->io_bitmap;
568 size_t idx;
569
570 if (!iobm)
571 goto fault;
572
573 for (idx = port; idx < port + size; ++idx) {
574 if (test_bit(idx, iobm->bitmap))
575 goto fault;
576 }
577 }
578
579 return ES_OK;
580
581 fault:
582 ctxt->fi.vector = X86_TRAP_GP;
583 ctxt->fi.error_code = 0;
584
585 return ES_EXCEPTION;
586 }
587
588 static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
589 {
590 long error_code = ctxt->fi.error_code;
591 int trapnr = ctxt->fi.vector;
592
593 ctxt->regs->orig_ax = ctxt->fi.error_code;
594
595 switch (trapnr) {
596 case X86_TRAP_GP:
597 exc_general_protection(ctxt->regs, error_code);
598 break;
599 case X86_TRAP_UD:
600 exc_invalid_op(ctxt->regs);
601 break;
602 case X86_TRAP_PF:
603 write_cr2(ctxt->fi.cr2);
604 exc_page_fault(ctxt->regs, error_code);
605 break;
606 case X86_TRAP_AC:
607 exc_alignment_check(ctxt->regs, error_code);
608 break;
609 default:
610 pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
611 BUG();
612 }
613 }
614
615 /* Include code shared with pre-decompression boot stage */
616 #include "shared.c"
617
618 static inline struct svsm_ca *svsm_get_caa(void)
619 {
620 /*
621 * Use rIP-relative references when called early in the boot. If
622 * ->use_cas is set, then it is late in the boot and no need
623 * to worry about rIP-relative references.
624 */
625 if (RIP_REL_REF(sev_cfg).use_cas)
626 return this_cpu_read(svsm_caa);
627 else
628 return RIP_REL_REF(boot_svsm_caa);
629 }
630
631 static u64 svsm_get_caa_pa(void)
632 {
633 /*
634 * Use rIP-relative references when called early in the boot. If
635 * ->use_cas is set, then it is late in the boot and no need
636 * to worry about rIP-relative references.
637 */
638 if (RIP_REL_REF(sev_cfg).use_cas)
639 return this_cpu_read(svsm_caa_pa);
640 else
641 return RIP_REL_REF(boot_svsm_caa_pa);
642 }
643
644 static noinstr void __sev_put_ghcb(struct ghcb_state *state)
645 {
646 struct sev_es_runtime_data *data;
647 struct ghcb *ghcb;
648
649 WARN_ON(!irqs_disabled());
650
651 data = this_cpu_read(runtime_data);
652 ghcb = &data->ghcb_page;
653
654 if (state->ghcb) {
655 /* Restore GHCB from Backup */
656 *ghcb = *state->ghcb;
657 data->backup_ghcb_active = false;
658 state->ghcb = NULL;
659 } else {
660 /*
661 * Invalidate the GHCB so a VMGEXIT instruction issued
662 * from userspace won't appear to be valid.
663 */
664 vc_ghcb_invalidate(ghcb);
665 data->ghcb_active = false;
666 }
667 }
668
669 static int svsm_perform_call_protocol(struct svsm_call *call)
670 {
671 struct ghcb_state state;
672 unsigned long flags;
673 struct ghcb *ghcb;
674 int ret;
675
676 /*
677 * This can be called very early in the boot, use native functions in
678 * order to avoid paravirt issues.
679 */
680 flags = native_local_irq_save();
681
682 /*
683 * Use rip-relative references when called early in the boot. If
684 * ghcbs_initialized is set, then it is late in the boot and no need
685 * to worry about rip-relative references in called functions.
686 */
687 if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
688 ghcb = __sev_get_ghcb(&state);
689 else if (RIP_REL_REF(boot_ghcb))
690 ghcb = RIP_REL_REF(boot_ghcb);
691 else
692 ghcb = NULL;
693
694 do {
695 ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
696 : svsm_perform_msr_protocol(call);
697 } while (ret == -EAGAIN);
698
699 if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
700 __sev_put_ghcb(&state);
701
702 native_local_irq_restore(flags);
703
704 return ret;
705 }
706
707 void noinstr __sev_es_nmi_complete(void)
708 {
709 struct ghcb_state state;
710 struct ghcb *ghcb;
711
712 ghcb = __sev_get_ghcb(&state);
713
714 vc_ghcb_invalidate(ghcb);
715 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
716 ghcb_set_sw_exit_info_1(ghcb, 0);
717 ghcb_set_sw_exit_info_2(ghcb, 0);
718
719 sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
720 VMGEXIT();
721
722 __sev_put_ghcb(&state);
723 }
724
725 static u64 __init get_secrets_page(void)
726 {
727 u64 pa_data = boot_params.cc_blob_address;
728 struct cc_blob_sev_info info;
729 void *map;
730
731 /*
732 * The CC blob contains the address of the secrets page, check if the
733 * blob is present.
734 */
735 if (!pa_data)
736 return 0;
737
738 map = early_memremap(pa_data, sizeof(info));
739 if (!map) {
740 pr_err("Unable to locate SNP secrets page: failed to map the Confidential Computing blob.\n");
741 return 0;
742 }
743 memcpy(&info, map, sizeof(info));
744 early_memunmap(map, sizeof(info));
745
746 /* smoke-test the secrets page passed */
747 if (!info.secrets_phys || info.secrets_len != PAGE_SIZE)
748 return 0;
749
750 return info.secrets_phys;
751 }
752
753 static u64 __init get_snp_jump_table_addr(void)
754 {
755 struct snp_secrets_page *secrets;
756 void __iomem *mem;
757 u64 pa, addr;
758
759 pa = get_secrets_page();
760 if (!pa)
761 return 0;
762
763 mem = ioremap_encrypted(pa, PAGE_SIZE);
764 if (!mem) {
765 pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
766 return 0;
767 }
768
769 secrets = (__force struct snp_secrets_page *)mem;
770
771 addr = secrets->os_area.ap_jump_table_pa;
772 iounmap(mem);
773
774 return addr;
775 }
776
777 static u64 __init get_jump_table_addr(void)
778 {
779 struct ghcb_state state;
780 unsigned long flags;
781 struct ghcb *ghcb;
782 u64 ret = 0;
783
784 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
785 return get_snp_jump_table_addr();
786
787 local_irq_save(flags);
788
789 ghcb = __sev_get_ghcb(&state);
790
791 vc_ghcb_invalidate(ghcb);
792 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
793 ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
794 ghcb_set_sw_exit_info_2(ghcb, 0);
795
796 sev_es_wr_ghcb_msr(__pa(ghcb));
797 VMGEXIT();
798
799 if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
800 ghcb_sw_exit_info_2_is_valid(ghcb))
801 ret = ghcb->save.sw_exit_info_2;
802
803 __sev_put_ghcb(&state);
804
805 local_irq_restore(flags);
806
807 return ret;
808 }
809
810 static void __head
811 early_set_pages_state(unsigned long vaddr, unsigned long paddr,
812 unsigned long npages, enum psc_op op)
813 {
814 unsigned long paddr_end;
815 u64 val;
816
817 vaddr = vaddr & PAGE_MASK;
818
819 paddr = paddr & PAGE_MASK;
820 paddr_end = paddr + (npages << PAGE_SHIFT);
821
822 while (paddr < paddr_end) {
823 /* Page validation must be rescinded before changing to shared */
824 if (op == SNP_PAGE_STATE_SHARED)
825 pvalidate_4k_page(vaddr, paddr, false);
826
827 /*
828 * Use the MSR protocol because this function can be called before
829 * the GHCB is established.
830 */
831 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
832 VMGEXIT();
833
834 val = sev_es_rd_ghcb_msr();
835
836 if (WARN(GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP,
837 "Wrong PSC response code: 0x%x\n",
838 (unsigned int)GHCB_RESP_CODE(val)))
839 goto e_term;
840
841 if (WARN(GHCB_MSR_PSC_RESP_VAL(val),
842 "Failed to change page state to '%s' paddr 0x%lx error 0x%llx\n",
843 op == SNP_PAGE_STATE_PRIVATE ? "private" : "shared",
844 paddr, GHCB_MSR_PSC_RESP_VAL(val)))
845 goto e_term;
846
847 /* Page validation must be performed after changing to private */
848 if (op == SNP_PAGE_STATE_PRIVATE)
849 pvalidate_4k_page(vaddr, paddr, true);
850
851 vaddr += PAGE_SIZE;
852 paddr += PAGE_SIZE;
853 }
854
855 return;
856
857 e_term:
858 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
859 }
860
861 void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
862 unsigned long npages)
863 {
864 /*
865 * This can be invoked in early boot while running identity mapped, so
866 * use an open coded check for SNP instead of using cc_platform_has().
867 * This eliminates worries about jump tables or checking boot_cpu_data
868 * in the cc_platform_has() function.
869 */
870 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
871 return;
872
873 /*
874 * Ask the hypervisor to mark the memory pages as private in the RMP
875 * table.
876 */
877 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE);
878 }
879
880 void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
881 unsigned long npages)
882 {
883 /*
884 * This can be invoked in early boot while running identity mapped, so
885 * use an open coded check for SNP instead of using cc_platform_has().
886 * This eliminates worries about jump tables or checking boot_cpu_data
887 * in the cc_platform_has() function.
888 */
889 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
890 return;
891
892 /* Ask hypervisor to mark the memory pages shared in the RMP table. */
893 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED);
894 }
895
896 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
897 unsigned long vaddr_end, int op)
898 {
899 struct ghcb_state state;
900 bool use_large_entry;
901 struct psc_hdr *hdr;
902 struct psc_entry *e;
903 unsigned long flags;
904 unsigned long pfn;
905 struct ghcb *ghcb;
906 int i;
907
908 hdr = &data->hdr;
909 e = data->entries;
910
911 memset(data, 0, sizeof(*data));
912 i = 0;
913
914 while (vaddr < vaddr_end && i < ARRAY_SIZE(data->entries)) {
915 hdr->end_entry = i;
916
917 if (is_vmalloc_addr((void *)vaddr)) {
918 pfn = vmalloc_to_pfn((void *)vaddr);
919 use_large_entry = false;
920 } else {
921 pfn = __pa(vaddr) >> PAGE_SHIFT;
922 use_large_entry = true;
923 }
924
925 e->gfn = pfn;
926 e->operation = op;
927
928 if (use_large_entry && IS_ALIGNED(vaddr, PMD_SIZE) &&
929 (vaddr_end - vaddr) >= PMD_SIZE) {
930 e->pagesize = RMP_PG_SIZE_2M;
931 vaddr += PMD_SIZE;
932 } else {
933 e->pagesize = RMP_PG_SIZE_4K;
934 vaddr += PAGE_SIZE;
935 }
936
937 e++;
938 i++;
939 }
940
941 /* Page validation must be rescinded before changing to shared */
942 if (op == SNP_PAGE_STATE_SHARED)
943 pvalidate_pages(data);
944
945 local_irq_save(flags);
946
947 if (sev_cfg.ghcbs_initialized)
948 ghcb = __sev_get_ghcb(&state);
949 else
950 ghcb = boot_ghcb;
951
952 /* Invoke the hypervisor to perform the page state changes */
953 if (!ghcb || vmgexit_psc(ghcb, data))
954 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
955
956 if (sev_cfg.ghcbs_initialized)
957 __sev_put_ghcb(&state);
958
959 local_irq_restore(flags);
960
961 /* Page validation must be performed after changing to private */
962 if (op == SNP_PAGE_STATE_PRIVATE)
963 pvalidate_pages(data);
964
965 return vaddr;
966 }
967
968 static void set_pages_state(unsigned long vaddr, unsigned long npages, int op)
969 {
970 struct snp_psc_desc desc;
971 unsigned long vaddr_end;
972
973 /* Use the MSR protocol when a GHCB is not available. */
974 if (!boot_ghcb)
975 return early_set_pages_state(vaddr, __pa(vaddr), npages, op);
976
977 vaddr = vaddr & PAGE_MASK;
978 vaddr_end = vaddr + (npages << PAGE_SHIFT);
979
980 while (vaddr < vaddr_end)
981 vaddr = __set_pages_state(&desc, vaddr, vaddr_end, op);
982 }
983
984 void snp_set_memory_shared(unsigned long vaddr, unsigned long npages)
985 {
986 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
987 return;
988
989 set_pages_state(vaddr, npages, SNP_PAGE_STATE_SHARED);
990 }
991
992 void snp_set_memory_private(unsigned long vaddr, unsigned long npages)
993 {
994 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
995 return;
996
997 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
998 }
999
1000 void snp_accept_memory(phys_addr_t start, phys_addr_t end)
1001 {
1002 unsigned long vaddr, npages;
1003
1004 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1005 return;
1006
1007 vaddr = (unsigned long)__va(start);
1008 npages = (end - start) >> PAGE_SHIFT;
1009
1010 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
1011 }
1012
1013 static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa)
1014 {
1015 int ret;
1016
1017 if (snp_vmpl) {
1018 struct svsm_call call = {};
1019 unsigned long flags;
1020
1021 local_irq_save(flags);
1022
1023 call.caa = this_cpu_read(svsm_caa);
1024 call.rcx = __pa(va);
1025
1026 if (make_vmsa) {
1027 /* Protocol 0, Call ID 2 */
1028 call.rax = SVSM_CORE_CALL(SVSM_CORE_CREATE_VCPU);
1029 call.rdx = __pa(caa);
1030 call.r8 = apic_id;
1031 } else {
1032 /* Protocol 0, Call ID 3 */
1033 call.rax = SVSM_CORE_CALL(SVSM_CORE_DELETE_VCPU);
1034 }
1035
1036 ret = svsm_perform_call_protocol(&call);
1037
1038 local_irq_restore(flags);
1039 } else {
1040 /*
1041 * If the kernel runs at VMPL0, it can change the VMSA
1042 * bit for a page using the RMPADJUST instruction.
1043 * However, for the instruction to succeed it must
1044 * target the permissions of a lesser privileged (higher
1045 * numbered) VMPL level, so use VMPL1.
1046 */
1047 u64 attrs = 1;
1048
1049 if (make_vmsa)
1050 attrs |= RMPADJUST_VMSA_PAGE_BIT;
1051
1052 ret = rmpadjust((unsigned long)va, RMP_PG_SIZE_4K, attrs);
1053 }
1054
1055 return ret;
1056 }
1057
1058 #define __ATTR_BASE (SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK)
1059 #define INIT_CS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_READ_MASK | SVM_SELECTOR_CODE_MASK)
1060 #define INIT_DS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_WRITE_MASK)
1061
1062 #define INIT_LDTR_ATTRIBS (SVM_SELECTOR_P_MASK | 2)
1063 #define INIT_TR_ATTRIBS (SVM_SELECTOR_P_MASK | 3)
1064
1065 static void *snp_alloc_vmsa_page(int cpu)
1066 {
1067 struct page *p;
1068
1069 /*
1070 * Allocate VMSA page to work around the SNP erratum where the CPU will
1071 * incorrectly signal an RMP violation #PF if a large page (2MB or 1GB)
1072 * collides with the RMP entry of VMSA page. The recommended workaround
1073 * is to not use a large page.
1074 *
1075 * Allocate an 8k page which is also 8k-aligned.
1076 */
1077 p = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL_ACCOUNT | __GFP_ZERO, 1);
1078 if (!p)
1079 return NULL;
1080
1081 split_page(p, 1);
1082
1083 /* Free the first 4k. This page may be 2M/1G aligned and cannot be used. */
1084 __free_page(p);
1085
1086 return page_address(p + 1);
1087 }
1088
1089 static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id)
1090 {
1091 int err;
1092
1093 err = snp_set_vmsa(vmsa, NULL, apic_id, false);
1094 if (err)
1095 pr_err("clear VMSA page failed (%u), leaking page\n", err);
1096 else
1097 free_page((unsigned long)vmsa);
1098 }
1099
1100 static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip)
1101 {
1102 struct sev_es_save_area *cur_vmsa, *vmsa;
1103 struct ghcb_state state;
1104 struct svsm_ca *caa;
1105 unsigned long flags;
1106 struct ghcb *ghcb;
1107 u8 sipi_vector;
1108 int cpu, ret;
1109 u64 cr4;
1110
1111 /*
1112 * The hypervisor SNP feature support check has happened earlier, just check
1113 * the AP_CREATION one here.
1114 */
1115 if (!(sev_hv_features & GHCB_HV_FT_SNP_AP_CREATION))
1116 return -EOPNOTSUPP;
1117
1118 /*
1119 * Verify the desired start IP against the known trampoline start IP
1120 * to catch any future new trampolines that may be introduced that
1121 * would require a new protected guest entry point.
1122 */
1123 if (WARN_ONCE(start_ip != real_mode_header->trampoline_start,
1124 "Unsupported SNP start_ip: %lx\n", start_ip))
1125 return -EINVAL;
1126
1127 /* Override start_ip with known protected guest start IP */
1128 start_ip = real_mode_header->sev_es_trampoline_start;
1129
1130 /* Find the logical CPU for the APIC ID */
1131 for_each_present_cpu(cpu) {
1132 if (arch_match_cpu_phys_id(cpu, apic_id))
1133 break;
1134 }
1135 if (cpu >= nr_cpu_ids)
1136 return -EINVAL;
1137
1138 cur_vmsa = per_cpu(sev_vmsa, cpu);
1139
1140 /*
1141 * A new VMSA is created each time because there is no guarantee that
1142 * the current VMSA is the kernels or that the vCPU is not running. If
1143 * an attempt was done to use the current VMSA with a running vCPU, a
1144 * #VMEXIT of that vCPU would wipe out all of the settings being done
1145 * here.
1146 */
1147 vmsa = (struct sev_es_save_area *)snp_alloc_vmsa_page(cpu);
1148 if (!vmsa)
1149 return -ENOMEM;
1150
1151 /* If an SVSM is present, the SVSM per-CPU CAA will be !NULL */
1152 caa = per_cpu(svsm_caa, cpu);
1153
1154 /* CR4 should maintain the MCE value */
1155 cr4 = native_read_cr4() & X86_CR4_MCE;
1156
1157 /* Set the CS value based on the start_ip converted to a SIPI vector */
1158 sipi_vector = (start_ip >> 12);
1159 vmsa->cs.base = sipi_vector << 12;
1160 vmsa->cs.limit = AP_INIT_CS_LIMIT;
1161 vmsa->cs.attrib = INIT_CS_ATTRIBS;
1162 vmsa->cs.selector = sipi_vector << 8;
1163
1164 /* Set the RIP value based on start_ip */
1165 vmsa->rip = start_ip & 0xfff;
1166
1167 /* Set AP INIT defaults as documented in the APM */
1168 vmsa->ds.limit = AP_INIT_DS_LIMIT;
1169 vmsa->ds.attrib = INIT_DS_ATTRIBS;
1170 vmsa->es = vmsa->ds;
1171 vmsa->fs = vmsa->ds;
1172 vmsa->gs = vmsa->ds;
1173 vmsa->ss = vmsa->ds;
1174
1175 vmsa->gdtr.limit = AP_INIT_GDTR_LIMIT;
1176 vmsa->ldtr.limit = AP_INIT_LDTR_LIMIT;
1177 vmsa->ldtr.attrib = INIT_LDTR_ATTRIBS;
1178 vmsa->idtr.limit = AP_INIT_IDTR_LIMIT;
1179 vmsa->tr.limit = AP_INIT_TR_LIMIT;
1180 vmsa->tr.attrib = INIT_TR_ATTRIBS;
1181
1182 vmsa->cr4 = cr4;
1183 vmsa->cr0 = AP_INIT_CR0_DEFAULT;
1184 vmsa->dr7 = DR7_RESET_VALUE;
1185 vmsa->dr6 = AP_INIT_DR6_DEFAULT;
1186 vmsa->rflags = AP_INIT_RFLAGS_DEFAULT;
1187 vmsa->g_pat = AP_INIT_GPAT_DEFAULT;
1188 vmsa->xcr0 = AP_INIT_XCR0_DEFAULT;
1189 vmsa->mxcsr = AP_INIT_MXCSR_DEFAULT;
1190 vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT;
1191 vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT;
1192
1193 /* SVME must be set. */
1194 vmsa->efer = EFER_SVME;
1195
1196 /*
1197 * Set the SNP-specific fields for this VMSA:
1198 * VMPL level
1199 * SEV_FEATURES (matches the SEV STATUS MSR right shifted 2 bits)
1200 */
1201 vmsa->vmpl = snp_vmpl;
1202 vmsa->sev_features = sev_status >> 2;
1203
1204 /* Switch the page over to a VMSA page now that it is initialized */
1205 ret = snp_set_vmsa(vmsa, caa, apic_id, true);
1206 if (ret) {
1207 pr_err("set VMSA page failed (%u)\n", ret);
1208 free_page((unsigned long)vmsa);
1209
1210 return -EINVAL;
1211 }
1212
1213 /* Issue VMGEXIT AP Creation NAE event */
1214 local_irq_save(flags);
1215
1216 ghcb = __sev_get_ghcb(&state);
1217
1218 vc_ghcb_invalidate(ghcb);
1219 ghcb_set_rax(ghcb, vmsa->sev_features);
1220 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION);
1221 ghcb_set_sw_exit_info_1(ghcb,
1222 ((u64)apic_id << 32) |
1223 ((u64)snp_vmpl << 16) |
1224 SVM_VMGEXIT_AP_CREATE);
1225 ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa));
1226
1227 sev_es_wr_ghcb_msr(__pa(ghcb));
1228 VMGEXIT();
1229
1230 if (!ghcb_sw_exit_info_1_is_valid(ghcb) ||
1231 lower_32_bits(ghcb->save.sw_exit_info_1)) {
1232 pr_err("SNP AP Creation error\n");
1233 ret = -EINVAL;
1234 }
1235
1236 __sev_put_ghcb(&state);
1237
1238 local_irq_restore(flags);
1239
1240 /* Perform cleanup if there was an error */
1241 if (ret) {
1242 snp_cleanup_vmsa(vmsa, apic_id);
1243 vmsa = NULL;
1244 }
1245
1246 /* Free up any previous VMSA page */
1247 if (cur_vmsa)
1248 snp_cleanup_vmsa(cur_vmsa, apic_id);
1249
1250 /* Record the current VMSA page */
1251 per_cpu(sev_vmsa, cpu) = vmsa;
1252
1253 return ret;
1254 }
1255
1256 void __init snp_set_wakeup_secondary_cpu(void)
1257 {
1258 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1259 return;
1260
1261 /*
1262 * Always set this override if SNP is enabled. This makes it the
1263 * required method to start APs under SNP. If the hypervisor does
1264 * not support AP creation, then no APs will be started.
1265 */
1266 apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit);
1267 }
1268
1269 int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh)
1270 {
1271 u16 startup_cs, startup_ip;
1272 phys_addr_t jump_table_pa;
1273 u64 jump_table_addr;
1274 u16 __iomem *jump_table;
1275
1276 jump_table_addr = get_jump_table_addr();
1277
1278 /* On UP guests there is no jump table so this is not a failure */
1279 if (!jump_table_addr)
1280 return 0;
1281
1282 /* Check if AP Jump Table is page-aligned */
1283 if (jump_table_addr & ~PAGE_MASK)
1284 return -EINVAL;
1285
1286 jump_table_pa = jump_table_addr & PAGE_MASK;
1287
1288 startup_cs = (u16)(rmh->trampoline_start >> 4);
1289 startup_ip = (u16)(rmh->sev_es_trampoline_start -
1290 rmh->trampoline_start);
1291
1292 jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE);
1293 if (!jump_table)
1294 return -EIO;
1295
1296 writew(startup_ip, &jump_table[0]);
1297 writew(startup_cs, &jump_table[1]);
1298
1299 iounmap(jump_table);
1300
1301 return 0;
1302 }
1303
1304 /*
1305 * This is needed by the OVMF UEFI firmware which will use whatever it finds in
1306 * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu
1307 * runtime GHCBs used by the kernel are also mapped in the EFI page-table.
1308 */
1309 int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
1310 {
1311 struct sev_es_runtime_data *data;
1312 unsigned long address, pflags;
1313 int cpu;
1314 u64 pfn;
1315
1316 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1317 return 0;
1318
1319 pflags = _PAGE_NX | _PAGE_RW;
1320
1321 for_each_possible_cpu(cpu) {
1322 data = per_cpu(runtime_data, cpu);
1323
1324 address = __pa(&data->ghcb_page);
1325 pfn = address >> PAGE_SHIFT;
1326
1327 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags))
1328 return 1;
1329 }
1330
1331 return 0;
1332 }
1333
1334 static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
1335 {
1336 struct pt_regs *regs = ctxt->regs;
1337 enum es_result ret;
1338 u64 exit_info_1;
1339
1340 /* Is it a WRMSR? */
1341 exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0;
1342
1343 if (regs->cx == MSR_SVSM_CAA) {
1344 /* Writes to the SVSM CAA msr are ignored */
1345 if (exit_info_1)
1346 return ES_OK;
1347
1348 regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
1349 regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
1350
1351 return ES_OK;
1352 }
1353
1354 ghcb_set_rcx(ghcb, regs->cx);
1355 if (exit_info_1) {
1356 ghcb_set_rax(ghcb, regs->ax);
1357 ghcb_set_rdx(ghcb, regs->dx);
1358 }
1359
1360 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0);
1361
1362 if ((ret == ES_OK) && (!exit_info_1)) {
1363 regs->ax = ghcb->save.rax;
1364 regs->dx = ghcb->save.rdx;
1365 }
1366
1367 return ret;
1368 }
1369
1370 static void snp_register_per_cpu_ghcb(void)
1371 {
1372 struct sev_es_runtime_data *data;
1373 struct ghcb *ghcb;
1374
1375 data = this_cpu_read(runtime_data);
1376 ghcb = &data->ghcb_page;
1377
1378 snp_register_ghcb_early(__pa(ghcb));
1379 }
1380
1381 void setup_ghcb(void)
1382 {
1383 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1384 return;
1385
1386 /*
1387 * Check whether the runtime #VC exception handler is active. It uses
1388 * the per-CPU GHCB page which is set up by sev_es_init_vc_handling().
1389 *
1390 * If SNP is active, register the per-CPU GHCB page so that the runtime
1391 * exception handler can use it.
1392 */
1393 if (initial_vc_handler == (unsigned long)kernel_exc_vmm_communication) {
1394 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1395 snp_register_per_cpu_ghcb();
1396
1397 sev_cfg.ghcbs_initialized = true;
1398
1399 return;
1400 }
1401
1402 /*
1403 * Make sure the hypervisor talks a supported protocol.
1404 * This gets called only in the BSP boot phase.
1405 */
1406 if (!sev_es_negotiate_protocol())
1407 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
1408
1409 /*
1410 * Clear the boot_ghcb. The first exception comes in before the bss
1411 * section is cleared.
1412 */
1413 memset(&boot_ghcb_page, 0, PAGE_SIZE);
1414
1415 /* Alright - Make the boot-ghcb public */
1416 boot_ghcb = &boot_ghcb_page;
1417
1418 /* SNP guest requires that GHCB GPA must be registered. */
1419 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1420 snp_register_ghcb_early(__pa(&boot_ghcb_page));
1421 }
1422
1423 #ifdef CONFIG_HOTPLUG_CPU
1424 static void sev_es_ap_hlt_loop(void)
1425 {
1426 struct ghcb_state state;
1427 struct ghcb *ghcb;
1428
1429 ghcb = __sev_get_ghcb(&state);
1430
1431 while (true) {
1432 vc_ghcb_invalidate(ghcb);
1433 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP);
1434 ghcb_set_sw_exit_info_1(ghcb, 0);
1435 ghcb_set_sw_exit_info_2(ghcb, 0);
1436
1437 sev_es_wr_ghcb_msr(__pa(ghcb));
1438 VMGEXIT();
1439
1440 /* Wakeup signal? */
1441 if (ghcb_sw_exit_info_2_is_valid(ghcb) &&
1442 ghcb->save.sw_exit_info_2)
1443 break;
1444 }
1445
1446 __sev_put_ghcb(&state);
1447 }
1448
1449 /*
1450 * Play_dead handler when running under SEV-ES. This is needed because
1451 * the hypervisor can't deliver an SIPI request to restart the AP.
1452 * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the
1453 * hypervisor wakes it up again.
1454 */
1455 static void sev_es_play_dead(void)
1456 {
1457 play_dead_common();
1458
1459 /* IRQs now disabled */
1460
1461 sev_es_ap_hlt_loop();
1462
1463 /*
1464 * If we get here, the VCPU was woken up again. Jump to CPU
1465 * startup code to get it back online.
1466 */
1467 soft_restart_cpu();
1468 }
1469 #else /* CONFIG_HOTPLUG_CPU */
1470 #define sev_es_play_dead native_play_dead
1471 #endif /* CONFIG_HOTPLUG_CPU */
1472
1473 #ifdef CONFIG_SMP
1474 static void __init sev_es_setup_play_dead(void)
1475 {
1476 smp_ops.play_dead = sev_es_play_dead;
1477 }
1478 #else
1479 static inline void sev_es_setup_play_dead(void) { }
1480 #endif
1481
1482 static void __init alloc_runtime_data(int cpu)
1483 {
1484 struct sev_es_runtime_data *data;
1485
1486 data = memblock_alloc_node(sizeof(*data), PAGE_SIZE, cpu_to_node(cpu));
1487 if (!data)
1488 panic("Can't allocate SEV-ES runtime data");
1489
1490 per_cpu(runtime_data, cpu) = data;
1491
1492 if (snp_vmpl) {
1493 struct svsm_ca *caa;
1494
1495 /* Allocate the SVSM CA page if an SVSM is present */
1496 caa = memblock_alloc(sizeof(*caa), PAGE_SIZE);
1497 if (!caa)
1498 panic("Can't allocate SVSM CA page\n");
1499
1500 per_cpu(svsm_caa, cpu) = caa;
1501 per_cpu(svsm_caa_pa, cpu) = __pa(caa);
1502 }
1503 }
1504
1505 static void __init init_ghcb(int cpu)
1506 {
1507 struct sev_es_runtime_data *data;
1508 int err;
1509
1510 data = per_cpu(runtime_data, cpu);
1511
1512 err = early_set_memory_decrypted((unsigned long)&data->ghcb_page,
1513 sizeof(data->ghcb_page));
1514 if (err)
1515 panic("Can't map GHCBs unencrypted");
1516
1517 memset(&data->ghcb_page, 0, sizeof(data->ghcb_page));
1518
1519 data->ghcb_active = false;
1520 data->backup_ghcb_active = false;
1521 }
1522
1523 void __init sev_es_init_vc_handling(void)
1524 {
1525 int cpu;
1526
1527 BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE);
1528
1529 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1530 return;
1531
1532 if (!sev_es_check_cpu_features())
1533 panic("SEV-ES CPU Features missing");
1534
1535 /*
1536 * SNP is supported in v2 of the GHCB spec which mandates support for HV
1537 * features.
1538 */
1539 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
1540 sev_hv_features = get_hv_features();
1541
1542 if (!(sev_hv_features & GHCB_HV_FT_SNP))
1543 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
1544 }
1545
1546 /* Initialize per-cpu GHCB pages */
1547 for_each_possible_cpu(cpu) {
1548 alloc_runtime_data(cpu);
1549 init_ghcb(cpu);
1550 }
1551
1552 /* If running under an SVSM, switch to the per-cpu CA */
1553 if (snp_vmpl) {
1554 struct svsm_call call = {};
1555 unsigned long flags;
1556 int ret;
1557
1558 local_irq_save(flags);
1559
1560 /*
1561 * SVSM_CORE_REMAP_CA call:
1562 * RAX = 0 (Protocol=0, CallID=0)
1563 * RCX = New CA GPA
1564 */
1565 call.caa = svsm_get_caa();
1566 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
1567 call.rcx = this_cpu_read(svsm_caa_pa);
1568 ret = svsm_perform_call_protocol(&call);
1569 if (ret)
1570 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n",
1571 ret, call.rax_out);
1572
1573 sev_cfg.use_cas = true;
1574
1575 local_irq_restore(flags);
1576 }
1577
1578 sev_es_setup_play_dead();
1579
1580 /* Secondary CPUs use the runtime #VC handler */
1581 initial_vc_handler = (unsigned long)kernel_exc_vmm_communication;
1582 }
1583
1584 static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
1585 {
1586 int trapnr = ctxt->fi.vector;
1587
1588 if (trapnr == X86_TRAP_PF)
1589 native_write_cr2(ctxt->fi.cr2);
1590
1591 ctxt->regs->orig_ax = ctxt->fi.error_code;
1592 do_early_exception(ctxt->regs, trapnr);
1593 }
1594
1595 static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
1596 {
1597 long *reg_array;
1598 int offset;
1599
1600 reg_array = (long *)ctxt->regs;
1601 offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
1602
1603 if (offset < 0)
1604 return NULL;
1605
1606 offset /= sizeof(long);
1607
1608 return reg_array + offset;
1609 }
1610 static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
1611 unsigned int bytes, bool read)
1612 {
1613 u64 exit_code, exit_info_1, exit_info_2;
1614 unsigned long ghcb_pa = __pa(ghcb);
1615 enum es_result res;
1616 phys_addr_t paddr;
1617 void __user *ref;
1618
1619 ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
1620 if (ref == (void __user *)-1L)
1621 return ES_UNSUPPORTED;
1622
1623 exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
1624
1625 res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
1626 if (res != ES_OK) {
1627 if (res == ES_EXCEPTION && !read)
1628 ctxt->fi.error_code |= X86_PF_WRITE;
1629
1630 return res;
1631 }
1632
1633 exit_info_1 = paddr;
1634 /* Can never be greater than 8 */
1635 exit_info_2 = bytes;
1636
1637 ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
1638
1639 return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
1640 }
1641
1642 /*
1643 * The MOVS instruction has two memory operands, which raises the
1644 * problem that it is not known whether the access to the source or the
1645 * destination caused the #VC exception (and hence whether an MMIO read
1646 * or write operation needs to be emulated).
1647 *
1648 * Instead of playing games with walking page-tables and trying to guess
1649 * whether the source or destination is an MMIO range, split the move
1650 * into two operations, a read and a write with only one memory operand.
1651 * This will cause a nested #VC exception on the MMIO address which can
1652 * then be handled.
1653 *
1654 * This implementation has the benefit that it also supports MOVS where
1655 * source _and_ destination are MMIO regions.
1656 *
1657 * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
1658 * rare operation. If it turns out to be a performance problem the split
1659 * operations can be moved to memcpy_fromio() and memcpy_toio().
1660 */
1661 static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
1662 unsigned int bytes)
1663 {
1664 unsigned long ds_base, es_base;
1665 unsigned char *src, *dst;
1666 unsigned char buffer[8];
1667 enum es_result ret;
1668 bool rep;
1669 int off;
1670
1671 ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
1672 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
1673
1674 if (ds_base == -1L || es_base == -1L) {
1675 ctxt->fi.vector = X86_TRAP_GP;
1676 ctxt->fi.error_code = 0;
1677 return ES_EXCEPTION;
1678 }
1679
1680 src = ds_base + (unsigned char *)ctxt->regs->si;
1681 dst = es_base + (unsigned char *)ctxt->regs->di;
1682
1683 ret = vc_read_mem(ctxt, src, buffer, bytes);
1684 if (ret != ES_OK)
1685 return ret;
1686
1687 ret = vc_write_mem(ctxt, dst, buffer, bytes);
1688 if (ret != ES_OK)
1689 return ret;
1690
1691 if (ctxt->regs->flags & X86_EFLAGS_DF)
1692 off = -bytes;
1693 else
1694 off = bytes;
1695
1696 ctxt->regs->si += off;
1697 ctxt->regs->di += off;
1698
1699 rep = insn_has_rep_prefix(&ctxt->insn);
1700 if (rep)
1701 ctxt->regs->cx -= 1;
1702
1703 if (!rep || ctxt->regs->cx == 0)
1704 return ES_OK;
1705 else
1706 return ES_RETRY;
1707 }
1708
1709 static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
1710 {
1711 struct insn *insn = &ctxt->insn;
1712 enum insn_mmio_type mmio;
1713 unsigned int bytes = 0;
1714 enum es_result ret;
1715 u8 sign_byte;
1716 long *reg_data;
1717
1718 mmio = insn_decode_mmio(insn, &bytes);
1719 if (mmio == INSN_MMIO_DECODE_FAILED)
1720 return ES_DECODE_FAILED;
1721
1722 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
1723 reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
1724 if (!reg_data)
1725 return ES_DECODE_FAILED;
1726 }
1727
1728 if (user_mode(ctxt->regs))
1729 return ES_UNSUPPORTED;
1730
1731 switch (mmio) {
1732 case INSN_MMIO_WRITE:
1733 memcpy(ghcb->shared_buffer, reg_data, bytes);
1734 ret = vc_do_mmio(ghcb, ctxt, bytes, false);
1735 break;
1736 case INSN_MMIO_WRITE_IMM:
1737 memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
1738 ret = vc_do_mmio(ghcb, ctxt, bytes, false);
1739 break;
1740 case INSN_MMIO_READ:
1741 ret = vc_do_mmio(ghcb, ctxt, bytes, true);
1742 if (ret)
1743 break;
1744
1745 /* Zero-extend for 32-bit operation */
1746 if (bytes == 4)
1747 *reg_data = 0;
1748
1749 memcpy(reg_data, ghcb->shared_buffer, bytes);
1750 break;
1751 case INSN_MMIO_READ_ZERO_EXTEND:
1752 ret = vc_do_mmio(ghcb, ctxt, bytes, true);
1753 if (ret)
1754 break;
1755
1756 /* Zero extend based on operand size */
1757 memset(reg_data, 0, insn->opnd_bytes);
1758 memcpy(reg_data, ghcb->shared_buffer, bytes);
1759 break;
1760 case INSN_MMIO_READ_SIGN_EXTEND:
1761 ret = vc_do_mmio(ghcb, ctxt, bytes, true);
1762 if (ret)
1763 break;
1764
1765 if (bytes == 1) {
1766 u8 *val = (u8 *)ghcb->shared_buffer;
1767
1768 sign_byte = (*val & 0x80) ? 0xff : 0x00;
1769 } else {
1770 u16 *val = (u16 *)ghcb->shared_buffer;
1771
1772 sign_byte = (*val & 0x8000) ? 0xff : 0x00;
1773 }
1774
1775 /* Sign extend based on operand size */
1776 memset(reg_data, sign_byte, insn->opnd_bytes);
1777 memcpy(reg_data, ghcb->shared_buffer, bytes);
1778 break;
1779 case INSN_MMIO_MOVS:
1780 ret = vc_handle_mmio_movs(ctxt, bytes);
1781 break;
1782 default:
1783 ret = ES_UNSUPPORTED;
1784 break;
1785 }
1786
1787 return ret;
1788 }
1789
1790 static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
1791 struct es_em_ctxt *ctxt)
1792 {
1793 struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
1794 long val, *reg = vc_insn_get_rm(ctxt);
1795 enum es_result ret;
1796
1797 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
1798 return ES_VMM_ERROR;
1799
1800 if (!reg)
1801 return ES_DECODE_FAILED;
1802
1803 val = *reg;
1804
1805 /* Upper 32 bits must be written as zeroes */
1806 if (val >> 32) {
1807 ctxt->fi.vector = X86_TRAP_GP;
1808 ctxt->fi.error_code = 0;
1809 return ES_EXCEPTION;
1810 }
1811
1812 /* Clear out other reserved bits and set bit 10 */
1813 val = (val & 0xffff23ffL) | BIT(10);
1814
1815 /* Early non-zero writes to DR7 are not supported */
1816 if (!data && (val & ~DR7_RESET_VALUE))
1817 return ES_UNSUPPORTED;
1818
1819 /* Using a value of 0 for ExitInfo1 means RAX holds the value */
1820 ghcb_set_rax(ghcb, val);
1821 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
1822 if (ret != ES_OK)
1823 return ret;
1824
1825 if (data)
1826 data->dr7 = val;
1827
1828 return ES_OK;
1829 }
1830
1831 static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
1832 struct es_em_ctxt *ctxt)
1833 {
1834 struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
1835 long *reg = vc_insn_get_rm(ctxt);
1836
1837 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
1838 return ES_VMM_ERROR;
1839
1840 if (!reg)
1841 return ES_DECODE_FAILED;
1842
1843 if (data)
1844 *reg = data->dr7;
1845 else
1846 *reg = DR7_RESET_VALUE;
1847
1848 return ES_OK;
1849 }
1850
1851 static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
1852 struct es_em_ctxt *ctxt)
1853 {
1854 return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
1855 }
1856
1857 static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
1858 {
1859 enum es_result ret;
1860
1861 ghcb_set_rcx(ghcb, ctxt->regs->cx);
1862
1863 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
1864 if (ret != ES_OK)
1865 return ret;
1866
1867 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
1868 return ES_VMM_ERROR;
1869
1870 ctxt->regs->ax = ghcb->save.rax;
1871 ctxt->regs->dx = ghcb->save.rdx;
1872
1873 return ES_OK;
1874 }
1875
1876 static enum es_result vc_handle_monitor(struct ghcb *ghcb,
1877 struct es_em_ctxt *ctxt)
1878 {
1879 /*
1880 * Treat it as a NOP and do not leak a physical address to the
1881 * hypervisor.
1882 */
1883 return ES_OK;
1884 }
1885
1886 static enum es_result vc_handle_mwait(struct ghcb *ghcb,
1887 struct es_em_ctxt *ctxt)
1888 {
1889 /* Treat the same as MONITOR/MONITORX */
1890 return ES_OK;
1891 }
1892
1893 static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
1894 struct es_em_ctxt *ctxt)
1895 {
1896 enum es_result ret;
1897
1898 ghcb_set_rax(ghcb, ctxt->regs->ax);
1899 ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
1900
1901 if (x86_platform.hyper.sev_es_hcall_prepare)
1902 x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
1903
1904 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
1905 if (ret != ES_OK)
1906 return ret;
1907
1908 if (!ghcb_rax_is_valid(ghcb))
1909 return ES_VMM_ERROR;
1910
1911 ctxt->regs->ax = ghcb->save.rax;
1912
1913 /*
1914 * Call sev_es_hcall_finish() after regs->ax is already set.
1915 * This allows the hypervisor handler to overwrite it again if
1916 * necessary.
1917 */
1918 if (x86_platform.hyper.sev_es_hcall_finish &&
1919 !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
1920 return ES_VMM_ERROR;
1921
1922 return ES_OK;
1923 }
1924
1925 static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
1926 struct es_em_ctxt *ctxt)
1927 {
1928 /*
1929 * Calling ecx_alignment_check() directly does not work, because it
1930 * enables IRQs and the GHCB is active. Forward the exception and call
1931 * it later from vc_forward_exception().
1932 */
1933 ctxt->fi.vector = X86_TRAP_AC;
1934 ctxt->fi.error_code = 0;
1935 return ES_EXCEPTION;
1936 }
1937
1938 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
1939 struct ghcb *ghcb,
1940 unsigned long exit_code)
1941 {
1942 enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
1943
1944 if (result != ES_OK)
1945 return result;
1946
1947 switch (exit_code) {
1948 case SVM_EXIT_READ_DR7:
1949 result = vc_handle_dr7_read(ghcb, ctxt);
1950 break;
1951 case SVM_EXIT_WRITE_DR7:
1952 result = vc_handle_dr7_write(ghcb, ctxt);
1953 break;
1954 case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
1955 result = vc_handle_trap_ac(ghcb, ctxt);
1956 break;
1957 case SVM_EXIT_RDTSC:
1958 case SVM_EXIT_RDTSCP:
1959 result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
1960 break;
1961 case SVM_EXIT_RDPMC:
1962 result = vc_handle_rdpmc(ghcb, ctxt);
1963 break;
1964 case SVM_EXIT_INVD:
1965 pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
1966 result = ES_UNSUPPORTED;
1967 break;
1968 case SVM_EXIT_CPUID:
1969 result = vc_handle_cpuid(ghcb, ctxt);
1970 break;
1971 case SVM_EXIT_IOIO:
1972 result = vc_handle_ioio(ghcb, ctxt);
1973 break;
1974 case SVM_EXIT_MSR:
1975 result = vc_handle_msr(ghcb, ctxt);
1976 break;
1977 case SVM_EXIT_VMMCALL:
1978 result = vc_handle_vmmcall(ghcb, ctxt);
1979 break;
1980 case SVM_EXIT_WBINVD:
1981 result = vc_handle_wbinvd(ghcb, ctxt);
1982 break;
1983 case SVM_EXIT_MONITOR:
1984 result = vc_handle_monitor(ghcb, ctxt);
1985 break;
1986 case SVM_EXIT_MWAIT:
1987 result = vc_handle_mwait(ghcb, ctxt);
1988 break;
1989 case SVM_EXIT_NPF:
1990 result = vc_handle_mmio(ghcb, ctxt);
1991 break;
1992 default:
1993 /*
1994 * Unexpected #VC exception
1995 */
1996 result = ES_UNSUPPORTED;
1997 }
1998
1999 return result;
2000 }
2001
2002 static __always_inline bool is_vc2_stack(unsigned long sp)
2003 {
2004 return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
2005 }
2006
2007 static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
2008 {
2009 unsigned long sp, prev_sp;
2010
2011 sp = (unsigned long)regs;
2012 prev_sp = regs->sp;
2013
2014 /*
2015 * If the code was already executing on the VC2 stack when the #VC
2016 * happened, let it proceed to the normal handling routine. This way the
2017 * code executing on the VC2 stack can cause #VC exceptions to get handled.
2018 */
2019 return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
2020 }
2021
2022 static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
2023 {
2024 struct ghcb_state state;
2025 struct es_em_ctxt ctxt;
2026 enum es_result result;
2027 struct ghcb *ghcb;
2028 bool ret = true;
2029
2030 ghcb = __sev_get_ghcb(&state);
2031
2032 vc_ghcb_invalidate(ghcb);
2033 result = vc_init_em_ctxt(&ctxt, regs, error_code);
2034
2035 if (result == ES_OK)
2036 result = vc_handle_exitcode(&ctxt, ghcb, error_code);
2037
2038 __sev_put_ghcb(&state);
2039
2040 /* Done - now check the result */
2041 switch (result) {
2042 case ES_OK:
2043 vc_finish_insn(&ctxt);
2044 break;
2045 case ES_UNSUPPORTED:
2046 pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
2047 error_code, regs->ip);
2048 ret = false;
2049 break;
2050 case ES_VMM_ERROR:
2051 pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
2052 error_code, regs->ip);
2053 ret = false;
2054 break;
2055 case ES_DECODE_FAILED:
2056 pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
2057 error_code, regs->ip);
2058 ret = false;
2059 break;
2060 case ES_EXCEPTION:
2061 vc_forward_exception(&ctxt);
2062 break;
2063 case ES_RETRY:
2064 /* Nothing to do */
2065 break;
2066 default:
2067 pr_emerg("Unknown result in %s():%d\n", __func__, result);
2068 /*
2069 * Emulating the instruction which caused the #VC exception
2070 * failed - can't continue so print debug information
2071 */
2072 BUG();
2073 }
2074
2075 return ret;
2076 }
2077
2078 static __always_inline bool vc_is_db(unsigned long error_code)
2079 {
2080 return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
2081 }
2082
2083 /*
2084 * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
2085 * and will panic when an error happens.
2086 */
2087 DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
2088 {
2089 irqentry_state_t irq_state;
2090
2091 /*
2092 * With the current implementation it is always possible to switch to a
2093 * safe stack because #VC exceptions only happen at known places, like
2094 * intercepted instructions or accesses to MMIO areas/IO ports. They can
2095 * also happen with code instrumentation when the hypervisor intercepts
2096 * #DB, but the critical paths are forbidden to be instrumented, so #DB
2097 * exceptions currently also only happen in safe places.
2098 *
2099 * But keep this here in case the noinstr annotations are violated due
2100 * to bug elsewhere.
2101 */
2102 if (unlikely(vc_from_invalid_context(regs))) {
2103 instrumentation_begin();
2104 panic("Can't handle #VC exception from unsupported context\n");
2105 instrumentation_end();
2106 }
2107
2108 /*
2109 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
2110 */
2111 if (vc_is_db(error_code)) {
2112 exc_debug(regs);
2113 return;
2114 }
2115
2116 irq_state = irqentry_nmi_enter(regs);
2117
2118 instrumentation_begin();
2119
2120 if (!vc_raw_handle_exception(regs, error_code)) {
2121 /* Show some debug info */
2122 show_regs(regs);
2123
2124 /* Ask hypervisor to sev_es_terminate */
2125 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
2126
2127 /* If that fails and we get here - just panic */
2128 panic("Returned from Terminate-Request to Hypervisor\n");
2129 }
2130
2131 instrumentation_end();
2132 irqentry_nmi_exit(regs, irq_state);
2133 }
2134
2135 /*
2136 * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
2137 * and will kill the current task with SIGBUS when an error happens.
2138 */
2139 DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
2140 {
2141 /*
2142 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
2143 */
2144 if (vc_is_db(error_code)) {
2145 noist_exc_debug(regs);
2146 return;
2147 }
2148
2149 irqentry_enter_from_user_mode(regs);
2150 instrumentation_begin();
2151
2152 if (!vc_raw_handle_exception(regs, error_code)) {
2153 /*
2154 * Do not kill the machine if user-space triggered the
2155 * exception. Send SIGBUS instead and let user-space deal with
2156 * it.
2157 */
2158 force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
2159 }
2160
2161 instrumentation_end();
2162 irqentry_exit_to_user_mode(regs);
2163 }
2164
2165 bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
2166 {
2167 unsigned long exit_code = regs->orig_ax;
2168 struct es_em_ctxt ctxt;
2169 enum es_result result;
2170
2171 vc_ghcb_invalidate(boot_ghcb);
2172
2173 result = vc_init_em_ctxt(&ctxt, regs, exit_code);
2174 if (result == ES_OK)
2175 result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
2176
2177 /* Done - now check the result */
2178 switch (result) {
2179 case ES_OK:
2180 vc_finish_insn(&ctxt);
2181 break;
2182 case ES_UNSUPPORTED:
2183 early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
2184 exit_code, regs->ip);
2185 goto fail;
2186 case ES_VMM_ERROR:
2187 early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
2188 exit_code, regs->ip);
2189 goto fail;
2190 case ES_DECODE_FAILED:
2191 early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
2192 exit_code, regs->ip);
2193 goto fail;
2194 case ES_EXCEPTION:
2195 vc_early_forward_exception(&ctxt);
2196 break;
2197 case ES_RETRY:
2198 /* Nothing to do */
2199 break;
2200 default:
2201 BUG();
2202 }
2203
2204 return true;
2205
2206 fail:
2207 show_regs(regs);
2208
2209 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
2210 }
2211
2212 /*
2213 * Initial set up of SNP relies on information provided by the
2214 * Confidential Computing blob, which can be passed to the kernel
2215 * in the following ways, depending on how it is booted:
2216 *
2217 * - when booted via the boot/decompress kernel:
2218 * - via boot_params
2219 *
2220 * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
2221 * - via a setup_data entry, as defined by the Linux Boot Protocol
2222 *
2223 * Scan for the blob in that order.
2224 */
2225 static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
2226 {
2227 struct cc_blob_sev_info *cc_info;
2228
2229 /* Boot kernel would have passed the CC blob via boot_params. */
2230 if (bp->cc_blob_address) {
2231 cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
2232 goto found_cc_info;
2233 }
2234
2235 /*
2236 * If kernel was booted directly, without the use of the
2237 * boot/decompression kernel, the CC blob may have been passed via
2238 * setup_data instead.
2239 */
2240 cc_info = find_cc_blob_setup_data(bp);
2241 if (!cc_info)
2242 return NULL;
2243
2244 found_cc_info:
2245 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
2246 snp_abort();
2247
2248 return cc_info;
2249 }
2250
2251 static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
2252 {
2253 struct svsm_call call = {};
2254 int ret;
2255 u64 pa;
2256
2257 /*
2258 * Record the SVSM Calling Area address (CAA) if the guest is not
2259 * running at VMPL0. The CA will be used to communicate with the
2260 * SVSM to perform the SVSM services.
2261 */
2262 if (!svsm_setup_ca(cc_info))
2263 return;
2264
2265 /*
2266 * It is very early in the boot and the kernel is running identity
2267 * mapped but without having adjusted the pagetables to where the
2268 * kernel was loaded (physbase), so the get the CA address using
2269 * RIP-relative addressing.
2270 */
2271 pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);
2272
2273 /*
2274 * Switch over to the boot SVSM CA while the current CA is still
2275 * addressable. There is no GHCB at this point so use the MSR protocol.
2276 *
2277 * SVSM_CORE_REMAP_CA call:
2278 * RAX = 0 (Protocol=0, CallID=0)
2279 * RCX = New CA GPA
2280 */
2281 call.caa = svsm_get_caa();
2282 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
2283 call.rcx = pa;
2284 ret = svsm_perform_call_protocol(&call);
2285 if (ret)
2286 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", ret, call.rax_out);
2287
2288 RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
2289 RIP_REL_REF(boot_svsm_caa_pa) = pa;
2290 }
2291
2292 bool __head snp_init(struct boot_params *bp)
2293 {
2294 struct cc_blob_sev_info *cc_info;
2295
2296 if (!bp)
2297 return false;
2298
2299 cc_info = find_cc_blob(bp);
2300 if (!cc_info)
2301 return false;
2302
2303 setup_cpuid_table(cc_info);
2304
2305 svsm_setup(cc_info);
2306
2307 /*
2308 * The CC blob will be used later to access the secrets page. Cache
2309 * it here like the boot kernel does.
2310 */
2311 bp->cc_blob_address = (u32)(unsigned long)cc_info;
2312
2313 return true;
2314 }
2315
2316 void __head __noreturn snp_abort(void)
2317 {
2318 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
2319 }
2320
2321 /*
2322 * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are
2323 * enabled, as the alternative (fallback) logic for DMI probing in the legacy
2324 * ROM region can cause a crash since this region is not pre-validated.
2325 */
2326 void __init snp_dmi_setup(void)
2327 {
2328 if (efi_enabled(EFI_CONFIG_TABLES))
2329 dmi_setup();
2330 }
2331
2332 static void dump_cpuid_table(void)
2333 {
2334 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
2335 int i = 0;
2336
2337 pr_info("count=%d reserved=0x%x reserved2=0x%llx\n",
2338 cpuid_table->count, cpuid_table->__reserved1, cpuid_table->__reserved2);
2339
2340 for (i = 0; i < SNP_CPUID_COUNT_MAX; i++) {
2341 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
2342
2343 pr_info("index=%3d fn=0x%08x subfn=0x%08x: eax=0x%08x ebx=0x%08x ecx=0x%08x edx=0x%08x xcr0_in=0x%016llx xss_in=0x%016llx reserved=0x%016llx\n",
2344 i, fn->eax_in, fn->ecx_in, fn->eax, fn->ebx, fn->ecx,
2345 fn->edx, fn->xcr0_in, fn->xss_in, fn->__reserved);
2346 }
2347 }
2348
2349 /*
2350 * It is useful from an auditing/testing perspective to provide an easy way
2351 * for the guest owner to know that the CPUID table has been initialized as
2352 * expected, but that initialization happens too early in boot to print any
2353 * sort of indicator, and there's not really any other good place to do it,
2354 * so do it here.
2355 *
2356 * If running as an SNP guest, report the current VM privilege level (VMPL).
2357 */
2358 static int __init report_snp_info(void)
2359 {
2360 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
2361
2362 if (cpuid_table->count) {
2363 pr_info("Using SNP CPUID table, %d entries present.\n",
2364 cpuid_table->count);
2365
2366 if (sev_cfg.debug)
2367 dump_cpuid_table();
2368 }
2369
2370 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
2371 pr_info("SNP running at VMPL%u.\n", snp_vmpl);
2372
2373 return 0;
2374 }
2375 arch_initcall(report_snp_info);
2376
2377 static int __init init_sev_config(char *str)
2378 {
2379 char *s;
2380
2381 while ((s = strsep(&str, ","))) {
2382 if (!strcmp(s, "debug")) {
2383 sev_cfg.debug = true;
2384 continue;
2385 }
2386
2387 pr_info("SEV command-line option '%s' was not recognized\n", s);
2388 }
2389
2390 return 1;
2391 }
2392 __setup("sev=", init_sev_config);
2393
2394 static void update_attest_input(struct svsm_call *call, struct svsm_attest_call *input)
2395 {
2396 /* If (new) lengths have been returned, propagate them up */
2397 if (call->rcx_out != call->rcx)
2398 input->manifest_buf.len = call->rcx_out;
2399
2400 if (call->rdx_out != call->rdx)
2401 input->certificates_buf.len = call->rdx_out;
2402
2403 if (call->r8_out != call->r8)
2404 input->report_buf.len = call->r8_out;
2405 }
2406
2407 int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call,
2408 struct svsm_attest_call *input)
2409 {
2410 struct svsm_attest_call *ac;
2411 unsigned long flags;
2412 u64 attest_call_pa;
2413 int ret;
2414
2415 if (!snp_vmpl)
2416 return -EINVAL;
2417
2418 local_irq_save(flags);
2419
2420 call->caa = svsm_get_caa();
2421
2422 ac = (struct svsm_attest_call *)call->caa->svsm_buffer;
2423 attest_call_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
2424
2425 *ac = *input;
2426
2427 /*
2428 * Set input registers for the request and set RDX and R8 to known
2429 * values in order to detect length values being returned in them.
2430 */
2431 call->rax = call_id;
2432 call->rcx = attest_call_pa;
2433 call->rdx = -1;
2434 call->r8 = -1;
2435 ret = svsm_perform_call_protocol(call);
2436 update_attest_input(call, input);
2437
2438 local_irq_restore(flags);
2439
2440 return ret;
2441 }
2442 EXPORT_SYMBOL_GPL(snp_issue_svsm_attest_req);
2443
2444 int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio)
2445 {
2446 struct ghcb_state state;
2447 struct es_em_ctxt ctxt;
2448 unsigned long flags;
2449 struct ghcb *ghcb;
2450 int ret;
2451
2452 rio->exitinfo2 = SEV_RET_NO_FW_CALL;
2453
2454 /*
2455 * __sev_get_ghcb() needs to run with IRQs disabled because it is using
2456 * a per-CPU GHCB.
2457 */
2458 local_irq_save(flags);
2459
2460 ghcb = __sev_get_ghcb(&state);
2461 if (!ghcb) {
2462 ret = -EIO;
2463 goto e_restore_irq;
2464 }
2465
2466 vc_ghcb_invalidate(ghcb);
2467
2468 if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
2469 ghcb_set_rax(ghcb, input->data_gpa);
2470 ghcb_set_rbx(ghcb, input->data_npages);
2471 }
2472
2473 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, exit_code, input->req_gpa, input->resp_gpa);
2474 if (ret)
2475 goto e_put;
2476
2477 rio->exitinfo2 = ghcb->save.sw_exit_info_2;
2478 switch (rio->exitinfo2) {
2479 case 0:
2480 break;
2481
2482 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_BUSY):
2483 ret = -EAGAIN;
2484 break;
2485
2486 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN):
2487 /* Number of expected pages are returned in RBX */
2488 if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
2489 input->data_npages = ghcb_get_rbx(ghcb);
2490 ret = -ENOSPC;
2491 break;
2492 }
2493 fallthrough;
2494 default:
2495 ret = -EIO;
2496 break;
2497 }
2498
2499 e_put:
2500 __sev_put_ghcb(&state);
2501 e_restore_irq:
2502 local_irq_restore(flags);
2503
2504 return ret;
2505 }
2506 EXPORT_SYMBOL_GPL(snp_issue_guest_request);
2507
2508 static struct platform_device sev_guest_device = {
2509 .name = "sev-guest",
2510 .id = -1,
2511 };
2512
2513 static int __init snp_init_platform_device(void)
2514 {
2515 struct sev_guest_platform_data data;
2516 u64 gpa;
2517
2518 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
2519 return -ENODEV;
2520
2521 gpa = get_secrets_page();
2522 if (!gpa)
2523 return -ENODEV;
2524
2525 data.secrets_gpa = gpa;
2526 if (platform_device_add_data(&sev_guest_device, &data, sizeof(data)))
2527 return -ENODEV;
2528
2529 if (platform_device_register(&sev_guest_device))
2530 return -ENODEV;
2531
2532 pr_info("SNP guest platform device initialized.\n");
2533 return 0;
2534 }
2535 device_initcall(snp_init_platform_device);
2536
2537 void sev_show_status(void)
2538 {
2539 int i;
2540
2541 pr_info("Status: ");
2542 for (i = 0; i < MSR_AMD64_SNP_RESV_BIT; i++) {
2543 if (sev_status & BIT_ULL(i)) {
2544 if (!sev_status_feat_names[i])
2545 continue;
2546
2547 pr_cont("%s ", sev_status_feat_names[i]);
2548 }
2549 }
2550 pr_cont("\n");
2551 }
2552
2553 void __init snp_update_svsm_ca(void)
2554 {
2555 if (!snp_vmpl)
2556 return;
2557
2558 /* Update the CAA to a proper kernel address */
2559 boot_svsm_caa = &boot_svsm_ca_page;
2560 }
2561
2562 #ifdef CONFIG_SYSFS
2563 static ssize_t vmpl_show(struct kobject *kobj,
2564 struct kobj_attribute *attr, char *buf)
2565 {
2566 return sysfs_emit(buf, "%d\n", snp_vmpl);
2567 }
2568
2569 static struct kobj_attribute vmpl_attr = __ATTR_RO(vmpl);
2570
2571 static struct attribute *vmpl_attrs[] = {
2572 &vmpl_attr.attr,
2573 NULL
2574 };
2575
2576 static struct attribute_group sev_attr_group = {
2577 .attrs = vmpl_attrs,
2578 };
2579
2580 static int __init sev_sysfs_init(void)
2581 {
2582 struct kobject *sev_kobj;
2583 struct device *dev_root;
2584 int ret;
2585
2586 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
2587 return -ENODEV;
2588
2589 dev_root = bus_get_dev_root(&cpu_subsys);
2590 if (!dev_root)
2591 return -ENODEV;
2592
2593 sev_kobj = kobject_create_and_add("sev", &dev_root->kobj);
2594 put_device(dev_root);
2595
2596 if (!sev_kobj)
2597 return -ENOMEM;
2598
2599 ret = sysfs_create_group(sev_kobj, &sev_attr_group);
2600 if (ret)
2601 kobject_put(sev_kobj);
2602
2603 return ret;
2604 }
2605 arch_initcall(sev_sysfs_init);
2606 #endif // CONFIG_SYSFS
2607
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.