1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Google LLC
4 * Author: Quentin Perret <qperret@google.com>
5 */
6
7 #include <linux/kvm_host.h>
8 #include <asm/kvm_emulate.h>
9 #include <asm/kvm_hyp.h>
10 #include <asm/kvm_mmu.h>
11 #include <asm/kvm_pgtable.h>
12 #include <asm/kvm_pkvm.h>
13 #include <asm/stage2_pgtable.h>
14
15 #include <hyp/fault.h>
16
17 #include <nvhe/gfp.h>
18 #include <nvhe/memory.h>
19 #include <nvhe/mem_protect.h>
20 #include <nvhe/mm.h>
21
22 #define KVM_HOST_S2_FLAGS (KVM_PGTABLE_S2_NOFWB | KVM_PGTABLE_S2_IDMAP)
23
24 struct host_mmu host_mmu;
25
26 static struct hyp_pool host_s2_pool;
27
28 static DEFINE_PER_CPU(struct pkvm_hyp_vm *, __current_vm);
29 #define current_vm (*this_cpu_ptr(&__current_vm))
30
31 static void guest_lock_component(struct pkvm_hyp_vm *vm)
32 {
33 hyp_spin_lock(&vm->lock);
34 current_vm = vm;
35 }
36
37 static void guest_unlock_component(struct pkvm_hyp_vm *vm)
38 {
39 current_vm = NULL;
40 hyp_spin_unlock(&vm->lock);
41 }
42
43 static void host_lock_component(void)
44 {
45 hyp_spin_lock(&host_mmu.lock);
46 }
47
48 static void host_unlock_component(void)
49 {
50 hyp_spin_unlock(&host_mmu.lock);
51 }
52
53 static void hyp_lock_component(void)
54 {
55 hyp_spin_lock(&pkvm_pgd_lock);
56 }
57
58 static void hyp_unlock_component(void)
59 {
60 hyp_spin_unlock(&pkvm_pgd_lock);
61 }
62
63 static void *host_s2_zalloc_pages_exact(size_t size)
64 {
65 void *addr = hyp_alloc_pages(&host_s2_pool, get_order(size));
66
67 hyp_split_page(hyp_virt_to_page(addr));
68
69 /*
70 * The size of concatenated PGDs is always a power of two of PAGE_SIZE,
71 * so there should be no need to free any of the tail pages to make the
72 * allocation exact.
73 */
74 WARN_ON(size != (PAGE_SIZE << get_order(size)));
75
76 return addr;
77 }
78
79 static void *host_s2_zalloc_page(void *pool)
80 {
81 return hyp_alloc_pages(pool, 0);
82 }
83
84 static void host_s2_get_page(void *addr)
85 {
86 hyp_get_page(&host_s2_pool, addr);
87 }
88
89 static void host_s2_put_page(void *addr)
90 {
91 hyp_put_page(&host_s2_pool, addr);
92 }
93
94 static void host_s2_free_unlinked_table(void *addr, s8 level)
95 {
96 kvm_pgtable_stage2_free_unlinked(&host_mmu.mm_ops, addr, level);
97 }
98
99 static int prepare_s2_pool(void *pgt_pool_base)
100 {
101 unsigned long nr_pages, pfn;
102 int ret;
103
104 pfn = hyp_virt_to_pfn(pgt_pool_base);
105 nr_pages = host_s2_pgtable_pages();
106 ret = hyp_pool_init(&host_s2_pool, pfn, nr_pages, 0);
107 if (ret)
108 return ret;
109
110 host_mmu.mm_ops = (struct kvm_pgtable_mm_ops) {
111 .zalloc_pages_exact = host_s2_zalloc_pages_exact,
112 .zalloc_page = host_s2_zalloc_page,
113 .free_unlinked_table = host_s2_free_unlinked_table,
114 .phys_to_virt = hyp_phys_to_virt,
115 .virt_to_phys = hyp_virt_to_phys,
116 .page_count = hyp_page_count,
117 .get_page = host_s2_get_page,
118 .put_page = host_s2_put_page,
119 };
120
121 return 0;
122 }
123
124 static void prepare_host_vtcr(void)
125 {
126 u32 parange, phys_shift;
127
128 /* The host stage 2 is id-mapped, so use parange for T0SZ */
129 parange = kvm_get_parange(id_aa64mmfr0_el1_sys_val);
130 phys_shift = id_aa64mmfr0_parange_to_phys_shift(parange);
131
132 host_mmu.arch.mmu.vtcr = kvm_get_vtcr(id_aa64mmfr0_el1_sys_val,
133 id_aa64mmfr1_el1_sys_val, phys_shift);
134 }
135
136 static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot prot);
137
138 int kvm_host_prepare_stage2(void *pgt_pool_base)
139 {
140 struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
141 int ret;
142
143 prepare_host_vtcr();
144 hyp_spin_lock_init(&host_mmu.lock);
145 mmu->arch = &host_mmu.arch;
146
147 ret = prepare_s2_pool(pgt_pool_base);
148 if (ret)
149 return ret;
150
151 ret = __kvm_pgtable_stage2_init(&host_mmu.pgt, mmu,
152 &host_mmu.mm_ops, KVM_HOST_S2_FLAGS,
153 host_stage2_force_pte_cb);
154 if (ret)
155 return ret;
156
157 mmu->pgd_phys = __hyp_pa(host_mmu.pgt.pgd);
158 mmu->pgt = &host_mmu.pgt;
159 atomic64_set(&mmu->vmid.id, 0);
160
161 return 0;
162 }
163
164 static bool guest_stage2_force_pte_cb(u64 addr, u64 end,
165 enum kvm_pgtable_prot prot)
166 {
167 return true;
168 }
169
170 static void *guest_s2_zalloc_pages_exact(size_t size)
171 {
172 void *addr = hyp_alloc_pages(¤t_vm->pool, get_order(size));
173
174 WARN_ON(size != (PAGE_SIZE << get_order(size)));
175 hyp_split_page(hyp_virt_to_page(addr));
176
177 return addr;
178 }
179
180 static void guest_s2_free_pages_exact(void *addr, unsigned long size)
181 {
182 u8 order = get_order(size);
183 unsigned int i;
184
185 for (i = 0; i < (1 << order); i++)
186 hyp_put_page(¤t_vm->pool, addr + (i * PAGE_SIZE));
187 }
188
189 static void *guest_s2_zalloc_page(void *mc)
190 {
191 struct hyp_page *p;
192 void *addr;
193
194 addr = hyp_alloc_pages(¤t_vm->pool, 0);
195 if (addr)
196 return addr;
197
198 addr = pop_hyp_memcache(mc, hyp_phys_to_virt);
199 if (!addr)
200 return addr;
201
202 memset(addr, 0, PAGE_SIZE);
203 p = hyp_virt_to_page(addr);
204 memset(p, 0, sizeof(*p));
205 p->refcount = 1;
206
207 return addr;
208 }
209
210 static void guest_s2_get_page(void *addr)
211 {
212 hyp_get_page(¤t_vm->pool, addr);
213 }
214
215 static void guest_s2_put_page(void *addr)
216 {
217 hyp_put_page(¤t_vm->pool, addr);
218 }
219
220 static void clean_dcache_guest_page(void *va, size_t size)
221 {
222 __clean_dcache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
223 hyp_fixmap_unmap();
224 }
225
226 static void invalidate_icache_guest_page(void *va, size_t size)
227 {
228 __invalidate_icache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
229 hyp_fixmap_unmap();
230 }
231
232 int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd)
233 {
234 struct kvm_s2_mmu *mmu = &vm->kvm.arch.mmu;
235 unsigned long nr_pages;
236 int ret;
237
238 nr_pages = kvm_pgtable_stage2_pgd_size(mmu->vtcr) >> PAGE_SHIFT;
239 ret = hyp_pool_init(&vm->pool, hyp_virt_to_pfn(pgd), nr_pages, 0);
240 if (ret)
241 return ret;
242
243 hyp_spin_lock_init(&vm->lock);
244 vm->mm_ops = (struct kvm_pgtable_mm_ops) {
245 .zalloc_pages_exact = guest_s2_zalloc_pages_exact,
246 .free_pages_exact = guest_s2_free_pages_exact,
247 .zalloc_page = guest_s2_zalloc_page,
248 .phys_to_virt = hyp_phys_to_virt,
249 .virt_to_phys = hyp_virt_to_phys,
250 .page_count = hyp_page_count,
251 .get_page = guest_s2_get_page,
252 .put_page = guest_s2_put_page,
253 .dcache_clean_inval_poc = clean_dcache_guest_page,
254 .icache_inval_pou = invalidate_icache_guest_page,
255 };
256
257 guest_lock_component(vm);
258 ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0,
259 guest_stage2_force_pte_cb);
260 guest_unlock_component(vm);
261 if (ret)
262 return ret;
263
264 vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);
265
266 return 0;
267 }
268
269 void reclaim_guest_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc)
270 {
271 void *addr;
272
273 /* Dump all pgtable pages in the hyp_pool */
274 guest_lock_component(vm);
275 kvm_pgtable_stage2_destroy(&vm->pgt);
276 vm->kvm.arch.mmu.pgd_phys = 0ULL;
277 guest_unlock_component(vm);
278
279 /* Drain the hyp_pool into the memcache */
280 addr = hyp_alloc_pages(&vm->pool, 0);
281 while (addr) {
282 memset(hyp_virt_to_page(addr), 0, sizeof(struct hyp_page));
283 push_hyp_memcache(mc, addr, hyp_virt_to_phys);
284 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(addr), 1));
285 addr = hyp_alloc_pages(&vm->pool, 0);
286 }
287 }
288
289 int __pkvm_prot_finalize(void)
290 {
291 struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
292 struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);
293
294 if (params->hcr_el2 & HCR_VM)
295 return -EPERM;
296
297 params->vttbr = kvm_get_vttbr(mmu);
298 params->vtcr = mmu->vtcr;
299 params->hcr_el2 |= HCR_VM;
300
301 /*
302 * The CMO below not only cleans the updated params to the
303 * PoC, but also provides the DSB that ensures ongoing
304 * page-table walks that have started before we trapped to EL2
305 * have completed.
306 */
307 kvm_flush_dcache_to_poc(params, sizeof(*params));
308
309 write_sysreg(params->hcr_el2, hcr_el2);
310 __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
311
312 /*
313 * Make sure to have an ISB before the TLB maintenance below but only
314 * when __load_stage2() doesn't include one already.
315 */
316 asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT));
317
318 /* Invalidate stale HCR bits that may be cached in TLBs */
319 __tlbi(vmalls12e1);
320 dsb(nsh);
321 isb();
322
323 return 0;
324 }
325
326 static int host_stage2_unmap_dev_all(void)
327 {
328 struct kvm_pgtable *pgt = &host_mmu.pgt;
329 struct memblock_region *reg;
330 u64 addr = 0;
331 int i, ret;
332
333 /* Unmap all non-memory regions to recycle the pages */
334 for (i = 0; i < hyp_memblock_nr; i++, addr = reg->base + reg->size) {
335 reg = &hyp_memory[i];
336 ret = kvm_pgtable_stage2_unmap(pgt, addr, reg->base - addr);
337 if (ret)
338 return ret;
339 }
340 return kvm_pgtable_stage2_unmap(pgt, addr, BIT(pgt->ia_bits) - addr);
341 }
342
343 struct kvm_mem_range {
344 u64 start;
345 u64 end;
346 };
347
348 static struct memblock_region *find_mem_range(phys_addr_t addr, struct kvm_mem_range *range)
349 {
350 int cur, left = 0, right = hyp_memblock_nr;
351 struct memblock_region *reg;
352 phys_addr_t end;
353
354 range->start = 0;
355 range->end = ULONG_MAX;
356
357 /* The list of memblock regions is sorted, binary search it */
358 while (left < right) {
359 cur = (left + right) >> 1;
360 reg = &hyp_memory[cur];
361 end = reg->base + reg->size;
362 if (addr < reg->base) {
363 right = cur;
364 range->end = reg->base;
365 } else if (addr >= end) {
366 left = cur + 1;
367 range->start = end;
368 } else {
369 range->start = reg->base;
370 range->end = end;
371 return reg;
372 }
373 }
374
375 return NULL;
376 }
377
378 bool addr_is_memory(phys_addr_t phys)
379 {
380 struct kvm_mem_range range;
381
382 return !!find_mem_range(phys, &range);
383 }
384
385 static bool addr_is_allowed_memory(phys_addr_t phys)
386 {
387 struct memblock_region *reg;
388 struct kvm_mem_range range;
389
390 reg = find_mem_range(phys, &range);
391
392 return reg && !(reg->flags & MEMBLOCK_NOMAP);
393 }
394
395 static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range)
396 {
397 return range->start <= addr && addr < range->end;
398 }
399
400 static bool range_is_memory(u64 start, u64 end)
401 {
402 struct kvm_mem_range r;
403
404 if (!find_mem_range(start, &r))
405 return false;
406
407 return is_in_mem_range(end - 1, &r);
408 }
409
410 static inline int __host_stage2_idmap(u64 start, u64 end,
411 enum kvm_pgtable_prot prot)
412 {
413 return kvm_pgtable_stage2_map(&host_mmu.pgt, start, end - start, start,
414 prot, &host_s2_pool, 0);
415 }
416
417 /*
418 * The pool has been provided with enough pages to cover all of memory with
419 * page granularity, but it is difficult to know how much of the MMIO range
420 * we will need to cover upfront, so we may need to 'recycle' the pages if we
421 * run out.
422 */
423 #define host_stage2_try(fn, ...) \
424 ({ \
425 int __ret; \
426 hyp_assert_lock_held(&host_mmu.lock); \
427 __ret = fn(__VA_ARGS__); \
428 if (__ret == -ENOMEM) { \
429 __ret = host_stage2_unmap_dev_all(); \
430 if (!__ret) \
431 __ret = fn(__VA_ARGS__); \
432 } \
433 __ret; \
434 })
435
436 static inline bool range_included(struct kvm_mem_range *child,
437 struct kvm_mem_range *parent)
438 {
439 return parent->start <= child->start && child->end <= parent->end;
440 }
441
442 static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range)
443 {
444 struct kvm_mem_range cur;
445 kvm_pte_t pte;
446 s8 level;
447 int ret;
448
449 hyp_assert_lock_held(&host_mmu.lock);
450 ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);
451 if (ret)
452 return ret;
453
454 if (kvm_pte_valid(pte))
455 return -EAGAIN;
456
457 if (pte)
458 return -EPERM;
459
460 do {
461 u64 granule = kvm_granule_size(level);
462 cur.start = ALIGN_DOWN(addr, granule);
463 cur.end = cur.start + granule;
464 level++;
465 } while ((level <= KVM_PGTABLE_LAST_LEVEL) &&
466 !(kvm_level_supports_block_mapping(level) &&
467 range_included(&cur, range)));
468
469 *range = cur;
470
471 return 0;
472 }
473
474 int host_stage2_idmap_locked(phys_addr_t addr, u64 size,
475 enum kvm_pgtable_prot prot)
476 {
477 return host_stage2_try(__host_stage2_idmap, addr, addr + size, prot);
478 }
479
480 int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id)
481 {
482 return host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,
483 addr, size, &host_s2_pool, owner_id);
484 }
485
486 static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot prot)
487 {
488 /*
489 * Block mappings must be used with care in the host stage-2 as a
490 * kvm_pgtable_stage2_map() operation targeting a page in the range of
491 * an existing block will delete the block under the assumption that
492 * mappings in the rest of the block range can always be rebuilt lazily.
493 * That assumption is correct for the host stage-2 with RWX mappings
494 * targeting memory or RW mappings targeting MMIO ranges (see
495 * host_stage2_idmap() below which implements some of the host memory
496 * abort logic). However, this is not safe for any other mappings where
497 * the host stage-2 page-table is in fact the only place where this
498 * state is stored. In all those cases, it is safer to use page-level
499 * mappings, hence avoiding to lose the state because of side-effects in
500 * kvm_pgtable_stage2_map().
501 */
502 if (range_is_memory(addr, end))
503 return prot != PKVM_HOST_MEM_PROT;
504 else
505 return prot != PKVM_HOST_MMIO_PROT;
506 }
507
508 static int host_stage2_idmap(u64 addr)
509 {
510 struct kvm_mem_range range;
511 bool is_memory = !!find_mem_range(addr, &range);
512 enum kvm_pgtable_prot prot;
513 int ret;
514
515 prot = is_memory ? PKVM_HOST_MEM_PROT : PKVM_HOST_MMIO_PROT;
516
517 host_lock_component();
518 ret = host_stage2_adjust_range(addr, &range);
519 if (ret)
520 goto unlock;
521
522 ret = host_stage2_idmap_locked(range.start, range.end - range.start, prot);
523 unlock:
524 host_unlock_component();
525
526 return ret;
527 }
528
529 void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
530 {
531 struct kvm_vcpu_fault_info fault;
532 u64 esr, addr;
533 int ret = 0;
534
535 esr = read_sysreg_el2(SYS_ESR);
536 if (!__get_fault_info(esr, &fault)) {
537 /*
538 * We've presumably raced with a page-table change which caused
539 * AT to fail, try again.
540 */
541 return;
542 }
543
544 addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;
545 ret = host_stage2_idmap(addr);
546 BUG_ON(ret && ret != -EAGAIN);
547 }
548
549 struct pkvm_mem_transition {
550 u64 nr_pages;
551
552 struct {
553 enum pkvm_component_id id;
554 /* Address in the initiator's address space */
555 u64 addr;
556
557 union {
558 struct {
559 /* Address in the completer's address space */
560 u64 completer_addr;
561 } host;
562 struct {
563 u64 completer_addr;
564 } hyp;
565 };
566 } initiator;
567
568 struct {
569 enum pkvm_component_id id;
570 } completer;
571 };
572
573 struct pkvm_mem_share {
574 const struct pkvm_mem_transition tx;
575 const enum kvm_pgtable_prot completer_prot;
576 };
577
578 struct pkvm_mem_donation {
579 const struct pkvm_mem_transition tx;
580 };
581
582 struct check_walk_data {
583 enum pkvm_page_state desired;
584 enum pkvm_page_state (*get_page_state)(kvm_pte_t pte, u64 addr);
585 };
586
587 static int __check_page_state_visitor(const struct kvm_pgtable_visit_ctx *ctx,
588 enum kvm_pgtable_walk_flags visit)
589 {
590 struct check_walk_data *d = ctx->arg;
591
592 return d->get_page_state(ctx->old, ctx->addr) == d->desired ? 0 : -EPERM;
593 }
594
595 static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,
596 struct check_walk_data *data)
597 {
598 struct kvm_pgtable_walker walker = {
599 .cb = __check_page_state_visitor,
600 .arg = data,
601 .flags = KVM_PGTABLE_WALK_LEAF,
602 };
603
604 return kvm_pgtable_walk(pgt, addr, size, &walker);
605 }
606
607 static enum pkvm_page_state host_get_page_state(kvm_pte_t pte, u64 addr)
608 {
609 if (!addr_is_allowed_memory(addr))
610 return PKVM_NOPAGE;
611
612 if (!kvm_pte_valid(pte) && pte)
613 return PKVM_NOPAGE;
614
615 return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte));
616 }
617
618 static int __host_check_page_state_range(u64 addr, u64 size,
619 enum pkvm_page_state state)
620 {
621 struct check_walk_data d = {
622 .desired = state,
623 .get_page_state = host_get_page_state,
624 };
625
626 hyp_assert_lock_held(&host_mmu.lock);
627 return check_page_state_range(&host_mmu.pgt, addr, size, &d);
628 }
629
630 static int __host_set_page_state_range(u64 addr, u64 size,
631 enum pkvm_page_state state)
632 {
633 enum kvm_pgtable_prot prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, state);
634
635 return host_stage2_idmap_locked(addr, size, prot);
636 }
637
638 static int host_request_owned_transition(u64 *completer_addr,
639 const struct pkvm_mem_transition *tx)
640 {
641 u64 size = tx->nr_pages * PAGE_SIZE;
642 u64 addr = tx->initiator.addr;
643
644 *completer_addr = tx->initiator.host.completer_addr;
645 return __host_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
646 }
647
648 static int host_request_unshare(u64 *completer_addr,
649 const struct pkvm_mem_transition *tx)
650 {
651 u64 size = tx->nr_pages * PAGE_SIZE;
652 u64 addr = tx->initiator.addr;
653
654 *completer_addr = tx->initiator.host.completer_addr;
655 return __host_check_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
656 }
657
658 static int host_initiate_share(u64 *completer_addr,
659 const struct pkvm_mem_transition *tx)
660 {
661 u64 size = tx->nr_pages * PAGE_SIZE;
662 u64 addr = tx->initiator.addr;
663
664 *completer_addr = tx->initiator.host.completer_addr;
665 return __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
666 }
667
668 static int host_initiate_unshare(u64 *completer_addr,
669 const struct pkvm_mem_transition *tx)
670 {
671 u64 size = tx->nr_pages * PAGE_SIZE;
672 u64 addr = tx->initiator.addr;
673
674 *completer_addr = tx->initiator.host.completer_addr;
675 return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED);
676 }
677
678 static int host_initiate_donation(u64 *completer_addr,
679 const struct pkvm_mem_transition *tx)
680 {
681 u8 owner_id = tx->completer.id;
682 u64 size = tx->nr_pages * PAGE_SIZE;
683
684 *completer_addr = tx->initiator.host.completer_addr;
685 return host_stage2_set_owner_locked(tx->initiator.addr, size, owner_id);
686 }
687
688 static bool __host_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
689 {
690 return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
691 tx->initiator.id != PKVM_ID_HYP);
692 }
693
694 static int __host_ack_transition(u64 addr, const struct pkvm_mem_transition *tx,
695 enum pkvm_page_state state)
696 {
697 u64 size = tx->nr_pages * PAGE_SIZE;
698
699 if (__host_ack_skip_pgtable_check(tx))
700 return 0;
701
702 return __host_check_page_state_range(addr, size, state);
703 }
704
705 static int host_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
706 {
707 return __host_ack_transition(addr, tx, PKVM_NOPAGE);
708 }
709
710 static int host_complete_donation(u64 addr, const struct pkvm_mem_transition *tx)
711 {
712 u64 size = tx->nr_pages * PAGE_SIZE;
713 u8 host_id = tx->completer.id;
714
715 return host_stage2_set_owner_locked(addr, size, host_id);
716 }
717
718 static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte, u64 addr)
719 {
720 if (!kvm_pte_valid(pte))
721 return PKVM_NOPAGE;
722
723 return pkvm_getstate(kvm_pgtable_hyp_pte_prot(pte));
724 }
725
726 static int __hyp_check_page_state_range(u64 addr, u64 size,
727 enum pkvm_page_state state)
728 {
729 struct check_walk_data d = {
730 .desired = state,
731 .get_page_state = hyp_get_page_state,
732 };
733
734 hyp_assert_lock_held(&pkvm_pgd_lock);
735 return check_page_state_range(&pkvm_pgtable, addr, size, &d);
736 }
737
738 static int hyp_request_donation(u64 *completer_addr,
739 const struct pkvm_mem_transition *tx)
740 {
741 u64 size = tx->nr_pages * PAGE_SIZE;
742 u64 addr = tx->initiator.addr;
743
744 *completer_addr = tx->initiator.hyp.completer_addr;
745 return __hyp_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
746 }
747
748 static int hyp_initiate_donation(u64 *completer_addr,
749 const struct pkvm_mem_transition *tx)
750 {
751 u64 size = tx->nr_pages * PAGE_SIZE;
752 int ret;
753
754 *completer_addr = tx->initiator.hyp.completer_addr;
755 ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, tx->initiator.addr, size);
756 return (ret != size) ? -EFAULT : 0;
757 }
758
759 static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
760 {
761 return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
762 tx->initiator.id != PKVM_ID_HOST);
763 }
764
765 static int hyp_ack_share(u64 addr, const struct pkvm_mem_transition *tx,
766 enum kvm_pgtable_prot perms)
767 {
768 u64 size = tx->nr_pages * PAGE_SIZE;
769
770 if (perms != PAGE_HYP)
771 return -EPERM;
772
773 if (__hyp_ack_skip_pgtable_check(tx))
774 return 0;
775
776 return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
777 }
778
779 static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx)
780 {
781 u64 size = tx->nr_pages * PAGE_SIZE;
782
783 if (tx->initiator.id == PKVM_ID_HOST && hyp_page_count((void *)addr))
784 return -EBUSY;
785
786 if (__hyp_ack_skip_pgtable_check(tx))
787 return 0;
788
789 return __hyp_check_page_state_range(addr, size,
790 PKVM_PAGE_SHARED_BORROWED);
791 }
792
793 static int hyp_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
794 {
795 u64 size = tx->nr_pages * PAGE_SIZE;
796
797 if (__hyp_ack_skip_pgtable_check(tx))
798 return 0;
799
800 return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
801 }
802
803 static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
804 enum kvm_pgtable_prot perms)
805 {
806 void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE);
807 enum kvm_pgtable_prot prot;
808
809 prot = pkvm_mkstate(perms, PKVM_PAGE_SHARED_BORROWED);
810 return pkvm_create_mappings_locked(start, end, prot);
811 }
812
813 static int hyp_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx)
814 {
815 u64 size = tx->nr_pages * PAGE_SIZE;
816 int ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size);
817
818 return (ret != size) ? -EFAULT : 0;
819 }
820
821 static int hyp_complete_donation(u64 addr,
822 const struct pkvm_mem_transition *tx)
823 {
824 void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE);
825 enum kvm_pgtable_prot prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_OWNED);
826
827 return pkvm_create_mappings_locked(start, end, prot);
828 }
829
830 static int check_share(struct pkvm_mem_share *share)
831 {
832 const struct pkvm_mem_transition *tx = &share->tx;
833 u64 completer_addr;
834 int ret;
835
836 switch (tx->initiator.id) {
837 case PKVM_ID_HOST:
838 ret = host_request_owned_transition(&completer_addr, tx);
839 break;
840 default:
841 ret = -EINVAL;
842 }
843
844 if (ret)
845 return ret;
846
847 switch (tx->completer.id) {
848 case PKVM_ID_HYP:
849 ret = hyp_ack_share(completer_addr, tx, share->completer_prot);
850 break;
851 case PKVM_ID_FFA:
852 /*
853 * We only check the host; the secure side will check the other
854 * end when we forward the FFA call.
855 */
856 ret = 0;
857 break;
858 default:
859 ret = -EINVAL;
860 }
861
862 return ret;
863 }
864
865 static int __do_share(struct pkvm_mem_share *share)
866 {
867 const struct pkvm_mem_transition *tx = &share->tx;
868 u64 completer_addr;
869 int ret;
870
871 switch (tx->initiator.id) {
872 case PKVM_ID_HOST:
873 ret = host_initiate_share(&completer_addr, tx);
874 break;
875 default:
876 ret = -EINVAL;
877 }
878
879 if (ret)
880 return ret;
881
882 switch (tx->completer.id) {
883 case PKVM_ID_HYP:
884 ret = hyp_complete_share(completer_addr, tx, share->completer_prot);
885 break;
886 case PKVM_ID_FFA:
887 /*
888 * We're not responsible for any secure page-tables, so there's
889 * nothing to do here.
890 */
891 ret = 0;
892 break;
893 default:
894 ret = -EINVAL;
895 }
896
897 return ret;
898 }
899
900 /*
901 * do_share():
902 *
903 * The page owner grants access to another component with a given set
904 * of permissions.
905 *
906 * Initiator: OWNED => SHARED_OWNED
907 * Completer: NOPAGE => SHARED_BORROWED
908 */
909 static int do_share(struct pkvm_mem_share *share)
910 {
911 int ret;
912
913 ret = check_share(share);
914 if (ret)
915 return ret;
916
917 return WARN_ON(__do_share(share));
918 }
919
920 static int check_unshare(struct pkvm_mem_share *share)
921 {
922 const struct pkvm_mem_transition *tx = &share->tx;
923 u64 completer_addr;
924 int ret;
925
926 switch (tx->initiator.id) {
927 case PKVM_ID_HOST:
928 ret = host_request_unshare(&completer_addr, tx);
929 break;
930 default:
931 ret = -EINVAL;
932 }
933
934 if (ret)
935 return ret;
936
937 switch (tx->completer.id) {
938 case PKVM_ID_HYP:
939 ret = hyp_ack_unshare(completer_addr, tx);
940 break;
941 case PKVM_ID_FFA:
942 /* See check_share() */
943 ret = 0;
944 break;
945 default:
946 ret = -EINVAL;
947 }
948
949 return ret;
950 }
951
952 static int __do_unshare(struct pkvm_mem_share *share)
953 {
954 const struct pkvm_mem_transition *tx = &share->tx;
955 u64 completer_addr;
956 int ret;
957
958 switch (tx->initiator.id) {
959 case PKVM_ID_HOST:
960 ret = host_initiate_unshare(&completer_addr, tx);
961 break;
962 default:
963 ret = -EINVAL;
964 }
965
966 if (ret)
967 return ret;
968
969 switch (tx->completer.id) {
970 case PKVM_ID_HYP:
971 ret = hyp_complete_unshare(completer_addr, tx);
972 break;
973 case PKVM_ID_FFA:
974 /* See __do_share() */
975 ret = 0;
976 break;
977 default:
978 ret = -EINVAL;
979 }
980
981 return ret;
982 }
983
984 /*
985 * do_unshare():
986 *
987 * The page owner revokes access from another component for a range of
988 * pages which were previously shared using do_share().
989 *
990 * Initiator: SHARED_OWNED => OWNED
991 * Completer: SHARED_BORROWED => NOPAGE
992 */
993 static int do_unshare(struct pkvm_mem_share *share)
994 {
995 int ret;
996
997 ret = check_unshare(share);
998 if (ret)
999 return ret;
1000
1001 return WARN_ON(__do_unshare(share));
1002 }
1003
1004 static int check_donation(struct pkvm_mem_donation *donation)
1005 {
1006 const struct pkvm_mem_transition *tx = &donation->tx;
1007 u64 completer_addr;
1008 int ret;
1009
1010 switch (tx->initiator.id) {
1011 case PKVM_ID_HOST:
1012 ret = host_request_owned_transition(&completer_addr, tx);
1013 break;
1014 case PKVM_ID_HYP:
1015 ret = hyp_request_donation(&completer_addr, tx);
1016 break;
1017 default:
1018 ret = -EINVAL;
1019 }
1020
1021 if (ret)
1022 return ret;
1023
1024 switch (tx->completer.id) {
1025 case PKVM_ID_HOST:
1026 ret = host_ack_donation(completer_addr, tx);
1027 break;
1028 case PKVM_ID_HYP:
1029 ret = hyp_ack_donation(completer_addr, tx);
1030 break;
1031 default:
1032 ret = -EINVAL;
1033 }
1034
1035 return ret;
1036 }
1037
1038 static int __do_donate(struct pkvm_mem_donation *donation)
1039 {
1040 const struct pkvm_mem_transition *tx = &donation->tx;
1041 u64 completer_addr;
1042 int ret;
1043
1044 switch (tx->initiator.id) {
1045 case PKVM_ID_HOST:
1046 ret = host_initiate_donation(&completer_addr, tx);
1047 break;
1048 case PKVM_ID_HYP:
1049 ret = hyp_initiate_donation(&completer_addr, tx);
1050 break;
1051 default:
1052 ret = -EINVAL;
1053 }
1054
1055 if (ret)
1056 return ret;
1057
1058 switch (tx->completer.id) {
1059 case PKVM_ID_HOST:
1060 ret = host_complete_donation(completer_addr, tx);
1061 break;
1062 case PKVM_ID_HYP:
1063 ret = hyp_complete_donation(completer_addr, tx);
1064 break;
1065 default:
1066 ret = -EINVAL;
1067 }
1068
1069 return ret;
1070 }
1071
1072 /*
1073 * do_donate():
1074 *
1075 * The page owner transfers ownership to another component, losing access
1076 * as a consequence.
1077 *
1078 * Initiator: OWNED => NOPAGE
1079 * Completer: NOPAGE => OWNED
1080 */
1081 static int do_donate(struct pkvm_mem_donation *donation)
1082 {
1083 int ret;
1084
1085 ret = check_donation(donation);
1086 if (ret)
1087 return ret;
1088
1089 return WARN_ON(__do_donate(donation));
1090 }
1091
1092 int __pkvm_host_share_hyp(u64 pfn)
1093 {
1094 int ret;
1095 u64 host_addr = hyp_pfn_to_phys(pfn);
1096 u64 hyp_addr = (u64)__hyp_va(host_addr);
1097 struct pkvm_mem_share share = {
1098 .tx = {
1099 .nr_pages = 1,
1100 .initiator = {
1101 .id = PKVM_ID_HOST,
1102 .addr = host_addr,
1103 .host = {
1104 .completer_addr = hyp_addr,
1105 },
1106 },
1107 .completer = {
1108 .id = PKVM_ID_HYP,
1109 },
1110 },
1111 .completer_prot = PAGE_HYP,
1112 };
1113
1114 host_lock_component();
1115 hyp_lock_component();
1116
1117 ret = do_share(&share);
1118
1119 hyp_unlock_component();
1120 host_unlock_component();
1121
1122 return ret;
1123 }
1124
1125 int __pkvm_host_unshare_hyp(u64 pfn)
1126 {
1127 int ret;
1128 u64 host_addr = hyp_pfn_to_phys(pfn);
1129 u64 hyp_addr = (u64)__hyp_va(host_addr);
1130 struct pkvm_mem_share share = {
1131 .tx = {
1132 .nr_pages = 1,
1133 .initiator = {
1134 .id = PKVM_ID_HOST,
1135 .addr = host_addr,
1136 .host = {
1137 .completer_addr = hyp_addr,
1138 },
1139 },
1140 .completer = {
1141 .id = PKVM_ID_HYP,
1142 },
1143 },
1144 .completer_prot = PAGE_HYP,
1145 };
1146
1147 host_lock_component();
1148 hyp_lock_component();
1149
1150 ret = do_unshare(&share);
1151
1152 hyp_unlock_component();
1153 host_unlock_component();
1154
1155 return ret;
1156 }
1157
1158 int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages)
1159 {
1160 int ret;
1161 u64 host_addr = hyp_pfn_to_phys(pfn);
1162 u64 hyp_addr = (u64)__hyp_va(host_addr);
1163 struct pkvm_mem_donation donation = {
1164 .tx = {
1165 .nr_pages = nr_pages,
1166 .initiator = {
1167 .id = PKVM_ID_HOST,
1168 .addr = host_addr,
1169 .host = {
1170 .completer_addr = hyp_addr,
1171 },
1172 },
1173 .completer = {
1174 .id = PKVM_ID_HYP,
1175 },
1176 },
1177 };
1178
1179 host_lock_component();
1180 hyp_lock_component();
1181
1182 ret = do_donate(&donation);
1183
1184 hyp_unlock_component();
1185 host_unlock_component();
1186
1187 return ret;
1188 }
1189
1190 int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages)
1191 {
1192 int ret;
1193 u64 host_addr = hyp_pfn_to_phys(pfn);
1194 u64 hyp_addr = (u64)__hyp_va(host_addr);
1195 struct pkvm_mem_donation donation = {
1196 .tx = {
1197 .nr_pages = nr_pages,
1198 .initiator = {
1199 .id = PKVM_ID_HYP,
1200 .addr = hyp_addr,
1201 .hyp = {
1202 .completer_addr = host_addr,
1203 },
1204 },
1205 .completer = {
1206 .id = PKVM_ID_HOST,
1207 },
1208 },
1209 };
1210
1211 host_lock_component();
1212 hyp_lock_component();
1213
1214 ret = do_donate(&donation);
1215
1216 hyp_unlock_component();
1217 host_unlock_component();
1218
1219 return ret;
1220 }
1221
1222 int hyp_pin_shared_mem(void *from, void *to)
1223 {
1224 u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
1225 u64 end = PAGE_ALIGN((u64)to);
1226 u64 size = end - start;
1227 int ret;
1228
1229 host_lock_component();
1230 hyp_lock_component();
1231
1232 ret = __host_check_page_state_range(__hyp_pa(start), size,
1233 PKVM_PAGE_SHARED_OWNED);
1234 if (ret)
1235 goto unlock;
1236
1237 ret = __hyp_check_page_state_range(start, size,
1238 PKVM_PAGE_SHARED_BORROWED);
1239 if (ret)
1240 goto unlock;
1241
1242 for (cur = start; cur < end; cur += PAGE_SIZE)
1243 hyp_page_ref_inc(hyp_virt_to_page(cur));
1244
1245 unlock:
1246 hyp_unlock_component();
1247 host_unlock_component();
1248
1249 return ret;
1250 }
1251
1252 void hyp_unpin_shared_mem(void *from, void *to)
1253 {
1254 u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
1255 u64 end = PAGE_ALIGN((u64)to);
1256
1257 host_lock_component();
1258 hyp_lock_component();
1259
1260 for (cur = start; cur < end; cur += PAGE_SIZE)
1261 hyp_page_ref_dec(hyp_virt_to_page(cur));
1262
1263 hyp_unlock_component();
1264 host_unlock_component();
1265 }
1266
1267 int __pkvm_host_share_ffa(u64 pfn, u64 nr_pages)
1268 {
1269 int ret;
1270 struct pkvm_mem_share share = {
1271 .tx = {
1272 .nr_pages = nr_pages,
1273 .initiator = {
1274 .id = PKVM_ID_HOST,
1275 .addr = hyp_pfn_to_phys(pfn),
1276 },
1277 .completer = {
1278 .id = PKVM_ID_FFA,
1279 },
1280 },
1281 };
1282
1283 host_lock_component();
1284 ret = do_share(&share);
1285 host_unlock_component();
1286
1287 return ret;
1288 }
1289
1290 int __pkvm_host_unshare_ffa(u64 pfn, u64 nr_pages)
1291 {
1292 int ret;
1293 struct pkvm_mem_share share = {
1294 .tx = {
1295 .nr_pages = nr_pages,
1296 .initiator = {
1297 .id = PKVM_ID_HOST,
1298 .addr = hyp_pfn_to_phys(pfn),
1299 },
1300 .completer = {
1301 .id = PKVM_ID_FFA,
1302 },
1303 },
1304 };
1305
1306 host_lock_component();
1307 ret = do_unshare(&share);
1308 host_unlock_component();
1309
1310 return ret;
1311 }
1312
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.