1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 *
4 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
5 * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
6 * Copyright 2016 Alexey Kardashevskiy, IBM Corporation <aik@au1.ibm.com>
7 */
8
9 #include <linux/types.h>
10 #include <linux/string.h>
11 #include <linux/kvm.h>
12 #include <linux/kvm_host.h>
13 #include <linux/highmem.h>
14 #include <linux/gfp.h>
15 #include <linux/slab.h>
16 #include <linux/sched/signal.h>
17 #include <linux/hugetlb.h>
18 #include <linux/list.h>
19 #include <linux/anon_inodes.h>
20 #include <linux/iommu.h>
21 #include <linux/file.h>
22 #include <linux/mm.h>
23 #include <linux/rcupdate_wait.h>
24
25 #include <asm/kvm_ppc.h>
26 #include <asm/kvm_book3s.h>
27 #include <asm/book3s/64/mmu-hash.h>
28 #include <asm/hvcall.h>
29 #include <asm/synch.h>
30 #include <asm/ppc-opcode.h>
31 #include <asm/udbg.h>
32 #include <asm/iommu.h>
33 #include <asm/tce.h>
34 #include <asm/mmu_context.h>
35
36 static struct kvmppc_spapr_tce_table *kvmppc_find_table(struct kvm *kvm,
37 unsigned long liobn)
38 {
39 struct kvmppc_spapr_tce_table *stt;
40
41 list_for_each_entry_lockless(stt, &kvm->arch.spapr_tce_tables, list)
42 if (stt->liobn == liobn)
43 return stt;
44
45 return NULL;
46 }
47
48 static unsigned long kvmppc_tce_pages(unsigned long iommu_pages)
49 {
50 return ALIGN(iommu_pages * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
51 }
52
53 static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
54 {
55 unsigned long stt_bytes = sizeof(struct kvmppc_spapr_tce_table) +
56 (tce_pages * sizeof(struct page *));
57
58 return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
59 }
60
61 static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
62 {
63 struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
64 struct kvmppc_spapr_tce_iommu_table, rcu);
65
66 iommu_tce_table_put(stit->tbl);
67
68 kfree(stit);
69 }
70
71 static void kvm_spapr_tce_liobn_put(struct kref *kref)
72 {
73 struct kvmppc_spapr_tce_iommu_table *stit = container_of(kref,
74 struct kvmppc_spapr_tce_iommu_table, kref);
75
76 list_del_rcu(&stit->next);
77
78 call_rcu(&stit->rcu, kvm_spapr_tce_iommu_table_free);
79 }
80
81 void kvm_spapr_tce_release_iommu_group(struct kvm *kvm,
82 struct iommu_group *grp)
83 {
84 int i;
85 struct kvmppc_spapr_tce_table *stt;
86 struct kvmppc_spapr_tce_iommu_table *stit, *tmp;
87 struct iommu_table_group *table_group = NULL;
88
89 rcu_read_lock();
90 list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {
91
92 table_group = iommu_group_get_iommudata(grp);
93 if (WARN_ON(!table_group))
94 continue;
95
96 list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {
97 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
98 if (table_group->tables[i] != stit->tbl)
99 continue;
100
101 kref_put(&stit->kref, kvm_spapr_tce_liobn_put);
102 }
103 }
104 cond_resched_rcu();
105 }
106 rcu_read_unlock();
107 }
108
109 long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd,
110 struct iommu_group *grp)
111 {
112 struct kvmppc_spapr_tce_table *stt = NULL;
113 bool found = false;
114 struct iommu_table *tbl = NULL;
115 struct iommu_table_group *table_group;
116 long i;
117 struct kvmppc_spapr_tce_iommu_table *stit;
118 struct fd f;
119
120 f = fdget(tablefd);
121 if (!f.file)
122 return -EBADF;
123
124 rcu_read_lock();
125 list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {
126 if (stt == f.file->private_data) {
127 found = true;
128 break;
129 }
130 }
131 rcu_read_unlock();
132
133 if (!found) {
134 fdput(f);
135 return -EINVAL;
136 }
137
138 table_group = iommu_group_get_iommudata(grp);
139 if (WARN_ON(!table_group)) {
140 fdput(f);
141 return -EFAULT;
142 }
143
144 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
145 struct iommu_table *tbltmp = table_group->tables[i];
146
147 if (!tbltmp)
148 continue;
149 /* Make sure hardware table parameters are compatible */
150 if ((tbltmp->it_page_shift <= stt->page_shift) &&
151 (tbltmp->it_offset << tbltmp->it_page_shift ==
152 stt->offset << stt->page_shift) &&
153 (tbltmp->it_size << tbltmp->it_page_shift >=
154 stt->size << stt->page_shift)) {
155 /*
156 * Reference the table to avoid races with
157 * add/remove DMA windows.
158 */
159 tbl = iommu_tce_table_get(tbltmp);
160 break;
161 }
162 }
163 if (!tbl) {
164 fdput(f);
165 return -EINVAL;
166 }
167
168 rcu_read_lock();
169 list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {
170 if (tbl != stit->tbl)
171 continue;
172
173 if (!kref_get_unless_zero(&stit->kref)) {
174 /* stit is being destroyed */
175 iommu_tce_table_put(tbl);
176 rcu_read_unlock();
177 fdput(f);
178 return -ENOTTY;
179 }
180 /*
181 * The table is already known to this KVM, we just increased
182 * its KVM reference counter and can return.
183 */
184 rcu_read_unlock();
185 fdput(f);
186 return 0;
187 }
188 rcu_read_unlock();
189
190 stit = kzalloc(sizeof(*stit), GFP_KERNEL);
191 if (!stit) {
192 iommu_tce_table_put(tbl);
193 fdput(f);
194 return -ENOMEM;
195 }
196
197 stit->tbl = tbl;
198 kref_init(&stit->kref);
199
200 list_add_rcu(&stit->next, &stt->iommu_tables);
201
202 fdput(f);
203 return 0;
204 }
205
206 static void release_spapr_tce_table(struct rcu_head *head)
207 {
208 struct kvmppc_spapr_tce_table *stt = container_of(head,
209 struct kvmppc_spapr_tce_table, rcu);
210 unsigned long i, npages = kvmppc_tce_pages(stt->size);
211
212 for (i = 0; i < npages; i++)
213 if (stt->pages[i])
214 __free_page(stt->pages[i]);
215
216 kfree(stt);
217 }
218
219 static struct page *kvm_spapr_get_tce_page(struct kvmppc_spapr_tce_table *stt,
220 unsigned long sttpage)
221 {
222 struct page *page = stt->pages[sttpage];
223
224 if (page)
225 return page;
226
227 mutex_lock(&stt->alloc_lock);
228 page = stt->pages[sttpage];
229 if (!page) {
230 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
231 WARN_ON_ONCE(!page);
232 if (page)
233 stt->pages[sttpage] = page;
234 }
235 mutex_unlock(&stt->alloc_lock);
236
237 return page;
238 }
239
240 static vm_fault_t kvm_spapr_tce_fault(struct vm_fault *vmf)
241 {
242 struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data;
243 struct page *page;
244
245 if (vmf->pgoff >= kvmppc_tce_pages(stt->size))
246 return VM_FAULT_SIGBUS;
247
248 page = kvm_spapr_get_tce_page(stt, vmf->pgoff);
249 if (!page)
250 return VM_FAULT_OOM;
251
252 get_page(page);
253 vmf->page = page;
254 return 0;
255 }
256
257 static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
258 .fault = kvm_spapr_tce_fault,
259 };
260
261 static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
262 {
263 vma->vm_ops = &kvm_spapr_tce_vm_ops;
264 return 0;
265 }
266
267 static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
268 {
269 struct kvmppc_spapr_tce_table *stt = filp->private_data;
270 struct kvmppc_spapr_tce_iommu_table *stit, *tmp;
271 struct kvm *kvm = stt->kvm;
272
273 mutex_lock(&kvm->lock);
274 list_del_rcu(&stt->list);
275 mutex_unlock(&kvm->lock);
276
277 list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {
278 WARN_ON(!kref_read(&stit->kref));
279 while (1) {
280 if (kref_put(&stit->kref, kvm_spapr_tce_liobn_put))
281 break;
282 }
283 }
284
285 account_locked_vm(kvm->mm,
286 kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
287
288 kvm_put_kvm(stt->kvm);
289
290 call_rcu(&stt->rcu, release_spapr_tce_table);
291
292 return 0;
293 }
294
295 static const struct file_operations kvm_spapr_tce_fops = {
296 .mmap = kvm_spapr_tce_mmap,
297 .release = kvm_spapr_tce_release,
298 };
299
300 int kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
301 struct kvm_create_spapr_tce_64 *args)
302 {
303 struct kvmppc_spapr_tce_table *stt = NULL;
304 struct kvmppc_spapr_tce_table *siter;
305 struct mm_struct *mm = kvm->mm;
306 unsigned long npages;
307 int ret;
308
309 if (!args->size || args->page_shift < 12 || args->page_shift > 34 ||
310 (args->offset + args->size > (ULLONG_MAX >> args->page_shift)))
311 return -EINVAL;
312
313 npages = kvmppc_tce_pages(args->size);
314 ret = account_locked_vm(mm, kvmppc_stt_pages(npages), true);
315 if (ret)
316 return ret;
317
318 ret = -ENOMEM;
319 stt = kzalloc(struct_size(stt, pages, npages), GFP_KERNEL | __GFP_NOWARN);
320 if (!stt)
321 goto fail_acct;
322
323 stt->liobn = args->liobn;
324 stt->page_shift = args->page_shift;
325 stt->offset = args->offset;
326 stt->size = args->size;
327 stt->kvm = kvm;
328 mutex_init(&stt->alloc_lock);
329 INIT_LIST_HEAD_RCU(&stt->iommu_tables);
330
331 mutex_lock(&kvm->lock);
332
333 /* Check this LIOBN hasn't been previously allocated */
334 ret = 0;
335 list_for_each_entry(siter, &kvm->arch.spapr_tce_tables, list) {
336 if (siter->liobn == args->liobn) {
337 ret = -EBUSY;
338 break;
339 }
340 }
341
342 kvm_get_kvm(kvm);
343 if (!ret)
344 ret = anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
345 stt, O_RDWR | O_CLOEXEC);
346
347 if (ret >= 0)
348 list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);
349 else
350 kvm_put_kvm_no_destroy(kvm);
351
352 mutex_unlock(&kvm->lock);
353
354 if (ret >= 0)
355 return ret;
356
357 kfree(stt);
358 fail_acct:
359 account_locked_vm(mm, kvmppc_stt_pages(npages), false);
360 return ret;
361 }
362
363 static long kvmppc_tce_to_ua(struct kvm *kvm, unsigned long tce,
364 unsigned long *ua)
365 {
366 unsigned long gfn = tce >> PAGE_SHIFT;
367 struct kvm_memory_slot *memslot;
368
369 memslot = __gfn_to_memslot(kvm_memslots(kvm), gfn);
370 if (!memslot)
371 return -EINVAL;
372
373 *ua = __gfn_to_hva_memslot(memslot, gfn) |
374 (tce & ~(PAGE_MASK | TCE_PCI_READ | TCE_PCI_WRITE));
375
376 return 0;
377 }
378
379 static long kvmppc_tce_validate(struct kvmppc_spapr_tce_table *stt,
380 unsigned long tce)
381 {
382 unsigned long gpa = tce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
383 enum dma_data_direction dir = iommu_tce_direction(tce);
384 struct kvmppc_spapr_tce_iommu_table *stit;
385 unsigned long ua = 0;
386
387 /* Allow userspace to poison TCE table */
388 if (dir == DMA_NONE)
389 return H_SUCCESS;
390
391 if (iommu_tce_check_gpa(stt->page_shift, gpa))
392 return H_TOO_HARD;
393
394 if (kvmppc_tce_to_ua(stt->kvm, tce, &ua))
395 return H_TOO_HARD;
396
397 rcu_read_lock();
398 list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {
399 unsigned long hpa = 0;
400 struct mm_iommu_table_group_mem_t *mem;
401 long shift = stit->tbl->it_page_shift;
402
403 mem = mm_iommu_lookup(stt->kvm->mm, ua, 1ULL << shift);
404 if (!mem || mm_iommu_ua_to_hpa(mem, ua, shift, &hpa)) {
405 rcu_read_unlock();
406 return H_TOO_HARD;
407 }
408 }
409 rcu_read_unlock();
410
411 return H_SUCCESS;
412 }
413
414 /*
415 * Handles TCE requests for emulated devices.
416 * Puts guest TCE values to the table and expects user space to convert them.
417 * Cannot fail so kvmppc_tce_validate must be called before it.
418 */
419 static void kvmppc_tce_put(struct kvmppc_spapr_tce_table *stt,
420 unsigned long idx, unsigned long tce)
421 {
422 struct page *page;
423 u64 *tbl;
424 unsigned long sttpage;
425
426 idx -= stt->offset;
427 sttpage = idx / TCES_PER_PAGE;
428 page = stt->pages[sttpage];
429
430 if (!page) {
431 /* We allow any TCE, not just with read|write permissions */
432 if (!tce)
433 return;
434
435 page = kvm_spapr_get_tce_page(stt, sttpage);
436 if (!page)
437 return;
438 }
439 tbl = page_to_virt(page);
440
441 tbl[idx % TCES_PER_PAGE] = tce;
442 }
443
444 static void kvmppc_clear_tce(struct mm_struct *mm, struct kvmppc_spapr_tce_table *stt,
445 struct iommu_table *tbl, unsigned long entry)
446 {
447 unsigned long i;
448 unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
449 unsigned long io_entry = entry << (stt->page_shift - tbl->it_page_shift);
450
451 for (i = 0; i < subpages; ++i) {
452 unsigned long hpa = 0;
453 enum dma_data_direction dir = DMA_NONE;
454
455 iommu_tce_xchg_no_kill(mm, tbl, io_entry + i, &hpa, &dir);
456 }
457 }
458
459 static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
460 struct iommu_table *tbl, unsigned long entry)
461 {
462 struct mm_iommu_table_group_mem_t *mem = NULL;
463 const unsigned long pgsize = 1ULL << tbl->it_page_shift;
464 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
465
466 if (!pua)
467 return H_SUCCESS;
468
469 mem = mm_iommu_lookup(kvm->mm, be64_to_cpu(*pua), pgsize);
470 if (!mem)
471 return H_TOO_HARD;
472
473 mm_iommu_mapped_dec(mem);
474
475 *pua = cpu_to_be64(0);
476
477 return H_SUCCESS;
478 }
479
480 static long kvmppc_tce_iommu_do_unmap(struct kvm *kvm,
481 struct iommu_table *tbl, unsigned long entry)
482 {
483 enum dma_data_direction dir = DMA_NONE;
484 unsigned long hpa = 0;
485 long ret;
486
487 if (WARN_ON_ONCE(iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa,
488 &dir)))
489 return H_TOO_HARD;
490
491 if (dir == DMA_NONE)
492 return H_SUCCESS;
493
494 ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
495 if (ret != H_SUCCESS)
496 iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);
497
498 return ret;
499 }
500
501 static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
502 struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,
503 unsigned long entry)
504 {
505 unsigned long i, ret = H_SUCCESS;
506 unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
507 unsigned long io_entry = entry * subpages;
508
509 for (i = 0; i < subpages; ++i) {
510 ret = kvmppc_tce_iommu_do_unmap(kvm, tbl, io_entry + i);
511 if (ret != H_SUCCESS)
512 break;
513 }
514
515 iommu_tce_kill(tbl, io_entry, subpages);
516
517 return ret;
518 }
519
520 static long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
521 unsigned long entry, unsigned long ua,
522 enum dma_data_direction dir)
523 {
524 long ret;
525 unsigned long hpa;
526 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
527 struct mm_iommu_table_group_mem_t *mem;
528
529 if (!pua)
530 /* it_userspace allocation might be delayed */
531 return H_TOO_HARD;
532
533 mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift);
534 if (!mem)
535 /* This only handles v2 IOMMU type, v1 is handled via ioctl() */
536 return H_TOO_HARD;
537
538 if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, tbl->it_page_shift, &hpa)))
539 return H_TOO_HARD;
540
541 if (mm_iommu_mapped_inc(mem))
542 return H_TOO_HARD;
543
544 ret = iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);
545 if (WARN_ON_ONCE(ret)) {
546 mm_iommu_mapped_dec(mem);
547 return H_TOO_HARD;
548 }
549
550 if (dir != DMA_NONE)
551 kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
552
553 *pua = cpu_to_be64(ua);
554
555 return 0;
556 }
557
558 static long kvmppc_tce_iommu_map(struct kvm *kvm,
559 struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,
560 unsigned long entry, unsigned long ua,
561 enum dma_data_direction dir)
562 {
563 unsigned long i, pgoff, ret = H_SUCCESS;
564 unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);
565 unsigned long io_entry = entry * subpages;
566
567 for (i = 0, pgoff = 0; i < subpages;
568 ++i, pgoff += IOMMU_PAGE_SIZE(tbl)) {
569
570 ret = kvmppc_tce_iommu_do_map(kvm, tbl,
571 io_entry + i, ua + pgoff, dir);
572 if (ret != H_SUCCESS)
573 break;
574 }
575
576 iommu_tce_kill(tbl, io_entry, subpages);
577
578 return ret;
579 }
580
581 long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
582 unsigned long ioba, unsigned long tce)
583 {
584 struct kvmppc_spapr_tce_table *stt;
585 long ret, idx;
586 struct kvmppc_spapr_tce_iommu_table *stit;
587 unsigned long entry, ua = 0;
588 enum dma_data_direction dir;
589
590 /* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
591 /* liobn, ioba, tce); */
592
593 stt = kvmppc_find_table(vcpu->kvm, liobn);
594 if (!stt)
595 return H_TOO_HARD;
596
597 ret = kvmppc_ioba_validate(stt, ioba, 1);
598 if (ret != H_SUCCESS)
599 return ret;
600
601 idx = srcu_read_lock(&vcpu->kvm->srcu);
602
603 ret = kvmppc_tce_validate(stt, tce);
604 if (ret != H_SUCCESS)
605 goto unlock_exit;
606
607 dir = iommu_tce_direction(tce);
608
609 if ((dir != DMA_NONE) && kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) {
610 ret = H_PARAMETER;
611 goto unlock_exit;
612 }
613
614 entry = ioba >> stt->page_shift;
615
616 list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
617 if (dir == DMA_NONE)
618 ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,
619 stit->tbl, entry);
620 else
621 ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,
622 entry, ua, dir);
623
624
625 if (ret != H_SUCCESS) {
626 kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry);
627 goto unlock_exit;
628 }
629 }
630
631 kvmppc_tce_put(stt, entry, tce);
632
633 unlock_exit:
634 srcu_read_unlock(&vcpu->kvm->srcu, idx);
635
636 return ret;
637 }
638 EXPORT_SYMBOL_GPL(kvmppc_h_put_tce);
639
640 long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
641 unsigned long liobn, unsigned long ioba,
642 unsigned long tce_list, unsigned long npages)
643 {
644 struct kvmppc_spapr_tce_table *stt;
645 long i, ret = H_SUCCESS, idx;
646 unsigned long entry, ua = 0;
647 u64 __user *tces;
648 u64 tce;
649 struct kvmppc_spapr_tce_iommu_table *stit;
650
651 stt = kvmppc_find_table(vcpu->kvm, liobn);
652 if (!stt)
653 return H_TOO_HARD;
654
655 entry = ioba >> stt->page_shift;
656 /*
657 * SPAPR spec says that the maximum size of the list is 512 TCEs
658 * so the whole table fits in 4K page
659 */
660 if (npages > 512)
661 return H_PARAMETER;
662
663 if (tce_list & (SZ_4K - 1))
664 return H_PARAMETER;
665
666 ret = kvmppc_ioba_validate(stt, ioba, npages);
667 if (ret != H_SUCCESS)
668 return ret;
669
670 idx = srcu_read_lock(&vcpu->kvm->srcu);
671 if (kvmppc_tce_to_ua(vcpu->kvm, tce_list, &ua)) {
672 ret = H_TOO_HARD;
673 goto unlock_exit;
674 }
675 tces = (u64 __user *) ua;
676
677 for (i = 0; i < npages; ++i) {
678 if (get_user(tce, tces + i)) {
679 ret = H_TOO_HARD;
680 goto unlock_exit;
681 }
682 tce = be64_to_cpu(tce);
683
684 ret = kvmppc_tce_validate(stt, tce);
685 if (ret != H_SUCCESS)
686 goto unlock_exit;
687 }
688
689 for (i = 0; i < npages; ++i) {
690 /*
691 * This looks unsafe, because we validate, then regrab
692 * the TCE from userspace which could have been changed by
693 * another thread.
694 *
695 * But it actually is safe, because the relevant checks will be
696 * re-executed in the following code. If userspace tries to
697 * change this dodgily it will result in a messier failure mode
698 * but won't threaten the host.
699 */
700 if (get_user(tce, tces + i)) {
701 ret = H_TOO_HARD;
702 goto unlock_exit;
703 }
704 tce = be64_to_cpu(tce);
705
706 if (kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) {
707 ret = H_PARAMETER;
708 goto unlock_exit;
709 }
710
711 list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
712 ret = kvmppc_tce_iommu_map(vcpu->kvm, stt,
713 stit->tbl, entry + i, ua,
714 iommu_tce_direction(tce));
715
716 if (ret != H_SUCCESS) {
717 kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl,
718 entry + i);
719 goto unlock_exit;
720 }
721 }
722
723 kvmppc_tce_put(stt, entry + i, tce);
724 }
725
726 unlock_exit:
727 srcu_read_unlock(&vcpu->kvm->srcu, idx);
728
729 return ret;
730 }
731 EXPORT_SYMBOL_GPL(kvmppc_h_put_tce_indirect);
732
733 long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
734 unsigned long liobn, unsigned long ioba,
735 unsigned long tce_value, unsigned long npages)
736 {
737 struct kvmppc_spapr_tce_table *stt;
738 long i, ret;
739 struct kvmppc_spapr_tce_iommu_table *stit;
740
741 stt = kvmppc_find_table(vcpu->kvm, liobn);
742 if (!stt)
743 return H_TOO_HARD;
744
745 ret = kvmppc_ioba_validate(stt, ioba, npages);
746 if (ret != H_SUCCESS)
747 return ret;
748
749 /* Check permission bits only to allow userspace poison TCE for debug */
750 if (tce_value & (TCE_PCI_WRITE | TCE_PCI_READ))
751 return H_PARAMETER;
752
753 list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
754 unsigned long entry = ioba >> stt->page_shift;
755
756 for (i = 0; i < npages; ++i) {
757 ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,
758 stit->tbl, entry + i);
759
760 if (ret == H_SUCCESS)
761 continue;
762
763 if (ret == H_TOO_HARD)
764 return ret;
765
766 WARN_ON_ONCE(1);
767 kvmppc_clear_tce(vcpu->kvm->mm, stt, stit->tbl, entry + i);
768 }
769 }
770
771 for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
772 kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);
773
774 return ret;
775 }
776 EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce);
777
778 long kvmppc_h_get_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
779 unsigned long ioba)
780 {
781 struct kvmppc_spapr_tce_table *stt;
782 long ret;
783 unsigned long idx;
784 struct page *page;
785 u64 *tbl;
786
787 stt = kvmppc_find_table(vcpu->kvm, liobn);
788 if (!stt)
789 return H_TOO_HARD;
790
791 ret = kvmppc_ioba_validate(stt, ioba, 1);
792 if (ret != H_SUCCESS)
793 return ret;
794
795 idx = (ioba >> stt->page_shift) - stt->offset;
796 page = stt->pages[idx / TCES_PER_PAGE];
797 if (!page) {
798 kvmppc_set_gpr(vcpu, 4, 0);
799 return H_SUCCESS;
800 }
801 tbl = (u64 *)page_address(page);
802
803 kvmppc_set_gpr(vcpu, 4, tbl[idx % TCES_PER_PAGE]);
804
805 return H_SUCCESS;
806 }
807 EXPORT_SYMBOL_GPL(kvmppc_h_get_tce);
808
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.