1 // SPDX-License-Identifier: GPL-2.0
2 #include <signal.h>
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <string.h>
6 #include <sys/ioctl.h>
7
8 #include <linux/sizes.h>
9
10 #include <kvm_util.h>
11 #include <processor.h>
12
13 #include "ucall_common.h"
14
15 struct kvm_coalesced_io {
16 struct kvm_coalesced_mmio_ring *ring;
17 uint32_t ring_size;
18 uint64_t mmio_gpa;
19 uint64_t *mmio;
20
21 /*
22 * x86-only, but define pio_port for all architectures to minimize the
23 * amount of #ifdeffery and complexity, without having to sacrifice
24 * verbose error messages.
25 */
26 uint8_t pio_port;
27 };
28
29 static struct kvm_coalesced_io kvm_builtin_io_ring;
30
31 #ifdef __x86_64__
32 static const int has_pio = 1;
33 #else
34 static const int has_pio = 0;
35 #endif
36
37 static void guest_code(struct kvm_coalesced_io *io)
38 {
39 int i, j;
40
41 for (;;) {
42 for (j = 0; j < 1 + has_pio; j++) {
43 /*
44 * KVM always leaves one free entry, i.e. exits to
45 * userspace before the last entry is filled.
46 */
47 for (i = 0; i < io->ring_size - 1; i++) {
48 #ifdef __x86_64__
49 if (i & 1)
50 outl(io->pio_port, io->pio_port + i);
51 else
52 #endif
53 WRITE_ONCE(*io->mmio, io->mmio_gpa + i);
54 }
55 #ifdef __x86_64__
56 if (j & 1)
57 outl(io->pio_port, io->pio_port + i);
58 else
59 #endif
60 WRITE_ONCE(*io->mmio, io->mmio_gpa + i);
61 }
62 GUEST_SYNC(0);
63
64 WRITE_ONCE(*io->mmio, io->mmio_gpa + i);
65 #ifdef __x86_64__
66 outl(io->pio_port, io->pio_port + i);
67 #endif
68 }
69 }
70
71 static void vcpu_run_and_verify_io_exit(struct kvm_vcpu *vcpu,
72 struct kvm_coalesced_io *io,
73 uint32_t ring_start,
74 uint32_t expected_exit)
75 {
76 const bool want_pio = expected_exit == KVM_EXIT_IO;
77 struct kvm_coalesced_mmio_ring *ring = io->ring;
78 struct kvm_run *run = vcpu->run;
79 uint32_t pio_value;
80
81 WRITE_ONCE(ring->first, ring_start);
82 WRITE_ONCE(ring->last, ring_start);
83
84 vcpu_run(vcpu);
85
86 /*
87 * Annoyingly, reading PIO data is safe only for PIO exits, otherwise
88 * data_offset is garbage, e.g. an MMIO gpa.
89 */
90 if (run->exit_reason == KVM_EXIT_IO)
91 pio_value = *(uint32_t *)((void *)run + run->io.data_offset);
92 else
93 pio_value = 0;
94
95 TEST_ASSERT((!want_pio && (run->exit_reason == KVM_EXIT_MMIO && run->mmio.is_write &&
96 run->mmio.phys_addr == io->mmio_gpa && run->mmio.len == 8 &&
97 *(uint64_t *)run->mmio.data == io->mmio_gpa + io->ring_size - 1)) ||
98 (want_pio && (run->exit_reason == KVM_EXIT_IO && run->io.port == io->pio_port &&
99 run->io.direction == KVM_EXIT_IO_OUT && run->io.count == 1 &&
100 pio_value == io->pio_port + io->ring_size - 1)),
101 "For start = %u, expected exit on %u-byte %s write 0x%llx = %lx, got exit_reason = %u (%s)\n "
102 "(MMIO addr = 0x%llx, write = %u, len = %u, data = %lx)\n "
103 "(PIO port = 0x%x, write = %u, len = %u, count = %u, data = %x",
104 ring_start, want_pio ? 4 : 8, want_pio ? "PIO" : "MMIO",
105 want_pio ? (unsigned long long)io->pio_port : io->mmio_gpa,
106 (want_pio ? io->pio_port : io->mmio_gpa) + io->ring_size - 1, run->exit_reason,
107 run->exit_reason == KVM_EXIT_MMIO ? "MMIO" : run->exit_reason == KVM_EXIT_IO ? "PIO" : "other",
108 run->mmio.phys_addr, run->mmio.is_write, run->mmio.len, *(uint64_t *)run->mmio.data,
109 run->io.port, run->io.direction, run->io.size, run->io.count, pio_value);
110 }
111
112 static void vcpu_run_and_verify_coalesced_io(struct kvm_vcpu *vcpu,
113 struct kvm_coalesced_io *io,
114 uint32_t ring_start,
115 uint32_t expected_exit)
116 {
117 struct kvm_coalesced_mmio_ring *ring = io->ring;
118 int i;
119
120 vcpu_run_and_verify_io_exit(vcpu, io, ring_start, expected_exit);
121
122 TEST_ASSERT((ring->last + 1) % io->ring_size == ring->first,
123 "Expected ring to be full (minus 1), first = %u, last = %u, max = %u, start = %u",
124 ring->first, ring->last, io->ring_size, ring_start);
125
126 for (i = 0; i < io->ring_size - 1; i++) {
127 uint32_t idx = (ring->first + i) % io->ring_size;
128 struct kvm_coalesced_mmio *entry = &ring->coalesced_mmio[idx];
129
130 #ifdef __x86_64__
131 if (i & 1)
132 TEST_ASSERT(entry->phys_addr == io->pio_port &&
133 entry->len == 4 && entry->pio &&
134 *(uint32_t *)entry->data == io->pio_port + i,
135 "Wanted 4-byte port I/O 0x%x = 0x%x in entry %u, got %u-byte %s 0x%llx = 0x%x",
136 io->pio_port, io->pio_port + i, i,
137 entry->len, entry->pio ? "PIO" : "MMIO",
138 entry->phys_addr, *(uint32_t *)entry->data);
139 else
140 #endif
141 TEST_ASSERT(entry->phys_addr == io->mmio_gpa &&
142 entry->len == 8 && !entry->pio,
143 "Wanted 8-byte MMIO to 0x%lx = %lx in entry %u, got %u-byte %s 0x%llx = 0x%lx",
144 io->mmio_gpa, io->mmio_gpa + i, i,
145 entry->len, entry->pio ? "PIO" : "MMIO",
146 entry->phys_addr, *(uint64_t *)entry->data);
147 }
148 }
149
150 static void test_coalesced_io(struct kvm_vcpu *vcpu,
151 struct kvm_coalesced_io *io, uint32_t ring_start)
152 {
153 struct kvm_coalesced_mmio_ring *ring = io->ring;
154
155 kvm_vm_register_coalesced_io(vcpu->vm, io->mmio_gpa, 8, false /* pio */);
156 #ifdef __x86_64__
157 kvm_vm_register_coalesced_io(vcpu->vm, io->pio_port, 8, true /* pio */);
158 #endif
159
160 vcpu_run_and_verify_coalesced_io(vcpu, io, ring_start, KVM_EXIT_MMIO);
161 #ifdef __x86_64__
162 vcpu_run_and_verify_coalesced_io(vcpu, io, ring_start, KVM_EXIT_IO);
163 #endif
164
165 /*
166 * Verify ucall, which may use non-coalesced MMIO or PIO, generates an
167 * immediate exit.
168 */
169 WRITE_ONCE(ring->first, ring_start);
170 WRITE_ONCE(ring->last, ring_start);
171 vcpu_run(vcpu);
172 TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC);
173 TEST_ASSERT_EQ(ring->first, ring_start);
174 TEST_ASSERT_EQ(ring->last, ring_start);
175
176 /* Verify that non-coalesced MMIO/PIO generates an exit to userspace. */
177 kvm_vm_unregister_coalesced_io(vcpu->vm, io->mmio_gpa, 8, false /* pio */);
178 vcpu_run_and_verify_io_exit(vcpu, io, ring_start, KVM_EXIT_MMIO);
179
180 #ifdef __x86_64__
181 kvm_vm_unregister_coalesced_io(vcpu->vm, io->pio_port, 8, true /* pio */);
182 vcpu_run_and_verify_io_exit(vcpu, io, ring_start, KVM_EXIT_IO);
183 #endif
184 }
185
186 int main(int argc, char *argv[])
187 {
188 struct kvm_vcpu *vcpu;
189 struct kvm_vm *vm;
190 int i;
191
192 TEST_REQUIRE(kvm_has_cap(KVM_CAP_COALESCED_MMIO));
193
194 #ifdef __x86_64__
195 TEST_REQUIRE(kvm_has_cap(KVM_CAP_COALESCED_PIO));
196 #endif
197
198 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
199
200 kvm_builtin_io_ring = (struct kvm_coalesced_io) {
201 /*
202 * The I/O ring is a kernel-allocated page whose address is
203 * relative to each vCPU's run page, with the page offset
204 * provided by KVM in the return of KVM_CAP_COALESCED_MMIO.
205 */
206 .ring = (void *)vcpu->run +
207 (kvm_check_cap(KVM_CAP_COALESCED_MMIO) * getpagesize()),
208
209 /*
210 * The size of the I/O ring is fixed, but KVM defines the sized
211 * based on the kernel's PAGE_SIZE. Thus, userspace must query
212 * the host's page size at runtime to compute the ring size.
213 */
214 .ring_size = (getpagesize() - sizeof(struct kvm_coalesced_mmio_ring)) /
215 sizeof(struct kvm_coalesced_mmio),
216
217 /*
218 * Arbitrary address+port (MMIO mustn't overlap memslots), with
219 * the MMIO GPA identity mapped in the guest.
220 */
221 .mmio_gpa = 4ull * SZ_1G,
222 .mmio = (uint64_t *)(4ull * SZ_1G),
223 .pio_port = 0x80,
224 };
225
226 virt_map(vm, (uint64_t)kvm_builtin_io_ring.mmio, kvm_builtin_io_ring.mmio_gpa, 1);
227
228 sync_global_to_guest(vm, kvm_builtin_io_ring);
229 vcpu_args_set(vcpu, 1, &kvm_builtin_io_ring);
230
231 for (i = 0; i < kvm_builtin_io_ring.ring_size; i++)
232 test_coalesced_io(vcpu, &kvm_builtin_io_ring, i);
233
234 kvm_vm_free(vm);
235 return 0;
236 }
237
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.