1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * GUP long-term page pinning tests.
4 *
5 * Copyright 2023, Red Hat, Inc.
6 *
7 * Author(s): David Hildenbrand <david@redhat.com>
8 */
9 #define _GNU_SOURCE
10 #include <stdlib.h>
11 #include <string.h>
12 #include <stdbool.h>
13 #include <stdint.h>
14 #include <unistd.h>
15 #include <errno.h>
16 #include <fcntl.h>
17 #include <assert.h>
18 #include <sys/mman.h>
19 #include <sys/ioctl.h>
20 #include <sys/vfs.h>
21 #include <linux/magic.h>
22 #include <linux/memfd.h>
23
24 #include "local_config.h"
25 #ifdef LOCAL_CONFIG_HAVE_LIBURING
26 #include <liburing.h>
27 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
28
29 #include "../../../../mm/gup_test.h"
30 #include "../kselftest.h"
31 #include "vm_util.h"
32
33 static size_t pagesize;
34 static int nr_hugetlbsizes;
35 static size_t hugetlbsizes[10];
36 static int gup_fd;
37
38 static __fsword_t get_fs_type(int fd)
39 {
40 struct statfs fs;
41 int ret;
42
43 do {
44 ret = fstatfs(fd, &fs);
45 } while (ret && errno == EINTR);
46
47 return ret ? 0 : fs.f_type;
48 }
49
50 static bool fs_is_unknown(__fsword_t fs_type)
51 {
52 /*
53 * We only support some filesystems in our tests when dealing with
54 * R/W long-term pinning. For these filesystems, we can be fairly sure
55 * whether they support it or not.
56 */
57 switch (fs_type) {
58 case TMPFS_MAGIC:
59 case HUGETLBFS_MAGIC:
60 case BTRFS_SUPER_MAGIC:
61 case EXT4_SUPER_MAGIC:
62 case XFS_SUPER_MAGIC:
63 return false;
64 default:
65 return true;
66 }
67 }
68
69 static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type)
70 {
71 assert(!fs_is_unknown(fs_type));
72 switch (fs_type) {
73 case TMPFS_MAGIC:
74 case HUGETLBFS_MAGIC:
75 return true;
76 default:
77 return false;
78 }
79 }
80
81 enum test_type {
82 TEST_TYPE_RO,
83 TEST_TYPE_RO_FAST,
84 TEST_TYPE_RW,
85 TEST_TYPE_RW_FAST,
86 #ifdef LOCAL_CONFIG_HAVE_LIBURING
87 TEST_TYPE_IOURING,
88 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
89 };
90
91 static void do_test(int fd, size_t size, enum test_type type, bool shared)
92 {
93 __fsword_t fs_type = get_fs_type(fd);
94 bool should_work;
95 char *mem;
96 int ret;
97
98 if (ftruncate(fd, size)) {
99 ksft_test_result_fail("ftruncate() failed\n");
100 return;
101 }
102
103 if (fallocate(fd, 0, 0, size)) {
104 if (size == pagesize)
105 ksft_test_result_fail("fallocate() failed\n");
106 else
107 ksft_test_result_skip("need more free huge pages\n");
108 return;
109 }
110
111 mem = mmap(NULL, size, PROT_READ | PROT_WRITE,
112 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0);
113 if (mem == MAP_FAILED) {
114 if (size == pagesize || shared)
115 ksft_test_result_fail("mmap() failed\n");
116 else
117 ksft_test_result_skip("need more free huge pages\n");
118 return;
119 }
120
121 /* Fault in the page such that GUP-fast can pin it directly. */
122 memset(mem, 0, size);
123
124 switch (type) {
125 case TEST_TYPE_RO:
126 case TEST_TYPE_RO_FAST:
127 /*
128 * Cover more cases regarding unsharing decisions when
129 * long-term R/O pinning by mapping the page R/O.
130 */
131 ret = mprotect(mem, size, PROT_READ);
132 if (ret) {
133 ksft_test_result_fail("mprotect() failed\n");
134 goto munmap;
135 }
136 /* FALLTHROUGH */
137 case TEST_TYPE_RW:
138 case TEST_TYPE_RW_FAST: {
139 struct pin_longterm_test args;
140 const bool fast = type == TEST_TYPE_RO_FAST ||
141 type == TEST_TYPE_RW_FAST;
142 const bool rw = type == TEST_TYPE_RW ||
143 type == TEST_TYPE_RW_FAST;
144
145 if (gup_fd < 0) {
146 ksft_test_result_skip("gup_test not available\n");
147 break;
148 }
149
150 if (rw && shared && fs_is_unknown(fs_type)) {
151 ksft_test_result_skip("Unknown filesystem\n");
152 return;
153 }
154 /*
155 * R/O pinning or pinning in a private mapping is always
156 * expected to work. Otherwise, we expect long-term R/W pinning
157 * to only succeed for special fielesystems.
158 */
159 should_work = !shared || !rw ||
160 fs_supports_writable_longterm_pinning(fs_type);
161
162 args.addr = (__u64)(uintptr_t)mem;
163 args.size = size;
164 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0;
165 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0;
166 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args);
167 if (ret && errno == EINVAL) {
168 ksft_test_result_skip("PIN_LONGTERM_TEST_START failed\n");
169 break;
170 } else if (ret && errno == EFAULT) {
171 ksft_test_result(!should_work, "Should have failed\n");
172 break;
173 } else if (ret) {
174 ksft_test_result_fail("PIN_LONGTERM_TEST_START failed\n");
175 break;
176 }
177
178 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP))
179 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed\n");
180
181 /*
182 * TODO: if the kernel ever supports long-term R/W pinning on
183 * some previously unsupported filesystems, we might want to
184 * perform some additional tests for possible data corruptions.
185 */
186 ksft_test_result(should_work, "Should have worked\n");
187 break;
188 }
189 #ifdef LOCAL_CONFIG_HAVE_LIBURING
190 case TEST_TYPE_IOURING: {
191 struct io_uring ring;
192 struct iovec iov;
193
194 /* io_uring always pins pages writable. */
195 if (shared && fs_is_unknown(fs_type)) {
196 ksft_test_result_skip("Unknown filesystem\n");
197 return;
198 }
199 should_work = !shared ||
200 fs_supports_writable_longterm_pinning(fs_type);
201
202 /* Skip on errors, as we might just lack kernel support. */
203 ret = io_uring_queue_init(1, &ring, 0);
204 if (ret < 0) {
205 ksft_test_result_skip("io_uring_queue_init() failed\n");
206 break;
207 }
208 /*
209 * Register the range as a fixed buffer. This will FOLL_WRITE |
210 * FOLL_PIN | FOLL_LONGTERM the range.
211 */
212 iov.iov_base = mem;
213 iov.iov_len = size;
214 ret = io_uring_register_buffers(&ring, &iov, 1);
215 /* Only new kernels return EFAULT. */
216 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP ||
217 errno == EFAULT)) {
218 ksft_test_result(!should_work, "Should have failed\n");
219 } else if (ret) {
220 /*
221 * We might just lack support or have insufficient
222 * MEMLOCK limits.
223 */
224 ksft_test_result_skip("io_uring_register_buffers() failed\n");
225 } else {
226 ksft_test_result(should_work, "Should have worked\n");
227 io_uring_unregister_buffers(&ring);
228 }
229
230 io_uring_queue_exit(&ring);
231 break;
232 }
233 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
234 default:
235 assert(false);
236 }
237
238 munmap:
239 munmap(mem, size);
240 }
241
242 typedef void (*test_fn)(int fd, size_t size);
243
244 static void run_with_memfd(test_fn fn, const char *desc)
245 {
246 int fd;
247
248 ksft_print_msg("[RUN] %s ... with memfd\n", desc);
249
250 fd = memfd_create("test", 0);
251 if (fd < 0) {
252 ksft_test_result_fail("memfd_create() failed\n");
253 return;
254 }
255
256 fn(fd, pagesize);
257 close(fd);
258 }
259
260 static void run_with_tmpfile(test_fn fn, const char *desc)
261 {
262 FILE *file;
263 int fd;
264
265 ksft_print_msg("[RUN] %s ... with tmpfile\n", desc);
266
267 file = tmpfile();
268 if (!file) {
269 ksft_test_result_fail("tmpfile() failed\n");
270 return;
271 }
272
273 fd = fileno(file);
274 if (fd < 0) {
275 ksft_test_result_fail("fileno() failed\n");
276 goto close;
277 }
278
279 fn(fd, pagesize);
280 close:
281 fclose(file);
282 }
283
284 static void run_with_local_tmpfile(test_fn fn, const char *desc)
285 {
286 char filename[] = __FILE__"_tmpfile_XXXXXX";
287 int fd;
288
289 ksft_print_msg("[RUN] %s ... with local tmpfile\n", desc);
290
291 fd = mkstemp(filename);
292 if (fd < 0) {
293 ksft_test_result_fail("mkstemp() failed\n");
294 return;
295 }
296
297 if (unlink(filename)) {
298 ksft_test_result_fail("unlink() failed\n");
299 goto close;
300 }
301
302 fn(fd, pagesize);
303 close:
304 close(fd);
305 }
306
307 static void run_with_memfd_hugetlb(test_fn fn, const char *desc,
308 size_t hugetlbsize)
309 {
310 int flags = MFD_HUGETLB;
311 int fd;
312
313 ksft_print_msg("[RUN] %s ... with memfd hugetlb (%zu kB)\n", desc,
314 hugetlbsize / 1024);
315
316 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT;
317
318 fd = memfd_create("test", flags);
319 if (fd < 0) {
320 ksft_test_result_skip("memfd_create() failed\n");
321 return;
322 }
323
324 fn(fd, hugetlbsize);
325 close(fd);
326 }
327
328 struct test_case {
329 const char *desc;
330 test_fn fn;
331 };
332
333 static void test_shared_rw_pin(int fd, size_t size)
334 {
335 do_test(fd, size, TEST_TYPE_RW, true);
336 }
337
338 static void test_shared_rw_fast_pin(int fd, size_t size)
339 {
340 do_test(fd, size, TEST_TYPE_RW_FAST, true);
341 }
342
343 static void test_shared_ro_pin(int fd, size_t size)
344 {
345 do_test(fd, size, TEST_TYPE_RO, true);
346 }
347
348 static void test_shared_ro_fast_pin(int fd, size_t size)
349 {
350 do_test(fd, size, TEST_TYPE_RO_FAST, true);
351 }
352
353 static void test_private_rw_pin(int fd, size_t size)
354 {
355 do_test(fd, size, TEST_TYPE_RW, false);
356 }
357
358 static void test_private_rw_fast_pin(int fd, size_t size)
359 {
360 do_test(fd, size, TEST_TYPE_RW_FAST, false);
361 }
362
363 static void test_private_ro_pin(int fd, size_t size)
364 {
365 do_test(fd, size, TEST_TYPE_RO, false);
366 }
367
368 static void test_private_ro_fast_pin(int fd, size_t size)
369 {
370 do_test(fd, size, TEST_TYPE_RO_FAST, false);
371 }
372
373 #ifdef LOCAL_CONFIG_HAVE_LIBURING
374 static void test_shared_iouring(int fd, size_t size)
375 {
376 do_test(fd, size, TEST_TYPE_IOURING, true);
377 }
378
379 static void test_private_iouring(int fd, size_t size)
380 {
381 do_test(fd, size, TEST_TYPE_IOURING, false);
382 }
383 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
384
385 static const struct test_case test_cases[] = {
386 {
387 "R/W longterm GUP pin in MAP_SHARED file mapping",
388 test_shared_rw_pin,
389 },
390 {
391 "R/W longterm GUP-fast pin in MAP_SHARED file mapping",
392 test_shared_rw_fast_pin,
393 },
394 {
395 "R/O longterm GUP pin in MAP_SHARED file mapping",
396 test_shared_ro_pin,
397 },
398 {
399 "R/O longterm GUP-fast pin in MAP_SHARED file mapping",
400 test_shared_ro_fast_pin,
401 },
402 {
403 "R/W longterm GUP pin in MAP_PRIVATE file mapping",
404 test_private_rw_pin,
405 },
406 {
407 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping",
408 test_private_rw_fast_pin,
409 },
410 {
411 "R/O longterm GUP pin in MAP_PRIVATE file mapping",
412 test_private_ro_pin,
413 },
414 {
415 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping",
416 test_private_ro_fast_pin,
417 },
418 #ifdef LOCAL_CONFIG_HAVE_LIBURING
419 {
420 "io_uring fixed buffer with MAP_SHARED file mapping",
421 test_shared_iouring,
422 },
423 {
424 "io_uring fixed buffer with MAP_PRIVATE file mapping",
425 test_private_iouring,
426 },
427 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
428 };
429
430 static void run_test_case(struct test_case const *test_case)
431 {
432 int i;
433
434 run_with_memfd(test_case->fn, test_case->desc);
435 run_with_tmpfile(test_case->fn, test_case->desc);
436 run_with_local_tmpfile(test_case->fn, test_case->desc);
437 for (i = 0; i < nr_hugetlbsizes; i++)
438 run_with_memfd_hugetlb(test_case->fn, test_case->desc,
439 hugetlbsizes[i]);
440 }
441
442 static int tests_per_test_case(void)
443 {
444 return 3 + nr_hugetlbsizes;
445 }
446
447 int main(int argc, char **argv)
448 {
449 int i, err;
450
451 pagesize = getpagesize();
452 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes,
453 ARRAY_SIZE(hugetlbsizes));
454
455 ksft_print_header();
456 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case());
457
458 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
459
460 for (i = 0; i < ARRAY_SIZE(test_cases); i++)
461 run_test_case(&test_cases[i]);
462
463 err = ksft_get_fail_cnt();
464 if (err)
465 ksft_exit_fail_msg("%d out of %d tests failed\n",
466 err, ksft_test_num());
467 ksft_exit_pass();
468 }
469
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.