1 // SPDX-License-Identifier: GPL-2.0
2 #include <string.h>
3 #include <fcntl.h>
4 #include <dirent.h>
5 #include <sys/ioctl.h>
6 #include <linux/userfaultfd.h>
7 #include <linux/fs.h>
8 #include <sys/syscall.h>
9 #include <unistd.h>
10 #include "../kselftest.h"
11 #include "vm_util.h"
12
13 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
14 #define SMAP_FILE_PATH "/proc/self/smaps"
15 #define MAX_LINE_LENGTH 500
16
17 unsigned int __page_size;
18 unsigned int __page_shift;
19
20 uint64_t pagemap_get_entry(int fd, char *start)
21 {
22 const unsigned long pfn = (unsigned long)start / getpagesize();
23 uint64_t entry;
24 int ret;
25
26 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
27 if (ret != sizeof(entry))
28 ksft_exit_fail_msg("reading pagemap failed\n");
29 return entry;
30 }
31
32 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
33 {
34 struct pm_scan_arg arg;
35
36 arg.start = (uintptr_t)start;
37 arg.end = (uintptr_t)(start + psize());
38 arg.vec = (uintptr_t)r;
39 arg.vec_len = 1;
40 arg.flags = 0;
41 arg.size = sizeof(struct pm_scan_arg);
42 arg.max_pages = 0;
43 arg.category_inverted = 0;
44 arg.category_mask = 0;
45 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
46 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
47 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
48 arg.return_mask = arg.category_anyof_mask;
49
50 return ioctl(fd, PAGEMAP_SCAN, &arg);
51 }
52
53 static uint64_t pagemap_scan_get_categories(int fd, char *start)
54 {
55 struct page_region r;
56 long ret;
57
58 ret = __pagemap_scan_get_categories(fd, start, &r);
59 if (ret < 0)
60 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
61 if (ret == 0)
62 return 0;
63 return r.categories;
64 }
65
66 /* `start` is any valid address. */
67 static bool pagemap_scan_supported(int fd, char *start)
68 {
69 static int supported = -1;
70 int ret;
71
72 if (supported != -1)
73 return supported;
74
75 /* Provide an invalid address in order to trigger EFAULT. */
76 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
77 if (ret == 0)
78 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
79
80 supported = errno == EFAULT;
81
82 return supported;
83 }
84
85 static bool page_entry_is(int fd, char *start, char *desc,
86 uint64_t pagemap_flags, uint64_t pagescan_flags)
87 {
88 bool m = pagemap_get_entry(fd, start) & pagemap_flags;
89
90 if (pagemap_scan_supported(fd, start)) {
91 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
92
93 if (m == s)
94 return m;
95
96 ksft_exit_fail_msg(
97 "read and ioctl return unmatched results for %s: %d %d", desc, m, s);
98 }
99 return m;
100 }
101
102 bool pagemap_is_softdirty(int fd, char *start)
103 {
104 return page_entry_is(fd, start, "soft-dirty",
105 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
106 }
107
108 bool pagemap_is_swapped(int fd, char *start)
109 {
110 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
111 }
112
113 bool pagemap_is_populated(int fd, char *start)
114 {
115 return page_entry_is(fd, start, "populated",
116 PM_PRESENT | PM_SWAP,
117 PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
118 }
119
120 unsigned long pagemap_get_pfn(int fd, char *start)
121 {
122 uint64_t entry = pagemap_get_entry(fd, start);
123
124 /* If present (63th bit), PFN is at bit 0 -- 54. */
125 if (entry & PM_PRESENT)
126 return entry & 0x007fffffffffffffull;
127 return -1ul;
128 }
129
130 void clear_softdirty(void)
131 {
132 int ret;
133 const char *ctrl = "4";
134 int fd = open("/proc/self/clear_refs", O_WRONLY);
135
136 if (fd < 0)
137 ksft_exit_fail_msg("opening clear_refs failed\n");
138 ret = write(fd, ctrl, strlen(ctrl));
139 close(fd);
140 if (ret != strlen(ctrl))
141 ksft_exit_fail_msg("writing clear_refs failed\n");
142 }
143
144 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
145 {
146 while (fgets(buf, len, fp)) {
147 if (!strncmp(buf, pattern, strlen(pattern)))
148 return true;
149 }
150 return false;
151 }
152
153 uint64_t read_pmd_pagesize(void)
154 {
155 int fd;
156 char buf[20];
157 ssize_t num_read;
158
159 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
160 if (fd == -1)
161 return 0;
162
163 num_read = read(fd, buf, 19);
164 if (num_read < 1) {
165 close(fd);
166 return 0;
167 }
168 buf[num_read] = '\0';
169 close(fd);
170
171 return strtoul(buf, NULL, 10);
172 }
173
174 bool __check_huge(void *addr, char *pattern, int nr_hpages,
175 uint64_t hpage_size)
176 {
177 uint64_t thp = -1;
178 int ret;
179 FILE *fp;
180 char buffer[MAX_LINE_LENGTH];
181 char addr_pattern[MAX_LINE_LENGTH];
182
183 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
184 (unsigned long) addr);
185 if (ret >= MAX_LINE_LENGTH)
186 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
187
188 fp = fopen(SMAP_FILE_PATH, "r");
189 if (!fp)
190 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
191
192 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
193 goto err_out;
194
195 /*
196 * Fetch the pattern in the same block and check the number of
197 * hugepages.
198 */
199 if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
200 goto err_out;
201
202 snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
203
204 if (sscanf(buffer, addr_pattern, &thp) != 1)
205 ksft_exit_fail_msg("Reading smap error\n");
206
207 err_out:
208 fclose(fp);
209 return thp == (nr_hpages * (hpage_size >> 10));
210 }
211
212 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
213 {
214 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
215 }
216
217 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
218 {
219 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
220 }
221
222 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
223 {
224 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
225 }
226
227 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
228 {
229 uint64_t ent[2];
230
231 /* drop pmd */
232 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
233 MAP_FIXED | MAP_ANONYMOUS |
234 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
235 ksft_exit_fail_msg("mmap transhuge\n");
236
237 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
238 ksft_exit_fail_msg("MADV_HUGEPAGE\n");
239
240 /* allocate transparent huge page */
241 *(volatile void **)ptr = ptr;
242
243 if (pread(pagemap_fd, ent, sizeof(ent),
244 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
245 ksft_exit_fail_msg("read pagemap\n");
246
247 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
248 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
249 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
250 return PAGEMAP_PFN(ent[0]);
251
252 return -1;
253 }
254
255 unsigned long default_huge_page_size(void)
256 {
257 unsigned long hps = 0;
258 char *line = NULL;
259 size_t linelen = 0;
260 FILE *f = fopen("/proc/meminfo", "r");
261
262 if (!f)
263 return 0;
264 while (getline(&line, &linelen, f) > 0) {
265 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
266 hps <<= 10;
267 break;
268 }
269 }
270
271 free(line);
272 fclose(f);
273 return hps;
274 }
275
276 int detect_hugetlb_page_sizes(size_t sizes[], int max)
277 {
278 DIR *dir = opendir("/sys/kernel/mm/hugepages/");
279 int count = 0;
280
281 if (!dir)
282 return 0;
283
284 while (count < max) {
285 struct dirent *entry = readdir(dir);
286 size_t kb;
287
288 if (!entry)
289 break;
290 if (entry->d_type != DT_DIR)
291 continue;
292 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
293 continue;
294 sizes[count++] = kb * 1024;
295 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
296 kb);
297 }
298 closedir(dir);
299 return count;
300 }
301
302 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
303 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
304 bool miss, bool wp, bool minor, uint64_t *ioctls)
305 {
306 struct uffdio_register uffdio_register = { 0 };
307 uint64_t mode = 0;
308 int ret = 0;
309
310 if (miss)
311 mode |= UFFDIO_REGISTER_MODE_MISSING;
312 if (wp)
313 mode |= UFFDIO_REGISTER_MODE_WP;
314 if (minor)
315 mode |= UFFDIO_REGISTER_MODE_MINOR;
316
317 uffdio_register.range.start = (unsigned long)addr;
318 uffdio_register.range.len = len;
319 uffdio_register.mode = mode;
320
321 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
322 ret = -errno;
323 else if (ioctls)
324 *ioctls = uffdio_register.ioctls;
325
326 return ret;
327 }
328
329 int uffd_register(int uffd, void *addr, uint64_t len,
330 bool miss, bool wp, bool minor)
331 {
332 return uffd_register_with_ioctls(uffd, addr, len,
333 miss, wp, minor, NULL);
334 }
335
336 int uffd_unregister(int uffd, void *addr, uint64_t len)
337 {
338 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
339 int ret = 0;
340
341 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
342 ret = -errno;
343
344 return ret;
345 }
346
347 unsigned long get_free_hugepages(void)
348 {
349 unsigned long fhp = 0;
350 char *line = NULL;
351 size_t linelen = 0;
352 FILE *f = fopen("/proc/meminfo", "r");
353
354 if (!f)
355 return fhp;
356 while (getline(&line, &linelen, f) > 0) {
357 if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1)
358 break;
359 }
360
361 free(line);
362 fclose(f);
363 return fhp;
364 }
365
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.