1 // SPDX-License-Identifier: GPL-2.0
2 #include <fcntl.h>
3 #include <limits.h>
4 #include <stdio.h>
5 #include <stdlib.h>
6 #include <string.h>
7 #include <unistd.h>
8
9 #include "thp_settings.h"
10
11 #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/"
12 #define MAX_SETTINGS_DEPTH 4
13 static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH];
14 static int settings_index;
15 static struct thp_settings saved_settings;
16 static char dev_queue_read_ahead_path[PATH_MAX];
17
18 static const char * const thp_enabled_strings[] = {
19 "never",
20 "always",
21 "inherit",
22 "madvise",
23 NULL
24 };
25
26 static const char * const thp_defrag_strings[] = {
27 "always",
28 "defer",
29 "defer+madvise",
30 "madvise",
31 "never",
32 NULL
33 };
34
35 static const char * const shmem_enabled_strings[] = {
36 "always",
37 "within_size",
38 "advise",
39 "never",
40 "deny",
41 "force",
42 NULL
43 };
44
45 int read_file(const char *path, char *buf, size_t buflen)
46 {
47 int fd;
48 ssize_t numread;
49
50 fd = open(path, O_RDONLY);
51 if (fd == -1)
52 return 0;
53
54 numread = read(fd, buf, buflen - 1);
55 if (numread < 1) {
56 close(fd);
57 return 0;
58 }
59
60 buf[numread] = '\0';
61 close(fd);
62
63 return (unsigned int) numread;
64 }
65
66 int write_file(const char *path, const char *buf, size_t buflen)
67 {
68 int fd;
69 ssize_t numwritten;
70
71 fd = open(path, O_WRONLY);
72 if (fd == -1) {
73 printf("open(%s)\n", path);
74 exit(EXIT_FAILURE);
75 return 0;
76 }
77
78 numwritten = write(fd, buf, buflen - 1);
79 close(fd);
80 if (numwritten < 1) {
81 printf("write(%s)\n", buf);
82 exit(EXIT_FAILURE);
83 return 0;
84 }
85
86 return (unsigned int) numwritten;
87 }
88
89 const unsigned long read_num(const char *path)
90 {
91 char buf[21];
92
93 if (read_file(path, buf, sizeof(buf)) < 0) {
94 perror("read_file()");
95 exit(EXIT_FAILURE);
96 }
97
98 return strtoul(buf, NULL, 10);
99 }
100
101 void write_num(const char *path, unsigned long num)
102 {
103 char buf[21];
104
105 sprintf(buf, "%ld", num);
106 if (!write_file(path, buf, strlen(buf) + 1)) {
107 perror(path);
108 exit(EXIT_FAILURE);
109 }
110 }
111
112 int thp_read_string(const char *name, const char * const strings[])
113 {
114 char path[PATH_MAX];
115 char buf[256];
116 char *c;
117 int ret;
118
119 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
120 if (ret >= PATH_MAX) {
121 printf("%s: Pathname is too long\n", __func__);
122 exit(EXIT_FAILURE);
123 }
124
125 if (!read_file(path, buf, sizeof(buf))) {
126 perror(path);
127 exit(EXIT_FAILURE);
128 }
129
130 c = strchr(buf, '[');
131 if (!c) {
132 printf("%s: Parse failure\n", __func__);
133 exit(EXIT_FAILURE);
134 }
135
136 c++;
137 memmove(buf, c, sizeof(buf) - (c - buf));
138
139 c = strchr(buf, ']');
140 if (!c) {
141 printf("%s: Parse failure\n", __func__);
142 exit(EXIT_FAILURE);
143 }
144 *c = '\0';
145
146 ret = 0;
147 while (strings[ret]) {
148 if (!strcmp(strings[ret], buf))
149 return ret;
150 ret++;
151 }
152
153 printf("Failed to parse %s\n", name);
154 exit(EXIT_FAILURE);
155 }
156
157 void thp_write_string(const char *name, const char *val)
158 {
159 char path[PATH_MAX];
160 int ret;
161
162 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
163 if (ret >= PATH_MAX) {
164 printf("%s: Pathname is too long\n", __func__);
165 exit(EXIT_FAILURE);
166 }
167
168 if (!write_file(path, val, strlen(val) + 1)) {
169 perror(path);
170 exit(EXIT_FAILURE);
171 }
172 }
173
174 const unsigned long thp_read_num(const char *name)
175 {
176 char path[PATH_MAX];
177 int ret;
178
179 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
180 if (ret >= PATH_MAX) {
181 printf("%s: Pathname is too long\n", __func__);
182 exit(EXIT_FAILURE);
183 }
184 return read_num(path);
185 }
186
187 void thp_write_num(const char *name, unsigned long num)
188 {
189 char path[PATH_MAX];
190 int ret;
191
192 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
193 if (ret >= PATH_MAX) {
194 printf("%s: Pathname is too long\n", __func__);
195 exit(EXIT_FAILURE);
196 }
197 write_num(path, num);
198 }
199
200 void thp_read_settings(struct thp_settings *settings)
201 {
202 unsigned long orders = thp_supported_orders();
203 char path[PATH_MAX];
204 int i;
205
206 *settings = (struct thp_settings) {
207 .thp_enabled = thp_read_string("enabled", thp_enabled_strings),
208 .thp_defrag = thp_read_string("defrag", thp_defrag_strings),
209 .shmem_enabled =
210 thp_read_string("shmem_enabled", shmem_enabled_strings),
211 .use_zero_page = thp_read_num("use_zero_page"),
212 };
213 settings->khugepaged = (struct khugepaged_settings) {
214 .defrag = thp_read_num("khugepaged/defrag"),
215 .alloc_sleep_millisecs =
216 thp_read_num("khugepaged/alloc_sleep_millisecs"),
217 .scan_sleep_millisecs =
218 thp_read_num("khugepaged/scan_sleep_millisecs"),
219 .max_ptes_none = thp_read_num("khugepaged/max_ptes_none"),
220 .max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"),
221 .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"),
222 .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"),
223 };
224 if (dev_queue_read_ahead_path[0])
225 settings->read_ahead_kb = read_num(dev_queue_read_ahead_path);
226
227 for (i = 0; i < NR_ORDERS; i++) {
228 if (!((1 << i) & orders)) {
229 settings->hugepages[i].enabled = THP_NEVER;
230 continue;
231 }
232 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",
233 (getpagesize() >> 10) << i);
234 settings->hugepages[i].enabled =
235 thp_read_string(path, thp_enabled_strings);
236 }
237 }
238
239 void thp_write_settings(struct thp_settings *settings)
240 {
241 struct khugepaged_settings *khugepaged = &settings->khugepaged;
242 unsigned long orders = thp_supported_orders();
243 char path[PATH_MAX];
244 int enabled;
245 int i;
246
247 thp_write_string("enabled", thp_enabled_strings[settings->thp_enabled]);
248 thp_write_string("defrag", thp_defrag_strings[settings->thp_defrag]);
249 thp_write_string("shmem_enabled",
250 shmem_enabled_strings[settings->shmem_enabled]);
251 thp_write_num("use_zero_page", settings->use_zero_page);
252
253 thp_write_num("khugepaged/defrag", khugepaged->defrag);
254 thp_write_num("khugepaged/alloc_sleep_millisecs",
255 khugepaged->alloc_sleep_millisecs);
256 thp_write_num("khugepaged/scan_sleep_millisecs",
257 khugepaged->scan_sleep_millisecs);
258 thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none);
259 thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap);
260 thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared);
261 thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan);
262
263 if (dev_queue_read_ahead_path[0])
264 write_num(dev_queue_read_ahead_path, settings->read_ahead_kb);
265
266 for (i = 0; i < NR_ORDERS; i++) {
267 if (!((1 << i) & orders))
268 continue;
269 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",
270 (getpagesize() >> 10) << i);
271 enabled = settings->hugepages[i].enabled;
272 thp_write_string(path, thp_enabled_strings[enabled]);
273 }
274 }
275
276 struct thp_settings *thp_current_settings(void)
277 {
278 if (!settings_index) {
279 printf("Fail: No settings set");
280 exit(EXIT_FAILURE);
281 }
282 return settings_stack + settings_index - 1;
283 }
284
285 void thp_push_settings(struct thp_settings *settings)
286 {
287 if (settings_index >= MAX_SETTINGS_DEPTH) {
288 printf("Fail: Settings stack exceeded");
289 exit(EXIT_FAILURE);
290 }
291 settings_stack[settings_index++] = *settings;
292 thp_write_settings(thp_current_settings());
293 }
294
295 void thp_pop_settings(void)
296 {
297 if (settings_index <= 0) {
298 printf("Fail: Settings stack empty");
299 exit(EXIT_FAILURE);
300 }
301 --settings_index;
302 thp_write_settings(thp_current_settings());
303 }
304
305 void thp_restore_settings(void)
306 {
307 thp_write_settings(&saved_settings);
308 }
309
310 void thp_save_settings(void)
311 {
312 thp_read_settings(&saved_settings);
313 }
314
315 void thp_set_read_ahead_path(char *path)
316 {
317 if (!path) {
318 dev_queue_read_ahead_path[0] = '\0';
319 return;
320 }
321
322 strncpy(dev_queue_read_ahead_path, path,
323 sizeof(dev_queue_read_ahead_path));
324 dev_queue_read_ahead_path[sizeof(dev_queue_read_ahead_path) - 1] = '\0';
325 }
326
327 unsigned long thp_supported_orders(void)
328 {
329 unsigned long orders = 0;
330 char path[PATH_MAX];
331 char buf[256];
332 int ret;
333 int i;
334
335 for (i = 0; i < NR_ORDERS; i++) {
336 ret = snprintf(path, PATH_MAX, THP_SYSFS "hugepages-%ukB/enabled",
337 (getpagesize() >> 10) << i);
338 if (ret >= PATH_MAX) {
339 printf("%s: Pathname is too long\n", __func__);
340 exit(EXIT_FAILURE);
341 }
342
343 ret = read_file(path, buf, sizeof(buf));
344 if (ret)
345 orders |= 1UL << i;
346 }
347
348 return orders;
349 }
350
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.