1 /* 1 /*
2 * memfd_create system call and file sealing s 2 * memfd_create system call and file sealing support
3 * 3 *
4 * Code was originally included in shmem.c, an 4 * Code was originally included in shmem.c, and broken out to facilitate
5 * use by hugetlbfs as well as tmpfs. 5 * use by hugetlbfs as well as tmpfs.
6 * 6 *
7 * This file is released under the GPL. 7 * This file is released under the GPL.
8 */ 8 */
9 9
10 #include <linux/fs.h> 10 #include <linux/fs.h>
11 #include <linux/vfs.h> 11 #include <linux/vfs.h>
12 #include <linux/pagemap.h> 12 #include <linux/pagemap.h>
13 #include <linux/file.h> 13 #include <linux/file.h>
14 #include <linux/mm.h> 14 #include <linux/mm.h>
15 #include <linux/sched/signal.h> 15 #include <linux/sched/signal.h>
16 #include <linux/khugepaged.h> 16 #include <linux/khugepaged.h>
17 #include <linux/syscalls.h> 17 #include <linux/syscalls.h>
18 #include <linux/hugetlb.h> 18 #include <linux/hugetlb.h>
19 #include <linux/shmem_fs.h> 19 #include <linux/shmem_fs.h>
20 #include <linux/memfd.h> 20 #include <linux/memfd.h>
21 #include <linux/pid_namespace.h> <<
22 #include <uapi/linux/memfd.h> 21 #include <uapi/linux/memfd.h>
23 22
24 /* 23 /*
25 * We need a tag: a new tag would expand every 24 * We need a tag: a new tag would expand every xa_node by 8 bytes,
26 * so reuse a tag which we firmly believe is n 25 * so reuse a tag which we firmly believe is never set or cleared on tmpfs
27 * or hugetlbfs because they are memory only f 26 * or hugetlbfs because they are memory only filesystems.
28 */ 27 */
29 #define MEMFD_TAG_PINNED PAGECACHE_TAG_ 28 #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE
30 #define LAST_SCAN 4 /* abo 29 #define LAST_SCAN 4 /* about 150ms max */
31 30
32 static bool memfd_folio_has_extra_refs(struct <<
33 { <<
34 return folio_ref_count(folio) - folio_ <<
35 folio_nr_pages(folio); <<
36 } <<
37 <<
38 static void memfd_tag_pins(struct xa_state *xa 31 static void memfd_tag_pins(struct xa_state *xas)
39 { 32 {
40 struct folio *folio; !! 33 struct page *page;
41 int latency = 0; !! 34 unsigned int tagged = 0;
42 35
43 lru_add_drain(); 36 lru_add_drain();
44 37
45 xas_lock_irq(xas); 38 xas_lock_irq(xas);
46 xas_for_each(xas, folio, ULONG_MAX) { !! 39 xas_for_each(xas, page, ULONG_MAX) {
47 if (!xa_is_value(folio) && mem !! 40 if (xa_is_value(page))
>> 41 continue;
>> 42 if (page_count(page) - page_mapcount(page) > 1)
48 xas_set_mark(xas, MEMF 43 xas_set_mark(xas, MEMFD_TAG_PINNED);
49 44
50 if (++latency < XA_CHECK_SCHED !! 45 if (++tagged % XA_CHECK_SCHED)
51 continue; 46 continue;
52 latency = 0; <<
53 47
54 xas_pause(xas); 48 xas_pause(xas);
55 xas_unlock_irq(xas); 49 xas_unlock_irq(xas);
56 cond_resched(); 50 cond_resched();
57 xas_lock_irq(xas); 51 xas_lock_irq(xas);
58 } 52 }
59 xas_unlock_irq(xas); 53 xas_unlock_irq(xas);
60 } 54 }
61 55
62 /* 56 /*
63 * This is a helper function used by memfd_pin <<
64 * It is mainly called to allocate a folio in <<
65 * (memfd_pin_folios()) cannot find a folio in <<
66 * index in the mapping. <<
67 */ <<
68 struct folio *memfd_alloc_folio(struct file *m <<
69 { <<
70 #ifdef CONFIG_HUGETLB_PAGE <<
71 struct folio *folio; <<
72 gfp_t gfp_mask; <<
73 int err; <<
74 <<
75 if (is_file_hugepages(memfd)) { <<
76 /* <<
77 * The folio would most likely <<
78 * therefore, we have zone mem <<
79 * alloc from. Also, the folio <<
80 * amount of time, so it is no <<
81 */ <<
82 struct hstate *h = hstate_file <<
83 <<
84 gfp_mask = htlb_alloc_mask(h); <<
85 gfp_mask &= ~(__GFP_HIGHMEM | <<
86 idx >>= huge_page_order(h); <<
87 <<
88 folio = alloc_hugetlb_folio_re <<
89 <<
90 <<
91 <<
92 if (folio) { <<
93 err = hugetlb_add_to_p <<
94 <<
95 <<
96 if (err) { <<
97 folio_put(foli <<
98 return ERR_PTR <<
99 } <<
100 folio_unlock(folio); <<
101 return folio; <<
102 } <<
103 return ERR_PTR(-ENOMEM); <<
104 } <<
105 #endif <<
106 return shmem_read_folio(memfd->f_mappi <<
107 } <<
108 <<
109 /* <<
110 * Setting SEAL_WRITE requires us to verify th 57 * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
111 * via get_user_pages(), drivers might have so 58 * via get_user_pages(), drivers might have some pending I/O without any active
112 * user-space mappings (eg., direct-IO, AIO). !! 59 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
113 * and see whether it has an elevated ref-coun 60 * and see whether it has an elevated ref-count. If so, we tag them and wait for
114 * them to be dropped. 61 * them to be dropped.
115 * The caller must guarantee that no new user 62 * The caller must guarantee that no new user will acquire writable references
116 * to those folios to avoid races. !! 63 * to those pages to avoid races.
117 */ 64 */
118 static int memfd_wait_for_pins(struct address_ 65 static int memfd_wait_for_pins(struct address_space *mapping)
119 { 66 {
120 XA_STATE(xas, &mapping->i_pages, 0); 67 XA_STATE(xas, &mapping->i_pages, 0);
121 struct folio *folio; !! 68 struct page *page;
122 int error, scan; 69 int error, scan;
123 70
124 memfd_tag_pins(&xas); 71 memfd_tag_pins(&xas);
125 72
126 error = 0; 73 error = 0;
127 for (scan = 0; scan <= LAST_SCAN; scan 74 for (scan = 0; scan <= LAST_SCAN; scan++) {
128 int latency = 0; !! 75 unsigned int tagged = 0;
129 76
130 if (!xas_marked(&xas, MEMFD_TA 77 if (!xas_marked(&xas, MEMFD_TAG_PINNED))
131 break; 78 break;
132 79
133 if (!scan) 80 if (!scan)
134 lru_add_drain_all(); 81 lru_add_drain_all();
135 else if (schedule_timeout_kill 82 else if (schedule_timeout_killable((HZ << scan) / 200))
136 scan = LAST_SCAN; 83 scan = LAST_SCAN;
137 84
138 xas_set(&xas, 0); 85 xas_set(&xas, 0);
139 xas_lock_irq(&xas); 86 xas_lock_irq(&xas);
140 xas_for_each_marked(&xas, foli !! 87 xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
141 bool clear = true; 88 bool clear = true;
142 !! 89 if (xa_is_value(page))
143 if (!xa_is_value(folio !! 90 continue;
144 memfd_folio_has_ex !! 91 if (page_count(page) - page_mapcount(page) != 1) {
145 /* 92 /*
146 * On the last 93 * On the last scan, we clean up all those tags
147 * we inserted 94 * we inserted; but make a note that we still
148 * found folio !! 95 * found pages pinned.
149 */ 96 */
150 if (scan == LA 97 if (scan == LAST_SCAN)
151 error 98 error = -EBUSY;
152 else 99 else
153 clear 100 clear = false;
154 } 101 }
155 if (clear) 102 if (clear)
156 xas_clear_mark 103 xas_clear_mark(&xas, MEMFD_TAG_PINNED);
157 !! 104 if (++tagged % XA_CHECK_SCHED)
158 if (++latency < XA_CHE <<
159 continue; 105 continue;
160 latency = 0; <<
161 106
162 xas_pause(&xas); 107 xas_pause(&xas);
163 xas_unlock_irq(&xas); 108 xas_unlock_irq(&xas);
164 cond_resched(); 109 cond_resched();
165 xas_lock_irq(&xas); 110 xas_lock_irq(&xas);
166 } 111 }
167 xas_unlock_irq(&xas); 112 xas_unlock_irq(&xas);
168 } 113 }
169 114
170 return error; 115 return error;
171 } 116 }
172 117
173 static unsigned int *memfd_file_seals_ptr(stru 118 static unsigned int *memfd_file_seals_ptr(struct file *file)
174 { 119 {
175 if (shmem_file(file)) 120 if (shmem_file(file))
176 return &SHMEM_I(file_inode(fil 121 return &SHMEM_I(file_inode(file))->seals;
177 122
178 #ifdef CONFIG_HUGETLBFS 123 #ifdef CONFIG_HUGETLBFS
179 if (is_file_hugepages(file)) 124 if (is_file_hugepages(file))
180 return &HUGETLBFS_I(file_inode 125 return &HUGETLBFS_I(file_inode(file))->seals;
181 #endif 126 #endif
182 127
183 return NULL; 128 return NULL;
184 } 129 }
185 130
186 #define F_ALL_SEALS (F_SEAL_SEAL | \ 131 #define F_ALL_SEALS (F_SEAL_SEAL | \
187 F_SEAL_EXEC | \ <<
188 F_SEAL_SHRINK | \ 132 F_SEAL_SHRINK | \
189 F_SEAL_GROW | \ 133 F_SEAL_GROW | \
190 F_SEAL_WRITE | \ !! 134 F_SEAL_WRITE)
191 F_SEAL_FUTURE_WRITE) <<
192 135
193 static int memfd_add_seals(struct file *file, 136 static int memfd_add_seals(struct file *file, unsigned int seals)
194 { 137 {
195 struct inode *inode = file_inode(file) 138 struct inode *inode = file_inode(file);
196 unsigned int *file_seals; 139 unsigned int *file_seals;
197 int error; 140 int error;
198 141
199 /* 142 /*
200 * SEALING 143 * SEALING
201 * Sealing allows multiple parties to 144 * Sealing allows multiple parties to share a tmpfs or hugetlbfs file
202 * but restrict access to a specific s 145 * but restrict access to a specific subset of file operations. Seals
203 * can only be added, but never remove 146 * can only be added, but never removed. This way, mutually untrusted
204 * parties can share common memory reg 147 * parties can share common memory regions with a well-defined policy.
205 * A malicious peer can thus never per 148 * A malicious peer can thus never perform unwanted operations on a
206 * shared object. 149 * shared object.
207 * 150 *
208 * Seals are only supported on special 151 * Seals are only supported on special tmpfs or hugetlbfs files and
209 * always affect the whole underlying 152 * always affect the whole underlying inode. Once a seal is set, it
210 * may prevent some kinds of access to 153 * may prevent some kinds of access to the file. Currently, the
211 * following seals are defined: 154 * following seals are defined:
212 * SEAL_SEAL: Prevent further seals 155 * SEAL_SEAL: Prevent further seals from being set on this file
213 * SEAL_SHRINK: Prevent the file fro 156 * SEAL_SHRINK: Prevent the file from shrinking
214 * SEAL_GROW: Prevent the file from 157 * SEAL_GROW: Prevent the file from growing
215 * SEAL_WRITE: Prevent write access 158 * SEAL_WRITE: Prevent write access to the file
216 * SEAL_EXEC: Prevent modification o <<
217 * 159 *
218 * As we don't require any trust relat 160 * As we don't require any trust relationship between two parties, we
219 * must prevent seals from being remov 161 * must prevent seals from being removed. Therefore, sealing a file
220 * only adds a given set of seals to t 162 * only adds a given set of seals to the file, it never touches
221 * existing seals. Furthermore, the "s 163 * existing seals. Furthermore, the "setting seals"-operation can be
222 * sealed itself, which basically prev 164 * sealed itself, which basically prevents any further seal from being
223 * added. 165 * added.
224 * 166 *
225 * Semantics of sealing are only defin 167 * Semantics of sealing are only defined on volatile files. Only
226 * anonymous tmpfs and hugetlbfs files 168 * anonymous tmpfs and hugetlbfs files support sealing. More
227 * importantly, seals are never writte 169 * importantly, seals are never written to disk. Therefore, there's
228 * no plan to support it on other file 170 * no plan to support it on other file types.
229 */ 171 */
230 172
231 if (!(file->f_mode & FMODE_WRITE)) 173 if (!(file->f_mode & FMODE_WRITE))
232 return -EPERM; 174 return -EPERM;
233 if (seals & ~(unsigned int)F_ALL_SEALS 175 if (seals & ~(unsigned int)F_ALL_SEALS)
234 return -EINVAL; 176 return -EINVAL;
235 177
236 inode_lock(inode); 178 inode_lock(inode);
237 179
238 file_seals = memfd_file_seals_ptr(file 180 file_seals = memfd_file_seals_ptr(file);
239 if (!file_seals) { 181 if (!file_seals) {
240 error = -EINVAL; 182 error = -EINVAL;
241 goto unlock; 183 goto unlock;
242 } 184 }
243 185
244 if (*file_seals & F_SEAL_SEAL) { 186 if (*file_seals & F_SEAL_SEAL) {
245 error = -EPERM; 187 error = -EPERM;
246 goto unlock; 188 goto unlock;
247 } 189 }
248 190
249 if ((seals & F_SEAL_WRITE) && !(*file_ 191 if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
250 error = mapping_deny_writable( 192 error = mapping_deny_writable(file->f_mapping);
251 if (error) 193 if (error)
252 goto unlock; 194 goto unlock;
253 195
254 error = memfd_wait_for_pins(fi 196 error = memfd_wait_for_pins(file->f_mapping);
255 if (error) { 197 if (error) {
256 mapping_allow_writable 198 mapping_allow_writable(file->f_mapping);
257 goto unlock; 199 goto unlock;
258 } 200 }
259 } 201 }
260 202
261 /* <<
262 * SEAL_EXEC implys SEAL_WRITE, making <<
263 */ <<
264 if (seals & F_SEAL_EXEC && inode->i_mo <<
265 seals |= F_SEAL_SHRINK|F_SEAL_ <<
266 <<
267 *file_seals |= seals; 203 *file_seals |= seals;
268 error = 0; 204 error = 0;
269 205
270 unlock: 206 unlock:
271 inode_unlock(inode); 207 inode_unlock(inode);
272 return error; 208 return error;
273 } 209 }
274 210
275 static int memfd_get_seals(struct file *file) 211 static int memfd_get_seals(struct file *file)
276 { 212 {
277 unsigned int *seals = memfd_file_seals 213 unsigned int *seals = memfd_file_seals_ptr(file);
278 214
279 return seals ? *seals : -EINVAL; 215 return seals ? *seals : -EINVAL;
280 } 216 }
281 217
282 long memfd_fcntl(struct file *file, unsigned i !! 218 long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
283 { 219 {
284 long error; 220 long error;
285 221
286 switch (cmd) { 222 switch (cmd) {
287 case F_ADD_SEALS: 223 case F_ADD_SEALS:
>> 224 /* disallow upper 32bit */
>> 225 if (arg > UINT_MAX)
>> 226 return -EINVAL;
>> 227
288 error = memfd_add_seals(file, 228 error = memfd_add_seals(file, arg);
289 break; 229 break;
290 case F_GET_SEALS: 230 case F_GET_SEALS:
291 error = memfd_get_seals(file); 231 error = memfd_get_seals(file);
292 break; 232 break;
293 default: 233 default:
294 error = -EINVAL; 234 error = -EINVAL;
295 break; 235 break;
296 } 236 }
297 237
298 return error; 238 return error;
299 } 239 }
300 240
301 #define MFD_NAME_PREFIX "memfd:" 241 #define MFD_NAME_PREFIX "memfd:"
302 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_P 242 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
303 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_ 243 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
304 244
305 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW !! 245 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB)
306 <<
307 static int check_sysctl_memfd_noexec(unsigned <<
308 { <<
309 #ifdef CONFIG_SYSCTL <<
310 struct pid_namespace *ns = task_active <<
311 int sysctl = pidns_memfd_noexec_scope( <<
312 <<
313 if (!(*flags & (MFD_EXEC | MFD_NOEXEC_ <<
314 if (sysctl >= MEMFD_NOEXEC_SCO <<
315 *flags |= MFD_NOEXEC_S <<
316 else <<
317 *flags |= MFD_EXEC; <<
318 } <<
319 <<
320 if (!(*flags & MFD_NOEXEC_SEAL) && sys <<
321 pr_err_ratelimited( <<
322 "%s[%d]: memfd_create( <<
323 current->comm, task_pi <<
324 return -EACCES; <<
325 } <<
326 #endif <<
327 return 0; <<
328 } <<
329 246
330 SYSCALL_DEFINE2(memfd_create, 247 SYSCALL_DEFINE2(memfd_create,
331 const char __user *, uname, 248 const char __user *, uname,
332 unsigned int, flags) 249 unsigned int, flags)
333 { 250 {
334 unsigned int *file_seals; 251 unsigned int *file_seals;
335 struct file *file; 252 struct file *file;
336 int fd, error; 253 int fd, error;
337 char *name; 254 char *name;
338 long len; 255 long len;
339 256
340 if (!(flags & MFD_HUGETLB)) { 257 if (!(flags & MFD_HUGETLB)) {
341 if (flags & ~(unsigned int)MFD 258 if (flags & ~(unsigned int)MFD_ALL_FLAGS)
342 return -EINVAL; 259 return -EINVAL;
343 } else { 260 } else {
344 /* Allow huge page size encodi 261 /* Allow huge page size encoding in flags. */
345 if (flags & ~(unsigned int)(MF 262 if (flags & ~(unsigned int)(MFD_ALL_FLAGS |
346 (MFD_HUGE_MASK 263 (MFD_HUGE_MASK << MFD_HUGE_SHIFT)))
347 return -EINVAL; 264 return -EINVAL;
348 } 265 }
349 266
350 /* Invalid if both EXEC and NOEXEC_SEA <<
351 if ((flags & MFD_EXEC) && (flags & MFD <<
352 return -EINVAL; <<
353 <<
354 error = check_sysctl_memfd_noexec(&fla <<
355 if (error < 0) <<
356 return error; <<
357 <<
358 /* length includes terminating zero */ 267 /* length includes terminating zero */
359 len = strnlen_user(uname, MFD_NAME_MAX 268 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
360 if (len <= 0) 269 if (len <= 0)
361 return -EFAULT; 270 return -EFAULT;
362 if (len > MFD_NAME_MAX_LEN + 1) 271 if (len > MFD_NAME_MAX_LEN + 1)
363 return -EINVAL; 272 return -EINVAL;
364 273
365 name = kmalloc(len + MFD_NAME_PREFIX_L 274 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL);
366 if (!name) 275 if (!name)
367 return -ENOMEM; 276 return -ENOMEM;
368 277
369 strcpy(name, MFD_NAME_PREFIX); 278 strcpy(name, MFD_NAME_PREFIX);
370 if (copy_from_user(&name[MFD_NAME_PREF 279 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
371 error = -EFAULT; 280 error = -EFAULT;
372 goto err_name; 281 goto err_name;
373 } 282 }
374 283
375 /* terminating-zero may have changed a 284 /* terminating-zero may have changed after strnlen_user() returned */
376 if (name[len + MFD_NAME_PREFIX_LEN - 1 285 if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
377 error = -EFAULT; 286 error = -EFAULT;
378 goto err_name; 287 goto err_name;
379 } 288 }
380 289
381 fd = get_unused_fd_flags((flags & MFD_ 290 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
382 if (fd < 0) { 291 if (fd < 0) {
383 error = fd; 292 error = fd;
384 goto err_name; 293 goto err_name;
385 } 294 }
386 295
387 if (flags & MFD_HUGETLB) { 296 if (flags & MFD_HUGETLB) {
388 file = hugetlb_file_setup(name !! 297 struct user_struct *user = NULL;
>> 298
>> 299 file = hugetlb_file_setup(name, 0, VM_NORESERVE, &user,
389 HUGETL 300 HUGETLB_ANONHUGE_INODE,
390 (flags 301 (flags >> MFD_HUGE_SHIFT) &
391 MFD_HU 302 MFD_HUGE_MASK);
392 } else 303 } else
393 file = shmem_file_setup(name, 304 file = shmem_file_setup(name, 0, VM_NORESERVE);
394 if (IS_ERR(file)) { 305 if (IS_ERR(file)) {
395 error = PTR_ERR(file); 306 error = PTR_ERR(file);
396 goto err_fd; 307 goto err_fd;
397 } 308 }
398 file->f_mode |= FMODE_LSEEK | FMODE_PR 309 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
399 file->f_flags |= O_LARGEFILE; 310 file->f_flags |= O_LARGEFILE;
400 311
401 if (flags & MFD_NOEXEC_SEAL) { !! 312 if (flags & MFD_ALLOW_SEALING) {
402 struct inode *inode = file_ino <<
403 <<
404 inode->i_mode &= ~0111; <<
405 file_seals = memfd_file_seals_ <<
406 if (file_seals) { <<
407 *file_seals &= ~F_SEAL <<
408 *file_seals |= F_SEAL_ <<
409 } <<
410 } else if (flags & MFD_ALLOW_SEALING) <<
411 /* MFD_EXEC and MFD_ALLOW_SEAL <<
412 file_seals = memfd_file_seals_ 313 file_seals = memfd_file_seals_ptr(file);
413 if (file_seals) !! 314 *file_seals &= ~F_SEAL_SEAL;
414 *file_seals &= ~F_SEAL <<
415 } 315 }
416 316
417 fd_install(fd, file); 317 fd_install(fd, file);
418 kfree(name); 318 kfree(name);
419 return fd; 319 return fd;
420 320
421 err_fd: 321 err_fd:
422 put_unused_fd(fd); 322 put_unused_fd(fd);
423 err_name: 323 err_name:
424 kfree(name); 324 kfree(name);
425 return error; 325 return error;
426 } 326 }
427 327
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.