1 /* 1 /*
2 * memfd_create system call and file sealing s 2 * memfd_create system call and file sealing support
3 * 3 *
4 * Code was originally included in shmem.c, an 4 * Code was originally included in shmem.c, and broken out to facilitate
5 * use by hugetlbfs as well as tmpfs. 5 * use by hugetlbfs as well as tmpfs.
6 * 6 *
7 * This file is released under the GPL. 7 * This file is released under the GPL.
8 */ 8 */
9 9
10 #include <linux/fs.h> 10 #include <linux/fs.h>
11 #include <linux/vfs.h> 11 #include <linux/vfs.h>
12 #include <linux/pagemap.h> 12 #include <linux/pagemap.h>
13 #include <linux/file.h> 13 #include <linux/file.h>
14 #include <linux/mm.h> 14 #include <linux/mm.h>
15 #include <linux/sched/signal.h> 15 #include <linux/sched/signal.h>
16 #include <linux/khugepaged.h> 16 #include <linux/khugepaged.h>
17 #include <linux/syscalls.h> 17 #include <linux/syscalls.h>
18 #include <linux/hugetlb.h> 18 #include <linux/hugetlb.h>
19 #include <linux/shmem_fs.h> 19 #include <linux/shmem_fs.h>
20 #include <linux/memfd.h> 20 #include <linux/memfd.h>
21 #include <linux/pid_namespace.h> <<
22 #include <uapi/linux/memfd.h> 21 #include <uapi/linux/memfd.h>
23 22
24 /* 23 /*
25 * We need a tag: a new tag would expand every 24 * We need a tag: a new tag would expand every xa_node by 8 bytes,
26 * so reuse a tag which we firmly believe is n 25 * so reuse a tag which we firmly believe is never set or cleared on tmpfs
27 * or hugetlbfs because they are memory only f 26 * or hugetlbfs because they are memory only filesystems.
28 */ 27 */
29 #define MEMFD_TAG_PINNED PAGECACHE_TAG_ 28 #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE
30 #define LAST_SCAN 4 /* abo 29 #define LAST_SCAN 4 /* about 150ms max */
31 30
32 static bool memfd_folio_has_extra_refs(struct <<
33 { <<
34 return folio_ref_count(folio) - folio_ <<
35 folio_nr_pages(folio); <<
36 } <<
37 <<
38 static void memfd_tag_pins(struct xa_state *xa 31 static void memfd_tag_pins(struct xa_state *xas)
39 { 32 {
40 struct folio *folio; !! 33 struct page *page;
41 int latency = 0; !! 34 unsigned int tagged = 0;
42 35
43 lru_add_drain(); 36 lru_add_drain();
44 37
45 xas_lock_irq(xas); 38 xas_lock_irq(xas);
46 xas_for_each(xas, folio, ULONG_MAX) { !! 39 xas_for_each(xas, page, ULONG_MAX) {
47 if (!xa_is_value(folio) && mem !! 40 if (xa_is_value(page))
>> 41 continue;
>> 42 page = find_subpage(page, xas->xa_index);
>> 43 if (page_count(page) - page_mapcount(page) > 1)
48 xas_set_mark(xas, MEMF 44 xas_set_mark(xas, MEMFD_TAG_PINNED);
49 45
50 if (++latency < XA_CHECK_SCHED !! 46 if (++tagged % XA_CHECK_SCHED)
51 continue; 47 continue;
52 latency = 0; <<
53 48
54 xas_pause(xas); 49 xas_pause(xas);
55 xas_unlock_irq(xas); 50 xas_unlock_irq(xas);
56 cond_resched(); 51 cond_resched();
57 xas_lock_irq(xas); 52 xas_lock_irq(xas);
58 } 53 }
59 xas_unlock_irq(xas); 54 xas_unlock_irq(xas);
60 } 55 }
61 56
62 /* 57 /*
63 * This is a helper function used by memfd_pin <<
64 * It is mainly called to allocate a folio in <<
65 * (memfd_pin_folios()) cannot find a folio in <<
66 * index in the mapping. <<
67 */ <<
68 struct folio *memfd_alloc_folio(struct file *m <<
69 { <<
70 #ifdef CONFIG_HUGETLB_PAGE <<
71 struct folio *folio; <<
72 gfp_t gfp_mask; <<
73 int err; <<
74 <<
75 if (is_file_hugepages(memfd)) { <<
76 /* <<
77 * The folio would most likely <<
78 * therefore, we have zone mem <<
79 * alloc from. Also, the folio <<
80 * amount of time, so it is no <<
81 */ <<
82 struct hstate *h = hstate_file <<
83 <<
84 gfp_mask = htlb_alloc_mask(h); <<
85 gfp_mask &= ~(__GFP_HIGHMEM | <<
86 idx >>= huge_page_order(h); <<
87 <<
88 folio = alloc_hugetlb_folio_re <<
89 <<
90 <<
91 <<
92 if (folio) { <<
93 err = hugetlb_add_to_p <<
94 <<
95 <<
96 if (err) { <<
97 folio_put(foli <<
98 return ERR_PTR <<
99 } <<
100 folio_unlock(folio); <<
101 return folio; <<
102 } <<
103 return ERR_PTR(-ENOMEM); <<
104 } <<
105 #endif <<
106 return shmem_read_folio(memfd->f_mappi <<
107 } <<
108 <<
109 /* <<
110 * Setting SEAL_WRITE requires us to verify th 58 * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
111 * via get_user_pages(), drivers might have so 59 * via get_user_pages(), drivers might have some pending I/O without any active
112 * user-space mappings (eg., direct-IO, AIO). !! 60 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
113 * and see whether it has an elevated ref-coun 61 * and see whether it has an elevated ref-count. If so, we tag them and wait for
114 * them to be dropped. 62 * them to be dropped.
115 * The caller must guarantee that no new user 63 * The caller must guarantee that no new user will acquire writable references
116 * to those folios to avoid races. !! 64 * to those pages to avoid races.
117 */ 65 */
118 static int memfd_wait_for_pins(struct address_ 66 static int memfd_wait_for_pins(struct address_space *mapping)
119 { 67 {
120 XA_STATE(xas, &mapping->i_pages, 0); 68 XA_STATE(xas, &mapping->i_pages, 0);
121 struct folio *folio; !! 69 struct page *page;
122 int error, scan; 70 int error, scan;
123 71
124 memfd_tag_pins(&xas); 72 memfd_tag_pins(&xas);
125 73
126 error = 0; 74 error = 0;
127 for (scan = 0; scan <= LAST_SCAN; scan 75 for (scan = 0; scan <= LAST_SCAN; scan++) {
128 int latency = 0; !! 76 unsigned int tagged = 0;
129 77
130 if (!xas_marked(&xas, MEMFD_TA 78 if (!xas_marked(&xas, MEMFD_TAG_PINNED))
131 break; 79 break;
132 80
133 if (!scan) 81 if (!scan)
134 lru_add_drain_all(); 82 lru_add_drain_all();
135 else if (schedule_timeout_kill 83 else if (schedule_timeout_killable((HZ << scan) / 200))
136 scan = LAST_SCAN; 84 scan = LAST_SCAN;
137 85
138 xas_set(&xas, 0); 86 xas_set(&xas, 0);
139 xas_lock_irq(&xas); 87 xas_lock_irq(&xas);
140 xas_for_each_marked(&xas, foli !! 88 xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
141 bool clear = true; 89 bool clear = true;
142 !! 90 if (xa_is_value(page))
143 if (!xa_is_value(folio !! 91 continue;
144 memfd_folio_has_ex !! 92 page = find_subpage(page, xas.xa_index);
>> 93 if (page_count(page) - page_mapcount(page) != 1) {
145 /* 94 /*
146 * On the last 95 * On the last scan, we clean up all those tags
147 * we inserted 96 * we inserted; but make a note that we still
148 * found folio !! 97 * found pages pinned.
149 */ 98 */
150 if (scan == LA 99 if (scan == LAST_SCAN)
151 error 100 error = -EBUSY;
152 else 101 else
153 clear 102 clear = false;
154 } 103 }
155 if (clear) 104 if (clear)
156 xas_clear_mark 105 xas_clear_mark(&xas, MEMFD_TAG_PINNED);
157 !! 106 if (++tagged % XA_CHECK_SCHED)
158 if (++latency < XA_CHE <<
159 continue; 107 continue;
160 latency = 0; <<
161 108
162 xas_pause(&xas); 109 xas_pause(&xas);
163 xas_unlock_irq(&xas); 110 xas_unlock_irq(&xas);
164 cond_resched(); 111 cond_resched();
165 xas_lock_irq(&xas); 112 xas_lock_irq(&xas);
166 } 113 }
167 xas_unlock_irq(&xas); 114 xas_unlock_irq(&xas);
168 } 115 }
169 116
170 return error; 117 return error;
171 } 118 }
172 119
173 static unsigned int *memfd_file_seals_ptr(stru 120 static unsigned int *memfd_file_seals_ptr(struct file *file)
174 { 121 {
175 if (shmem_file(file)) 122 if (shmem_file(file))
176 return &SHMEM_I(file_inode(fil 123 return &SHMEM_I(file_inode(file))->seals;
177 124
178 #ifdef CONFIG_HUGETLBFS 125 #ifdef CONFIG_HUGETLBFS
179 if (is_file_hugepages(file)) 126 if (is_file_hugepages(file))
180 return &HUGETLBFS_I(file_inode 127 return &HUGETLBFS_I(file_inode(file))->seals;
181 #endif 128 #endif
182 129
183 return NULL; 130 return NULL;
184 } 131 }
185 132
186 #define F_ALL_SEALS (F_SEAL_SEAL | \ 133 #define F_ALL_SEALS (F_SEAL_SEAL | \
187 F_SEAL_EXEC | \ <<
188 F_SEAL_SHRINK | \ 134 F_SEAL_SHRINK | \
189 F_SEAL_GROW | \ 135 F_SEAL_GROW | \
190 F_SEAL_WRITE | \ 136 F_SEAL_WRITE | \
191 F_SEAL_FUTURE_WRITE) 137 F_SEAL_FUTURE_WRITE)
192 138
193 static int memfd_add_seals(struct file *file, 139 static int memfd_add_seals(struct file *file, unsigned int seals)
194 { 140 {
195 struct inode *inode = file_inode(file) 141 struct inode *inode = file_inode(file);
196 unsigned int *file_seals; 142 unsigned int *file_seals;
197 int error; 143 int error;
198 144
199 /* 145 /*
200 * SEALING 146 * SEALING
201 * Sealing allows multiple parties to 147 * Sealing allows multiple parties to share a tmpfs or hugetlbfs file
202 * but restrict access to a specific s 148 * but restrict access to a specific subset of file operations. Seals
203 * can only be added, but never remove 149 * can only be added, but never removed. This way, mutually untrusted
204 * parties can share common memory reg 150 * parties can share common memory regions with a well-defined policy.
205 * A malicious peer can thus never per 151 * A malicious peer can thus never perform unwanted operations on a
206 * shared object. 152 * shared object.
207 * 153 *
208 * Seals are only supported on special 154 * Seals are only supported on special tmpfs or hugetlbfs files and
209 * always affect the whole underlying 155 * always affect the whole underlying inode. Once a seal is set, it
210 * may prevent some kinds of access to 156 * may prevent some kinds of access to the file. Currently, the
211 * following seals are defined: 157 * following seals are defined:
212 * SEAL_SEAL: Prevent further seals 158 * SEAL_SEAL: Prevent further seals from being set on this file
213 * SEAL_SHRINK: Prevent the file fro 159 * SEAL_SHRINK: Prevent the file from shrinking
214 * SEAL_GROW: Prevent the file from 160 * SEAL_GROW: Prevent the file from growing
215 * SEAL_WRITE: Prevent write access 161 * SEAL_WRITE: Prevent write access to the file
216 * SEAL_EXEC: Prevent modification o <<
217 * 162 *
218 * As we don't require any trust relat 163 * As we don't require any trust relationship between two parties, we
219 * must prevent seals from being remov 164 * must prevent seals from being removed. Therefore, sealing a file
220 * only adds a given set of seals to t 165 * only adds a given set of seals to the file, it never touches
221 * existing seals. Furthermore, the "s 166 * existing seals. Furthermore, the "setting seals"-operation can be
222 * sealed itself, which basically prev 167 * sealed itself, which basically prevents any further seal from being
223 * added. 168 * added.
224 * 169 *
225 * Semantics of sealing are only defin 170 * Semantics of sealing are only defined on volatile files. Only
226 * anonymous tmpfs and hugetlbfs files 171 * anonymous tmpfs and hugetlbfs files support sealing. More
227 * importantly, seals are never writte 172 * importantly, seals are never written to disk. Therefore, there's
228 * no plan to support it on other file 173 * no plan to support it on other file types.
229 */ 174 */
230 175
231 if (!(file->f_mode & FMODE_WRITE)) 176 if (!(file->f_mode & FMODE_WRITE))
232 return -EPERM; 177 return -EPERM;
233 if (seals & ~(unsigned int)F_ALL_SEALS 178 if (seals & ~(unsigned int)F_ALL_SEALS)
234 return -EINVAL; 179 return -EINVAL;
235 180
236 inode_lock(inode); 181 inode_lock(inode);
237 182
238 file_seals = memfd_file_seals_ptr(file 183 file_seals = memfd_file_seals_ptr(file);
239 if (!file_seals) { 184 if (!file_seals) {
240 error = -EINVAL; 185 error = -EINVAL;
241 goto unlock; 186 goto unlock;
242 } 187 }
243 188
244 if (*file_seals & F_SEAL_SEAL) { 189 if (*file_seals & F_SEAL_SEAL) {
245 error = -EPERM; 190 error = -EPERM;
246 goto unlock; 191 goto unlock;
247 } 192 }
248 193
249 if ((seals & F_SEAL_WRITE) && !(*file_ 194 if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
250 error = mapping_deny_writable( 195 error = mapping_deny_writable(file->f_mapping);
251 if (error) 196 if (error)
252 goto unlock; 197 goto unlock;
253 198
254 error = memfd_wait_for_pins(fi 199 error = memfd_wait_for_pins(file->f_mapping);
255 if (error) { 200 if (error) {
256 mapping_allow_writable 201 mapping_allow_writable(file->f_mapping);
257 goto unlock; 202 goto unlock;
258 } 203 }
259 } 204 }
260 205
261 /* <<
262 * SEAL_EXEC implys SEAL_WRITE, making <<
263 */ <<
264 if (seals & F_SEAL_EXEC && inode->i_mo <<
265 seals |= F_SEAL_SHRINK|F_SEAL_ <<
266 <<
267 *file_seals |= seals; 206 *file_seals |= seals;
268 error = 0; 207 error = 0;
269 208
270 unlock: 209 unlock:
271 inode_unlock(inode); 210 inode_unlock(inode);
272 return error; 211 return error;
273 } 212 }
274 213
275 static int memfd_get_seals(struct file *file) 214 static int memfd_get_seals(struct file *file)
276 { 215 {
277 unsigned int *seals = memfd_file_seals 216 unsigned int *seals = memfd_file_seals_ptr(file);
278 217
279 return seals ? *seals : -EINVAL; 218 return seals ? *seals : -EINVAL;
280 } 219 }
281 220
282 long memfd_fcntl(struct file *file, unsigned i !! 221 long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
283 { 222 {
284 long error; 223 long error;
285 224
286 switch (cmd) { 225 switch (cmd) {
287 case F_ADD_SEALS: 226 case F_ADD_SEALS:
>> 227 /* disallow upper 32bit */
>> 228 if (arg > UINT_MAX)
>> 229 return -EINVAL;
>> 230
288 error = memfd_add_seals(file, 231 error = memfd_add_seals(file, arg);
289 break; 232 break;
290 case F_GET_SEALS: 233 case F_GET_SEALS:
291 error = memfd_get_seals(file); 234 error = memfd_get_seals(file);
292 break; 235 break;
293 default: 236 default:
294 error = -EINVAL; 237 error = -EINVAL;
295 break; 238 break;
296 } 239 }
297 240
298 return error; 241 return error;
299 } 242 }
300 243
301 #define MFD_NAME_PREFIX "memfd:" 244 #define MFD_NAME_PREFIX "memfd:"
302 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_P 245 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
303 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_ 246 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
304 247
305 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW !! 248 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB)
306 <<
307 static int check_sysctl_memfd_noexec(unsigned <<
308 { <<
309 #ifdef CONFIG_SYSCTL <<
310 struct pid_namespace *ns = task_active <<
311 int sysctl = pidns_memfd_noexec_scope( <<
312 <<
313 if (!(*flags & (MFD_EXEC | MFD_NOEXEC_ <<
314 if (sysctl >= MEMFD_NOEXEC_SCO <<
315 *flags |= MFD_NOEXEC_S <<
316 else <<
317 *flags |= MFD_EXEC; <<
318 } <<
319 <<
320 if (!(*flags & MFD_NOEXEC_SEAL) && sys <<
321 pr_err_ratelimited( <<
322 "%s[%d]: memfd_create( <<
323 current->comm, task_pi <<
324 return -EACCES; <<
325 } <<
326 #endif <<
327 return 0; <<
328 } <<
329 249
330 SYSCALL_DEFINE2(memfd_create, 250 SYSCALL_DEFINE2(memfd_create,
331 const char __user *, uname, 251 const char __user *, uname,
332 unsigned int, flags) 252 unsigned int, flags)
333 { 253 {
334 unsigned int *file_seals; 254 unsigned int *file_seals;
335 struct file *file; 255 struct file *file;
336 int fd, error; 256 int fd, error;
337 char *name; 257 char *name;
338 long len; 258 long len;
339 259
340 if (!(flags & MFD_HUGETLB)) { 260 if (!(flags & MFD_HUGETLB)) {
341 if (flags & ~(unsigned int)MFD 261 if (flags & ~(unsigned int)MFD_ALL_FLAGS)
342 return -EINVAL; 262 return -EINVAL;
343 } else { 263 } else {
344 /* Allow huge page size encodi 264 /* Allow huge page size encoding in flags. */
345 if (flags & ~(unsigned int)(MF 265 if (flags & ~(unsigned int)(MFD_ALL_FLAGS |
346 (MFD_HUGE_MASK 266 (MFD_HUGE_MASK << MFD_HUGE_SHIFT)))
347 return -EINVAL; 267 return -EINVAL;
348 } 268 }
349 269
350 /* Invalid if both EXEC and NOEXEC_SEA <<
351 if ((flags & MFD_EXEC) && (flags & MFD <<
352 return -EINVAL; <<
353 <<
354 error = check_sysctl_memfd_noexec(&fla <<
355 if (error < 0) <<
356 return error; <<
357 <<
358 /* length includes terminating zero */ 270 /* length includes terminating zero */
359 len = strnlen_user(uname, MFD_NAME_MAX 271 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
360 if (len <= 0) 272 if (len <= 0)
361 return -EFAULT; 273 return -EFAULT;
362 if (len > MFD_NAME_MAX_LEN + 1) 274 if (len > MFD_NAME_MAX_LEN + 1)
363 return -EINVAL; 275 return -EINVAL;
364 276
365 name = kmalloc(len + MFD_NAME_PREFIX_L 277 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL);
366 if (!name) 278 if (!name)
367 return -ENOMEM; 279 return -ENOMEM;
368 280
369 strcpy(name, MFD_NAME_PREFIX); 281 strcpy(name, MFD_NAME_PREFIX);
370 if (copy_from_user(&name[MFD_NAME_PREF 282 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
371 error = -EFAULT; 283 error = -EFAULT;
372 goto err_name; 284 goto err_name;
373 } 285 }
374 286
375 /* terminating-zero may have changed a 287 /* terminating-zero may have changed after strnlen_user() returned */
376 if (name[len + MFD_NAME_PREFIX_LEN - 1 288 if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
377 error = -EFAULT; 289 error = -EFAULT;
378 goto err_name; 290 goto err_name;
379 } 291 }
380 292
381 fd = get_unused_fd_flags((flags & MFD_ 293 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
382 if (fd < 0) { 294 if (fd < 0) {
383 error = fd; 295 error = fd;
384 goto err_name; 296 goto err_name;
385 } 297 }
386 298
387 if (flags & MFD_HUGETLB) { 299 if (flags & MFD_HUGETLB) {
388 file = hugetlb_file_setup(name !! 300 struct user_struct *user = NULL;
>> 301
>> 302 file = hugetlb_file_setup(name, 0, VM_NORESERVE, &user,
389 HUGETL 303 HUGETLB_ANONHUGE_INODE,
390 (flags 304 (flags >> MFD_HUGE_SHIFT) &
391 MFD_HU 305 MFD_HUGE_MASK);
392 } else 306 } else
393 file = shmem_file_setup(name, 307 file = shmem_file_setup(name, 0, VM_NORESERVE);
394 if (IS_ERR(file)) { 308 if (IS_ERR(file)) {
395 error = PTR_ERR(file); 309 error = PTR_ERR(file);
396 goto err_fd; 310 goto err_fd;
397 } 311 }
398 file->f_mode |= FMODE_LSEEK | FMODE_PR 312 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
399 file->f_flags |= O_LARGEFILE; 313 file->f_flags |= O_LARGEFILE;
400 314
401 if (flags & MFD_NOEXEC_SEAL) { !! 315 if (flags & MFD_ALLOW_SEALING) {
402 struct inode *inode = file_ino <<
403 <<
404 inode->i_mode &= ~0111; <<
405 file_seals = memfd_file_seals_ <<
406 if (file_seals) { <<
407 *file_seals &= ~F_SEAL <<
408 *file_seals |= F_SEAL_ <<
409 } <<
410 } else if (flags & MFD_ALLOW_SEALING) <<
411 /* MFD_EXEC and MFD_ALLOW_SEAL <<
412 file_seals = memfd_file_seals_ 316 file_seals = memfd_file_seals_ptr(file);
413 if (file_seals) !! 317 *file_seals &= ~F_SEAL_SEAL;
414 *file_seals &= ~F_SEAL <<
415 } 318 }
416 319
417 fd_install(fd, file); 320 fd_install(fd, file);
418 kfree(name); 321 kfree(name);
419 return fd; 322 return fd;
420 323
421 err_fd: 324 err_fd:
422 put_unused_fd(fd); 325 put_unused_fd(fd);
423 err_name: 326 err_name:
424 kfree(name); 327 kfree(name);
425 return error; 328 return error;
426 } 329 }
427 330
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.