1 /* 1 /*
2 * memfd_create system call and file sealing s 2 * memfd_create system call and file sealing support
3 * 3 *
4 * Code was originally included in shmem.c, an 4 * Code was originally included in shmem.c, and broken out to facilitate
5 * use by hugetlbfs as well as tmpfs. 5 * use by hugetlbfs as well as tmpfs.
6 * 6 *
7 * This file is released under the GPL. 7 * This file is released under the GPL.
8 */ 8 */
9 9
10 #include <linux/fs.h> 10 #include <linux/fs.h>
11 #include <linux/vfs.h> 11 #include <linux/vfs.h>
12 #include <linux/pagemap.h> 12 #include <linux/pagemap.h>
13 #include <linux/file.h> 13 #include <linux/file.h>
14 #include <linux/mm.h> 14 #include <linux/mm.h>
15 #include <linux/sched/signal.h> 15 #include <linux/sched/signal.h>
16 #include <linux/khugepaged.h> 16 #include <linux/khugepaged.h>
17 #include <linux/syscalls.h> 17 #include <linux/syscalls.h>
18 #include <linux/hugetlb.h> 18 #include <linux/hugetlb.h>
19 #include <linux/shmem_fs.h> 19 #include <linux/shmem_fs.h>
20 #include <linux/memfd.h> 20 #include <linux/memfd.h>
21 #include <linux/pid_namespace.h> <<
22 #include <uapi/linux/memfd.h> 21 #include <uapi/linux/memfd.h>
23 22
24 /* 23 /*
25 * We need a tag: a new tag would expand every 24 * We need a tag: a new tag would expand every xa_node by 8 bytes,
26 * so reuse a tag which we firmly believe is n 25 * so reuse a tag which we firmly believe is never set or cleared on tmpfs
27 * or hugetlbfs because they are memory only f 26 * or hugetlbfs because they are memory only filesystems.
28 */ 27 */
29 #define MEMFD_TAG_PINNED PAGECACHE_TAG_ 28 #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE
30 #define LAST_SCAN 4 /* abo 29 #define LAST_SCAN 4 /* about 150ms max */
31 30
32 static bool memfd_folio_has_extra_refs(struct <<
33 { <<
34 return folio_ref_count(folio) - folio_ <<
35 folio_nr_pages(folio); <<
36 } <<
37 <<
38 static void memfd_tag_pins(struct xa_state *xa 31 static void memfd_tag_pins(struct xa_state *xas)
39 { 32 {
40 struct folio *folio; !! 33 struct page *page;
41 int latency = 0; 34 int latency = 0;
>> 35 int cache_count;
42 36
43 lru_add_drain(); 37 lru_add_drain();
44 38
45 xas_lock_irq(xas); 39 xas_lock_irq(xas);
46 xas_for_each(xas, folio, ULONG_MAX) { !! 40 xas_for_each(xas, page, ULONG_MAX) {
47 if (!xa_is_value(folio) && mem !! 41 cache_count = 1;
>> 42 if (!xa_is_value(page) &&
>> 43 PageTransHuge(page) && !PageHuge(page))
>> 44 cache_count = HPAGE_PMD_NR;
>> 45
>> 46 if (!xa_is_value(page) &&
>> 47 page_count(page) - total_mapcount(page) != cache_count)
48 xas_set_mark(xas, MEMF 48 xas_set_mark(xas, MEMFD_TAG_PINNED);
>> 49 if (cache_count != 1)
>> 50 xas_set(xas, page->index + cache_count);
49 51
50 if (++latency < XA_CHECK_SCHED !! 52 latency += cache_count;
>> 53 if (latency < XA_CHECK_SCHED)
51 continue; 54 continue;
52 latency = 0; 55 latency = 0;
53 56
54 xas_pause(xas); 57 xas_pause(xas);
55 xas_unlock_irq(xas); 58 xas_unlock_irq(xas);
56 cond_resched(); 59 cond_resched();
57 xas_lock_irq(xas); 60 xas_lock_irq(xas);
58 } 61 }
59 xas_unlock_irq(xas); 62 xas_unlock_irq(xas);
60 } 63 }
61 64
62 /* 65 /*
63 * This is a helper function used by memfd_pin <<
64 * It is mainly called to allocate a folio in <<
65 * (memfd_pin_folios()) cannot find a folio in <<
66 * index in the mapping. <<
67 */ <<
68 struct folio *memfd_alloc_folio(struct file *m <<
69 { <<
70 #ifdef CONFIG_HUGETLB_PAGE <<
71 struct folio *folio; <<
72 gfp_t gfp_mask; <<
73 int err; <<
74 <<
75 if (is_file_hugepages(memfd)) { <<
76 /* <<
77 * The folio would most likely <<
78 * therefore, we have zone mem <<
79 * alloc from. Also, the folio <<
80 * amount of time, so it is no <<
81 */ <<
82 struct hstate *h = hstate_file <<
83 <<
84 gfp_mask = htlb_alloc_mask(h); <<
85 gfp_mask &= ~(__GFP_HIGHMEM | <<
86 idx >>= huge_page_order(h); <<
87 <<
88 folio = alloc_hugetlb_folio_re <<
89 <<
90 <<
91 <<
92 if (folio) { <<
93 err = hugetlb_add_to_p <<
94 <<
95 <<
96 if (err) { <<
97 folio_put(foli <<
98 return ERR_PTR <<
99 } <<
100 folio_unlock(folio); <<
101 return folio; <<
102 } <<
103 return ERR_PTR(-ENOMEM); <<
104 } <<
105 #endif <<
106 return shmem_read_folio(memfd->f_mappi <<
107 } <<
108 <<
109 /* <<
110 * Setting SEAL_WRITE requires us to verify th 66 * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
111 * via get_user_pages(), drivers might have so 67 * via get_user_pages(), drivers might have some pending I/O without any active
112 * user-space mappings (eg., direct-IO, AIO). !! 68 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
113 * and see whether it has an elevated ref-coun 69 * and see whether it has an elevated ref-count. If so, we tag them and wait for
114 * them to be dropped. 70 * them to be dropped.
115 * The caller must guarantee that no new user 71 * The caller must guarantee that no new user will acquire writable references
116 * to those folios to avoid races. !! 72 * to those pages to avoid races.
117 */ 73 */
118 static int memfd_wait_for_pins(struct address_ 74 static int memfd_wait_for_pins(struct address_space *mapping)
119 { 75 {
120 XA_STATE(xas, &mapping->i_pages, 0); 76 XA_STATE(xas, &mapping->i_pages, 0);
121 struct folio *folio; !! 77 struct page *page;
122 int error, scan; 78 int error, scan;
123 79
124 memfd_tag_pins(&xas); 80 memfd_tag_pins(&xas);
125 81
126 error = 0; 82 error = 0;
127 for (scan = 0; scan <= LAST_SCAN; scan 83 for (scan = 0; scan <= LAST_SCAN; scan++) {
128 int latency = 0; 84 int latency = 0;
>> 85 int cache_count;
129 86
130 if (!xas_marked(&xas, MEMFD_TA 87 if (!xas_marked(&xas, MEMFD_TAG_PINNED))
131 break; 88 break;
132 89
133 if (!scan) 90 if (!scan)
134 lru_add_drain_all(); 91 lru_add_drain_all();
135 else if (schedule_timeout_kill 92 else if (schedule_timeout_killable((HZ << scan) / 200))
136 scan = LAST_SCAN; 93 scan = LAST_SCAN;
137 94
138 xas_set(&xas, 0); 95 xas_set(&xas, 0);
139 xas_lock_irq(&xas); 96 xas_lock_irq(&xas);
140 xas_for_each_marked(&xas, foli !! 97 xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
141 bool clear = true; 98 bool clear = true;
142 99
143 if (!xa_is_value(folio !! 100 cache_count = 1;
144 memfd_folio_has_ex !! 101 if (!xa_is_value(page) &&
>> 102 PageTransHuge(page) && !PageHuge(page))
>> 103 cache_count = HPAGE_PMD_NR;
>> 104
>> 105 if (!xa_is_value(page) && cache_count !=
>> 106 page_count(page) - total_mapcount(page)) {
145 /* 107 /*
146 * On the last 108 * On the last scan, we clean up all those tags
147 * we inserted 109 * we inserted; but make a note that we still
148 * found folio !! 110 * found pages pinned.
149 */ 111 */
150 if (scan == LA 112 if (scan == LAST_SCAN)
151 error 113 error = -EBUSY;
152 else 114 else
153 clear 115 clear = false;
154 } 116 }
155 if (clear) 117 if (clear)
156 xas_clear_mark 118 xas_clear_mark(&xas, MEMFD_TAG_PINNED);
157 119
158 if (++latency < XA_CHE !! 120 latency += cache_count;
>> 121 if (latency < XA_CHECK_SCHED)
159 continue; 122 continue;
160 latency = 0; 123 latency = 0;
161 124
162 xas_pause(&xas); 125 xas_pause(&xas);
163 xas_unlock_irq(&xas); 126 xas_unlock_irq(&xas);
164 cond_resched(); 127 cond_resched();
165 xas_lock_irq(&xas); 128 xas_lock_irq(&xas);
166 } 129 }
167 xas_unlock_irq(&xas); 130 xas_unlock_irq(&xas);
168 } 131 }
169 132
170 return error; 133 return error;
171 } 134 }
172 135
173 static unsigned int *memfd_file_seals_ptr(stru 136 static unsigned int *memfd_file_seals_ptr(struct file *file)
174 { 137 {
175 if (shmem_file(file)) 138 if (shmem_file(file))
176 return &SHMEM_I(file_inode(fil 139 return &SHMEM_I(file_inode(file))->seals;
177 140
178 #ifdef CONFIG_HUGETLBFS 141 #ifdef CONFIG_HUGETLBFS
179 if (is_file_hugepages(file)) 142 if (is_file_hugepages(file))
180 return &HUGETLBFS_I(file_inode 143 return &HUGETLBFS_I(file_inode(file))->seals;
181 #endif 144 #endif
182 145
183 return NULL; 146 return NULL;
184 } 147 }
185 148
186 #define F_ALL_SEALS (F_SEAL_SEAL | \ 149 #define F_ALL_SEALS (F_SEAL_SEAL | \
187 F_SEAL_EXEC | \ <<
188 F_SEAL_SHRINK | \ 150 F_SEAL_SHRINK | \
189 F_SEAL_GROW | \ 151 F_SEAL_GROW | \
190 F_SEAL_WRITE | \ 152 F_SEAL_WRITE | \
191 F_SEAL_FUTURE_WRITE) 153 F_SEAL_FUTURE_WRITE)
192 154
193 static int memfd_add_seals(struct file *file, 155 static int memfd_add_seals(struct file *file, unsigned int seals)
194 { 156 {
195 struct inode *inode = file_inode(file) 157 struct inode *inode = file_inode(file);
196 unsigned int *file_seals; 158 unsigned int *file_seals;
197 int error; 159 int error;
198 160
199 /* 161 /*
200 * SEALING 162 * SEALING
201 * Sealing allows multiple parties to 163 * Sealing allows multiple parties to share a tmpfs or hugetlbfs file
202 * but restrict access to a specific s 164 * but restrict access to a specific subset of file operations. Seals
203 * can only be added, but never remove 165 * can only be added, but never removed. This way, mutually untrusted
204 * parties can share common memory reg 166 * parties can share common memory regions with a well-defined policy.
205 * A malicious peer can thus never per 167 * A malicious peer can thus never perform unwanted operations on a
206 * shared object. 168 * shared object.
207 * 169 *
208 * Seals are only supported on special 170 * Seals are only supported on special tmpfs or hugetlbfs files and
209 * always affect the whole underlying 171 * always affect the whole underlying inode. Once a seal is set, it
210 * may prevent some kinds of access to 172 * may prevent some kinds of access to the file. Currently, the
211 * following seals are defined: 173 * following seals are defined:
212 * SEAL_SEAL: Prevent further seals 174 * SEAL_SEAL: Prevent further seals from being set on this file
213 * SEAL_SHRINK: Prevent the file fro 175 * SEAL_SHRINK: Prevent the file from shrinking
214 * SEAL_GROW: Prevent the file from 176 * SEAL_GROW: Prevent the file from growing
215 * SEAL_WRITE: Prevent write access 177 * SEAL_WRITE: Prevent write access to the file
216 * SEAL_EXEC: Prevent modification o <<
217 * 178 *
218 * As we don't require any trust relat 179 * As we don't require any trust relationship between two parties, we
219 * must prevent seals from being remov 180 * must prevent seals from being removed. Therefore, sealing a file
220 * only adds a given set of seals to t 181 * only adds a given set of seals to the file, it never touches
221 * existing seals. Furthermore, the "s 182 * existing seals. Furthermore, the "setting seals"-operation can be
222 * sealed itself, which basically prev 183 * sealed itself, which basically prevents any further seal from being
223 * added. 184 * added.
224 * 185 *
225 * Semantics of sealing are only defin 186 * Semantics of sealing are only defined on volatile files. Only
226 * anonymous tmpfs and hugetlbfs files 187 * anonymous tmpfs and hugetlbfs files support sealing. More
227 * importantly, seals are never writte 188 * importantly, seals are never written to disk. Therefore, there's
228 * no plan to support it on other file 189 * no plan to support it on other file types.
229 */ 190 */
230 191
231 if (!(file->f_mode & FMODE_WRITE)) 192 if (!(file->f_mode & FMODE_WRITE))
232 return -EPERM; 193 return -EPERM;
233 if (seals & ~(unsigned int)F_ALL_SEALS 194 if (seals & ~(unsigned int)F_ALL_SEALS)
234 return -EINVAL; 195 return -EINVAL;
235 196
236 inode_lock(inode); 197 inode_lock(inode);
237 198
238 file_seals = memfd_file_seals_ptr(file 199 file_seals = memfd_file_seals_ptr(file);
239 if (!file_seals) { 200 if (!file_seals) {
240 error = -EINVAL; 201 error = -EINVAL;
241 goto unlock; 202 goto unlock;
242 } 203 }
243 204
244 if (*file_seals & F_SEAL_SEAL) { 205 if (*file_seals & F_SEAL_SEAL) {
245 error = -EPERM; 206 error = -EPERM;
246 goto unlock; 207 goto unlock;
247 } 208 }
248 209
249 if ((seals & F_SEAL_WRITE) && !(*file_ 210 if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
250 error = mapping_deny_writable( 211 error = mapping_deny_writable(file->f_mapping);
251 if (error) 212 if (error)
252 goto unlock; 213 goto unlock;
253 214
254 error = memfd_wait_for_pins(fi 215 error = memfd_wait_for_pins(file->f_mapping);
255 if (error) { 216 if (error) {
256 mapping_allow_writable 217 mapping_allow_writable(file->f_mapping);
257 goto unlock; 218 goto unlock;
258 } 219 }
259 } 220 }
260 221
261 /* <<
262 * SEAL_EXEC implys SEAL_WRITE, making <<
263 */ <<
264 if (seals & F_SEAL_EXEC && inode->i_mo <<
265 seals |= F_SEAL_SHRINK|F_SEAL_ <<
266 <<
267 *file_seals |= seals; 222 *file_seals |= seals;
268 error = 0; 223 error = 0;
269 224
270 unlock: 225 unlock:
271 inode_unlock(inode); 226 inode_unlock(inode);
272 return error; 227 return error;
273 } 228 }
274 229
275 static int memfd_get_seals(struct file *file) 230 static int memfd_get_seals(struct file *file)
276 { 231 {
277 unsigned int *seals = memfd_file_seals 232 unsigned int *seals = memfd_file_seals_ptr(file);
278 233
279 return seals ? *seals : -EINVAL; 234 return seals ? *seals : -EINVAL;
280 } 235 }
281 236
282 long memfd_fcntl(struct file *file, unsigned i !! 237 long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
283 { 238 {
284 long error; 239 long error;
285 240
286 switch (cmd) { 241 switch (cmd) {
287 case F_ADD_SEALS: 242 case F_ADD_SEALS:
>> 243 /* disallow upper 32bit */
>> 244 if (arg > UINT_MAX)
>> 245 return -EINVAL;
>> 246
288 error = memfd_add_seals(file, 247 error = memfd_add_seals(file, arg);
289 break; 248 break;
290 case F_GET_SEALS: 249 case F_GET_SEALS:
291 error = memfd_get_seals(file); 250 error = memfd_get_seals(file);
292 break; 251 break;
293 default: 252 default:
294 error = -EINVAL; 253 error = -EINVAL;
295 break; 254 break;
296 } 255 }
297 256
298 return error; 257 return error;
299 } 258 }
300 259
301 #define MFD_NAME_PREFIX "memfd:" 260 #define MFD_NAME_PREFIX "memfd:"
302 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_P 261 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
303 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_ 262 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
304 263
305 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW !! 264 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB)
306 <<
307 static int check_sysctl_memfd_noexec(unsigned <<
308 { <<
309 #ifdef CONFIG_SYSCTL <<
310 struct pid_namespace *ns = task_active <<
311 int sysctl = pidns_memfd_noexec_scope( <<
312 <<
313 if (!(*flags & (MFD_EXEC | MFD_NOEXEC_ <<
314 if (sysctl >= MEMFD_NOEXEC_SCO <<
315 *flags |= MFD_NOEXEC_S <<
316 else <<
317 *flags |= MFD_EXEC; <<
318 } <<
319 <<
320 if (!(*flags & MFD_NOEXEC_SEAL) && sys <<
321 pr_err_ratelimited( <<
322 "%s[%d]: memfd_create( <<
323 current->comm, task_pi <<
324 return -EACCES; <<
325 } <<
326 #endif <<
327 return 0; <<
328 } <<
329 265
330 SYSCALL_DEFINE2(memfd_create, 266 SYSCALL_DEFINE2(memfd_create,
331 const char __user *, uname, 267 const char __user *, uname,
332 unsigned int, flags) 268 unsigned int, flags)
333 { 269 {
334 unsigned int *file_seals; 270 unsigned int *file_seals;
335 struct file *file; 271 struct file *file;
336 int fd, error; 272 int fd, error;
337 char *name; 273 char *name;
338 long len; 274 long len;
339 275
340 if (!(flags & MFD_HUGETLB)) { 276 if (!(flags & MFD_HUGETLB)) {
341 if (flags & ~(unsigned int)MFD 277 if (flags & ~(unsigned int)MFD_ALL_FLAGS)
342 return -EINVAL; 278 return -EINVAL;
343 } else { 279 } else {
344 /* Allow huge page size encodi 280 /* Allow huge page size encoding in flags. */
345 if (flags & ~(unsigned int)(MF 281 if (flags & ~(unsigned int)(MFD_ALL_FLAGS |
346 (MFD_HUGE_MASK 282 (MFD_HUGE_MASK << MFD_HUGE_SHIFT)))
347 return -EINVAL; 283 return -EINVAL;
348 } 284 }
349 285
350 /* Invalid if both EXEC and NOEXEC_SEA <<
351 if ((flags & MFD_EXEC) && (flags & MFD <<
352 return -EINVAL; <<
353 <<
354 error = check_sysctl_memfd_noexec(&fla <<
355 if (error < 0) <<
356 return error; <<
357 <<
358 /* length includes terminating zero */ 286 /* length includes terminating zero */
359 len = strnlen_user(uname, MFD_NAME_MAX 287 len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
360 if (len <= 0) 288 if (len <= 0)
361 return -EFAULT; 289 return -EFAULT;
362 if (len > MFD_NAME_MAX_LEN + 1) 290 if (len > MFD_NAME_MAX_LEN + 1)
363 return -EINVAL; 291 return -EINVAL;
364 292
365 name = kmalloc(len + MFD_NAME_PREFIX_L 293 name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL);
366 if (!name) 294 if (!name)
367 return -ENOMEM; 295 return -ENOMEM;
368 296
369 strcpy(name, MFD_NAME_PREFIX); 297 strcpy(name, MFD_NAME_PREFIX);
370 if (copy_from_user(&name[MFD_NAME_PREF 298 if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
371 error = -EFAULT; 299 error = -EFAULT;
372 goto err_name; 300 goto err_name;
373 } 301 }
374 302
375 /* terminating-zero may have changed a 303 /* terminating-zero may have changed after strnlen_user() returned */
376 if (name[len + MFD_NAME_PREFIX_LEN - 1 304 if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
377 error = -EFAULT; 305 error = -EFAULT;
378 goto err_name; 306 goto err_name;
379 } 307 }
380 308
381 fd = get_unused_fd_flags((flags & MFD_ 309 fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
382 if (fd < 0) { 310 if (fd < 0) {
383 error = fd; 311 error = fd;
384 goto err_name; 312 goto err_name;
385 } 313 }
386 314
387 if (flags & MFD_HUGETLB) { 315 if (flags & MFD_HUGETLB) {
388 file = hugetlb_file_setup(name 316 file = hugetlb_file_setup(name, 0, VM_NORESERVE,
389 HUGETL 317 HUGETLB_ANONHUGE_INODE,
390 (flags 318 (flags >> MFD_HUGE_SHIFT) &
391 MFD_HU 319 MFD_HUGE_MASK);
392 } else 320 } else
393 file = shmem_file_setup(name, 321 file = shmem_file_setup(name, 0, VM_NORESERVE);
394 if (IS_ERR(file)) { 322 if (IS_ERR(file)) {
395 error = PTR_ERR(file); 323 error = PTR_ERR(file);
396 goto err_fd; 324 goto err_fd;
397 } 325 }
398 file->f_mode |= FMODE_LSEEK | FMODE_PR 326 file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
399 file->f_flags |= O_LARGEFILE; 327 file->f_flags |= O_LARGEFILE;
400 328
401 if (flags & MFD_NOEXEC_SEAL) { !! 329 if (flags & MFD_ALLOW_SEALING) {
402 struct inode *inode = file_ino <<
403 <<
404 inode->i_mode &= ~0111; <<
405 file_seals = memfd_file_seals_ <<
406 if (file_seals) { <<
407 *file_seals &= ~F_SEAL <<
408 *file_seals |= F_SEAL_ <<
409 } <<
410 } else if (flags & MFD_ALLOW_SEALING) <<
411 /* MFD_EXEC and MFD_ALLOW_SEAL <<
412 file_seals = memfd_file_seals_ 330 file_seals = memfd_file_seals_ptr(file);
413 if (file_seals) !! 331 *file_seals &= ~F_SEAL_SEAL;
414 *file_seals &= ~F_SEAL <<
415 } 332 }
416 333
417 fd_install(fd, file); 334 fd_install(fd, file);
418 kfree(name); 335 kfree(name);
419 return fd; 336 return fd;
420 337
421 err_fd: 338 err_fd:
422 put_unused_fd(fd); 339 put_unused_fd(fd);
423 err_name: 340 err_name:
424 kfree(name); 341 kfree(name);
425 return error; 342 return error;
426 } 343 }
427 344
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.