1 // SPDX-License-Identifier: GPL-2.0 1
2 /* Watch queue and general notification mechan
3 *
4 * Copyright (C) 2020 Red Hat, Inc. All Rights
5 * Written by David Howells (dhowells@redhat.c
6 *
7 * See Documentation/core-api/watch_queue.rst
8 */
9
10 #define pr_fmt(fmt) "watchq: " fmt
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/sched.h>
14 #include <linux/slab.h>
15 #include <linux/printk.h>
16 #include <linux/miscdevice.h>
17 #include <linux/fs.h>
18 #include <linux/mm.h>
19 #include <linux/pagemap.h>
20 #include <linux/poll.h>
21 #include <linux/uaccess.h>
22 #include <linux/vmalloc.h>
23 #include <linux/file.h>
24 #include <linux/security.h>
25 #include <linux/cred.h>
26 #include <linux/sched/signal.h>
27 #include <linux/watch_queue.h>
28 #include <linux/pipe_fs_i.h>
29
30 MODULE_DESCRIPTION("Watch queue");
31 MODULE_AUTHOR("Red Hat, Inc.");
32
33 #define WATCH_QUEUE_NOTE_SIZE 128
34 #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE
35
36 /*
37 * This must be called under the RCU read-lock
38 * sure that the wqueue still exists. It can t
39 * and check that the wqueue hasn't been destr
40 * turn makes sure that the notification pipe
41 */
42 static inline bool lock_wqueue(struct watch_qu
43 {
44 spin_lock_bh(&wqueue->lock);
45 if (unlikely(!wqueue->pipe)) {
46 spin_unlock_bh(&wqueue->lock);
47 return false;
48 }
49 return true;
50 }
51
52 static inline void unlock_wqueue(struct watch_
53 {
54 spin_unlock_bh(&wqueue->lock);
55 }
56
57 static void watch_queue_pipe_buf_release(struc
58 struc
59 {
60 struct watch_queue *wqueue = (struct w
61 struct page *page;
62 unsigned int bit;
63
64 /* We need to work out which note with
65 * the note might have been maximum si
66 * off doesn't work. OTOH, the note m
67 */
68 bit = buf->offset + buf->len;
69 if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1)
70 bit -= WATCH_QUEUE_NOTE_SIZE;
71 bit /= WATCH_QUEUE_NOTE_SIZE;
72
73 page = buf->page;
74 bit += page->index;
75
76 set_bit(bit, wqueue->notes_bitmap);
77 generic_pipe_buf_release(pipe, buf);
78 }
79
80 // No try_steal function => no stealing
81 #define watch_queue_pipe_buf_try_steal NULL
82
83 /* New data written to a pipe may be appended
84 static const struct pipe_buf_operations watch_
85 .release = watch_queue_pipe_buf
86 .try_steal = watch_queue_pipe_buf
87 .get = generic_pipe_buf_get
88 };
89
90 /*
91 * Post a notification to a watch queue.
92 *
93 * Must be called with the RCU lock for readin
94 * watch_queue lock held, which guarantees tha
95 * hasn't been released.
96 */
97 static bool post_one_notification(struct watch
98 struct watch
99 {
100 void *p;
101 struct pipe_inode_info *pipe = wqueue-
102 struct pipe_buffer *buf;
103 struct page *page;
104 unsigned int head, tail, mask, note, o
105 bool done = false;
106
107 spin_lock_irq(&pipe->rd_wait.lock);
108
109 mask = pipe->ring_size - 1;
110 head = pipe->head;
111 tail = pipe->tail;
112 if (pipe_full(head, tail, pipe->ring_s
113 goto lost;
114
115 note = find_first_bit(wqueue->notes_bi
116 if (note >= wqueue->nr_notes)
117 goto lost;
118
119 page = wqueue->notes[note / WATCH_QUEU
120 offset = note % WATCH_QUEUE_NOTES_PER_
121 get_page(page);
122 len = n->info & WATCH_INFO_LENGTH;
123 p = kmap_atomic(page);
124 memcpy(p + offset, n, len);
125 kunmap_atomic(p);
126
127 buf = &pipe->bufs[head & mask];
128 buf->page = page;
129 buf->private = (unsigned long)wqueue;
130 buf->ops = &watch_queue_pipe_buf_ops;
131 buf->offset = offset;
132 buf->len = len;
133 buf->flags = PIPE_BUF_FLAG_WHOLE;
134 smp_store_release(&pipe->head, head +
135
136 if (!test_and_clear_bit(note, wqueue->
137 spin_unlock_irq(&pipe->rd_wait
138 BUG();
139 }
140 wake_up_interruptible_sync_poll_locked
141 done = true;
142
143 out:
144 spin_unlock_irq(&pipe->rd_wait.lock);
145 if (done)
146 kill_fasync(&pipe->fasync_read
147 return done;
148
149 lost:
150 buf = &pipe->bufs[(head - 1) & mask];
151 buf->flags |= PIPE_BUF_FLAG_LOSS;
152 goto out;
153 }
154
155 /*
156 * Apply filter rules to a notification.
157 */
158 static bool filter_watch_notification(const st
159 const st
160 {
161 const struct watch_type_filter *wt;
162 unsigned int st_bits = sizeof(wt->subt
163 unsigned int st_index = n->subtype / s
164 unsigned int st_bit = 1U << (n->subtyp
165 int i;
166
167 if (!test_bit(n->type, wf->type_filter
168 return false;
169
170 for (i = 0; i < wf->nr_filters; i++) {
171 wt = &wf->filters[i];
172 if (n->type == wt->type &&
173 (wt->subtype_filter[st_ind
174 (n->info & wt->info_mask)
175 return true;
176 }
177
178 return false; /* If there is a filter,
179 }
180
181 /**
182 * __post_watch_notification - Post an event n
183 * @wlist: The watch list to post the event to
184 * @n: The notification record to post.
185 * @cred: The creds of the process that trigge
186 * @id: The ID to match on the watch.
187 *
188 * Post a notification of an event into a set
189 * know.
190 *
191 * The size of the notification should be set
192 * should be in units of sizeof(*n).
193 */
194 void __post_watch_notification(struct watch_li
195 struct watch_no
196 const struct cr
197 u64 id)
198 {
199 const struct watch_filter *wf;
200 struct watch_queue *wqueue;
201 struct watch *watch;
202
203 if (((n->info & WATCH_INFO_LENGTH) >>
204 WARN_ON(1);
205 return;
206 }
207
208 rcu_read_lock();
209
210 hlist_for_each_entry_rcu(watch, &wlist
211 if (watch->id != id)
212 continue;
213 n->info &= ~WATCH_INFO_ID;
214 n->info |= watch->info_id;
215
216 wqueue = rcu_dereference(watch
217 wf = rcu_dereference(wqueue->f
218 if (wf && !filter_watch_notifi
219 continue;
220
221 if (security_post_notification
222 continue;
223
224 if (lock_wqueue(wqueue)) {
225 post_one_notification(
226 unlock_wqueue(wqueue);
227 }
228 }
229
230 rcu_read_unlock();
231 }
232 EXPORT_SYMBOL(__post_watch_notification);
233
234 /*
235 * Allocate sufficient pages to preallocation
236 * notifications.
237 */
238 long watch_queue_set_size(struct pipe_inode_in
239 {
240 struct watch_queue *wqueue = pipe->wat
241 struct page **pages;
242 unsigned long *bitmap;
243 unsigned long user_bufs;
244 int ret, i, nr_pages;
245
246 if (!wqueue)
247 return -ENODEV;
248 if (wqueue->notes)
249 return -EBUSY;
250
251 if (nr_notes < 1 ||
252 nr_notes > 512) /* TODO: choose a
253 return -EINVAL;
254
255 nr_pages = (nr_notes + WATCH_QUEUE_NOT
256 nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE
257 user_bufs = account_pipe_buffers(pipe-
258
259 if (nr_pages > pipe->max_usage &&
260 (too_many_pipe_buffers_hard(user_b
261 too_many_pipe_buffers_soft(user_b
262 pipe_is_unprivileged_user()) {
263 ret = -EPERM;
264 goto error;
265 }
266
267 nr_notes = nr_pages * WATCH_QUEUE_NOTE
268 ret = pipe_resize_ring(pipe, roundup_p
269 if (ret < 0)
270 goto error;
271
272 ret = -ENOMEM;
273 pages = kcalloc(nr_pages, sizeof(struc
274 if (!pages)
275 goto error;
276
277 for (i = 0; i < nr_pages; i++) {
278 pages[i] = alloc_page(GFP_KERN
279 if (!pages[i])
280 goto error_p;
281 pages[i]->index = i * WATCH_QU
282 }
283
284 bitmap = bitmap_alloc(nr_notes, GFP_KE
285 if (!bitmap)
286 goto error_p;
287
288 bitmap_fill(bitmap, nr_notes);
289 wqueue->notes = pages;
290 wqueue->notes_bitmap = bitmap;
291 wqueue->nr_pages = nr_pages;
292 wqueue->nr_notes = nr_notes;
293 return 0;
294
295 error_p:
296 while (--i >= 0)
297 __free_page(pages[i]);
298 kfree(pages);
299 error:
300 (void) account_pipe_buffers(pipe->user
301 return ret;
302 }
303
304 /*
305 * Set the filter on a watch queue.
306 */
307 long watch_queue_set_filter(struct pipe_inode_
308 struct watch_notif
309 {
310 struct watch_notification_type_filter
311 struct watch_notification_filter filte
312 struct watch_type_filter *q;
313 struct watch_filter *wfilter;
314 struct watch_queue *wqueue = pipe->wat
315 int ret, nr_filter = 0, i;
316
317 if (!wqueue)
318 return -ENODEV;
319
320 if (!_filter) {
321 /* Remove the old filter */
322 wfilter = NULL;
323 goto set;
324 }
325
326 /* Grab the user's filter specificatio
327 if (copy_from_user(&filter, _filter, s
328 return -EFAULT;
329 if (filter.nr_filters == 0 ||
330 filter.nr_filters > 16 ||
331 filter.__reserved != 0)
332 return -EINVAL;
333
334 tf = memdup_array_user(_filter->filter
335 if (IS_ERR(tf))
336 return PTR_ERR(tf);
337
338 ret = -EINVAL;
339 for (i = 0; i < filter.nr_filters; i++
340 if ((tf[i].info_filter & ~tf[i
341 tf[i].info_mask & WATCH_IN
342 goto err_filter;
343 /* Ignore any unknown types */
344 if (tf[i].type >= WATCH_TYPE__
345 continue;
346 nr_filter++;
347 }
348
349 /* Now we need to build the internal f
350 * user-specified filters.
351 */
352 ret = -ENOMEM;
353 wfilter = kzalloc(struct_size(wfilter,
354 if (!wfilter)
355 goto err_filter;
356 wfilter->nr_filters = nr_filter;
357
358 q = wfilter->filters;
359 for (i = 0; i < filter.nr_filters; i++
360 if (tf[i].type >= WATCH_TYPE__
361 continue;
362
363 q->type = tf[i
364 q->info_filter = tf[i
365 q->info_mask = tf[i
366 q->subtype_filter[0] = tf[i
367 __set_bit(q->type, wfilter->ty
368 q++;
369 }
370
371 kfree(tf);
372 set:
373 pipe_lock(pipe);
374 wfilter = rcu_replace_pointer(wqueue->
375 lockdep_
376 pipe_unlock(pipe);
377 if (wfilter)
378 kfree_rcu(wfilter, rcu);
379 return 0;
380
381 err_filter:
382 kfree(tf);
383 return ret;
384 }
385
386 static void __put_watch_queue(struct kref *kre
387 {
388 struct watch_queue *wqueue =
389 container_of(kref, struct watc
390 struct watch_filter *wfilter;
391 int i;
392
393 for (i = 0; i < wqueue->nr_pages; i++)
394 __free_page(wqueue->notes[i]);
395 kfree(wqueue->notes);
396 bitmap_free(wqueue->notes_bitmap);
397
398 wfilter = rcu_access_pointer(wqueue->f
399 if (wfilter)
400 kfree_rcu(wfilter, rcu);
401 kfree_rcu(wqueue, rcu);
402 }
403
404 /**
405 * put_watch_queue - Dispose of a ref on a wat
406 * @wqueue: The watch queue to unref.
407 */
408 void put_watch_queue(struct watch_queue *wqueu
409 {
410 kref_put(&wqueue->usage, __put_watch_q
411 }
412 EXPORT_SYMBOL(put_watch_queue);
413
414 static void free_watch(struct rcu_head *rcu)
415 {
416 struct watch *watch = container_of(rcu
417
418 put_watch_queue(rcu_access_pointer(wat
419 atomic_dec(&watch->cred->user->nr_watc
420 put_cred(watch->cred);
421 kfree(watch);
422 }
423
424 static void __put_watch(struct kref *kref)
425 {
426 struct watch *watch = container_of(kre
427
428 call_rcu(&watch->rcu, free_watch);
429 }
430
431 /*
432 * Discard a watch.
433 */
434 static void put_watch(struct watch *watch)
435 {
436 kref_put(&watch->usage, __put_watch);
437 }
438
439 /**
440 * init_watch - Initialise a watch
441 * @watch: The watch to initialise.
442 * @wqueue: The queue to assign.
443 *
444 * Initialise a watch and set the watch queue.
445 */
446 void init_watch(struct watch *watch, struct wa
447 {
448 kref_init(&watch->usage);
449 INIT_HLIST_NODE(&watch->list_node);
450 INIT_HLIST_NODE(&watch->queue_node);
451 rcu_assign_pointer(watch->queue, wqueu
452 }
453
454 static int add_one_watch(struct watch *watch,
455 {
456 const struct cred *cred;
457 struct watch *w;
458
459 hlist_for_each_entry(w, &wlist->watche
460 struct watch_queue *wq = rcu_a
461 if (wqueue == wq && watch->id
462 return -EBUSY;
463 }
464
465 cred = current_cred();
466 if (atomic_inc_return(&cred->user->nr_
467 atomic_dec(&cred->user->nr_wat
468 return -EAGAIN;
469 }
470
471 watch->cred = get_cred(cred);
472 rcu_assign_pointer(watch->watch_list,
473
474 kref_get(&wqueue->usage);
475 kref_get(&watch->usage);
476 hlist_add_head(&watch->queue_node, &wq
477 hlist_add_head_rcu(&watch->list_node,
478 return 0;
479 }
480
481 /**
482 * add_watch_to_object - Add a watch on an obj
483 * @watch: The watch to add
484 * @wlist: The watch list to add to
485 *
486 * @watch->queue must have been set to point t
487 * to and the watch list of the object to be w
488 * have been set to the appropriate credential
489 *
490 * The caller must pin the queue and the list
491 * locked against racing watch additions/remov
492 */
493 int add_watch_to_object(struct watch *watch, s
494 {
495 struct watch_queue *wqueue;
496 int ret = -ENOENT;
497
498 rcu_read_lock();
499
500 wqueue = rcu_access_pointer(watch->que
501 if (lock_wqueue(wqueue)) {
502 spin_lock(&wlist->lock);
503 ret = add_one_watch(watch, wli
504 spin_unlock(&wlist->lock);
505 unlock_wqueue(wqueue);
506 }
507
508 rcu_read_unlock();
509 return ret;
510 }
511 EXPORT_SYMBOL(add_watch_to_object);
512
513 /**
514 * remove_watch_from_object - Remove a watch o
515 * @wlist: The watch list to remove from
516 * @wq: The watch queue of interest (ignored i
517 * @id: The ID of the watch to remove (ignored
518 * @all: True to remove all objects
519 *
520 * Remove a specific watch or all watches from
521 * sent to the watcher to tell them that this
522 */
523 int remove_watch_from_object(struct watch_list
524 u64 id, bool all)
525 {
526 struct watch_notification_removal n;
527 struct watch_queue *wqueue;
528 struct watch *watch;
529 int ret = -EBADSLT;
530
531 rcu_read_lock();
532
533 again:
534 spin_lock(&wlist->lock);
535 hlist_for_each_entry(watch, &wlist->wa
536 if (all ||
537 (watch->id == id && rcu_ac
538 goto found;
539 }
540 spin_unlock(&wlist->lock);
541 goto out;
542
543 found:
544 ret = 0;
545 hlist_del_init_rcu(&watch->list_node);
546 rcu_assign_pointer(watch->watch_list,
547 spin_unlock(&wlist->lock);
548
549 /* We now own the reference on watch t
550
551 n.watch.type = WATCH_TYPE_META;
552 n.watch.subtype = WATCH_META_REMOVAL_N
553 n.watch.info = watch->info_id | watch_
554 n.id = id;
555 if (id != 0)
556 n.watch.info = watch->info_id
557
558 wqueue = rcu_dereference(watch->queue)
559
560 if (lock_wqueue(wqueue)) {
561 post_one_notification(wqueue,
562
563 if (!hlist_unhashed(&watch->qu
564 hlist_del_init_rcu(&wa
565 put_watch(watch);
566 }
567
568 unlock_wqueue(wqueue);
569 }
570
571 if (wlist->release_watch) {
572 void (*release_watch)(struct w
573
574 release_watch = wlist->release
575 rcu_read_unlock();
576 (*release_watch)(watch);
577 rcu_read_lock();
578 }
579 put_watch(watch);
580
581 if (all && !hlist_empty(&wlist->watche
582 goto again;
583 out:
584 rcu_read_unlock();
585 return ret;
586 }
587 EXPORT_SYMBOL(remove_watch_from_object);
588
589 /*
590 * Remove all the watches that are contributor
591 * potential to race with removal of the watch
592 * objects being watched or with the distribut
593 */
594 void watch_queue_clear(struct watch_queue *wqu
595 {
596 struct watch_list *wlist;
597 struct watch *watch;
598 bool release;
599
600 rcu_read_lock();
601 spin_lock_bh(&wqueue->lock);
602
603 /*
604 * This pipe can be freed by callers l
605 * Removing this reference also preven
606 */
607 wqueue->pipe = NULL;
608
609 while (!hlist_empty(&wqueue->watches))
610 watch = hlist_entry(wqueue->wa
611 hlist_del_init_rcu(&watch->que
612 /* We now own a ref on the wat
613 spin_unlock_bh(&wqueue->lock);
614
615 /* We can't do the next bit un
616 * get the list lock - which w
617 * was removing from the oppos
618 * posting a notification.
619 */
620 wlist = rcu_dereference(watch-
621 if (wlist) {
622 void (*release_watch)(
623
624 spin_lock(&wlist->lock
625
626 release = !hlist_unhas
627 if (release) {
628 hlist_del_init
629 rcu_assign_poi
630
631 /* We now own
632 }
633
634 release_watch = wlist-
635 spin_unlock(&wlist->lo
636
637 if (release) {
638 if (release_wa
639 rcu_re
640 /* Thi
641 * we
642 */
643 (*rele
644 rcu_re
645 }
646 put_watch(watc
647 }
648 }
649
650 put_watch(watch);
651 spin_lock_bh(&wqueue->lock);
652 }
653
654 spin_unlock_bh(&wqueue->lock);
655 rcu_read_unlock();
656 }
657
658 /**
659 * get_watch_queue - Get a watch queue from it
660 * @fd: The fd to query.
661 */
662 struct watch_queue *get_watch_queue(int fd)
663 {
664 struct pipe_inode_info *pipe;
665 struct watch_queue *wqueue = ERR_PTR(-
666 struct fd f;
667
668 f = fdget(fd);
669 if (fd_file(f)) {
670 pipe = get_pipe_info(fd_file(f
671 if (pipe && pipe->watch_queue)
672 wqueue = pipe->watch_q
673 kref_get(&wqueue->usag
674 }
675 fdput(f);
676 }
677
678 return wqueue;
679 }
680 EXPORT_SYMBOL(get_watch_queue);
681
682 /*
683 * Initialise a watch queue
684 */
685 int watch_queue_init(struct pipe_inode_info *p
686 {
687 struct watch_queue *wqueue;
688
689 wqueue = kzalloc(sizeof(*wqueue), GFP_
690 if (!wqueue)
691 return -ENOMEM;
692
693 wqueue->pipe = pipe;
694 kref_init(&wqueue->usage);
695 spin_lock_init(&wqueue->lock);
696 INIT_HLIST_HEAD(&wqueue->watches);
697
698 pipe->watch_queue = wqueue;
699 return 0;
700 }
701
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.