1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 /* Watch queue and general notification mechan 2 /* Watch queue and general notification mechanism, built on pipes
3 * 3 *
4 * Copyright (C) 2020 Red Hat, Inc. All Rights 4 * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.c 5 * Written by David Howells (dhowells@redhat.com)
6 * 6 *
7 * See Documentation/core-api/watch_queue.rst !! 7 * See Documentation/watch_queue.rst
8 */ 8 */
9 9
10 #define pr_fmt(fmt) "watchq: " fmt 10 #define pr_fmt(fmt) "watchq: " fmt
11 #include <linux/module.h> 11 #include <linux/module.h>
12 #include <linux/init.h> 12 #include <linux/init.h>
13 #include <linux/sched.h> 13 #include <linux/sched.h>
14 #include <linux/slab.h> 14 #include <linux/slab.h>
15 #include <linux/printk.h> 15 #include <linux/printk.h>
16 #include <linux/miscdevice.h> 16 #include <linux/miscdevice.h>
17 #include <linux/fs.h> 17 #include <linux/fs.h>
18 #include <linux/mm.h> 18 #include <linux/mm.h>
19 #include <linux/pagemap.h> 19 #include <linux/pagemap.h>
20 #include <linux/poll.h> 20 #include <linux/poll.h>
21 #include <linux/uaccess.h> 21 #include <linux/uaccess.h>
22 #include <linux/vmalloc.h> 22 #include <linux/vmalloc.h>
23 #include <linux/file.h> 23 #include <linux/file.h>
24 #include <linux/security.h> 24 #include <linux/security.h>
25 #include <linux/cred.h> 25 #include <linux/cred.h>
26 #include <linux/sched/signal.h> 26 #include <linux/sched/signal.h>
27 #include <linux/watch_queue.h> 27 #include <linux/watch_queue.h>
28 #include <linux/pipe_fs_i.h> 28 #include <linux/pipe_fs_i.h>
29 29
30 MODULE_DESCRIPTION("Watch queue"); 30 MODULE_DESCRIPTION("Watch queue");
31 MODULE_AUTHOR("Red Hat, Inc."); 31 MODULE_AUTHOR("Red Hat, Inc.");
>> 32 MODULE_LICENSE("GPL");
32 33
33 #define WATCH_QUEUE_NOTE_SIZE 128 34 #define WATCH_QUEUE_NOTE_SIZE 128
34 #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE 35 #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE / WATCH_QUEUE_NOTE_SIZE)
35 36
36 /* <<
37 * This must be called under the RCU read-lock <<
38 * sure that the wqueue still exists. It can t <<
39 * and check that the wqueue hasn't been destr <<
40 * turn makes sure that the notification pipe <<
41 */ <<
42 static inline bool lock_wqueue(struct watch_qu <<
43 { <<
44 spin_lock_bh(&wqueue->lock); <<
45 if (unlikely(!wqueue->pipe)) { <<
46 spin_unlock_bh(&wqueue->lock); <<
47 return false; <<
48 } <<
49 return true; <<
50 } <<
51 <<
52 static inline void unlock_wqueue(struct watch_ <<
53 { <<
54 spin_unlock_bh(&wqueue->lock); <<
55 } <<
56 <<
57 static void watch_queue_pipe_buf_release(struc 37 static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe,
58 struc 38 struct pipe_buffer *buf)
59 { 39 {
60 struct watch_queue *wqueue = (struct w 40 struct watch_queue *wqueue = (struct watch_queue *)buf->private;
61 struct page *page; 41 struct page *page;
62 unsigned int bit; 42 unsigned int bit;
63 43
64 /* We need to work out which note with 44 /* We need to work out which note within the page this refers to, but
65 * the note might have been maximum si 45 * the note might have been maximum size, so merely ANDing the offset
66 * off doesn't work. OTOH, the note m 46 * off doesn't work. OTOH, the note must've been more than zero size.
67 */ 47 */
68 bit = buf->offset + buf->len; 48 bit = buf->offset + buf->len;
69 if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1) 49 if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1)) == 0)
70 bit -= WATCH_QUEUE_NOTE_SIZE; 50 bit -= WATCH_QUEUE_NOTE_SIZE;
71 bit /= WATCH_QUEUE_NOTE_SIZE; 51 bit /= WATCH_QUEUE_NOTE_SIZE;
72 52
73 page = buf->page; 53 page = buf->page;
74 bit += page->index; 54 bit += page->index;
75 55
76 set_bit(bit, wqueue->notes_bitmap); 56 set_bit(bit, wqueue->notes_bitmap);
77 generic_pipe_buf_release(pipe, buf); <<
78 } 57 }
79 58
80 // No try_steal function => no stealing 59 // No try_steal function => no stealing
81 #define watch_queue_pipe_buf_try_steal NULL 60 #define watch_queue_pipe_buf_try_steal NULL
82 61
83 /* New data written to a pipe may be appended 62 /* New data written to a pipe may be appended to a buffer with this type. */
84 static const struct pipe_buf_operations watch_ 63 static const struct pipe_buf_operations watch_queue_pipe_buf_ops = {
85 .release = watch_queue_pipe_buf 64 .release = watch_queue_pipe_buf_release,
86 .try_steal = watch_queue_pipe_buf 65 .try_steal = watch_queue_pipe_buf_try_steal,
87 .get = generic_pipe_buf_get 66 .get = generic_pipe_buf_get,
88 }; 67 };
89 68
90 /* 69 /*
91 * Post a notification to a watch queue. 70 * Post a notification to a watch queue.
92 * <<
93 * Must be called with the RCU lock for readin <<
94 * watch_queue lock held, which guarantees tha <<
95 * hasn't been released. <<
96 */ 71 */
97 static bool post_one_notification(struct watch 72 static bool post_one_notification(struct watch_queue *wqueue,
98 struct watch 73 struct watch_notification *n)
99 { 74 {
100 void *p; 75 void *p;
101 struct pipe_inode_info *pipe = wqueue- 76 struct pipe_inode_info *pipe = wqueue->pipe;
102 struct pipe_buffer *buf; 77 struct pipe_buffer *buf;
103 struct page *page; 78 struct page *page;
104 unsigned int head, tail, mask, note, o 79 unsigned int head, tail, mask, note, offset, len;
105 bool done = false; 80 bool done = false;
106 81
>> 82 if (!pipe)
>> 83 return false;
>> 84
107 spin_lock_irq(&pipe->rd_wait.lock); 85 spin_lock_irq(&pipe->rd_wait.lock);
108 86
>> 87 if (wqueue->defunct)
>> 88 goto out;
>> 89
109 mask = pipe->ring_size - 1; 90 mask = pipe->ring_size - 1;
110 head = pipe->head; 91 head = pipe->head;
111 tail = pipe->tail; 92 tail = pipe->tail;
112 if (pipe_full(head, tail, pipe->ring_s 93 if (pipe_full(head, tail, pipe->ring_size))
113 goto lost; 94 goto lost;
114 95
115 note = find_first_bit(wqueue->notes_bi 96 note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);
116 if (note >= wqueue->nr_notes) 97 if (note >= wqueue->nr_notes)
117 goto lost; 98 goto lost;
118 99
119 page = wqueue->notes[note / WATCH_QUEU 100 page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE];
120 offset = note % WATCH_QUEUE_NOTES_PER_ 101 offset = note % WATCH_QUEUE_NOTES_PER_PAGE * WATCH_QUEUE_NOTE_SIZE;
121 get_page(page); 102 get_page(page);
122 len = n->info & WATCH_INFO_LENGTH; 103 len = n->info & WATCH_INFO_LENGTH;
123 p = kmap_atomic(page); 104 p = kmap_atomic(page);
124 memcpy(p + offset, n, len); 105 memcpy(p + offset, n, len);
125 kunmap_atomic(p); 106 kunmap_atomic(p);
126 107
127 buf = &pipe->bufs[head & mask]; 108 buf = &pipe->bufs[head & mask];
128 buf->page = page; 109 buf->page = page;
129 buf->private = (unsigned long)wqueue; 110 buf->private = (unsigned long)wqueue;
130 buf->ops = &watch_queue_pipe_buf_ops; 111 buf->ops = &watch_queue_pipe_buf_ops;
131 buf->offset = offset; 112 buf->offset = offset;
132 buf->len = len; 113 buf->len = len;
133 buf->flags = PIPE_BUF_FLAG_WHOLE; 114 buf->flags = PIPE_BUF_FLAG_WHOLE;
134 smp_store_release(&pipe->head, head + !! 115 pipe->head = head + 1;
135 116
136 if (!test_and_clear_bit(note, wqueue-> 117 if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {
137 spin_unlock_irq(&pipe->rd_wait 118 spin_unlock_irq(&pipe->rd_wait.lock);
138 BUG(); 119 BUG();
139 } 120 }
140 wake_up_interruptible_sync_poll_locked 121 wake_up_interruptible_sync_poll_locked(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
141 done = true; 122 done = true;
142 123
143 out: 124 out:
144 spin_unlock_irq(&pipe->rd_wait.lock); 125 spin_unlock_irq(&pipe->rd_wait.lock);
145 if (done) 126 if (done)
146 kill_fasync(&pipe->fasync_read 127 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
147 return done; 128 return done;
148 129
149 lost: 130 lost:
150 buf = &pipe->bufs[(head - 1) & mask]; 131 buf = &pipe->bufs[(head - 1) & mask];
151 buf->flags |= PIPE_BUF_FLAG_LOSS; 132 buf->flags |= PIPE_BUF_FLAG_LOSS;
152 goto out; 133 goto out;
153 } 134 }
154 135
155 /* 136 /*
156 * Apply filter rules to a notification. 137 * Apply filter rules to a notification.
157 */ 138 */
158 static bool filter_watch_notification(const st 139 static bool filter_watch_notification(const struct watch_filter *wf,
159 const st 140 const struct watch_notification *n)
160 { 141 {
161 const struct watch_type_filter *wt; 142 const struct watch_type_filter *wt;
162 unsigned int st_bits = sizeof(wt->subt 143 unsigned int st_bits = sizeof(wt->subtype_filter[0]) * 8;
163 unsigned int st_index = n->subtype / s 144 unsigned int st_index = n->subtype / st_bits;
164 unsigned int st_bit = 1U << (n->subtyp 145 unsigned int st_bit = 1U << (n->subtype % st_bits);
165 int i; 146 int i;
166 147
167 if (!test_bit(n->type, wf->type_filter 148 if (!test_bit(n->type, wf->type_filter))
168 return false; 149 return false;
169 150
170 for (i = 0; i < wf->nr_filters; i++) { 151 for (i = 0; i < wf->nr_filters; i++) {
171 wt = &wf->filters[i]; 152 wt = &wf->filters[i];
172 if (n->type == wt->type && 153 if (n->type == wt->type &&
173 (wt->subtype_filter[st_ind 154 (wt->subtype_filter[st_index] & st_bit) &&
174 (n->info & wt->info_mask) 155 (n->info & wt->info_mask) == wt->info_filter)
175 return true; 156 return true;
176 } 157 }
177 158
178 return false; /* If there is a filter, 159 return false; /* If there is a filter, the default is to reject. */
179 } 160 }
180 161
181 /** 162 /**
182 * __post_watch_notification - Post an event n 163 * __post_watch_notification - Post an event notification
183 * @wlist: The watch list to post the event to 164 * @wlist: The watch list to post the event to.
184 * @n: The notification record to post. 165 * @n: The notification record to post.
185 * @cred: The creds of the process that trigge 166 * @cred: The creds of the process that triggered the notification.
186 * @id: The ID to match on the watch. 167 * @id: The ID to match on the watch.
187 * 168 *
188 * Post a notification of an event into a set 169 * Post a notification of an event into a set of watch queues and let the users
189 * know. 170 * know.
190 * 171 *
191 * The size of the notification should be set 172 * The size of the notification should be set in n->info & WATCH_INFO_LENGTH and
192 * should be in units of sizeof(*n). 173 * should be in units of sizeof(*n).
193 */ 174 */
194 void __post_watch_notification(struct watch_li 175 void __post_watch_notification(struct watch_list *wlist,
195 struct watch_no 176 struct watch_notification *n,
196 const struct cr 177 const struct cred *cred,
197 u64 id) 178 u64 id)
198 { 179 {
199 const struct watch_filter *wf; 180 const struct watch_filter *wf;
200 struct watch_queue *wqueue; 181 struct watch_queue *wqueue;
201 struct watch *watch; 182 struct watch *watch;
202 183
203 if (((n->info & WATCH_INFO_LENGTH) >> 184 if (((n->info & WATCH_INFO_LENGTH) >> WATCH_INFO_LENGTH__SHIFT) == 0) {
204 WARN_ON(1); 185 WARN_ON(1);
205 return; 186 return;
206 } 187 }
207 188
208 rcu_read_lock(); 189 rcu_read_lock();
209 190
210 hlist_for_each_entry_rcu(watch, &wlist 191 hlist_for_each_entry_rcu(watch, &wlist->watchers, list_node) {
211 if (watch->id != id) 192 if (watch->id != id)
212 continue; 193 continue;
213 n->info &= ~WATCH_INFO_ID; 194 n->info &= ~WATCH_INFO_ID;
214 n->info |= watch->info_id; 195 n->info |= watch->info_id;
215 196
216 wqueue = rcu_dereference(watch 197 wqueue = rcu_dereference(watch->queue);
217 wf = rcu_dereference(wqueue->f 198 wf = rcu_dereference(wqueue->filter);
218 if (wf && !filter_watch_notifi 199 if (wf && !filter_watch_notification(wf, n))
219 continue; 200 continue;
220 201
221 if (security_post_notification 202 if (security_post_notification(watch->cred, cred, n) < 0)
222 continue; 203 continue;
223 204
224 if (lock_wqueue(wqueue)) { !! 205 post_one_notification(wqueue, n);
225 post_one_notification( <<
226 unlock_wqueue(wqueue); <<
227 } <<
228 } 206 }
229 207
230 rcu_read_unlock(); 208 rcu_read_unlock();
231 } 209 }
232 EXPORT_SYMBOL(__post_watch_notification); 210 EXPORT_SYMBOL(__post_watch_notification);
233 211
234 /* 212 /*
235 * Allocate sufficient pages to preallocation 213 * Allocate sufficient pages to preallocation for the requested number of
236 * notifications. 214 * notifications.
237 */ 215 */
238 long watch_queue_set_size(struct pipe_inode_in 216 long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes)
239 { 217 {
240 struct watch_queue *wqueue = pipe->wat 218 struct watch_queue *wqueue = pipe->watch_queue;
241 struct page **pages; 219 struct page **pages;
242 unsigned long *bitmap; 220 unsigned long *bitmap;
243 unsigned long user_bufs; 221 unsigned long user_bufs;
>> 222 unsigned int bmsize;
244 int ret, i, nr_pages; 223 int ret, i, nr_pages;
245 224
246 if (!wqueue) 225 if (!wqueue)
247 return -ENODEV; 226 return -ENODEV;
248 if (wqueue->notes) 227 if (wqueue->notes)
249 return -EBUSY; 228 return -EBUSY;
250 229
251 if (nr_notes < 1 || 230 if (nr_notes < 1 ||
252 nr_notes > 512) /* TODO: choose a 231 nr_notes > 512) /* TODO: choose a better hard limit */
253 return -EINVAL; 232 return -EINVAL;
254 233
255 nr_pages = (nr_notes + WATCH_QUEUE_NOT 234 nr_pages = (nr_notes + WATCH_QUEUE_NOTES_PER_PAGE - 1);
256 nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE 235 nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE;
257 user_bufs = account_pipe_buffers(pipe- 236 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_pages);
258 237
259 if (nr_pages > pipe->max_usage && 238 if (nr_pages > pipe->max_usage &&
260 (too_many_pipe_buffers_hard(user_b 239 (too_many_pipe_buffers_hard(user_bufs) ||
261 too_many_pipe_buffers_soft(user_b 240 too_many_pipe_buffers_soft(user_bufs)) &&
262 pipe_is_unprivileged_user()) { 241 pipe_is_unprivileged_user()) {
263 ret = -EPERM; 242 ret = -EPERM;
264 goto error; 243 goto error;
265 } 244 }
266 245
267 nr_notes = nr_pages * WATCH_QUEUE_NOTE !! 246 ret = pipe_resize_ring(pipe, nr_notes);
268 ret = pipe_resize_ring(pipe, roundup_p <<
269 if (ret < 0) 247 if (ret < 0)
270 goto error; 248 goto error;
271 249
272 ret = -ENOMEM; !! 250 pages = kcalloc(sizeof(struct page *), nr_pages, GFP_KERNEL);
273 pages = kcalloc(nr_pages, sizeof(struc <<
274 if (!pages) 251 if (!pages)
275 goto error; 252 goto error;
276 253
277 for (i = 0; i < nr_pages; i++) { 254 for (i = 0; i < nr_pages; i++) {
278 pages[i] = alloc_page(GFP_KERN 255 pages[i] = alloc_page(GFP_KERNEL);
279 if (!pages[i]) 256 if (!pages[i])
280 goto error_p; 257 goto error_p;
281 pages[i]->index = i * WATCH_QU 258 pages[i]->index = i * WATCH_QUEUE_NOTES_PER_PAGE;
282 } 259 }
283 260
284 bitmap = bitmap_alloc(nr_notes, GFP_KE !! 261 bmsize = (nr_notes + BITS_PER_LONG - 1) / BITS_PER_LONG;
>> 262 bmsize *= sizeof(unsigned long);
>> 263 bitmap = kmalloc(bmsize, GFP_KERNEL);
285 if (!bitmap) 264 if (!bitmap)
286 goto error_p; 265 goto error_p;
287 266
288 bitmap_fill(bitmap, nr_notes); !! 267 memset(bitmap, 0xff, bmsize);
289 wqueue->notes = pages; 268 wqueue->notes = pages;
290 wqueue->notes_bitmap = bitmap; 269 wqueue->notes_bitmap = bitmap;
291 wqueue->nr_pages = nr_pages; 270 wqueue->nr_pages = nr_pages;
292 wqueue->nr_notes = nr_notes; !! 271 wqueue->nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE;
293 return 0; 272 return 0;
294 273
295 error_p: 274 error_p:
296 while (--i >= 0) !! 275 for (i = 0; i < nr_pages; i++)
297 __free_page(pages[i]); 276 __free_page(pages[i]);
298 kfree(pages); 277 kfree(pages);
299 error: 278 error:
300 (void) account_pipe_buffers(pipe->user 279 (void) account_pipe_buffers(pipe->user, nr_pages, pipe->nr_accounted);
301 return ret; 280 return ret;
302 } 281 }
303 282
304 /* 283 /*
305 * Set the filter on a watch queue. 284 * Set the filter on a watch queue.
306 */ 285 */
307 long watch_queue_set_filter(struct pipe_inode_ 286 long watch_queue_set_filter(struct pipe_inode_info *pipe,
308 struct watch_notif 287 struct watch_notification_filter __user *_filter)
309 { 288 {
310 struct watch_notification_type_filter 289 struct watch_notification_type_filter *tf;
311 struct watch_notification_filter filte 290 struct watch_notification_filter filter;
312 struct watch_type_filter *q; 291 struct watch_type_filter *q;
313 struct watch_filter *wfilter; 292 struct watch_filter *wfilter;
314 struct watch_queue *wqueue = pipe->wat 293 struct watch_queue *wqueue = pipe->watch_queue;
315 int ret, nr_filter = 0, i; 294 int ret, nr_filter = 0, i;
316 295
317 if (!wqueue) 296 if (!wqueue)
318 return -ENODEV; 297 return -ENODEV;
319 298
320 if (!_filter) { 299 if (!_filter) {
321 /* Remove the old filter */ 300 /* Remove the old filter */
322 wfilter = NULL; 301 wfilter = NULL;
323 goto set; 302 goto set;
324 } 303 }
325 304
326 /* Grab the user's filter specificatio 305 /* Grab the user's filter specification */
327 if (copy_from_user(&filter, _filter, s 306 if (copy_from_user(&filter, _filter, sizeof(filter)) != 0)
328 return -EFAULT; 307 return -EFAULT;
329 if (filter.nr_filters == 0 || 308 if (filter.nr_filters == 0 ||
330 filter.nr_filters > 16 || 309 filter.nr_filters > 16 ||
331 filter.__reserved != 0) 310 filter.__reserved != 0)
332 return -EINVAL; 311 return -EINVAL;
333 312
334 tf = memdup_array_user(_filter->filter !! 313 tf = memdup_user(_filter->filters, filter.nr_filters * sizeof(*tf));
335 if (IS_ERR(tf)) 314 if (IS_ERR(tf))
336 return PTR_ERR(tf); 315 return PTR_ERR(tf);
337 316
338 ret = -EINVAL; 317 ret = -EINVAL;
339 for (i = 0; i < filter.nr_filters; i++ 318 for (i = 0; i < filter.nr_filters; i++) {
340 if ((tf[i].info_filter & ~tf[i 319 if ((tf[i].info_filter & ~tf[i].info_mask) ||
341 tf[i].info_mask & WATCH_IN 320 tf[i].info_mask & WATCH_INFO_LENGTH)
342 goto err_filter; 321 goto err_filter;
343 /* Ignore any unknown types */ 322 /* Ignore any unknown types */
344 if (tf[i].type >= WATCH_TYPE__ !! 323 if (tf[i].type >= sizeof(wfilter->type_filter) * 8)
345 continue; 324 continue;
346 nr_filter++; 325 nr_filter++;
347 } 326 }
348 327
349 /* Now we need to build the internal f 328 /* Now we need to build the internal filter from only the relevant
350 * user-specified filters. 329 * user-specified filters.
351 */ 330 */
352 ret = -ENOMEM; 331 ret = -ENOMEM;
353 wfilter = kzalloc(struct_size(wfilter, 332 wfilter = kzalloc(struct_size(wfilter, filters, nr_filter), GFP_KERNEL);
354 if (!wfilter) 333 if (!wfilter)
355 goto err_filter; 334 goto err_filter;
356 wfilter->nr_filters = nr_filter; 335 wfilter->nr_filters = nr_filter;
357 336
358 q = wfilter->filters; 337 q = wfilter->filters;
359 for (i = 0; i < filter.nr_filters; i++ 338 for (i = 0; i < filter.nr_filters; i++) {
360 if (tf[i].type >= WATCH_TYPE__ !! 339 if (tf[i].type >= sizeof(wfilter->type_filter) * BITS_PER_LONG)
361 continue; 340 continue;
362 341
363 q->type = tf[i 342 q->type = tf[i].type;
364 q->info_filter = tf[i 343 q->info_filter = tf[i].info_filter;
365 q->info_mask = tf[i 344 q->info_mask = tf[i].info_mask;
366 q->subtype_filter[0] = tf[i 345 q->subtype_filter[0] = tf[i].subtype_filter[0];
367 __set_bit(q->type, wfilter->ty 346 __set_bit(q->type, wfilter->type_filter);
368 q++; 347 q++;
369 } 348 }
370 349
371 kfree(tf); 350 kfree(tf);
372 set: 351 set:
373 pipe_lock(pipe); 352 pipe_lock(pipe);
374 wfilter = rcu_replace_pointer(wqueue-> 353 wfilter = rcu_replace_pointer(wqueue->filter, wfilter,
375 lockdep_ 354 lockdep_is_held(&pipe->mutex));
376 pipe_unlock(pipe); 355 pipe_unlock(pipe);
377 if (wfilter) 356 if (wfilter)
378 kfree_rcu(wfilter, rcu); 357 kfree_rcu(wfilter, rcu);
379 return 0; 358 return 0;
380 359
381 err_filter: 360 err_filter:
382 kfree(tf); 361 kfree(tf);
383 return ret; 362 return ret;
384 } 363 }
385 364
386 static void __put_watch_queue(struct kref *kre 365 static void __put_watch_queue(struct kref *kref)
387 { 366 {
388 struct watch_queue *wqueue = 367 struct watch_queue *wqueue =
389 container_of(kref, struct watc 368 container_of(kref, struct watch_queue, usage);
390 struct watch_filter *wfilter; 369 struct watch_filter *wfilter;
391 int i; 370 int i;
392 371
393 for (i = 0; i < wqueue->nr_pages; i++) 372 for (i = 0; i < wqueue->nr_pages; i++)
394 __free_page(wqueue->notes[i]); 373 __free_page(wqueue->notes[i]);
395 kfree(wqueue->notes); <<
396 bitmap_free(wqueue->notes_bitmap); <<
397 374
398 wfilter = rcu_access_pointer(wqueue->f 375 wfilter = rcu_access_pointer(wqueue->filter);
399 if (wfilter) 376 if (wfilter)
400 kfree_rcu(wfilter, rcu); 377 kfree_rcu(wfilter, rcu);
401 kfree_rcu(wqueue, rcu); 378 kfree_rcu(wqueue, rcu);
402 } 379 }
403 380
404 /** 381 /**
405 * put_watch_queue - Dispose of a ref on a wat 382 * put_watch_queue - Dispose of a ref on a watchqueue.
406 * @wqueue: The watch queue to unref. 383 * @wqueue: The watch queue to unref.
407 */ 384 */
408 void put_watch_queue(struct watch_queue *wqueu 385 void put_watch_queue(struct watch_queue *wqueue)
409 { 386 {
410 kref_put(&wqueue->usage, __put_watch_q 387 kref_put(&wqueue->usage, __put_watch_queue);
411 } 388 }
412 EXPORT_SYMBOL(put_watch_queue); 389 EXPORT_SYMBOL(put_watch_queue);
413 390
414 static void free_watch(struct rcu_head *rcu) 391 static void free_watch(struct rcu_head *rcu)
415 { 392 {
416 struct watch *watch = container_of(rcu 393 struct watch *watch = container_of(rcu, struct watch, rcu);
417 394
418 put_watch_queue(rcu_access_pointer(wat 395 put_watch_queue(rcu_access_pointer(watch->queue));
419 atomic_dec(&watch->cred->user->nr_watc 396 atomic_dec(&watch->cred->user->nr_watches);
420 put_cred(watch->cred); 397 put_cred(watch->cred);
421 kfree(watch); <<
422 } 398 }
423 399
424 static void __put_watch(struct kref *kref) 400 static void __put_watch(struct kref *kref)
425 { 401 {
426 struct watch *watch = container_of(kre 402 struct watch *watch = container_of(kref, struct watch, usage);
427 403
428 call_rcu(&watch->rcu, free_watch); 404 call_rcu(&watch->rcu, free_watch);
429 } 405 }
430 406
431 /* 407 /*
432 * Discard a watch. 408 * Discard a watch.
433 */ 409 */
434 static void put_watch(struct watch *watch) 410 static void put_watch(struct watch *watch)
435 { 411 {
436 kref_put(&watch->usage, __put_watch); 412 kref_put(&watch->usage, __put_watch);
437 } 413 }
438 414
439 /** 415 /**
440 * init_watch - Initialise a watch !! 416 * init_watch_queue - Initialise a watch
441 * @watch: The watch to initialise. 417 * @watch: The watch to initialise.
442 * @wqueue: The queue to assign. 418 * @wqueue: The queue to assign.
443 * 419 *
444 * Initialise a watch and set the watch queue. 420 * Initialise a watch and set the watch queue.
445 */ 421 */
446 void init_watch(struct watch *watch, struct wa 422 void init_watch(struct watch *watch, struct watch_queue *wqueue)
447 { 423 {
448 kref_init(&watch->usage); 424 kref_init(&watch->usage);
449 INIT_HLIST_NODE(&watch->list_node); 425 INIT_HLIST_NODE(&watch->list_node);
450 INIT_HLIST_NODE(&watch->queue_node); 426 INIT_HLIST_NODE(&watch->queue_node);
451 rcu_assign_pointer(watch->queue, wqueu 427 rcu_assign_pointer(watch->queue, wqueue);
452 } 428 }
453 429
454 static int add_one_watch(struct watch *watch, <<
455 { <<
456 const struct cred *cred; <<
457 struct watch *w; <<
458 <<
459 hlist_for_each_entry(w, &wlist->watche <<
460 struct watch_queue *wq = rcu_a <<
461 if (wqueue == wq && watch->id <<
462 return -EBUSY; <<
463 } <<
464 <<
465 cred = current_cred(); <<
466 if (atomic_inc_return(&cred->user->nr_ <<
467 atomic_dec(&cred->user->nr_wat <<
468 return -EAGAIN; <<
469 } <<
470 <<
471 watch->cred = get_cred(cred); <<
472 rcu_assign_pointer(watch->watch_list, <<
473 <<
474 kref_get(&wqueue->usage); <<
475 kref_get(&watch->usage); <<
476 hlist_add_head(&watch->queue_node, &wq <<
477 hlist_add_head_rcu(&watch->list_node, <<
478 return 0; <<
479 } <<
480 <<
481 /** 430 /**
482 * add_watch_to_object - Add a watch on an obj 431 * add_watch_to_object - Add a watch on an object to a watch list
483 * @watch: The watch to add 432 * @watch: The watch to add
484 * @wlist: The watch list to add to 433 * @wlist: The watch list to add to
485 * 434 *
486 * @watch->queue must have been set to point t 435 * @watch->queue must have been set to point to the queue to post notifications
487 * to and the watch list of the object to be w 436 * to and the watch list of the object to be watched. @watch->cred must also
488 * have been set to the appropriate credential 437 * have been set to the appropriate credentials and a ref taken on them.
489 * 438 *
490 * The caller must pin the queue and the list 439 * The caller must pin the queue and the list both and must hold the list
491 * locked against racing watch additions/remov 440 * locked against racing watch additions/removals.
492 */ 441 */
493 int add_watch_to_object(struct watch *watch, s 442 int add_watch_to_object(struct watch *watch, struct watch_list *wlist)
494 { 443 {
495 struct watch_queue *wqueue; !! 444 struct watch_queue *wqueue = rcu_access_pointer(watch->queue);
496 int ret = -ENOENT; !! 445 struct watch *w;
497 446
498 rcu_read_lock(); !! 447 hlist_for_each_entry(w, &wlist->watchers, list_node) {
>> 448 struct watch_queue *wq = rcu_access_pointer(w->queue);
>> 449 if (wqueue == wq && watch->id == w->id)
>> 450 return -EBUSY;
>> 451 }
>> 452
>> 453 watch->cred = get_current_cred();
>> 454 rcu_assign_pointer(watch->watch_list, wlist);
499 455
500 wqueue = rcu_access_pointer(watch->que !! 456 if (atomic_inc_return(&watch->cred->user->nr_watches) >
501 if (lock_wqueue(wqueue)) { !! 457 task_rlimit(current, RLIMIT_NOFILE)) {
502 spin_lock(&wlist->lock); !! 458 atomic_dec(&watch->cred->user->nr_watches);
503 ret = add_one_watch(watch, wli !! 459 put_cred(watch->cred);
504 spin_unlock(&wlist->lock); !! 460 return -EAGAIN;
505 unlock_wqueue(wqueue); <<
506 } 461 }
507 462
508 rcu_read_unlock(); !! 463 spin_lock_bh(&wqueue->lock);
509 return ret; !! 464 kref_get(&wqueue->usage);
>> 465 kref_get(&watch->usage);
>> 466 hlist_add_head(&watch->queue_node, &wqueue->watches);
>> 467 spin_unlock_bh(&wqueue->lock);
>> 468
>> 469 hlist_add_head(&watch->list_node, &wlist->watchers);
>> 470 return 0;
510 } 471 }
511 EXPORT_SYMBOL(add_watch_to_object); 472 EXPORT_SYMBOL(add_watch_to_object);
512 473
513 /** 474 /**
514 * remove_watch_from_object - Remove a watch o 475 * remove_watch_from_object - Remove a watch or all watches from an object.
515 * @wlist: The watch list to remove from 476 * @wlist: The watch list to remove from
516 * @wq: The watch queue of interest (ignored i 477 * @wq: The watch queue of interest (ignored if @all is true)
517 * @id: The ID of the watch to remove (ignored 478 * @id: The ID of the watch to remove (ignored if @all is true)
518 * @all: True to remove all objects 479 * @all: True to remove all objects
519 * 480 *
520 * Remove a specific watch or all watches from 481 * Remove a specific watch or all watches from an object. A notification is
521 * sent to the watcher to tell them that this 482 * sent to the watcher to tell them that this happened.
522 */ 483 */
523 int remove_watch_from_object(struct watch_list 484 int remove_watch_from_object(struct watch_list *wlist, struct watch_queue *wq,
524 u64 id, bool all) 485 u64 id, bool all)
525 { 486 {
526 struct watch_notification_removal n; 487 struct watch_notification_removal n;
527 struct watch_queue *wqueue; 488 struct watch_queue *wqueue;
528 struct watch *watch; 489 struct watch *watch;
529 int ret = -EBADSLT; 490 int ret = -EBADSLT;
530 491
531 rcu_read_lock(); 492 rcu_read_lock();
532 493
533 again: 494 again:
534 spin_lock(&wlist->lock); 495 spin_lock(&wlist->lock);
535 hlist_for_each_entry(watch, &wlist->wa 496 hlist_for_each_entry(watch, &wlist->watchers, list_node) {
536 if (all || 497 if (all ||
537 (watch->id == id && rcu_ac 498 (watch->id == id && rcu_access_pointer(watch->queue) == wq))
538 goto found; 499 goto found;
539 } 500 }
540 spin_unlock(&wlist->lock); 501 spin_unlock(&wlist->lock);
541 goto out; 502 goto out;
542 503
543 found: 504 found:
544 ret = 0; 505 ret = 0;
545 hlist_del_init_rcu(&watch->list_node); 506 hlist_del_init_rcu(&watch->list_node);
546 rcu_assign_pointer(watch->watch_list, 507 rcu_assign_pointer(watch->watch_list, NULL);
547 spin_unlock(&wlist->lock); 508 spin_unlock(&wlist->lock);
548 509
549 /* We now own the reference on watch t 510 /* We now own the reference on watch that used to belong to wlist. */
550 511
551 n.watch.type = WATCH_TYPE_META; 512 n.watch.type = WATCH_TYPE_META;
552 n.watch.subtype = WATCH_META_REMOVAL_N 513 n.watch.subtype = WATCH_META_REMOVAL_NOTIFICATION;
553 n.watch.info = watch->info_id | watch_ 514 n.watch.info = watch->info_id | watch_sizeof(n.watch);
554 n.id = id; 515 n.id = id;
555 if (id != 0) 516 if (id != 0)
556 n.watch.info = watch->info_id 517 n.watch.info = watch->info_id | watch_sizeof(n);
557 518
558 wqueue = rcu_dereference(watch->queue) 519 wqueue = rcu_dereference(watch->queue);
559 520
560 if (lock_wqueue(wqueue)) { !! 521 /* We don't need the watch list lock for the next bit as RCU is
>> 522 * protecting *wqueue from deallocation.
>> 523 */
>> 524 if (wqueue) {
561 post_one_notification(wqueue, 525 post_one_notification(wqueue, &n.watch);
562 526
>> 527 spin_lock_bh(&wqueue->lock);
>> 528
563 if (!hlist_unhashed(&watch->qu 529 if (!hlist_unhashed(&watch->queue_node)) {
564 hlist_del_init_rcu(&wa 530 hlist_del_init_rcu(&watch->queue_node);
565 put_watch(watch); 531 put_watch(watch);
566 } 532 }
567 533
568 unlock_wqueue(wqueue); !! 534 spin_unlock_bh(&wqueue->lock);
569 } 535 }
570 536
571 if (wlist->release_watch) { 537 if (wlist->release_watch) {
572 void (*release_watch)(struct w 538 void (*release_watch)(struct watch *);
573 539
574 release_watch = wlist->release 540 release_watch = wlist->release_watch;
575 rcu_read_unlock(); 541 rcu_read_unlock();
576 (*release_watch)(watch); 542 (*release_watch)(watch);
577 rcu_read_lock(); 543 rcu_read_lock();
578 } 544 }
579 put_watch(watch); 545 put_watch(watch);
580 546
581 if (all && !hlist_empty(&wlist->watche 547 if (all && !hlist_empty(&wlist->watchers))
582 goto again; 548 goto again;
583 out: 549 out:
584 rcu_read_unlock(); 550 rcu_read_unlock();
585 return ret; 551 return ret;
586 } 552 }
587 EXPORT_SYMBOL(remove_watch_from_object); 553 EXPORT_SYMBOL(remove_watch_from_object);
588 554
589 /* 555 /*
590 * Remove all the watches that are contributor 556 * Remove all the watches that are contributory to a queue. This has the
591 * potential to race with removal of the watch 557 * potential to race with removal of the watches by the destruction of the
592 * objects being watched or with the distribut 558 * objects being watched or with the distribution of notifications.
593 */ 559 */
594 void watch_queue_clear(struct watch_queue *wqu 560 void watch_queue_clear(struct watch_queue *wqueue)
595 { 561 {
596 struct watch_list *wlist; 562 struct watch_list *wlist;
597 struct watch *watch; 563 struct watch *watch;
598 bool release; 564 bool release;
599 565
600 rcu_read_lock(); 566 rcu_read_lock();
601 spin_lock_bh(&wqueue->lock); 567 spin_lock_bh(&wqueue->lock);
602 568
603 /* !! 569 /* Prevent new additions and prevent notifications from happening */
604 * This pipe can be freed by callers l !! 570 wqueue->defunct = true;
605 * Removing this reference also preven <<
606 */ <<
607 wqueue->pipe = NULL; <<
608 571
609 while (!hlist_empty(&wqueue->watches)) 572 while (!hlist_empty(&wqueue->watches)) {
610 watch = hlist_entry(wqueue->wa 573 watch = hlist_entry(wqueue->watches.first, struct watch, queue_node);
611 hlist_del_init_rcu(&watch->que 574 hlist_del_init_rcu(&watch->queue_node);
612 /* We now own a ref on the wat 575 /* We now own a ref on the watch. */
613 spin_unlock_bh(&wqueue->lock); 576 spin_unlock_bh(&wqueue->lock);
614 577
615 /* We can't do the next bit un 578 /* We can't do the next bit under the queue lock as we need to
616 * get the list lock - which w 579 * get the list lock - which would cause a deadlock if someone
617 * was removing from the oppos 580 * was removing from the opposite direction at the same time or
618 * posting a notification. 581 * posting a notification.
619 */ 582 */
620 wlist = rcu_dereference(watch- 583 wlist = rcu_dereference(watch->watch_list);
621 if (wlist) { 584 if (wlist) {
622 void (*release_watch)( 585 void (*release_watch)(struct watch *);
623 586
624 spin_lock(&wlist->lock 587 spin_lock(&wlist->lock);
625 588
626 release = !hlist_unhas 589 release = !hlist_unhashed(&watch->list_node);
627 if (release) { 590 if (release) {
628 hlist_del_init 591 hlist_del_init_rcu(&watch->list_node);
629 rcu_assign_poi 592 rcu_assign_pointer(watch->watch_list, NULL);
630 593
631 /* We now own 594 /* We now own a second ref on the watch. */
632 } 595 }
633 596
634 release_watch = wlist- 597 release_watch = wlist->release_watch;
635 spin_unlock(&wlist->lo 598 spin_unlock(&wlist->lock);
636 599
637 if (release) { 600 if (release) {
638 if (release_wa 601 if (release_watch) {
639 rcu_re 602 rcu_read_unlock();
640 /* Thi 603 /* This might need to call dput(), so
641 * we 604 * we have to drop all the locks.
642 */ 605 */
643 (*rele 606 (*release_watch)(watch);
644 rcu_re 607 rcu_read_lock();
645 } 608 }
646 put_watch(watc 609 put_watch(watch);
647 } 610 }
648 } 611 }
649 612
650 put_watch(watch); 613 put_watch(watch);
651 spin_lock_bh(&wqueue->lock); 614 spin_lock_bh(&wqueue->lock);
652 } 615 }
653 616
654 spin_unlock_bh(&wqueue->lock); 617 spin_unlock_bh(&wqueue->lock);
655 rcu_read_unlock(); 618 rcu_read_unlock();
656 } 619 }
657 620
658 /** 621 /**
659 * get_watch_queue - Get a watch queue from it 622 * get_watch_queue - Get a watch queue from its file descriptor.
660 * @fd: The fd to query. 623 * @fd: The fd to query.
661 */ 624 */
662 struct watch_queue *get_watch_queue(int fd) 625 struct watch_queue *get_watch_queue(int fd)
663 { 626 {
664 struct pipe_inode_info *pipe; 627 struct pipe_inode_info *pipe;
665 struct watch_queue *wqueue = ERR_PTR(- 628 struct watch_queue *wqueue = ERR_PTR(-EINVAL);
666 struct fd f; 629 struct fd f;
667 630
668 f = fdget(fd); 631 f = fdget(fd);
669 if (fd_file(f)) { !! 632 if (f.file) {
670 pipe = get_pipe_info(fd_file(f !! 633 pipe = get_pipe_info(f.file, false);
671 if (pipe && pipe->watch_queue) 634 if (pipe && pipe->watch_queue) {
672 wqueue = pipe->watch_q 635 wqueue = pipe->watch_queue;
673 kref_get(&wqueue->usag 636 kref_get(&wqueue->usage);
674 } 637 }
675 fdput(f); 638 fdput(f);
676 } 639 }
677 640
678 return wqueue; 641 return wqueue;
679 } 642 }
680 EXPORT_SYMBOL(get_watch_queue); 643 EXPORT_SYMBOL(get_watch_queue);
681 644
682 /* 645 /*
683 * Initialise a watch queue 646 * Initialise a watch queue
684 */ 647 */
685 int watch_queue_init(struct pipe_inode_info *p 648 int watch_queue_init(struct pipe_inode_info *pipe)
686 { 649 {
687 struct watch_queue *wqueue; 650 struct watch_queue *wqueue;
688 651
689 wqueue = kzalloc(sizeof(*wqueue), GFP_ 652 wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL);
690 if (!wqueue) 653 if (!wqueue)
691 return -ENOMEM; 654 return -ENOMEM;
692 655
693 wqueue->pipe = pipe; 656 wqueue->pipe = pipe;
694 kref_init(&wqueue->usage); 657 kref_init(&wqueue->usage);
695 spin_lock_init(&wqueue->lock); 658 spin_lock_init(&wqueue->lock);
696 INIT_HLIST_HEAD(&wqueue->watches); 659 INIT_HLIST_HEAD(&wqueue->watches);
697 660
698 pipe->watch_queue = wqueue; 661 pipe->watch_queue = wqueue;
699 return 0; 662 return 0;
700 } 663 }
701 664
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.