1 // SPDX-License-Identifier: GPL-2.0+ <<
2 /* 1 /*
3 * User-space Probes (UProbes) 2 * User-space Probes (UProbes)
4 * 3 *
>> 4 * This program is free software; you can redistribute it and/or modify
>> 5 * it under the terms of the GNU General Public License as published by
>> 6 * the Free Software Foundation; either version 2 of the License, or
>> 7 * (at your option) any later version.
>> 8 *
>> 9 * This program is distributed in the hope that it will be useful,
>> 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> 12 * GNU General Public License for more details.
>> 13 *
>> 14 * You should have received a copy of the GNU General Public License
>> 15 * along with this program; if not, write to the Free Software
>> 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
>> 17 *
5 * Copyright (C) IBM Corporation, 2008-2012 18 * Copyright (C) IBM Corporation, 2008-2012
6 * Authors: 19 * Authors:
7 * Srikar Dronamraju 20 * Srikar Dronamraju
8 * Jim Keniston 21 * Jim Keniston
9 * Copyright (C) 2011-2012 Red Hat, Inc., Pete 22 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
10 */ 23 */
11 24
12 #include <linux/kernel.h> 25 #include <linux/kernel.h>
13 #include <linux/highmem.h> 26 #include <linux/highmem.h>
14 #include <linux/pagemap.h> /* read_mappin 27 #include <linux/pagemap.h> /* read_mapping_page */
15 #include <linux/slab.h> 28 #include <linux/slab.h>
16 #include <linux/sched.h> 29 #include <linux/sched.h>
17 #include <linux/sched/mm.h> 30 #include <linux/sched/mm.h>
18 #include <linux/sched/coredump.h> 31 #include <linux/sched/coredump.h>
19 #include <linux/export.h> 32 #include <linux/export.h>
20 #include <linux/rmap.h> /* anon_vma_pr 33 #include <linux/rmap.h> /* anon_vma_prepare */
21 #include <linux/mmu_notifier.h> !! 34 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
22 #include <linux/swap.h> /* folio_free_ !! 35 #include <linux/swap.h> /* try_to_free_swap */
23 #include <linux/ptrace.h> /* user_enable 36 #include <linux/ptrace.h> /* user_enable_single_step */
24 #include <linux/kdebug.h> /* notifier me 37 #include <linux/kdebug.h> /* notifier mechanism */
>> 38 #include "../../mm/internal.h" /* munlock_vma_page */
25 #include <linux/percpu-rwsem.h> 39 #include <linux/percpu-rwsem.h>
26 #include <linux/task_work.h> 40 #include <linux/task_work.h>
27 #include <linux/shmem_fs.h> 41 #include <linux/shmem_fs.h>
28 #include <linux/khugepaged.h> <<
29 42
30 #include <linux/uprobes.h> 43 #include <linux/uprobes.h>
31 44
32 #define UINSNS_PER_PAGE (PAGE_ 45 #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
33 #define MAX_UPROBE_XOL_SLOTS UINSNS 46 #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
34 47
35 static struct rb_root uprobes_tree = RB_ROOT; 48 static struct rb_root uprobes_tree = RB_ROOT;
36 /* 49 /*
37 * allows us to skip the uprobe_mmap if there 50 * allows us to skip the uprobe_mmap if there are no uprobe events active
38 * at this time. Probably a fine grained per 51 * at this time. Probably a fine grained per inode count is better?
39 */ 52 */
40 #define no_uprobe_events() RB_EMPTY_ROOT( 53 #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree)
41 54
42 static DEFINE_RWLOCK(uprobes_treelock); /* ser !! 55 static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
43 static seqcount_rwlock_t uprobes_seqcount = SE <<
44 <<
45 DEFINE_STATIC_SRCU(uprobes_srcu); <<
46 56
47 #define UPROBES_HASH_SZ 13 57 #define UPROBES_HASH_SZ 13
48 /* serialize uprobe->pending_list */ 58 /* serialize uprobe->pending_list */
49 static struct mutex uprobes_mmap_mutex[UPROBES 59 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
50 #define uprobes_mmap_hash(v) (&uprobes_mmap 60 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
51 61
52 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); !! 62 static struct percpu_rw_semaphore dup_mmap_sem;
53 63
54 /* Have a copy of original instruction */ 64 /* Have a copy of original instruction */
55 #define UPROBE_COPY_INSN 0 65 #define UPROBE_COPY_INSN 0
56 66
57 struct uprobe { 67 struct uprobe {
58 struct rb_node rb_node; 68 struct rb_node rb_node; /* node in the rb tree */
59 refcount_t ref; !! 69 atomic_t ref;
60 struct rw_semaphore register_rwsem 70 struct rw_semaphore register_rwsem;
61 struct rw_semaphore consumer_rwsem 71 struct rw_semaphore consumer_rwsem;
62 struct list_head pending_list; 72 struct list_head pending_list;
63 struct list_head consumers; !! 73 struct uprobe_consumer *consumers;
64 struct inode *inode; 74 struct inode *inode; /* Also hold a ref to inode */
65 struct rcu_head rcu; <<
66 loff_t offset; 75 loff_t offset;
67 loff_t ref_ctr_offset <<
68 unsigned long flags; 76 unsigned long flags;
69 77
70 /* 78 /*
71 * The generic code assumes that it ha 79 * The generic code assumes that it has two members of unknown type
72 * owned by the arch-specific code: 80 * owned by the arch-specific code:
73 * 81 *
74 * insn - copy_insn() saves the 82 * insn - copy_insn() saves the original instruction here for
75 * arch_uprobe_analyze_in 83 * arch_uprobe_analyze_insn().
76 * 84 *
77 * ixol - potentially modified i 85 * ixol - potentially modified instruction to execute out of
78 * line, copied to xol_ar 86 * line, copied to xol_area by xol_get_insn_slot().
79 */ 87 */
80 struct arch_uprobe arch; 88 struct arch_uprobe arch;
81 }; 89 };
82 90
83 struct delayed_uprobe { <<
84 struct list_head list; <<
85 struct uprobe *uprobe; <<
86 struct mm_struct *mm; <<
87 }; <<
88 <<
89 static DEFINE_MUTEX(delayed_uprobe_lock); <<
90 static LIST_HEAD(delayed_uprobe_list); <<
91 <<
92 /* 91 /*
93 * Execute out of line area: anonymous executa 92 * Execute out of line area: anonymous executable mapping installed
94 * by the probed task to execute the copy of t 93 * by the probed task to execute the copy of the original instruction
95 * mangled by set_swbp(). 94 * mangled by set_swbp().
96 * 95 *
97 * On a breakpoint hit, thread contests for a 96 * On a breakpoint hit, thread contests for a slot. It frees the
98 * slot after singlestep. Currently a fixed nu 97 * slot after singlestep. Currently a fixed number of slots are
99 * allocated. 98 * allocated.
100 */ 99 */
101 struct xol_area { 100 struct xol_area {
102 wait_queue_head_t wq; 101 wait_queue_head_t wq; /* if all slots are busy */
103 atomic_t slot_c 102 atomic_t slot_count; /* number of in-use slots */
104 unsigned long *bitma 103 unsigned long *bitmap; /* 0 = free slot */
105 104
106 struct page *page; !! 105 struct vm_special_mapping xol_mapping;
>> 106 struct page *pages[2];
107 /* 107 /*
108 * We keep the vma's vm_start rather t 108 * We keep the vma's vm_start rather than a pointer to the vma
109 * itself. The probed process or a na 109 * itself. The probed process or a naughty kernel module could make
110 * the vma go away, and we must handle 110 * the vma go away, and we must handle that reasonably gracefully.
111 */ 111 */
112 unsigned long vaddr; 112 unsigned long vaddr; /* Page(s) of instruction slots */
113 }; 113 };
114 114
115 static void uprobe_warn(struct task_struct *t, <<
116 { <<
117 pr_warn("uprobe: %s:%d failed to %s\n" <<
118 } <<
119 <<
120 /* 115 /*
121 * valid_vma: Verify if the specified vma is a 116 * valid_vma: Verify if the specified vma is an executable vma
122 * Relax restrictions while unregistering: vm_ 117 * Relax restrictions while unregistering: vm_flags might have
123 * changed after breakpoint was inserted. 118 * changed after breakpoint was inserted.
124 * - is_register: indicates if we are in 119 * - is_register: indicates if we are in register context.
125 * - Return 1 if the specified virtual ad 120 * - Return 1 if the specified virtual address is in an
126 * executable vma. 121 * executable vma.
127 */ 122 */
128 static bool valid_vma(struct vm_area_struct *v 123 static bool valid_vma(struct vm_area_struct *vma, bool is_register)
129 { 124 {
130 vm_flags_t flags = VM_HUGETLB | VM_MAY 125 vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
131 126
132 if (is_register) 127 if (is_register)
133 flags |= VM_WRITE; 128 flags |= VM_WRITE;
134 129
135 return vma->vm_file && (vma->vm_flags 130 return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
136 } 131 }
137 132
138 static unsigned long offset_to_vaddr(struct vm 133 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
139 { 134 {
140 return vma->vm_start + offset - ((loff 135 return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
141 } 136 }
142 137
143 static loff_t vaddr_to_offset(struct vm_area_s 138 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
144 { 139 {
145 return ((loff_t)vma->vm_pgoff << PAGE_ 140 return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
146 } 141 }
147 142
148 /** 143 /**
149 * __replace_page - replace page in vma by new 144 * __replace_page - replace page in vma by new page.
150 * based on replace_page in mm/ksm.c 145 * based on replace_page in mm/ksm.c
151 * 146 *
152 * @vma: vma that holds the pte pointing 147 * @vma: vma that holds the pte pointing to page
153 * @addr: address the old @page is mapped 148 * @addr: address the old @page is mapped at
154 * @old_page: the page we are replacing by new !! 149 * @page: the cowed page we are replacing by kpage
155 * @new_page: the modified page we replace pag !! 150 * @kpage: the modified page we replace page by
156 * 151 *
157 * If @new_page is NULL, only unmap @old_page. !! 152 * Returns 0 on success, -EFAULT on failure.
158 * <<
159 * Returns 0 on success, negative error code o <<
160 */ 153 */
161 static int __replace_page(struct vm_area_struc 154 static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
162 struct page *o 155 struct page *old_page, struct page *new_page)
163 { 156 {
164 struct folio *old_folio = page_folio(o <<
165 struct folio *new_folio; <<
166 struct mm_struct *mm = vma->vm_mm; 157 struct mm_struct *mm = vma->vm_mm;
167 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, !! 158 struct page_vma_mapped_walk pvmw = {
>> 159 .page = old_page,
>> 160 .vma = vma,
>> 161 .address = addr,
>> 162 };
168 int err; 163 int err;
169 struct mmu_notifier_range range; !! 164 /* For mmu_notifiers */
170 !! 165 const unsigned long mmun_start = addr;
171 mmu_notifier_range_init(&range, MMU_NO !! 166 const unsigned long mmun_end = addr + PAGE_SIZE;
172 addr + PAGE_SI !! 167 struct mem_cgroup *memcg;
173 !! 168
174 if (new_page) { !! 169 VM_BUG_ON_PAGE(PageTransHuge(old_page), old_page);
175 new_folio = page_folio(new_pag !! 170
176 err = mem_cgroup_charge(new_fo !! 171 err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL, &memcg,
177 if (err) !! 172 false);
178 return err; !! 173 if (err)
179 } !! 174 return err;
180 175
181 /* For folio_free_swap() below */ !! 176 /* For try_to_free_swap() and munlock_vma_page() below */
182 folio_lock(old_folio); !! 177 lock_page(old_page);
183 178
184 mmu_notifier_invalidate_range_start(&r !! 179 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
185 err = -EAGAIN; 180 err = -EAGAIN;
186 if (!page_vma_mapped_walk(&pvmw)) !! 181 if (!page_vma_mapped_walk(&pvmw)) {
>> 182 mem_cgroup_cancel_charge(new_page, memcg, false);
187 goto unlock; 183 goto unlock;
>> 184 }
188 VM_BUG_ON_PAGE(addr != pvmw.address, o 185 VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
189 186
190 if (new_page) { !! 187 get_page(new_page);
191 folio_get(new_folio); !! 188 page_add_new_anon_rmap(new_page, vma, addr, false);
192 folio_add_new_anon_rmap(new_fo !! 189 mem_cgroup_commit_charge(new_page, memcg, false, false);
193 folio_add_lru_vma(new_folio, v !! 190 lru_cache_add_active_or_unevictable(new_page, vma);
194 } else <<
195 /* no new page, just dec_mm_co <<
196 dec_mm_counter(mm, MM_ANONPAGE <<
197 191
198 if (!folio_test_anon(old_folio)) { !! 192 if (!PageAnon(old_page)) {
199 dec_mm_counter(mm, mm_counter_ !! 193 dec_mm_counter(mm, mm_counter_file(old_page));
200 inc_mm_counter(mm, MM_ANONPAGE 194 inc_mm_counter(mm, MM_ANONPAGES);
201 } 195 }
202 196
203 flush_cache_page(vma, addr, pte_pfn(pt !! 197 flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
204 ptep_clear_flush(vma, addr, pvmw.pte); !! 198 ptep_clear_flush_notify(vma, addr, pvmw.pte);
205 if (new_page) !! 199 set_pte_at_notify(mm, addr, pvmw.pte,
206 set_pte_at(mm, addr, pvmw.pte, !! 200 mk_pte(new_page, vma->vm_page_prot));
207 mk_pte(new_page, vm !! 201
208 !! 202 page_remove_rmap(old_page, false);
209 folio_remove_rmap_pte(old_folio, old_p !! 203 if (!page_mapped(old_page))
210 if (!folio_mapped(old_folio)) !! 204 try_to_free_swap(old_page);
211 folio_free_swap(old_folio); <<
212 page_vma_mapped_walk_done(&pvmw); 205 page_vma_mapped_walk_done(&pvmw);
213 folio_put(old_folio); !! 206
>> 207 if (vma->vm_flags & VM_LOCKED)
>> 208 munlock_vma_page(old_page);
>> 209 put_page(old_page);
214 210
215 err = 0; 211 err = 0;
216 unlock: 212 unlock:
217 mmu_notifier_invalidate_range_end(&ran !! 213 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
218 folio_unlock(old_folio); !! 214 unlock_page(old_page);
219 return err; 215 return err;
220 } 216 }
221 217
222 /** 218 /**
223 * is_swbp_insn - check if instruction is brea 219 * is_swbp_insn - check if instruction is breakpoint instruction.
224 * @insn: instruction to be checked. 220 * @insn: instruction to be checked.
225 * Default implementation of is_swbp_insn 221 * Default implementation of is_swbp_insn
226 * Returns true if @insn is a breakpoint instr 222 * Returns true if @insn is a breakpoint instruction.
227 */ 223 */
228 bool __weak is_swbp_insn(uprobe_opcode_t *insn 224 bool __weak is_swbp_insn(uprobe_opcode_t *insn)
229 { 225 {
230 return *insn == UPROBE_SWBP_INSN; 226 return *insn == UPROBE_SWBP_INSN;
231 } 227 }
232 228
233 /** 229 /**
234 * is_trap_insn - check if instruction is brea 230 * is_trap_insn - check if instruction is breakpoint instruction.
235 * @insn: instruction to be checked. 231 * @insn: instruction to be checked.
236 * Default implementation of is_trap_insn 232 * Default implementation of is_trap_insn
237 * Returns true if @insn is a breakpoint instr 233 * Returns true if @insn is a breakpoint instruction.
238 * 234 *
239 * This function is needed for the case where 235 * This function is needed for the case where an architecture has multiple
240 * trap instructions (like powerpc). 236 * trap instructions (like powerpc).
241 */ 237 */
242 bool __weak is_trap_insn(uprobe_opcode_t *insn 238 bool __weak is_trap_insn(uprobe_opcode_t *insn)
243 { 239 {
244 return is_swbp_insn(insn); 240 return is_swbp_insn(insn);
245 } 241 }
246 242
247 static void copy_from_page(struct page *page, 243 static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
248 { 244 {
249 void *kaddr = kmap_atomic(page); 245 void *kaddr = kmap_atomic(page);
250 memcpy(dst, kaddr + (vaddr & ~PAGE_MAS 246 memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
251 kunmap_atomic(kaddr); 247 kunmap_atomic(kaddr);
252 } 248 }
253 249
254 static void copy_to_page(struct page *page, un 250 static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
255 { 251 {
256 void *kaddr = kmap_atomic(page); 252 void *kaddr = kmap_atomic(page);
257 memcpy(kaddr + (vaddr & ~PAGE_MASK), s 253 memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
258 kunmap_atomic(kaddr); 254 kunmap_atomic(kaddr);
259 } 255 }
260 256
261 static int verify_opcode(struct page *page, un 257 static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
262 { 258 {
263 uprobe_opcode_t old_opcode; 259 uprobe_opcode_t old_opcode;
264 bool is_swbp; 260 bool is_swbp;
265 261
266 /* 262 /*
267 * Note: We only check if the old_opco 263 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
268 * We do not check if it is any other 264 * We do not check if it is any other 'trap variant' which could
269 * be conditional trap instruction suc 265 * be conditional trap instruction such as the one powerpc supports.
270 * 266 *
271 * The logic is that we do not care if 267 * The logic is that we do not care if the underlying instruction
272 * is a trap variant; uprobes always w 268 * is a trap variant; uprobes always wins over any other (gdb)
273 * breakpoint. 269 * breakpoint.
274 */ 270 */
275 copy_from_page(page, vaddr, &old_opcod 271 copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
276 is_swbp = is_swbp_insn(&old_opcode); 272 is_swbp = is_swbp_insn(&old_opcode);
277 273
278 if (is_swbp_insn(new_opcode)) { 274 if (is_swbp_insn(new_opcode)) {
279 if (is_swbp) /* reg 275 if (is_swbp) /* register: already installed? */
280 return 0; 276 return 0;
281 } else { 277 } else {
282 if (!is_swbp) /* unr 278 if (!is_swbp) /* unregister: was it changed by us? */
283 return 0; 279 return 0;
284 } 280 }
285 281
286 return 1; 282 return 1;
287 } 283 }
288 284
289 static struct delayed_uprobe * <<
290 delayed_uprobe_check(struct uprobe *uprobe, st <<
291 { <<
292 struct delayed_uprobe *du; <<
293 <<
294 list_for_each_entry(du, &delayed_uprob <<
295 if (du->uprobe == uprobe && du <<
296 return du; <<
297 return NULL; <<
298 } <<
299 <<
300 static int delayed_uprobe_add(struct uprobe *u <<
301 { <<
302 struct delayed_uprobe *du; <<
303 <<
304 if (delayed_uprobe_check(uprobe, mm)) <<
305 return 0; <<
306 <<
307 du = kzalloc(sizeof(*du), GFP_KERNEL) <<
308 if (!du) <<
309 return -ENOMEM; <<
310 <<
311 du->uprobe = uprobe; <<
312 du->mm = mm; <<
313 list_add(&du->list, &delayed_uprobe_li <<
314 return 0; <<
315 } <<
316 <<
317 static void delayed_uprobe_delete(struct delay <<
318 { <<
319 if (WARN_ON(!du)) <<
320 return; <<
321 list_del(&du->list); <<
322 kfree(du); <<
323 } <<
324 <<
325 static void delayed_uprobe_remove(struct uprob <<
326 { <<
327 struct list_head *pos, *q; <<
328 struct delayed_uprobe *du; <<
329 <<
330 if (!uprobe && !mm) <<
331 return; <<
332 <<
333 list_for_each_safe(pos, q, &delayed_up <<
334 du = list_entry(pos, struct de <<
335 <<
336 if (uprobe && du->uprobe != up <<
337 continue; <<
338 if (mm && du->mm != mm) <<
339 continue; <<
340 <<
341 delayed_uprobe_delete(du); <<
342 } <<
343 } <<
344 <<
345 static bool valid_ref_ctr_vma(struct uprobe *u <<
346 struct vm_area_s <<
347 { <<
348 unsigned long vaddr = offset_to_vaddr( <<
349 <<
350 return uprobe->ref_ctr_offset && <<
351 vma->vm_file && <<
352 file_inode(vma->vm_file) == up <<
353 (vma->vm_flags & (VM_WRITE|VM_ <<
354 vma->vm_start <= vaddr && <<
355 vma->vm_end > vaddr; <<
356 } <<
357 <<
358 static struct vm_area_struct * <<
359 find_ref_ctr_vma(struct uprobe *uprobe, struct <<
360 { <<
361 VMA_ITERATOR(vmi, mm, 0); <<
362 struct vm_area_struct *tmp; <<
363 <<
364 for_each_vma(vmi, tmp) <<
365 if (valid_ref_ctr_vma(uprobe, <<
366 return tmp; <<
367 <<
368 return NULL; <<
369 } <<
370 <<
371 static int <<
372 __update_ref_ctr(struct mm_struct *mm, unsigne <<
373 { <<
374 void *kaddr; <<
375 struct page *page; <<
376 int ret; <<
377 short *ptr; <<
378 <<
379 if (!vaddr || !d) <<
380 return -EINVAL; <<
381 <<
382 ret = get_user_pages_remote(mm, vaddr, <<
383 FOLL_WRITE <<
384 if (unlikely(ret <= 0)) { <<
385 /* <<
386 * We are asking for 1 page. I <<
387 * it may return 0, in that ca <<
388 */ <<
389 return ret == 0 ? -EBUSY : ret <<
390 } <<
391 <<
392 kaddr = kmap_atomic(page); <<
393 ptr = kaddr + (vaddr & ~PAGE_MASK); <<
394 <<
395 if (unlikely(*ptr + d < 0)) { <<
396 pr_warn("ref_ctr going negativ <<
397 "curr val: %d, delta: <<
398 ret = -EINVAL; <<
399 goto out; <<
400 } <<
401 <<
402 *ptr += d; <<
403 ret = 0; <<
404 out: <<
405 kunmap_atomic(kaddr); <<
406 put_page(page); <<
407 return ret; <<
408 } <<
409 <<
410 static void update_ref_ctr_warn(struct uprobe <<
411 struct mm_stru <<
412 { <<
413 pr_warn("ref_ctr %s failed for inode: <<
414 "0x%llx ref_ctr_offset: 0x%llx <<
415 d > 0 ? "increment" : "decreme <<
416 (unsigned long long) uprobe->o <<
417 (unsigned long long) uprobe->r <<
418 } <<
419 <<
420 static int update_ref_ctr(struct uprobe *uprob <<
421 short d) <<
422 { <<
423 struct vm_area_struct *rc_vma; <<
424 unsigned long rc_vaddr; <<
425 int ret = 0; <<
426 <<
427 rc_vma = find_ref_ctr_vma(uprobe, mm); <<
428 <<
429 if (rc_vma) { <<
430 rc_vaddr = offset_to_vaddr(rc_ <<
431 ret = __update_ref_ctr(mm, rc_ <<
432 if (ret) <<
433 update_ref_ctr_warn(up <<
434 <<
435 if (d > 0) <<
436 return ret; <<
437 } <<
438 <<
439 mutex_lock(&delayed_uprobe_lock); <<
440 if (d > 0) <<
441 ret = delayed_uprobe_add(uprob <<
442 else <<
443 delayed_uprobe_remove(uprobe, <<
444 mutex_unlock(&delayed_uprobe_lock); <<
445 <<
446 return ret; <<
447 } <<
448 <<
449 /* 285 /*
450 * NOTE: 286 * NOTE:
451 * Expect the breakpoint instruction to be the 287 * Expect the breakpoint instruction to be the smallest size instruction for
452 * the architecture. If an arch has variable l 288 * the architecture. If an arch has variable length instruction and the
453 * breakpoint instruction is not of the smalle 289 * breakpoint instruction is not of the smallest length instruction
454 * supported by that architecture then we need 290 * supported by that architecture then we need to modify is_trap_at_addr and
455 * uprobe_write_opcode accordingly. This would 291 * uprobe_write_opcode accordingly. This would never be a problem for archs
456 * that have fixed length instructions. 292 * that have fixed length instructions.
457 * 293 *
458 * uprobe_write_opcode - write the opcode at a 294 * uprobe_write_opcode - write the opcode at a given virtual address.
459 * @auprobe: arch specific probepoint informat <<
460 * @mm: the probed process address space. 295 * @mm: the probed process address space.
461 * @vaddr: the virtual address to store the op 296 * @vaddr: the virtual address to store the opcode.
462 * @opcode: opcode to be written at @vaddr. 297 * @opcode: opcode to be written at @vaddr.
463 * 298 *
464 * Called with mm->mmap_lock held for read or !! 299 * Called with mm->mmap_sem held for write.
465 * Return 0 (success) or a negative errno. 300 * Return 0 (success) or a negative errno.
466 */ 301 */
467 int uprobe_write_opcode(struct arch_uprobe *au !! 302 int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
468 unsigned long vaddr, u !! 303 uprobe_opcode_t opcode)
469 { 304 {
470 struct uprobe *uprobe; <<
471 struct page *old_page, *new_page; 305 struct page *old_page, *new_page;
472 struct vm_area_struct *vma; 306 struct vm_area_struct *vma;
473 int ret, is_register, ref_ctr_updated !! 307 int ret;
474 bool orig_page_huge = false; <<
475 unsigned int gup_flags = FOLL_FORCE; <<
476 <<
477 is_register = is_swbp_insn(&opcode); <<
478 uprobe = container_of(auprobe, struct <<
479 308
480 retry: 309 retry:
481 if (is_register) <<
482 gup_flags |= FOLL_SPLIT_PMD; <<
483 /* Read the page with vaddr into memor 310 /* Read the page with vaddr into memory */
484 old_page = get_user_page_vma_remote(mm !! 311 ret = get_user_pages_remote(NULL, mm, vaddr, 1,
485 if (IS_ERR(old_page)) !! 312 FOLL_FORCE | FOLL_SPLIT, &old_page, &vma, NULL);
486 return PTR_ERR(old_page); !! 313 if (ret <= 0)
>> 314 return ret;
487 315
488 ret = verify_opcode(old_page, vaddr, & 316 ret = verify_opcode(old_page, vaddr, &opcode);
489 if (ret <= 0) 317 if (ret <= 0)
490 goto put_old; 318 goto put_old;
491 319
492 if (WARN(!is_register && PageCompound( <<
493 "uprobe unregister should nev <<
494 ret = -EINVAL; <<
495 goto put_old; <<
496 } <<
497 <<
498 /* We are going to replace instruction <<
499 if (!ref_ctr_updated && uprobe->ref_ct <<
500 ret = update_ref_ctr(uprobe, m <<
501 if (ret) <<
502 goto put_old; <<
503 <<
504 ref_ctr_updated = 1; <<
505 } <<
506 <<
507 ret = 0; <<
508 if (!is_register && !PageAnon(old_page <<
509 goto put_old; <<
510 <<
511 ret = anon_vma_prepare(vma); 320 ret = anon_vma_prepare(vma);
512 if (ret) 321 if (ret)
513 goto put_old; 322 goto put_old;
514 323
515 ret = -ENOMEM; 324 ret = -ENOMEM;
516 new_page = alloc_page_vma(GFP_HIGHUSER 325 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
517 if (!new_page) 326 if (!new_page)
518 goto put_old; 327 goto put_old;
519 328
520 __SetPageUptodate(new_page); 329 __SetPageUptodate(new_page);
521 copy_highpage(new_page, old_page); 330 copy_highpage(new_page, old_page);
522 copy_to_page(new_page, vaddr, &opcode, 331 copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
523 332
524 if (!is_register) { !! 333 ret = __replace_page(vma, vaddr, old_page, new_page);
525 struct page *orig_page; !! 334 put_page(new_page);
526 pgoff_t index; <<
527 <<
528 VM_BUG_ON_PAGE(!PageAnon(old_p <<
529 <<
530 index = vaddr_to_offset(vma, v <<
531 orig_page = find_get_page(vma- <<
532 inde <<
533 <<
534 if (orig_page) { <<
535 if (PageUptodate(orig_ <<
536 pages_identical(ne <<
537 /* let go new_ <<
538 put_page(new_p <<
539 new_page = NUL <<
540 <<
541 if (PageCompou <<
542 orig_p <<
543 } <<
544 put_page(orig_page); <<
545 } <<
546 } <<
547 <<
548 ret = __replace_page(vma, vaddr & PAGE <<
549 if (new_page) <<
550 put_page(new_page); <<
551 put_old: 335 put_old:
552 put_page(old_page); 336 put_page(old_page);
553 337
554 if (unlikely(ret == -EAGAIN)) 338 if (unlikely(ret == -EAGAIN))
555 goto retry; 339 goto retry;
556 <<
557 /* Revert back reference counter if in <<
558 if (ret && is_register && ref_ctr_upda <<
559 update_ref_ctr(uprobe, mm, -1) <<
560 <<
561 /* try collapse pmd for compound page <<
562 if (!ret && orig_page_huge) <<
563 collapse_pte_mapped_thp(mm, va <<
564 <<
565 return ret; 340 return ret;
566 } 341 }
567 342
568 /** 343 /**
569 * set_swbp - store breakpoint at a given addr 344 * set_swbp - store breakpoint at a given address.
570 * @auprobe: arch specific probepoint informat 345 * @auprobe: arch specific probepoint information.
571 * @mm: the probed process address space. 346 * @mm: the probed process address space.
572 * @vaddr: the virtual address to insert the o 347 * @vaddr: the virtual address to insert the opcode.
573 * 348 *
574 * For mm @mm, store the breakpoint instructio 349 * For mm @mm, store the breakpoint instruction at @vaddr.
575 * Return 0 (success) or a negative errno. 350 * Return 0 (success) or a negative errno.
576 */ 351 */
577 int __weak set_swbp(struct arch_uprobe *auprob 352 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
578 { 353 {
579 return uprobe_write_opcode(auprobe, mm !! 354 return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN);
580 } 355 }
581 356
582 /** 357 /**
583 * set_orig_insn - Restore the original instru 358 * set_orig_insn - Restore the original instruction.
584 * @mm: the probed process address space. 359 * @mm: the probed process address space.
585 * @auprobe: arch specific probepoint informat 360 * @auprobe: arch specific probepoint information.
586 * @vaddr: the virtual address to insert the o 361 * @vaddr: the virtual address to insert the opcode.
587 * 362 *
588 * For mm @mm, restore the original opcode (op 363 * For mm @mm, restore the original opcode (opcode) at @vaddr.
589 * Return 0 (success) or a negative errno. 364 * Return 0 (success) or a negative errno.
590 */ 365 */
591 int __weak 366 int __weak
592 set_orig_insn(struct arch_uprobe *auprobe, str 367 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
593 { 368 {
594 return uprobe_write_opcode(auprobe, mm !! 369 return uprobe_write_opcode(mm, vaddr, *(uprobe_opcode_t *)&auprobe->insn);
595 *(uprobe_opcode_t *)&a <<
596 } 370 }
597 371
598 /* uprobe should have guaranteed positive refc <<
599 static struct uprobe *get_uprobe(struct uprobe 372 static struct uprobe *get_uprobe(struct uprobe *uprobe)
600 { 373 {
601 refcount_inc(&uprobe->ref); !! 374 atomic_inc(&uprobe->ref);
602 return uprobe; 375 return uprobe;
603 } 376 }
604 377
605 /* <<
606 * uprobe should have guaranteed lifetime, whi <<
607 * - caller already has refcount taken (and <<
608 * - uprobe is RCU protected and won't be fr <<
609 * - we are holding uprobes_treelock (for re <<
610 */ <<
611 static struct uprobe *try_get_uprobe(struct up <<
612 { <<
613 if (refcount_inc_not_zero(&uprobe->ref <<
614 return uprobe; <<
615 return NULL; <<
616 } <<
617 <<
618 static inline bool uprobe_is_active(struct upr <<
619 { <<
620 return !RB_EMPTY_NODE(&uprobe->rb_node <<
621 } <<
622 <<
623 static void uprobe_free_rcu(struct rcu_head *r <<
624 { <<
625 struct uprobe *uprobe = container_of(r <<
626 <<
627 kfree(uprobe); <<
628 } <<
629 <<
630 static void put_uprobe(struct uprobe *uprobe) 378 static void put_uprobe(struct uprobe *uprobe)
631 { 379 {
632 if (!refcount_dec_and_test(&uprobe->re !! 380 if (atomic_dec_and_test(&uprobe->ref))
633 return; !! 381 kfree(uprobe);
634 <<
635 write_lock(&uprobes_treelock); <<
636 <<
637 if (uprobe_is_active(uprobe)) { <<
638 write_seqcount_begin(&uprobes_ <<
639 rb_erase(&uprobe->rb_node, &up <<
640 write_seqcount_end(&uprobes_se <<
641 } <<
642 <<
643 write_unlock(&uprobes_treelock); <<
644 <<
645 /* <<
646 * If application munmap(exec_vma) bef <<
647 * gets called, we don't get a chance <<
648 * delayed_uprobe_list from remove_bre <<
649 */ <<
650 mutex_lock(&delayed_uprobe_lock); <<
651 delayed_uprobe_remove(uprobe, NULL); <<
652 mutex_unlock(&delayed_uprobe_lock); <<
653 <<
654 call_srcu(&uprobes_srcu, &uprobe->rcu, <<
655 } 382 }
656 383
657 static __always_inline !! 384 static int match_uprobe(struct uprobe *l, struct uprobe *r)
658 int uprobe_cmp(const struct inode *l_inode, co <<
659 const struct uprobe *r) <<
660 { 385 {
661 if (l_inode < r->inode) !! 386 if (l->inode < r->inode)
662 return -1; 387 return -1;
663 388
664 if (l_inode > r->inode) !! 389 if (l->inode > r->inode)
665 return 1; 390 return 1;
666 391
667 if (l_offset < r->offset) !! 392 if (l->offset < r->offset)
668 return -1; 393 return -1;
669 394
670 if (l_offset > r->offset) !! 395 if (l->offset > r->offset)
671 return 1; 396 return 1;
672 397
673 return 0; 398 return 0;
674 } 399 }
675 400
676 #define __node_2_uprobe(node) \ !! 401 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
677 rb_entry((node), struct uprobe, rb_nod <<
678 <<
679 struct __uprobe_key { <<
680 struct inode *inode; <<
681 loff_t offset; <<
682 }; <<
683 <<
684 static inline int __uprobe_cmp_key(const void <<
685 { 402 {
686 const struct __uprobe_key *a = key; !! 403 struct uprobe u = { .inode = inode, .offset = offset };
687 return uprobe_cmp(a->inode, a->offset, !! 404 struct rb_node *n = uprobes_tree.rb_node;
688 } !! 405 struct uprobe *uprobe;
>> 406 int match;
689 407
690 static inline int __uprobe_cmp(struct rb_node !! 408 while (n) {
691 { !! 409 uprobe = rb_entry(n, struct uprobe, rb_node);
692 struct uprobe *u = __node_2_uprobe(a); !! 410 match = match_uprobe(&u, uprobe);
693 return uprobe_cmp(u->inode, u->offset, !! 411 if (!match)
>> 412 return get_uprobe(uprobe);
>> 413
>> 414 if (match < 0)
>> 415 n = n->rb_left;
>> 416 else
>> 417 n = n->rb_right;
>> 418 }
>> 419 return NULL;
694 } 420 }
695 421
696 /* 422 /*
697 * Assumes being inside RCU protected region. !! 423 * Find a uprobe corresponding to a given inode:offset
698 * No refcount is taken on returned uprobe. !! 424 * Acquires uprobes_treelock
699 */ 425 */
700 static struct uprobe *find_uprobe_rcu(struct i !! 426 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
701 { 427 {
702 struct __uprobe_key key = { !! 428 struct uprobe *uprobe;
703 .inode = inode, <<
704 .offset = offset, <<
705 }; <<
706 struct rb_node *node; <<
707 unsigned int seq; <<
708 429
709 lockdep_assert(srcu_read_lock_held(&up !! 430 spin_lock(&uprobes_treelock);
>> 431 uprobe = __find_uprobe(inode, offset);
>> 432 spin_unlock(&uprobes_treelock);
710 433
711 do { !! 434 return uprobe;
712 seq = read_seqcount_begin(&upr <<
713 node = rb_find_rcu(&key, &upro <<
714 /* <<
715 * Lockless RB-tree lookups ca <<
716 * If the element is found, it <<
717 * under RCU protection. If we <<
718 * validate that seqcount didn <<
719 * try again as we might have <<
720 * negative). If seqcount is u <<
721 */ <<
722 if (node) <<
723 return __node_2_uprobe <<
724 } while (read_seqcount_retry(&uprobes_ <<
725 <<
726 return NULL; <<
727 } 435 }
728 436
729 /* <<
730 * Attempt to insert a new uprobe into uprobes <<
731 * <<
732 * If uprobe already exists (for given inode+o <<
733 * refcount of previously existing uprobe. <<
734 * <<
735 * If not, a provided new instance of uprobe i <<
736 * assumed initial refcount == 1). <<
737 * <<
738 * In any case, we return a uprobe instance th <<
739 * Caller has to clean up new uprobe instance, <<
740 * inserted into the tree. <<
741 * <<
742 * We assume that uprobes_treelock is held for <<
743 */ <<
744 static struct uprobe *__insert_uprobe(struct u 437 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
745 { 438 {
746 struct rb_node *node; !! 439 struct rb_node **p = &uprobes_tree.rb_node;
747 again: !! 440 struct rb_node *parent = NULL;
748 node = rb_find_add_rcu(&uprobe->rb_nod !! 441 struct uprobe *u;
749 if (node) { !! 442 int match;
750 struct uprobe *u = __node_2_up !! 443
751 !! 444 while (*p) {
752 if (!try_get_uprobe(u)) { !! 445 parent = *p;
753 rb_erase(node, &uprobe !! 446 u = rb_entry(parent, struct uprobe, rb_node);
754 RB_CLEAR_NODE(&u->rb_n !! 447 match = match_uprobe(uprobe, u);
755 goto again; !! 448 if (!match)
756 } !! 449 return get_uprobe(u);
>> 450
>> 451 if (match < 0)
>> 452 p = &parent->rb_left;
>> 453 else
>> 454 p = &parent->rb_right;
757 455
758 return u; <<
759 } 456 }
760 457
761 return uprobe; !! 458 u = NULL;
>> 459 rb_link_node(&uprobe->rb_node, parent, p);
>> 460 rb_insert_color(&uprobe->rb_node, &uprobes_tree);
>> 461 /* get access + creation ref */
>> 462 atomic_set(&uprobe->ref, 2);
>> 463
>> 464 return u;
762 } 465 }
763 466
764 /* 467 /*
765 * Acquire uprobes_treelock and insert uprobe !! 468 * Acquire uprobes_treelock.
766 * (or reuse existing one, see __insert_uprobe !! 469 * Matching uprobe already exists in rbtree;
>> 470 * increment (access refcount) and return the matching uprobe.
>> 471 *
>> 472 * No matching uprobe; insert the uprobe in rb_tree;
>> 473 * get a double refcount (access + creation) and return NULL.
767 */ 474 */
768 static struct uprobe *insert_uprobe(struct upr 475 static struct uprobe *insert_uprobe(struct uprobe *uprobe)
769 { 476 {
770 struct uprobe *u; 477 struct uprobe *u;
771 478
772 write_lock(&uprobes_treelock); !! 479 spin_lock(&uprobes_treelock);
773 write_seqcount_begin(&uprobes_seqcount <<
774 u = __insert_uprobe(uprobe); 480 u = __insert_uprobe(uprobe);
775 write_seqcount_end(&uprobes_seqcount); !! 481 spin_unlock(&uprobes_treelock);
776 write_unlock(&uprobes_treelock); <<
777 482
778 return u; 483 return u;
779 } 484 }
780 485
781 static void !! 486 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
782 ref_ctr_mismatch_warn(struct uprobe *cur_uprob <<
783 { <<
784 pr_warn("ref_ctr_offset mismatch. inod <<
785 "ref_ctr_offset(old): 0x%llx r <<
786 uprobe->inode->i_ino, (unsigne <<
787 (unsigned long long) cur_uprob <<
788 (unsigned long long) uprobe->r <<
789 } <<
790 <<
791 static struct uprobe *alloc_uprobe(struct inod <<
792 loff_t ref_ <<
793 { 487 {
794 struct uprobe *uprobe, *cur_uprobe; 488 struct uprobe *uprobe, *cur_uprobe;
795 489
796 uprobe = kzalloc(sizeof(struct uprobe) 490 uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
797 if (!uprobe) 491 if (!uprobe)
798 return ERR_PTR(-ENOMEM); !! 492 return NULL;
799 493
800 uprobe->inode = inode; !! 494 uprobe->inode = igrab(inode);
801 uprobe->offset = offset; 495 uprobe->offset = offset;
802 uprobe->ref_ctr_offset = ref_ctr_offse <<
803 INIT_LIST_HEAD(&uprobe->consumers); <<
804 init_rwsem(&uprobe->register_rwsem); 496 init_rwsem(&uprobe->register_rwsem);
805 init_rwsem(&uprobe->consumer_rwsem); 497 init_rwsem(&uprobe->consumer_rwsem);
806 RB_CLEAR_NODE(&uprobe->rb_node); <<
807 refcount_set(&uprobe->ref, 1); <<
808 498
809 /* add to uprobes_tree, sorted on inod 499 /* add to uprobes_tree, sorted on inode:offset */
810 cur_uprobe = insert_uprobe(uprobe); 500 cur_uprobe = insert_uprobe(uprobe);
811 /* a uprobe exists for this inode:offs 501 /* a uprobe exists for this inode:offset combination */
812 if (cur_uprobe != uprobe) { !! 502 if (cur_uprobe) {
813 if (cur_uprobe->ref_ctr_offset <<
814 ref_ctr_mismatch_warn( <<
815 put_uprobe(cur_uprobe) <<
816 kfree(uprobe); <<
817 return ERR_PTR(-EINVAL <<
818 } <<
819 kfree(uprobe); 503 kfree(uprobe);
820 uprobe = cur_uprobe; 504 uprobe = cur_uprobe;
>> 505 iput(inode);
821 } 506 }
822 507
823 return uprobe; 508 return uprobe;
824 } 509 }
825 510
826 static void consumer_add(struct uprobe *uprobe 511 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
827 { 512 {
828 down_write(&uprobe->consumer_rwsem); 513 down_write(&uprobe->consumer_rwsem);
829 list_add_rcu(&uc->cons_node, &uprobe-> !! 514 uc->next = uprobe->consumers;
>> 515 uprobe->consumers = uc;
830 up_write(&uprobe->consumer_rwsem); 516 up_write(&uprobe->consumer_rwsem);
831 } 517 }
832 518
833 /* 519 /*
834 * For uprobe @uprobe, delete the consumer @uc 520 * For uprobe @uprobe, delete the consumer @uc.
835 * Should never be called with consumer that's !! 521 * Return true if the @uc is deleted successfully
>> 522 * or return false.
836 */ 523 */
837 static void consumer_del(struct uprobe *uprobe !! 524 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
838 { 525 {
>> 526 struct uprobe_consumer **con;
>> 527 bool ret = false;
>> 528
839 down_write(&uprobe->consumer_rwsem); 529 down_write(&uprobe->consumer_rwsem);
840 list_del_rcu(&uc->cons_node); !! 530 for (con = &uprobe->consumers; *con; con = &(*con)->next) {
>> 531 if (*con == uc) {
>> 532 *con = uc->next;
>> 533 ret = true;
>> 534 break;
>> 535 }
>> 536 }
841 up_write(&uprobe->consumer_rwsem); 537 up_write(&uprobe->consumer_rwsem);
>> 538
>> 539 return ret;
842 } 540 }
843 541
844 static int __copy_insn(struct address_space *m 542 static int __copy_insn(struct address_space *mapping, struct file *filp,
845 void *insn, int nbytes 543 void *insn, int nbytes, loff_t offset)
846 { 544 {
847 struct page *page; 545 struct page *page;
848 /* 546 /*
849 * Ensure that the page that has the o 547 * Ensure that the page that has the original instruction is populated
850 * and in page-cache. If ->read_folio !! 548 * and in page-cache. If ->readpage == NULL it must be shmem_mapping(),
851 * see uprobe_register(). 549 * see uprobe_register().
852 */ 550 */
853 if (mapping->a_ops->read_folio) !! 551 if (mapping->a_ops->readpage)
854 page = read_mapping_page(mappi 552 page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
855 else 553 else
856 page = shmem_read_mapping_page 554 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
857 if (IS_ERR(page)) 555 if (IS_ERR(page))
858 return PTR_ERR(page); 556 return PTR_ERR(page);
859 557
860 copy_from_page(page, offset, insn, nby 558 copy_from_page(page, offset, insn, nbytes);
861 put_page(page); 559 put_page(page);
862 560
863 return 0; 561 return 0;
864 } 562 }
865 563
866 static int copy_insn(struct uprobe *uprobe, st 564 static int copy_insn(struct uprobe *uprobe, struct file *filp)
867 { 565 {
868 struct address_space *mapping = uprobe 566 struct address_space *mapping = uprobe->inode->i_mapping;
869 loff_t offs = uprobe->offset; 567 loff_t offs = uprobe->offset;
870 void *insn = &uprobe->arch.insn; 568 void *insn = &uprobe->arch.insn;
871 int size = sizeof(uprobe->arch.insn); 569 int size = sizeof(uprobe->arch.insn);
872 int len, err = -EIO; 570 int len, err = -EIO;
873 571
874 /* Copy only available bytes, -EIO if 572 /* Copy only available bytes, -EIO if nothing was read */
875 do { 573 do {
876 if (offs >= i_size_read(uprobe 574 if (offs >= i_size_read(uprobe->inode))
877 break; 575 break;
878 576
879 len = min_t(int, size, PAGE_SI 577 len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
880 err = __copy_insn(mapping, fil 578 err = __copy_insn(mapping, filp, insn, len, offs);
881 if (err) 579 if (err)
882 break; 580 break;
883 581
884 insn += len; 582 insn += len;
885 offs += len; 583 offs += len;
886 size -= len; 584 size -= len;
887 } while (size); 585 } while (size);
888 586
889 return err; 587 return err;
890 } 588 }
891 589
892 static int prepare_uprobe(struct uprobe *uprob 590 static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
893 struct mm_stru 591 struct mm_struct *mm, unsigned long vaddr)
894 { 592 {
895 int ret = 0; 593 int ret = 0;
896 594
897 if (test_bit(UPROBE_COPY_INSN, &uprobe 595 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
898 return ret; 596 return ret;
899 597
900 /* TODO: move this into _register, unt 598 /* TODO: move this into _register, until then we abuse this sem. */
901 down_write(&uprobe->consumer_rwsem); 599 down_write(&uprobe->consumer_rwsem);
902 if (test_bit(UPROBE_COPY_INSN, &uprobe 600 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
903 goto out; 601 goto out;
904 602
905 ret = copy_insn(uprobe, file); 603 ret = copy_insn(uprobe, file);
906 if (ret) 604 if (ret)
907 goto out; 605 goto out;
908 606
909 ret = -ENOTSUPP; 607 ret = -ENOTSUPP;
910 if (is_trap_insn((uprobe_opcode_t *)&u 608 if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
911 goto out; 609 goto out;
912 610
913 ret = arch_uprobe_analyze_insn(&uprobe 611 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
914 if (ret) 612 if (ret)
915 goto out; 613 goto out;
916 614
917 smp_wmb(); /* pairs with the smp_rmb() !! 615 /* uprobe_write_opcode() assumes we don't cross page boundary */
>> 616 BUG_ON((uprobe->offset & ~PAGE_MASK) +
>> 617 UPROBE_SWBP_INSN_SIZE > PAGE_SIZE);
>> 618
>> 619 smp_wmb(); /* pairs with rmb() in find_active_uprobe() */
918 set_bit(UPROBE_COPY_INSN, &uprobe->fla 620 set_bit(UPROBE_COPY_INSN, &uprobe->flags);
919 621
920 out: 622 out:
921 up_write(&uprobe->consumer_rwsem); 623 up_write(&uprobe->consumer_rwsem);
922 624
923 return ret; 625 return ret;
924 } 626 }
925 627
926 static inline bool consumer_filter(struct upro !! 628 static inline bool consumer_filter(struct uprobe_consumer *uc,
>> 629 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
927 { 630 {
928 return !uc->filter || uc->filter(uc, m !! 631 return !uc->filter || uc->filter(uc, ctx, mm);
929 } 632 }
930 633
931 static bool filter_chain(struct uprobe *uprobe !! 634 static bool filter_chain(struct uprobe *uprobe,
>> 635 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
932 { 636 {
933 struct uprobe_consumer *uc; 637 struct uprobe_consumer *uc;
934 bool ret = false; 638 bool ret = false;
935 639
936 down_read(&uprobe->consumer_rwsem); 640 down_read(&uprobe->consumer_rwsem);
937 list_for_each_entry_srcu(uc, &uprobe-> !! 641 for (uc = uprobe->consumers; uc; uc = uc->next) {
938 srcu_read_loc !! 642 ret = consumer_filter(uc, ctx, mm);
939 ret = consumer_filter(uc, mm); <<
940 if (ret) 643 if (ret)
941 break; 644 break;
942 } 645 }
943 up_read(&uprobe->consumer_rwsem); 646 up_read(&uprobe->consumer_rwsem);
944 647
945 return ret; 648 return ret;
946 } 649 }
947 650
948 static int 651 static int
949 install_breakpoint(struct uprobe *uprobe, stru 652 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
950 struct vm_area_struct 653 struct vm_area_struct *vma, unsigned long vaddr)
951 { 654 {
952 bool first_uprobe; 655 bool first_uprobe;
953 int ret; 656 int ret;
954 657
955 ret = prepare_uprobe(uprobe, vma->vm_f 658 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
956 if (ret) 659 if (ret)
957 return ret; 660 return ret;
958 661
959 /* 662 /*
960 * set MMF_HAS_UPROBES in advance for 663 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
961 * the task can hit this breakpoint ri 664 * the task can hit this breakpoint right after __replace_page().
962 */ 665 */
963 first_uprobe = !test_bit(MMF_HAS_UPROB 666 first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
964 if (first_uprobe) 667 if (first_uprobe)
965 set_bit(MMF_HAS_UPROBES, &mm-> 668 set_bit(MMF_HAS_UPROBES, &mm->flags);
966 669
967 ret = set_swbp(&uprobe->arch, mm, vadd 670 ret = set_swbp(&uprobe->arch, mm, vaddr);
968 if (!ret) 671 if (!ret)
969 clear_bit(MMF_RECALC_UPROBES, 672 clear_bit(MMF_RECALC_UPROBES, &mm->flags);
970 else if (first_uprobe) 673 else if (first_uprobe)
971 clear_bit(MMF_HAS_UPROBES, &mm 674 clear_bit(MMF_HAS_UPROBES, &mm->flags);
972 675
973 return ret; 676 return ret;
974 } 677 }
975 678
976 static int 679 static int
977 remove_breakpoint(struct uprobe *uprobe, struc 680 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
978 { 681 {
979 set_bit(MMF_RECALC_UPROBES, &mm->flags 682 set_bit(MMF_RECALC_UPROBES, &mm->flags);
980 return set_orig_insn(&uprobe->arch, mm 683 return set_orig_insn(&uprobe->arch, mm, vaddr);
981 } 684 }
982 685
>> 686 static inline bool uprobe_is_active(struct uprobe *uprobe)
>> 687 {
>> 688 return !RB_EMPTY_NODE(&uprobe->rb_node);
>> 689 }
>> 690 /*
>> 691 * There could be threads that have already hit the breakpoint. They
>> 692 * will recheck the current insn and restart if find_uprobe() fails.
>> 693 * See find_active_uprobe().
>> 694 */
>> 695 static void delete_uprobe(struct uprobe *uprobe)
>> 696 {
>> 697 if (WARN_ON(!uprobe_is_active(uprobe)))
>> 698 return;
>> 699
>> 700 spin_lock(&uprobes_treelock);
>> 701 rb_erase(&uprobe->rb_node, &uprobes_tree);
>> 702 spin_unlock(&uprobes_treelock);
>> 703 RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
>> 704 iput(uprobe->inode);
>> 705 put_uprobe(uprobe);
>> 706 }
>> 707
983 struct map_info { 708 struct map_info {
984 struct map_info *next; 709 struct map_info *next;
985 struct mm_struct *mm; 710 struct mm_struct *mm;
986 unsigned long vaddr; 711 unsigned long vaddr;
987 }; 712 };
988 713
989 static inline struct map_info *free_map_info(s 714 static inline struct map_info *free_map_info(struct map_info *info)
990 { 715 {
991 struct map_info *next = info->next; 716 struct map_info *next = info->next;
992 kfree(info); 717 kfree(info);
993 return next; 718 return next;
994 } 719 }
995 720
996 static struct map_info * 721 static struct map_info *
997 build_map_info(struct address_space *mapping, 722 build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
998 { 723 {
999 unsigned long pgoff = offset >> PAGE_S 724 unsigned long pgoff = offset >> PAGE_SHIFT;
1000 struct vm_area_struct *vma; 725 struct vm_area_struct *vma;
1001 struct map_info *curr = NULL; 726 struct map_info *curr = NULL;
1002 struct map_info *prev = NULL; 727 struct map_info *prev = NULL;
1003 struct map_info *info; 728 struct map_info *info;
1004 int more = 0; 729 int more = 0;
1005 730
1006 again: 731 again:
1007 i_mmap_lock_read(mapping); 732 i_mmap_lock_read(mapping);
1008 vma_interval_tree_foreach(vma, &mappi 733 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1009 if (!valid_vma(vma, is_regist 734 if (!valid_vma(vma, is_register))
1010 continue; 735 continue;
1011 736
1012 if (!prev && !more) { 737 if (!prev && !more) {
1013 /* 738 /*
1014 * Needs GFP_NOWAIT t 739 * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
1015 * reclaim. This is o 740 * reclaim. This is optimistic, no harm done if it fails.
1016 */ 741 */
1017 prev = kmalloc(sizeof 742 prev = kmalloc(sizeof(struct map_info),
1018 GFP_N 743 GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
1019 if (prev) 744 if (prev)
1020 prev->next = 745 prev->next = NULL;
1021 } 746 }
1022 if (!prev) { 747 if (!prev) {
1023 more++; 748 more++;
1024 continue; 749 continue;
1025 } 750 }
1026 751
1027 if (!mmget_not_zero(vma->vm_m 752 if (!mmget_not_zero(vma->vm_mm))
1028 continue; 753 continue;
1029 754
1030 info = prev; 755 info = prev;
1031 prev = prev->next; 756 prev = prev->next;
1032 info->next = curr; 757 info->next = curr;
1033 curr = info; 758 curr = info;
1034 759
1035 info->mm = vma->vm_mm; 760 info->mm = vma->vm_mm;
1036 info->vaddr = offset_to_vaddr 761 info->vaddr = offset_to_vaddr(vma, offset);
1037 } 762 }
1038 i_mmap_unlock_read(mapping); 763 i_mmap_unlock_read(mapping);
1039 764
1040 if (!more) 765 if (!more)
1041 goto out; 766 goto out;
1042 767
1043 prev = curr; 768 prev = curr;
1044 while (curr) { 769 while (curr) {
1045 mmput(curr->mm); 770 mmput(curr->mm);
1046 curr = curr->next; 771 curr = curr->next;
1047 } 772 }
1048 773
1049 do { 774 do {
1050 info = kmalloc(sizeof(struct 775 info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
1051 if (!info) { 776 if (!info) {
1052 curr = ERR_PTR(-ENOME 777 curr = ERR_PTR(-ENOMEM);
1053 goto out; 778 goto out;
1054 } 779 }
1055 info->next = prev; 780 info->next = prev;
1056 prev = info; 781 prev = info;
1057 } while (--more); 782 } while (--more);
1058 783
1059 goto again; 784 goto again;
1060 out: 785 out:
1061 while (prev) 786 while (prev)
1062 prev = free_map_info(prev); 787 prev = free_map_info(prev);
1063 return curr; 788 return curr;
1064 } 789 }
1065 790
1066 static int 791 static int
1067 register_for_each_vma(struct uprobe *uprobe, 792 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
1068 { 793 {
1069 bool is_register = !!new; 794 bool is_register = !!new;
1070 struct map_info *info; 795 struct map_info *info;
1071 int err = 0; 796 int err = 0;
1072 797
1073 percpu_down_write(&dup_mmap_sem); 798 percpu_down_write(&dup_mmap_sem);
1074 info = build_map_info(uprobe->inode-> 799 info = build_map_info(uprobe->inode->i_mapping,
1075 uprob 800 uprobe->offset, is_register);
1076 if (IS_ERR(info)) { 801 if (IS_ERR(info)) {
1077 err = PTR_ERR(info); 802 err = PTR_ERR(info);
1078 goto out; 803 goto out;
1079 } 804 }
1080 805
1081 while (info) { 806 while (info) {
1082 struct mm_struct *mm = info-> 807 struct mm_struct *mm = info->mm;
1083 struct vm_area_struct *vma; 808 struct vm_area_struct *vma;
1084 809
1085 if (err && is_register) 810 if (err && is_register)
1086 goto free; 811 goto free;
1087 /* !! 812
1088 * We take mmap_lock for writ !! 813 down_write(&mm->mmap_sem);
1089 * find_active_uprobe_rcu() w <<
1090 * Thus this install_breakpoi <<
1091 * is_trap_at_addr() true rig <<
1092 * returns NULL in find_activ <<
1093 */ <<
1094 mmap_write_lock(mm); <<
1095 vma = find_vma(mm, info->vadd 814 vma = find_vma(mm, info->vaddr);
1096 if (!vma || !valid_vma(vma, i 815 if (!vma || !valid_vma(vma, is_register) ||
1097 file_inode(vma->vm_file) 816 file_inode(vma->vm_file) != uprobe->inode)
1098 goto unlock; 817 goto unlock;
1099 818
1100 if (vma->vm_start > info->vad 819 if (vma->vm_start > info->vaddr ||
1101 vaddr_to_offset(vma, info 820 vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
1102 goto unlock; 821 goto unlock;
1103 822
1104 if (is_register) { 823 if (is_register) {
1105 /* consult only the " 824 /* consult only the "caller", new consumer. */
1106 if (consumer_filter(n !! 825 if (consumer_filter(new,
>> 826 UPROBE_FILTER_REGISTER, mm))
1107 err = install 827 err = install_breakpoint(uprobe, mm, vma, info->vaddr);
1108 } else if (test_bit(MMF_HAS_U 828 } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
1109 if (!filter_chain(upr !! 829 if (!filter_chain(uprobe,
>> 830 UPROBE_FILTER_UNREGISTER, mm))
1110 err |= remove 831 err |= remove_breakpoint(uprobe, mm, info->vaddr);
1111 } 832 }
1112 833
1113 unlock: 834 unlock:
1114 mmap_write_unlock(mm); !! 835 up_write(&mm->mmap_sem);
1115 free: 836 free:
1116 mmput(mm); 837 mmput(mm);
1117 info = free_map_info(info); 838 info = free_map_info(info);
1118 } 839 }
1119 out: 840 out:
1120 percpu_up_write(&dup_mmap_sem); 841 percpu_up_write(&dup_mmap_sem);
1121 return err; 842 return err;
1122 } 843 }
1123 844
1124 /** !! 845 static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc)
1125 * uprobe_unregister_nosync - unregister an a <<
1126 * @uprobe: uprobe to remove <<
1127 * @uc: identify which probe if multiple prob <<
1128 */ <<
1129 void uprobe_unregister_nosync(struct uprobe * <<
1130 { 846 {
1131 int err; !! 847 consumer_add(uprobe, uc);
>> 848 return register_for_each_vma(uprobe, uc);
>> 849 }
1132 850
1133 down_write(&uprobe->register_rwsem); !! 851 static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
1134 consumer_del(uprobe, uc); !! 852 {
1135 err = register_for_each_vma(uprobe, N !! 853 int err;
1136 up_write(&uprobe->register_rwsem); <<
1137 854
1138 /* TODO : cant unregister? schedule a !! 855 if (WARN_ON(!consumer_del(uprobe, uc)))
1139 if (unlikely(err)) { <<
1140 uprobe_warn(current, "unregis <<
1141 return; 856 return;
1142 } <<
1143 857
1144 put_uprobe(uprobe); !! 858 err = register_for_each_vma(uprobe, NULL);
1145 } !! 859 /* TODO : cant unregister? schedule a worker thread */
1146 EXPORT_SYMBOL_GPL(uprobe_unregister_nosync); !! 860 if (!uprobe->consumers && !err)
1147 !! 861 delete_uprobe(uprobe);
1148 void uprobe_unregister_sync(void) <<
1149 { <<
1150 /* <<
1151 * Now that handler_chain() and handl <<
1152 * uprobe->consumers list under RCU p <<
1153 * uprobe->register_rwsem, we need to <<
1154 * make sure that we can't call into <<
1155 * uprobe_consumer's callbacks anymor <<
1156 * unlucky enough caller can free con <<
1157 * handler_chain() or handle_uretprob <<
1158 */ <<
1159 synchronize_srcu(&uprobes_srcu); <<
1160 } 862 }
1161 EXPORT_SYMBOL_GPL(uprobe_unregister_sync); <<
1162 863
1163 /** !! 864 /*
1164 * uprobe_register - register a probe 865 * uprobe_register - register a probe
1165 * @inode: the file in which the probe has to 866 * @inode: the file in which the probe has to be placed.
1166 * @offset: offset from the start of the file 867 * @offset: offset from the start of the file.
1167 * @ref_ctr_offset: offset of SDT marker / re <<
1168 * @uc: information on howto handle the probe 868 * @uc: information on howto handle the probe..
1169 * 869 *
1170 * Apart from the access refcount, uprobe_reg 870 * Apart from the access refcount, uprobe_register() takes a creation
1171 * refcount (thro alloc_uprobe) if and only i 871 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
1172 * inserted into the rbtree (i.e first consum 872 * inserted into the rbtree (i.e first consumer for a @inode:@offset
1173 * tuple). Creation refcount stops uprobe_un 873 * tuple). Creation refcount stops uprobe_unregister from freeing the
1174 * @uprobe even before the register operation 874 * @uprobe even before the register operation is complete. Creation
1175 * refcount is released when the last @uc for 875 * refcount is released when the last @uc for the @uprobe
1176 * unregisters. Caller of uprobe_register() i !! 876 * unregisters.
1177 * (and the containing mount) referenced. <<
1178 * 877 *
1179 * Return: pointer to the new uprobe on succe !! 878 * Return errno if it cannot successully install probes
>> 879 * else return 0 (success)
1180 */ 880 */
1181 struct uprobe *uprobe_register(struct inode * !! 881 int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
1182 loff_t offset <<
1183 struct uprobe <<
1184 { 882 {
1185 struct uprobe *uprobe; 883 struct uprobe *uprobe;
1186 int ret; 884 int ret;
1187 885
1188 /* Uprobe must have at least one set 886 /* Uprobe must have at least one set consumer */
1189 if (!uc->handler && !uc->ret_handler) 887 if (!uc->handler && !uc->ret_handler)
1190 return ERR_PTR(-EINVAL); !! 888 return -EINVAL;
1191 889
1192 /* copy_insn() uses read_mapping_page 890 /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
1193 if (!inode->i_mapping->a_ops->read_fo !! 891 if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
1194 !shmem_mapping(inode->i_mapping)) !! 892 return -EIO;
1195 return ERR_PTR(-EIO); <<
1196 /* Racy, just to catch the obvious mi 893 /* Racy, just to catch the obvious mistakes */
1197 if (offset > i_size_read(inode)) 894 if (offset > i_size_read(inode))
1198 return ERR_PTR(-EINVAL); !! 895 return -EINVAL;
1199 896
>> 897 retry:
>> 898 uprobe = alloc_uprobe(inode, offset);
>> 899 if (!uprobe)
>> 900 return -ENOMEM;
1200 /* 901 /*
1201 * This ensures that copy_from_page() !! 902 * We can race with uprobe_unregister()->delete_uprobe().
1202 * __update_ref_ctr() can't cross pag !! 903 * Check uprobe_is_active() and retry if it is false.
1203 */ 904 */
1204 if (!IS_ALIGNED(offset, UPROBE_SWBP_I <<
1205 return ERR_PTR(-EINVAL); <<
1206 if (!IS_ALIGNED(ref_ctr_offset, sizeo <<
1207 return ERR_PTR(-EINVAL); <<
1208 <<
1209 uprobe = alloc_uprobe(inode, offset, <<
1210 if (IS_ERR(uprobe)) <<
1211 return uprobe; <<
1212 <<
1213 down_write(&uprobe->register_rwsem); 905 down_write(&uprobe->register_rwsem);
1214 consumer_add(uprobe, uc); !! 906 ret = -EAGAIN;
1215 ret = register_for_each_vma(uprobe, u !! 907 if (likely(uprobe_is_active(uprobe))) {
1216 up_write(&uprobe->register_rwsem); !! 908 ret = __uprobe_register(uprobe, uc);
1217 !! 909 if (ret)
1218 if (ret) { !! 910 __uprobe_unregister(uprobe, uc);
1219 uprobe_unregister_nosync(upro <<
1220 /* <<
1221 * Registration might have pa <<
1222 * this consumer being called <<
1223 * sync here. It's ok, it's u <<
1224 */ <<
1225 uprobe_unregister_sync(); <<
1226 return ERR_PTR(ret); <<
1227 } 911 }
>> 912 up_write(&uprobe->register_rwsem);
>> 913 put_uprobe(uprobe);
1228 914
1229 return uprobe; !! 915 if (unlikely(ret == -EAGAIN))
>> 916 goto retry;
>> 917 return ret;
1230 } 918 }
1231 EXPORT_SYMBOL_GPL(uprobe_register); 919 EXPORT_SYMBOL_GPL(uprobe_register);
1232 920
1233 /** !! 921 /*
1234 * uprobe_apply - add or remove the breakpoin !! 922 * uprobe_apply - unregister a already registered probe.
1235 * @uprobe: uprobe which "owns" the breakpoin !! 923 * @inode: the file in which the probe has to be removed.
>> 924 * @offset: offset from the start of the file.
1236 * @uc: consumer which wants to add more or r 925 * @uc: consumer which wants to add more or remove some breakpoints
1237 * @add: add or remove the breakpoints 926 * @add: add or remove the breakpoints
1238 * Return: 0 on success or negative error cod <<
1239 */ 927 */
1240 int uprobe_apply(struct uprobe *uprobe, struc !! 928 int uprobe_apply(struct inode *inode, loff_t offset,
>> 929 struct uprobe_consumer *uc, bool add)
1241 { 930 {
>> 931 struct uprobe *uprobe;
1242 struct uprobe_consumer *con; 932 struct uprobe_consumer *con;
1243 int ret = -ENOENT, srcu_idx; !! 933 int ret = -ENOENT;
1244 934
1245 down_write(&uprobe->register_rwsem); !! 935 uprobe = find_uprobe(inode, offset);
1246 !! 936 if (WARN_ON(!uprobe))
1247 srcu_idx = srcu_read_lock(&uprobes_sr !! 937 return ret;
1248 list_for_each_entry_srcu(con, &uprobe <<
1249 srcu_read_lo <<
1250 if (con == uc) { <<
1251 ret = register_for_ea <<
1252 break; <<
1253 } <<
1254 } <<
1255 srcu_read_unlock(&uprobes_srcu, srcu_ <<
1256 938
>> 939 down_write(&uprobe->register_rwsem);
>> 940 for (con = uprobe->consumers; con && con != uc ; con = con->next)
>> 941 ;
>> 942 if (con)
>> 943 ret = register_for_each_vma(uprobe, add ? uc : NULL);
1257 up_write(&uprobe->register_rwsem); 944 up_write(&uprobe->register_rwsem);
>> 945 put_uprobe(uprobe);
1258 946
1259 return ret; 947 return ret;
1260 } 948 }
1261 949
>> 950 /*
>> 951 * uprobe_unregister - unregister a already registered probe.
>> 952 * @inode: the file in which the probe has to be removed.
>> 953 * @offset: offset from the start of the file.
>> 954 * @uc: identify which probe if multiple probes are colocated.
>> 955 */
>> 956 void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
>> 957 {
>> 958 struct uprobe *uprobe;
>> 959
>> 960 uprobe = find_uprobe(inode, offset);
>> 961 if (WARN_ON(!uprobe))
>> 962 return;
>> 963
>> 964 down_write(&uprobe->register_rwsem);
>> 965 __uprobe_unregister(uprobe, uc);
>> 966 up_write(&uprobe->register_rwsem);
>> 967 put_uprobe(uprobe);
>> 968 }
>> 969 EXPORT_SYMBOL_GPL(uprobe_unregister);
>> 970
1262 static int unapply_uprobe(struct uprobe *upro 971 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
1263 { 972 {
1264 VMA_ITERATOR(vmi, mm, 0); <<
1265 struct vm_area_struct *vma; 973 struct vm_area_struct *vma;
1266 int err = 0; 974 int err = 0;
1267 975
1268 mmap_read_lock(mm); !! 976 down_read(&mm->mmap_sem);
1269 for_each_vma(vmi, vma) { !! 977 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1270 unsigned long vaddr; 978 unsigned long vaddr;
1271 loff_t offset; 979 loff_t offset;
1272 980
1273 if (!valid_vma(vma, false) || 981 if (!valid_vma(vma, false) ||
1274 file_inode(vma->vm_file) 982 file_inode(vma->vm_file) != uprobe->inode)
1275 continue; 983 continue;
1276 984
1277 offset = (loff_t)vma->vm_pgof 985 offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1278 if (uprobe->offset < offset 986 if (uprobe->offset < offset ||
1279 uprobe->offset >= offset 987 uprobe->offset >= offset + vma->vm_end - vma->vm_start)
1280 continue; 988 continue;
1281 989
1282 vaddr = offset_to_vaddr(vma, 990 vaddr = offset_to_vaddr(vma, uprobe->offset);
1283 err |= remove_breakpoint(upro 991 err |= remove_breakpoint(uprobe, mm, vaddr);
1284 } 992 }
1285 mmap_read_unlock(mm); !! 993 up_read(&mm->mmap_sem);
1286 994
1287 return err; 995 return err;
1288 } 996 }
1289 997
1290 static struct rb_node * 998 static struct rb_node *
1291 find_node_in_range(struct inode *inode, loff_ 999 find_node_in_range(struct inode *inode, loff_t min, loff_t max)
1292 { 1000 {
1293 struct rb_node *n = uprobes_tree.rb_n 1001 struct rb_node *n = uprobes_tree.rb_node;
1294 1002
1295 while (n) { 1003 while (n) {
1296 struct uprobe *u = rb_entry(n 1004 struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
1297 1005
1298 if (inode < u->inode) { 1006 if (inode < u->inode) {
1299 n = n->rb_left; 1007 n = n->rb_left;
1300 } else if (inode > u->inode) 1008 } else if (inode > u->inode) {
1301 n = n->rb_right; 1009 n = n->rb_right;
1302 } else { 1010 } else {
1303 if (max < u->offset) 1011 if (max < u->offset)
1304 n = n->rb_lef 1012 n = n->rb_left;
1305 else if (min > u->off 1013 else if (min > u->offset)
1306 n = n->rb_rig 1014 n = n->rb_right;
1307 else 1015 else
1308 break; 1016 break;
1309 } 1017 }
1310 } 1018 }
1311 1019
1312 return n; 1020 return n;
1313 } 1021 }
1314 1022
1315 /* 1023 /*
1316 * For a given range in vma, build a list of 1024 * For a given range in vma, build a list of probes that need to be inserted.
1317 */ 1025 */
1318 static void build_probe_list(struct inode *in 1026 static void build_probe_list(struct inode *inode,
1319 struct vm_are 1027 struct vm_area_struct *vma,
1320 unsigned long 1028 unsigned long start, unsigned long end,
1321 struct list_h 1029 struct list_head *head)
1322 { 1030 {
1323 loff_t min, max; 1031 loff_t min, max;
1324 struct rb_node *n, *t; 1032 struct rb_node *n, *t;
1325 struct uprobe *u; 1033 struct uprobe *u;
1326 1034
1327 INIT_LIST_HEAD(head); 1035 INIT_LIST_HEAD(head);
1328 min = vaddr_to_offset(vma, start); 1036 min = vaddr_to_offset(vma, start);
1329 max = min + (end - start) - 1; 1037 max = min + (end - start) - 1;
1330 1038
1331 read_lock(&uprobes_treelock); !! 1039 spin_lock(&uprobes_treelock);
1332 n = find_node_in_range(inode, min, ma 1040 n = find_node_in_range(inode, min, max);
1333 if (n) { 1041 if (n) {
1334 for (t = n; t; t = rb_prev(t) 1042 for (t = n; t; t = rb_prev(t)) {
1335 u = rb_entry(t, struc 1043 u = rb_entry(t, struct uprobe, rb_node);
1336 if (u->inode != inode 1044 if (u->inode != inode || u->offset < min)
1337 break; 1045 break;
1338 /* if uprobe went awa !! 1046 list_add(&u->pending_list, head);
1339 if (try_get_uprobe(u) !! 1047 get_uprobe(u);
1340 list_add(&u-> <<
1341 } 1048 }
1342 for (t = n; (t = rb_next(t)); 1049 for (t = n; (t = rb_next(t)); ) {
1343 u = rb_entry(t, struc 1050 u = rb_entry(t, struct uprobe, rb_node);
1344 if (u->inode != inode 1051 if (u->inode != inode || u->offset > max)
1345 break; 1052 break;
1346 /* if uprobe went awa !! 1053 list_add(&u->pending_list, head);
1347 if (try_get_uprobe(u) !! 1054 get_uprobe(u);
1348 list_add(&u-> <<
1349 } 1055 }
1350 } 1056 }
1351 read_unlock(&uprobes_treelock); !! 1057 spin_unlock(&uprobes_treelock);
1352 } <<
1353 <<
1354 /* @vma contains reference counter, not the p <<
1355 static int delayed_ref_ctr_inc(struct vm_area <<
1356 { <<
1357 struct list_head *pos, *q; <<
1358 struct delayed_uprobe *du; <<
1359 unsigned long vaddr; <<
1360 int ret = 0, err = 0; <<
1361 <<
1362 mutex_lock(&delayed_uprobe_lock); <<
1363 list_for_each_safe(pos, q, &delayed_u <<
1364 du = list_entry(pos, struct d <<
1365 <<
1366 if (du->mm != vma->vm_mm || <<
1367 !valid_ref_ctr_vma(du->up <<
1368 continue; <<
1369 <<
1370 vaddr = offset_to_vaddr(vma, <<
1371 ret = __update_ref_ctr(vma->v <<
1372 if (ret) { <<
1373 update_ref_ctr_warn(d <<
1374 if (!err) <<
1375 err = ret; <<
1376 } <<
1377 delayed_uprobe_delete(du); <<
1378 } <<
1379 mutex_unlock(&delayed_uprobe_lock); <<
1380 return err; <<
1381 } 1058 }
1382 1059
1383 /* 1060 /*
1384 * Called from mmap_region/vma_merge with mm- !! 1061 * Called from mmap_region/vma_adjust with mm->mmap_sem acquired.
1385 * 1062 *
1386 * Currently we ignore all errors and always 1063 * Currently we ignore all errors and always return 0, the callers
1387 * can't handle the failure anyway. 1064 * can't handle the failure anyway.
1388 */ 1065 */
1389 int uprobe_mmap(struct vm_area_struct *vma) 1066 int uprobe_mmap(struct vm_area_struct *vma)
1390 { 1067 {
1391 struct list_head tmp_list; 1068 struct list_head tmp_list;
1392 struct uprobe *uprobe, *u; 1069 struct uprobe *uprobe, *u;
1393 struct inode *inode; 1070 struct inode *inode;
1394 1071
1395 if (no_uprobe_events()) !! 1072 if (no_uprobe_events() || !valid_vma(vma, true))
1396 return 0; <<
1397 <<
1398 if (vma->vm_file && <<
1399 (vma->vm_flags & (VM_WRITE|VM_SHA <<
1400 test_bit(MMF_HAS_UPROBES, &vma->v <<
1401 delayed_ref_ctr_inc(vma); <<
1402 <<
1403 if (!valid_vma(vma, true)) <<
1404 return 0; 1073 return 0;
1405 1074
1406 inode = file_inode(vma->vm_file); 1075 inode = file_inode(vma->vm_file);
1407 if (!inode) 1076 if (!inode)
1408 return 0; 1077 return 0;
1409 1078
1410 mutex_lock(uprobes_mmap_hash(inode)); 1079 mutex_lock(uprobes_mmap_hash(inode));
1411 build_probe_list(inode, vma, vma->vm_ 1080 build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
1412 /* 1081 /*
1413 * We can race with uprobe_unregister 1082 * We can race with uprobe_unregister(), this uprobe can be already
1414 * removed. But in this case filter_c 1083 * removed. But in this case filter_chain() must return false, all
1415 * consumers have gone away. 1084 * consumers have gone away.
1416 */ 1085 */
1417 list_for_each_entry_safe(uprobe, u, & 1086 list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
1418 if (!fatal_signal_pending(cur 1087 if (!fatal_signal_pending(current) &&
1419 filter_chain(uprobe, vma- !! 1088 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
1420 unsigned long vaddr = 1089 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
1421 install_breakpoint(up 1090 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
1422 } 1091 }
1423 put_uprobe(uprobe); 1092 put_uprobe(uprobe);
1424 } 1093 }
1425 mutex_unlock(uprobes_mmap_hash(inode) 1094 mutex_unlock(uprobes_mmap_hash(inode));
1426 1095
1427 return 0; 1096 return 0;
1428 } 1097 }
1429 1098
1430 static bool 1099 static bool
1431 vma_has_uprobes(struct vm_area_struct *vma, u 1100 vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1432 { 1101 {
1433 loff_t min, max; 1102 loff_t min, max;
1434 struct inode *inode; 1103 struct inode *inode;
1435 struct rb_node *n; 1104 struct rb_node *n;
1436 1105
1437 inode = file_inode(vma->vm_file); 1106 inode = file_inode(vma->vm_file);
1438 1107
1439 min = vaddr_to_offset(vma, start); 1108 min = vaddr_to_offset(vma, start);
1440 max = min + (end - start) - 1; 1109 max = min + (end - start) - 1;
1441 1110
1442 read_lock(&uprobes_treelock); !! 1111 spin_lock(&uprobes_treelock);
1443 n = find_node_in_range(inode, min, ma 1112 n = find_node_in_range(inode, min, max);
1444 read_unlock(&uprobes_treelock); !! 1113 spin_unlock(&uprobes_treelock);
1445 1114
1446 return !!n; 1115 return !!n;
1447 } 1116 }
1448 1117
1449 /* 1118 /*
1450 * Called in context of a munmap of a vma. 1119 * Called in context of a munmap of a vma.
1451 */ 1120 */
1452 void uprobe_munmap(struct vm_area_struct *vma 1121 void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1453 { 1122 {
1454 if (no_uprobe_events() || !valid_vma( 1123 if (no_uprobe_events() || !valid_vma(vma, false))
1455 return; 1124 return;
1456 1125
1457 if (!atomic_read(&vma->vm_mm->mm_user 1126 if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
1458 return; 1127 return;
1459 1128
1460 if (!test_bit(MMF_HAS_UPROBES, &vma-> 1129 if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
1461 test_bit(MMF_RECALC_UPROBES, &vm 1130 test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
1462 return; 1131 return;
1463 1132
1464 if (vma_has_uprobes(vma, start, end)) 1133 if (vma_has_uprobes(vma, start, end))
1465 set_bit(MMF_RECALC_UPROBES, & 1134 set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
1466 } 1135 }
1467 1136
1468 static vm_fault_t xol_fault(const struct vm_s <<
1469 struct vm_area_st <<
1470 { <<
1471 struct xol_area *area = vma->vm_mm->u <<
1472 <<
1473 vmf->page = area->page; <<
1474 get_page(vmf->page); <<
1475 return 0; <<
1476 } <<
1477 <<
1478 static const struct vm_special_mapping xol_ma <<
1479 .name = "[uprobes]", <<
1480 .fault = xol_fault, <<
1481 }; <<
1482 <<
1483 /* Slot allocation for XOL */ 1137 /* Slot allocation for XOL */
1484 static int xol_add_vma(struct mm_struct *mm, 1138 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1485 { 1139 {
1486 struct vm_area_struct *vma; 1140 struct vm_area_struct *vma;
1487 int ret; 1141 int ret;
1488 1142
1489 if (mmap_write_lock_killable(mm)) !! 1143 if (down_write_killable(&mm->mmap_sem))
1490 return -EINTR; 1144 return -EINTR;
1491 1145
1492 if (mm->uprobes_state.xol_area) { 1146 if (mm->uprobes_state.xol_area) {
1493 ret = -EALREADY; 1147 ret = -EALREADY;
1494 goto fail; 1148 goto fail;
1495 } 1149 }
1496 1150
1497 if (!area->vaddr) { 1151 if (!area->vaddr) {
1498 /* Try to map as high as poss 1152 /* Try to map as high as possible, this is only a hint. */
1499 area->vaddr = get_unmapped_ar 1153 area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1500 1154 PAGE_SIZE, 0, 0);
1501 if (IS_ERR_VALUE(area->vaddr) !! 1155 if (area->vaddr & ~PAGE_MASK) {
1502 ret = area->vaddr; 1156 ret = area->vaddr;
1503 goto fail; 1157 goto fail;
1504 } 1158 }
1505 } 1159 }
1506 1160
1507 vma = _install_special_mapping(mm, ar 1161 vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1508 VM_EXEC|VM_MA 1162 VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
1509 &xol_mapping) !! 1163 &area->xol_mapping);
1510 if (IS_ERR(vma)) { 1164 if (IS_ERR(vma)) {
1511 ret = PTR_ERR(vma); 1165 ret = PTR_ERR(vma);
1512 goto fail; 1166 goto fail;
1513 } 1167 }
1514 1168
1515 ret = 0; 1169 ret = 0;
1516 /* pairs with get_xol_area() */ 1170 /* pairs with get_xol_area() */
1517 smp_store_release(&mm->uprobes_state. 1171 smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
1518 fail: 1172 fail:
1519 mmap_write_unlock(mm); !! 1173 up_write(&mm->mmap_sem);
1520 1174
1521 return ret; 1175 return ret;
1522 } 1176 }
1523 1177
1524 void * __weak arch_uprobe_trampoline(unsigned <<
1525 { <<
1526 static uprobe_opcode_t insn = UPROBE_ <<
1527 <<
1528 *psize = UPROBE_SWBP_INSN_SIZE; <<
1529 return &insn; <<
1530 } <<
1531 <<
1532 static struct xol_area *__create_xol_area(uns 1178 static struct xol_area *__create_xol_area(unsigned long vaddr)
1533 { 1179 {
1534 struct mm_struct *mm = current->mm; 1180 struct mm_struct *mm = current->mm;
1535 unsigned long insns_size; !! 1181 uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1536 struct xol_area *area; 1182 struct xol_area *area;
1537 void *insns; <<
1538 1183
1539 area = kzalloc(sizeof(*area), GFP_KER !! 1184 area = kmalloc(sizeof(*area), GFP_KERNEL);
1540 if (unlikely(!area)) 1185 if (unlikely(!area))
1541 goto out; 1186 goto out;
1542 1187
1543 area->bitmap = kcalloc(BITS_TO_LONGS( !! 1188 area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL);
1544 GFP_KERNEL); <<
1545 if (!area->bitmap) 1189 if (!area->bitmap)
1546 goto free_area; 1190 goto free_area;
1547 1191
1548 area->page = alloc_page(GFP_HIGHUSER !! 1192 area->xol_mapping.name = "[uprobes]";
1549 if (!area->page) !! 1193 area->xol_mapping.fault = NULL;
>> 1194 area->xol_mapping.pages = area->pages;
>> 1195 area->pages[0] = alloc_page(GFP_HIGHUSER);
>> 1196 if (!area->pages[0])
1550 goto free_bitmap; 1197 goto free_bitmap;
>> 1198 area->pages[1] = NULL;
1551 1199
1552 area->vaddr = vaddr; 1200 area->vaddr = vaddr;
1553 init_waitqueue_head(&area->wq); 1201 init_waitqueue_head(&area->wq);
1554 /* Reserve the 1st slot for get_tramp 1202 /* Reserve the 1st slot for get_trampoline_vaddr() */
1555 set_bit(0, area->bitmap); 1203 set_bit(0, area->bitmap);
1556 atomic_set(&area->slot_count, 1); 1204 atomic_set(&area->slot_count, 1);
1557 insns = arch_uprobe_trampoline(&insns !! 1205 arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
1558 arch_uprobe_copy_ixol(area->page, 0, <<
1559 1206
1560 if (!xol_add_vma(mm, area)) 1207 if (!xol_add_vma(mm, area))
1561 return area; 1208 return area;
1562 1209
1563 __free_page(area->page); !! 1210 __free_page(area->pages[0]);
1564 free_bitmap: 1211 free_bitmap:
1565 kfree(area->bitmap); 1212 kfree(area->bitmap);
1566 free_area: 1213 free_area:
1567 kfree(area); 1214 kfree(area);
1568 out: 1215 out:
1569 return NULL; 1216 return NULL;
1570 } 1217 }
1571 1218
1572 /* 1219 /*
1573 * get_xol_area - Allocate process's xol_area 1220 * get_xol_area - Allocate process's xol_area if necessary.
1574 * This area will be used for storing instruc 1221 * This area will be used for storing instructions for execution out of line.
1575 * 1222 *
1576 * Returns the allocated area or NULL. 1223 * Returns the allocated area or NULL.
1577 */ 1224 */
1578 static struct xol_area *get_xol_area(void) 1225 static struct xol_area *get_xol_area(void)
1579 { 1226 {
1580 struct mm_struct *mm = current->mm; 1227 struct mm_struct *mm = current->mm;
1581 struct xol_area *area; 1228 struct xol_area *area;
1582 1229
1583 if (!mm->uprobes_state.xol_area) 1230 if (!mm->uprobes_state.xol_area)
1584 __create_xol_area(0); 1231 __create_xol_area(0);
1585 1232
1586 /* Pairs with xol_add_vma() smp_store 1233 /* Pairs with xol_add_vma() smp_store_release() */
1587 area = READ_ONCE(mm->uprobes_state.xo 1234 area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
1588 return area; 1235 return area;
1589 } 1236 }
1590 1237
1591 /* 1238 /*
1592 * uprobe_clear_state - Free the area allocat 1239 * uprobe_clear_state - Free the area allocated for slots.
1593 */ 1240 */
1594 void uprobe_clear_state(struct mm_struct *mm) 1241 void uprobe_clear_state(struct mm_struct *mm)
1595 { 1242 {
1596 struct xol_area *area = mm->uprobes_s 1243 struct xol_area *area = mm->uprobes_state.xol_area;
1597 1244
1598 mutex_lock(&delayed_uprobe_lock); <<
1599 delayed_uprobe_remove(NULL, mm); <<
1600 mutex_unlock(&delayed_uprobe_lock); <<
1601 <<
1602 if (!area) 1245 if (!area)
1603 return; 1246 return;
1604 1247
1605 put_page(area->page); !! 1248 put_page(area->pages[0]);
1606 kfree(area->bitmap); 1249 kfree(area->bitmap);
1607 kfree(area); 1250 kfree(area);
1608 } 1251 }
1609 1252
1610 void uprobe_start_dup_mmap(void) 1253 void uprobe_start_dup_mmap(void)
1611 { 1254 {
1612 percpu_down_read(&dup_mmap_sem); 1255 percpu_down_read(&dup_mmap_sem);
1613 } 1256 }
1614 1257
1615 void uprobe_end_dup_mmap(void) 1258 void uprobe_end_dup_mmap(void)
1616 { 1259 {
1617 percpu_up_read(&dup_mmap_sem); 1260 percpu_up_read(&dup_mmap_sem);
1618 } 1261 }
1619 1262
1620 void uprobe_dup_mmap(struct mm_struct *oldmm, 1263 void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
1621 { 1264 {
1622 if (test_bit(MMF_HAS_UPROBES, &oldmm- 1265 if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
1623 set_bit(MMF_HAS_UPROBES, &new 1266 set_bit(MMF_HAS_UPROBES, &newmm->flags);
1624 /* unconditionally, dup_mmap( 1267 /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
1625 set_bit(MMF_RECALC_UPROBES, & 1268 set_bit(MMF_RECALC_UPROBES, &newmm->flags);
1626 } 1269 }
1627 } 1270 }
1628 1271
1629 /* 1272 /*
1630 * - search for a free slot. 1273 * - search for a free slot.
1631 */ 1274 */
1632 static unsigned long xol_take_insn_slot(struc 1275 static unsigned long xol_take_insn_slot(struct xol_area *area)
1633 { 1276 {
1634 unsigned long slot_addr; 1277 unsigned long slot_addr;
1635 int slot_nr; 1278 int slot_nr;
1636 1279
1637 do { 1280 do {
1638 slot_nr = find_first_zero_bit 1281 slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1639 if (slot_nr < UINSNS_PER_PAGE 1282 if (slot_nr < UINSNS_PER_PAGE) {
1640 if (!test_and_set_bit 1283 if (!test_and_set_bit(slot_nr, area->bitmap))
1641 break; 1284 break;
1642 1285
1643 slot_nr = UINSNS_PER_ 1286 slot_nr = UINSNS_PER_PAGE;
1644 continue; 1287 continue;
1645 } 1288 }
1646 wait_event(area->wq, (atomic_ 1289 wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1647 } while (slot_nr >= UINSNS_PER_PAGE); 1290 } while (slot_nr >= UINSNS_PER_PAGE);
1648 1291
1649 slot_addr = area->vaddr + (slot_nr * 1292 slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1650 atomic_inc(&area->slot_count); 1293 atomic_inc(&area->slot_count);
1651 1294
1652 return slot_addr; 1295 return slot_addr;
1653 } 1296 }
1654 1297
1655 /* 1298 /*
1656 * xol_get_insn_slot - allocate a slot for xo 1299 * xol_get_insn_slot - allocate a slot for xol.
1657 * Returns the allocated slot address or 0. 1300 * Returns the allocated slot address or 0.
1658 */ 1301 */
1659 static unsigned long xol_get_insn_slot(struct 1302 static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
1660 { 1303 {
1661 struct xol_area *area; 1304 struct xol_area *area;
1662 unsigned long xol_vaddr; 1305 unsigned long xol_vaddr;
1663 1306
1664 area = get_xol_area(); 1307 area = get_xol_area();
1665 if (!area) 1308 if (!area)
1666 return 0; 1309 return 0;
1667 1310
1668 xol_vaddr = xol_take_insn_slot(area); 1311 xol_vaddr = xol_take_insn_slot(area);
1669 if (unlikely(!xol_vaddr)) 1312 if (unlikely(!xol_vaddr))
1670 return 0; 1313 return 0;
1671 1314
1672 arch_uprobe_copy_ixol(area->page, xol !! 1315 arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
1673 &uprobe->arch.i 1316 &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1674 1317
1675 return xol_vaddr; 1318 return xol_vaddr;
1676 } 1319 }
1677 1320
1678 /* 1321 /*
1679 * xol_free_insn_slot - If slot was earlier a 1322 * xol_free_insn_slot - If slot was earlier allocated by
1680 * @xol_get_insn_slot(), make the slot availa 1323 * @xol_get_insn_slot(), make the slot available for
1681 * subsequent requests. 1324 * subsequent requests.
1682 */ 1325 */
1683 static void xol_free_insn_slot(struct task_st 1326 static void xol_free_insn_slot(struct task_struct *tsk)
1684 { 1327 {
1685 struct xol_area *area; 1328 struct xol_area *area;
1686 unsigned long vma_end; 1329 unsigned long vma_end;
1687 unsigned long slot_addr; 1330 unsigned long slot_addr;
1688 1331
1689 if (!tsk->mm || !tsk->mm->uprobes_sta 1332 if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
1690 return; 1333 return;
1691 1334
1692 slot_addr = tsk->utask->xol_vaddr; 1335 slot_addr = tsk->utask->xol_vaddr;
1693 if (unlikely(!slot_addr)) 1336 if (unlikely(!slot_addr))
1694 return; 1337 return;
1695 1338
1696 area = tsk->mm->uprobes_state.xol_are 1339 area = tsk->mm->uprobes_state.xol_area;
1697 vma_end = area->vaddr + PAGE_SIZE; 1340 vma_end = area->vaddr + PAGE_SIZE;
1698 if (area->vaddr <= slot_addr && slot_ 1341 if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1699 unsigned long offset; 1342 unsigned long offset;
1700 int slot_nr; 1343 int slot_nr;
1701 1344
1702 offset = slot_addr - area->va 1345 offset = slot_addr - area->vaddr;
1703 slot_nr = offset / UPROBE_XOL 1346 slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1704 if (slot_nr >= UINSNS_PER_PAG 1347 if (slot_nr >= UINSNS_PER_PAGE)
1705 return; 1348 return;
1706 1349
1707 clear_bit(slot_nr, area->bitm 1350 clear_bit(slot_nr, area->bitmap);
1708 atomic_dec(&area->slot_count) 1351 atomic_dec(&area->slot_count);
1709 smp_mb__after_atomic(); /* pa 1352 smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1710 if (waitqueue_active(&area->w 1353 if (waitqueue_active(&area->wq))
1711 wake_up(&area->wq); 1354 wake_up(&area->wq);
1712 1355
1713 tsk->utask->xol_vaddr = 0; 1356 tsk->utask->xol_vaddr = 0;
1714 } 1357 }
1715 } 1358 }
1716 1359
1717 void __weak arch_uprobe_copy_ixol(struct page 1360 void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
1718 void *src, 1361 void *src, unsigned long len)
1719 { 1362 {
1720 /* Initialize the slot */ 1363 /* Initialize the slot */
1721 copy_to_page(page, vaddr, src, len); 1364 copy_to_page(page, vaddr, src, len);
1722 1365
1723 /* 1366 /*
1724 * We probably need flush_icache_user !! 1367 * We probably need flush_icache_user_range() but it needs vma.
1725 * This should work on most of archit 1368 * This should work on most of architectures by default. If
1726 * architecture needs to do something 1369 * architecture needs to do something different it can define
1727 * its own version of the function. 1370 * its own version of the function.
1728 */ 1371 */
1729 flush_dcache_page(page); 1372 flush_dcache_page(page);
1730 } 1373 }
1731 1374
1732 /** 1375 /**
1733 * uprobe_get_swbp_addr - compute address of 1376 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1734 * @regs: Reflects the saved state of the tas 1377 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1735 * instruction. 1378 * instruction.
1736 * Return the address of the breakpoint instr 1379 * Return the address of the breakpoint instruction.
1737 */ 1380 */
1738 unsigned long __weak uprobe_get_swbp_addr(str 1381 unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1739 { 1382 {
1740 return instruction_pointer(regs) - UP 1383 return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1741 } 1384 }
1742 1385
1743 unsigned long uprobe_get_trap_addr(struct pt_ 1386 unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
1744 { 1387 {
1745 struct uprobe_task *utask = current-> 1388 struct uprobe_task *utask = current->utask;
1746 1389
1747 if (unlikely(utask && utask->active_u 1390 if (unlikely(utask && utask->active_uprobe))
1748 return utask->vaddr; 1391 return utask->vaddr;
1749 1392
1750 return instruction_pointer(regs); 1393 return instruction_pointer(regs);
1751 } 1394 }
1752 1395
1753 static struct return_instance *free_ret_insta 1396 static struct return_instance *free_ret_instance(struct return_instance *ri)
1754 { 1397 {
1755 struct return_instance *next = ri->ne 1398 struct return_instance *next = ri->next;
1756 put_uprobe(ri->uprobe); 1399 put_uprobe(ri->uprobe);
1757 kfree(ri); 1400 kfree(ri);
1758 return next; 1401 return next;
1759 } 1402 }
1760 1403
1761 /* 1404 /*
1762 * Called with no locks held. 1405 * Called with no locks held.
1763 * Called in context of an exiting or an exec !! 1406 * Called in context of a exiting or a exec-ing thread.
1764 */ 1407 */
1765 void uprobe_free_utask(struct task_struct *t) 1408 void uprobe_free_utask(struct task_struct *t)
1766 { 1409 {
1767 struct uprobe_task *utask = t->utask; 1410 struct uprobe_task *utask = t->utask;
1768 struct return_instance *ri; 1411 struct return_instance *ri;
1769 1412
1770 if (!utask) 1413 if (!utask)
1771 return; 1414 return;
1772 1415
1773 if (utask->active_uprobe) 1416 if (utask->active_uprobe)
1774 put_uprobe(utask->active_upro 1417 put_uprobe(utask->active_uprobe);
1775 1418
1776 ri = utask->return_instances; 1419 ri = utask->return_instances;
1777 while (ri) 1420 while (ri)
1778 ri = free_ret_instance(ri); 1421 ri = free_ret_instance(ri);
1779 1422
1780 xol_free_insn_slot(t); 1423 xol_free_insn_slot(t);
1781 kfree(utask); 1424 kfree(utask);
1782 t->utask = NULL; 1425 t->utask = NULL;
1783 } 1426 }
1784 1427
1785 /* 1428 /*
1786 * Allocate a uprobe_task object for the task !! 1429 * Allocate a uprobe_task object for the task if if necessary.
1787 * Called when the thread hits a breakpoint. 1430 * Called when the thread hits a breakpoint.
1788 * 1431 *
1789 * Returns: 1432 * Returns:
1790 * - pointer to new uprobe_task on success 1433 * - pointer to new uprobe_task on success
1791 * - NULL otherwise 1434 * - NULL otherwise
1792 */ 1435 */
1793 static struct uprobe_task *get_utask(void) 1436 static struct uprobe_task *get_utask(void)
1794 { 1437 {
1795 if (!current->utask) 1438 if (!current->utask)
1796 current->utask = kzalloc(size 1439 current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1797 return current->utask; 1440 return current->utask;
1798 } 1441 }
1799 1442
1800 static int dup_utask(struct task_struct *t, s 1443 static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
1801 { 1444 {
1802 struct uprobe_task *n_utask; 1445 struct uprobe_task *n_utask;
1803 struct return_instance **p, *o, *n; 1446 struct return_instance **p, *o, *n;
1804 1447
1805 n_utask = kzalloc(sizeof(struct uprob 1448 n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1806 if (!n_utask) 1449 if (!n_utask)
1807 return -ENOMEM; 1450 return -ENOMEM;
1808 t->utask = n_utask; 1451 t->utask = n_utask;
1809 1452
1810 p = &n_utask->return_instances; 1453 p = &n_utask->return_instances;
1811 for (o = o_utask->return_instances; o 1454 for (o = o_utask->return_instances; o; o = o->next) {
1812 n = kmalloc(sizeof(struct ret 1455 n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1813 if (!n) 1456 if (!n)
1814 return -ENOMEM; 1457 return -ENOMEM;
1815 1458
1816 *n = *o; 1459 *n = *o;
1817 /* <<
1818 * uprobe's refcnt has to be <<
1819 * utask->return_instances it <<
1820 * removed right now, as task <<
1821 * get_uprobe() is safe to us <<
1822 */ <<
1823 get_uprobe(n->uprobe); 1460 get_uprobe(n->uprobe);
1824 n->next = NULL; 1461 n->next = NULL;
1825 1462
1826 *p = n; 1463 *p = n;
1827 p = &n->next; 1464 p = &n->next;
1828 n_utask->depth++; 1465 n_utask->depth++;
1829 } 1466 }
1830 1467
1831 return 0; 1468 return 0;
1832 } 1469 }
1833 1470
>> 1471 static void uprobe_warn(struct task_struct *t, const char *msg)
>> 1472 {
>> 1473 pr_warn("uprobe: %s:%d failed to %s\n",
>> 1474 current->comm, current->pid, msg);
>> 1475 }
>> 1476
1834 static void dup_xol_work(struct callback_head 1477 static void dup_xol_work(struct callback_head *work)
1835 { 1478 {
1836 if (current->flags & PF_EXITING) 1479 if (current->flags & PF_EXITING)
1837 return; 1480 return;
1838 1481
1839 if (!__create_xol_area(current->utask 1482 if (!__create_xol_area(current->utask->dup_xol_addr) &&
1840 !fatal_signal_pending 1483 !fatal_signal_pending(current))
1841 uprobe_warn(current, "dup xol 1484 uprobe_warn(current, "dup xol area");
1842 } 1485 }
1843 1486
1844 /* 1487 /*
1845 * Called in context of a new clone/fork from 1488 * Called in context of a new clone/fork from copy_process.
1846 */ 1489 */
1847 void uprobe_copy_process(struct task_struct * 1490 void uprobe_copy_process(struct task_struct *t, unsigned long flags)
1848 { 1491 {
1849 struct uprobe_task *utask = current-> 1492 struct uprobe_task *utask = current->utask;
1850 struct mm_struct *mm = current->mm; 1493 struct mm_struct *mm = current->mm;
1851 struct xol_area *area; 1494 struct xol_area *area;
1852 1495
1853 t->utask = NULL; 1496 t->utask = NULL;
1854 1497
1855 if (!utask || !utask->return_instance 1498 if (!utask || !utask->return_instances)
1856 return; 1499 return;
1857 1500
1858 if (mm == t->mm && !(flags & CLONE_VF 1501 if (mm == t->mm && !(flags & CLONE_VFORK))
1859 return; 1502 return;
1860 1503
1861 if (dup_utask(t, utask)) 1504 if (dup_utask(t, utask))
1862 return uprobe_warn(t, "dup re 1505 return uprobe_warn(t, "dup ret instances");
1863 1506
1864 /* The task can fork() after dup_xol_ 1507 /* The task can fork() after dup_xol_work() fails */
1865 area = mm->uprobes_state.xol_area; 1508 area = mm->uprobes_state.xol_area;
1866 if (!area) 1509 if (!area)
1867 return uprobe_warn(t, "dup xo 1510 return uprobe_warn(t, "dup xol area");
1868 1511
1869 if (mm == t->mm) 1512 if (mm == t->mm)
1870 return; 1513 return;
1871 1514
1872 t->utask->dup_xol_addr = area->vaddr; 1515 t->utask->dup_xol_addr = area->vaddr;
1873 init_task_work(&t->utask->dup_xol_wor 1516 init_task_work(&t->utask->dup_xol_work, dup_xol_work);
1874 task_work_add(t, &t->utask->dup_xol_w !! 1517 task_work_add(t, &t->utask->dup_xol_work, true);
1875 } 1518 }
1876 1519
1877 /* 1520 /*
1878 * Current area->vaddr notion assume the tram 1521 * Current area->vaddr notion assume the trampoline address is always
1879 * equal area->vaddr. 1522 * equal area->vaddr.
1880 * 1523 *
1881 * Returns -1 in case the xol_area is not all 1524 * Returns -1 in case the xol_area is not allocated.
1882 */ 1525 */
1883 unsigned long uprobe_get_trampoline_vaddr(voi !! 1526 static unsigned long get_trampoline_vaddr(void)
1884 { 1527 {
1885 struct xol_area *area; 1528 struct xol_area *area;
1886 unsigned long trampoline_vaddr = -1; 1529 unsigned long trampoline_vaddr = -1;
1887 1530
1888 /* Pairs with xol_add_vma() smp_store 1531 /* Pairs with xol_add_vma() smp_store_release() */
1889 area = READ_ONCE(current->mm->uprobes 1532 area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
1890 if (area) 1533 if (area)
1891 trampoline_vaddr = area->vadd 1534 trampoline_vaddr = area->vaddr;
1892 1535
1893 return trampoline_vaddr; 1536 return trampoline_vaddr;
1894 } 1537 }
1895 1538
1896 static void cleanup_return_instances(struct u 1539 static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
1897 struc 1540 struct pt_regs *regs)
1898 { 1541 {
1899 struct return_instance *ri = utask->r 1542 struct return_instance *ri = utask->return_instances;
1900 enum rp_check ctx = chained ? RP_CHEC 1543 enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
1901 1544
1902 while (ri && !arch_uretprobe_is_alive 1545 while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
1903 ri = free_ret_instance(ri); 1546 ri = free_ret_instance(ri);
1904 utask->depth--; 1547 utask->depth--;
1905 } 1548 }
1906 utask->return_instances = ri; 1549 utask->return_instances = ri;
1907 } 1550 }
1908 1551
1909 static void prepare_uretprobe(struct uprobe * 1552 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
1910 { 1553 {
1911 struct return_instance *ri; 1554 struct return_instance *ri;
1912 struct uprobe_task *utask; 1555 struct uprobe_task *utask;
1913 unsigned long orig_ret_vaddr, trampol 1556 unsigned long orig_ret_vaddr, trampoline_vaddr;
1914 bool chained; 1557 bool chained;
1915 1558
1916 if (!get_xol_area()) 1559 if (!get_xol_area())
1917 return; 1560 return;
1918 1561
1919 utask = get_utask(); 1562 utask = get_utask();
1920 if (!utask) 1563 if (!utask)
1921 return; 1564 return;
1922 1565
1923 if (utask->depth >= MAX_URETPROBE_DEP 1566 if (utask->depth >= MAX_URETPROBE_DEPTH) {
1924 printk_ratelimited(KERN_INFO 1567 printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
1925 " nestedness 1568 " nestedness limit pid/tgid=%d/%d\n",
1926 current->pid, 1569 current->pid, current->tgid);
1927 return; 1570 return;
1928 } 1571 }
1929 1572
1930 /* we need to bump refcount to store <<
1931 if (!try_get_uprobe(uprobe)) <<
1932 return; <<
1933 <<
1934 ri = kmalloc(sizeof(struct return_ins 1573 ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1935 if (!ri) 1574 if (!ri)
1936 goto fail; !! 1575 return;
1937 1576
1938 trampoline_vaddr = uprobe_get_trampol !! 1577 trampoline_vaddr = get_trampoline_vaddr();
1939 orig_ret_vaddr = arch_uretprobe_hijac 1578 orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1940 if (orig_ret_vaddr == -1) 1579 if (orig_ret_vaddr == -1)
1941 goto fail; 1580 goto fail;
1942 1581
1943 /* drop the entries invalidated by lo 1582 /* drop the entries invalidated by longjmp() */
1944 chained = (orig_ret_vaddr == trampoli 1583 chained = (orig_ret_vaddr == trampoline_vaddr);
1945 cleanup_return_instances(utask, chain 1584 cleanup_return_instances(utask, chained, regs);
1946 1585
1947 /* 1586 /*
1948 * We don't want to keep trampoline a 1587 * We don't want to keep trampoline address in stack, rather keep the
1949 * original return address of first c 1588 * original return address of first caller thru all the consequent
1950 * instances. This also makes breakpo 1589 * instances. This also makes breakpoint unwrapping easier.
1951 */ 1590 */
1952 if (chained) { 1591 if (chained) {
1953 if (!utask->return_instances) 1592 if (!utask->return_instances) {
1954 /* 1593 /*
1955 * This situation is 1594 * This situation is not possible. Likely we have an
1956 * attack from user-s 1595 * attack from user-space.
1957 */ 1596 */
1958 uprobe_warn(current, 1597 uprobe_warn(current, "handle tail call");
1959 goto fail; 1598 goto fail;
1960 } 1599 }
1961 orig_ret_vaddr = utask->retur 1600 orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
1962 } 1601 }
1963 ri->uprobe = uprobe; !! 1602
>> 1603 ri->uprobe = get_uprobe(uprobe);
1964 ri->func = instruction_pointer(regs); 1604 ri->func = instruction_pointer(regs);
1965 ri->stack = user_stack_pointer(regs); 1605 ri->stack = user_stack_pointer(regs);
1966 ri->orig_ret_vaddr = orig_ret_vaddr; 1606 ri->orig_ret_vaddr = orig_ret_vaddr;
1967 ri->chained = chained; 1607 ri->chained = chained;
1968 1608
1969 utask->depth++; 1609 utask->depth++;
1970 ri->next = utask->return_instances; 1610 ri->next = utask->return_instances;
1971 utask->return_instances = ri; 1611 utask->return_instances = ri;
1972 1612
1973 return; 1613 return;
1974 fail: !! 1614 fail:
1975 kfree(ri); 1615 kfree(ri);
1976 put_uprobe(uprobe); <<
1977 } 1616 }
1978 1617
1979 /* Prepare to single-step probed instruction 1618 /* Prepare to single-step probed instruction out of line. */
1980 static int 1619 static int
1981 pre_ssout(struct uprobe *uprobe, struct pt_re 1620 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
1982 { 1621 {
1983 struct uprobe_task *utask; 1622 struct uprobe_task *utask;
1984 unsigned long xol_vaddr; 1623 unsigned long xol_vaddr;
1985 int err; 1624 int err;
1986 1625
1987 utask = get_utask(); 1626 utask = get_utask();
1988 if (!utask) 1627 if (!utask)
1989 return -ENOMEM; 1628 return -ENOMEM;
1990 1629
1991 if (!try_get_uprobe(uprobe)) <<
1992 return -EINVAL; <<
1993 <<
1994 xol_vaddr = xol_get_insn_slot(uprobe) 1630 xol_vaddr = xol_get_insn_slot(uprobe);
1995 if (!xol_vaddr) { !! 1631 if (!xol_vaddr)
1996 err = -ENOMEM; !! 1632 return -ENOMEM;
1997 goto err_out; <<
1998 } <<
1999 1633
2000 utask->xol_vaddr = xol_vaddr; 1634 utask->xol_vaddr = xol_vaddr;
2001 utask->vaddr = bp_vaddr; 1635 utask->vaddr = bp_vaddr;
2002 1636
2003 err = arch_uprobe_pre_xol(&uprobe->ar 1637 err = arch_uprobe_pre_xol(&uprobe->arch, regs);
2004 if (unlikely(err)) { 1638 if (unlikely(err)) {
2005 xol_free_insn_slot(current); 1639 xol_free_insn_slot(current);
2006 goto err_out; !! 1640 return err;
2007 } 1641 }
2008 1642
2009 utask->active_uprobe = uprobe; 1643 utask->active_uprobe = uprobe;
2010 utask->state = UTASK_SSTEP; 1644 utask->state = UTASK_SSTEP;
2011 return 0; 1645 return 0;
2012 err_out: <<
2013 put_uprobe(uprobe); <<
2014 return err; <<
2015 } 1646 }
2016 1647
2017 /* 1648 /*
2018 * If we are singlestepping, then ensure this 1649 * If we are singlestepping, then ensure this thread is not connected to
2019 * non-fatal signals until completion of sing 1650 * non-fatal signals until completion of singlestep. When xol insn itself
2020 * triggers the signal, restart the original 1651 * triggers the signal, restart the original insn even if the task is
2021 * already SIGKILL'ed (since coredump should 1652 * already SIGKILL'ed (since coredump should report the correct ip). This
2022 * is even more important if the task has a h 1653 * is even more important if the task has a handler for SIGSEGV/etc, The
2023 * _same_ instruction should be repeated agai 1654 * _same_ instruction should be repeated again after return from the signal
2024 * handler, and SSTEP can never finish in thi 1655 * handler, and SSTEP can never finish in this case.
2025 */ 1656 */
2026 bool uprobe_deny_signal(void) 1657 bool uprobe_deny_signal(void)
2027 { 1658 {
2028 struct task_struct *t = current; 1659 struct task_struct *t = current;
2029 struct uprobe_task *utask = t->utask; 1660 struct uprobe_task *utask = t->utask;
2030 1661
2031 if (likely(!utask || !utask->active_u 1662 if (likely(!utask || !utask->active_uprobe))
2032 return false; 1663 return false;
2033 1664
2034 WARN_ON_ONCE(utask->state != UTASK_SS 1665 WARN_ON_ONCE(utask->state != UTASK_SSTEP);
2035 1666
2036 if (task_sigpending(t)) { !! 1667 if (signal_pending(t)) {
2037 spin_lock_irq(&t->sighand->si 1668 spin_lock_irq(&t->sighand->siglock);
2038 clear_tsk_thread_flag(t, TIF_ 1669 clear_tsk_thread_flag(t, TIF_SIGPENDING);
2039 spin_unlock_irq(&t->sighand-> 1670 spin_unlock_irq(&t->sighand->siglock);
2040 1671
2041 if (__fatal_signal_pending(t) 1672 if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
2042 utask->state = UTASK_ 1673 utask->state = UTASK_SSTEP_TRAPPED;
2043 set_tsk_thread_flag(t 1674 set_tsk_thread_flag(t, TIF_UPROBE);
2044 } 1675 }
2045 } 1676 }
2046 1677
2047 return true; 1678 return true;
2048 } 1679 }
2049 1680
2050 static void mmf_recalc_uprobes(struct mm_stru 1681 static void mmf_recalc_uprobes(struct mm_struct *mm)
2051 { 1682 {
2052 VMA_ITERATOR(vmi, mm, 0); <<
2053 struct vm_area_struct *vma; 1683 struct vm_area_struct *vma;
2054 1684
2055 for_each_vma(vmi, vma) { !! 1685 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2056 if (!valid_vma(vma, false)) 1686 if (!valid_vma(vma, false))
2057 continue; 1687 continue;
2058 /* 1688 /*
2059 * This is not strictly accur 1689 * This is not strictly accurate, we can race with
2060 * uprobe_unregister() and se 1690 * uprobe_unregister() and see the already removed
2061 * uprobe if delete_uprobe() 1691 * uprobe if delete_uprobe() was not yet called.
2062 * Or this uprobe can be filt 1692 * Or this uprobe can be filtered out.
2063 */ 1693 */
2064 if (vma_has_uprobes(vma, vma- 1694 if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
2065 return; 1695 return;
2066 } 1696 }
2067 1697
2068 clear_bit(MMF_HAS_UPROBES, &mm->flags 1698 clear_bit(MMF_HAS_UPROBES, &mm->flags);
2069 } 1699 }
2070 1700
2071 static int is_trap_at_addr(struct mm_struct * 1701 static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
2072 { 1702 {
2073 struct page *page; 1703 struct page *page;
2074 uprobe_opcode_t opcode; 1704 uprobe_opcode_t opcode;
2075 int result; 1705 int result;
2076 1706
2077 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, U <<
2078 return -EINVAL; <<
2079 <<
2080 pagefault_disable(); 1707 pagefault_disable();
2081 result = __get_user(opcode, (uprobe_o 1708 result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
2082 pagefault_enable(); 1709 pagefault_enable();
2083 1710
2084 if (likely(result == 0)) 1711 if (likely(result == 0))
2085 goto out; 1712 goto out;
2086 1713
2087 result = get_user_pages(vaddr, 1, FOL !! 1714 /*
>> 1715 * The NULL 'tsk' here ensures that any faults that occur here
>> 1716 * will not be accounted to the task. 'mm' *is* current->mm,
>> 1717 * but we treat this as a 'remote' access since it is
>> 1718 * essentially a kernel access to the memory.
>> 1719 */
>> 1720 result = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &page,
>> 1721 NULL, NULL);
2088 if (result < 0) 1722 if (result < 0)
2089 return result; 1723 return result;
2090 1724
2091 copy_from_page(page, vaddr, &opcode, 1725 copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
2092 put_page(page); 1726 put_page(page);
2093 out: 1727 out:
2094 /* This needs to return true for any 1728 /* This needs to return true for any variant of the trap insn */
2095 return is_trap_insn(&opcode); 1729 return is_trap_insn(&opcode);
2096 } 1730 }
2097 1731
2098 /* assumes being inside RCU protected region !! 1732 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
2099 static struct uprobe *find_active_uprobe_rcu( <<
2100 { 1733 {
2101 struct mm_struct *mm = current->mm; 1734 struct mm_struct *mm = current->mm;
2102 struct uprobe *uprobe = NULL; 1735 struct uprobe *uprobe = NULL;
2103 struct vm_area_struct *vma; 1736 struct vm_area_struct *vma;
2104 1737
2105 mmap_read_lock(mm); !! 1738 down_read(&mm->mmap_sem);
2106 vma = vma_lookup(mm, bp_vaddr); !! 1739 vma = find_vma(mm, bp_vaddr);
2107 if (vma) { !! 1740 if (vma && vma->vm_start <= bp_vaddr) {
2108 if (valid_vma(vma, false)) { 1741 if (valid_vma(vma, false)) {
2109 struct inode *inode = 1742 struct inode *inode = file_inode(vma->vm_file);
2110 loff_t offset = vaddr 1743 loff_t offset = vaddr_to_offset(vma, bp_vaddr);
2111 1744
2112 uprobe = find_uprobe_ !! 1745 uprobe = find_uprobe(inode, offset);
2113 } 1746 }
2114 1747
2115 if (!uprobe) 1748 if (!uprobe)
2116 *is_swbp = is_trap_at 1749 *is_swbp = is_trap_at_addr(mm, bp_vaddr);
2117 } else { 1750 } else {
2118 *is_swbp = -EFAULT; 1751 *is_swbp = -EFAULT;
2119 } 1752 }
2120 1753
2121 if (!uprobe && test_and_clear_bit(MMF 1754 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
2122 mmf_recalc_uprobes(mm); 1755 mmf_recalc_uprobes(mm);
2123 mmap_read_unlock(mm); !! 1756 up_read(&mm->mmap_sem);
2124 1757
2125 return uprobe; 1758 return uprobe;
2126 } 1759 }
2127 1760
2128 static void handler_chain(struct uprobe *upro 1761 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
2129 { 1762 {
2130 struct uprobe_consumer *uc; 1763 struct uprobe_consumer *uc;
2131 int remove = UPROBE_HANDLER_REMOVE; 1764 int remove = UPROBE_HANDLER_REMOVE;
2132 bool need_prep = false; /* prepare re 1765 bool need_prep = false; /* prepare return uprobe, when needed */
2133 bool has_consumers = false; <<
2134 <<
2135 current->utask->auprobe = &uprobe->ar <<
2136 1766
2137 list_for_each_entry_srcu(uc, &uprobe- !! 1767 down_read(&uprobe->register_rwsem);
2138 srcu_read_lo !! 1768 for (uc = uprobe->consumers; uc; uc = uc->next) {
2139 int rc = 0; 1769 int rc = 0;
2140 1770
2141 if (uc->handler) { 1771 if (uc->handler) {
2142 rc = uc->handler(uc, 1772 rc = uc->handler(uc, regs);
2143 WARN(rc & ~UPROBE_HAN 1773 WARN(rc & ~UPROBE_HANDLER_MASK,
2144 "bad rc=0x%x !! 1774 "bad rc=0x%x from %pf()\n", rc, uc->handler);
2145 } 1775 }
2146 1776
2147 if (uc->ret_handler) 1777 if (uc->ret_handler)
2148 need_prep = true; 1778 need_prep = true;
2149 1779
2150 remove &= rc; 1780 remove &= rc;
2151 has_consumers = true; <<
2152 } 1781 }
2153 current->utask->auprobe = NULL; <<
2154 1782
2155 if (need_prep && !remove) 1783 if (need_prep && !remove)
2156 prepare_uretprobe(uprobe, reg 1784 prepare_uretprobe(uprobe, regs); /* put bp at return */
2157 1785
2158 if (remove && has_consumers) { !! 1786 if (remove && uprobe->consumers) {
2159 down_read(&uprobe->register_r !! 1787 WARN_ON(!uprobe_is_active(uprobe));
2160 !! 1788 unapply_uprobe(uprobe, current->mm);
2161 /* re-check that removal is s <<
2162 if (!filter_chain(uprobe, cur <<
2163 WARN_ON(!uprobe_is_ac <<
2164 unapply_uprobe(uprobe <<
2165 } <<
2166 <<
2167 up_read(&uprobe->register_rws <<
2168 } 1789 }
>> 1790 up_read(&uprobe->register_rwsem);
2169 } 1791 }
2170 1792
2171 static void 1793 static void
2172 handle_uretprobe_chain(struct return_instance 1794 handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
2173 { 1795 {
2174 struct uprobe *uprobe = ri->uprobe; 1796 struct uprobe *uprobe = ri->uprobe;
2175 struct uprobe_consumer *uc; 1797 struct uprobe_consumer *uc;
2176 int srcu_idx; <<
2177 1798
2178 srcu_idx = srcu_read_lock(&uprobes_sr !! 1799 down_read(&uprobe->register_rwsem);
2179 list_for_each_entry_srcu(uc, &uprobe- !! 1800 for (uc = uprobe->consumers; uc; uc = uc->next) {
2180 srcu_read_lo <<
2181 if (uc->ret_handler) 1801 if (uc->ret_handler)
2182 uc->ret_handler(uc, r 1802 uc->ret_handler(uc, ri->func, regs);
2183 } 1803 }
2184 srcu_read_unlock(&uprobes_srcu, srcu_ !! 1804 up_read(&uprobe->register_rwsem);
2185 } 1805 }
2186 1806
2187 static struct return_instance *find_next_ret_ 1807 static struct return_instance *find_next_ret_chain(struct return_instance *ri)
2188 { 1808 {
2189 bool chained; 1809 bool chained;
2190 1810
2191 do { 1811 do {
2192 chained = ri->chained; 1812 chained = ri->chained;
2193 ri = ri->next; /* can't be N 1813 ri = ri->next; /* can't be NULL if chained */
2194 } while (chained); 1814 } while (chained);
2195 1815
2196 return ri; 1816 return ri;
2197 } 1817 }
2198 1818
2199 void uprobe_handle_trampoline(struct pt_regs !! 1819 static void handle_trampoline(struct pt_regs *regs)
2200 { 1820 {
2201 struct uprobe_task *utask; 1821 struct uprobe_task *utask;
2202 struct return_instance *ri, *next; 1822 struct return_instance *ri, *next;
2203 bool valid; 1823 bool valid;
2204 1824
2205 utask = current->utask; 1825 utask = current->utask;
2206 if (!utask) 1826 if (!utask)
2207 goto sigill; 1827 goto sigill;
2208 1828
2209 ri = utask->return_instances; 1829 ri = utask->return_instances;
2210 if (!ri) 1830 if (!ri)
2211 goto sigill; 1831 goto sigill;
2212 1832
2213 do { 1833 do {
2214 /* 1834 /*
2215 * We should throw out the fr 1835 * We should throw out the frames invalidated by longjmp().
2216 * If this chain is valid, th 1836 * If this chain is valid, then the next one should be alive
2217 * or NULL; the latter case m 1837 * or NULL; the latter case means that nobody but ri->func
2218 * could hit this trampoline 1838 * could hit this trampoline on return. TODO: sigaltstack().
2219 */ 1839 */
2220 next = find_next_ret_chain(ri 1840 next = find_next_ret_chain(ri);
2221 valid = !next || arch_uretpro 1841 valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
2222 1842
2223 instruction_pointer_set(regs, 1843 instruction_pointer_set(regs, ri->orig_ret_vaddr);
2224 do { 1844 do {
2225 /* pop current instan <<
2226 * as it's not pendin <<
2227 * instruction pointe <<
2228 * this allows fixup_ <<
2229 * captured stack tra <<
2230 * trampoline address <<
2231 * original return ad <<
2232 */ <<
2233 utask->return_instanc <<
2234 if (valid) 1845 if (valid)
2235 handle_uretpr 1846 handle_uretprobe_chain(ri, regs);
2236 ri = free_ret_instanc 1847 ri = free_ret_instance(ri);
2237 utask->depth--; 1848 utask->depth--;
2238 } while (ri != next); 1849 } while (ri != next);
2239 } while (!valid); 1850 } while (!valid);
2240 1851
2241 utask->return_instances = ri; 1852 utask->return_instances = ri;
2242 return; 1853 return;
2243 1854
2244 sigill: 1855 sigill:
2245 uprobe_warn(current, "handle uretprob 1856 uprobe_warn(current, "handle uretprobe, sending SIGILL.");
2246 force_sig(SIGILL); !! 1857 force_sig_info(SIGILL, SEND_SIG_FORCED, current);
2247 1858
2248 } 1859 }
2249 1860
2250 bool __weak arch_uprobe_ignore(struct arch_up 1861 bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
2251 { 1862 {
2252 return false; 1863 return false;
2253 } 1864 }
2254 1865
2255 bool __weak arch_uretprobe_is_alive(struct re 1866 bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
2256 struc 1867 struct pt_regs *regs)
2257 { 1868 {
2258 return true; 1869 return true;
2259 } 1870 }
2260 1871
2261 /* 1872 /*
2262 * Run handler and ask thread to singlestep. 1873 * Run handler and ask thread to singlestep.
2263 * Ensure all non-fatal signals cannot interr 1874 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
2264 */ 1875 */
2265 static void handle_swbp(struct pt_regs *regs) 1876 static void handle_swbp(struct pt_regs *regs)
2266 { 1877 {
2267 struct uprobe *uprobe; 1878 struct uprobe *uprobe;
2268 unsigned long bp_vaddr; 1879 unsigned long bp_vaddr;
2269 int is_swbp, srcu_idx; !! 1880 int uninitialized_var(is_swbp);
2270 1881
2271 bp_vaddr = uprobe_get_swbp_addr(regs) 1882 bp_vaddr = uprobe_get_swbp_addr(regs);
2272 if (bp_vaddr == uprobe_get_trampoline !! 1883 if (bp_vaddr == get_trampoline_vaddr())
2273 return uprobe_handle_trampoli !! 1884 return handle_trampoline(regs);
2274 <<
2275 srcu_idx = srcu_read_lock(&uprobes_sr <<
2276 1885
2277 uprobe = find_active_uprobe_rcu(bp_va !! 1886 uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
2278 if (!uprobe) { 1887 if (!uprobe) {
2279 if (is_swbp > 0) { 1888 if (is_swbp > 0) {
2280 /* No matching uprobe 1889 /* No matching uprobe; signal SIGTRAP. */
2281 force_sig(SIGTRAP); !! 1890 send_sig(SIGTRAP, current, 0);
2282 } else { 1891 } else {
2283 /* 1892 /*
2284 * Either we raced wi 1893 * Either we raced with uprobe_unregister() or we can't
2285 * access this memory 1894 * access this memory. The latter is only possible if
2286 * another thread pla 1895 * another thread plays with our ->mm. In both cases
2287 * we can simply rest 1896 * we can simply restart. If this vma was unmapped we
2288 * can pretend this i 1897 * can pretend this insn was not executed yet and get
2289 * the (correct) SIGS 1898 * the (correct) SIGSEGV after restart.
2290 */ 1899 */
2291 instruction_pointer_s 1900 instruction_pointer_set(regs, bp_vaddr);
2292 } 1901 }
2293 goto out; !! 1902 return;
2294 } 1903 }
2295 1904
2296 /* change it in advance for ->handler 1905 /* change it in advance for ->handler() and restart */
2297 instruction_pointer_set(regs, bp_vadd 1906 instruction_pointer_set(regs, bp_vaddr);
2298 1907
2299 /* 1908 /*
2300 * TODO: move copy_insn/etc into _reg 1909 * TODO: move copy_insn/etc into _register and remove this hack.
2301 * After we hit the bp, _unregister + 1910 * After we hit the bp, _unregister + _register can install the
2302 * new and not-yet-analyzed uprobe at 1911 * new and not-yet-analyzed uprobe at the same address, restart.
2303 */ 1912 */
>> 1913 smp_rmb(); /* pairs with wmb() in install_breakpoint() */
2304 if (unlikely(!test_bit(UPROBE_COPY_IN 1914 if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
2305 goto out; 1915 goto out;
2306 1916
2307 /* <<
2308 * Pairs with the smp_wmb() in prepar <<
2309 * <<
2310 * Guarantees that if we see the UPRO <<
2311 * we must also see the stores to &up <<
2312 * prepare_uprobe() call. <<
2313 */ <<
2314 smp_rmb(); <<
2315 <<
2316 /* Tracing handlers use ->utask to co 1917 /* Tracing handlers use ->utask to communicate with fetch methods */
2317 if (!get_utask()) 1918 if (!get_utask())
2318 goto out; 1919 goto out;
2319 1920
2320 if (arch_uprobe_ignore(&uprobe->arch, 1921 if (arch_uprobe_ignore(&uprobe->arch, regs))
2321 goto out; 1922 goto out;
2322 1923
2323 handler_chain(uprobe, regs); 1924 handler_chain(uprobe, regs);
2324 1925
2325 if (arch_uprobe_skip_sstep(&uprobe->a 1926 if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
2326 goto out; 1927 goto out;
2327 1928
2328 if (pre_ssout(uprobe, regs, bp_vaddr) !! 1929 if (!pre_ssout(uprobe, regs, bp_vaddr))
2329 goto out; !! 1930 return;
2330 1931
2331 out: <<
2332 /* arch_uprobe_skip_sstep() succeeded 1932 /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
2333 srcu_read_unlock(&uprobes_srcu, srcu_ !! 1933 out:
>> 1934 put_uprobe(uprobe);
2334 } 1935 }
2335 1936
2336 /* 1937 /*
2337 * Perform required fix-ups and disable singl 1938 * Perform required fix-ups and disable singlestep.
2338 * Allow pending signals to take effect. 1939 * Allow pending signals to take effect.
2339 */ 1940 */
2340 static void handle_singlestep(struct uprobe_t 1941 static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
2341 { 1942 {
2342 struct uprobe *uprobe; 1943 struct uprobe *uprobe;
2343 int err = 0; 1944 int err = 0;
2344 1945
2345 uprobe = utask->active_uprobe; 1946 uprobe = utask->active_uprobe;
2346 if (utask->state == UTASK_SSTEP_ACK) 1947 if (utask->state == UTASK_SSTEP_ACK)
2347 err = arch_uprobe_post_xol(&u 1948 err = arch_uprobe_post_xol(&uprobe->arch, regs);
2348 else if (utask->state == UTASK_SSTEP_ 1949 else if (utask->state == UTASK_SSTEP_TRAPPED)
2349 arch_uprobe_abort_xol(&uprobe 1950 arch_uprobe_abort_xol(&uprobe->arch, regs);
2350 else 1951 else
2351 WARN_ON_ONCE(1); 1952 WARN_ON_ONCE(1);
2352 1953
2353 put_uprobe(uprobe); 1954 put_uprobe(uprobe);
2354 utask->active_uprobe = NULL; 1955 utask->active_uprobe = NULL;
2355 utask->state = UTASK_RUNNING; 1956 utask->state = UTASK_RUNNING;
2356 xol_free_insn_slot(current); 1957 xol_free_insn_slot(current);
2357 1958
2358 spin_lock_irq(¤t->sighand->sigl 1959 spin_lock_irq(¤t->sighand->siglock);
2359 recalc_sigpending(); /* see uprobe_de 1960 recalc_sigpending(); /* see uprobe_deny_signal() */
2360 spin_unlock_irq(¤t->sighand->si 1961 spin_unlock_irq(¤t->sighand->siglock);
2361 1962
2362 if (unlikely(err)) { 1963 if (unlikely(err)) {
2363 uprobe_warn(current, "execute 1964 uprobe_warn(current, "execute the probed insn, sending SIGILL.");
2364 force_sig(SIGILL); !! 1965 force_sig_info(SIGILL, SEND_SIG_FORCED, current);
2365 } 1966 }
2366 } 1967 }
2367 1968
2368 /* 1969 /*
2369 * On breakpoint hit, breakpoint notifier set 1970 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
2370 * allows the thread to return from interrupt 1971 * allows the thread to return from interrupt. After that handle_swbp()
2371 * sets utask->active_uprobe. 1972 * sets utask->active_uprobe.
2372 * 1973 *
2373 * On singlestep exception, singlestep notifi 1974 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
2374 * and allows the thread to return from inter 1975 * and allows the thread to return from interrupt.
2375 * 1976 *
2376 * While returning to userspace, thread notic 1977 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
2377 * uprobe_notify_resume(). 1978 * uprobe_notify_resume().
2378 */ 1979 */
2379 void uprobe_notify_resume(struct pt_regs *reg 1980 void uprobe_notify_resume(struct pt_regs *regs)
2380 { 1981 {
2381 struct uprobe_task *utask; 1982 struct uprobe_task *utask;
2382 1983
2383 clear_thread_flag(TIF_UPROBE); 1984 clear_thread_flag(TIF_UPROBE);
2384 1985
2385 utask = current->utask; 1986 utask = current->utask;
2386 if (utask && utask->active_uprobe) 1987 if (utask && utask->active_uprobe)
2387 handle_singlestep(utask, regs 1988 handle_singlestep(utask, regs);
2388 else 1989 else
2389 handle_swbp(regs); 1990 handle_swbp(regs);
2390 } 1991 }
2391 1992
2392 /* 1993 /*
2393 * uprobe_pre_sstep_notifier gets called from 1994 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
2394 * notifier mechanism. Set TIF_UPROBE flag an 1995 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
2395 */ 1996 */
2396 int uprobe_pre_sstep_notifier(struct pt_regs 1997 int uprobe_pre_sstep_notifier(struct pt_regs *regs)
2397 { 1998 {
2398 if (!current->mm) 1999 if (!current->mm)
2399 return 0; 2000 return 0;
2400 2001
2401 if (!test_bit(MMF_HAS_UPROBES, &curre 2002 if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) &&
2402 (!current->utask || !current->uta 2003 (!current->utask || !current->utask->return_instances))
2403 return 0; 2004 return 0;
2404 2005
2405 set_thread_flag(TIF_UPROBE); 2006 set_thread_flag(TIF_UPROBE);
2406 return 1; 2007 return 1;
2407 } 2008 }
2408 2009
2409 /* 2010 /*
2410 * uprobe_post_sstep_notifier gets called in 2011 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
2411 * mechanism. Set TIF_UPROBE flag and indicat 2012 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
2412 */ 2013 */
2413 int uprobe_post_sstep_notifier(struct pt_regs 2014 int uprobe_post_sstep_notifier(struct pt_regs *regs)
2414 { 2015 {
2415 struct uprobe_task *utask = current-> 2016 struct uprobe_task *utask = current->utask;
2416 2017
2417 if (!current->mm || !utask || !utask- 2018 if (!current->mm || !utask || !utask->active_uprobe)
2418 /* task is currently not upro 2019 /* task is currently not uprobed */
2419 return 0; 2020 return 0;
2420 2021
2421 utask->state = UTASK_SSTEP_ACK; 2022 utask->state = UTASK_SSTEP_ACK;
2422 set_thread_flag(TIF_UPROBE); 2023 set_thread_flag(TIF_UPROBE);
2423 return 1; 2024 return 1;
2424 } 2025 }
2425 2026
2426 static struct notifier_block uprobe_exception 2027 static struct notifier_block uprobe_exception_nb = {
2427 .notifier_call = arch_uprobe 2028 .notifier_call = arch_uprobe_exception_notify,
2428 .priority = INT_MAX-1, 2029 .priority = INT_MAX-1, /* notified after kprobes, kgdb */
2429 }; 2030 };
2430 2031
2431 void __init uprobes_init(void) !! 2032 static int __init init_uprobes(void)
2432 { 2033 {
2433 int i; 2034 int i;
2434 2035
2435 for (i = 0; i < UPROBES_HASH_SZ; i++) 2036 for (i = 0; i < UPROBES_HASH_SZ; i++)
2436 mutex_init(&uprobes_mmap_mute 2037 mutex_init(&uprobes_mmap_mutex[i]);
2437 2038
2438 BUG_ON(register_die_notifier(&uprobe_ !! 2039 if (percpu_init_rwsem(&dup_mmap_sem))
>> 2040 return -ENOMEM;
>> 2041
>> 2042 return register_die_notifier(&uprobe_exception_nb);
2439 } 2043 }
>> 2044 __initcall(init_uprobes);
2440 2045
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.