1 // SPDX-License-Identifier: GPL-2.0+ 1 // SPDX-License-Identifier: GPL-2.0+
2 /* 2 /*
3 * User-space Probes (UProbes) 3 * User-space Probes (UProbes)
4 * 4 *
5 * Copyright (C) IBM Corporation, 2008-2012 5 * Copyright (C) IBM Corporation, 2008-2012
6 * Authors: 6 * Authors:
7 * Srikar Dronamraju 7 * Srikar Dronamraju
8 * Jim Keniston 8 * Jim Keniston
9 * Copyright (C) 2011-2012 Red Hat, Inc., Pete 9 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
10 */ 10 */
11 11
12 #include <linux/kernel.h> 12 #include <linux/kernel.h>
13 #include <linux/highmem.h> 13 #include <linux/highmem.h>
14 #include <linux/pagemap.h> /* read_mappin 14 #include <linux/pagemap.h> /* read_mapping_page */
15 #include <linux/slab.h> 15 #include <linux/slab.h>
16 #include <linux/sched.h> 16 #include <linux/sched.h>
17 #include <linux/sched/mm.h> 17 #include <linux/sched/mm.h>
18 #include <linux/sched/coredump.h> 18 #include <linux/sched/coredump.h>
19 #include <linux/export.h> 19 #include <linux/export.h>
20 #include <linux/rmap.h> /* anon_vma_pr 20 #include <linux/rmap.h> /* anon_vma_prepare */
21 #include <linux/mmu_notifier.h> !! 21 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
22 #include <linux/swap.h> /* folio_free_ !! 22 #include <linux/swap.h> /* try_to_free_swap */
23 #include <linux/ptrace.h> /* user_enable 23 #include <linux/ptrace.h> /* user_enable_single_step */
24 #include <linux/kdebug.h> /* notifier me 24 #include <linux/kdebug.h> /* notifier mechanism */
>> 25 #include "../../mm/internal.h" /* munlock_vma_page */
25 #include <linux/percpu-rwsem.h> 26 #include <linux/percpu-rwsem.h>
26 #include <linux/task_work.h> 27 #include <linux/task_work.h>
27 #include <linux/shmem_fs.h> 28 #include <linux/shmem_fs.h>
28 #include <linux/khugepaged.h> 29 #include <linux/khugepaged.h>
29 30
30 #include <linux/uprobes.h> 31 #include <linux/uprobes.h>
31 32
32 #define UINSNS_PER_PAGE (PAGE_ 33 #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
33 #define MAX_UPROBE_XOL_SLOTS UINSNS 34 #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
34 35
35 static struct rb_root uprobes_tree = RB_ROOT; 36 static struct rb_root uprobes_tree = RB_ROOT;
36 /* 37 /*
37 * allows us to skip the uprobe_mmap if there 38 * allows us to skip the uprobe_mmap if there are no uprobe events active
38 * at this time. Probably a fine grained per 39 * at this time. Probably a fine grained per inode count is better?
39 */ 40 */
40 #define no_uprobe_events() RB_EMPTY_ROOT( 41 #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree)
41 42
42 static DEFINE_RWLOCK(uprobes_treelock); /* ser !! 43 static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
43 static seqcount_rwlock_t uprobes_seqcount = SE <<
44 <<
45 DEFINE_STATIC_SRCU(uprobes_srcu); <<
46 44
47 #define UPROBES_HASH_SZ 13 45 #define UPROBES_HASH_SZ 13
48 /* serialize uprobe->pending_list */ 46 /* serialize uprobe->pending_list */
49 static struct mutex uprobes_mmap_mutex[UPROBES 47 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
50 #define uprobes_mmap_hash(v) (&uprobes_mmap 48 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
51 49
52 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); 50 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
53 51
54 /* Have a copy of original instruction */ 52 /* Have a copy of original instruction */
55 #define UPROBE_COPY_INSN 0 53 #define UPROBE_COPY_INSN 0
56 54
57 struct uprobe { 55 struct uprobe {
58 struct rb_node rb_node; 56 struct rb_node rb_node; /* node in the rb tree */
59 refcount_t ref; 57 refcount_t ref;
60 struct rw_semaphore register_rwsem 58 struct rw_semaphore register_rwsem;
61 struct rw_semaphore consumer_rwsem 59 struct rw_semaphore consumer_rwsem;
62 struct list_head pending_list; 60 struct list_head pending_list;
63 struct list_head consumers; !! 61 struct uprobe_consumer *consumers;
64 struct inode *inode; 62 struct inode *inode; /* Also hold a ref to inode */
65 struct rcu_head rcu; <<
66 loff_t offset; 63 loff_t offset;
67 loff_t ref_ctr_offset 64 loff_t ref_ctr_offset;
68 unsigned long flags; 65 unsigned long flags;
69 66
70 /* 67 /*
71 * The generic code assumes that it ha 68 * The generic code assumes that it has two members of unknown type
72 * owned by the arch-specific code: 69 * owned by the arch-specific code:
73 * 70 *
74 * insn - copy_insn() saves the 71 * insn - copy_insn() saves the original instruction here for
75 * arch_uprobe_analyze_in 72 * arch_uprobe_analyze_insn().
76 * 73 *
77 * ixol - potentially modified i 74 * ixol - potentially modified instruction to execute out of
78 * line, copied to xol_ar 75 * line, copied to xol_area by xol_get_insn_slot().
79 */ 76 */
80 struct arch_uprobe arch; 77 struct arch_uprobe arch;
81 }; 78 };
82 79
83 struct delayed_uprobe { 80 struct delayed_uprobe {
84 struct list_head list; 81 struct list_head list;
85 struct uprobe *uprobe; 82 struct uprobe *uprobe;
86 struct mm_struct *mm; 83 struct mm_struct *mm;
87 }; 84 };
88 85
89 static DEFINE_MUTEX(delayed_uprobe_lock); 86 static DEFINE_MUTEX(delayed_uprobe_lock);
90 static LIST_HEAD(delayed_uprobe_list); 87 static LIST_HEAD(delayed_uprobe_list);
91 88
92 /* 89 /*
93 * Execute out of line area: anonymous executa 90 * Execute out of line area: anonymous executable mapping installed
94 * by the probed task to execute the copy of t 91 * by the probed task to execute the copy of the original instruction
95 * mangled by set_swbp(). 92 * mangled by set_swbp().
96 * 93 *
97 * On a breakpoint hit, thread contests for a 94 * On a breakpoint hit, thread contests for a slot. It frees the
98 * slot after singlestep. Currently a fixed nu 95 * slot after singlestep. Currently a fixed number of slots are
99 * allocated. 96 * allocated.
100 */ 97 */
101 struct xol_area { 98 struct xol_area {
102 wait_queue_head_t wq; 99 wait_queue_head_t wq; /* if all slots are busy */
103 atomic_t slot_c 100 atomic_t slot_count; /* number of in-use slots */
104 unsigned long *bitma 101 unsigned long *bitmap; /* 0 = free slot */
105 102
106 struct page *page; !! 103 struct vm_special_mapping xol_mapping;
>> 104 struct page *pages[2];
107 /* 105 /*
108 * We keep the vma's vm_start rather t 106 * We keep the vma's vm_start rather than a pointer to the vma
109 * itself. The probed process or a na 107 * itself. The probed process or a naughty kernel module could make
110 * the vma go away, and we must handle 108 * the vma go away, and we must handle that reasonably gracefully.
111 */ 109 */
112 unsigned long vaddr; 110 unsigned long vaddr; /* Page(s) of instruction slots */
113 }; 111 };
114 112
115 static void uprobe_warn(struct task_struct *t, <<
116 { <<
117 pr_warn("uprobe: %s:%d failed to %s\n" <<
118 } <<
119 <<
120 /* 113 /*
121 * valid_vma: Verify if the specified vma is a 114 * valid_vma: Verify if the specified vma is an executable vma
122 * Relax restrictions while unregistering: vm_ 115 * Relax restrictions while unregistering: vm_flags might have
123 * changed after breakpoint was inserted. 116 * changed after breakpoint was inserted.
124 * - is_register: indicates if we are in 117 * - is_register: indicates if we are in register context.
125 * - Return 1 if the specified virtual ad 118 * - Return 1 if the specified virtual address is in an
126 * executable vma. 119 * executable vma.
127 */ 120 */
128 static bool valid_vma(struct vm_area_struct *v 121 static bool valid_vma(struct vm_area_struct *vma, bool is_register)
129 { 122 {
130 vm_flags_t flags = VM_HUGETLB | VM_MAY 123 vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
131 124
132 if (is_register) 125 if (is_register)
133 flags |= VM_WRITE; 126 flags |= VM_WRITE;
134 127
135 return vma->vm_file && (vma->vm_flags 128 return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
136 } 129 }
137 130
138 static unsigned long offset_to_vaddr(struct vm 131 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
139 { 132 {
140 return vma->vm_start + offset - ((loff 133 return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
141 } 134 }
142 135
143 static loff_t vaddr_to_offset(struct vm_area_s 136 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
144 { 137 {
145 return ((loff_t)vma->vm_pgoff << PAGE_ 138 return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
146 } 139 }
147 140
148 /** 141 /**
149 * __replace_page - replace page in vma by new 142 * __replace_page - replace page in vma by new page.
150 * based on replace_page in mm/ksm.c 143 * based on replace_page in mm/ksm.c
151 * 144 *
152 * @vma: vma that holds the pte pointing 145 * @vma: vma that holds the pte pointing to page
153 * @addr: address the old @page is mapped 146 * @addr: address the old @page is mapped at
154 * @old_page: the page we are replacing by new 147 * @old_page: the page we are replacing by new_page
155 * @new_page: the modified page we replace pag 148 * @new_page: the modified page we replace page by
156 * 149 *
157 * If @new_page is NULL, only unmap @old_page. 150 * If @new_page is NULL, only unmap @old_page.
158 * 151 *
159 * Returns 0 on success, negative error code o 152 * Returns 0 on success, negative error code otherwise.
160 */ 153 */
161 static int __replace_page(struct vm_area_struc 154 static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
162 struct page *o 155 struct page *old_page, struct page *new_page)
163 { 156 {
164 struct folio *old_folio = page_folio(o <<
165 struct folio *new_folio; <<
166 struct mm_struct *mm = vma->vm_mm; 157 struct mm_struct *mm = vma->vm_mm;
167 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, !! 158 struct page_vma_mapped_walk pvmw = {
>> 159 .page = compound_head(old_page),
>> 160 .vma = vma,
>> 161 .address = addr,
>> 162 };
168 int err; 163 int err;
169 struct mmu_notifier_range range; 164 struct mmu_notifier_range range;
170 165
171 mmu_notifier_range_init(&range, MMU_NO !! 166 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
172 addr + PAGE_SI 167 addr + PAGE_SIZE);
173 168
174 if (new_page) { 169 if (new_page) {
175 new_folio = page_folio(new_pag !! 170 err = mem_cgroup_charge(page_folio(new_page), vma->vm_mm,
176 err = mem_cgroup_charge(new_fo !! 171 GFP_KERNEL);
177 if (err) 172 if (err)
178 return err; 173 return err;
179 } 174 }
180 175
181 /* For folio_free_swap() below */ !! 176 /* For try_to_free_swap() and munlock_vma_page() below */
182 folio_lock(old_folio); !! 177 lock_page(old_page);
183 178
184 mmu_notifier_invalidate_range_start(&r 179 mmu_notifier_invalidate_range_start(&range);
185 err = -EAGAIN; 180 err = -EAGAIN;
186 if (!page_vma_mapped_walk(&pvmw)) 181 if (!page_vma_mapped_walk(&pvmw))
187 goto unlock; 182 goto unlock;
188 VM_BUG_ON_PAGE(addr != pvmw.address, o 183 VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
189 184
190 if (new_page) { 185 if (new_page) {
191 folio_get(new_folio); !! 186 get_page(new_page);
192 folio_add_new_anon_rmap(new_fo !! 187 page_add_new_anon_rmap(new_page, vma, addr, false);
193 folio_add_lru_vma(new_folio, v !! 188 lru_cache_add_inactive_or_unevictable(new_page, vma);
194 } else 189 } else
195 /* no new page, just dec_mm_co 190 /* no new page, just dec_mm_counter for old_page */
196 dec_mm_counter(mm, MM_ANONPAGE 191 dec_mm_counter(mm, MM_ANONPAGES);
197 192
198 if (!folio_test_anon(old_folio)) { !! 193 if (!PageAnon(old_page)) {
199 dec_mm_counter(mm, mm_counter_ !! 194 dec_mm_counter(mm, mm_counter_file(old_page));
200 inc_mm_counter(mm, MM_ANONPAGE 195 inc_mm_counter(mm, MM_ANONPAGES);
201 } 196 }
202 197
203 flush_cache_page(vma, addr, pte_pfn(pt !! 198 flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
204 ptep_clear_flush(vma, addr, pvmw.pte); !! 199 ptep_clear_flush_notify(vma, addr, pvmw.pte);
205 if (new_page) 200 if (new_page)
206 set_pte_at(mm, addr, pvmw.pte, !! 201 set_pte_at_notify(mm, addr, pvmw.pte,
207 mk_pte(new_page, vm !! 202 mk_pte(new_page, vma->vm_page_prot));
208 203
209 folio_remove_rmap_pte(old_folio, old_p !! 204 page_remove_rmap(old_page, false);
210 if (!folio_mapped(old_folio)) !! 205 if (!page_mapped(old_page))
211 folio_free_swap(old_folio); !! 206 try_to_free_swap(old_page);
212 page_vma_mapped_walk_done(&pvmw); 207 page_vma_mapped_walk_done(&pvmw);
213 folio_put(old_folio); !! 208
>> 209 if ((vma->vm_flags & VM_LOCKED) && !PageCompound(old_page))
>> 210 munlock_vma_page(old_page);
>> 211 put_page(old_page);
214 212
215 err = 0; 213 err = 0;
216 unlock: 214 unlock:
217 mmu_notifier_invalidate_range_end(&ran 215 mmu_notifier_invalidate_range_end(&range);
218 folio_unlock(old_folio); !! 216 unlock_page(old_page);
219 return err; 217 return err;
220 } 218 }
221 219
222 /** 220 /**
223 * is_swbp_insn - check if instruction is brea 221 * is_swbp_insn - check if instruction is breakpoint instruction.
224 * @insn: instruction to be checked. 222 * @insn: instruction to be checked.
225 * Default implementation of is_swbp_insn 223 * Default implementation of is_swbp_insn
226 * Returns true if @insn is a breakpoint instr 224 * Returns true if @insn is a breakpoint instruction.
227 */ 225 */
228 bool __weak is_swbp_insn(uprobe_opcode_t *insn 226 bool __weak is_swbp_insn(uprobe_opcode_t *insn)
229 { 227 {
230 return *insn == UPROBE_SWBP_INSN; 228 return *insn == UPROBE_SWBP_INSN;
231 } 229 }
232 230
233 /** 231 /**
234 * is_trap_insn - check if instruction is brea 232 * is_trap_insn - check if instruction is breakpoint instruction.
235 * @insn: instruction to be checked. 233 * @insn: instruction to be checked.
236 * Default implementation of is_trap_insn 234 * Default implementation of is_trap_insn
237 * Returns true if @insn is a breakpoint instr 235 * Returns true if @insn is a breakpoint instruction.
238 * 236 *
239 * This function is needed for the case where 237 * This function is needed for the case where an architecture has multiple
240 * trap instructions (like powerpc). 238 * trap instructions (like powerpc).
241 */ 239 */
242 bool __weak is_trap_insn(uprobe_opcode_t *insn 240 bool __weak is_trap_insn(uprobe_opcode_t *insn)
243 { 241 {
244 return is_swbp_insn(insn); 242 return is_swbp_insn(insn);
245 } 243 }
246 244
247 static void copy_from_page(struct page *page, 245 static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
248 { 246 {
249 void *kaddr = kmap_atomic(page); 247 void *kaddr = kmap_atomic(page);
250 memcpy(dst, kaddr + (vaddr & ~PAGE_MAS 248 memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
251 kunmap_atomic(kaddr); 249 kunmap_atomic(kaddr);
252 } 250 }
253 251
254 static void copy_to_page(struct page *page, un 252 static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
255 { 253 {
256 void *kaddr = kmap_atomic(page); 254 void *kaddr = kmap_atomic(page);
257 memcpy(kaddr + (vaddr & ~PAGE_MASK), s 255 memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
258 kunmap_atomic(kaddr); 256 kunmap_atomic(kaddr);
259 } 257 }
260 258
261 static int verify_opcode(struct page *page, un 259 static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
262 { 260 {
263 uprobe_opcode_t old_opcode; 261 uprobe_opcode_t old_opcode;
264 bool is_swbp; 262 bool is_swbp;
265 263
266 /* 264 /*
267 * Note: We only check if the old_opco 265 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
268 * We do not check if it is any other 266 * We do not check if it is any other 'trap variant' which could
269 * be conditional trap instruction suc 267 * be conditional trap instruction such as the one powerpc supports.
270 * 268 *
271 * The logic is that we do not care if 269 * The logic is that we do not care if the underlying instruction
272 * is a trap variant; uprobes always w 270 * is a trap variant; uprobes always wins over any other (gdb)
273 * breakpoint. 271 * breakpoint.
274 */ 272 */
275 copy_from_page(page, vaddr, &old_opcod 273 copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
276 is_swbp = is_swbp_insn(&old_opcode); 274 is_swbp = is_swbp_insn(&old_opcode);
277 275
278 if (is_swbp_insn(new_opcode)) { 276 if (is_swbp_insn(new_opcode)) {
279 if (is_swbp) /* reg 277 if (is_swbp) /* register: already installed? */
280 return 0; 278 return 0;
281 } else { 279 } else {
282 if (!is_swbp) /* unr 280 if (!is_swbp) /* unregister: was it changed by us? */
283 return 0; 281 return 0;
284 } 282 }
285 283
286 return 1; 284 return 1;
287 } 285 }
288 286
289 static struct delayed_uprobe * 287 static struct delayed_uprobe *
290 delayed_uprobe_check(struct uprobe *uprobe, st 288 delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
291 { 289 {
292 struct delayed_uprobe *du; 290 struct delayed_uprobe *du;
293 291
294 list_for_each_entry(du, &delayed_uprob 292 list_for_each_entry(du, &delayed_uprobe_list, list)
295 if (du->uprobe == uprobe && du 293 if (du->uprobe == uprobe && du->mm == mm)
296 return du; 294 return du;
297 return NULL; 295 return NULL;
298 } 296 }
299 297
300 static int delayed_uprobe_add(struct uprobe *u 298 static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
301 { 299 {
302 struct delayed_uprobe *du; 300 struct delayed_uprobe *du;
303 301
304 if (delayed_uprobe_check(uprobe, mm)) 302 if (delayed_uprobe_check(uprobe, mm))
305 return 0; 303 return 0;
306 304
307 du = kzalloc(sizeof(*du), GFP_KERNEL) 305 du = kzalloc(sizeof(*du), GFP_KERNEL);
308 if (!du) 306 if (!du)
309 return -ENOMEM; 307 return -ENOMEM;
310 308
311 du->uprobe = uprobe; 309 du->uprobe = uprobe;
312 du->mm = mm; 310 du->mm = mm;
313 list_add(&du->list, &delayed_uprobe_li 311 list_add(&du->list, &delayed_uprobe_list);
314 return 0; 312 return 0;
315 } 313 }
316 314
317 static void delayed_uprobe_delete(struct delay 315 static void delayed_uprobe_delete(struct delayed_uprobe *du)
318 { 316 {
319 if (WARN_ON(!du)) 317 if (WARN_ON(!du))
320 return; 318 return;
321 list_del(&du->list); 319 list_del(&du->list);
322 kfree(du); 320 kfree(du);
323 } 321 }
324 322
325 static void delayed_uprobe_remove(struct uprob 323 static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
326 { 324 {
327 struct list_head *pos, *q; 325 struct list_head *pos, *q;
328 struct delayed_uprobe *du; 326 struct delayed_uprobe *du;
329 327
330 if (!uprobe && !mm) 328 if (!uprobe && !mm)
331 return; 329 return;
332 330
333 list_for_each_safe(pos, q, &delayed_up 331 list_for_each_safe(pos, q, &delayed_uprobe_list) {
334 du = list_entry(pos, struct de 332 du = list_entry(pos, struct delayed_uprobe, list);
335 333
336 if (uprobe && du->uprobe != up 334 if (uprobe && du->uprobe != uprobe)
337 continue; 335 continue;
338 if (mm && du->mm != mm) 336 if (mm && du->mm != mm)
339 continue; 337 continue;
340 338
341 delayed_uprobe_delete(du); 339 delayed_uprobe_delete(du);
342 } 340 }
343 } 341 }
344 342
345 static bool valid_ref_ctr_vma(struct uprobe *u 343 static bool valid_ref_ctr_vma(struct uprobe *uprobe,
346 struct vm_area_s 344 struct vm_area_struct *vma)
347 { 345 {
348 unsigned long vaddr = offset_to_vaddr( 346 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
349 347
350 return uprobe->ref_ctr_offset && 348 return uprobe->ref_ctr_offset &&
351 vma->vm_file && 349 vma->vm_file &&
352 file_inode(vma->vm_file) == up 350 file_inode(vma->vm_file) == uprobe->inode &&
353 (vma->vm_flags & (VM_WRITE|VM_ 351 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
354 vma->vm_start <= vaddr && 352 vma->vm_start <= vaddr &&
355 vma->vm_end > vaddr; 353 vma->vm_end > vaddr;
356 } 354 }
357 355
358 static struct vm_area_struct * 356 static struct vm_area_struct *
359 find_ref_ctr_vma(struct uprobe *uprobe, struct 357 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
360 { 358 {
361 VMA_ITERATOR(vmi, mm, 0); <<
362 struct vm_area_struct *tmp; 359 struct vm_area_struct *tmp;
363 360
364 for_each_vma(vmi, tmp) !! 361 for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
365 if (valid_ref_ctr_vma(uprobe, 362 if (valid_ref_ctr_vma(uprobe, tmp))
366 return tmp; 363 return tmp;
367 364
368 return NULL; 365 return NULL;
369 } 366 }
370 367
371 static int 368 static int
372 __update_ref_ctr(struct mm_struct *mm, unsigne 369 __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
373 { 370 {
374 void *kaddr; 371 void *kaddr;
375 struct page *page; 372 struct page *page;
>> 373 struct vm_area_struct *vma;
376 int ret; 374 int ret;
377 short *ptr; 375 short *ptr;
378 376
379 if (!vaddr || !d) 377 if (!vaddr || !d)
380 return -EINVAL; 378 return -EINVAL;
381 379
382 ret = get_user_pages_remote(mm, vaddr, 380 ret = get_user_pages_remote(mm, vaddr, 1,
383 FOLL_WRITE !! 381 FOLL_WRITE, &page, &vma, NULL);
384 if (unlikely(ret <= 0)) { 382 if (unlikely(ret <= 0)) {
385 /* 383 /*
386 * We are asking for 1 page. I 384 * We are asking for 1 page. If get_user_pages_remote() fails,
387 * it may return 0, in that ca 385 * it may return 0, in that case we have to return error.
388 */ 386 */
389 return ret == 0 ? -EBUSY : ret 387 return ret == 0 ? -EBUSY : ret;
390 } 388 }
391 389
392 kaddr = kmap_atomic(page); 390 kaddr = kmap_atomic(page);
393 ptr = kaddr + (vaddr & ~PAGE_MASK); 391 ptr = kaddr + (vaddr & ~PAGE_MASK);
394 392
395 if (unlikely(*ptr + d < 0)) { 393 if (unlikely(*ptr + d < 0)) {
396 pr_warn("ref_ctr going negativ 394 pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
397 "curr val: %d, delta: 395 "curr val: %d, delta: %d\n", vaddr, *ptr, d);
398 ret = -EINVAL; 396 ret = -EINVAL;
399 goto out; 397 goto out;
400 } 398 }
401 399
402 *ptr += d; 400 *ptr += d;
403 ret = 0; 401 ret = 0;
404 out: 402 out:
405 kunmap_atomic(kaddr); 403 kunmap_atomic(kaddr);
406 put_page(page); 404 put_page(page);
407 return ret; 405 return ret;
408 } 406 }
409 407
410 static void update_ref_ctr_warn(struct uprobe 408 static void update_ref_ctr_warn(struct uprobe *uprobe,
411 struct mm_stru 409 struct mm_struct *mm, short d)
412 { 410 {
413 pr_warn("ref_ctr %s failed for inode: 411 pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
414 "0x%llx ref_ctr_offset: 0x%llx 412 "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
415 d > 0 ? "increment" : "decreme 413 d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
416 (unsigned long long) uprobe->o 414 (unsigned long long) uprobe->offset,
417 (unsigned long long) uprobe->r 415 (unsigned long long) uprobe->ref_ctr_offset, mm);
418 } 416 }
419 417
420 static int update_ref_ctr(struct uprobe *uprob 418 static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
421 short d) 419 short d)
422 { 420 {
423 struct vm_area_struct *rc_vma; 421 struct vm_area_struct *rc_vma;
424 unsigned long rc_vaddr; 422 unsigned long rc_vaddr;
425 int ret = 0; 423 int ret = 0;
426 424
427 rc_vma = find_ref_ctr_vma(uprobe, mm); 425 rc_vma = find_ref_ctr_vma(uprobe, mm);
428 426
429 if (rc_vma) { 427 if (rc_vma) {
430 rc_vaddr = offset_to_vaddr(rc_ 428 rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
431 ret = __update_ref_ctr(mm, rc_ 429 ret = __update_ref_ctr(mm, rc_vaddr, d);
432 if (ret) 430 if (ret)
433 update_ref_ctr_warn(up 431 update_ref_ctr_warn(uprobe, mm, d);
434 432
435 if (d > 0) 433 if (d > 0)
436 return ret; 434 return ret;
437 } 435 }
438 436
439 mutex_lock(&delayed_uprobe_lock); 437 mutex_lock(&delayed_uprobe_lock);
440 if (d > 0) 438 if (d > 0)
441 ret = delayed_uprobe_add(uprob 439 ret = delayed_uprobe_add(uprobe, mm);
442 else 440 else
443 delayed_uprobe_remove(uprobe, 441 delayed_uprobe_remove(uprobe, mm);
444 mutex_unlock(&delayed_uprobe_lock); 442 mutex_unlock(&delayed_uprobe_lock);
445 443
446 return ret; 444 return ret;
447 } 445 }
448 446
449 /* 447 /*
450 * NOTE: 448 * NOTE:
451 * Expect the breakpoint instruction to be the 449 * Expect the breakpoint instruction to be the smallest size instruction for
452 * the architecture. If an arch has variable l 450 * the architecture. If an arch has variable length instruction and the
453 * breakpoint instruction is not of the smalle 451 * breakpoint instruction is not of the smallest length instruction
454 * supported by that architecture then we need 452 * supported by that architecture then we need to modify is_trap_at_addr and
455 * uprobe_write_opcode accordingly. This would 453 * uprobe_write_opcode accordingly. This would never be a problem for archs
456 * that have fixed length instructions. 454 * that have fixed length instructions.
457 * 455 *
458 * uprobe_write_opcode - write the opcode at a 456 * uprobe_write_opcode - write the opcode at a given virtual address.
459 * @auprobe: arch specific probepoint informat 457 * @auprobe: arch specific probepoint information.
460 * @mm: the probed process address space. 458 * @mm: the probed process address space.
461 * @vaddr: the virtual address to store the op 459 * @vaddr: the virtual address to store the opcode.
462 * @opcode: opcode to be written at @vaddr. 460 * @opcode: opcode to be written at @vaddr.
463 * 461 *
464 * Called with mm->mmap_lock held for read or !! 462 * Called with mm->mmap_lock held for write.
465 * Return 0 (success) or a negative errno. 463 * Return 0 (success) or a negative errno.
466 */ 464 */
467 int uprobe_write_opcode(struct arch_uprobe *au 465 int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
468 unsigned long vaddr, u 466 unsigned long vaddr, uprobe_opcode_t opcode)
469 { 467 {
470 struct uprobe *uprobe; 468 struct uprobe *uprobe;
471 struct page *old_page, *new_page; 469 struct page *old_page, *new_page;
472 struct vm_area_struct *vma; 470 struct vm_area_struct *vma;
473 int ret, is_register, ref_ctr_updated 471 int ret, is_register, ref_ctr_updated = 0;
474 bool orig_page_huge = false; 472 bool orig_page_huge = false;
475 unsigned int gup_flags = FOLL_FORCE; 473 unsigned int gup_flags = FOLL_FORCE;
476 474
477 is_register = is_swbp_insn(&opcode); 475 is_register = is_swbp_insn(&opcode);
478 uprobe = container_of(auprobe, struct 476 uprobe = container_of(auprobe, struct uprobe, arch);
479 477
480 retry: 478 retry:
481 if (is_register) 479 if (is_register)
482 gup_flags |= FOLL_SPLIT_PMD; 480 gup_flags |= FOLL_SPLIT_PMD;
483 /* Read the page with vaddr into memor 481 /* Read the page with vaddr into memory */
484 old_page = get_user_page_vma_remote(mm !! 482 ret = get_user_pages_remote(mm, vaddr, 1, gup_flags,
485 if (IS_ERR(old_page)) !! 483 &old_page, &vma, NULL);
486 return PTR_ERR(old_page); !! 484 if (ret <= 0)
>> 485 return ret;
487 486
488 ret = verify_opcode(old_page, vaddr, & 487 ret = verify_opcode(old_page, vaddr, &opcode);
489 if (ret <= 0) 488 if (ret <= 0)
490 goto put_old; 489 goto put_old;
491 490
492 if (WARN(!is_register && PageCompound( 491 if (WARN(!is_register && PageCompound(old_page),
493 "uprobe unregister should nev 492 "uprobe unregister should never work on compound page\n")) {
494 ret = -EINVAL; 493 ret = -EINVAL;
495 goto put_old; 494 goto put_old;
496 } 495 }
497 496
498 /* We are going to replace instruction 497 /* We are going to replace instruction, update ref_ctr. */
499 if (!ref_ctr_updated && uprobe->ref_ct 498 if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
500 ret = update_ref_ctr(uprobe, m 499 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
501 if (ret) 500 if (ret)
502 goto put_old; 501 goto put_old;
503 502
504 ref_ctr_updated = 1; 503 ref_ctr_updated = 1;
505 } 504 }
506 505
507 ret = 0; 506 ret = 0;
508 if (!is_register && !PageAnon(old_page 507 if (!is_register && !PageAnon(old_page))
509 goto put_old; 508 goto put_old;
510 509
511 ret = anon_vma_prepare(vma); 510 ret = anon_vma_prepare(vma);
512 if (ret) 511 if (ret)
513 goto put_old; 512 goto put_old;
514 513
515 ret = -ENOMEM; 514 ret = -ENOMEM;
516 new_page = alloc_page_vma(GFP_HIGHUSER 515 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
517 if (!new_page) 516 if (!new_page)
518 goto put_old; 517 goto put_old;
519 518
520 __SetPageUptodate(new_page); 519 __SetPageUptodate(new_page);
521 copy_highpage(new_page, old_page); 520 copy_highpage(new_page, old_page);
522 copy_to_page(new_page, vaddr, &opcode, 521 copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
523 522
524 if (!is_register) { 523 if (!is_register) {
525 struct page *orig_page; 524 struct page *orig_page;
526 pgoff_t index; 525 pgoff_t index;
527 526
528 VM_BUG_ON_PAGE(!PageAnon(old_p 527 VM_BUG_ON_PAGE(!PageAnon(old_page), old_page);
529 528
530 index = vaddr_to_offset(vma, v 529 index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT;
531 orig_page = find_get_page(vma- 530 orig_page = find_get_page(vma->vm_file->f_inode->i_mapping,
532 inde 531 index);
533 532
534 if (orig_page) { 533 if (orig_page) {
535 if (PageUptodate(orig_ 534 if (PageUptodate(orig_page) &&
536 pages_identical(ne 535 pages_identical(new_page, orig_page)) {
537 /* let go new_ 536 /* let go new_page */
538 put_page(new_p 537 put_page(new_page);
539 new_page = NUL 538 new_page = NULL;
540 539
541 if (PageCompou 540 if (PageCompound(orig_page))
542 orig_p 541 orig_page_huge = true;
543 } 542 }
544 put_page(orig_page); 543 put_page(orig_page);
545 } 544 }
546 } 545 }
547 546
548 ret = __replace_page(vma, vaddr & PAGE !! 547 ret = __replace_page(vma, vaddr, old_page, new_page);
549 if (new_page) 548 if (new_page)
550 put_page(new_page); 549 put_page(new_page);
551 put_old: 550 put_old:
552 put_page(old_page); 551 put_page(old_page);
553 552
554 if (unlikely(ret == -EAGAIN)) 553 if (unlikely(ret == -EAGAIN))
555 goto retry; 554 goto retry;
556 555
557 /* Revert back reference counter if in 556 /* Revert back reference counter if instruction update failed. */
558 if (ret && is_register && ref_ctr_upda 557 if (ret && is_register && ref_ctr_updated)
559 update_ref_ctr(uprobe, mm, -1) 558 update_ref_ctr(uprobe, mm, -1);
560 559
561 /* try collapse pmd for compound page 560 /* try collapse pmd for compound page */
562 if (!ret && orig_page_huge) 561 if (!ret && orig_page_huge)
563 collapse_pte_mapped_thp(mm, va !! 562 collapse_pte_mapped_thp(mm, vaddr);
564 563
565 return ret; 564 return ret;
566 } 565 }
567 566
568 /** 567 /**
569 * set_swbp - store breakpoint at a given addr 568 * set_swbp - store breakpoint at a given address.
570 * @auprobe: arch specific probepoint informat 569 * @auprobe: arch specific probepoint information.
571 * @mm: the probed process address space. 570 * @mm: the probed process address space.
572 * @vaddr: the virtual address to insert the o 571 * @vaddr: the virtual address to insert the opcode.
573 * 572 *
574 * For mm @mm, store the breakpoint instructio 573 * For mm @mm, store the breakpoint instruction at @vaddr.
575 * Return 0 (success) or a negative errno. 574 * Return 0 (success) or a negative errno.
576 */ 575 */
577 int __weak set_swbp(struct arch_uprobe *auprob 576 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
578 { 577 {
579 return uprobe_write_opcode(auprobe, mm 578 return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);
580 } 579 }
581 580
582 /** 581 /**
583 * set_orig_insn - Restore the original instru 582 * set_orig_insn - Restore the original instruction.
584 * @mm: the probed process address space. 583 * @mm: the probed process address space.
585 * @auprobe: arch specific probepoint informat 584 * @auprobe: arch specific probepoint information.
586 * @vaddr: the virtual address to insert the o 585 * @vaddr: the virtual address to insert the opcode.
587 * 586 *
588 * For mm @mm, restore the original opcode (op 587 * For mm @mm, restore the original opcode (opcode) at @vaddr.
589 * Return 0 (success) or a negative errno. 588 * Return 0 (success) or a negative errno.
590 */ 589 */
591 int __weak 590 int __weak
592 set_orig_insn(struct arch_uprobe *auprobe, str 591 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
593 { 592 {
594 return uprobe_write_opcode(auprobe, mm 593 return uprobe_write_opcode(auprobe, mm, vaddr,
595 *(uprobe_opcode_t *)&a 594 *(uprobe_opcode_t *)&auprobe->insn);
596 } 595 }
597 596
598 /* uprobe should have guaranteed positive refc <<
599 static struct uprobe *get_uprobe(struct uprobe 597 static struct uprobe *get_uprobe(struct uprobe *uprobe)
600 { 598 {
601 refcount_inc(&uprobe->ref); 599 refcount_inc(&uprobe->ref);
602 return uprobe; 600 return uprobe;
603 } 601 }
604 602
605 /* <<
606 * uprobe should have guaranteed lifetime, whi <<
607 * - caller already has refcount taken (and <<
608 * - uprobe is RCU protected and won't be fr <<
609 * - we are holding uprobes_treelock (for re <<
610 */ <<
611 static struct uprobe *try_get_uprobe(struct up <<
612 { <<
613 if (refcount_inc_not_zero(&uprobe->ref <<
614 return uprobe; <<
615 return NULL; <<
616 } <<
617 <<
618 static inline bool uprobe_is_active(struct upr <<
619 { <<
620 return !RB_EMPTY_NODE(&uprobe->rb_node <<
621 } <<
622 <<
623 static void uprobe_free_rcu(struct rcu_head *r <<
624 { <<
625 struct uprobe *uprobe = container_of(r <<
626 <<
627 kfree(uprobe); <<
628 } <<
629 <<
630 static void put_uprobe(struct uprobe *uprobe) 603 static void put_uprobe(struct uprobe *uprobe)
631 { 604 {
632 if (!refcount_dec_and_test(&uprobe->re !! 605 if (refcount_dec_and_test(&uprobe->ref)) {
633 return; !! 606 /*
634 !! 607 * If application munmap(exec_vma) before uprobe_unregister()
635 write_lock(&uprobes_treelock); !! 608 * gets called, we don't get a chance to remove uprobe from
636 !! 609 * delayed_uprobe_list from remove_breakpoint(). Do it here.
637 if (uprobe_is_active(uprobe)) { !! 610 */
638 write_seqcount_begin(&uprobes_ !! 611 mutex_lock(&delayed_uprobe_lock);
639 rb_erase(&uprobe->rb_node, &up !! 612 delayed_uprobe_remove(uprobe, NULL);
640 write_seqcount_end(&uprobes_se !! 613 mutex_unlock(&delayed_uprobe_lock);
>> 614 kfree(uprobe);
641 } 615 }
642 <<
643 write_unlock(&uprobes_treelock); <<
644 <<
645 /* <<
646 * If application munmap(exec_vma) bef <<
647 * gets called, we don't get a chance <<
648 * delayed_uprobe_list from remove_bre <<
649 */ <<
650 mutex_lock(&delayed_uprobe_lock); <<
651 delayed_uprobe_remove(uprobe, NULL); <<
652 mutex_unlock(&delayed_uprobe_lock); <<
653 <<
654 call_srcu(&uprobes_srcu, &uprobe->rcu, <<
655 } 616 }
656 617
657 static __always_inline 618 static __always_inline
658 int uprobe_cmp(const struct inode *l_inode, co 619 int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
659 const struct uprobe *r) 620 const struct uprobe *r)
660 { 621 {
661 if (l_inode < r->inode) 622 if (l_inode < r->inode)
662 return -1; 623 return -1;
663 624
664 if (l_inode > r->inode) 625 if (l_inode > r->inode)
665 return 1; 626 return 1;
666 627
667 if (l_offset < r->offset) 628 if (l_offset < r->offset)
668 return -1; 629 return -1;
669 630
670 if (l_offset > r->offset) 631 if (l_offset > r->offset)
671 return 1; 632 return 1;
672 633
673 return 0; 634 return 0;
674 } 635 }
675 636
676 #define __node_2_uprobe(node) \ 637 #define __node_2_uprobe(node) \
677 rb_entry((node), struct uprobe, rb_nod 638 rb_entry((node), struct uprobe, rb_node)
678 639
679 struct __uprobe_key { 640 struct __uprobe_key {
680 struct inode *inode; 641 struct inode *inode;
681 loff_t offset; 642 loff_t offset;
682 }; 643 };
683 644
684 static inline int __uprobe_cmp_key(const void 645 static inline int __uprobe_cmp_key(const void *key, const struct rb_node *b)
685 { 646 {
686 const struct __uprobe_key *a = key; 647 const struct __uprobe_key *a = key;
687 return uprobe_cmp(a->inode, a->offset, 648 return uprobe_cmp(a->inode, a->offset, __node_2_uprobe(b));
688 } 649 }
689 650
690 static inline int __uprobe_cmp(struct rb_node 651 static inline int __uprobe_cmp(struct rb_node *a, const struct rb_node *b)
691 { 652 {
692 struct uprobe *u = __node_2_uprobe(a); 653 struct uprobe *u = __node_2_uprobe(a);
693 return uprobe_cmp(u->inode, u->offset, 654 return uprobe_cmp(u->inode, u->offset, __node_2_uprobe(b));
694 } 655 }
695 656
696 /* !! 657 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
697 * Assumes being inside RCU protected region. <<
698 * No refcount is taken on returned uprobe. <<
699 */ <<
700 static struct uprobe *find_uprobe_rcu(struct i <<
701 { 658 {
702 struct __uprobe_key key = { 659 struct __uprobe_key key = {
703 .inode = inode, 660 .inode = inode,
704 .offset = offset, 661 .offset = offset,
705 }; 662 };
706 struct rb_node *node; !! 663 struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key);
707 unsigned int seq; <<
708 <<
709 lockdep_assert(srcu_read_lock_held(&up <<
710 664
711 do { !! 665 if (node)
712 seq = read_seqcount_begin(&upr !! 666 return get_uprobe(__node_2_uprobe(node));
713 node = rb_find_rcu(&key, &upro <<
714 /* <<
715 * Lockless RB-tree lookups ca <<
716 * If the element is found, it <<
717 * under RCU protection. If we <<
718 * validate that seqcount didn <<
719 * try again as we might have <<
720 * negative). If seqcount is u <<
721 */ <<
722 if (node) <<
723 return __node_2_uprobe <<
724 } while (read_seqcount_retry(&uprobes_ <<
725 667
726 return NULL; 668 return NULL;
727 } 669 }
728 670
729 /* 671 /*
730 * Attempt to insert a new uprobe into uprobes !! 672 * Find a uprobe corresponding to a given inode:offset
731 * !! 673 * Acquires uprobes_treelock
732 * If uprobe already exists (for given inode+o <<
733 * refcount of previously existing uprobe. <<
734 * <<
735 * If not, a provided new instance of uprobe i <<
736 * assumed initial refcount == 1). <<
737 * <<
738 * In any case, we return a uprobe instance th <<
739 * Caller has to clean up new uprobe instance, <<
740 * inserted into the tree. <<
741 * <<
742 * We assume that uprobes_treelock is held for <<
743 */ 674 */
>> 675 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
>> 676 {
>> 677 struct uprobe *uprobe;
>> 678
>> 679 spin_lock(&uprobes_treelock);
>> 680 uprobe = __find_uprobe(inode, offset);
>> 681 spin_unlock(&uprobes_treelock);
>> 682
>> 683 return uprobe;
>> 684 }
>> 685
744 static struct uprobe *__insert_uprobe(struct u 686 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
745 { 687 {
746 struct rb_node *node; 688 struct rb_node *node;
747 again: <<
748 node = rb_find_add_rcu(&uprobe->rb_nod <<
749 if (node) { <<
750 struct uprobe *u = __node_2_up <<
751 <<
752 if (!try_get_uprobe(u)) { <<
753 rb_erase(node, &uprobe <<
754 RB_CLEAR_NODE(&u->rb_n <<
755 goto again; <<
756 } <<
757 689
758 return u; !! 690 node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
759 } !! 691 if (node)
>> 692 return get_uprobe(__node_2_uprobe(node));
760 693
761 return uprobe; !! 694 /* get access + creation ref */
>> 695 refcount_set(&uprobe->ref, 2);
>> 696 return NULL;
762 } 697 }
763 698
764 /* 699 /*
765 * Acquire uprobes_treelock and insert uprobe !! 700 * Acquire uprobes_treelock.
766 * (or reuse existing one, see __insert_uprobe !! 701 * Matching uprobe already exists in rbtree;
>> 702 * increment (access refcount) and return the matching uprobe.
>> 703 *
>> 704 * No matching uprobe; insert the uprobe in rb_tree;
>> 705 * get a double refcount (access + creation) and return NULL.
767 */ 706 */
768 static struct uprobe *insert_uprobe(struct upr 707 static struct uprobe *insert_uprobe(struct uprobe *uprobe)
769 { 708 {
770 struct uprobe *u; 709 struct uprobe *u;
771 710
772 write_lock(&uprobes_treelock); !! 711 spin_lock(&uprobes_treelock);
773 write_seqcount_begin(&uprobes_seqcount <<
774 u = __insert_uprobe(uprobe); 712 u = __insert_uprobe(uprobe);
775 write_seqcount_end(&uprobes_seqcount); !! 713 spin_unlock(&uprobes_treelock);
776 write_unlock(&uprobes_treelock); <<
777 714
778 return u; 715 return u;
779 } 716 }
780 717
781 static void 718 static void
782 ref_ctr_mismatch_warn(struct uprobe *cur_uprob 719 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
783 { 720 {
784 pr_warn("ref_ctr_offset mismatch. inod 721 pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
785 "ref_ctr_offset(old): 0x%llx r 722 "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
786 uprobe->inode->i_ino, (unsigne 723 uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
787 (unsigned long long) cur_uprob 724 (unsigned long long) cur_uprobe->ref_ctr_offset,
788 (unsigned long long) uprobe->r 725 (unsigned long long) uprobe->ref_ctr_offset);
789 } 726 }
790 727
791 static struct uprobe *alloc_uprobe(struct inod 728 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
792 loff_t ref_ 729 loff_t ref_ctr_offset)
793 { 730 {
794 struct uprobe *uprobe, *cur_uprobe; 731 struct uprobe *uprobe, *cur_uprobe;
795 732
796 uprobe = kzalloc(sizeof(struct uprobe) 733 uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
797 if (!uprobe) 734 if (!uprobe)
798 return ERR_PTR(-ENOMEM); !! 735 return NULL;
799 736
800 uprobe->inode = inode; 737 uprobe->inode = inode;
801 uprobe->offset = offset; 738 uprobe->offset = offset;
802 uprobe->ref_ctr_offset = ref_ctr_offse 739 uprobe->ref_ctr_offset = ref_ctr_offset;
803 INIT_LIST_HEAD(&uprobe->consumers); <<
804 init_rwsem(&uprobe->register_rwsem); 740 init_rwsem(&uprobe->register_rwsem);
805 init_rwsem(&uprobe->consumer_rwsem); 741 init_rwsem(&uprobe->consumer_rwsem);
806 RB_CLEAR_NODE(&uprobe->rb_node); <<
807 refcount_set(&uprobe->ref, 1); <<
808 742
809 /* add to uprobes_tree, sorted on inod 743 /* add to uprobes_tree, sorted on inode:offset */
810 cur_uprobe = insert_uprobe(uprobe); 744 cur_uprobe = insert_uprobe(uprobe);
811 /* a uprobe exists for this inode:offs 745 /* a uprobe exists for this inode:offset combination */
812 if (cur_uprobe != uprobe) { !! 746 if (cur_uprobe) {
813 if (cur_uprobe->ref_ctr_offset 747 if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
814 ref_ctr_mismatch_warn( 748 ref_ctr_mismatch_warn(cur_uprobe, uprobe);
815 put_uprobe(cur_uprobe) 749 put_uprobe(cur_uprobe);
816 kfree(uprobe); 750 kfree(uprobe);
817 return ERR_PTR(-EINVAL 751 return ERR_PTR(-EINVAL);
818 } 752 }
819 kfree(uprobe); 753 kfree(uprobe);
820 uprobe = cur_uprobe; 754 uprobe = cur_uprobe;
821 } 755 }
822 756
823 return uprobe; 757 return uprobe;
824 } 758 }
825 759
826 static void consumer_add(struct uprobe *uprobe 760 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
827 { 761 {
828 down_write(&uprobe->consumer_rwsem); 762 down_write(&uprobe->consumer_rwsem);
829 list_add_rcu(&uc->cons_node, &uprobe-> !! 763 uc->next = uprobe->consumers;
>> 764 uprobe->consumers = uc;
830 up_write(&uprobe->consumer_rwsem); 765 up_write(&uprobe->consumer_rwsem);
831 } 766 }
832 767
833 /* 768 /*
834 * For uprobe @uprobe, delete the consumer @uc 769 * For uprobe @uprobe, delete the consumer @uc.
835 * Should never be called with consumer that's !! 770 * Return true if the @uc is deleted successfully
>> 771 * or return false.
836 */ 772 */
837 static void consumer_del(struct uprobe *uprobe !! 773 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
838 { 774 {
>> 775 struct uprobe_consumer **con;
>> 776 bool ret = false;
>> 777
839 down_write(&uprobe->consumer_rwsem); 778 down_write(&uprobe->consumer_rwsem);
840 list_del_rcu(&uc->cons_node); !! 779 for (con = &uprobe->consumers; *con; con = &(*con)->next) {
>> 780 if (*con == uc) {
>> 781 *con = uc->next;
>> 782 ret = true;
>> 783 break;
>> 784 }
>> 785 }
841 up_write(&uprobe->consumer_rwsem); 786 up_write(&uprobe->consumer_rwsem);
>> 787
>> 788 return ret;
842 } 789 }
843 790
844 static int __copy_insn(struct address_space *m 791 static int __copy_insn(struct address_space *mapping, struct file *filp,
845 void *insn, int nbytes 792 void *insn, int nbytes, loff_t offset)
846 { 793 {
847 struct page *page; 794 struct page *page;
848 /* 795 /*
849 * Ensure that the page that has the o 796 * Ensure that the page that has the original instruction is populated
850 * and in page-cache. If ->read_folio !! 797 * and in page-cache. If ->readpage == NULL it must be shmem_mapping(),
851 * see uprobe_register(). 798 * see uprobe_register().
852 */ 799 */
853 if (mapping->a_ops->read_folio) !! 800 if (mapping->a_ops->readpage)
854 page = read_mapping_page(mappi 801 page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
855 else 802 else
856 page = shmem_read_mapping_page 803 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
857 if (IS_ERR(page)) 804 if (IS_ERR(page))
858 return PTR_ERR(page); 805 return PTR_ERR(page);
859 806
860 copy_from_page(page, offset, insn, nby 807 copy_from_page(page, offset, insn, nbytes);
861 put_page(page); 808 put_page(page);
862 809
863 return 0; 810 return 0;
864 } 811 }
865 812
866 static int copy_insn(struct uprobe *uprobe, st 813 static int copy_insn(struct uprobe *uprobe, struct file *filp)
867 { 814 {
868 struct address_space *mapping = uprobe 815 struct address_space *mapping = uprobe->inode->i_mapping;
869 loff_t offs = uprobe->offset; 816 loff_t offs = uprobe->offset;
870 void *insn = &uprobe->arch.insn; 817 void *insn = &uprobe->arch.insn;
871 int size = sizeof(uprobe->arch.insn); 818 int size = sizeof(uprobe->arch.insn);
872 int len, err = -EIO; 819 int len, err = -EIO;
873 820
874 /* Copy only available bytes, -EIO if 821 /* Copy only available bytes, -EIO if nothing was read */
875 do { 822 do {
876 if (offs >= i_size_read(uprobe 823 if (offs >= i_size_read(uprobe->inode))
877 break; 824 break;
878 825
879 len = min_t(int, size, PAGE_SI 826 len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
880 err = __copy_insn(mapping, fil 827 err = __copy_insn(mapping, filp, insn, len, offs);
881 if (err) 828 if (err)
882 break; 829 break;
883 830
884 insn += len; 831 insn += len;
885 offs += len; 832 offs += len;
886 size -= len; 833 size -= len;
887 } while (size); 834 } while (size);
888 835
889 return err; 836 return err;
890 } 837 }
891 838
892 static int prepare_uprobe(struct uprobe *uprob 839 static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
893 struct mm_stru 840 struct mm_struct *mm, unsigned long vaddr)
894 { 841 {
895 int ret = 0; 842 int ret = 0;
896 843
897 if (test_bit(UPROBE_COPY_INSN, &uprobe 844 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
898 return ret; 845 return ret;
899 846
900 /* TODO: move this into _register, unt 847 /* TODO: move this into _register, until then we abuse this sem. */
901 down_write(&uprobe->consumer_rwsem); 848 down_write(&uprobe->consumer_rwsem);
902 if (test_bit(UPROBE_COPY_INSN, &uprobe 849 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
903 goto out; 850 goto out;
904 851
905 ret = copy_insn(uprobe, file); 852 ret = copy_insn(uprobe, file);
906 if (ret) 853 if (ret)
907 goto out; 854 goto out;
908 855
909 ret = -ENOTSUPP; 856 ret = -ENOTSUPP;
910 if (is_trap_insn((uprobe_opcode_t *)&u 857 if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
911 goto out; 858 goto out;
912 859
913 ret = arch_uprobe_analyze_insn(&uprobe 860 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
914 if (ret) 861 if (ret)
915 goto out; 862 goto out;
916 863
917 smp_wmb(); /* pairs with the smp_rmb() 864 smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
918 set_bit(UPROBE_COPY_INSN, &uprobe->fla 865 set_bit(UPROBE_COPY_INSN, &uprobe->flags);
919 866
920 out: 867 out:
921 up_write(&uprobe->consumer_rwsem); 868 up_write(&uprobe->consumer_rwsem);
922 869
923 return ret; 870 return ret;
924 } 871 }
925 872
926 static inline bool consumer_filter(struct upro !! 873 static inline bool consumer_filter(struct uprobe_consumer *uc,
>> 874 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
927 { 875 {
928 return !uc->filter || uc->filter(uc, m !! 876 return !uc->filter || uc->filter(uc, ctx, mm);
929 } 877 }
930 878
931 static bool filter_chain(struct uprobe *uprobe !! 879 static bool filter_chain(struct uprobe *uprobe,
>> 880 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
932 { 881 {
933 struct uprobe_consumer *uc; 882 struct uprobe_consumer *uc;
934 bool ret = false; 883 bool ret = false;
935 884
936 down_read(&uprobe->consumer_rwsem); 885 down_read(&uprobe->consumer_rwsem);
937 list_for_each_entry_srcu(uc, &uprobe-> !! 886 for (uc = uprobe->consumers; uc; uc = uc->next) {
938 srcu_read_loc !! 887 ret = consumer_filter(uc, ctx, mm);
939 ret = consumer_filter(uc, mm); <<
940 if (ret) 888 if (ret)
941 break; 889 break;
942 } 890 }
943 up_read(&uprobe->consumer_rwsem); 891 up_read(&uprobe->consumer_rwsem);
944 892
945 return ret; 893 return ret;
946 } 894 }
947 895
948 static int 896 static int
949 install_breakpoint(struct uprobe *uprobe, stru 897 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
950 struct vm_area_struct 898 struct vm_area_struct *vma, unsigned long vaddr)
951 { 899 {
952 bool first_uprobe; 900 bool first_uprobe;
953 int ret; 901 int ret;
954 902
955 ret = prepare_uprobe(uprobe, vma->vm_f 903 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
956 if (ret) 904 if (ret)
957 return ret; 905 return ret;
958 906
959 /* 907 /*
960 * set MMF_HAS_UPROBES in advance for 908 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
961 * the task can hit this breakpoint ri 909 * the task can hit this breakpoint right after __replace_page().
962 */ 910 */
963 first_uprobe = !test_bit(MMF_HAS_UPROB 911 first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
964 if (first_uprobe) 912 if (first_uprobe)
965 set_bit(MMF_HAS_UPROBES, &mm-> 913 set_bit(MMF_HAS_UPROBES, &mm->flags);
966 914
967 ret = set_swbp(&uprobe->arch, mm, vadd 915 ret = set_swbp(&uprobe->arch, mm, vaddr);
968 if (!ret) 916 if (!ret)
969 clear_bit(MMF_RECALC_UPROBES, 917 clear_bit(MMF_RECALC_UPROBES, &mm->flags);
970 else if (first_uprobe) 918 else if (first_uprobe)
971 clear_bit(MMF_HAS_UPROBES, &mm 919 clear_bit(MMF_HAS_UPROBES, &mm->flags);
972 920
973 return ret; 921 return ret;
974 } 922 }
975 923
976 static int 924 static int
977 remove_breakpoint(struct uprobe *uprobe, struc 925 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
978 { 926 {
979 set_bit(MMF_RECALC_UPROBES, &mm->flags 927 set_bit(MMF_RECALC_UPROBES, &mm->flags);
980 return set_orig_insn(&uprobe->arch, mm 928 return set_orig_insn(&uprobe->arch, mm, vaddr);
981 } 929 }
982 930
>> 931 static inline bool uprobe_is_active(struct uprobe *uprobe)
>> 932 {
>> 933 return !RB_EMPTY_NODE(&uprobe->rb_node);
>> 934 }
>> 935 /*
>> 936 * There could be threads that have already hit the breakpoint. They
>> 937 * will recheck the current insn and restart if find_uprobe() fails.
>> 938 * See find_active_uprobe().
>> 939 */
>> 940 static void delete_uprobe(struct uprobe *uprobe)
>> 941 {
>> 942 if (WARN_ON(!uprobe_is_active(uprobe)))
>> 943 return;
>> 944
>> 945 spin_lock(&uprobes_treelock);
>> 946 rb_erase(&uprobe->rb_node, &uprobes_tree);
>> 947 spin_unlock(&uprobes_treelock);
>> 948 RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
>> 949 put_uprobe(uprobe);
>> 950 }
>> 951
983 struct map_info { 952 struct map_info {
984 struct map_info *next; 953 struct map_info *next;
985 struct mm_struct *mm; 954 struct mm_struct *mm;
986 unsigned long vaddr; 955 unsigned long vaddr;
987 }; 956 };
988 957
989 static inline struct map_info *free_map_info(s 958 static inline struct map_info *free_map_info(struct map_info *info)
990 { 959 {
991 struct map_info *next = info->next; 960 struct map_info *next = info->next;
992 kfree(info); 961 kfree(info);
993 return next; 962 return next;
994 } 963 }
995 964
996 static struct map_info * 965 static struct map_info *
997 build_map_info(struct address_space *mapping, 966 build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
998 { 967 {
999 unsigned long pgoff = offset >> PAGE_S 968 unsigned long pgoff = offset >> PAGE_SHIFT;
1000 struct vm_area_struct *vma; 969 struct vm_area_struct *vma;
1001 struct map_info *curr = NULL; 970 struct map_info *curr = NULL;
1002 struct map_info *prev = NULL; 971 struct map_info *prev = NULL;
1003 struct map_info *info; 972 struct map_info *info;
1004 int more = 0; 973 int more = 0;
1005 974
1006 again: 975 again:
1007 i_mmap_lock_read(mapping); 976 i_mmap_lock_read(mapping);
1008 vma_interval_tree_foreach(vma, &mappi 977 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1009 if (!valid_vma(vma, is_regist 978 if (!valid_vma(vma, is_register))
1010 continue; 979 continue;
1011 980
1012 if (!prev && !more) { 981 if (!prev && !more) {
1013 /* 982 /*
1014 * Needs GFP_NOWAIT t 983 * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
1015 * reclaim. This is o 984 * reclaim. This is optimistic, no harm done if it fails.
1016 */ 985 */
1017 prev = kmalloc(sizeof 986 prev = kmalloc(sizeof(struct map_info),
1018 GFP_N 987 GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
1019 if (prev) 988 if (prev)
1020 prev->next = 989 prev->next = NULL;
1021 } 990 }
1022 if (!prev) { 991 if (!prev) {
1023 more++; 992 more++;
1024 continue; 993 continue;
1025 } 994 }
1026 995
1027 if (!mmget_not_zero(vma->vm_m 996 if (!mmget_not_zero(vma->vm_mm))
1028 continue; 997 continue;
1029 998
1030 info = prev; 999 info = prev;
1031 prev = prev->next; 1000 prev = prev->next;
1032 info->next = curr; 1001 info->next = curr;
1033 curr = info; 1002 curr = info;
1034 1003
1035 info->mm = vma->vm_mm; 1004 info->mm = vma->vm_mm;
1036 info->vaddr = offset_to_vaddr 1005 info->vaddr = offset_to_vaddr(vma, offset);
1037 } 1006 }
1038 i_mmap_unlock_read(mapping); 1007 i_mmap_unlock_read(mapping);
1039 1008
1040 if (!more) 1009 if (!more)
1041 goto out; 1010 goto out;
1042 1011
1043 prev = curr; 1012 prev = curr;
1044 while (curr) { 1013 while (curr) {
1045 mmput(curr->mm); 1014 mmput(curr->mm);
1046 curr = curr->next; 1015 curr = curr->next;
1047 } 1016 }
1048 1017
1049 do { 1018 do {
1050 info = kmalloc(sizeof(struct 1019 info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
1051 if (!info) { 1020 if (!info) {
1052 curr = ERR_PTR(-ENOME 1021 curr = ERR_PTR(-ENOMEM);
1053 goto out; 1022 goto out;
1054 } 1023 }
1055 info->next = prev; 1024 info->next = prev;
1056 prev = info; 1025 prev = info;
1057 } while (--more); 1026 } while (--more);
1058 1027
1059 goto again; 1028 goto again;
1060 out: 1029 out:
1061 while (prev) 1030 while (prev)
1062 prev = free_map_info(prev); 1031 prev = free_map_info(prev);
1063 return curr; 1032 return curr;
1064 } 1033 }
1065 1034
1066 static int 1035 static int
1067 register_for_each_vma(struct uprobe *uprobe, 1036 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
1068 { 1037 {
1069 bool is_register = !!new; 1038 bool is_register = !!new;
1070 struct map_info *info; 1039 struct map_info *info;
1071 int err = 0; 1040 int err = 0;
1072 1041
1073 percpu_down_write(&dup_mmap_sem); 1042 percpu_down_write(&dup_mmap_sem);
1074 info = build_map_info(uprobe->inode-> 1043 info = build_map_info(uprobe->inode->i_mapping,
1075 uprob 1044 uprobe->offset, is_register);
1076 if (IS_ERR(info)) { 1045 if (IS_ERR(info)) {
1077 err = PTR_ERR(info); 1046 err = PTR_ERR(info);
1078 goto out; 1047 goto out;
1079 } 1048 }
1080 1049
1081 while (info) { 1050 while (info) {
1082 struct mm_struct *mm = info-> 1051 struct mm_struct *mm = info->mm;
1083 struct vm_area_struct *vma; 1052 struct vm_area_struct *vma;
1084 1053
1085 if (err && is_register) 1054 if (err && is_register)
1086 goto free; 1055 goto free;
1087 /* !! 1056
1088 * We take mmap_lock for writ <<
1089 * find_active_uprobe_rcu() w <<
1090 * Thus this install_breakpoi <<
1091 * is_trap_at_addr() true rig <<
1092 * returns NULL in find_activ <<
1093 */ <<
1094 mmap_write_lock(mm); 1057 mmap_write_lock(mm);
1095 vma = find_vma(mm, info->vadd 1058 vma = find_vma(mm, info->vaddr);
1096 if (!vma || !valid_vma(vma, i 1059 if (!vma || !valid_vma(vma, is_register) ||
1097 file_inode(vma->vm_file) 1060 file_inode(vma->vm_file) != uprobe->inode)
1098 goto unlock; 1061 goto unlock;
1099 1062
1100 if (vma->vm_start > info->vad 1063 if (vma->vm_start > info->vaddr ||
1101 vaddr_to_offset(vma, info 1064 vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
1102 goto unlock; 1065 goto unlock;
1103 1066
1104 if (is_register) { 1067 if (is_register) {
1105 /* consult only the " 1068 /* consult only the "caller", new consumer. */
1106 if (consumer_filter(n !! 1069 if (consumer_filter(new,
>> 1070 UPROBE_FILTER_REGISTER, mm))
1107 err = install 1071 err = install_breakpoint(uprobe, mm, vma, info->vaddr);
1108 } else if (test_bit(MMF_HAS_U 1072 } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
1109 if (!filter_chain(upr !! 1073 if (!filter_chain(uprobe,
>> 1074 UPROBE_FILTER_UNREGISTER, mm))
1110 err |= remove 1075 err |= remove_breakpoint(uprobe, mm, info->vaddr);
1111 } 1076 }
1112 1077
1113 unlock: 1078 unlock:
1114 mmap_write_unlock(mm); 1079 mmap_write_unlock(mm);
1115 free: 1080 free:
1116 mmput(mm); 1081 mmput(mm);
1117 info = free_map_info(info); 1082 info = free_map_info(info);
1118 } 1083 }
1119 out: 1084 out:
1120 percpu_up_write(&dup_mmap_sem); 1085 percpu_up_write(&dup_mmap_sem);
1121 return err; 1086 return err;
1122 } 1087 }
1123 1088
1124 /** !! 1089 static void
1125 * uprobe_unregister_nosync - unregister an a !! 1090 __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
1126 * @uprobe: uprobe to remove <<
1127 * @uc: identify which probe if multiple prob <<
1128 */ <<
1129 void uprobe_unregister_nosync(struct uprobe * <<
1130 { 1091 {
1131 int err; 1092 int err;
1132 1093
1133 down_write(&uprobe->register_rwsem); !! 1094 if (WARN_ON(!consumer_del(uprobe, uc)))
1134 consumer_del(uprobe, uc); <<
1135 err = register_for_each_vma(uprobe, N <<
1136 up_write(&uprobe->register_rwsem); <<
1137 <<
1138 /* TODO : cant unregister? schedule a <<
1139 if (unlikely(err)) { <<
1140 uprobe_warn(current, "unregis <<
1141 return; 1095 return;
1142 } <<
1143 1096
1144 put_uprobe(uprobe); !! 1097 err = register_for_each_vma(uprobe, NULL);
>> 1098 /* TODO : cant unregister? schedule a worker thread */
>> 1099 if (!uprobe->consumers && !err)
>> 1100 delete_uprobe(uprobe);
1145 } 1101 }
1146 EXPORT_SYMBOL_GPL(uprobe_unregister_nosync); <<
1147 1102
1148 void uprobe_unregister_sync(void) !! 1103 /*
>> 1104 * uprobe_unregister - unregister an already registered probe.
>> 1105 * @inode: the file in which the probe has to be removed.
>> 1106 * @offset: offset from the start of the file.
>> 1107 * @uc: identify which probe if multiple probes are colocated.
>> 1108 */
>> 1109 void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
1149 { 1110 {
1150 /* !! 1111 struct uprobe *uprobe;
1151 * Now that handler_chain() and handl !! 1112
1152 * uprobe->consumers list under RCU p !! 1113 uprobe = find_uprobe(inode, offset);
1153 * uprobe->register_rwsem, we need to !! 1114 if (WARN_ON(!uprobe))
1154 * make sure that we can't call into !! 1115 return;
1155 * uprobe_consumer's callbacks anymor !! 1116
1156 * unlucky enough caller can free con !! 1117 down_write(&uprobe->register_rwsem);
1157 * handler_chain() or handle_uretprob !! 1118 __uprobe_unregister(uprobe, uc);
1158 */ !! 1119 up_write(&uprobe->register_rwsem);
1159 synchronize_srcu(&uprobes_srcu); !! 1120 put_uprobe(uprobe);
1160 } 1121 }
1161 EXPORT_SYMBOL_GPL(uprobe_unregister_sync); !! 1122 EXPORT_SYMBOL_GPL(uprobe_unregister);
1162 1123
1163 /** !! 1124 /*
1164 * uprobe_register - register a probe !! 1125 * __uprobe_register - register a probe
1165 * @inode: the file in which the probe has to 1126 * @inode: the file in which the probe has to be placed.
1166 * @offset: offset from the start of the file 1127 * @offset: offset from the start of the file.
1167 * @ref_ctr_offset: offset of SDT marker / re <<
1168 * @uc: information on howto handle the probe 1128 * @uc: information on howto handle the probe..
1169 * 1129 *
1170 * Apart from the access refcount, uprobe_reg !! 1130 * Apart from the access refcount, __uprobe_register() takes a creation
1171 * refcount (thro alloc_uprobe) if and only i 1131 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
1172 * inserted into the rbtree (i.e first consum 1132 * inserted into the rbtree (i.e first consumer for a @inode:@offset
1173 * tuple). Creation refcount stops uprobe_un 1133 * tuple). Creation refcount stops uprobe_unregister from freeing the
1174 * @uprobe even before the register operation 1134 * @uprobe even before the register operation is complete. Creation
1175 * refcount is released when the last @uc for 1135 * refcount is released when the last @uc for the @uprobe
1176 * unregisters. Caller of uprobe_register() i !! 1136 * unregisters. Caller of __uprobe_register() is required to keep @inode
1177 * (and the containing mount) referenced. 1137 * (and the containing mount) referenced.
1178 * 1138 *
1179 * Return: pointer to the new uprobe on succe !! 1139 * Return errno if it cannot successully install probes
>> 1140 * else return 0 (success)
1180 */ 1141 */
1181 struct uprobe *uprobe_register(struct inode * !! 1142 static int __uprobe_register(struct inode *inode, loff_t offset,
1182 loff_t offset !! 1143 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1183 struct uprobe <<
1184 { 1144 {
1185 struct uprobe *uprobe; 1145 struct uprobe *uprobe;
1186 int ret; 1146 int ret;
1187 1147
1188 /* Uprobe must have at least one set 1148 /* Uprobe must have at least one set consumer */
1189 if (!uc->handler && !uc->ret_handler) 1149 if (!uc->handler && !uc->ret_handler)
1190 return ERR_PTR(-EINVAL); !! 1150 return -EINVAL;
1191 1151
1192 /* copy_insn() uses read_mapping_page 1152 /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
1193 if (!inode->i_mapping->a_ops->read_fo !! 1153 if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
1194 !shmem_mapping(inode->i_mapping)) !! 1154 return -EIO;
1195 return ERR_PTR(-EIO); <<
1196 /* Racy, just to catch the obvious mi 1155 /* Racy, just to catch the obvious mistakes */
1197 if (offset > i_size_read(inode)) 1156 if (offset > i_size_read(inode))
1198 return ERR_PTR(-EINVAL); !! 1157 return -EINVAL;
1199 1158
1200 /* 1159 /*
1201 * This ensures that copy_from_page() 1160 * This ensures that copy_from_page(), copy_to_page() and
1202 * __update_ref_ctr() can't cross pag 1161 * __update_ref_ctr() can't cross page boundary.
1203 */ 1162 */
1204 if (!IS_ALIGNED(offset, UPROBE_SWBP_I 1163 if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
1205 return ERR_PTR(-EINVAL); !! 1164 return -EINVAL;
1206 if (!IS_ALIGNED(ref_ctr_offset, sizeo 1165 if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
1207 return ERR_PTR(-EINVAL); !! 1166 return -EINVAL;
1208 1167
>> 1168 retry:
1209 uprobe = alloc_uprobe(inode, offset, 1169 uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
>> 1170 if (!uprobe)
>> 1171 return -ENOMEM;
1210 if (IS_ERR(uprobe)) 1172 if (IS_ERR(uprobe))
1211 return uprobe; !! 1173 return PTR_ERR(uprobe);
1212 1174
>> 1175 /*
>> 1176 * We can race with uprobe_unregister()->delete_uprobe().
>> 1177 * Check uprobe_is_active() and retry if it is false.
>> 1178 */
1213 down_write(&uprobe->register_rwsem); 1179 down_write(&uprobe->register_rwsem);
1214 consumer_add(uprobe, uc); !! 1180 ret = -EAGAIN;
1215 ret = register_for_each_vma(uprobe, u !! 1181 if (likely(uprobe_is_active(uprobe))) {
>> 1182 consumer_add(uprobe, uc);
>> 1183 ret = register_for_each_vma(uprobe, uc);
>> 1184 if (ret)
>> 1185 __uprobe_unregister(uprobe, uc);
>> 1186 }
1216 up_write(&uprobe->register_rwsem); 1187 up_write(&uprobe->register_rwsem);
>> 1188 put_uprobe(uprobe);
1217 1189
1218 if (ret) { !! 1190 if (unlikely(ret == -EAGAIN))
1219 uprobe_unregister_nosync(upro !! 1191 goto retry;
1220 /* !! 1192 return ret;
1221 * Registration might have pa !! 1193 }
1222 * this consumer being called <<
1223 * sync here. It's ok, it's u <<
1224 */ <<
1225 uprobe_unregister_sync(); <<
1226 return ERR_PTR(ret); <<
1227 } <<
1228 1194
1229 return uprobe; !! 1195 int uprobe_register(struct inode *inode, loff_t offset,
>> 1196 struct uprobe_consumer *uc)
>> 1197 {
>> 1198 return __uprobe_register(inode, offset, 0, uc);
1230 } 1199 }
1231 EXPORT_SYMBOL_GPL(uprobe_register); 1200 EXPORT_SYMBOL_GPL(uprobe_register);
1232 1201
1233 /** !! 1202 int uprobe_register_refctr(struct inode *inode, loff_t offset,
1234 * uprobe_apply - add or remove the breakpoin !! 1203 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1235 * @uprobe: uprobe which "owns" the breakpoin !! 1204 {
>> 1205 return __uprobe_register(inode, offset, ref_ctr_offset, uc);
>> 1206 }
>> 1207 EXPORT_SYMBOL_GPL(uprobe_register_refctr);
>> 1208
>> 1209 /*
>> 1210 * uprobe_apply - unregister an already registered probe.
>> 1211 * @inode: the file in which the probe has to be removed.
>> 1212 * @offset: offset from the start of the file.
1236 * @uc: consumer which wants to add more or r 1213 * @uc: consumer which wants to add more or remove some breakpoints
1237 * @add: add or remove the breakpoints 1214 * @add: add or remove the breakpoints
1238 * Return: 0 on success or negative error cod <<
1239 */ 1215 */
1240 int uprobe_apply(struct uprobe *uprobe, struc !! 1216 int uprobe_apply(struct inode *inode, loff_t offset,
>> 1217 struct uprobe_consumer *uc, bool add)
1241 { 1218 {
>> 1219 struct uprobe *uprobe;
1242 struct uprobe_consumer *con; 1220 struct uprobe_consumer *con;
1243 int ret = -ENOENT, srcu_idx; !! 1221 int ret = -ENOENT;
1244 <<
1245 down_write(&uprobe->register_rwsem); <<
1246 1222
1247 srcu_idx = srcu_read_lock(&uprobes_sr !! 1223 uprobe = find_uprobe(inode, offset);
1248 list_for_each_entry_srcu(con, &uprobe !! 1224 if (WARN_ON(!uprobe))
1249 srcu_read_lo !! 1225 return ret;
1250 if (con == uc) { <<
1251 ret = register_for_ea <<
1252 break; <<
1253 } <<
1254 } <<
1255 srcu_read_unlock(&uprobes_srcu, srcu_ <<
1256 1226
>> 1227 down_write(&uprobe->register_rwsem);
>> 1228 for (con = uprobe->consumers; con && con != uc ; con = con->next)
>> 1229 ;
>> 1230 if (con)
>> 1231 ret = register_for_each_vma(uprobe, add ? uc : NULL);
1257 up_write(&uprobe->register_rwsem); 1232 up_write(&uprobe->register_rwsem);
>> 1233 put_uprobe(uprobe);
1258 1234
1259 return ret; 1235 return ret;
1260 } 1236 }
1261 1237
1262 static int unapply_uprobe(struct uprobe *upro 1238 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
1263 { 1239 {
1264 VMA_ITERATOR(vmi, mm, 0); <<
1265 struct vm_area_struct *vma; 1240 struct vm_area_struct *vma;
1266 int err = 0; 1241 int err = 0;
1267 1242
1268 mmap_read_lock(mm); 1243 mmap_read_lock(mm);
1269 for_each_vma(vmi, vma) { !! 1244 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1270 unsigned long vaddr; 1245 unsigned long vaddr;
1271 loff_t offset; 1246 loff_t offset;
1272 1247
1273 if (!valid_vma(vma, false) || 1248 if (!valid_vma(vma, false) ||
1274 file_inode(vma->vm_file) 1249 file_inode(vma->vm_file) != uprobe->inode)
1275 continue; 1250 continue;
1276 1251
1277 offset = (loff_t)vma->vm_pgof 1252 offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1278 if (uprobe->offset < offset 1253 if (uprobe->offset < offset ||
1279 uprobe->offset >= offset 1254 uprobe->offset >= offset + vma->vm_end - vma->vm_start)
1280 continue; 1255 continue;
1281 1256
1282 vaddr = offset_to_vaddr(vma, 1257 vaddr = offset_to_vaddr(vma, uprobe->offset);
1283 err |= remove_breakpoint(upro 1258 err |= remove_breakpoint(uprobe, mm, vaddr);
1284 } 1259 }
1285 mmap_read_unlock(mm); 1260 mmap_read_unlock(mm);
1286 1261
1287 return err; 1262 return err;
1288 } 1263 }
1289 1264
1290 static struct rb_node * 1265 static struct rb_node *
1291 find_node_in_range(struct inode *inode, loff_ 1266 find_node_in_range(struct inode *inode, loff_t min, loff_t max)
1292 { 1267 {
1293 struct rb_node *n = uprobes_tree.rb_n 1268 struct rb_node *n = uprobes_tree.rb_node;
1294 1269
1295 while (n) { 1270 while (n) {
1296 struct uprobe *u = rb_entry(n 1271 struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
1297 1272
1298 if (inode < u->inode) { 1273 if (inode < u->inode) {
1299 n = n->rb_left; 1274 n = n->rb_left;
1300 } else if (inode > u->inode) 1275 } else if (inode > u->inode) {
1301 n = n->rb_right; 1276 n = n->rb_right;
1302 } else { 1277 } else {
1303 if (max < u->offset) 1278 if (max < u->offset)
1304 n = n->rb_lef 1279 n = n->rb_left;
1305 else if (min > u->off 1280 else if (min > u->offset)
1306 n = n->rb_rig 1281 n = n->rb_right;
1307 else 1282 else
1308 break; 1283 break;
1309 } 1284 }
1310 } 1285 }
1311 1286
1312 return n; 1287 return n;
1313 } 1288 }
1314 1289
1315 /* 1290 /*
1316 * For a given range in vma, build a list of 1291 * For a given range in vma, build a list of probes that need to be inserted.
1317 */ 1292 */
1318 static void build_probe_list(struct inode *in 1293 static void build_probe_list(struct inode *inode,
1319 struct vm_are 1294 struct vm_area_struct *vma,
1320 unsigned long 1295 unsigned long start, unsigned long end,
1321 struct list_h 1296 struct list_head *head)
1322 { 1297 {
1323 loff_t min, max; 1298 loff_t min, max;
1324 struct rb_node *n, *t; 1299 struct rb_node *n, *t;
1325 struct uprobe *u; 1300 struct uprobe *u;
1326 1301
1327 INIT_LIST_HEAD(head); 1302 INIT_LIST_HEAD(head);
1328 min = vaddr_to_offset(vma, start); 1303 min = vaddr_to_offset(vma, start);
1329 max = min + (end - start) - 1; 1304 max = min + (end - start) - 1;
1330 1305
1331 read_lock(&uprobes_treelock); !! 1306 spin_lock(&uprobes_treelock);
1332 n = find_node_in_range(inode, min, ma 1307 n = find_node_in_range(inode, min, max);
1333 if (n) { 1308 if (n) {
1334 for (t = n; t; t = rb_prev(t) 1309 for (t = n; t; t = rb_prev(t)) {
1335 u = rb_entry(t, struc 1310 u = rb_entry(t, struct uprobe, rb_node);
1336 if (u->inode != inode 1311 if (u->inode != inode || u->offset < min)
1337 break; 1312 break;
1338 /* if uprobe went awa !! 1313 list_add(&u->pending_list, head);
1339 if (try_get_uprobe(u) !! 1314 get_uprobe(u);
1340 list_add(&u-> <<
1341 } 1315 }
1342 for (t = n; (t = rb_next(t)); 1316 for (t = n; (t = rb_next(t)); ) {
1343 u = rb_entry(t, struc 1317 u = rb_entry(t, struct uprobe, rb_node);
1344 if (u->inode != inode 1318 if (u->inode != inode || u->offset > max)
1345 break; 1319 break;
1346 /* if uprobe went awa !! 1320 list_add(&u->pending_list, head);
1347 if (try_get_uprobe(u) !! 1321 get_uprobe(u);
1348 list_add(&u-> <<
1349 } 1322 }
1350 } 1323 }
1351 read_unlock(&uprobes_treelock); !! 1324 spin_unlock(&uprobes_treelock);
1352 } 1325 }
1353 1326
1354 /* @vma contains reference counter, not the p 1327 /* @vma contains reference counter, not the probed instruction. */
1355 static int delayed_ref_ctr_inc(struct vm_area 1328 static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
1356 { 1329 {
1357 struct list_head *pos, *q; 1330 struct list_head *pos, *q;
1358 struct delayed_uprobe *du; 1331 struct delayed_uprobe *du;
1359 unsigned long vaddr; 1332 unsigned long vaddr;
1360 int ret = 0, err = 0; 1333 int ret = 0, err = 0;
1361 1334
1362 mutex_lock(&delayed_uprobe_lock); 1335 mutex_lock(&delayed_uprobe_lock);
1363 list_for_each_safe(pos, q, &delayed_u 1336 list_for_each_safe(pos, q, &delayed_uprobe_list) {
1364 du = list_entry(pos, struct d 1337 du = list_entry(pos, struct delayed_uprobe, list);
1365 1338
1366 if (du->mm != vma->vm_mm || 1339 if (du->mm != vma->vm_mm ||
1367 !valid_ref_ctr_vma(du->up 1340 !valid_ref_ctr_vma(du->uprobe, vma))
1368 continue; 1341 continue;
1369 1342
1370 vaddr = offset_to_vaddr(vma, 1343 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
1371 ret = __update_ref_ctr(vma->v 1344 ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
1372 if (ret) { 1345 if (ret) {
1373 update_ref_ctr_warn(d 1346 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
1374 if (!err) 1347 if (!err)
1375 err = ret; 1348 err = ret;
1376 } 1349 }
1377 delayed_uprobe_delete(du); 1350 delayed_uprobe_delete(du);
1378 } 1351 }
1379 mutex_unlock(&delayed_uprobe_lock); 1352 mutex_unlock(&delayed_uprobe_lock);
1380 return err; 1353 return err;
1381 } 1354 }
1382 1355
1383 /* 1356 /*
1384 * Called from mmap_region/vma_merge with mm- !! 1357 * Called from mmap_region/vma_adjust with mm->mmap_lock acquired.
1385 * 1358 *
1386 * Currently we ignore all errors and always 1359 * Currently we ignore all errors and always return 0, the callers
1387 * can't handle the failure anyway. 1360 * can't handle the failure anyway.
1388 */ 1361 */
1389 int uprobe_mmap(struct vm_area_struct *vma) 1362 int uprobe_mmap(struct vm_area_struct *vma)
1390 { 1363 {
1391 struct list_head tmp_list; 1364 struct list_head tmp_list;
1392 struct uprobe *uprobe, *u; 1365 struct uprobe *uprobe, *u;
1393 struct inode *inode; 1366 struct inode *inode;
1394 1367
1395 if (no_uprobe_events()) 1368 if (no_uprobe_events())
1396 return 0; 1369 return 0;
1397 1370
1398 if (vma->vm_file && 1371 if (vma->vm_file &&
1399 (vma->vm_flags & (VM_WRITE|VM_SHA 1372 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
1400 test_bit(MMF_HAS_UPROBES, &vma->v 1373 test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
1401 delayed_ref_ctr_inc(vma); 1374 delayed_ref_ctr_inc(vma);
1402 1375
1403 if (!valid_vma(vma, true)) 1376 if (!valid_vma(vma, true))
1404 return 0; 1377 return 0;
1405 1378
1406 inode = file_inode(vma->vm_file); 1379 inode = file_inode(vma->vm_file);
1407 if (!inode) 1380 if (!inode)
1408 return 0; 1381 return 0;
1409 1382
1410 mutex_lock(uprobes_mmap_hash(inode)); 1383 mutex_lock(uprobes_mmap_hash(inode));
1411 build_probe_list(inode, vma, vma->vm_ 1384 build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
1412 /* 1385 /*
1413 * We can race with uprobe_unregister 1386 * We can race with uprobe_unregister(), this uprobe can be already
1414 * removed. But in this case filter_c 1387 * removed. But in this case filter_chain() must return false, all
1415 * consumers have gone away. 1388 * consumers have gone away.
1416 */ 1389 */
1417 list_for_each_entry_safe(uprobe, u, & 1390 list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
1418 if (!fatal_signal_pending(cur 1391 if (!fatal_signal_pending(current) &&
1419 filter_chain(uprobe, vma- !! 1392 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
1420 unsigned long vaddr = 1393 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
1421 install_breakpoint(up 1394 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
1422 } 1395 }
1423 put_uprobe(uprobe); 1396 put_uprobe(uprobe);
1424 } 1397 }
1425 mutex_unlock(uprobes_mmap_hash(inode) 1398 mutex_unlock(uprobes_mmap_hash(inode));
1426 1399
1427 return 0; 1400 return 0;
1428 } 1401 }
1429 1402
1430 static bool 1403 static bool
1431 vma_has_uprobes(struct vm_area_struct *vma, u 1404 vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1432 { 1405 {
1433 loff_t min, max; 1406 loff_t min, max;
1434 struct inode *inode; 1407 struct inode *inode;
1435 struct rb_node *n; 1408 struct rb_node *n;
1436 1409
1437 inode = file_inode(vma->vm_file); 1410 inode = file_inode(vma->vm_file);
1438 1411
1439 min = vaddr_to_offset(vma, start); 1412 min = vaddr_to_offset(vma, start);
1440 max = min + (end - start) - 1; 1413 max = min + (end - start) - 1;
1441 1414
1442 read_lock(&uprobes_treelock); !! 1415 spin_lock(&uprobes_treelock);
1443 n = find_node_in_range(inode, min, ma 1416 n = find_node_in_range(inode, min, max);
1444 read_unlock(&uprobes_treelock); !! 1417 spin_unlock(&uprobes_treelock);
1445 1418
1446 return !!n; 1419 return !!n;
1447 } 1420 }
1448 1421
1449 /* 1422 /*
1450 * Called in context of a munmap of a vma. 1423 * Called in context of a munmap of a vma.
1451 */ 1424 */
1452 void uprobe_munmap(struct vm_area_struct *vma 1425 void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1453 { 1426 {
1454 if (no_uprobe_events() || !valid_vma( 1427 if (no_uprobe_events() || !valid_vma(vma, false))
1455 return; 1428 return;
1456 1429
1457 if (!atomic_read(&vma->vm_mm->mm_user 1430 if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
1458 return; 1431 return;
1459 1432
1460 if (!test_bit(MMF_HAS_UPROBES, &vma-> 1433 if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
1461 test_bit(MMF_RECALC_UPROBES, &vm 1434 test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
1462 return; 1435 return;
1463 1436
1464 if (vma_has_uprobes(vma, start, end)) 1437 if (vma_has_uprobes(vma, start, end))
1465 set_bit(MMF_RECALC_UPROBES, & 1438 set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
1466 } 1439 }
1467 1440
1468 static vm_fault_t xol_fault(const struct vm_s <<
1469 struct vm_area_st <<
1470 { <<
1471 struct xol_area *area = vma->vm_mm->u <<
1472 <<
1473 vmf->page = area->page; <<
1474 get_page(vmf->page); <<
1475 return 0; <<
1476 } <<
1477 <<
1478 static const struct vm_special_mapping xol_ma <<
1479 .name = "[uprobes]", <<
1480 .fault = xol_fault, <<
1481 }; <<
1482 <<
1483 /* Slot allocation for XOL */ 1441 /* Slot allocation for XOL */
1484 static int xol_add_vma(struct mm_struct *mm, 1442 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1485 { 1443 {
1486 struct vm_area_struct *vma; 1444 struct vm_area_struct *vma;
1487 int ret; 1445 int ret;
1488 1446
1489 if (mmap_write_lock_killable(mm)) 1447 if (mmap_write_lock_killable(mm))
1490 return -EINTR; 1448 return -EINTR;
1491 1449
1492 if (mm->uprobes_state.xol_area) { 1450 if (mm->uprobes_state.xol_area) {
1493 ret = -EALREADY; 1451 ret = -EALREADY;
1494 goto fail; 1452 goto fail;
1495 } 1453 }
1496 1454
1497 if (!area->vaddr) { 1455 if (!area->vaddr) {
1498 /* Try to map as high as poss 1456 /* Try to map as high as possible, this is only a hint. */
1499 area->vaddr = get_unmapped_ar 1457 area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1500 1458 PAGE_SIZE, 0, 0);
1501 if (IS_ERR_VALUE(area->vaddr) 1459 if (IS_ERR_VALUE(area->vaddr)) {
1502 ret = area->vaddr; 1460 ret = area->vaddr;
1503 goto fail; 1461 goto fail;
1504 } 1462 }
1505 } 1463 }
1506 1464
1507 vma = _install_special_mapping(mm, ar 1465 vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1508 VM_EXEC|VM_MA 1466 VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
1509 &xol_mapping) !! 1467 &area->xol_mapping);
1510 if (IS_ERR(vma)) { 1468 if (IS_ERR(vma)) {
1511 ret = PTR_ERR(vma); 1469 ret = PTR_ERR(vma);
1512 goto fail; 1470 goto fail;
1513 } 1471 }
1514 1472
1515 ret = 0; 1473 ret = 0;
1516 /* pairs with get_xol_area() */ 1474 /* pairs with get_xol_area() */
1517 smp_store_release(&mm->uprobes_state. 1475 smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
1518 fail: 1476 fail:
1519 mmap_write_unlock(mm); 1477 mmap_write_unlock(mm);
1520 1478
1521 return ret; 1479 return ret;
1522 } 1480 }
1523 1481
1524 void * __weak arch_uprobe_trampoline(unsigned <<
1525 { <<
1526 static uprobe_opcode_t insn = UPROBE_ <<
1527 <<
1528 *psize = UPROBE_SWBP_INSN_SIZE; <<
1529 return &insn; <<
1530 } <<
1531 <<
1532 static struct xol_area *__create_xol_area(uns 1482 static struct xol_area *__create_xol_area(unsigned long vaddr)
1533 { 1483 {
1534 struct mm_struct *mm = current->mm; 1484 struct mm_struct *mm = current->mm;
1535 unsigned long insns_size; !! 1485 uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1536 struct xol_area *area; 1486 struct xol_area *area;
1537 void *insns; <<
1538 1487
1539 area = kzalloc(sizeof(*area), GFP_KER !! 1488 area = kmalloc(sizeof(*area), GFP_KERNEL);
1540 if (unlikely(!area)) 1489 if (unlikely(!area))
1541 goto out; 1490 goto out;
1542 1491
1543 area->bitmap = kcalloc(BITS_TO_LONGS( 1492 area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
1544 GFP_KERNEL); 1493 GFP_KERNEL);
1545 if (!area->bitmap) 1494 if (!area->bitmap)
1546 goto free_area; 1495 goto free_area;
1547 1496
1548 area->page = alloc_page(GFP_HIGHUSER !! 1497 area->xol_mapping.name = "[uprobes]";
1549 if (!area->page) !! 1498 area->xol_mapping.fault = NULL;
>> 1499 area->xol_mapping.pages = area->pages;
>> 1500 area->pages[0] = alloc_page(GFP_HIGHUSER);
>> 1501 if (!area->pages[0])
1550 goto free_bitmap; 1502 goto free_bitmap;
>> 1503 area->pages[1] = NULL;
1551 1504
1552 area->vaddr = vaddr; 1505 area->vaddr = vaddr;
1553 init_waitqueue_head(&area->wq); 1506 init_waitqueue_head(&area->wq);
1554 /* Reserve the 1st slot for get_tramp 1507 /* Reserve the 1st slot for get_trampoline_vaddr() */
1555 set_bit(0, area->bitmap); 1508 set_bit(0, area->bitmap);
1556 atomic_set(&area->slot_count, 1); 1509 atomic_set(&area->slot_count, 1);
1557 insns = arch_uprobe_trampoline(&insns !! 1510 arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
1558 arch_uprobe_copy_ixol(area->page, 0, <<
1559 1511
1560 if (!xol_add_vma(mm, area)) 1512 if (!xol_add_vma(mm, area))
1561 return area; 1513 return area;
1562 1514
1563 __free_page(area->page); !! 1515 __free_page(area->pages[0]);
1564 free_bitmap: 1516 free_bitmap:
1565 kfree(area->bitmap); 1517 kfree(area->bitmap);
1566 free_area: 1518 free_area:
1567 kfree(area); 1519 kfree(area);
1568 out: 1520 out:
1569 return NULL; 1521 return NULL;
1570 } 1522 }
1571 1523
1572 /* 1524 /*
1573 * get_xol_area - Allocate process's xol_area 1525 * get_xol_area - Allocate process's xol_area if necessary.
1574 * This area will be used for storing instruc 1526 * This area will be used for storing instructions for execution out of line.
1575 * 1527 *
1576 * Returns the allocated area or NULL. 1528 * Returns the allocated area or NULL.
1577 */ 1529 */
1578 static struct xol_area *get_xol_area(void) 1530 static struct xol_area *get_xol_area(void)
1579 { 1531 {
1580 struct mm_struct *mm = current->mm; 1532 struct mm_struct *mm = current->mm;
1581 struct xol_area *area; 1533 struct xol_area *area;
1582 1534
1583 if (!mm->uprobes_state.xol_area) 1535 if (!mm->uprobes_state.xol_area)
1584 __create_xol_area(0); 1536 __create_xol_area(0);
1585 1537
1586 /* Pairs with xol_add_vma() smp_store 1538 /* Pairs with xol_add_vma() smp_store_release() */
1587 area = READ_ONCE(mm->uprobes_state.xo 1539 area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
1588 return area; 1540 return area;
1589 } 1541 }
1590 1542
1591 /* 1543 /*
1592 * uprobe_clear_state - Free the area allocat 1544 * uprobe_clear_state - Free the area allocated for slots.
1593 */ 1545 */
1594 void uprobe_clear_state(struct mm_struct *mm) 1546 void uprobe_clear_state(struct mm_struct *mm)
1595 { 1547 {
1596 struct xol_area *area = mm->uprobes_s 1548 struct xol_area *area = mm->uprobes_state.xol_area;
1597 1549
1598 mutex_lock(&delayed_uprobe_lock); 1550 mutex_lock(&delayed_uprobe_lock);
1599 delayed_uprobe_remove(NULL, mm); 1551 delayed_uprobe_remove(NULL, mm);
1600 mutex_unlock(&delayed_uprobe_lock); 1552 mutex_unlock(&delayed_uprobe_lock);
1601 1553
1602 if (!area) 1554 if (!area)
1603 return; 1555 return;
1604 1556
1605 put_page(area->page); !! 1557 put_page(area->pages[0]);
1606 kfree(area->bitmap); 1558 kfree(area->bitmap);
1607 kfree(area); 1559 kfree(area);
1608 } 1560 }
1609 1561
1610 void uprobe_start_dup_mmap(void) 1562 void uprobe_start_dup_mmap(void)
1611 { 1563 {
1612 percpu_down_read(&dup_mmap_sem); 1564 percpu_down_read(&dup_mmap_sem);
1613 } 1565 }
1614 1566
1615 void uprobe_end_dup_mmap(void) 1567 void uprobe_end_dup_mmap(void)
1616 { 1568 {
1617 percpu_up_read(&dup_mmap_sem); 1569 percpu_up_read(&dup_mmap_sem);
1618 } 1570 }
1619 1571
1620 void uprobe_dup_mmap(struct mm_struct *oldmm, 1572 void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
1621 { 1573 {
1622 if (test_bit(MMF_HAS_UPROBES, &oldmm- 1574 if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
1623 set_bit(MMF_HAS_UPROBES, &new 1575 set_bit(MMF_HAS_UPROBES, &newmm->flags);
1624 /* unconditionally, dup_mmap( 1576 /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
1625 set_bit(MMF_RECALC_UPROBES, & 1577 set_bit(MMF_RECALC_UPROBES, &newmm->flags);
1626 } 1578 }
1627 } 1579 }
1628 1580
1629 /* 1581 /*
1630 * - search for a free slot. 1582 * - search for a free slot.
1631 */ 1583 */
1632 static unsigned long xol_take_insn_slot(struc 1584 static unsigned long xol_take_insn_slot(struct xol_area *area)
1633 { 1585 {
1634 unsigned long slot_addr; 1586 unsigned long slot_addr;
1635 int slot_nr; 1587 int slot_nr;
1636 1588
1637 do { 1589 do {
1638 slot_nr = find_first_zero_bit 1590 slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1639 if (slot_nr < UINSNS_PER_PAGE 1591 if (slot_nr < UINSNS_PER_PAGE) {
1640 if (!test_and_set_bit 1592 if (!test_and_set_bit(slot_nr, area->bitmap))
1641 break; 1593 break;
1642 1594
1643 slot_nr = UINSNS_PER_ 1595 slot_nr = UINSNS_PER_PAGE;
1644 continue; 1596 continue;
1645 } 1597 }
1646 wait_event(area->wq, (atomic_ 1598 wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1647 } while (slot_nr >= UINSNS_PER_PAGE); 1599 } while (slot_nr >= UINSNS_PER_PAGE);
1648 1600
1649 slot_addr = area->vaddr + (slot_nr * 1601 slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1650 atomic_inc(&area->slot_count); 1602 atomic_inc(&area->slot_count);
1651 1603
1652 return slot_addr; 1604 return slot_addr;
1653 } 1605 }
1654 1606
1655 /* 1607 /*
1656 * xol_get_insn_slot - allocate a slot for xo 1608 * xol_get_insn_slot - allocate a slot for xol.
1657 * Returns the allocated slot address or 0. 1609 * Returns the allocated slot address or 0.
1658 */ 1610 */
1659 static unsigned long xol_get_insn_slot(struct 1611 static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
1660 { 1612 {
1661 struct xol_area *area; 1613 struct xol_area *area;
1662 unsigned long xol_vaddr; 1614 unsigned long xol_vaddr;
1663 1615
1664 area = get_xol_area(); 1616 area = get_xol_area();
1665 if (!area) 1617 if (!area)
1666 return 0; 1618 return 0;
1667 1619
1668 xol_vaddr = xol_take_insn_slot(area); 1620 xol_vaddr = xol_take_insn_slot(area);
1669 if (unlikely(!xol_vaddr)) 1621 if (unlikely(!xol_vaddr))
1670 return 0; 1622 return 0;
1671 1623
1672 arch_uprobe_copy_ixol(area->page, xol !! 1624 arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
1673 &uprobe->arch.i 1625 &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1674 1626
1675 return xol_vaddr; 1627 return xol_vaddr;
1676 } 1628 }
1677 1629
1678 /* 1630 /*
1679 * xol_free_insn_slot - If slot was earlier a 1631 * xol_free_insn_slot - If slot was earlier allocated by
1680 * @xol_get_insn_slot(), make the slot availa 1632 * @xol_get_insn_slot(), make the slot available for
1681 * subsequent requests. 1633 * subsequent requests.
1682 */ 1634 */
1683 static void xol_free_insn_slot(struct task_st 1635 static void xol_free_insn_slot(struct task_struct *tsk)
1684 { 1636 {
1685 struct xol_area *area; 1637 struct xol_area *area;
1686 unsigned long vma_end; 1638 unsigned long vma_end;
1687 unsigned long slot_addr; 1639 unsigned long slot_addr;
1688 1640
1689 if (!tsk->mm || !tsk->mm->uprobes_sta 1641 if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
1690 return; 1642 return;
1691 1643
1692 slot_addr = tsk->utask->xol_vaddr; 1644 slot_addr = tsk->utask->xol_vaddr;
1693 if (unlikely(!slot_addr)) 1645 if (unlikely(!slot_addr))
1694 return; 1646 return;
1695 1647
1696 area = tsk->mm->uprobes_state.xol_are 1648 area = tsk->mm->uprobes_state.xol_area;
1697 vma_end = area->vaddr + PAGE_SIZE; 1649 vma_end = area->vaddr + PAGE_SIZE;
1698 if (area->vaddr <= slot_addr && slot_ 1650 if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1699 unsigned long offset; 1651 unsigned long offset;
1700 int slot_nr; 1652 int slot_nr;
1701 1653
1702 offset = slot_addr - area->va 1654 offset = slot_addr - area->vaddr;
1703 slot_nr = offset / UPROBE_XOL 1655 slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1704 if (slot_nr >= UINSNS_PER_PAG 1656 if (slot_nr >= UINSNS_PER_PAGE)
1705 return; 1657 return;
1706 1658
1707 clear_bit(slot_nr, area->bitm 1659 clear_bit(slot_nr, area->bitmap);
1708 atomic_dec(&area->slot_count) 1660 atomic_dec(&area->slot_count);
1709 smp_mb__after_atomic(); /* pa 1661 smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1710 if (waitqueue_active(&area->w 1662 if (waitqueue_active(&area->wq))
1711 wake_up(&area->wq); 1663 wake_up(&area->wq);
1712 1664
1713 tsk->utask->xol_vaddr = 0; 1665 tsk->utask->xol_vaddr = 0;
1714 } 1666 }
1715 } 1667 }
1716 1668
1717 void __weak arch_uprobe_copy_ixol(struct page 1669 void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
1718 void *src, 1670 void *src, unsigned long len)
1719 { 1671 {
1720 /* Initialize the slot */ 1672 /* Initialize the slot */
1721 copy_to_page(page, vaddr, src, len); 1673 copy_to_page(page, vaddr, src, len);
1722 1674
1723 /* 1675 /*
1724 * We probably need flush_icache_user 1676 * We probably need flush_icache_user_page() but it needs vma.
1725 * This should work on most of archit 1677 * This should work on most of architectures by default. If
1726 * architecture needs to do something 1678 * architecture needs to do something different it can define
1727 * its own version of the function. 1679 * its own version of the function.
1728 */ 1680 */
1729 flush_dcache_page(page); 1681 flush_dcache_page(page);
1730 } 1682 }
1731 1683
1732 /** 1684 /**
1733 * uprobe_get_swbp_addr - compute address of 1685 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1734 * @regs: Reflects the saved state of the tas 1686 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1735 * instruction. 1687 * instruction.
1736 * Return the address of the breakpoint instr 1688 * Return the address of the breakpoint instruction.
1737 */ 1689 */
1738 unsigned long __weak uprobe_get_swbp_addr(str 1690 unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1739 { 1691 {
1740 return instruction_pointer(regs) - UP 1692 return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1741 } 1693 }
1742 1694
1743 unsigned long uprobe_get_trap_addr(struct pt_ 1695 unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
1744 { 1696 {
1745 struct uprobe_task *utask = current-> 1697 struct uprobe_task *utask = current->utask;
1746 1698
1747 if (unlikely(utask && utask->active_u 1699 if (unlikely(utask && utask->active_uprobe))
1748 return utask->vaddr; 1700 return utask->vaddr;
1749 1701
1750 return instruction_pointer(regs); 1702 return instruction_pointer(regs);
1751 } 1703 }
1752 1704
1753 static struct return_instance *free_ret_insta 1705 static struct return_instance *free_ret_instance(struct return_instance *ri)
1754 { 1706 {
1755 struct return_instance *next = ri->ne 1707 struct return_instance *next = ri->next;
1756 put_uprobe(ri->uprobe); 1708 put_uprobe(ri->uprobe);
1757 kfree(ri); 1709 kfree(ri);
1758 return next; 1710 return next;
1759 } 1711 }
1760 1712
1761 /* 1713 /*
1762 * Called with no locks held. 1714 * Called with no locks held.
1763 * Called in context of an exiting or an exec 1715 * Called in context of an exiting or an exec-ing thread.
1764 */ 1716 */
1765 void uprobe_free_utask(struct task_struct *t) 1717 void uprobe_free_utask(struct task_struct *t)
1766 { 1718 {
1767 struct uprobe_task *utask = t->utask; 1719 struct uprobe_task *utask = t->utask;
1768 struct return_instance *ri; 1720 struct return_instance *ri;
1769 1721
1770 if (!utask) 1722 if (!utask)
1771 return; 1723 return;
1772 1724
1773 if (utask->active_uprobe) 1725 if (utask->active_uprobe)
1774 put_uprobe(utask->active_upro 1726 put_uprobe(utask->active_uprobe);
1775 1727
1776 ri = utask->return_instances; 1728 ri = utask->return_instances;
1777 while (ri) 1729 while (ri)
1778 ri = free_ret_instance(ri); 1730 ri = free_ret_instance(ri);
1779 1731
1780 xol_free_insn_slot(t); 1732 xol_free_insn_slot(t);
1781 kfree(utask); 1733 kfree(utask);
1782 t->utask = NULL; 1734 t->utask = NULL;
1783 } 1735 }
1784 1736
1785 /* 1737 /*
1786 * Allocate a uprobe_task object for the task 1738 * Allocate a uprobe_task object for the task if necessary.
1787 * Called when the thread hits a breakpoint. 1739 * Called when the thread hits a breakpoint.
1788 * 1740 *
1789 * Returns: 1741 * Returns:
1790 * - pointer to new uprobe_task on success 1742 * - pointer to new uprobe_task on success
1791 * - NULL otherwise 1743 * - NULL otherwise
1792 */ 1744 */
1793 static struct uprobe_task *get_utask(void) 1745 static struct uprobe_task *get_utask(void)
1794 { 1746 {
1795 if (!current->utask) 1747 if (!current->utask)
1796 current->utask = kzalloc(size 1748 current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1797 return current->utask; 1749 return current->utask;
1798 } 1750 }
1799 1751
1800 static int dup_utask(struct task_struct *t, s 1752 static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
1801 { 1753 {
1802 struct uprobe_task *n_utask; 1754 struct uprobe_task *n_utask;
1803 struct return_instance **p, *o, *n; 1755 struct return_instance **p, *o, *n;
1804 1756
1805 n_utask = kzalloc(sizeof(struct uprob 1757 n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1806 if (!n_utask) 1758 if (!n_utask)
1807 return -ENOMEM; 1759 return -ENOMEM;
1808 t->utask = n_utask; 1760 t->utask = n_utask;
1809 1761
1810 p = &n_utask->return_instances; 1762 p = &n_utask->return_instances;
1811 for (o = o_utask->return_instances; o 1763 for (o = o_utask->return_instances; o; o = o->next) {
1812 n = kmalloc(sizeof(struct ret 1764 n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1813 if (!n) 1765 if (!n)
1814 return -ENOMEM; 1766 return -ENOMEM;
1815 1767
1816 *n = *o; 1768 *n = *o;
1817 /* <<
1818 * uprobe's refcnt has to be <<
1819 * utask->return_instances it <<
1820 * removed right now, as task <<
1821 * get_uprobe() is safe to us <<
1822 */ <<
1823 get_uprobe(n->uprobe); 1769 get_uprobe(n->uprobe);
1824 n->next = NULL; 1770 n->next = NULL;
1825 1771
1826 *p = n; 1772 *p = n;
1827 p = &n->next; 1773 p = &n->next;
1828 n_utask->depth++; 1774 n_utask->depth++;
1829 } 1775 }
1830 1776
1831 return 0; 1777 return 0;
1832 } 1778 }
1833 1779
>> 1780 static void uprobe_warn(struct task_struct *t, const char *msg)
>> 1781 {
>> 1782 pr_warn("uprobe: %s:%d failed to %s\n",
>> 1783 current->comm, current->pid, msg);
>> 1784 }
>> 1785
1834 static void dup_xol_work(struct callback_head 1786 static void dup_xol_work(struct callback_head *work)
1835 { 1787 {
1836 if (current->flags & PF_EXITING) 1788 if (current->flags & PF_EXITING)
1837 return; 1789 return;
1838 1790
1839 if (!__create_xol_area(current->utask 1791 if (!__create_xol_area(current->utask->dup_xol_addr) &&
1840 !fatal_signal_pending 1792 !fatal_signal_pending(current))
1841 uprobe_warn(current, "dup xol 1793 uprobe_warn(current, "dup xol area");
1842 } 1794 }
1843 1795
1844 /* 1796 /*
1845 * Called in context of a new clone/fork from 1797 * Called in context of a new clone/fork from copy_process.
1846 */ 1798 */
1847 void uprobe_copy_process(struct task_struct * 1799 void uprobe_copy_process(struct task_struct *t, unsigned long flags)
1848 { 1800 {
1849 struct uprobe_task *utask = current-> 1801 struct uprobe_task *utask = current->utask;
1850 struct mm_struct *mm = current->mm; 1802 struct mm_struct *mm = current->mm;
1851 struct xol_area *area; 1803 struct xol_area *area;
1852 1804
1853 t->utask = NULL; 1805 t->utask = NULL;
1854 1806
1855 if (!utask || !utask->return_instance 1807 if (!utask || !utask->return_instances)
1856 return; 1808 return;
1857 1809
1858 if (mm == t->mm && !(flags & CLONE_VF 1810 if (mm == t->mm && !(flags & CLONE_VFORK))
1859 return; 1811 return;
1860 1812
1861 if (dup_utask(t, utask)) 1813 if (dup_utask(t, utask))
1862 return uprobe_warn(t, "dup re 1814 return uprobe_warn(t, "dup ret instances");
1863 1815
1864 /* The task can fork() after dup_xol_ 1816 /* The task can fork() after dup_xol_work() fails */
1865 area = mm->uprobes_state.xol_area; 1817 area = mm->uprobes_state.xol_area;
1866 if (!area) 1818 if (!area)
1867 return uprobe_warn(t, "dup xo 1819 return uprobe_warn(t, "dup xol area");
1868 1820
1869 if (mm == t->mm) 1821 if (mm == t->mm)
1870 return; 1822 return;
1871 1823
1872 t->utask->dup_xol_addr = area->vaddr; 1824 t->utask->dup_xol_addr = area->vaddr;
1873 init_task_work(&t->utask->dup_xol_wor 1825 init_task_work(&t->utask->dup_xol_work, dup_xol_work);
1874 task_work_add(t, &t->utask->dup_xol_w 1826 task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME);
1875 } 1827 }
1876 1828
1877 /* 1829 /*
1878 * Current area->vaddr notion assume the tram 1830 * Current area->vaddr notion assume the trampoline address is always
1879 * equal area->vaddr. 1831 * equal area->vaddr.
1880 * 1832 *
1881 * Returns -1 in case the xol_area is not all 1833 * Returns -1 in case the xol_area is not allocated.
1882 */ 1834 */
1883 unsigned long uprobe_get_trampoline_vaddr(voi !! 1835 static unsigned long get_trampoline_vaddr(void)
1884 { 1836 {
1885 struct xol_area *area; 1837 struct xol_area *area;
1886 unsigned long trampoline_vaddr = -1; 1838 unsigned long trampoline_vaddr = -1;
1887 1839
1888 /* Pairs with xol_add_vma() smp_store 1840 /* Pairs with xol_add_vma() smp_store_release() */
1889 area = READ_ONCE(current->mm->uprobes 1841 area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
1890 if (area) 1842 if (area)
1891 trampoline_vaddr = area->vadd 1843 trampoline_vaddr = area->vaddr;
1892 1844
1893 return trampoline_vaddr; 1845 return trampoline_vaddr;
1894 } 1846 }
1895 1847
1896 static void cleanup_return_instances(struct u 1848 static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
1897 struc 1849 struct pt_regs *regs)
1898 { 1850 {
1899 struct return_instance *ri = utask->r 1851 struct return_instance *ri = utask->return_instances;
1900 enum rp_check ctx = chained ? RP_CHEC 1852 enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
1901 1853
1902 while (ri && !arch_uretprobe_is_alive 1854 while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
1903 ri = free_ret_instance(ri); 1855 ri = free_ret_instance(ri);
1904 utask->depth--; 1856 utask->depth--;
1905 } 1857 }
1906 utask->return_instances = ri; 1858 utask->return_instances = ri;
1907 } 1859 }
1908 1860
1909 static void prepare_uretprobe(struct uprobe * 1861 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
1910 { 1862 {
1911 struct return_instance *ri; 1863 struct return_instance *ri;
1912 struct uprobe_task *utask; 1864 struct uprobe_task *utask;
1913 unsigned long orig_ret_vaddr, trampol 1865 unsigned long orig_ret_vaddr, trampoline_vaddr;
1914 bool chained; 1866 bool chained;
1915 1867
1916 if (!get_xol_area()) 1868 if (!get_xol_area())
1917 return; 1869 return;
1918 1870
1919 utask = get_utask(); 1871 utask = get_utask();
1920 if (!utask) 1872 if (!utask)
1921 return; 1873 return;
1922 1874
1923 if (utask->depth >= MAX_URETPROBE_DEP 1875 if (utask->depth >= MAX_URETPROBE_DEPTH) {
1924 printk_ratelimited(KERN_INFO 1876 printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
1925 " nestedness 1877 " nestedness limit pid/tgid=%d/%d\n",
1926 current->pid, 1878 current->pid, current->tgid);
1927 return; 1879 return;
1928 } 1880 }
1929 1881
1930 /* we need to bump refcount to store <<
1931 if (!try_get_uprobe(uprobe)) <<
1932 return; <<
1933 <<
1934 ri = kmalloc(sizeof(struct return_ins 1882 ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1935 if (!ri) 1883 if (!ri)
1936 goto fail; !! 1884 return;
1937 1885
1938 trampoline_vaddr = uprobe_get_trampol !! 1886 trampoline_vaddr = get_trampoline_vaddr();
1939 orig_ret_vaddr = arch_uretprobe_hijac 1887 orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1940 if (orig_ret_vaddr == -1) 1888 if (orig_ret_vaddr == -1)
1941 goto fail; 1889 goto fail;
1942 1890
1943 /* drop the entries invalidated by lo 1891 /* drop the entries invalidated by longjmp() */
1944 chained = (orig_ret_vaddr == trampoli 1892 chained = (orig_ret_vaddr == trampoline_vaddr);
1945 cleanup_return_instances(utask, chain 1893 cleanup_return_instances(utask, chained, regs);
1946 1894
1947 /* 1895 /*
1948 * We don't want to keep trampoline a 1896 * We don't want to keep trampoline address in stack, rather keep the
1949 * original return address of first c 1897 * original return address of first caller thru all the consequent
1950 * instances. This also makes breakpo 1898 * instances. This also makes breakpoint unwrapping easier.
1951 */ 1899 */
1952 if (chained) { 1900 if (chained) {
1953 if (!utask->return_instances) 1901 if (!utask->return_instances) {
1954 /* 1902 /*
1955 * This situation is 1903 * This situation is not possible. Likely we have an
1956 * attack from user-s 1904 * attack from user-space.
1957 */ 1905 */
1958 uprobe_warn(current, 1906 uprobe_warn(current, "handle tail call");
1959 goto fail; 1907 goto fail;
1960 } 1908 }
1961 orig_ret_vaddr = utask->retur 1909 orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
1962 } 1910 }
1963 ri->uprobe = uprobe; !! 1911
>> 1912 ri->uprobe = get_uprobe(uprobe);
1964 ri->func = instruction_pointer(regs); 1913 ri->func = instruction_pointer(regs);
1965 ri->stack = user_stack_pointer(regs); 1914 ri->stack = user_stack_pointer(regs);
1966 ri->orig_ret_vaddr = orig_ret_vaddr; 1915 ri->orig_ret_vaddr = orig_ret_vaddr;
1967 ri->chained = chained; 1916 ri->chained = chained;
1968 1917
1969 utask->depth++; 1918 utask->depth++;
1970 ri->next = utask->return_instances; 1919 ri->next = utask->return_instances;
1971 utask->return_instances = ri; 1920 utask->return_instances = ri;
1972 1921
1973 return; 1922 return;
1974 fail: !! 1923 fail:
1975 kfree(ri); 1924 kfree(ri);
1976 put_uprobe(uprobe); <<
1977 } 1925 }
1978 1926
1979 /* Prepare to single-step probed instruction 1927 /* Prepare to single-step probed instruction out of line. */
1980 static int 1928 static int
1981 pre_ssout(struct uprobe *uprobe, struct pt_re 1929 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
1982 { 1930 {
1983 struct uprobe_task *utask; 1931 struct uprobe_task *utask;
1984 unsigned long xol_vaddr; 1932 unsigned long xol_vaddr;
1985 int err; 1933 int err;
1986 1934
1987 utask = get_utask(); 1935 utask = get_utask();
1988 if (!utask) 1936 if (!utask)
1989 return -ENOMEM; 1937 return -ENOMEM;
1990 1938
1991 if (!try_get_uprobe(uprobe)) <<
1992 return -EINVAL; <<
1993 <<
1994 xol_vaddr = xol_get_insn_slot(uprobe) 1939 xol_vaddr = xol_get_insn_slot(uprobe);
1995 if (!xol_vaddr) { !! 1940 if (!xol_vaddr)
1996 err = -ENOMEM; !! 1941 return -ENOMEM;
1997 goto err_out; <<
1998 } <<
1999 1942
2000 utask->xol_vaddr = xol_vaddr; 1943 utask->xol_vaddr = xol_vaddr;
2001 utask->vaddr = bp_vaddr; 1944 utask->vaddr = bp_vaddr;
2002 1945
2003 err = arch_uprobe_pre_xol(&uprobe->ar 1946 err = arch_uprobe_pre_xol(&uprobe->arch, regs);
2004 if (unlikely(err)) { 1947 if (unlikely(err)) {
2005 xol_free_insn_slot(current); 1948 xol_free_insn_slot(current);
2006 goto err_out; !! 1949 return err;
2007 } 1950 }
2008 1951
2009 utask->active_uprobe = uprobe; 1952 utask->active_uprobe = uprobe;
2010 utask->state = UTASK_SSTEP; 1953 utask->state = UTASK_SSTEP;
2011 return 0; 1954 return 0;
2012 err_out: <<
2013 put_uprobe(uprobe); <<
2014 return err; <<
2015 } 1955 }
2016 1956
2017 /* 1957 /*
2018 * If we are singlestepping, then ensure this 1958 * If we are singlestepping, then ensure this thread is not connected to
2019 * non-fatal signals until completion of sing 1959 * non-fatal signals until completion of singlestep. When xol insn itself
2020 * triggers the signal, restart the original 1960 * triggers the signal, restart the original insn even if the task is
2021 * already SIGKILL'ed (since coredump should 1961 * already SIGKILL'ed (since coredump should report the correct ip). This
2022 * is even more important if the task has a h 1962 * is even more important if the task has a handler for SIGSEGV/etc, The
2023 * _same_ instruction should be repeated agai 1963 * _same_ instruction should be repeated again after return from the signal
2024 * handler, and SSTEP can never finish in thi 1964 * handler, and SSTEP can never finish in this case.
2025 */ 1965 */
2026 bool uprobe_deny_signal(void) 1966 bool uprobe_deny_signal(void)
2027 { 1967 {
2028 struct task_struct *t = current; 1968 struct task_struct *t = current;
2029 struct uprobe_task *utask = t->utask; 1969 struct uprobe_task *utask = t->utask;
2030 1970
2031 if (likely(!utask || !utask->active_u 1971 if (likely(!utask || !utask->active_uprobe))
2032 return false; 1972 return false;
2033 1973
2034 WARN_ON_ONCE(utask->state != UTASK_SS 1974 WARN_ON_ONCE(utask->state != UTASK_SSTEP);
2035 1975
2036 if (task_sigpending(t)) { 1976 if (task_sigpending(t)) {
2037 spin_lock_irq(&t->sighand->si 1977 spin_lock_irq(&t->sighand->siglock);
2038 clear_tsk_thread_flag(t, TIF_ 1978 clear_tsk_thread_flag(t, TIF_SIGPENDING);
2039 spin_unlock_irq(&t->sighand-> 1979 spin_unlock_irq(&t->sighand->siglock);
2040 1980
2041 if (__fatal_signal_pending(t) 1981 if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
2042 utask->state = UTASK_ 1982 utask->state = UTASK_SSTEP_TRAPPED;
2043 set_tsk_thread_flag(t 1983 set_tsk_thread_flag(t, TIF_UPROBE);
2044 } 1984 }
2045 } 1985 }
2046 1986
2047 return true; 1987 return true;
2048 } 1988 }
2049 1989
2050 static void mmf_recalc_uprobes(struct mm_stru 1990 static void mmf_recalc_uprobes(struct mm_struct *mm)
2051 { 1991 {
2052 VMA_ITERATOR(vmi, mm, 0); <<
2053 struct vm_area_struct *vma; 1992 struct vm_area_struct *vma;
2054 1993
2055 for_each_vma(vmi, vma) { !! 1994 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2056 if (!valid_vma(vma, false)) 1995 if (!valid_vma(vma, false))
2057 continue; 1996 continue;
2058 /* 1997 /*
2059 * This is not strictly accur 1998 * This is not strictly accurate, we can race with
2060 * uprobe_unregister() and se 1999 * uprobe_unregister() and see the already removed
2061 * uprobe if delete_uprobe() 2000 * uprobe if delete_uprobe() was not yet called.
2062 * Or this uprobe can be filt 2001 * Or this uprobe can be filtered out.
2063 */ 2002 */
2064 if (vma_has_uprobes(vma, vma- 2003 if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
2065 return; 2004 return;
2066 } 2005 }
2067 2006
2068 clear_bit(MMF_HAS_UPROBES, &mm->flags 2007 clear_bit(MMF_HAS_UPROBES, &mm->flags);
2069 } 2008 }
2070 2009
2071 static int is_trap_at_addr(struct mm_struct * 2010 static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
2072 { 2011 {
2073 struct page *page; 2012 struct page *page;
2074 uprobe_opcode_t opcode; 2013 uprobe_opcode_t opcode;
2075 int result; 2014 int result;
2076 2015
2077 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, U 2016 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE)))
2078 return -EINVAL; 2017 return -EINVAL;
2079 2018
2080 pagefault_disable(); 2019 pagefault_disable();
2081 result = __get_user(opcode, (uprobe_o 2020 result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
2082 pagefault_enable(); 2021 pagefault_enable();
2083 2022
2084 if (likely(result == 0)) 2023 if (likely(result == 0))
2085 goto out; 2024 goto out;
2086 2025
2087 result = get_user_pages(vaddr, 1, FOL !! 2026 /*
>> 2027 * The NULL 'tsk' here ensures that any faults that occur here
>> 2028 * will not be accounted to the task. 'mm' *is* current->mm,
>> 2029 * but we treat this as a 'remote' access since it is
>> 2030 * essentially a kernel access to the memory.
>> 2031 */
>> 2032 result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page,
>> 2033 NULL, NULL);
2088 if (result < 0) 2034 if (result < 0)
2089 return result; 2035 return result;
2090 2036
2091 copy_from_page(page, vaddr, &opcode, 2037 copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
2092 put_page(page); 2038 put_page(page);
2093 out: 2039 out:
2094 /* This needs to return true for any 2040 /* This needs to return true for any variant of the trap insn */
2095 return is_trap_insn(&opcode); 2041 return is_trap_insn(&opcode);
2096 } 2042 }
2097 2043
2098 /* assumes being inside RCU protected region !! 2044 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
2099 static struct uprobe *find_active_uprobe_rcu( <<
2100 { 2045 {
2101 struct mm_struct *mm = current->mm; 2046 struct mm_struct *mm = current->mm;
2102 struct uprobe *uprobe = NULL; 2047 struct uprobe *uprobe = NULL;
2103 struct vm_area_struct *vma; 2048 struct vm_area_struct *vma;
2104 2049
2105 mmap_read_lock(mm); 2050 mmap_read_lock(mm);
2106 vma = vma_lookup(mm, bp_vaddr); 2051 vma = vma_lookup(mm, bp_vaddr);
2107 if (vma) { 2052 if (vma) {
2108 if (valid_vma(vma, false)) { 2053 if (valid_vma(vma, false)) {
2109 struct inode *inode = 2054 struct inode *inode = file_inode(vma->vm_file);
2110 loff_t offset = vaddr 2055 loff_t offset = vaddr_to_offset(vma, bp_vaddr);
2111 2056
2112 uprobe = find_uprobe_ !! 2057 uprobe = find_uprobe(inode, offset);
2113 } 2058 }
2114 2059
2115 if (!uprobe) 2060 if (!uprobe)
2116 *is_swbp = is_trap_at 2061 *is_swbp = is_trap_at_addr(mm, bp_vaddr);
2117 } else { 2062 } else {
2118 *is_swbp = -EFAULT; 2063 *is_swbp = -EFAULT;
2119 } 2064 }
2120 2065
2121 if (!uprobe && test_and_clear_bit(MMF 2066 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
2122 mmf_recalc_uprobes(mm); 2067 mmf_recalc_uprobes(mm);
2123 mmap_read_unlock(mm); 2068 mmap_read_unlock(mm);
2124 2069
2125 return uprobe; 2070 return uprobe;
2126 } 2071 }
2127 2072
2128 static void handler_chain(struct uprobe *upro 2073 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
2129 { 2074 {
2130 struct uprobe_consumer *uc; 2075 struct uprobe_consumer *uc;
2131 int remove = UPROBE_HANDLER_REMOVE; 2076 int remove = UPROBE_HANDLER_REMOVE;
2132 bool need_prep = false; /* prepare re 2077 bool need_prep = false; /* prepare return uprobe, when needed */
2133 bool has_consumers = false; <<
2134 <<
2135 current->utask->auprobe = &uprobe->ar <<
2136 2078
2137 list_for_each_entry_srcu(uc, &uprobe- !! 2079 down_read(&uprobe->register_rwsem);
2138 srcu_read_lo !! 2080 for (uc = uprobe->consumers; uc; uc = uc->next) {
2139 int rc = 0; 2081 int rc = 0;
2140 2082
2141 if (uc->handler) { 2083 if (uc->handler) {
2142 rc = uc->handler(uc, 2084 rc = uc->handler(uc, regs);
2143 WARN(rc & ~UPROBE_HAN 2085 WARN(rc & ~UPROBE_HANDLER_MASK,
2144 "bad rc=0x%x 2086 "bad rc=0x%x from %ps()\n", rc, uc->handler);
2145 } 2087 }
2146 2088
2147 if (uc->ret_handler) 2089 if (uc->ret_handler)
2148 need_prep = true; 2090 need_prep = true;
2149 2091
2150 remove &= rc; 2092 remove &= rc;
2151 has_consumers = true; <<
2152 } 2093 }
2153 current->utask->auprobe = NULL; <<
2154 2094
2155 if (need_prep && !remove) 2095 if (need_prep && !remove)
2156 prepare_uretprobe(uprobe, reg 2096 prepare_uretprobe(uprobe, regs); /* put bp at return */
2157 2097
2158 if (remove && has_consumers) { !! 2098 if (remove && uprobe->consumers) {
2159 down_read(&uprobe->register_r !! 2099 WARN_ON(!uprobe_is_active(uprobe));
2160 !! 2100 unapply_uprobe(uprobe, current->mm);
2161 /* re-check that removal is s <<
2162 if (!filter_chain(uprobe, cur <<
2163 WARN_ON(!uprobe_is_ac <<
2164 unapply_uprobe(uprobe <<
2165 } <<
2166 <<
2167 up_read(&uprobe->register_rws <<
2168 } 2101 }
>> 2102 up_read(&uprobe->register_rwsem);
2169 } 2103 }
2170 2104
2171 static void 2105 static void
2172 handle_uretprobe_chain(struct return_instance 2106 handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
2173 { 2107 {
2174 struct uprobe *uprobe = ri->uprobe; 2108 struct uprobe *uprobe = ri->uprobe;
2175 struct uprobe_consumer *uc; 2109 struct uprobe_consumer *uc;
2176 int srcu_idx; <<
2177 2110
2178 srcu_idx = srcu_read_lock(&uprobes_sr !! 2111 down_read(&uprobe->register_rwsem);
2179 list_for_each_entry_srcu(uc, &uprobe- !! 2112 for (uc = uprobe->consumers; uc; uc = uc->next) {
2180 srcu_read_lo <<
2181 if (uc->ret_handler) 2113 if (uc->ret_handler)
2182 uc->ret_handler(uc, r 2114 uc->ret_handler(uc, ri->func, regs);
2183 } 2115 }
2184 srcu_read_unlock(&uprobes_srcu, srcu_ !! 2116 up_read(&uprobe->register_rwsem);
2185 } 2117 }
2186 2118
2187 static struct return_instance *find_next_ret_ 2119 static struct return_instance *find_next_ret_chain(struct return_instance *ri)
2188 { 2120 {
2189 bool chained; 2121 bool chained;
2190 2122
2191 do { 2123 do {
2192 chained = ri->chained; 2124 chained = ri->chained;
2193 ri = ri->next; /* can't be N 2125 ri = ri->next; /* can't be NULL if chained */
2194 } while (chained); 2126 } while (chained);
2195 2127
2196 return ri; 2128 return ri;
2197 } 2129 }
2198 2130
2199 void uprobe_handle_trampoline(struct pt_regs !! 2131 static void handle_trampoline(struct pt_regs *regs)
2200 { 2132 {
2201 struct uprobe_task *utask; 2133 struct uprobe_task *utask;
2202 struct return_instance *ri, *next; 2134 struct return_instance *ri, *next;
2203 bool valid; 2135 bool valid;
2204 2136
2205 utask = current->utask; 2137 utask = current->utask;
2206 if (!utask) 2138 if (!utask)
2207 goto sigill; 2139 goto sigill;
2208 2140
2209 ri = utask->return_instances; 2141 ri = utask->return_instances;
2210 if (!ri) 2142 if (!ri)
2211 goto sigill; 2143 goto sigill;
2212 2144
2213 do { 2145 do {
2214 /* 2146 /*
2215 * We should throw out the fr 2147 * We should throw out the frames invalidated by longjmp().
2216 * If this chain is valid, th 2148 * If this chain is valid, then the next one should be alive
2217 * or NULL; the latter case m 2149 * or NULL; the latter case means that nobody but ri->func
2218 * could hit this trampoline 2150 * could hit this trampoline on return. TODO: sigaltstack().
2219 */ 2151 */
2220 next = find_next_ret_chain(ri 2152 next = find_next_ret_chain(ri);
2221 valid = !next || arch_uretpro 2153 valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
2222 2154
2223 instruction_pointer_set(regs, 2155 instruction_pointer_set(regs, ri->orig_ret_vaddr);
2224 do { 2156 do {
2225 /* pop current instan <<
2226 * as it's not pendin <<
2227 * instruction pointe <<
2228 * this allows fixup_ <<
2229 * captured stack tra <<
2230 * trampoline address <<
2231 * original return ad <<
2232 */ <<
2233 utask->return_instanc <<
2234 if (valid) 2157 if (valid)
2235 handle_uretpr 2158 handle_uretprobe_chain(ri, regs);
2236 ri = free_ret_instanc 2159 ri = free_ret_instance(ri);
2237 utask->depth--; 2160 utask->depth--;
2238 } while (ri != next); 2161 } while (ri != next);
2239 } while (!valid); 2162 } while (!valid);
2240 2163
2241 utask->return_instances = ri; 2164 utask->return_instances = ri;
2242 return; 2165 return;
2243 2166
2244 sigill: 2167 sigill:
2245 uprobe_warn(current, "handle uretprob 2168 uprobe_warn(current, "handle uretprobe, sending SIGILL.");
2246 force_sig(SIGILL); 2169 force_sig(SIGILL);
2247 2170
2248 } 2171 }
2249 2172
2250 bool __weak arch_uprobe_ignore(struct arch_up 2173 bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
2251 { 2174 {
2252 return false; 2175 return false;
2253 } 2176 }
2254 2177
2255 bool __weak arch_uretprobe_is_alive(struct re 2178 bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
2256 struc 2179 struct pt_regs *regs)
2257 { 2180 {
2258 return true; 2181 return true;
2259 } 2182 }
2260 2183
2261 /* 2184 /*
2262 * Run handler and ask thread to singlestep. 2185 * Run handler and ask thread to singlestep.
2263 * Ensure all non-fatal signals cannot interr 2186 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
2264 */ 2187 */
2265 static void handle_swbp(struct pt_regs *regs) 2188 static void handle_swbp(struct pt_regs *regs)
2266 { 2189 {
2267 struct uprobe *uprobe; 2190 struct uprobe *uprobe;
2268 unsigned long bp_vaddr; 2191 unsigned long bp_vaddr;
2269 int is_swbp, srcu_idx; !! 2192 int is_swbp;
2270 2193
2271 bp_vaddr = uprobe_get_swbp_addr(regs) 2194 bp_vaddr = uprobe_get_swbp_addr(regs);
2272 if (bp_vaddr == uprobe_get_trampoline !! 2195 if (bp_vaddr == get_trampoline_vaddr())
2273 return uprobe_handle_trampoli !! 2196 return handle_trampoline(regs);
2274 <<
2275 srcu_idx = srcu_read_lock(&uprobes_sr <<
2276 2197
2277 uprobe = find_active_uprobe_rcu(bp_va !! 2198 uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
2278 if (!uprobe) { 2199 if (!uprobe) {
2279 if (is_swbp > 0) { 2200 if (is_swbp > 0) {
2280 /* No matching uprobe 2201 /* No matching uprobe; signal SIGTRAP. */
2281 force_sig(SIGTRAP); 2202 force_sig(SIGTRAP);
2282 } else { 2203 } else {
2283 /* 2204 /*
2284 * Either we raced wi 2205 * Either we raced with uprobe_unregister() or we can't
2285 * access this memory 2206 * access this memory. The latter is only possible if
2286 * another thread pla 2207 * another thread plays with our ->mm. In both cases
2287 * we can simply rest 2208 * we can simply restart. If this vma was unmapped we
2288 * can pretend this i 2209 * can pretend this insn was not executed yet and get
2289 * the (correct) SIGS 2210 * the (correct) SIGSEGV after restart.
2290 */ 2211 */
2291 instruction_pointer_s 2212 instruction_pointer_set(regs, bp_vaddr);
2292 } 2213 }
2293 goto out; !! 2214 return;
2294 } 2215 }
2295 2216
2296 /* change it in advance for ->handler 2217 /* change it in advance for ->handler() and restart */
2297 instruction_pointer_set(regs, bp_vadd 2218 instruction_pointer_set(regs, bp_vaddr);
2298 2219
2299 /* 2220 /*
2300 * TODO: move copy_insn/etc into _reg 2221 * TODO: move copy_insn/etc into _register and remove this hack.
2301 * After we hit the bp, _unregister + 2222 * After we hit the bp, _unregister + _register can install the
2302 * new and not-yet-analyzed uprobe at 2223 * new and not-yet-analyzed uprobe at the same address, restart.
2303 */ 2224 */
2304 if (unlikely(!test_bit(UPROBE_COPY_IN 2225 if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
2305 goto out; 2226 goto out;
2306 2227
2307 /* 2228 /*
2308 * Pairs with the smp_wmb() in prepar 2229 * Pairs with the smp_wmb() in prepare_uprobe().
2309 * 2230 *
2310 * Guarantees that if we see the UPRO 2231 * Guarantees that if we see the UPROBE_COPY_INSN bit set, then
2311 * we must also see the stores to &up 2232 * we must also see the stores to &uprobe->arch performed by the
2312 * prepare_uprobe() call. 2233 * prepare_uprobe() call.
2313 */ 2234 */
2314 smp_rmb(); 2235 smp_rmb();
2315 2236
2316 /* Tracing handlers use ->utask to co 2237 /* Tracing handlers use ->utask to communicate with fetch methods */
2317 if (!get_utask()) 2238 if (!get_utask())
2318 goto out; 2239 goto out;
2319 2240
2320 if (arch_uprobe_ignore(&uprobe->arch, 2241 if (arch_uprobe_ignore(&uprobe->arch, regs))
2321 goto out; 2242 goto out;
2322 2243
2323 handler_chain(uprobe, regs); 2244 handler_chain(uprobe, regs);
2324 2245
2325 if (arch_uprobe_skip_sstep(&uprobe->a 2246 if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
2326 goto out; 2247 goto out;
2327 2248
2328 if (pre_ssout(uprobe, regs, bp_vaddr) !! 2249 if (!pre_ssout(uprobe, regs, bp_vaddr))
2329 goto out; !! 2250 return;
2330 2251
2331 out: <<
2332 /* arch_uprobe_skip_sstep() succeeded 2252 /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
2333 srcu_read_unlock(&uprobes_srcu, srcu_ !! 2253 out:
>> 2254 put_uprobe(uprobe);
2334 } 2255 }
2335 2256
2336 /* 2257 /*
2337 * Perform required fix-ups and disable singl 2258 * Perform required fix-ups and disable singlestep.
2338 * Allow pending signals to take effect. 2259 * Allow pending signals to take effect.
2339 */ 2260 */
2340 static void handle_singlestep(struct uprobe_t 2261 static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
2341 { 2262 {
2342 struct uprobe *uprobe; 2263 struct uprobe *uprobe;
2343 int err = 0; 2264 int err = 0;
2344 2265
2345 uprobe = utask->active_uprobe; 2266 uprobe = utask->active_uprobe;
2346 if (utask->state == UTASK_SSTEP_ACK) 2267 if (utask->state == UTASK_SSTEP_ACK)
2347 err = arch_uprobe_post_xol(&u 2268 err = arch_uprobe_post_xol(&uprobe->arch, regs);
2348 else if (utask->state == UTASK_SSTEP_ 2269 else if (utask->state == UTASK_SSTEP_TRAPPED)
2349 arch_uprobe_abort_xol(&uprobe 2270 arch_uprobe_abort_xol(&uprobe->arch, regs);
2350 else 2271 else
2351 WARN_ON_ONCE(1); 2272 WARN_ON_ONCE(1);
2352 2273
2353 put_uprobe(uprobe); 2274 put_uprobe(uprobe);
2354 utask->active_uprobe = NULL; 2275 utask->active_uprobe = NULL;
2355 utask->state = UTASK_RUNNING; 2276 utask->state = UTASK_RUNNING;
2356 xol_free_insn_slot(current); 2277 xol_free_insn_slot(current);
2357 2278
2358 spin_lock_irq(¤t->sighand->sigl 2279 spin_lock_irq(¤t->sighand->siglock);
2359 recalc_sigpending(); /* see uprobe_de 2280 recalc_sigpending(); /* see uprobe_deny_signal() */
2360 spin_unlock_irq(¤t->sighand->si 2281 spin_unlock_irq(¤t->sighand->siglock);
2361 2282
2362 if (unlikely(err)) { 2283 if (unlikely(err)) {
2363 uprobe_warn(current, "execute 2284 uprobe_warn(current, "execute the probed insn, sending SIGILL.");
2364 force_sig(SIGILL); 2285 force_sig(SIGILL);
2365 } 2286 }
2366 } 2287 }
2367 2288
2368 /* 2289 /*
2369 * On breakpoint hit, breakpoint notifier set 2290 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
2370 * allows the thread to return from interrupt 2291 * allows the thread to return from interrupt. After that handle_swbp()
2371 * sets utask->active_uprobe. 2292 * sets utask->active_uprobe.
2372 * 2293 *
2373 * On singlestep exception, singlestep notifi 2294 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
2374 * and allows the thread to return from inter 2295 * and allows the thread to return from interrupt.
2375 * 2296 *
2376 * While returning to userspace, thread notic 2297 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
2377 * uprobe_notify_resume(). 2298 * uprobe_notify_resume().
2378 */ 2299 */
2379 void uprobe_notify_resume(struct pt_regs *reg 2300 void uprobe_notify_resume(struct pt_regs *regs)
2380 { 2301 {
2381 struct uprobe_task *utask; 2302 struct uprobe_task *utask;
2382 2303
2383 clear_thread_flag(TIF_UPROBE); 2304 clear_thread_flag(TIF_UPROBE);
2384 2305
2385 utask = current->utask; 2306 utask = current->utask;
2386 if (utask && utask->active_uprobe) 2307 if (utask && utask->active_uprobe)
2387 handle_singlestep(utask, regs 2308 handle_singlestep(utask, regs);
2388 else 2309 else
2389 handle_swbp(regs); 2310 handle_swbp(regs);
2390 } 2311 }
2391 2312
2392 /* 2313 /*
2393 * uprobe_pre_sstep_notifier gets called from 2314 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
2394 * notifier mechanism. Set TIF_UPROBE flag an 2315 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
2395 */ 2316 */
2396 int uprobe_pre_sstep_notifier(struct pt_regs 2317 int uprobe_pre_sstep_notifier(struct pt_regs *regs)
2397 { 2318 {
2398 if (!current->mm) 2319 if (!current->mm)
2399 return 0; 2320 return 0;
2400 2321
2401 if (!test_bit(MMF_HAS_UPROBES, &curre 2322 if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) &&
2402 (!current->utask || !current->uta 2323 (!current->utask || !current->utask->return_instances))
2403 return 0; 2324 return 0;
2404 2325
2405 set_thread_flag(TIF_UPROBE); 2326 set_thread_flag(TIF_UPROBE);
2406 return 1; 2327 return 1;
2407 } 2328 }
2408 2329
2409 /* 2330 /*
2410 * uprobe_post_sstep_notifier gets called in 2331 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
2411 * mechanism. Set TIF_UPROBE flag and indicat 2332 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
2412 */ 2333 */
2413 int uprobe_post_sstep_notifier(struct pt_regs 2334 int uprobe_post_sstep_notifier(struct pt_regs *regs)
2414 { 2335 {
2415 struct uprobe_task *utask = current-> 2336 struct uprobe_task *utask = current->utask;
2416 2337
2417 if (!current->mm || !utask || !utask- 2338 if (!current->mm || !utask || !utask->active_uprobe)
2418 /* task is currently not upro 2339 /* task is currently not uprobed */
2419 return 0; 2340 return 0;
2420 2341
2421 utask->state = UTASK_SSTEP_ACK; 2342 utask->state = UTASK_SSTEP_ACK;
2422 set_thread_flag(TIF_UPROBE); 2343 set_thread_flag(TIF_UPROBE);
2423 return 1; 2344 return 1;
2424 } 2345 }
2425 2346
2426 static struct notifier_block uprobe_exception 2347 static struct notifier_block uprobe_exception_nb = {
2427 .notifier_call = arch_uprobe 2348 .notifier_call = arch_uprobe_exception_notify,
2428 .priority = INT_MAX-1, 2349 .priority = INT_MAX-1, /* notified after kprobes, kgdb */
2429 }; 2350 };
2430 2351
2431 void __init uprobes_init(void) 2352 void __init uprobes_init(void)
2432 { 2353 {
2433 int i; 2354 int i;
2434 2355
2435 for (i = 0; i < UPROBES_HASH_SZ; i++) 2356 for (i = 0; i < UPROBES_HASH_SZ; i++)
2436 mutex_init(&uprobes_mmap_mute 2357 mutex_init(&uprobes_mmap_mutex[i]);
2437 2358
2438 BUG_ON(register_die_notifier(&uprobe_ 2359 BUG_ON(register_die_notifier(&uprobe_exception_nb));
2439 } 2360 }
2440 2361
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.