1 // SPDX-License-Identifier: GPL-2.0+ 1 // SPDX-License-Identifier: GPL-2.0+
2 /* 2 /*
3 * User-space Probes (UProbes) 3 * User-space Probes (UProbes)
4 * 4 *
5 * Copyright (C) IBM Corporation, 2008-2012 5 * Copyright (C) IBM Corporation, 2008-2012
6 * Authors: 6 * Authors:
7 * Srikar Dronamraju 7 * Srikar Dronamraju
8 * Jim Keniston 8 * Jim Keniston
9 * Copyright (C) 2011-2012 Red Hat, Inc., Pete 9 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
10 */ 10 */
11 11
12 #include <linux/kernel.h> 12 #include <linux/kernel.h>
13 #include <linux/highmem.h> 13 #include <linux/highmem.h>
14 #include <linux/pagemap.h> /* read_mappin 14 #include <linux/pagemap.h> /* read_mapping_page */
15 #include <linux/slab.h> 15 #include <linux/slab.h>
16 #include <linux/sched.h> 16 #include <linux/sched.h>
17 #include <linux/sched/mm.h> 17 #include <linux/sched/mm.h>
18 #include <linux/sched/coredump.h> 18 #include <linux/sched/coredump.h>
19 #include <linux/export.h> 19 #include <linux/export.h>
20 #include <linux/rmap.h> /* anon_vma_pr 20 #include <linux/rmap.h> /* anon_vma_prepare */
21 #include <linux/mmu_notifier.h> !! 21 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
22 #include <linux/swap.h> /* folio_free_ 22 #include <linux/swap.h> /* folio_free_swap */
23 #include <linux/ptrace.h> /* user_enable 23 #include <linux/ptrace.h> /* user_enable_single_step */
24 #include <linux/kdebug.h> /* notifier me 24 #include <linux/kdebug.h> /* notifier mechanism */
25 #include <linux/percpu-rwsem.h> 25 #include <linux/percpu-rwsem.h>
26 #include <linux/task_work.h> 26 #include <linux/task_work.h>
27 #include <linux/shmem_fs.h> 27 #include <linux/shmem_fs.h>
28 #include <linux/khugepaged.h> 28 #include <linux/khugepaged.h>
29 29
30 #include <linux/uprobes.h> 30 #include <linux/uprobes.h>
31 31
32 #define UINSNS_PER_PAGE (PAGE_ 32 #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
33 #define MAX_UPROBE_XOL_SLOTS UINSNS 33 #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
34 34
35 static struct rb_root uprobes_tree = RB_ROOT; 35 static struct rb_root uprobes_tree = RB_ROOT;
36 /* 36 /*
37 * allows us to skip the uprobe_mmap if there 37 * allows us to skip the uprobe_mmap if there are no uprobe events active
38 * at this time. Probably a fine grained per 38 * at this time. Probably a fine grained per inode count is better?
39 */ 39 */
40 #define no_uprobe_events() RB_EMPTY_ROOT( 40 #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree)
41 41
42 static DEFINE_RWLOCK(uprobes_treelock); /* ser !! 42 static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
43 static seqcount_rwlock_t uprobes_seqcount = SE <<
44 <<
45 DEFINE_STATIC_SRCU(uprobes_srcu); <<
46 43
47 #define UPROBES_HASH_SZ 13 44 #define UPROBES_HASH_SZ 13
48 /* serialize uprobe->pending_list */ 45 /* serialize uprobe->pending_list */
49 static struct mutex uprobes_mmap_mutex[UPROBES 46 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
50 #define uprobes_mmap_hash(v) (&uprobes_mmap 47 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
51 48
52 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); 49 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
53 50
54 /* Have a copy of original instruction */ 51 /* Have a copy of original instruction */
55 #define UPROBE_COPY_INSN 0 52 #define UPROBE_COPY_INSN 0
56 53
57 struct uprobe { 54 struct uprobe {
58 struct rb_node rb_node; 55 struct rb_node rb_node; /* node in the rb tree */
59 refcount_t ref; 56 refcount_t ref;
60 struct rw_semaphore register_rwsem 57 struct rw_semaphore register_rwsem;
61 struct rw_semaphore consumer_rwsem 58 struct rw_semaphore consumer_rwsem;
62 struct list_head pending_list; 59 struct list_head pending_list;
63 struct list_head consumers; !! 60 struct uprobe_consumer *consumers;
64 struct inode *inode; 61 struct inode *inode; /* Also hold a ref to inode */
65 struct rcu_head rcu; <<
66 loff_t offset; 62 loff_t offset;
67 loff_t ref_ctr_offset 63 loff_t ref_ctr_offset;
68 unsigned long flags; 64 unsigned long flags;
69 65
70 /* 66 /*
71 * The generic code assumes that it ha 67 * The generic code assumes that it has two members of unknown type
72 * owned by the arch-specific code: 68 * owned by the arch-specific code:
73 * 69 *
74 * insn - copy_insn() saves the 70 * insn - copy_insn() saves the original instruction here for
75 * arch_uprobe_analyze_in 71 * arch_uprobe_analyze_insn().
76 * 72 *
77 * ixol - potentially modified i 73 * ixol - potentially modified instruction to execute out of
78 * line, copied to xol_ar 74 * line, copied to xol_area by xol_get_insn_slot().
79 */ 75 */
80 struct arch_uprobe arch; 76 struct arch_uprobe arch;
81 }; 77 };
82 78
83 struct delayed_uprobe { 79 struct delayed_uprobe {
84 struct list_head list; 80 struct list_head list;
85 struct uprobe *uprobe; 81 struct uprobe *uprobe;
86 struct mm_struct *mm; 82 struct mm_struct *mm;
87 }; 83 };
88 84
89 static DEFINE_MUTEX(delayed_uprobe_lock); 85 static DEFINE_MUTEX(delayed_uprobe_lock);
90 static LIST_HEAD(delayed_uprobe_list); 86 static LIST_HEAD(delayed_uprobe_list);
91 87
92 /* 88 /*
93 * Execute out of line area: anonymous executa 89 * Execute out of line area: anonymous executable mapping installed
94 * by the probed task to execute the copy of t 90 * by the probed task to execute the copy of the original instruction
95 * mangled by set_swbp(). 91 * mangled by set_swbp().
96 * 92 *
97 * On a breakpoint hit, thread contests for a 93 * On a breakpoint hit, thread contests for a slot. It frees the
98 * slot after singlestep. Currently a fixed nu 94 * slot after singlestep. Currently a fixed number of slots are
99 * allocated. 95 * allocated.
100 */ 96 */
101 struct xol_area { 97 struct xol_area {
102 wait_queue_head_t wq; 98 wait_queue_head_t wq; /* if all slots are busy */
103 atomic_t slot_c 99 atomic_t slot_count; /* number of in-use slots */
104 unsigned long *bitma 100 unsigned long *bitmap; /* 0 = free slot */
105 101
106 struct page *page; !! 102 struct vm_special_mapping xol_mapping;
>> 103 struct page *pages[2];
107 /* 104 /*
108 * We keep the vma's vm_start rather t 105 * We keep the vma's vm_start rather than a pointer to the vma
109 * itself. The probed process or a na 106 * itself. The probed process or a naughty kernel module could make
110 * the vma go away, and we must handle 107 * the vma go away, and we must handle that reasonably gracefully.
111 */ 108 */
112 unsigned long vaddr; 109 unsigned long vaddr; /* Page(s) of instruction slots */
113 }; 110 };
114 111
115 static void uprobe_warn(struct task_struct *t, <<
116 { <<
117 pr_warn("uprobe: %s:%d failed to %s\n" <<
118 } <<
119 <<
120 /* 112 /*
121 * valid_vma: Verify if the specified vma is a 113 * valid_vma: Verify if the specified vma is an executable vma
122 * Relax restrictions while unregistering: vm_ 114 * Relax restrictions while unregistering: vm_flags might have
123 * changed after breakpoint was inserted. 115 * changed after breakpoint was inserted.
124 * - is_register: indicates if we are in 116 * - is_register: indicates if we are in register context.
125 * - Return 1 if the specified virtual ad 117 * - Return 1 if the specified virtual address is in an
126 * executable vma. 118 * executable vma.
127 */ 119 */
128 static bool valid_vma(struct vm_area_struct *v 120 static bool valid_vma(struct vm_area_struct *vma, bool is_register)
129 { 121 {
130 vm_flags_t flags = VM_HUGETLB | VM_MAY 122 vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
131 123
132 if (is_register) 124 if (is_register)
133 flags |= VM_WRITE; 125 flags |= VM_WRITE;
134 126
135 return vma->vm_file && (vma->vm_flags 127 return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
136 } 128 }
137 129
138 static unsigned long offset_to_vaddr(struct vm 130 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
139 { 131 {
140 return vma->vm_start + offset - ((loff 132 return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
141 } 133 }
142 134
143 static loff_t vaddr_to_offset(struct vm_area_s 135 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
144 { 136 {
145 return ((loff_t)vma->vm_pgoff << PAGE_ 137 return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
146 } 138 }
147 139
148 /** 140 /**
149 * __replace_page - replace page in vma by new 141 * __replace_page - replace page in vma by new page.
150 * based on replace_page in mm/ksm.c 142 * based on replace_page in mm/ksm.c
151 * 143 *
152 * @vma: vma that holds the pte pointing 144 * @vma: vma that holds the pte pointing to page
153 * @addr: address the old @page is mapped 145 * @addr: address the old @page is mapped at
154 * @old_page: the page we are replacing by new 146 * @old_page: the page we are replacing by new_page
155 * @new_page: the modified page we replace pag 147 * @new_page: the modified page we replace page by
156 * 148 *
157 * If @new_page is NULL, only unmap @old_page. 149 * If @new_page is NULL, only unmap @old_page.
158 * 150 *
159 * Returns 0 on success, negative error code o 151 * Returns 0 on success, negative error code otherwise.
160 */ 152 */
161 static int __replace_page(struct vm_area_struc 153 static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
162 struct page *o 154 struct page *old_page, struct page *new_page)
163 { 155 {
164 struct folio *old_folio = page_folio(o 156 struct folio *old_folio = page_folio(old_page);
165 struct folio *new_folio; 157 struct folio *new_folio;
166 struct mm_struct *mm = vma->vm_mm; 158 struct mm_struct *mm = vma->vm_mm;
167 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, 159 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0);
168 int err; 160 int err;
169 struct mmu_notifier_range range; 161 struct mmu_notifier_range range;
170 162
171 mmu_notifier_range_init(&range, MMU_NO 163 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, addr,
172 addr + PAGE_SI 164 addr + PAGE_SIZE);
173 165
174 if (new_page) { 166 if (new_page) {
175 new_folio = page_folio(new_pag 167 new_folio = page_folio(new_page);
176 err = mem_cgroup_charge(new_fo 168 err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL);
177 if (err) 169 if (err)
178 return err; 170 return err;
179 } 171 }
180 172
181 /* For folio_free_swap() below */ 173 /* For folio_free_swap() below */
182 folio_lock(old_folio); 174 folio_lock(old_folio);
183 175
184 mmu_notifier_invalidate_range_start(&r 176 mmu_notifier_invalidate_range_start(&range);
185 err = -EAGAIN; 177 err = -EAGAIN;
186 if (!page_vma_mapped_walk(&pvmw)) 178 if (!page_vma_mapped_walk(&pvmw))
187 goto unlock; 179 goto unlock;
188 VM_BUG_ON_PAGE(addr != pvmw.address, o 180 VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
189 181
190 if (new_page) { 182 if (new_page) {
191 folio_get(new_folio); 183 folio_get(new_folio);
192 folio_add_new_anon_rmap(new_fo !! 184 page_add_new_anon_rmap(new_page, vma, addr);
193 folio_add_lru_vma(new_folio, v 185 folio_add_lru_vma(new_folio, vma);
194 } else 186 } else
195 /* no new page, just dec_mm_co 187 /* no new page, just dec_mm_counter for old_page */
196 dec_mm_counter(mm, MM_ANONPAGE 188 dec_mm_counter(mm, MM_ANONPAGES);
197 189
198 if (!folio_test_anon(old_folio)) { 190 if (!folio_test_anon(old_folio)) {
199 dec_mm_counter(mm, mm_counter_ !! 191 dec_mm_counter(mm, mm_counter_file(old_page));
200 inc_mm_counter(mm, MM_ANONPAGE 192 inc_mm_counter(mm, MM_ANONPAGES);
201 } 193 }
202 194
203 flush_cache_page(vma, addr, pte_pfn(pt !! 195 flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
204 ptep_clear_flush(vma, addr, pvmw.pte); !! 196 ptep_clear_flush_notify(vma, addr, pvmw.pte);
205 if (new_page) 197 if (new_page)
206 set_pte_at(mm, addr, pvmw.pte, !! 198 set_pte_at_notify(mm, addr, pvmw.pte,
207 mk_pte(new_page, vm !! 199 mk_pte(new_page, vma->vm_page_prot));
208 200
209 folio_remove_rmap_pte(old_folio, old_p !! 201 page_remove_rmap(old_page, vma, false);
210 if (!folio_mapped(old_folio)) 202 if (!folio_mapped(old_folio))
211 folio_free_swap(old_folio); 203 folio_free_swap(old_folio);
212 page_vma_mapped_walk_done(&pvmw); 204 page_vma_mapped_walk_done(&pvmw);
213 folio_put(old_folio); 205 folio_put(old_folio);
214 206
215 err = 0; 207 err = 0;
216 unlock: 208 unlock:
217 mmu_notifier_invalidate_range_end(&ran 209 mmu_notifier_invalidate_range_end(&range);
218 folio_unlock(old_folio); 210 folio_unlock(old_folio);
219 return err; 211 return err;
220 } 212 }
221 213
222 /** 214 /**
223 * is_swbp_insn - check if instruction is brea 215 * is_swbp_insn - check if instruction is breakpoint instruction.
224 * @insn: instruction to be checked. 216 * @insn: instruction to be checked.
225 * Default implementation of is_swbp_insn 217 * Default implementation of is_swbp_insn
226 * Returns true if @insn is a breakpoint instr 218 * Returns true if @insn is a breakpoint instruction.
227 */ 219 */
228 bool __weak is_swbp_insn(uprobe_opcode_t *insn 220 bool __weak is_swbp_insn(uprobe_opcode_t *insn)
229 { 221 {
230 return *insn == UPROBE_SWBP_INSN; 222 return *insn == UPROBE_SWBP_INSN;
231 } 223 }
232 224
233 /** 225 /**
234 * is_trap_insn - check if instruction is brea 226 * is_trap_insn - check if instruction is breakpoint instruction.
235 * @insn: instruction to be checked. 227 * @insn: instruction to be checked.
236 * Default implementation of is_trap_insn 228 * Default implementation of is_trap_insn
237 * Returns true if @insn is a breakpoint instr 229 * Returns true if @insn is a breakpoint instruction.
238 * 230 *
239 * This function is needed for the case where 231 * This function is needed for the case where an architecture has multiple
240 * trap instructions (like powerpc). 232 * trap instructions (like powerpc).
241 */ 233 */
242 bool __weak is_trap_insn(uprobe_opcode_t *insn 234 bool __weak is_trap_insn(uprobe_opcode_t *insn)
243 { 235 {
244 return is_swbp_insn(insn); 236 return is_swbp_insn(insn);
245 } 237 }
246 238
247 static void copy_from_page(struct page *page, 239 static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
248 { 240 {
249 void *kaddr = kmap_atomic(page); 241 void *kaddr = kmap_atomic(page);
250 memcpy(dst, kaddr + (vaddr & ~PAGE_MAS 242 memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
251 kunmap_atomic(kaddr); 243 kunmap_atomic(kaddr);
252 } 244 }
253 245
254 static void copy_to_page(struct page *page, un 246 static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
255 { 247 {
256 void *kaddr = kmap_atomic(page); 248 void *kaddr = kmap_atomic(page);
257 memcpy(kaddr + (vaddr & ~PAGE_MASK), s 249 memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
258 kunmap_atomic(kaddr); 250 kunmap_atomic(kaddr);
259 } 251 }
260 252
261 static int verify_opcode(struct page *page, un 253 static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
262 { 254 {
263 uprobe_opcode_t old_opcode; 255 uprobe_opcode_t old_opcode;
264 bool is_swbp; 256 bool is_swbp;
265 257
266 /* 258 /*
267 * Note: We only check if the old_opco 259 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
268 * We do not check if it is any other 260 * We do not check if it is any other 'trap variant' which could
269 * be conditional trap instruction suc 261 * be conditional trap instruction such as the one powerpc supports.
270 * 262 *
271 * The logic is that we do not care if 263 * The logic is that we do not care if the underlying instruction
272 * is a trap variant; uprobes always w 264 * is a trap variant; uprobes always wins over any other (gdb)
273 * breakpoint. 265 * breakpoint.
274 */ 266 */
275 copy_from_page(page, vaddr, &old_opcod 267 copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
276 is_swbp = is_swbp_insn(&old_opcode); 268 is_swbp = is_swbp_insn(&old_opcode);
277 269
278 if (is_swbp_insn(new_opcode)) { 270 if (is_swbp_insn(new_opcode)) {
279 if (is_swbp) /* reg 271 if (is_swbp) /* register: already installed? */
280 return 0; 272 return 0;
281 } else { 273 } else {
282 if (!is_swbp) /* unr 274 if (!is_swbp) /* unregister: was it changed by us? */
283 return 0; 275 return 0;
284 } 276 }
285 277
286 return 1; 278 return 1;
287 } 279 }
288 280
289 static struct delayed_uprobe * 281 static struct delayed_uprobe *
290 delayed_uprobe_check(struct uprobe *uprobe, st 282 delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
291 { 283 {
292 struct delayed_uprobe *du; 284 struct delayed_uprobe *du;
293 285
294 list_for_each_entry(du, &delayed_uprob 286 list_for_each_entry(du, &delayed_uprobe_list, list)
295 if (du->uprobe == uprobe && du 287 if (du->uprobe == uprobe && du->mm == mm)
296 return du; 288 return du;
297 return NULL; 289 return NULL;
298 } 290 }
299 291
300 static int delayed_uprobe_add(struct uprobe *u 292 static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
301 { 293 {
302 struct delayed_uprobe *du; 294 struct delayed_uprobe *du;
303 295
304 if (delayed_uprobe_check(uprobe, mm)) 296 if (delayed_uprobe_check(uprobe, mm))
305 return 0; 297 return 0;
306 298
307 du = kzalloc(sizeof(*du), GFP_KERNEL) 299 du = kzalloc(sizeof(*du), GFP_KERNEL);
308 if (!du) 300 if (!du)
309 return -ENOMEM; 301 return -ENOMEM;
310 302
311 du->uprobe = uprobe; 303 du->uprobe = uprobe;
312 du->mm = mm; 304 du->mm = mm;
313 list_add(&du->list, &delayed_uprobe_li 305 list_add(&du->list, &delayed_uprobe_list);
314 return 0; 306 return 0;
315 } 307 }
316 308
317 static void delayed_uprobe_delete(struct delay 309 static void delayed_uprobe_delete(struct delayed_uprobe *du)
318 { 310 {
319 if (WARN_ON(!du)) 311 if (WARN_ON(!du))
320 return; 312 return;
321 list_del(&du->list); 313 list_del(&du->list);
322 kfree(du); 314 kfree(du);
323 } 315 }
324 316
325 static void delayed_uprobe_remove(struct uprob 317 static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
326 { 318 {
327 struct list_head *pos, *q; 319 struct list_head *pos, *q;
328 struct delayed_uprobe *du; 320 struct delayed_uprobe *du;
329 321
330 if (!uprobe && !mm) 322 if (!uprobe && !mm)
331 return; 323 return;
332 324
333 list_for_each_safe(pos, q, &delayed_up 325 list_for_each_safe(pos, q, &delayed_uprobe_list) {
334 du = list_entry(pos, struct de 326 du = list_entry(pos, struct delayed_uprobe, list);
335 327
336 if (uprobe && du->uprobe != up 328 if (uprobe && du->uprobe != uprobe)
337 continue; 329 continue;
338 if (mm && du->mm != mm) 330 if (mm && du->mm != mm)
339 continue; 331 continue;
340 332
341 delayed_uprobe_delete(du); 333 delayed_uprobe_delete(du);
342 } 334 }
343 } 335 }
344 336
345 static bool valid_ref_ctr_vma(struct uprobe *u 337 static bool valid_ref_ctr_vma(struct uprobe *uprobe,
346 struct vm_area_s 338 struct vm_area_struct *vma)
347 { 339 {
348 unsigned long vaddr = offset_to_vaddr( 340 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
349 341
350 return uprobe->ref_ctr_offset && 342 return uprobe->ref_ctr_offset &&
351 vma->vm_file && 343 vma->vm_file &&
352 file_inode(vma->vm_file) == up 344 file_inode(vma->vm_file) == uprobe->inode &&
353 (vma->vm_flags & (VM_WRITE|VM_ 345 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
354 vma->vm_start <= vaddr && 346 vma->vm_start <= vaddr &&
355 vma->vm_end > vaddr; 347 vma->vm_end > vaddr;
356 } 348 }
357 349
358 static struct vm_area_struct * 350 static struct vm_area_struct *
359 find_ref_ctr_vma(struct uprobe *uprobe, struct 351 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
360 { 352 {
361 VMA_ITERATOR(vmi, mm, 0); 353 VMA_ITERATOR(vmi, mm, 0);
362 struct vm_area_struct *tmp; 354 struct vm_area_struct *tmp;
363 355
364 for_each_vma(vmi, tmp) 356 for_each_vma(vmi, tmp)
365 if (valid_ref_ctr_vma(uprobe, 357 if (valid_ref_ctr_vma(uprobe, tmp))
366 return tmp; 358 return tmp;
367 359
368 return NULL; 360 return NULL;
369 } 361 }
370 362
371 static int 363 static int
372 __update_ref_ctr(struct mm_struct *mm, unsigne 364 __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
373 { 365 {
374 void *kaddr; 366 void *kaddr;
375 struct page *page; 367 struct page *page;
>> 368 struct vm_area_struct *vma;
376 int ret; 369 int ret;
377 short *ptr; 370 short *ptr;
378 371
379 if (!vaddr || !d) 372 if (!vaddr || !d)
380 return -EINVAL; 373 return -EINVAL;
381 374
382 ret = get_user_pages_remote(mm, vaddr, 375 ret = get_user_pages_remote(mm, vaddr, 1,
383 FOLL_WRITE !! 376 FOLL_WRITE, &page, &vma, NULL);
384 if (unlikely(ret <= 0)) { 377 if (unlikely(ret <= 0)) {
385 /* 378 /*
386 * We are asking for 1 page. I 379 * We are asking for 1 page. If get_user_pages_remote() fails,
387 * it may return 0, in that ca 380 * it may return 0, in that case we have to return error.
388 */ 381 */
389 return ret == 0 ? -EBUSY : ret 382 return ret == 0 ? -EBUSY : ret;
390 } 383 }
391 384
392 kaddr = kmap_atomic(page); 385 kaddr = kmap_atomic(page);
393 ptr = kaddr + (vaddr & ~PAGE_MASK); 386 ptr = kaddr + (vaddr & ~PAGE_MASK);
394 387
395 if (unlikely(*ptr + d < 0)) { 388 if (unlikely(*ptr + d < 0)) {
396 pr_warn("ref_ctr going negativ 389 pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
397 "curr val: %d, delta: 390 "curr val: %d, delta: %d\n", vaddr, *ptr, d);
398 ret = -EINVAL; 391 ret = -EINVAL;
399 goto out; 392 goto out;
400 } 393 }
401 394
402 *ptr += d; 395 *ptr += d;
403 ret = 0; 396 ret = 0;
404 out: 397 out:
405 kunmap_atomic(kaddr); 398 kunmap_atomic(kaddr);
406 put_page(page); 399 put_page(page);
407 return ret; 400 return ret;
408 } 401 }
409 402
410 static void update_ref_ctr_warn(struct uprobe 403 static void update_ref_ctr_warn(struct uprobe *uprobe,
411 struct mm_stru 404 struct mm_struct *mm, short d)
412 { 405 {
413 pr_warn("ref_ctr %s failed for inode: 406 pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
414 "0x%llx ref_ctr_offset: 0x%llx 407 "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
415 d > 0 ? "increment" : "decreme 408 d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
416 (unsigned long long) uprobe->o 409 (unsigned long long) uprobe->offset,
417 (unsigned long long) uprobe->r 410 (unsigned long long) uprobe->ref_ctr_offset, mm);
418 } 411 }
419 412
420 static int update_ref_ctr(struct uprobe *uprob 413 static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
421 short d) 414 short d)
422 { 415 {
423 struct vm_area_struct *rc_vma; 416 struct vm_area_struct *rc_vma;
424 unsigned long rc_vaddr; 417 unsigned long rc_vaddr;
425 int ret = 0; 418 int ret = 0;
426 419
427 rc_vma = find_ref_ctr_vma(uprobe, mm); 420 rc_vma = find_ref_ctr_vma(uprobe, mm);
428 421
429 if (rc_vma) { 422 if (rc_vma) {
430 rc_vaddr = offset_to_vaddr(rc_ 423 rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
431 ret = __update_ref_ctr(mm, rc_ 424 ret = __update_ref_ctr(mm, rc_vaddr, d);
432 if (ret) 425 if (ret)
433 update_ref_ctr_warn(up 426 update_ref_ctr_warn(uprobe, mm, d);
434 427
435 if (d > 0) 428 if (d > 0)
436 return ret; 429 return ret;
437 } 430 }
438 431
439 mutex_lock(&delayed_uprobe_lock); 432 mutex_lock(&delayed_uprobe_lock);
440 if (d > 0) 433 if (d > 0)
441 ret = delayed_uprobe_add(uprob 434 ret = delayed_uprobe_add(uprobe, mm);
442 else 435 else
443 delayed_uprobe_remove(uprobe, 436 delayed_uprobe_remove(uprobe, mm);
444 mutex_unlock(&delayed_uprobe_lock); 437 mutex_unlock(&delayed_uprobe_lock);
445 438
446 return ret; 439 return ret;
447 } 440 }
448 441
449 /* 442 /*
450 * NOTE: 443 * NOTE:
451 * Expect the breakpoint instruction to be the 444 * Expect the breakpoint instruction to be the smallest size instruction for
452 * the architecture. If an arch has variable l 445 * the architecture. If an arch has variable length instruction and the
453 * breakpoint instruction is not of the smalle 446 * breakpoint instruction is not of the smallest length instruction
454 * supported by that architecture then we need 447 * supported by that architecture then we need to modify is_trap_at_addr and
455 * uprobe_write_opcode accordingly. This would 448 * uprobe_write_opcode accordingly. This would never be a problem for archs
456 * that have fixed length instructions. 449 * that have fixed length instructions.
457 * 450 *
458 * uprobe_write_opcode - write the opcode at a 451 * uprobe_write_opcode - write the opcode at a given virtual address.
459 * @auprobe: arch specific probepoint informat 452 * @auprobe: arch specific probepoint information.
460 * @mm: the probed process address space. 453 * @mm: the probed process address space.
461 * @vaddr: the virtual address to store the op 454 * @vaddr: the virtual address to store the opcode.
462 * @opcode: opcode to be written at @vaddr. 455 * @opcode: opcode to be written at @vaddr.
463 * 456 *
464 * Called with mm->mmap_lock held for read or !! 457 * Called with mm->mmap_lock held for write.
465 * Return 0 (success) or a negative errno. 458 * Return 0 (success) or a negative errno.
466 */ 459 */
467 int uprobe_write_opcode(struct arch_uprobe *au 460 int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
468 unsigned long vaddr, u 461 unsigned long vaddr, uprobe_opcode_t opcode)
469 { 462 {
470 struct uprobe *uprobe; 463 struct uprobe *uprobe;
471 struct page *old_page, *new_page; 464 struct page *old_page, *new_page;
472 struct vm_area_struct *vma; 465 struct vm_area_struct *vma;
473 int ret, is_register, ref_ctr_updated 466 int ret, is_register, ref_ctr_updated = 0;
474 bool orig_page_huge = false; 467 bool orig_page_huge = false;
475 unsigned int gup_flags = FOLL_FORCE; 468 unsigned int gup_flags = FOLL_FORCE;
476 469
477 is_register = is_swbp_insn(&opcode); 470 is_register = is_swbp_insn(&opcode);
478 uprobe = container_of(auprobe, struct 471 uprobe = container_of(auprobe, struct uprobe, arch);
479 472
480 retry: 473 retry:
481 if (is_register) 474 if (is_register)
482 gup_flags |= FOLL_SPLIT_PMD; 475 gup_flags |= FOLL_SPLIT_PMD;
483 /* Read the page with vaddr into memor 476 /* Read the page with vaddr into memory */
484 old_page = get_user_page_vma_remote(mm !! 477 ret = get_user_pages_remote(mm, vaddr, 1, gup_flags,
485 if (IS_ERR(old_page)) !! 478 &old_page, &vma, NULL);
486 return PTR_ERR(old_page); !! 479 if (ret <= 0)
>> 480 return ret;
487 481
488 ret = verify_opcode(old_page, vaddr, & 482 ret = verify_opcode(old_page, vaddr, &opcode);
489 if (ret <= 0) 483 if (ret <= 0)
490 goto put_old; 484 goto put_old;
491 485
492 if (WARN(!is_register && PageCompound( 486 if (WARN(!is_register && PageCompound(old_page),
493 "uprobe unregister should nev 487 "uprobe unregister should never work on compound page\n")) {
494 ret = -EINVAL; 488 ret = -EINVAL;
495 goto put_old; 489 goto put_old;
496 } 490 }
497 491
498 /* We are going to replace instruction 492 /* We are going to replace instruction, update ref_ctr. */
499 if (!ref_ctr_updated && uprobe->ref_ct 493 if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
500 ret = update_ref_ctr(uprobe, m 494 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
501 if (ret) 495 if (ret)
502 goto put_old; 496 goto put_old;
503 497
504 ref_ctr_updated = 1; 498 ref_ctr_updated = 1;
505 } 499 }
506 500
507 ret = 0; 501 ret = 0;
508 if (!is_register && !PageAnon(old_page 502 if (!is_register && !PageAnon(old_page))
509 goto put_old; 503 goto put_old;
510 504
511 ret = anon_vma_prepare(vma); 505 ret = anon_vma_prepare(vma);
512 if (ret) 506 if (ret)
513 goto put_old; 507 goto put_old;
514 508
515 ret = -ENOMEM; 509 ret = -ENOMEM;
516 new_page = alloc_page_vma(GFP_HIGHUSER 510 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
517 if (!new_page) 511 if (!new_page)
518 goto put_old; 512 goto put_old;
519 513
520 __SetPageUptodate(new_page); 514 __SetPageUptodate(new_page);
521 copy_highpage(new_page, old_page); 515 copy_highpage(new_page, old_page);
522 copy_to_page(new_page, vaddr, &opcode, 516 copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
523 517
524 if (!is_register) { 518 if (!is_register) {
525 struct page *orig_page; 519 struct page *orig_page;
526 pgoff_t index; 520 pgoff_t index;
527 521
528 VM_BUG_ON_PAGE(!PageAnon(old_p 522 VM_BUG_ON_PAGE(!PageAnon(old_page), old_page);
529 523
530 index = vaddr_to_offset(vma, v 524 index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT;
531 orig_page = find_get_page(vma- 525 orig_page = find_get_page(vma->vm_file->f_inode->i_mapping,
532 inde 526 index);
533 527
534 if (orig_page) { 528 if (orig_page) {
535 if (PageUptodate(orig_ 529 if (PageUptodate(orig_page) &&
536 pages_identical(ne 530 pages_identical(new_page, orig_page)) {
537 /* let go new_ 531 /* let go new_page */
538 put_page(new_p 532 put_page(new_page);
539 new_page = NUL 533 new_page = NULL;
540 534
541 if (PageCompou 535 if (PageCompound(orig_page))
542 orig_p 536 orig_page_huge = true;
543 } 537 }
544 put_page(orig_page); 538 put_page(orig_page);
545 } 539 }
546 } 540 }
547 541
548 ret = __replace_page(vma, vaddr & PAGE !! 542 ret = __replace_page(vma, vaddr, old_page, new_page);
549 if (new_page) 543 if (new_page)
550 put_page(new_page); 544 put_page(new_page);
551 put_old: 545 put_old:
552 put_page(old_page); 546 put_page(old_page);
553 547
554 if (unlikely(ret == -EAGAIN)) 548 if (unlikely(ret == -EAGAIN))
555 goto retry; 549 goto retry;
556 550
557 /* Revert back reference counter if in 551 /* Revert back reference counter if instruction update failed. */
558 if (ret && is_register && ref_ctr_upda 552 if (ret && is_register && ref_ctr_updated)
559 update_ref_ctr(uprobe, mm, -1) 553 update_ref_ctr(uprobe, mm, -1);
560 554
561 /* try collapse pmd for compound page 555 /* try collapse pmd for compound page */
562 if (!ret && orig_page_huge) 556 if (!ret && orig_page_huge)
563 collapse_pte_mapped_thp(mm, va 557 collapse_pte_mapped_thp(mm, vaddr, false);
564 558
565 return ret; 559 return ret;
566 } 560 }
567 561
568 /** 562 /**
569 * set_swbp - store breakpoint at a given addr 563 * set_swbp - store breakpoint at a given address.
570 * @auprobe: arch specific probepoint informat 564 * @auprobe: arch specific probepoint information.
571 * @mm: the probed process address space. 565 * @mm: the probed process address space.
572 * @vaddr: the virtual address to insert the o 566 * @vaddr: the virtual address to insert the opcode.
573 * 567 *
574 * For mm @mm, store the breakpoint instructio 568 * For mm @mm, store the breakpoint instruction at @vaddr.
575 * Return 0 (success) or a negative errno. 569 * Return 0 (success) or a negative errno.
576 */ 570 */
577 int __weak set_swbp(struct arch_uprobe *auprob 571 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
578 { 572 {
579 return uprobe_write_opcode(auprobe, mm 573 return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);
580 } 574 }
581 575
582 /** 576 /**
583 * set_orig_insn - Restore the original instru 577 * set_orig_insn - Restore the original instruction.
584 * @mm: the probed process address space. 578 * @mm: the probed process address space.
585 * @auprobe: arch specific probepoint informat 579 * @auprobe: arch specific probepoint information.
586 * @vaddr: the virtual address to insert the o 580 * @vaddr: the virtual address to insert the opcode.
587 * 581 *
588 * For mm @mm, restore the original opcode (op 582 * For mm @mm, restore the original opcode (opcode) at @vaddr.
589 * Return 0 (success) or a negative errno. 583 * Return 0 (success) or a negative errno.
590 */ 584 */
591 int __weak 585 int __weak
592 set_orig_insn(struct arch_uprobe *auprobe, str 586 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
593 { 587 {
594 return uprobe_write_opcode(auprobe, mm 588 return uprobe_write_opcode(auprobe, mm, vaddr,
595 *(uprobe_opcode_t *)&a 589 *(uprobe_opcode_t *)&auprobe->insn);
596 } 590 }
597 591
598 /* uprobe should have guaranteed positive refc <<
599 static struct uprobe *get_uprobe(struct uprobe 592 static struct uprobe *get_uprobe(struct uprobe *uprobe)
600 { 593 {
601 refcount_inc(&uprobe->ref); 594 refcount_inc(&uprobe->ref);
602 return uprobe; 595 return uprobe;
603 } 596 }
604 597
605 /* <<
606 * uprobe should have guaranteed lifetime, whi <<
607 * - caller already has refcount taken (and <<
608 * - uprobe is RCU protected and won't be fr <<
609 * - we are holding uprobes_treelock (for re <<
610 */ <<
611 static struct uprobe *try_get_uprobe(struct up <<
612 { <<
613 if (refcount_inc_not_zero(&uprobe->ref <<
614 return uprobe; <<
615 return NULL; <<
616 } <<
617 <<
618 static inline bool uprobe_is_active(struct upr <<
619 { <<
620 return !RB_EMPTY_NODE(&uprobe->rb_node <<
621 } <<
622 <<
623 static void uprobe_free_rcu(struct rcu_head *r <<
624 { <<
625 struct uprobe *uprobe = container_of(r <<
626 <<
627 kfree(uprobe); <<
628 } <<
629 <<
630 static void put_uprobe(struct uprobe *uprobe) 598 static void put_uprobe(struct uprobe *uprobe)
631 { 599 {
632 if (!refcount_dec_and_test(&uprobe->re !! 600 if (refcount_dec_and_test(&uprobe->ref)) {
633 return; !! 601 /*
634 !! 602 * If application munmap(exec_vma) before uprobe_unregister()
635 write_lock(&uprobes_treelock); !! 603 * gets called, we don't get a chance to remove uprobe from
636 !! 604 * delayed_uprobe_list from remove_breakpoint(). Do it here.
637 if (uprobe_is_active(uprobe)) { !! 605 */
638 write_seqcount_begin(&uprobes_ !! 606 mutex_lock(&delayed_uprobe_lock);
639 rb_erase(&uprobe->rb_node, &up !! 607 delayed_uprobe_remove(uprobe, NULL);
640 write_seqcount_end(&uprobes_se !! 608 mutex_unlock(&delayed_uprobe_lock);
>> 609 kfree(uprobe);
641 } 610 }
642 <<
643 write_unlock(&uprobes_treelock); <<
644 <<
645 /* <<
646 * If application munmap(exec_vma) bef <<
647 * gets called, we don't get a chance <<
648 * delayed_uprobe_list from remove_bre <<
649 */ <<
650 mutex_lock(&delayed_uprobe_lock); <<
651 delayed_uprobe_remove(uprobe, NULL); <<
652 mutex_unlock(&delayed_uprobe_lock); <<
653 <<
654 call_srcu(&uprobes_srcu, &uprobe->rcu, <<
655 } 611 }
656 612
657 static __always_inline 613 static __always_inline
658 int uprobe_cmp(const struct inode *l_inode, co 614 int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
659 const struct uprobe *r) 615 const struct uprobe *r)
660 { 616 {
661 if (l_inode < r->inode) 617 if (l_inode < r->inode)
662 return -1; 618 return -1;
663 619
664 if (l_inode > r->inode) 620 if (l_inode > r->inode)
665 return 1; 621 return 1;
666 622
667 if (l_offset < r->offset) 623 if (l_offset < r->offset)
668 return -1; 624 return -1;
669 625
670 if (l_offset > r->offset) 626 if (l_offset > r->offset)
671 return 1; 627 return 1;
672 628
673 return 0; 629 return 0;
674 } 630 }
675 631
676 #define __node_2_uprobe(node) \ 632 #define __node_2_uprobe(node) \
677 rb_entry((node), struct uprobe, rb_nod 633 rb_entry((node), struct uprobe, rb_node)
678 634
679 struct __uprobe_key { 635 struct __uprobe_key {
680 struct inode *inode; 636 struct inode *inode;
681 loff_t offset; 637 loff_t offset;
682 }; 638 };
683 639
684 static inline int __uprobe_cmp_key(const void 640 static inline int __uprobe_cmp_key(const void *key, const struct rb_node *b)
685 { 641 {
686 const struct __uprobe_key *a = key; 642 const struct __uprobe_key *a = key;
687 return uprobe_cmp(a->inode, a->offset, 643 return uprobe_cmp(a->inode, a->offset, __node_2_uprobe(b));
688 } 644 }
689 645
690 static inline int __uprobe_cmp(struct rb_node 646 static inline int __uprobe_cmp(struct rb_node *a, const struct rb_node *b)
691 { 647 {
692 struct uprobe *u = __node_2_uprobe(a); 648 struct uprobe *u = __node_2_uprobe(a);
693 return uprobe_cmp(u->inode, u->offset, 649 return uprobe_cmp(u->inode, u->offset, __node_2_uprobe(b));
694 } 650 }
695 651
696 /* !! 652 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
697 * Assumes being inside RCU protected region. <<
698 * No refcount is taken on returned uprobe. <<
699 */ <<
700 static struct uprobe *find_uprobe_rcu(struct i <<
701 { 653 {
702 struct __uprobe_key key = { 654 struct __uprobe_key key = {
703 .inode = inode, 655 .inode = inode,
704 .offset = offset, 656 .offset = offset,
705 }; 657 };
706 struct rb_node *node; !! 658 struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key);
707 unsigned int seq; <<
708 659
709 lockdep_assert(srcu_read_lock_held(&up !! 660 if (node)
710 !! 661 return get_uprobe(__node_2_uprobe(node));
711 do { <<
712 seq = read_seqcount_begin(&upr <<
713 node = rb_find_rcu(&key, &upro <<
714 /* <<
715 * Lockless RB-tree lookups ca <<
716 * If the element is found, it <<
717 * under RCU protection. If we <<
718 * validate that seqcount didn <<
719 * try again as we might have <<
720 * negative). If seqcount is u <<
721 */ <<
722 if (node) <<
723 return __node_2_uprobe <<
724 } while (read_seqcount_retry(&uprobes_ <<
725 662
726 return NULL; 663 return NULL;
727 } 664 }
728 665
729 /* 666 /*
730 * Attempt to insert a new uprobe into uprobes !! 667 * Find a uprobe corresponding to a given inode:offset
731 * !! 668 * Acquires uprobes_treelock
732 * If uprobe already exists (for given inode+o <<
733 * refcount of previously existing uprobe. <<
734 * <<
735 * If not, a provided new instance of uprobe i <<
736 * assumed initial refcount == 1). <<
737 * <<
738 * In any case, we return a uprobe instance th <<
739 * Caller has to clean up new uprobe instance, <<
740 * inserted into the tree. <<
741 * <<
742 * We assume that uprobes_treelock is held for <<
743 */ 669 */
>> 670 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
>> 671 {
>> 672 struct uprobe *uprobe;
>> 673
>> 674 spin_lock(&uprobes_treelock);
>> 675 uprobe = __find_uprobe(inode, offset);
>> 676 spin_unlock(&uprobes_treelock);
>> 677
>> 678 return uprobe;
>> 679 }
>> 680
744 static struct uprobe *__insert_uprobe(struct u 681 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
745 { 682 {
746 struct rb_node *node; 683 struct rb_node *node;
747 again: <<
748 node = rb_find_add_rcu(&uprobe->rb_nod <<
749 if (node) { <<
750 struct uprobe *u = __node_2_up <<
751 <<
752 if (!try_get_uprobe(u)) { <<
753 rb_erase(node, &uprobe <<
754 RB_CLEAR_NODE(&u->rb_n <<
755 goto again; <<
756 } <<
757 684
758 return u; !! 685 node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
759 } !! 686 if (node)
>> 687 return get_uprobe(__node_2_uprobe(node));
760 688
761 return uprobe; !! 689 /* get access + creation ref */
>> 690 refcount_set(&uprobe->ref, 2);
>> 691 return NULL;
762 } 692 }
763 693
764 /* 694 /*
765 * Acquire uprobes_treelock and insert uprobe !! 695 * Acquire uprobes_treelock.
766 * (or reuse existing one, see __insert_uprobe !! 696 * Matching uprobe already exists in rbtree;
>> 697 * increment (access refcount) and return the matching uprobe.
>> 698 *
>> 699 * No matching uprobe; insert the uprobe in rb_tree;
>> 700 * get a double refcount (access + creation) and return NULL.
767 */ 701 */
768 static struct uprobe *insert_uprobe(struct upr 702 static struct uprobe *insert_uprobe(struct uprobe *uprobe)
769 { 703 {
770 struct uprobe *u; 704 struct uprobe *u;
771 705
772 write_lock(&uprobes_treelock); !! 706 spin_lock(&uprobes_treelock);
773 write_seqcount_begin(&uprobes_seqcount <<
774 u = __insert_uprobe(uprobe); 707 u = __insert_uprobe(uprobe);
775 write_seqcount_end(&uprobes_seqcount); !! 708 spin_unlock(&uprobes_treelock);
776 write_unlock(&uprobes_treelock); <<
777 709
778 return u; 710 return u;
779 } 711 }
780 712
781 static void 713 static void
782 ref_ctr_mismatch_warn(struct uprobe *cur_uprob 714 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
783 { 715 {
784 pr_warn("ref_ctr_offset mismatch. inod 716 pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
785 "ref_ctr_offset(old): 0x%llx r 717 "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
786 uprobe->inode->i_ino, (unsigne 718 uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
787 (unsigned long long) cur_uprob 719 (unsigned long long) cur_uprobe->ref_ctr_offset,
788 (unsigned long long) uprobe->r 720 (unsigned long long) uprobe->ref_ctr_offset);
789 } 721 }
790 722
791 static struct uprobe *alloc_uprobe(struct inod 723 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
792 loff_t ref_ 724 loff_t ref_ctr_offset)
793 { 725 {
794 struct uprobe *uprobe, *cur_uprobe; 726 struct uprobe *uprobe, *cur_uprobe;
795 727
796 uprobe = kzalloc(sizeof(struct uprobe) 728 uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
797 if (!uprobe) 729 if (!uprobe)
798 return ERR_PTR(-ENOMEM); !! 730 return NULL;
799 731
800 uprobe->inode = inode; 732 uprobe->inode = inode;
801 uprobe->offset = offset; 733 uprobe->offset = offset;
802 uprobe->ref_ctr_offset = ref_ctr_offse 734 uprobe->ref_ctr_offset = ref_ctr_offset;
803 INIT_LIST_HEAD(&uprobe->consumers); <<
804 init_rwsem(&uprobe->register_rwsem); 735 init_rwsem(&uprobe->register_rwsem);
805 init_rwsem(&uprobe->consumer_rwsem); 736 init_rwsem(&uprobe->consumer_rwsem);
806 RB_CLEAR_NODE(&uprobe->rb_node); <<
807 refcount_set(&uprobe->ref, 1); <<
808 737
809 /* add to uprobes_tree, sorted on inod 738 /* add to uprobes_tree, sorted on inode:offset */
810 cur_uprobe = insert_uprobe(uprobe); 739 cur_uprobe = insert_uprobe(uprobe);
811 /* a uprobe exists for this inode:offs 740 /* a uprobe exists for this inode:offset combination */
812 if (cur_uprobe != uprobe) { !! 741 if (cur_uprobe) {
813 if (cur_uprobe->ref_ctr_offset 742 if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
814 ref_ctr_mismatch_warn( 743 ref_ctr_mismatch_warn(cur_uprobe, uprobe);
815 put_uprobe(cur_uprobe) 744 put_uprobe(cur_uprobe);
816 kfree(uprobe); 745 kfree(uprobe);
817 return ERR_PTR(-EINVAL 746 return ERR_PTR(-EINVAL);
818 } 747 }
819 kfree(uprobe); 748 kfree(uprobe);
820 uprobe = cur_uprobe; 749 uprobe = cur_uprobe;
821 } 750 }
822 751
823 return uprobe; 752 return uprobe;
824 } 753 }
825 754
826 static void consumer_add(struct uprobe *uprobe 755 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
827 { 756 {
828 down_write(&uprobe->consumer_rwsem); 757 down_write(&uprobe->consumer_rwsem);
829 list_add_rcu(&uc->cons_node, &uprobe-> !! 758 uc->next = uprobe->consumers;
>> 759 uprobe->consumers = uc;
830 up_write(&uprobe->consumer_rwsem); 760 up_write(&uprobe->consumer_rwsem);
831 } 761 }
832 762
833 /* 763 /*
834 * For uprobe @uprobe, delete the consumer @uc 764 * For uprobe @uprobe, delete the consumer @uc.
835 * Should never be called with consumer that's !! 765 * Return true if the @uc is deleted successfully
>> 766 * or return false.
836 */ 767 */
837 static void consumer_del(struct uprobe *uprobe !! 768 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
838 { 769 {
>> 770 struct uprobe_consumer **con;
>> 771 bool ret = false;
>> 772
839 down_write(&uprobe->consumer_rwsem); 773 down_write(&uprobe->consumer_rwsem);
840 list_del_rcu(&uc->cons_node); !! 774 for (con = &uprobe->consumers; *con; con = &(*con)->next) {
>> 775 if (*con == uc) {
>> 776 *con = uc->next;
>> 777 ret = true;
>> 778 break;
>> 779 }
>> 780 }
841 up_write(&uprobe->consumer_rwsem); 781 up_write(&uprobe->consumer_rwsem);
>> 782
>> 783 return ret;
842 } 784 }
843 785
844 static int __copy_insn(struct address_space *m 786 static int __copy_insn(struct address_space *mapping, struct file *filp,
845 void *insn, int nbytes 787 void *insn, int nbytes, loff_t offset)
846 { 788 {
847 struct page *page; 789 struct page *page;
848 /* 790 /*
849 * Ensure that the page that has the o 791 * Ensure that the page that has the original instruction is populated
850 * and in page-cache. If ->read_folio 792 * and in page-cache. If ->read_folio == NULL it must be shmem_mapping(),
851 * see uprobe_register(). 793 * see uprobe_register().
852 */ 794 */
853 if (mapping->a_ops->read_folio) 795 if (mapping->a_ops->read_folio)
854 page = read_mapping_page(mappi 796 page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
855 else 797 else
856 page = shmem_read_mapping_page 798 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
857 if (IS_ERR(page)) 799 if (IS_ERR(page))
858 return PTR_ERR(page); 800 return PTR_ERR(page);
859 801
860 copy_from_page(page, offset, insn, nby 802 copy_from_page(page, offset, insn, nbytes);
861 put_page(page); 803 put_page(page);
862 804
863 return 0; 805 return 0;
864 } 806 }
865 807
866 static int copy_insn(struct uprobe *uprobe, st 808 static int copy_insn(struct uprobe *uprobe, struct file *filp)
867 { 809 {
868 struct address_space *mapping = uprobe 810 struct address_space *mapping = uprobe->inode->i_mapping;
869 loff_t offs = uprobe->offset; 811 loff_t offs = uprobe->offset;
870 void *insn = &uprobe->arch.insn; 812 void *insn = &uprobe->arch.insn;
871 int size = sizeof(uprobe->arch.insn); 813 int size = sizeof(uprobe->arch.insn);
872 int len, err = -EIO; 814 int len, err = -EIO;
873 815
874 /* Copy only available bytes, -EIO if 816 /* Copy only available bytes, -EIO if nothing was read */
875 do { 817 do {
876 if (offs >= i_size_read(uprobe 818 if (offs >= i_size_read(uprobe->inode))
877 break; 819 break;
878 820
879 len = min_t(int, size, PAGE_SI 821 len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
880 err = __copy_insn(mapping, fil 822 err = __copy_insn(mapping, filp, insn, len, offs);
881 if (err) 823 if (err)
882 break; 824 break;
883 825
884 insn += len; 826 insn += len;
885 offs += len; 827 offs += len;
886 size -= len; 828 size -= len;
887 } while (size); 829 } while (size);
888 830
889 return err; 831 return err;
890 } 832 }
891 833
892 static int prepare_uprobe(struct uprobe *uprob 834 static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
893 struct mm_stru 835 struct mm_struct *mm, unsigned long vaddr)
894 { 836 {
895 int ret = 0; 837 int ret = 0;
896 838
897 if (test_bit(UPROBE_COPY_INSN, &uprobe 839 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
898 return ret; 840 return ret;
899 841
900 /* TODO: move this into _register, unt 842 /* TODO: move this into _register, until then we abuse this sem. */
901 down_write(&uprobe->consumer_rwsem); 843 down_write(&uprobe->consumer_rwsem);
902 if (test_bit(UPROBE_COPY_INSN, &uprobe 844 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
903 goto out; 845 goto out;
904 846
905 ret = copy_insn(uprobe, file); 847 ret = copy_insn(uprobe, file);
906 if (ret) 848 if (ret)
907 goto out; 849 goto out;
908 850
909 ret = -ENOTSUPP; 851 ret = -ENOTSUPP;
910 if (is_trap_insn((uprobe_opcode_t *)&u 852 if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
911 goto out; 853 goto out;
912 854
913 ret = arch_uprobe_analyze_insn(&uprobe 855 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
914 if (ret) 856 if (ret)
915 goto out; 857 goto out;
916 858
917 smp_wmb(); /* pairs with the smp_rmb() 859 smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
918 set_bit(UPROBE_COPY_INSN, &uprobe->fla 860 set_bit(UPROBE_COPY_INSN, &uprobe->flags);
919 861
920 out: 862 out:
921 up_write(&uprobe->consumer_rwsem); 863 up_write(&uprobe->consumer_rwsem);
922 864
923 return ret; 865 return ret;
924 } 866 }
925 867
926 static inline bool consumer_filter(struct upro !! 868 static inline bool consumer_filter(struct uprobe_consumer *uc,
>> 869 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
927 { 870 {
928 return !uc->filter || uc->filter(uc, m !! 871 return !uc->filter || uc->filter(uc, ctx, mm);
929 } 872 }
930 873
931 static bool filter_chain(struct uprobe *uprobe !! 874 static bool filter_chain(struct uprobe *uprobe,
>> 875 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
932 { 876 {
933 struct uprobe_consumer *uc; 877 struct uprobe_consumer *uc;
934 bool ret = false; 878 bool ret = false;
935 879
936 down_read(&uprobe->consumer_rwsem); 880 down_read(&uprobe->consumer_rwsem);
937 list_for_each_entry_srcu(uc, &uprobe-> !! 881 for (uc = uprobe->consumers; uc; uc = uc->next) {
938 srcu_read_loc !! 882 ret = consumer_filter(uc, ctx, mm);
939 ret = consumer_filter(uc, mm); <<
940 if (ret) 883 if (ret)
941 break; 884 break;
942 } 885 }
943 up_read(&uprobe->consumer_rwsem); 886 up_read(&uprobe->consumer_rwsem);
944 887
945 return ret; 888 return ret;
946 } 889 }
947 890
948 static int 891 static int
949 install_breakpoint(struct uprobe *uprobe, stru 892 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
950 struct vm_area_struct 893 struct vm_area_struct *vma, unsigned long vaddr)
951 { 894 {
952 bool first_uprobe; 895 bool first_uprobe;
953 int ret; 896 int ret;
954 897
955 ret = prepare_uprobe(uprobe, vma->vm_f 898 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
956 if (ret) 899 if (ret)
957 return ret; 900 return ret;
958 901
959 /* 902 /*
960 * set MMF_HAS_UPROBES in advance for 903 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
961 * the task can hit this breakpoint ri 904 * the task can hit this breakpoint right after __replace_page().
962 */ 905 */
963 first_uprobe = !test_bit(MMF_HAS_UPROB 906 first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
964 if (first_uprobe) 907 if (first_uprobe)
965 set_bit(MMF_HAS_UPROBES, &mm-> 908 set_bit(MMF_HAS_UPROBES, &mm->flags);
966 909
967 ret = set_swbp(&uprobe->arch, mm, vadd 910 ret = set_swbp(&uprobe->arch, mm, vaddr);
968 if (!ret) 911 if (!ret)
969 clear_bit(MMF_RECALC_UPROBES, 912 clear_bit(MMF_RECALC_UPROBES, &mm->flags);
970 else if (first_uprobe) 913 else if (first_uprobe)
971 clear_bit(MMF_HAS_UPROBES, &mm 914 clear_bit(MMF_HAS_UPROBES, &mm->flags);
972 915
973 return ret; 916 return ret;
974 } 917 }
975 918
976 static int 919 static int
977 remove_breakpoint(struct uprobe *uprobe, struc 920 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
978 { 921 {
979 set_bit(MMF_RECALC_UPROBES, &mm->flags 922 set_bit(MMF_RECALC_UPROBES, &mm->flags);
980 return set_orig_insn(&uprobe->arch, mm 923 return set_orig_insn(&uprobe->arch, mm, vaddr);
981 } 924 }
982 925
>> 926 static inline bool uprobe_is_active(struct uprobe *uprobe)
>> 927 {
>> 928 return !RB_EMPTY_NODE(&uprobe->rb_node);
>> 929 }
>> 930 /*
>> 931 * There could be threads that have already hit the breakpoint. They
>> 932 * will recheck the current insn and restart if find_uprobe() fails.
>> 933 * See find_active_uprobe().
>> 934 */
>> 935 static void delete_uprobe(struct uprobe *uprobe)
>> 936 {
>> 937 if (WARN_ON(!uprobe_is_active(uprobe)))
>> 938 return;
>> 939
>> 940 spin_lock(&uprobes_treelock);
>> 941 rb_erase(&uprobe->rb_node, &uprobes_tree);
>> 942 spin_unlock(&uprobes_treelock);
>> 943 RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
>> 944 put_uprobe(uprobe);
>> 945 }
>> 946
983 struct map_info { 947 struct map_info {
984 struct map_info *next; 948 struct map_info *next;
985 struct mm_struct *mm; 949 struct mm_struct *mm;
986 unsigned long vaddr; 950 unsigned long vaddr;
987 }; 951 };
988 952
989 static inline struct map_info *free_map_info(s 953 static inline struct map_info *free_map_info(struct map_info *info)
990 { 954 {
991 struct map_info *next = info->next; 955 struct map_info *next = info->next;
992 kfree(info); 956 kfree(info);
993 return next; 957 return next;
994 } 958 }
995 959
996 static struct map_info * 960 static struct map_info *
997 build_map_info(struct address_space *mapping, 961 build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
998 { 962 {
999 unsigned long pgoff = offset >> PAGE_S 963 unsigned long pgoff = offset >> PAGE_SHIFT;
1000 struct vm_area_struct *vma; 964 struct vm_area_struct *vma;
1001 struct map_info *curr = NULL; 965 struct map_info *curr = NULL;
1002 struct map_info *prev = NULL; 966 struct map_info *prev = NULL;
1003 struct map_info *info; 967 struct map_info *info;
1004 int more = 0; 968 int more = 0;
1005 969
1006 again: 970 again:
1007 i_mmap_lock_read(mapping); 971 i_mmap_lock_read(mapping);
1008 vma_interval_tree_foreach(vma, &mappi 972 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1009 if (!valid_vma(vma, is_regist 973 if (!valid_vma(vma, is_register))
1010 continue; 974 continue;
1011 975
1012 if (!prev && !more) { 976 if (!prev && !more) {
1013 /* 977 /*
1014 * Needs GFP_NOWAIT t 978 * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
1015 * reclaim. This is o 979 * reclaim. This is optimistic, no harm done if it fails.
1016 */ 980 */
1017 prev = kmalloc(sizeof 981 prev = kmalloc(sizeof(struct map_info),
1018 GFP_N 982 GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
1019 if (prev) 983 if (prev)
1020 prev->next = 984 prev->next = NULL;
1021 } 985 }
1022 if (!prev) { 986 if (!prev) {
1023 more++; 987 more++;
1024 continue; 988 continue;
1025 } 989 }
1026 990
1027 if (!mmget_not_zero(vma->vm_m 991 if (!mmget_not_zero(vma->vm_mm))
1028 continue; 992 continue;
1029 993
1030 info = prev; 994 info = prev;
1031 prev = prev->next; 995 prev = prev->next;
1032 info->next = curr; 996 info->next = curr;
1033 curr = info; 997 curr = info;
1034 998
1035 info->mm = vma->vm_mm; 999 info->mm = vma->vm_mm;
1036 info->vaddr = offset_to_vaddr 1000 info->vaddr = offset_to_vaddr(vma, offset);
1037 } 1001 }
1038 i_mmap_unlock_read(mapping); 1002 i_mmap_unlock_read(mapping);
1039 1003
1040 if (!more) 1004 if (!more)
1041 goto out; 1005 goto out;
1042 1006
1043 prev = curr; 1007 prev = curr;
1044 while (curr) { 1008 while (curr) {
1045 mmput(curr->mm); 1009 mmput(curr->mm);
1046 curr = curr->next; 1010 curr = curr->next;
1047 } 1011 }
1048 1012
1049 do { 1013 do {
1050 info = kmalloc(sizeof(struct 1014 info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
1051 if (!info) { 1015 if (!info) {
1052 curr = ERR_PTR(-ENOME 1016 curr = ERR_PTR(-ENOMEM);
1053 goto out; 1017 goto out;
1054 } 1018 }
1055 info->next = prev; 1019 info->next = prev;
1056 prev = info; 1020 prev = info;
1057 } while (--more); 1021 } while (--more);
1058 1022
1059 goto again; 1023 goto again;
1060 out: 1024 out:
1061 while (prev) 1025 while (prev)
1062 prev = free_map_info(prev); 1026 prev = free_map_info(prev);
1063 return curr; 1027 return curr;
1064 } 1028 }
1065 1029
1066 static int 1030 static int
1067 register_for_each_vma(struct uprobe *uprobe, 1031 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
1068 { 1032 {
1069 bool is_register = !!new; 1033 bool is_register = !!new;
1070 struct map_info *info; 1034 struct map_info *info;
1071 int err = 0; 1035 int err = 0;
1072 1036
1073 percpu_down_write(&dup_mmap_sem); 1037 percpu_down_write(&dup_mmap_sem);
1074 info = build_map_info(uprobe->inode-> 1038 info = build_map_info(uprobe->inode->i_mapping,
1075 uprob 1039 uprobe->offset, is_register);
1076 if (IS_ERR(info)) { 1040 if (IS_ERR(info)) {
1077 err = PTR_ERR(info); 1041 err = PTR_ERR(info);
1078 goto out; 1042 goto out;
1079 } 1043 }
1080 1044
1081 while (info) { 1045 while (info) {
1082 struct mm_struct *mm = info-> 1046 struct mm_struct *mm = info->mm;
1083 struct vm_area_struct *vma; 1047 struct vm_area_struct *vma;
1084 1048
1085 if (err && is_register) 1049 if (err && is_register)
1086 goto free; 1050 goto free;
1087 /* !! 1051
1088 * We take mmap_lock for writ <<
1089 * find_active_uprobe_rcu() w <<
1090 * Thus this install_breakpoi <<
1091 * is_trap_at_addr() true rig <<
1092 * returns NULL in find_activ <<
1093 */ <<
1094 mmap_write_lock(mm); 1052 mmap_write_lock(mm);
1095 vma = find_vma(mm, info->vadd 1053 vma = find_vma(mm, info->vaddr);
1096 if (!vma || !valid_vma(vma, i 1054 if (!vma || !valid_vma(vma, is_register) ||
1097 file_inode(vma->vm_file) 1055 file_inode(vma->vm_file) != uprobe->inode)
1098 goto unlock; 1056 goto unlock;
1099 1057
1100 if (vma->vm_start > info->vad 1058 if (vma->vm_start > info->vaddr ||
1101 vaddr_to_offset(vma, info 1059 vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
1102 goto unlock; 1060 goto unlock;
1103 1061
1104 if (is_register) { 1062 if (is_register) {
1105 /* consult only the " 1063 /* consult only the "caller", new consumer. */
1106 if (consumer_filter(n !! 1064 if (consumer_filter(new,
>> 1065 UPROBE_FILTER_REGISTER, mm))
1107 err = install 1066 err = install_breakpoint(uprobe, mm, vma, info->vaddr);
1108 } else if (test_bit(MMF_HAS_U 1067 } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
1109 if (!filter_chain(upr !! 1068 if (!filter_chain(uprobe,
>> 1069 UPROBE_FILTER_UNREGISTER, mm))
1110 err |= remove 1070 err |= remove_breakpoint(uprobe, mm, info->vaddr);
1111 } 1071 }
1112 1072
1113 unlock: 1073 unlock:
1114 mmap_write_unlock(mm); 1074 mmap_write_unlock(mm);
1115 free: 1075 free:
1116 mmput(mm); 1076 mmput(mm);
1117 info = free_map_info(info); 1077 info = free_map_info(info);
1118 } 1078 }
1119 out: 1079 out:
1120 percpu_up_write(&dup_mmap_sem); 1080 percpu_up_write(&dup_mmap_sem);
1121 return err; 1081 return err;
1122 } 1082 }
1123 1083
1124 /** !! 1084 static void
1125 * uprobe_unregister_nosync - unregister an a !! 1085 __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
1126 * @uprobe: uprobe to remove <<
1127 * @uc: identify which probe if multiple prob <<
1128 */ <<
1129 void uprobe_unregister_nosync(struct uprobe * <<
1130 { 1086 {
1131 int err; 1087 int err;
1132 1088
1133 down_write(&uprobe->register_rwsem); !! 1089 if (WARN_ON(!consumer_del(uprobe, uc)))
1134 consumer_del(uprobe, uc); <<
1135 err = register_for_each_vma(uprobe, N <<
1136 up_write(&uprobe->register_rwsem); <<
1137 <<
1138 /* TODO : cant unregister? schedule a <<
1139 if (unlikely(err)) { <<
1140 uprobe_warn(current, "unregis <<
1141 return; 1090 return;
1142 } <<
1143 1091
1144 put_uprobe(uprobe); !! 1092 err = register_for_each_vma(uprobe, NULL);
>> 1093 /* TODO : cant unregister? schedule a worker thread */
>> 1094 if (!uprobe->consumers && !err)
>> 1095 delete_uprobe(uprobe);
1145 } 1096 }
1146 EXPORT_SYMBOL_GPL(uprobe_unregister_nosync); <<
1147 1097
1148 void uprobe_unregister_sync(void) !! 1098 /*
>> 1099 * uprobe_unregister - unregister an already registered probe.
>> 1100 * @inode: the file in which the probe has to be removed.
>> 1101 * @offset: offset from the start of the file.
>> 1102 * @uc: identify which probe if multiple probes are colocated.
>> 1103 */
>> 1104 void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
1149 { 1105 {
1150 /* !! 1106 struct uprobe *uprobe;
1151 * Now that handler_chain() and handl !! 1107
1152 * uprobe->consumers list under RCU p !! 1108 uprobe = find_uprobe(inode, offset);
1153 * uprobe->register_rwsem, we need to !! 1109 if (WARN_ON(!uprobe))
1154 * make sure that we can't call into !! 1110 return;
1155 * uprobe_consumer's callbacks anymor !! 1111
1156 * unlucky enough caller can free con !! 1112 down_write(&uprobe->register_rwsem);
1157 * handler_chain() or handle_uretprob !! 1113 __uprobe_unregister(uprobe, uc);
1158 */ !! 1114 up_write(&uprobe->register_rwsem);
1159 synchronize_srcu(&uprobes_srcu); !! 1115 put_uprobe(uprobe);
1160 } 1116 }
1161 EXPORT_SYMBOL_GPL(uprobe_unregister_sync); !! 1117 EXPORT_SYMBOL_GPL(uprobe_unregister);
1162 1118
1163 /** !! 1119 /*
1164 * uprobe_register - register a probe !! 1120 * __uprobe_register - register a probe
1165 * @inode: the file in which the probe has to 1121 * @inode: the file in which the probe has to be placed.
1166 * @offset: offset from the start of the file 1122 * @offset: offset from the start of the file.
1167 * @ref_ctr_offset: offset of SDT marker / re <<
1168 * @uc: information on howto handle the probe 1123 * @uc: information on howto handle the probe..
1169 * 1124 *
1170 * Apart from the access refcount, uprobe_reg !! 1125 * Apart from the access refcount, __uprobe_register() takes a creation
1171 * refcount (thro alloc_uprobe) if and only i 1126 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
1172 * inserted into the rbtree (i.e first consum 1127 * inserted into the rbtree (i.e first consumer for a @inode:@offset
1173 * tuple). Creation refcount stops uprobe_un 1128 * tuple). Creation refcount stops uprobe_unregister from freeing the
1174 * @uprobe even before the register operation 1129 * @uprobe even before the register operation is complete. Creation
1175 * refcount is released when the last @uc for 1130 * refcount is released when the last @uc for the @uprobe
1176 * unregisters. Caller of uprobe_register() i !! 1131 * unregisters. Caller of __uprobe_register() is required to keep @inode
1177 * (and the containing mount) referenced. 1132 * (and the containing mount) referenced.
1178 * 1133 *
1179 * Return: pointer to the new uprobe on succe !! 1134 * Return errno if it cannot successully install probes
>> 1135 * else return 0 (success)
1180 */ 1136 */
1181 struct uprobe *uprobe_register(struct inode * !! 1137 static int __uprobe_register(struct inode *inode, loff_t offset,
1182 loff_t offset !! 1138 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1183 struct uprobe <<
1184 { 1139 {
1185 struct uprobe *uprobe; 1140 struct uprobe *uprobe;
1186 int ret; 1141 int ret;
1187 1142
1188 /* Uprobe must have at least one set 1143 /* Uprobe must have at least one set consumer */
1189 if (!uc->handler && !uc->ret_handler) 1144 if (!uc->handler && !uc->ret_handler)
1190 return ERR_PTR(-EINVAL); !! 1145 return -EINVAL;
1191 1146
1192 /* copy_insn() uses read_mapping_page 1147 /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
1193 if (!inode->i_mapping->a_ops->read_fo 1148 if (!inode->i_mapping->a_ops->read_folio &&
1194 !shmem_mapping(inode->i_mapping)) 1149 !shmem_mapping(inode->i_mapping))
1195 return ERR_PTR(-EIO); !! 1150 return -EIO;
1196 /* Racy, just to catch the obvious mi 1151 /* Racy, just to catch the obvious mistakes */
1197 if (offset > i_size_read(inode)) 1152 if (offset > i_size_read(inode))
1198 return ERR_PTR(-EINVAL); !! 1153 return -EINVAL;
1199 1154
1200 /* 1155 /*
1201 * This ensures that copy_from_page() 1156 * This ensures that copy_from_page(), copy_to_page() and
1202 * __update_ref_ctr() can't cross pag 1157 * __update_ref_ctr() can't cross page boundary.
1203 */ 1158 */
1204 if (!IS_ALIGNED(offset, UPROBE_SWBP_I 1159 if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
1205 return ERR_PTR(-EINVAL); !! 1160 return -EINVAL;
1206 if (!IS_ALIGNED(ref_ctr_offset, sizeo 1161 if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
1207 return ERR_PTR(-EINVAL); !! 1162 return -EINVAL;
1208 1163
>> 1164 retry:
1209 uprobe = alloc_uprobe(inode, offset, 1165 uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
>> 1166 if (!uprobe)
>> 1167 return -ENOMEM;
1210 if (IS_ERR(uprobe)) 1168 if (IS_ERR(uprobe))
1211 return uprobe; !! 1169 return PTR_ERR(uprobe);
1212 1170
>> 1171 /*
>> 1172 * We can race with uprobe_unregister()->delete_uprobe().
>> 1173 * Check uprobe_is_active() and retry if it is false.
>> 1174 */
1213 down_write(&uprobe->register_rwsem); 1175 down_write(&uprobe->register_rwsem);
1214 consumer_add(uprobe, uc); !! 1176 ret = -EAGAIN;
1215 ret = register_for_each_vma(uprobe, u !! 1177 if (likely(uprobe_is_active(uprobe))) {
>> 1178 consumer_add(uprobe, uc);
>> 1179 ret = register_for_each_vma(uprobe, uc);
>> 1180 if (ret)
>> 1181 __uprobe_unregister(uprobe, uc);
>> 1182 }
1216 up_write(&uprobe->register_rwsem); 1183 up_write(&uprobe->register_rwsem);
>> 1184 put_uprobe(uprobe);
1217 1185
1218 if (ret) { !! 1186 if (unlikely(ret == -EAGAIN))
1219 uprobe_unregister_nosync(upro !! 1187 goto retry;
1220 /* !! 1188 return ret;
1221 * Registration might have pa !! 1189 }
1222 * this consumer being called <<
1223 * sync here. It's ok, it's u <<
1224 */ <<
1225 uprobe_unregister_sync(); <<
1226 return ERR_PTR(ret); <<
1227 } <<
1228 1190
1229 return uprobe; !! 1191 int uprobe_register(struct inode *inode, loff_t offset,
>> 1192 struct uprobe_consumer *uc)
>> 1193 {
>> 1194 return __uprobe_register(inode, offset, 0, uc);
1230 } 1195 }
1231 EXPORT_SYMBOL_GPL(uprobe_register); 1196 EXPORT_SYMBOL_GPL(uprobe_register);
1232 1197
1233 /** !! 1198 int uprobe_register_refctr(struct inode *inode, loff_t offset,
1234 * uprobe_apply - add or remove the breakpoin !! 1199 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1235 * @uprobe: uprobe which "owns" the breakpoin !! 1200 {
>> 1201 return __uprobe_register(inode, offset, ref_ctr_offset, uc);
>> 1202 }
>> 1203 EXPORT_SYMBOL_GPL(uprobe_register_refctr);
>> 1204
>> 1205 /*
>> 1206 * uprobe_apply - unregister an already registered probe.
>> 1207 * @inode: the file in which the probe has to be removed.
>> 1208 * @offset: offset from the start of the file.
1236 * @uc: consumer which wants to add more or r 1209 * @uc: consumer which wants to add more or remove some breakpoints
1237 * @add: add or remove the breakpoints 1210 * @add: add or remove the breakpoints
1238 * Return: 0 on success or negative error cod <<
1239 */ 1211 */
1240 int uprobe_apply(struct uprobe *uprobe, struc !! 1212 int uprobe_apply(struct inode *inode, loff_t offset,
>> 1213 struct uprobe_consumer *uc, bool add)
1241 { 1214 {
>> 1215 struct uprobe *uprobe;
1242 struct uprobe_consumer *con; 1216 struct uprobe_consumer *con;
1243 int ret = -ENOENT, srcu_idx; !! 1217 int ret = -ENOENT;
1244 1218
1245 down_write(&uprobe->register_rwsem); !! 1219 uprobe = find_uprobe(inode, offset);
1246 !! 1220 if (WARN_ON(!uprobe))
1247 srcu_idx = srcu_read_lock(&uprobes_sr !! 1221 return ret;
1248 list_for_each_entry_srcu(con, &uprobe <<
1249 srcu_read_lo <<
1250 if (con == uc) { <<
1251 ret = register_for_ea <<
1252 break; <<
1253 } <<
1254 } <<
1255 srcu_read_unlock(&uprobes_srcu, srcu_ <<
1256 1222
>> 1223 down_write(&uprobe->register_rwsem);
>> 1224 for (con = uprobe->consumers; con && con != uc ; con = con->next)
>> 1225 ;
>> 1226 if (con)
>> 1227 ret = register_for_each_vma(uprobe, add ? uc : NULL);
1257 up_write(&uprobe->register_rwsem); 1228 up_write(&uprobe->register_rwsem);
>> 1229 put_uprobe(uprobe);
1258 1230
1259 return ret; 1231 return ret;
1260 } 1232 }
1261 1233
1262 static int unapply_uprobe(struct uprobe *upro 1234 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
1263 { 1235 {
1264 VMA_ITERATOR(vmi, mm, 0); 1236 VMA_ITERATOR(vmi, mm, 0);
1265 struct vm_area_struct *vma; 1237 struct vm_area_struct *vma;
1266 int err = 0; 1238 int err = 0;
1267 1239
1268 mmap_read_lock(mm); 1240 mmap_read_lock(mm);
1269 for_each_vma(vmi, vma) { 1241 for_each_vma(vmi, vma) {
1270 unsigned long vaddr; 1242 unsigned long vaddr;
1271 loff_t offset; 1243 loff_t offset;
1272 1244
1273 if (!valid_vma(vma, false) || 1245 if (!valid_vma(vma, false) ||
1274 file_inode(vma->vm_file) 1246 file_inode(vma->vm_file) != uprobe->inode)
1275 continue; 1247 continue;
1276 1248
1277 offset = (loff_t)vma->vm_pgof 1249 offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1278 if (uprobe->offset < offset 1250 if (uprobe->offset < offset ||
1279 uprobe->offset >= offset 1251 uprobe->offset >= offset + vma->vm_end - vma->vm_start)
1280 continue; 1252 continue;
1281 1253
1282 vaddr = offset_to_vaddr(vma, 1254 vaddr = offset_to_vaddr(vma, uprobe->offset);
1283 err |= remove_breakpoint(upro 1255 err |= remove_breakpoint(uprobe, mm, vaddr);
1284 } 1256 }
1285 mmap_read_unlock(mm); 1257 mmap_read_unlock(mm);
1286 1258
1287 return err; 1259 return err;
1288 } 1260 }
1289 1261
1290 static struct rb_node * 1262 static struct rb_node *
1291 find_node_in_range(struct inode *inode, loff_ 1263 find_node_in_range(struct inode *inode, loff_t min, loff_t max)
1292 { 1264 {
1293 struct rb_node *n = uprobes_tree.rb_n 1265 struct rb_node *n = uprobes_tree.rb_node;
1294 1266
1295 while (n) { 1267 while (n) {
1296 struct uprobe *u = rb_entry(n 1268 struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
1297 1269
1298 if (inode < u->inode) { 1270 if (inode < u->inode) {
1299 n = n->rb_left; 1271 n = n->rb_left;
1300 } else if (inode > u->inode) 1272 } else if (inode > u->inode) {
1301 n = n->rb_right; 1273 n = n->rb_right;
1302 } else { 1274 } else {
1303 if (max < u->offset) 1275 if (max < u->offset)
1304 n = n->rb_lef 1276 n = n->rb_left;
1305 else if (min > u->off 1277 else if (min > u->offset)
1306 n = n->rb_rig 1278 n = n->rb_right;
1307 else 1279 else
1308 break; 1280 break;
1309 } 1281 }
1310 } 1282 }
1311 1283
1312 return n; 1284 return n;
1313 } 1285 }
1314 1286
1315 /* 1287 /*
1316 * For a given range in vma, build a list of 1288 * For a given range in vma, build a list of probes that need to be inserted.
1317 */ 1289 */
1318 static void build_probe_list(struct inode *in 1290 static void build_probe_list(struct inode *inode,
1319 struct vm_are 1291 struct vm_area_struct *vma,
1320 unsigned long 1292 unsigned long start, unsigned long end,
1321 struct list_h 1293 struct list_head *head)
1322 { 1294 {
1323 loff_t min, max; 1295 loff_t min, max;
1324 struct rb_node *n, *t; 1296 struct rb_node *n, *t;
1325 struct uprobe *u; 1297 struct uprobe *u;
1326 1298
1327 INIT_LIST_HEAD(head); 1299 INIT_LIST_HEAD(head);
1328 min = vaddr_to_offset(vma, start); 1300 min = vaddr_to_offset(vma, start);
1329 max = min + (end - start) - 1; 1301 max = min + (end - start) - 1;
1330 1302
1331 read_lock(&uprobes_treelock); !! 1303 spin_lock(&uprobes_treelock);
1332 n = find_node_in_range(inode, min, ma 1304 n = find_node_in_range(inode, min, max);
1333 if (n) { 1305 if (n) {
1334 for (t = n; t; t = rb_prev(t) 1306 for (t = n; t; t = rb_prev(t)) {
1335 u = rb_entry(t, struc 1307 u = rb_entry(t, struct uprobe, rb_node);
1336 if (u->inode != inode 1308 if (u->inode != inode || u->offset < min)
1337 break; 1309 break;
1338 /* if uprobe went awa !! 1310 list_add(&u->pending_list, head);
1339 if (try_get_uprobe(u) !! 1311 get_uprobe(u);
1340 list_add(&u-> <<
1341 } 1312 }
1342 for (t = n; (t = rb_next(t)); 1313 for (t = n; (t = rb_next(t)); ) {
1343 u = rb_entry(t, struc 1314 u = rb_entry(t, struct uprobe, rb_node);
1344 if (u->inode != inode 1315 if (u->inode != inode || u->offset > max)
1345 break; 1316 break;
1346 /* if uprobe went awa !! 1317 list_add(&u->pending_list, head);
1347 if (try_get_uprobe(u) !! 1318 get_uprobe(u);
1348 list_add(&u-> <<
1349 } 1319 }
1350 } 1320 }
1351 read_unlock(&uprobes_treelock); !! 1321 spin_unlock(&uprobes_treelock);
1352 } 1322 }
1353 1323
1354 /* @vma contains reference counter, not the p 1324 /* @vma contains reference counter, not the probed instruction. */
1355 static int delayed_ref_ctr_inc(struct vm_area 1325 static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
1356 { 1326 {
1357 struct list_head *pos, *q; 1327 struct list_head *pos, *q;
1358 struct delayed_uprobe *du; 1328 struct delayed_uprobe *du;
1359 unsigned long vaddr; 1329 unsigned long vaddr;
1360 int ret = 0, err = 0; 1330 int ret = 0, err = 0;
1361 1331
1362 mutex_lock(&delayed_uprobe_lock); 1332 mutex_lock(&delayed_uprobe_lock);
1363 list_for_each_safe(pos, q, &delayed_u 1333 list_for_each_safe(pos, q, &delayed_uprobe_list) {
1364 du = list_entry(pos, struct d 1334 du = list_entry(pos, struct delayed_uprobe, list);
1365 1335
1366 if (du->mm != vma->vm_mm || 1336 if (du->mm != vma->vm_mm ||
1367 !valid_ref_ctr_vma(du->up 1337 !valid_ref_ctr_vma(du->uprobe, vma))
1368 continue; 1338 continue;
1369 1339
1370 vaddr = offset_to_vaddr(vma, 1340 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
1371 ret = __update_ref_ctr(vma->v 1341 ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
1372 if (ret) { 1342 if (ret) {
1373 update_ref_ctr_warn(d 1343 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
1374 if (!err) 1344 if (!err)
1375 err = ret; 1345 err = ret;
1376 } 1346 }
1377 delayed_uprobe_delete(du); 1347 delayed_uprobe_delete(du);
1378 } 1348 }
1379 mutex_unlock(&delayed_uprobe_lock); 1349 mutex_unlock(&delayed_uprobe_lock);
1380 return err; 1350 return err;
1381 } 1351 }
1382 1352
1383 /* 1353 /*
1384 * Called from mmap_region/vma_merge with mm- 1354 * Called from mmap_region/vma_merge with mm->mmap_lock acquired.
1385 * 1355 *
1386 * Currently we ignore all errors and always 1356 * Currently we ignore all errors and always return 0, the callers
1387 * can't handle the failure anyway. 1357 * can't handle the failure anyway.
1388 */ 1358 */
1389 int uprobe_mmap(struct vm_area_struct *vma) 1359 int uprobe_mmap(struct vm_area_struct *vma)
1390 { 1360 {
1391 struct list_head tmp_list; 1361 struct list_head tmp_list;
1392 struct uprobe *uprobe, *u; 1362 struct uprobe *uprobe, *u;
1393 struct inode *inode; 1363 struct inode *inode;
1394 1364
1395 if (no_uprobe_events()) 1365 if (no_uprobe_events())
1396 return 0; 1366 return 0;
1397 1367
1398 if (vma->vm_file && 1368 if (vma->vm_file &&
1399 (vma->vm_flags & (VM_WRITE|VM_SHA 1369 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
1400 test_bit(MMF_HAS_UPROBES, &vma->v 1370 test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
1401 delayed_ref_ctr_inc(vma); 1371 delayed_ref_ctr_inc(vma);
1402 1372
1403 if (!valid_vma(vma, true)) 1373 if (!valid_vma(vma, true))
1404 return 0; 1374 return 0;
1405 1375
1406 inode = file_inode(vma->vm_file); 1376 inode = file_inode(vma->vm_file);
1407 if (!inode) 1377 if (!inode)
1408 return 0; 1378 return 0;
1409 1379
1410 mutex_lock(uprobes_mmap_hash(inode)); 1380 mutex_lock(uprobes_mmap_hash(inode));
1411 build_probe_list(inode, vma, vma->vm_ 1381 build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
1412 /* 1382 /*
1413 * We can race with uprobe_unregister 1383 * We can race with uprobe_unregister(), this uprobe can be already
1414 * removed. But in this case filter_c 1384 * removed. But in this case filter_chain() must return false, all
1415 * consumers have gone away. 1385 * consumers have gone away.
1416 */ 1386 */
1417 list_for_each_entry_safe(uprobe, u, & 1387 list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
1418 if (!fatal_signal_pending(cur 1388 if (!fatal_signal_pending(current) &&
1419 filter_chain(uprobe, vma- !! 1389 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
1420 unsigned long vaddr = 1390 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
1421 install_breakpoint(up 1391 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
1422 } 1392 }
1423 put_uprobe(uprobe); 1393 put_uprobe(uprobe);
1424 } 1394 }
1425 mutex_unlock(uprobes_mmap_hash(inode) 1395 mutex_unlock(uprobes_mmap_hash(inode));
1426 1396
1427 return 0; 1397 return 0;
1428 } 1398 }
1429 1399
1430 static bool 1400 static bool
1431 vma_has_uprobes(struct vm_area_struct *vma, u 1401 vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1432 { 1402 {
1433 loff_t min, max; 1403 loff_t min, max;
1434 struct inode *inode; 1404 struct inode *inode;
1435 struct rb_node *n; 1405 struct rb_node *n;
1436 1406
1437 inode = file_inode(vma->vm_file); 1407 inode = file_inode(vma->vm_file);
1438 1408
1439 min = vaddr_to_offset(vma, start); 1409 min = vaddr_to_offset(vma, start);
1440 max = min + (end - start) - 1; 1410 max = min + (end - start) - 1;
1441 1411
1442 read_lock(&uprobes_treelock); !! 1412 spin_lock(&uprobes_treelock);
1443 n = find_node_in_range(inode, min, ma 1413 n = find_node_in_range(inode, min, max);
1444 read_unlock(&uprobes_treelock); !! 1414 spin_unlock(&uprobes_treelock);
1445 1415
1446 return !!n; 1416 return !!n;
1447 } 1417 }
1448 1418
1449 /* 1419 /*
1450 * Called in context of a munmap of a vma. 1420 * Called in context of a munmap of a vma.
1451 */ 1421 */
1452 void uprobe_munmap(struct vm_area_struct *vma 1422 void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1453 { 1423 {
1454 if (no_uprobe_events() || !valid_vma( 1424 if (no_uprobe_events() || !valid_vma(vma, false))
1455 return; 1425 return;
1456 1426
1457 if (!atomic_read(&vma->vm_mm->mm_user 1427 if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
1458 return; 1428 return;
1459 1429
1460 if (!test_bit(MMF_HAS_UPROBES, &vma-> 1430 if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
1461 test_bit(MMF_RECALC_UPROBES, &vm 1431 test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
1462 return; 1432 return;
1463 1433
1464 if (vma_has_uprobes(vma, start, end)) 1434 if (vma_has_uprobes(vma, start, end))
1465 set_bit(MMF_RECALC_UPROBES, & 1435 set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
1466 } 1436 }
1467 1437
1468 static vm_fault_t xol_fault(const struct vm_s <<
1469 struct vm_area_st <<
1470 { <<
1471 struct xol_area *area = vma->vm_mm->u <<
1472 <<
1473 vmf->page = area->page; <<
1474 get_page(vmf->page); <<
1475 return 0; <<
1476 } <<
1477 <<
1478 static const struct vm_special_mapping xol_ma <<
1479 .name = "[uprobes]", <<
1480 .fault = xol_fault, <<
1481 }; <<
1482 <<
1483 /* Slot allocation for XOL */ 1438 /* Slot allocation for XOL */
1484 static int xol_add_vma(struct mm_struct *mm, 1439 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1485 { 1440 {
1486 struct vm_area_struct *vma; 1441 struct vm_area_struct *vma;
1487 int ret; 1442 int ret;
1488 1443
1489 if (mmap_write_lock_killable(mm)) 1444 if (mmap_write_lock_killable(mm))
1490 return -EINTR; 1445 return -EINTR;
1491 1446
1492 if (mm->uprobes_state.xol_area) { 1447 if (mm->uprobes_state.xol_area) {
1493 ret = -EALREADY; 1448 ret = -EALREADY;
1494 goto fail; 1449 goto fail;
1495 } 1450 }
1496 1451
1497 if (!area->vaddr) { 1452 if (!area->vaddr) {
1498 /* Try to map as high as poss 1453 /* Try to map as high as possible, this is only a hint. */
1499 area->vaddr = get_unmapped_ar 1454 area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1500 1455 PAGE_SIZE, 0, 0);
1501 if (IS_ERR_VALUE(area->vaddr) 1456 if (IS_ERR_VALUE(area->vaddr)) {
1502 ret = area->vaddr; 1457 ret = area->vaddr;
1503 goto fail; 1458 goto fail;
1504 } 1459 }
1505 } 1460 }
1506 1461
1507 vma = _install_special_mapping(mm, ar 1462 vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1508 VM_EXEC|VM_MA 1463 VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
1509 &xol_mapping) !! 1464 &area->xol_mapping);
1510 if (IS_ERR(vma)) { 1465 if (IS_ERR(vma)) {
1511 ret = PTR_ERR(vma); 1466 ret = PTR_ERR(vma);
1512 goto fail; 1467 goto fail;
1513 } 1468 }
1514 1469
1515 ret = 0; 1470 ret = 0;
1516 /* pairs with get_xol_area() */ 1471 /* pairs with get_xol_area() */
1517 smp_store_release(&mm->uprobes_state. 1472 smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
1518 fail: 1473 fail:
1519 mmap_write_unlock(mm); 1474 mmap_write_unlock(mm);
1520 1475
1521 return ret; 1476 return ret;
1522 } 1477 }
1523 1478
1524 void * __weak arch_uprobe_trampoline(unsigned <<
1525 { <<
1526 static uprobe_opcode_t insn = UPROBE_ <<
1527 <<
1528 *psize = UPROBE_SWBP_INSN_SIZE; <<
1529 return &insn; <<
1530 } <<
1531 <<
1532 static struct xol_area *__create_xol_area(uns 1479 static struct xol_area *__create_xol_area(unsigned long vaddr)
1533 { 1480 {
1534 struct mm_struct *mm = current->mm; 1481 struct mm_struct *mm = current->mm;
1535 unsigned long insns_size; !! 1482 uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1536 struct xol_area *area; 1483 struct xol_area *area;
1537 void *insns; <<
1538 1484
1539 area = kzalloc(sizeof(*area), GFP_KER !! 1485 area = kmalloc(sizeof(*area), GFP_KERNEL);
1540 if (unlikely(!area)) 1486 if (unlikely(!area))
1541 goto out; 1487 goto out;
1542 1488
1543 area->bitmap = kcalloc(BITS_TO_LONGS( 1489 area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
1544 GFP_KERNEL); 1490 GFP_KERNEL);
1545 if (!area->bitmap) 1491 if (!area->bitmap)
1546 goto free_area; 1492 goto free_area;
1547 1493
1548 area->page = alloc_page(GFP_HIGHUSER !! 1494 area->xol_mapping.name = "[uprobes]";
1549 if (!area->page) !! 1495 area->xol_mapping.fault = NULL;
>> 1496 area->xol_mapping.pages = area->pages;
>> 1497 area->pages[0] = alloc_page(GFP_HIGHUSER);
>> 1498 if (!area->pages[0])
1550 goto free_bitmap; 1499 goto free_bitmap;
>> 1500 area->pages[1] = NULL;
1551 1501
1552 area->vaddr = vaddr; 1502 area->vaddr = vaddr;
1553 init_waitqueue_head(&area->wq); 1503 init_waitqueue_head(&area->wq);
1554 /* Reserve the 1st slot for get_tramp 1504 /* Reserve the 1st slot for get_trampoline_vaddr() */
1555 set_bit(0, area->bitmap); 1505 set_bit(0, area->bitmap);
1556 atomic_set(&area->slot_count, 1); 1506 atomic_set(&area->slot_count, 1);
1557 insns = arch_uprobe_trampoline(&insns !! 1507 arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
1558 arch_uprobe_copy_ixol(area->page, 0, <<
1559 1508
1560 if (!xol_add_vma(mm, area)) 1509 if (!xol_add_vma(mm, area))
1561 return area; 1510 return area;
1562 1511
1563 __free_page(area->page); !! 1512 __free_page(area->pages[0]);
1564 free_bitmap: 1513 free_bitmap:
1565 kfree(area->bitmap); 1514 kfree(area->bitmap);
1566 free_area: 1515 free_area:
1567 kfree(area); 1516 kfree(area);
1568 out: 1517 out:
1569 return NULL; 1518 return NULL;
1570 } 1519 }
1571 1520
1572 /* 1521 /*
1573 * get_xol_area - Allocate process's xol_area 1522 * get_xol_area - Allocate process's xol_area if necessary.
1574 * This area will be used for storing instruc 1523 * This area will be used for storing instructions for execution out of line.
1575 * 1524 *
1576 * Returns the allocated area or NULL. 1525 * Returns the allocated area or NULL.
1577 */ 1526 */
1578 static struct xol_area *get_xol_area(void) 1527 static struct xol_area *get_xol_area(void)
1579 { 1528 {
1580 struct mm_struct *mm = current->mm; 1529 struct mm_struct *mm = current->mm;
1581 struct xol_area *area; 1530 struct xol_area *area;
1582 1531
1583 if (!mm->uprobes_state.xol_area) 1532 if (!mm->uprobes_state.xol_area)
1584 __create_xol_area(0); 1533 __create_xol_area(0);
1585 1534
1586 /* Pairs with xol_add_vma() smp_store 1535 /* Pairs with xol_add_vma() smp_store_release() */
1587 area = READ_ONCE(mm->uprobes_state.xo 1536 area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
1588 return area; 1537 return area;
1589 } 1538 }
1590 1539
1591 /* 1540 /*
1592 * uprobe_clear_state - Free the area allocat 1541 * uprobe_clear_state - Free the area allocated for slots.
1593 */ 1542 */
1594 void uprobe_clear_state(struct mm_struct *mm) 1543 void uprobe_clear_state(struct mm_struct *mm)
1595 { 1544 {
1596 struct xol_area *area = mm->uprobes_s 1545 struct xol_area *area = mm->uprobes_state.xol_area;
1597 1546
1598 mutex_lock(&delayed_uprobe_lock); 1547 mutex_lock(&delayed_uprobe_lock);
1599 delayed_uprobe_remove(NULL, mm); 1548 delayed_uprobe_remove(NULL, mm);
1600 mutex_unlock(&delayed_uprobe_lock); 1549 mutex_unlock(&delayed_uprobe_lock);
1601 1550
1602 if (!area) 1551 if (!area)
1603 return; 1552 return;
1604 1553
1605 put_page(area->page); !! 1554 put_page(area->pages[0]);
1606 kfree(area->bitmap); 1555 kfree(area->bitmap);
1607 kfree(area); 1556 kfree(area);
1608 } 1557 }
1609 1558
1610 void uprobe_start_dup_mmap(void) 1559 void uprobe_start_dup_mmap(void)
1611 { 1560 {
1612 percpu_down_read(&dup_mmap_sem); 1561 percpu_down_read(&dup_mmap_sem);
1613 } 1562 }
1614 1563
1615 void uprobe_end_dup_mmap(void) 1564 void uprobe_end_dup_mmap(void)
1616 { 1565 {
1617 percpu_up_read(&dup_mmap_sem); 1566 percpu_up_read(&dup_mmap_sem);
1618 } 1567 }
1619 1568
1620 void uprobe_dup_mmap(struct mm_struct *oldmm, 1569 void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
1621 { 1570 {
1622 if (test_bit(MMF_HAS_UPROBES, &oldmm- 1571 if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
1623 set_bit(MMF_HAS_UPROBES, &new 1572 set_bit(MMF_HAS_UPROBES, &newmm->flags);
1624 /* unconditionally, dup_mmap( 1573 /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
1625 set_bit(MMF_RECALC_UPROBES, & 1574 set_bit(MMF_RECALC_UPROBES, &newmm->flags);
1626 } 1575 }
1627 } 1576 }
1628 1577
1629 /* 1578 /*
1630 * - search for a free slot. 1579 * - search for a free slot.
1631 */ 1580 */
1632 static unsigned long xol_take_insn_slot(struc 1581 static unsigned long xol_take_insn_slot(struct xol_area *area)
1633 { 1582 {
1634 unsigned long slot_addr; 1583 unsigned long slot_addr;
1635 int slot_nr; 1584 int slot_nr;
1636 1585
1637 do { 1586 do {
1638 slot_nr = find_first_zero_bit 1587 slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1639 if (slot_nr < UINSNS_PER_PAGE 1588 if (slot_nr < UINSNS_PER_PAGE) {
1640 if (!test_and_set_bit 1589 if (!test_and_set_bit(slot_nr, area->bitmap))
1641 break; 1590 break;
1642 1591
1643 slot_nr = UINSNS_PER_ 1592 slot_nr = UINSNS_PER_PAGE;
1644 continue; 1593 continue;
1645 } 1594 }
1646 wait_event(area->wq, (atomic_ 1595 wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1647 } while (slot_nr >= UINSNS_PER_PAGE); 1596 } while (slot_nr >= UINSNS_PER_PAGE);
1648 1597
1649 slot_addr = area->vaddr + (slot_nr * 1598 slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1650 atomic_inc(&area->slot_count); 1599 atomic_inc(&area->slot_count);
1651 1600
1652 return slot_addr; 1601 return slot_addr;
1653 } 1602 }
1654 1603
1655 /* 1604 /*
1656 * xol_get_insn_slot - allocate a slot for xo 1605 * xol_get_insn_slot - allocate a slot for xol.
1657 * Returns the allocated slot address or 0. 1606 * Returns the allocated slot address or 0.
1658 */ 1607 */
1659 static unsigned long xol_get_insn_slot(struct 1608 static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
1660 { 1609 {
1661 struct xol_area *area; 1610 struct xol_area *area;
1662 unsigned long xol_vaddr; 1611 unsigned long xol_vaddr;
1663 1612
1664 area = get_xol_area(); 1613 area = get_xol_area();
1665 if (!area) 1614 if (!area)
1666 return 0; 1615 return 0;
1667 1616
1668 xol_vaddr = xol_take_insn_slot(area); 1617 xol_vaddr = xol_take_insn_slot(area);
1669 if (unlikely(!xol_vaddr)) 1618 if (unlikely(!xol_vaddr))
1670 return 0; 1619 return 0;
1671 1620
1672 arch_uprobe_copy_ixol(area->page, xol !! 1621 arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
1673 &uprobe->arch.i 1622 &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1674 1623
1675 return xol_vaddr; 1624 return xol_vaddr;
1676 } 1625 }
1677 1626
1678 /* 1627 /*
1679 * xol_free_insn_slot - If slot was earlier a 1628 * xol_free_insn_slot - If slot was earlier allocated by
1680 * @xol_get_insn_slot(), make the slot availa 1629 * @xol_get_insn_slot(), make the slot available for
1681 * subsequent requests. 1630 * subsequent requests.
1682 */ 1631 */
1683 static void xol_free_insn_slot(struct task_st 1632 static void xol_free_insn_slot(struct task_struct *tsk)
1684 { 1633 {
1685 struct xol_area *area; 1634 struct xol_area *area;
1686 unsigned long vma_end; 1635 unsigned long vma_end;
1687 unsigned long slot_addr; 1636 unsigned long slot_addr;
1688 1637
1689 if (!tsk->mm || !tsk->mm->uprobes_sta 1638 if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
1690 return; 1639 return;
1691 1640
1692 slot_addr = tsk->utask->xol_vaddr; 1641 slot_addr = tsk->utask->xol_vaddr;
1693 if (unlikely(!slot_addr)) 1642 if (unlikely(!slot_addr))
1694 return; 1643 return;
1695 1644
1696 area = tsk->mm->uprobes_state.xol_are 1645 area = tsk->mm->uprobes_state.xol_area;
1697 vma_end = area->vaddr + PAGE_SIZE; 1646 vma_end = area->vaddr + PAGE_SIZE;
1698 if (area->vaddr <= slot_addr && slot_ 1647 if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1699 unsigned long offset; 1648 unsigned long offset;
1700 int slot_nr; 1649 int slot_nr;
1701 1650
1702 offset = slot_addr - area->va 1651 offset = slot_addr - area->vaddr;
1703 slot_nr = offset / UPROBE_XOL 1652 slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1704 if (slot_nr >= UINSNS_PER_PAG 1653 if (slot_nr >= UINSNS_PER_PAGE)
1705 return; 1654 return;
1706 1655
1707 clear_bit(slot_nr, area->bitm 1656 clear_bit(slot_nr, area->bitmap);
1708 atomic_dec(&area->slot_count) 1657 atomic_dec(&area->slot_count);
1709 smp_mb__after_atomic(); /* pa 1658 smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1710 if (waitqueue_active(&area->w 1659 if (waitqueue_active(&area->wq))
1711 wake_up(&area->wq); 1660 wake_up(&area->wq);
1712 1661
1713 tsk->utask->xol_vaddr = 0; 1662 tsk->utask->xol_vaddr = 0;
1714 } 1663 }
1715 } 1664 }
1716 1665
1717 void __weak arch_uprobe_copy_ixol(struct page 1666 void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
1718 void *src, 1667 void *src, unsigned long len)
1719 { 1668 {
1720 /* Initialize the slot */ 1669 /* Initialize the slot */
1721 copy_to_page(page, vaddr, src, len); 1670 copy_to_page(page, vaddr, src, len);
1722 1671
1723 /* 1672 /*
1724 * We probably need flush_icache_user 1673 * We probably need flush_icache_user_page() but it needs vma.
1725 * This should work on most of archit 1674 * This should work on most of architectures by default. If
1726 * architecture needs to do something 1675 * architecture needs to do something different it can define
1727 * its own version of the function. 1676 * its own version of the function.
1728 */ 1677 */
1729 flush_dcache_page(page); 1678 flush_dcache_page(page);
1730 } 1679 }
1731 1680
1732 /** 1681 /**
1733 * uprobe_get_swbp_addr - compute address of 1682 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1734 * @regs: Reflects the saved state of the tas 1683 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1735 * instruction. 1684 * instruction.
1736 * Return the address of the breakpoint instr 1685 * Return the address of the breakpoint instruction.
1737 */ 1686 */
1738 unsigned long __weak uprobe_get_swbp_addr(str 1687 unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1739 { 1688 {
1740 return instruction_pointer(regs) - UP 1689 return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1741 } 1690 }
1742 1691
1743 unsigned long uprobe_get_trap_addr(struct pt_ 1692 unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
1744 { 1693 {
1745 struct uprobe_task *utask = current-> 1694 struct uprobe_task *utask = current->utask;
1746 1695
1747 if (unlikely(utask && utask->active_u 1696 if (unlikely(utask && utask->active_uprobe))
1748 return utask->vaddr; 1697 return utask->vaddr;
1749 1698
1750 return instruction_pointer(regs); 1699 return instruction_pointer(regs);
1751 } 1700 }
1752 1701
1753 static struct return_instance *free_ret_insta 1702 static struct return_instance *free_ret_instance(struct return_instance *ri)
1754 { 1703 {
1755 struct return_instance *next = ri->ne 1704 struct return_instance *next = ri->next;
1756 put_uprobe(ri->uprobe); 1705 put_uprobe(ri->uprobe);
1757 kfree(ri); 1706 kfree(ri);
1758 return next; 1707 return next;
1759 } 1708 }
1760 1709
1761 /* 1710 /*
1762 * Called with no locks held. 1711 * Called with no locks held.
1763 * Called in context of an exiting or an exec 1712 * Called in context of an exiting or an exec-ing thread.
1764 */ 1713 */
1765 void uprobe_free_utask(struct task_struct *t) 1714 void uprobe_free_utask(struct task_struct *t)
1766 { 1715 {
1767 struct uprobe_task *utask = t->utask; 1716 struct uprobe_task *utask = t->utask;
1768 struct return_instance *ri; 1717 struct return_instance *ri;
1769 1718
1770 if (!utask) 1719 if (!utask)
1771 return; 1720 return;
1772 1721
1773 if (utask->active_uprobe) 1722 if (utask->active_uprobe)
1774 put_uprobe(utask->active_upro 1723 put_uprobe(utask->active_uprobe);
1775 1724
1776 ri = utask->return_instances; 1725 ri = utask->return_instances;
1777 while (ri) 1726 while (ri)
1778 ri = free_ret_instance(ri); 1727 ri = free_ret_instance(ri);
1779 1728
1780 xol_free_insn_slot(t); 1729 xol_free_insn_slot(t);
1781 kfree(utask); 1730 kfree(utask);
1782 t->utask = NULL; 1731 t->utask = NULL;
1783 } 1732 }
1784 1733
1785 /* 1734 /*
1786 * Allocate a uprobe_task object for the task 1735 * Allocate a uprobe_task object for the task if necessary.
1787 * Called when the thread hits a breakpoint. 1736 * Called when the thread hits a breakpoint.
1788 * 1737 *
1789 * Returns: 1738 * Returns:
1790 * - pointer to new uprobe_task on success 1739 * - pointer to new uprobe_task on success
1791 * - NULL otherwise 1740 * - NULL otherwise
1792 */ 1741 */
1793 static struct uprobe_task *get_utask(void) 1742 static struct uprobe_task *get_utask(void)
1794 { 1743 {
1795 if (!current->utask) 1744 if (!current->utask)
1796 current->utask = kzalloc(size 1745 current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1797 return current->utask; 1746 return current->utask;
1798 } 1747 }
1799 1748
1800 static int dup_utask(struct task_struct *t, s 1749 static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
1801 { 1750 {
1802 struct uprobe_task *n_utask; 1751 struct uprobe_task *n_utask;
1803 struct return_instance **p, *o, *n; 1752 struct return_instance **p, *o, *n;
1804 1753
1805 n_utask = kzalloc(sizeof(struct uprob 1754 n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1806 if (!n_utask) 1755 if (!n_utask)
1807 return -ENOMEM; 1756 return -ENOMEM;
1808 t->utask = n_utask; 1757 t->utask = n_utask;
1809 1758
1810 p = &n_utask->return_instances; 1759 p = &n_utask->return_instances;
1811 for (o = o_utask->return_instances; o 1760 for (o = o_utask->return_instances; o; o = o->next) {
1812 n = kmalloc(sizeof(struct ret 1761 n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1813 if (!n) 1762 if (!n)
1814 return -ENOMEM; 1763 return -ENOMEM;
1815 1764
1816 *n = *o; 1765 *n = *o;
1817 /* <<
1818 * uprobe's refcnt has to be <<
1819 * utask->return_instances it <<
1820 * removed right now, as task <<
1821 * get_uprobe() is safe to us <<
1822 */ <<
1823 get_uprobe(n->uprobe); 1766 get_uprobe(n->uprobe);
1824 n->next = NULL; 1767 n->next = NULL;
1825 1768
1826 *p = n; 1769 *p = n;
1827 p = &n->next; 1770 p = &n->next;
1828 n_utask->depth++; 1771 n_utask->depth++;
1829 } 1772 }
1830 1773
1831 return 0; 1774 return 0;
1832 } 1775 }
1833 1776
>> 1777 static void uprobe_warn(struct task_struct *t, const char *msg)
>> 1778 {
>> 1779 pr_warn("uprobe: %s:%d failed to %s\n",
>> 1780 current->comm, current->pid, msg);
>> 1781 }
>> 1782
1834 static void dup_xol_work(struct callback_head 1783 static void dup_xol_work(struct callback_head *work)
1835 { 1784 {
1836 if (current->flags & PF_EXITING) 1785 if (current->flags & PF_EXITING)
1837 return; 1786 return;
1838 1787
1839 if (!__create_xol_area(current->utask 1788 if (!__create_xol_area(current->utask->dup_xol_addr) &&
1840 !fatal_signal_pending 1789 !fatal_signal_pending(current))
1841 uprobe_warn(current, "dup xol 1790 uprobe_warn(current, "dup xol area");
1842 } 1791 }
1843 1792
1844 /* 1793 /*
1845 * Called in context of a new clone/fork from 1794 * Called in context of a new clone/fork from copy_process.
1846 */ 1795 */
1847 void uprobe_copy_process(struct task_struct * 1796 void uprobe_copy_process(struct task_struct *t, unsigned long flags)
1848 { 1797 {
1849 struct uprobe_task *utask = current-> 1798 struct uprobe_task *utask = current->utask;
1850 struct mm_struct *mm = current->mm; 1799 struct mm_struct *mm = current->mm;
1851 struct xol_area *area; 1800 struct xol_area *area;
1852 1801
1853 t->utask = NULL; 1802 t->utask = NULL;
1854 1803
1855 if (!utask || !utask->return_instance 1804 if (!utask || !utask->return_instances)
1856 return; 1805 return;
1857 1806
1858 if (mm == t->mm && !(flags & CLONE_VF 1807 if (mm == t->mm && !(flags & CLONE_VFORK))
1859 return; 1808 return;
1860 1809
1861 if (dup_utask(t, utask)) 1810 if (dup_utask(t, utask))
1862 return uprobe_warn(t, "dup re 1811 return uprobe_warn(t, "dup ret instances");
1863 1812
1864 /* The task can fork() after dup_xol_ 1813 /* The task can fork() after dup_xol_work() fails */
1865 area = mm->uprobes_state.xol_area; 1814 area = mm->uprobes_state.xol_area;
1866 if (!area) 1815 if (!area)
1867 return uprobe_warn(t, "dup xo 1816 return uprobe_warn(t, "dup xol area");
1868 1817
1869 if (mm == t->mm) 1818 if (mm == t->mm)
1870 return; 1819 return;
1871 1820
1872 t->utask->dup_xol_addr = area->vaddr; 1821 t->utask->dup_xol_addr = area->vaddr;
1873 init_task_work(&t->utask->dup_xol_wor 1822 init_task_work(&t->utask->dup_xol_work, dup_xol_work);
1874 task_work_add(t, &t->utask->dup_xol_w 1823 task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME);
1875 } 1824 }
1876 1825
1877 /* 1826 /*
1878 * Current area->vaddr notion assume the tram 1827 * Current area->vaddr notion assume the trampoline address is always
1879 * equal area->vaddr. 1828 * equal area->vaddr.
1880 * 1829 *
1881 * Returns -1 in case the xol_area is not all 1830 * Returns -1 in case the xol_area is not allocated.
1882 */ 1831 */
1883 unsigned long uprobe_get_trampoline_vaddr(voi !! 1832 static unsigned long get_trampoline_vaddr(void)
1884 { 1833 {
1885 struct xol_area *area; 1834 struct xol_area *area;
1886 unsigned long trampoline_vaddr = -1; 1835 unsigned long trampoline_vaddr = -1;
1887 1836
1888 /* Pairs with xol_add_vma() smp_store 1837 /* Pairs with xol_add_vma() smp_store_release() */
1889 area = READ_ONCE(current->mm->uprobes 1838 area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
1890 if (area) 1839 if (area)
1891 trampoline_vaddr = area->vadd 1840 trampoline_vaddr = area->vaddr;
1892 1841
1893 return trampoline_vaddr; 1842 return trampoline_vaddr;
1894 } 1843 }
1895 1844
1896 static void cleanup_return_instances(struct u 1845 static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
1897 struc 1846 struct pt_regs *regs)
1898 { 1847 {
1899 struct return_instance *ri = utask->r 1848 struct return_instance *ri = utask->return_instances;
1900 enum rp_check ctx = chained ? RP_CHEC 1849 enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
1901 1850
1902 while (ri && !arch_uretprobe_is_alive 1851 while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
1903 ri = free_ret_instance(ri); 1852 ri = free_ret_instance(ri);
1904 utask->depth--; 1853 utask->depth--;
1905 } 1854 }
1906 utask->return_instances = ri; 1855 utask->return_instances = ri;
1907 } 1856 }
1908 1857
1909 static void prepare_uretprobe(struct uprobe * 1858 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
1910 { 1859 {
1911 struct return_instance *ri; 1860 struct return_instance *ri;
1912 struct uprobe_task *utask; 1861 struct uprobe_task *utask;
1913 unsigned long orig_ret_vaddr, trampol 1862 unsigned long orig_ret_vaddr, trampoline_vaddr;
1914 bool chained; 1863 bool chained;
1915 1864
1916 if (!get_xol_area()) 1865 if (!get_xol_area())
1917 return; 1866 return;
1918 1867
1919 utask = get_utask(); 1868 utask = get_utask();
1920 if (!utask) 1869 if (!utask)
1921 return; 1870 return;
1922 1871
1923 if (utask->depth >= MAX_URETPROBE_DEP 1872 if (utask->depth >= MAX_URETPROBE_DEPTH) {
1924 printk_ratelimited(KERN_INFO 1873 printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
1925 " nestedness 1874 " nestedness limit pid/tgid=%d/%d\n",
1926 current->pid, 1875 current->pid, current->tgid);
1927 return; 1876 return;
1928 } 1877 }
1929 1878
1930 /* we need to bump refcount to store <<
1931 if (!try_get_uprobe(uprobe)) <<
1932 return; <<
1933 <<
1934 ri = kmalloc(sizeof(struct return_ins 1879 ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1935 if (!ri) 1880 if (!ri)
1936 goto fail; !! 1881 return;
1937 1882
1938 trampoline_vaddr = uprobe_get_trampol !! 1883 trampoline_vaddr = get_trampoline_vaddr();
1939 orig_ret_vaddr = arch_uretprobe_hijac 1884 orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1940 if (orig_ret_vaddr == -1) 1885 if (orig_ret_vaddr == -1)
1941 goto fail; 1886 goto fail;
1942 1887
1943 /* drop the entries invalidated by lo 1888 /* drop the entries invalidated by longjmp() */
1944 chained = (orig_ret_vaddr == trampoli 1889 chained = (orig_ret_vaddr == trampoline_vaddr);
1945 cleanup_return_instances(utask, chain 1890 cleanup_return_instances(utask, chained, regs);
1946 1891
1947 /* 1892 /*
1948 * We don't want to keep trampoline a 1893 * We don't want to keep trampoline address in stack, rather keep the
1949 * original return address of first c 1894 * original return address of first caller thru all the consequent
1950 * instances. This also makes breakpo 1895 * instances. This also makes breakpoint unwrapping easier.
1951 */ 1896 */
1952 if (chained) { 1897 if (chained) {
1953 if (!utask->return_instances) 1898 if (!utask->return_instances) {
1954 /* 1899 /*
1955 * This situation is 1900 * This situation is not possible. Likely we have an
1956 * attack from user-s 1901 * attack from user-space.
1957 */ 1902 */
1958 uprobe_warn(current, 1903 uprobe_warn(current, "handle tail call");
1959 goto fail; 1904 goto fail;
1960 } 1905 }
1961 orig_ret_vaddr = utask->retur 1906 orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
1962 } 1907 }
1963 ri->uprobe = uprobe; !! 1908
>> 1909 ri->uprobe = get_uprobe(uprobe);
1964 ri->func = instruction_pointer(regs); 1910 ri->func = instruction_pointer(regs);
1965 ri->stack = user_stack_pointer(regs); 1911 ri->stack = user_stack_pointer(regs);
1966 ri->orig_ret_vaddr = orig_ret_vaddr; 1912 ri->orig_ret_vaddr = orig_ret_vaddr;
1967 ri->chained = chained; 1913 ri->chained = chained;
1968 1914
1969 utask->depth++; 1915 utask->depth++;
1970 ri->next = utask->return_instances; 1916 ri->next = utask->return_instances;
1971 utask->return_instances = ri; 1917 utask->return_instances = ri;
1972 1918
1973 return; 1919 return;
1974 fail: !! 1920 fail:
1975 kfree(ri); 1921 kfree(ri);
1976 put_uprobe(uprobe); <<
1977 } 1922 }
1978 1923
1979 /* Prepare to single-step probed instruction 1924 /* Prepare to single-step probed instruction out of line. */
1980 static int 1925 static int
1981 pre_ssout(struct uprobe *uprobe, struct pt_re 1926 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
1982 { 1927 {
1983 struct uprobe_task *utask; 1928 struct uprobe_task *utask;
1984 unsigned long xol_vaddr; 1929 unsigned long xol_vaddr;
1985 int err; 1930 int err;
1986 1931
1987 utask = get_utask(); 1932 utask = get_utask();
1988 if (!utask) 1933 if (!utask)
1989 return -ENOMEM; 1934 return -ENOMEM;
1990 1935
1991 if (!try_get_uprobe(uprobe)) <<
1992 return -EINVAL; <<
1993 <<
1994 xol_vaddr = xol_get_insn_slot(uprobe) 1936 xol_vaddr = xol_get_insn_slot(uprobe);
1995 if (!xol_vaddr) { !! 1937 if (!xol_vaddr)
1996 err = -ENOMEM; !! 1938 return -ENOMEM;
1997 goto err_out; <<
1998 } <<
1999 1939
2000 utask->xol_vaddr = xol_vaddr; 1940 utask->xol_vaddr = xol_vaddr;
2001 utask->vaddr = bp_vaddr; 1941 utask->vaddr = bp_vaddr;
2002 1942
2003 err = arch_uprobe_pre_xol(&uprobe->ar 1943 err = arch_uprobe_pre_xol(&uprobe->arch, regs);
2004 if (unlikely(err)) { 1944 if (unlikely(err)) {
2005 xol_free_insn_slot(current); 1945 xol_free_insn_slot(current);
2006 goto err_out; !! 1946 return err;
2007 } 1947 }
2008 1948
2009 utask->active_uprobe = uprobe; 1949 utask->active_uprobe = uprobe;
2010 utask->state = UTASK_SSTEP; 1950 utask->state = UTASK_SSTEP;
2011 return 0; 1951 return 0;
2012 err_out: <<
2013 put_uprobe(uprobe); <<
2014 return err; <<
2015 } 1952 }
2016 1953
2017 /* 1954 /*
2018 * If we are singlestepping, then ensure this 1955 * If we are singlestepping, then ensure this thread is not connected to
2019 * non-fatal signals until completion of sing 1956 * non-fatal signals until completion of singlestep. When xol insn itself
2020 * triggers the signal, restart the original 1957 * triggers the signal, restart the original insn even if the task is
2021 * already SIGKILL'ed (since coredump should 1958 * already SIGKILL'ed (since coredump should report the correct ip). This
2022 * is even more important if the task has a h 1959 * is even more important if the task has a handler for SIGSEGV/etc, The
2023 * _same_ instruction should be repeated agai 1960 * _same_ instruction should be repeated again after return from the signal
2024 * handler, and SSTEP can never finish in thi 1961 * handler, and SSTEP can never finish in this case.
2025 */ 1962 */
2026 bool uprobe_deny_signal(void) 1963 bool uprobe_deny_signal(void)
2027 { 1964 {
2028 struct task_struct *t = current; 1965 struct task_struct *t = current;
2029 struct uprobe_task *utask = t->utask; 1966 struct uprobe_task *utask = t->utask;
2030 1967
2031 if (likely(!utask || !utask->active_u 1968 if (likely(!utask || !utask->active_uprobe))
2032 return false; 1969 return false;
2033 1970
2034 WARN_ON_ONCE(utask->state != UTASK_SS 1971 WARN_ON_ONCE(utask->state != UTASK_SSTEP);
2035 1972
2036 if (task_sigpending(t)) { 1973 if (task_sigpending(t)) {
2037 spin_lock_irq(&t->sighand->si 1974 spin_lock_irq(&t->sighand->siglock);
2038 clear_tsk_thread_flag(t, TIF_ 1975 clear_tsk_thread_flag(t, TIF_SIGPENDING);
2039 spin_unlock_irq(&t->sighand-> 1976 spin_unlock_irq(&t->sighand->siglock);
2040 1977
2041 if (__fatal_signal_pending(t) 1978 if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
2042 utask->state = UTASK_ 1979 utask->state = UTASK_SSTEP_TRAPPED;
2043 set_tsk_thread_flag(t 1980 set_tsk_thread_flag(t, TIF_UPROBE);
2044 } 1981 }
2045 } 1982 }
2046 1983
2047 return true; 1984 return true;
2048 } 1985 }
2049 1986
2050 static void mmf_recalc_uprobes(struct mm_stru 1987 static void mmf_recalc_uprobes(struct mm_struct *mm)
2051 { 1988 {
2052 VMA_ITERATOR(vmi, mm, 0); 1989 VMA_ITERATOR(vmi, mm, 0);
2053 struct vm_area_struct *vma; 1990 struct vm_area_struct *vma;
2054 1991
2055 for_each_vma(vmi, vma) { 1992 for_each_vma(vmi, vma) {
2056 if (!valid_vma(vma, false)) 1993 if (!valid_vma(vma, false))
2057 continue; 1994 continue;
2058 /* 1995 /*
2059 * This is not strictly accur 1996 * This is not strictly accurate, we can race with
2060 * uprobe_unregister() and se 1997 * uprobe_unregister() and see the already removed
2061 * uprobe if delete_uprobe() 1998 * uprobe if delete_uprobe() was not yet called.
2062 * Or this uprobe can be filt 1999 * Or this uprobe can be filtered out.
2063 */ 2000 */
2064 if (vma_has_uprobes(vma, vma- 2001 if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
2065 return; 2002 return;
2066 } 2003 }
2067 2004
2068 clear_bit(MMF_HAS_UPROBES, &mm->flags 2005 clear_bit(MMF_HAS_UPROBES, &mm->flags);
2069 } 2006 }
2070 2007
2071 static int is_trap_at_addr(struct mm_struct * 2008 static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
2072 { 2009 {
2073 struct page *page; 2010 struct page *page;
2074 uprobe_opcode_t opcode; 2011 uprobe_opcode_t opcode;
2075 int result; 2012 int result;
2076 2013
2077 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, U 2014 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE)))
2078 return -EINVAL; 2015 return -EINVAL;
2079 2016
2080 pagefault_disable(); 2017 pagefault_disable();
2081 result = __get_user(opcode, (uprobe_o 2018 result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
2082 pagefault_enable(); 2019 pagefault_enable();
2083 2020
2084 if (likely(result == 0)) 2021 if (likely(result == 0))
2085 goto out; 2022 goto out;
2086 2023
2087 result = get_user_pages(vaddr, 1, FOL !! 2024 /*
>> 2025 * The NULL 'tsk' here ensures that any faults that occur here
>> 2026 * will not be accounted to the task. 'mm' *is* current->mm,
>> 2027 * but we treat this as a 'remote' access since it is
>> 2028 * essentially a kernel access to the memory.
>> 2029 */
>> 2030 result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page,
>> 2031 NULL, NULL);
2088 if (result < 0) 2032 if (result < 0)
2089 return result; 2033 return result;
2090 2034
2091 copy_from_page(page, vaddr, &opcode, 2035 copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
2092 put_page(page); 2036 put_page(page);
2093 out: 2037 out:
2094 /* This needs to return true for any 2038 /* This needs to return true for any variant of the trap insn */
2095 return is_trap_insn(&opcode); 2039 return is_trap_insn(&opcode);
2096 } 2040 }
2097 2041
2098 /* assumes being inside RCU protected region !! 2042 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
2099 static struct uprobe *find_active_uprobe_rcu( <<
2100 { 2043 {
2101 struct mm_struct *mm = current->mm; 2044 struct mm_struct *mm = current->mm;
2102 struct uprobe *uprobe = NULL; 2045 struct uprobe *uprobe = NULL;
2103 struct vm_area_struct *vma; 2046 struct vm_area_struct *vma;
2104 2047
2105 mmap_read_lock(mm); 2048 mmap_read_lock(mm);
2106 vma = vma_lookup(mm, bp_vaddr); 2049 vma = vma_lookup(mm, bp_vaddr);
2107 if (vma) { 2050 if (vma) {
2108 if (valid_vma(vma, false)) { 2051 if (valid_vma(vma, false)) {
2109 struct inode *inode = 2052 struct inode *inode = file_inode(vma->vm_file);
2110 loff_t offset = vaddr 2053 loff_t offset = vaddr_to_offset(vma, bp_vaddr);
2111 2054
2112 uprobe = find_uprobe_ !! 2055 uprobe = find_uprobe(inode, offset);
2113 } 2056 }
2114 2057
2115 if (!uprobe) 2058 if (!uprobe)
2116 *is_swbp = is_trap_at 2059 *is_swbp = is_trap_at_addr(mm, bp_vaddr);
2117 } else { 2060 } else {
2118 *is_swbp = -EFAULT; 2061 *is_swbp = -EFAULT;
2119 } 2062 }
2120 2063
2121 if (!uprobe && test_and_clear_bit(MMF 2064 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
2122 mmf_recalc_uprobes(mm); 2065 mmf_recalc_uprobes(mm);
2123 mmap_read_unlock(mm); 2066 mmap_read_unlock(mm);
2124 2067
2125 return uprobe; 2068 return uprobe;
2126 } 2069 }
2127 2070
2128 static void handler_chain(struct uprobe *upro 2071 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
2129 { 2072 {
2130 struct uprobe_consumer *uc; 2073 struct uprobe_consumer *uc;
2131 int remove = UPROBE_HANDLER_REMOVE; 2074 int remove = UPROBE_HANDLER_REMOVE;
2132 bool need_prep = false; /* prepare re 2075 bool need_prep = false; /* prepare return uprobe, when needed */
2133 bool has_consumers = false; <<
2134 <<
2135 current->utask->auprobe = &uprobe->ar <<
2136 2076
2137 list_for_each_entry_srcu(uc, &uprobe- !! 2077 down_read(&uprobe->register_rwsem);
2138 srcu_read_lo !! 2078 for (uc = uprobe->consumers; uc; uc = uc->next) {
2139 int rc = 0; 2079 int rc = 0;
2140 2080
2141 if (uc->handler) { 2081 if (uc->handler) {
2142 rc = uc->handler(uc, 2082 rc = uc->handler(uc, regs);
2143 WARN(rc & ~UPROBE_HAN 2083 WARN(rc & ~UPROBE_HANDLER_MASK,
2144 "bad rc=0x%x 2084 "bad rc=0x%x from %ps()\n", rc, uc->handler);
2145 } 2085 }
2146 2086
2147 if (uc->ret_handler) 2087 if (uc->ret_handler)
2148 need_prep = true; 2088 need_prep = true;
2149 2089
2150 remove &= rc; 2090 remove &= rc;
2151 has_consumers = true; <<
2152 } 2091 }
2153 current->utask->auprobe = NULL; <<
2154 2092
2155 if (need_prep && !remove) 2093 if (need_prep && !remove)
2156 prepare_uretprobe(uprobe, reg 2094 prepare_uretprobe(uprobe, regs); /* put bp at return */
2157 2095
2158 if (remove && has_consumers) { !! 2096 if (remove && uprobe->consumers) {
2159 down_read(&uprobe->register_r !! 2097 WARN_ON(!uprobe_is_active(uprobe));
2160 !! 2098 unapply_uprobe(uprobe, current->mm);
2161 /* re-check that removal is s <<
2162 if (!filter_chain(uprobe, cur <<
2163 WARN_ON(!uprobe_is_ac <<
2164 unapply_uprobe(uprobe <<
2165 } <<
2166 <<
2167 up_read(&uprobe->register_rws <<
2168 } 2099 }
>> 2100 up_read(&uprobe->register_rwsem);
2169 } 2101 }
2170 2102
2171 static void 2103 static void
2172 handle_uretprobe_chain(struct return_instance 2104 handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
2173 { 2105 {
2174 struct uprobe *uprobe = ri->uprobe; 2106 struct uprobe *uprobe = ri->uprobe;
2175 struct uprobe_consumer *uc; 2107 struct uprobe_consumer *uc;
2176 int srcu_idx; <<
2177 2108
2178 srcu_idx = srcu_read_lock(&uprobes_sr !! 2109 down_read(&uprobe->register_rwsem);
2179 list_for_each_entry_srcu(uc, &uprobe- !! 2110 for (uc = uprobe->consumers; uc; uc = uc->next) {
2180 srcu_read_lo <<
2181 if (uc->ret_handler) 2111 if (uc->ret_handler)
2182 uc->ret_handler(uc, r 2112 uc->ret_handler(uc, ri->func, regs);
2183 } 2113 }
2184 srcu_read_unlock(&uprobes_srcu, srcu_ !! 2114 up_read(&uprobe->register_rwsem);
2185 } 2115 }
2186 2116
2187 static struct return_instance *find_next_ret_ 2117 static struct return_instance *find_next_ret_chain(struct return_instance *ri)
2188 { 2118 {
2189 bool chained; 2119 bool chained;
2190 2120
2191 do { 2121 do {
2192 chained = ri->chained; 2122 chained = ri->chained;
2193 ri = ri->next; /* can't be N 2123 ri = ri->next; /* can't be NULL if chained */
2194 } while (chained); 2124 } while (chained);
2195 2125
2196 return ri; 2126 return ri;
2197 } 2127 }
2198 2128
2199 void uprobe_handle_trampoline(struct pt_regs !! 2129 static void handle_trampoline(struct pt_regs *regs)
2200 { 2130 {
2201 struct uprobe_task *utask; 2131 struct uprobe_task *utask;
2202 struct return_instance *ri, *next; 2132 struct return_instance *ri, *next;
2203 bool valid; 2133 bool valid;
2204 2134
2205 utask = current->utask; 2135 utask = current->utask;
2206 if (!utask) 2136 if (!utask)
2207 goto sigill; 2137 goto sigill;
2208 2138
2209 ri = utask->return_instances; 2139 ri = utask->return_instances;
2210 if (!ri) 2140 if (!ri)
2211 goto sigill; 2141 goto sigill;
2212 2142
2213 do { 2143 do {
2214 /* 2144 /*
2215 * We should throw out the fr 2145 * We should throw out the frames invalidated by longjmp().
2216 * If this chain is valid, th 2146 * If this chain is valid, then the next one should be alive
2217 * or NULL; the latter case m 2147 * or NULL; the latter case means that nobody but ri->func
2218 * could hit this trampoline 2148 * could hit this trampoline on return. TODO: sigaltstack().
2219 */ 2149 */
2220 next = find_next_ret_chain(ri 2150 next = find_next_ret_chain(ri);
2221 valid = !next || arch_uretpro 2151 valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
2222 2152
2223 instruction_pointer_set(regs, 2153 instruction_pointer_set(regs, ri->orig_ret_vaddr);
2224 do { 2154 do {
2225 /* pop current instan <<
2226 * as it's not pendin <<
2227 * instruction pointe <<
2228 * this allows fixup_ <<
2229 * captured stack tra <<
2230 * trampoline address <<
2231 * original return ad <<
2232 */ <<
2233 utask->return_instanc <<
2234 if (valid) 2155 if (valid)
2235 handle_uretpr 2156 handle_uretprobe_chain(ri, regs);
2236 ri = free_ret_instanc 2157 ri = free_ret_instance(ri);
2237 utask->depth--; 2158 utask->depth--;
2238 } while (ri != next); 2159 } while (ri != next);
2239 } while (!valid); 2160 } while (!valid);
2240 2161
2241 utask->return_instances = ri; 2162 utask->return_instances = ri;
2242 return; 2163 return;
2243 2164
2244 sigill: 2165 sigill:
2245 uprobe_warn(current, "handle uretprob 2166 uprobe_warn(current, "handle uretprobe, sending SIGILL.");
2246 force_sig(SIGILL); 2167 force_sig(SIGILL);
2247 2168
2248 } 2169 }
2249 2170
2250 bool __weak arch_uprobe_ignore(struct arch_up 2171 bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
2251 { 2172 {
2252 return false; 2173 return false;
2253 } 2174 }
2254 2175
2255 bool __weak arch_uretprobe_is_alive(struct re 2176 bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
2256 struc 2177 struct pt_regs *regs)
2257 { 2178 {
2258 return true; 2179 return true;
2259 } 2180 }
2260 2181
2261 /* 2182 /*
2262 * Run handler and ask thread to singlestep. 2183 * Run handler and ask thread to singlestep.
2263 * Ensure all non-fatal signals cannot interr 2184 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
2264 */ 2185 */
2265 static void handle_swbp(struct pt_regs *regs) 2186 static void handle_swbp(struct pt_regs *regs)
2266 { 2187 {
2267 struct uprobe *uprobe; 2188 struct uprobe *uprobe;
2268 unsigned long bp_vaddr; 2189 unsigned long bp_vaddr;
2269 int is_swbp, srcu_idx; !! 2190 int is_swbp;
2270 2191
2271 bp_vaddr = uprobe_get_swbp_addr(regs) 2192 bp_vaddr = uprobe_get_swbp_addr(regs);
2272 if (bp_vaddr == uprobe_get_trampoline !! 2193 if (bp_vaddr == get_trampoline_vaddr())
2273 return uprobe_handle_trampoli !! 2194 return handle_trampoline(regs);
2274 2195
2275 srcu_idx = srcu_read_lock(&uprobes_sr !! 2196 uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
2276 <<
2277 uprobe = find_active_uprobe_rcu(bp_va <<
2278 if (!uprobe) { 2197 if (!uprobe) {
2279 if (is_swbp > 0) { 2198 if (is_swbp > 0) {
2280 /* No matching uprobe 2199 /* No matching uprobe; signal SIGTRAP. */
2281 force_sig(SIGTRAP); 2200 force_sig(SIGTRAP);
2282 } else { 2201 } else {
2283 /* 2202 /*
2284 * Either we raced wi 2203 * Either we raced with uprobe_unregister() or we can't
2285 * access this memory 2204 * access this memory. The latter is only possible if
2286 * another thread pla 2205 * another thread plays with our ->mm. In both cases
2287 * we can simply rest 2206 * we can simply restart. If this vma was unmapped we
2288 * can pretend this i 2207 * can pretend this insn was not executed yet and get
2289 * the (correct) SIGS 2208 * the (correct) SIGSEGV after restart.
2290 */ 2209 */
2291 instruction_pointer_s 2210 instruction_pointer_set(regs, bp_vaddr);
2292 } 2211 }
2293 goto out; !! 2212 return;
2294 } 2213 }
2295 2214
2296 /* change it in advance for ->handler 2215 /* change it in advance for ->handler() and restart */
2297 instruction_pointer_set(regs, bp_vadd 2216 instruction_pointer_set(regs, bp_vaddr);
2298 2217
2299 /* 2218 /*
2300 * TODO: move copy_insn/etc into _reg 2219 * TODO: move copy_insn/etc into _register and remove this hack.
2301 * After we hit the bp, _unregister + 2220 * After we hit the bp, _unregister + _register can install the
2302 * new and not-yet-analyzed uprobe at 2221 * new and not-yet-analyzed uprobe at the same address, restart.
2303 */ 2222 */
2304 if (unlikely(!test_bit(UPROBE_COPY_IN 2223 if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
2305 goto out; 2224 goto out;
2306 2225
2307 /* 2226 /*
2308 * Pairs with the smp_wmb() in prepar 2227 * Pairs with the smp_wmb() in prepare_uprobe().
2309 * 2228 *
2310 * Guarantees that if we see the UPRO 2229 * Guarantees that if we see the UPROBE_COPY_INSN bit set, then
2311 * we must also see the stores to &up 2230 * we must also see the stores to &uprobe->arch performed by the
2312 * prepare_uprobe() call. 2231 * prepare_uprobe() call.
2313 */ 2232 */
2314 smp_rmb(); 2233 smp_rmb();
2315 2234
2316 /* Tracing handlers use ->utask to co 2235 /* Tracing handlers use ->utask to communicate with fetch methods */
2317 if (!get_utask()) 2236 if (!get_utask())
2318 goto out; 2237 goto out;
2319 2238
2320 if (arch_uprobe_ignore(&uprobe->arch, 2239 if (arch_uprobe_ignore(&uprobe->arch, regs))
2321 goto out; 2240 goto out;
2322 2241
2323 handler_chain(uprobe, regs); 2242 handler_chain(uprobe, regs);
2324 2243
2325 if (arch_uprobe_skip_sstep(&uprobe->a 2244 if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
2326 goto out; 2245 goto out;
2327 2246
2328 if (pre_ssout(uprobe, regs, bp_vaddr) !! 2247 if (!pre_ssout(uprobe, regs, bp_vaddr))
2329 goto out; !! 2248 return;
2330 2249
2331 out: <<
2332 /* arch_uprobe_skip_sstep() succeeded 2250 /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
2333 srcu_read_unlock(&uprobes_srcu, srcu_ !! 2251 out:
>> 2252 put_uprobe(uprobe);
2334 } 2253 }
2335 2254
2336 /* 2255 /*
2337 * Perform required fix-ups and disable singl 2256 * Perform required fix-ups and disable singlestep.
2338 * Allow pending signals to take effect. 2257 * Allow pending signals to take effect.
2339 */ 2258 */
2340 static void handle_singlestep(struct uprobe_t 2259 static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
2341 { 2260 {
2342 struct uprobe *uprobe; 2261 struct uprobe *uprobe;
2343 int err = 0; 2262 int err = 0;
2344 2263
2345 uprobe = utask->active_uprobe; 2264 uprobe = utask->active_uprobe;
2346 if (utask->state == UTASK_SSTEP_ACK) 2265 if (utask->state == UTASK_SSTEP_ACK)
2347 err = arch_uprobe_post_xol(&u 2266 err = arch_uprobe_post_xol(&uprobe->arch, regs);
2348 else if (utask->state == UTASK_SSTEP_ 2267 else if (utask->state == UTASK_SSTEP_TRAPPED)
2349 arch_uprobe_abort_xol(&uprobe 2268 arch_uprobe_abort_xol(&uprobe->arch, regs);
2350 else 2269 else
2351 WARN_ON_ONCE(1); 2270 WARN_ON_ONCE(1);
2352 2271
2353 put_uprobe(uprobe); 2272 put_uprobe(uprobe);
2354 utask->active_uprobe = NULL; 2273 utask->active_uprobe = NULL;
2355 utask->state = UTASK_RUNNING; 2274 utask->state = UTASK_RUNNING;
2356 xol_free_insn_slot(current); 2275 xol_free_insn_slot(current);
2357 2276
2358 spin_lock_irq(¤t->sighand->sigl 2277 spin_lock_irq(¤t->sighand->siglock);
2359 recalc_sigpending(); /* see uprobe_de 2278 recalc_sigpending(); /* see uprobe_deny_signal() */
2360 spin_unlock_irq(¤t->sighand->si 2279 spin_unlock_irq(¤t->sighand->siglock);
2361 2280
2362 if (unlikely(err)) { 2281 if (unlikely(err)) {
2363 uprobe_warn(current, "execute 2282 uprobe_warn(current, "execute the probed insn, sending SIGILL.");
2364 force_sig(SIGILL); 2283 force_sig(SIGILL);
2365 } 2284 }
2366 } 2285 }
2367 2286
2368 /* 2287 /*
2369 * On breakpoint hit, breakpoint notifier set 2288 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
2370 * allows the thread to return from interrupt 2289 * allows the thread to return from interrupt. After that handle_swbp()
2371 * sets utask->active_uprobe. 2290 * sets utask->active_uprobe.
2372 * 2291 *
2373 * On singlestep exception, singlestep notifi 2292 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
2374 * and allows the thread to return from inter 2293 * and allows the thread to return from interrupt.
2375 * 2294 *
2376 * While returning to userspace, thread notic 2295 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
2377 * uprobe_notify_resume(). 2296 * uprobe_notify_resume().
2378 */ 2297 */
2379 void uprobe_notify_resume(struct pt_regs *reg 2298 void uprobe_notify_resume(struct pt_regs *regs)
2380 { 2299 {
2381 struct uprobe_task *utask; 2300 struct uprobe_task *utask;
2382 2301
2383 clear_thread_flag(TIF_UPROBE); 2302 clear_thread_flag(TIF_UPROBE);
2384 2303
2385 utask = current->utask; 2304 utask = current->utask;
2386 if (utask && utask->active_uprobe) 2305 if (utask && utask->active_uprobe)
2387 handle_singlestep(utask, regs 2306 handle_singlestep(utask, regs);
2388 else 2307 else
2389 handle_swbp(regs); 2308 handle_swbp(regs);
2390 } 2309 }
2391 2310
2392 /* 2311 /*
2393 * uprobe_pre_sstep_notifier gets called from 2312 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
2394 * notifier mechanism. Set TIF_UPROBE flag an 2313 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
2395 */ 2314 */
2396 int uprobe_pre_sstep_notifier(struct pt_regs 2315 int uprobe_pre_sstep_notifier(struct pt_regs *regs)
2397 { 2316 {
2398 if (!current->mm) 2317 if (!current->mm)
2399 return 0; 2318 return 0;
2400 2319
2401 if (!test_bit(MMF_HAS_UPROBES, &curre 2320 if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) &&
2402 (!current->utask || !current->uta 2321 (!current->utask || !current->utask->return_instances))
2403 return 0; 2322 return 0;
2404 2323
2405 set_thread_flag(TIF_UPROBE); 2324 set_thread_flag(TIF_UPROBE);
2406 return 1; 2325 return 1;
2407 } 2326 }
2408 2327
2409 /* 2328 /*
2410 * uprobe_post_sstep_notifier gets called in 2329 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
2411 * mechanism. Set TIF_UPROBE flag and indicat 2330 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
2412 */ 2331 */
2413 int uprobe_post_sstep_notifier(struct pt_regs 2332 int uprobe_post_sstep_notifier(struct pt_regs *regs)
2414 { 2333 {
2415 struct uprobe_task *utask = current-> 2334 struct uprobe_task *utask = current->utask;
2416 2335
2417 if (!current->mm || !utask || !utask- 2336 if (!current->mm || !utask || !utask->active_uprobe)
2418 /* task is currently not upro 2337 /* task is currently not uprobed */
2419 return 0; 2338 return 0;
2420 2339
2421 utask->state = UTASK_SSTEP_ACK; 2340 utask->state = UTASK_SSTEP_ACK;
2422 set_thread_flag(TIF_UPROBE); 2341 set_thread_flag(TIF_UPROBE);
2423 return 1; 2342 return 1;
2424 } 2343 }
2425 2344
2426 static struct notifier_block uprobe_exception 2345 static struct notifier_block uprobe_exception_nb = {
2427 .notifier_call = arch_uprobe 2346 .notifier_call = arch_uprobe_exception_notify,
2428 .priority = INT_MAX-1, 2347 .priority = INT_MAX-1, /* notified after kprobes, kgdb */
2429 }; 2348 };
2430 2349
2431 void __init uprobes_init(void) 2350 void __init uprobes_init(void)
2432 { 2351 {
2433 int i; 2352 int i;
2434 2353
2435 for (i = 0; i < UPROBES_HASH_SZ; i++) 2354 for (i = 0; i < UPROBES_HASH_SZ; i++)
2436 mutex_init(&uprobes_mmap_mute 2355 mutex_init(&uprobes_mmap_mutex[i]);
2437 2356
2438 BUG_ON(register_die_notifier(&uprobe_ 2357 BUG_ON(register_die_notifier(&uprobe_exception_nb));
2439 } 2358 }
2440 2359
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.