1 // SPDX-License-Identifier: GPL-2.0+ 1 // SPDX-License-Identifier: GPL-2.0+
2 /* 2 /*
3 * User-space Probes (UProbes) 3 * User-space Probes (UProbes)
4 * 4 *
5 * Copyright (C) IBM Corporation, 2008-2012 5 * Copyright (C) IBM Corporation, 2008-2012
6 * Authors: 6 * Authors:
7 * Srikar Dronamraju 7 * Srikar Dronamraju
8 * Jim Keniston 8 * Jim Keniston
9 * Copyright (C) 2011-2012 Red Hat, Inc., Pete 9 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
10 */ 10 */
11 11
12 #include <linux/kernel.h> 12 #include <linux/kernel.h>
13 #include <linux/highmem.h> 13 #include <linux/highmem.h>
14 #include <linux/pagemap.h> /* read_mappin 14 #include <linux/pagemap.h> /* read_mapping_page */
15 #include <linux/slab.h> 15 #include <linux/slab.h>
16 #include <linux/sched.h> 16 #include <linux/sched.h>
17 #include <linux/sched/mm.h> 17 #include <linux/sched/mm.h>
18 #include <linux/sched/coredump.h> 18 #include <linux/sched/coredump.h>
19 #include <linux/export.h> 19 #include <linux/export.h>
20 #include <linux/rmap.h> /* anon_vma_pr 20 #include <linux/rmap.h> /* anon_vma_prepare */
21 #include <linux/mmu_notifier.h> !! 21 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
22 #include <linux/swap.h> /* folio_free_ !! 22 #include <linux/swap.h> /* try_to_free_swap */
23 #include <linux/ptrace.h> /* user_enable 23 #include <linux/ptrace.h> /* user_enable_single_step */
24 #include <linux/kdebug.h> /* notifier me 24 #include <linux/kdebug.h> /* notifier mechanism */
>> 25 #include "../../mm/internal.h" /* munlock_vma_page */
25 #include <linux/percpu-rwsem.h> 26 #include <linux/percpu-rwsem.h>
26 #include <linux/task_work.h> 27 #include <linux/task_work.h>
27 #include <linux/shmem_fs.h> 28 #include <linux/shmem_fs.h>
28 #include <linux/khugepaged.h> 29 #include <linux/khugepaged.h>
29 30
30 #include <linux/uprobes.h> 31 #include <linux/uprobes.h>
31 32
32 #define UINSNS_PER_PAGE (PAGE_ 33 #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
33 #define MAX_UPROBE_XOL_SLOTS UINSNS 34 #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
34 35
35 static struct rb_root uprobes_tree = RB_ROOT; 36 static struct rb_root uprobes_tree = RB_ROOT;
36 /* 37 /*
37 * allows us to skip the uprobe_mmap if there 38 * allows us to skip the uprobe_mmap if there are no uprobe events active
38 * at this time. Probably a fine grained per 39 * at this time. Probably a fine grained per inode count is better?
39 */ 40 */
40 #define no_uprobe_events() RB_EMPTY_ROOT( 41 #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree)
41 42
42 static DEFINE_RWLOCK(uprobes_treelock); /* ser !! 43 static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
43 static seqcount_rwlock_t uprobes_seqcount = SE <<
44 <<
45 DEFINE_STATIC_SRCU(uprobes_srcu); <<
46 44
47 #define UPROBES_HASH_SZ 13 45 #define UPROBES_HASH_SZ 13
48 /* serialize uprobe->pending_list */ 46 /* serialize uprobe->pending_list */
49 static struct mutex uprobes_mmap_mutex[UPROBES 47 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
50 #define uprobes_mmap_hash(v) (&uprobes_mmap 48 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
51 49
52 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); 50 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
53 51
54 /* Have a copy of original instruction */ 52 /* Have a copy of original instruction */
55 #define UPROBE_COPY_INSN 0 53 #define UPROBE_COPY_INSN 0
56 54
57 struct uprobe { 55 struct uprobe {
58 struct rb_node rb_node; 56 struct rb_node rb_node; /* node in the rb tree */
59 refcount_t ref; 57 refcount_t ref;
60 struct rw_semaphore register_rwsem 58 struct rw_semaphore register_rwsem;
61 struct rw_semaphore consumer_rwsem 59 struct rw_semaphore consumer_rwsem;
62 struct list_head pending_list; 60 struct list_head pending_list;
63 struct list_head consumers; !! 61 struct uprobe_consumer *consumers;
64 struct inode *inode; 62 struct inode *inode; /* Also hold a ref to inode */
65 struct rcu_head rcu; <<
66 loff_t offset; 63 loff_t offset;
67 loff_t ref_ctr_offset 64 loff_t ref_ctr_offset;
68 unsigned long flags; 65 unsigned long flags;
69 66
70 /* 67 /*
71 * The generic code assumes that it ha 68 * The generic code assumes that it has two members of unknown type
72 * owned by the arch-specific code: 69 * owned by the arch-specific code:
73 * 70 *
74 * insn - copy_insn() saves the 71 * insn - copy_insn() saves the original instruction here for
75 * arch_uprobe_analyze_in 72 * arch_uprobe_analyze_insn().
76 * 73 *
77 * ixol - potentially modified i 74 * ixol - potentially modified instruction to execute out of
78 * line, copied to xol_ar 75 * line, copied to xol_area by xol_get_insn_slot().
79 */ 76 */
80 struct arch_uprobe arch; 77 struct arch_uprobe arch;
81 }; 78 };
82 79
83 struct delayed_uprobe { 80 struct delayed_uprobe {
84 struct list_head list; 81 struct list_head list;
85 struct uprobe *uprobe; 82 struct uprobe *uprobe;
86 struct mm_struct *mm; 83 struct mm_struct *mm;
87 }; 84 };
88 85
89 static DEFINE_MUTEX(delayed_uprobe_lock); 86 static DEFINE_MUTEX(delayed_uprobe_lock);
90 static LIST_HEAD(delayed_uprobe_list); 87 static LIST_HEAD(delayed_uprobe_list);
91 88
92 /* 89 /*
93 * Execute out of line area: anonymous executa 90 * Execute out of line area: anonymous executable mapping installed
94 * by the probed task to execute the copy of t 91 * by the probed task to execute the copy of the original instruction
95 * mangled by set_swbp(). 92 * mangled by set_swbp().
96 * 93 *
97 * On a breakpoint hit, thread contests for a 94 * On a breakpoint hit, thread contests for a slot. It frees the
98 * slot after singlestep. Currently a fixed nu 95 * slot after singlestep. Currently a fixed number of slots are
99 * allocated. 96 * allocated.
100 */ 97 */
101 struct xol_area { 98 struct xol_area {
102 wait_queue_head_t wq; 99 wait_queue_head_t wq; /* if all slots are busy */
103 atomic_t slot_c 100 atomic_t slot_count; /* number of in-use slots */
104 unsigned long *bitma 101 unsigned long *bitmap; /* 0 = free slot */
105 102
106 struct page *page; !! 103 struct vm_special_mapping xol_mapping;
>> 104 struct page *pages[2];
107 /* 105 /*
108 * We keep the vma's vm_start rather t 106 * We keep the vma's vm_start rather than a pointer to the vma
109 * itself. The probed process or a na 107 * itself. The probed process or a naughty kernel module could make
110 * the vma go away, and we must handle 108 * the vma go away, and we must handle that reasonably gracefully.
111 */ 109 */
112 unsigned long vaddr; 110 unsigned long vaddr; /* Page(s) of instruction slots */
113 }; 111 };
114 112
115 static void uprobe_warn(struct task_struct *t, <<
116 { <<
117 pr_warn("uprobe: %s:%d failed to %s\n" <<
118 } <<
119 <<
120 /* 113 /*
121 * valid_vma: Verify if the specified vma is a 114 * valid_vma: Verify if the specified vma is an executable vma
122 * Relax restrictions while unregistering: vm_ 115 * Relax restrictions while unregistering: vm_flags might have
123 * changed after breakpoint was inserted. 116 * changed after breakpoint was inserted.
124 * - is_register: indicates if we are in 117 * - is_register: indicates if we are in register context.
125 * - Return 1 if the specified virtual ad 118 * - Return 1 if the specified virtual address is in an
126 * executable vma. 119 * executable vma.
127 */ 120 */
128 static bool valid_vma(struct vm_area_struct *v 121 static bool valid_vma(struct vm_area_struct *vma, bool is_register)
129 { 122 {
130 vm_flags_t flags = VM_HUGETLB | VM_MAY 123 vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
131 124
132 if (is_register) 125 if (is_register)
133 flags |= VM_WRITE; 126 flags |= VM_WRITE;
134 127
135 return vma->vm_file && (vma->vm_flags 128 return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
136 } 129 }
137 130
138 static unsigned long offset_to_vaddr(struct vm 131 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
139 { 132 {
140 return vma->vm_start + offset - ((loff 133 return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
141 } 134 }
142 135
143 static loff_t vaddr_to_offset(struct vm_area_s 136 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
144 { 137 {
145 return ((loff_t)vma->vm_pgoff << PAGE_ 138 return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
146 } 139 }
147 140
148 /** 141 /**
149 * __replace_page - replace page in vma by new 142 * __replace_page - replace page in vma by new page.
150 * based on replace_page in mm/ksm.c 143 * based on replace_page in mm/ksm.c
151 * 144 *
152 * @vma: vma that holds the pte pointing 145 * @vma: vma that holds the pte pointing to page
153 * @addr: address the old @page is mapped 146 * @addr: address the old @page is mapped at
154 * @old_page: the page we are replacing by new 147 * @old_page: the page we are replacing by new_page
155 * @new_page: the modified page we replace pag 148 * @new_page: the modified page we replace page by
156 * 149 *
157 * If @new_page is NULL, only unmap @old_page. 150 * If @new_page is NULL, only unmap @old_page.
158 * 151 *
159 * Returns 0 on success, negative error code o 152 * Returns 0 on success, negative error code otherwise.
160 */ 153 */
161 static int __replace_page(struct vm_area_struc 154 static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
162 struct page *o 155 struct page *old_page, struct page *new_page)
163 { 156 {
164 struct folio *old_folio = page_folio(o <<
165 struct folio *new_folio; <<
166 struct mm_struct *mm = vma->vm_mm; 157 struct mm_struct *mm = vma->vm_mm;
167 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, !! 158 struct page_vma_mapped_walk pvmw = {
>> 159 .page = compound_head(old_page),
>> 160 .vma = vma,
>> 161 .address = addr,
>> 162 };
168 int err; 163 int err;
169 struct mmu_notifier_range range; 164 struct mmu_notifier_range range;
170 165
171 mmu_notifier_range_init(&range, MMU_NO !! 166 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
172 addr + PAGE_SI 167 addr + PAGE_SIZE);
173 168
174 if (new_page) { 169 if (new_page) {
175 new_folio = page_folio(new_pag !! 170 err = mem_cgroup_charge(new_page, vma->vm_mm, GFP_KERNEL);
176 err = mem_cgroup_charge(new_fo <<
177 if (err) 171 if (err)
178 return err; 172 return err;
179 } 173 }
180 174
181 /* For folio_free_swap() below */ !! 175 /* For try_to_free_swap() and munlock_vma_page() below */
182 folio_lock(old_folio); !! 176 lock_page(old_page);
183 177
184 mmu_notifier_invalidate_range_start(&r 178 mmu_notifier_invalidate_range_start(&range);
185 err = -EAGAIN; 179 err = -EAGAIN;
186 if (!page_vma_mapped_walk(&pvmw)) 180 if (!page_vma_mapped_walk(&pvmw))
187 goto unlock; 181 goto unlock;
188 VM_BUG_ON_PAGE(addr != pvmw.address, o 182 VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
189 183
190 if (new_page) { 184 if (new_page) {
191 folio_get(new_folio); !! 185 get_page(new_page);
192 folio_add_new_anon_rmap(new_fo !! 186 page_add_new_anon_rmap(new_page, vma, addr, false);
193 folio_add_lru_vma(new_folio, v !! 187 lru_cache_add_inactive_or_unevictable(new_page, vma);
194 } else 188 } else
195 /* no new page, just dec_mm_co 189 /* no new page, just dec_mm_counter for old_page */
196 dec_mm_counter(mm, MM_ANONPAGE 190 dec_mm_counter(mm, MM_ANONPAGES);
197 191
198 if (!folio_test_anon(old_folio)) { !! 192 if (!PageAnon(old_page)) {
199 dec_mm_counter(mm, mm_counter_ !! 193 dec_mm_counter(mm, mm_counter_file(old_page));
200 inc_mm_counter(mm, MM_ANONPAGE 194 inc_mm_counter(mm, MM_ANONPAGES);
201 } 195 }
202 196
203 flush_cache_page(vma, addr, pte_pfn(pt !! 197 flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
204 ptep_clear_flush(vma, addr, pvmw.pte); !! 198 ptep_clear_flush_notify(vma, addr, pvmw.pte);
205 if (new_page) 199 if (new_page)
206 set_pte_at(mm, addr, pvmw.pte, !! 200 set_pte_at_notify(mm, addr, pvmw.pte,
207 mk_pte(new_page, vm !! 201 mk_pte(new_page, vma->vm_page_prot));
208 202
209 folio_remove_rmap_pte(old_folio, old_p !! 203 page_remove_rmap(old_page, false);
210 if (!folio_mapped(old_folio)) !! 204 if (!page_mapped(old_page))
211 folio_free_swap(old_folio); !! 205 try_to_free_swap(old_page);
212 page_vma_mapped_walk_done(&pvmw); 206 page_vma_mapped_walk_done(&pvmw);
213 folio_put(old_folio); !! 207
>> 208 if ((vma->vm_flags & VM_LOCKED) && !PageCompound(old_page))
>> 209 munlock_vma_page(old_page);
>> 210 put_page(old_page);
214 211
215 err = 0; 212 err = 0;
216 unlock: 213 unlock:
217 mmu_notifier_invalidate_range_end(&ran 214 mmu_notifier_invalidate_range_end(&range);
218 folio_unlock(old_folio); !! 215 unlock_page(old_page);
219 return err; 216 return err;
220 } 217 }
221 218
222 /** 219 /**
223 * is_swbp_insn - check if instruction is brea 220 * is_swbp_insn - check if instruction is breakpoint instruction.
224 * @insn: instruction to be checked. 221 * @insn: instruction to be checked.
225 * Default implementation of is_swbp_insn 222 * Default implementation of is_swbp_insn
226 * Returns true if @insn is a breakpoint instr 223 * Returns true if @insn is a breakpoint instruction.
227 */ 224 */
228 bool __weak is_swbp_insn(uprobe_opcode_t *insn 225 bool __weak is_swbp_insn(uprobe_opcode_t *insn)
229 { 226 {
230 return *insn == UPROBE_SWBP_INSN; 227 return *insn == UPROBE_SWBP_INSN;
231 } 228 }
232 229
233 /** 230 /**
234 * is_trap_insn - check if instruction is brea 231 * is_trap_insn - check if instruction is breakpoint instruction.
235 * @insn: instruction to be checked. 232 * @insn: instruction to be checked.
236 * Default implementation of is_trap_insn 233 * Default implementation of is_trap_insn
237 * Returns true if @insn is a breakpoint instr 234 * Returns true if @insn is a breakpoint instruction.
238 * 235 *
239 * This function is needed for the case where 236 * This function is needed for the case where an architecture has multiple
240 * trap instructions (like powerpc). 237 * trap instructions (like powerpc).
241 */ 238 */
242 bool __weak is_trap_insn(uprobe_opcode_t *insn 239 bool __weak is_trap_insn(uprobe_opcode_t *insn)
243 { 240 {
244 return is_swbp_insn(insn); 241 return is_swbp_insn(insn);
245 } 242 }
246 243
247 static void copy_from_page(struct page *page, 244 static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
248 { 245 {
249 void *kaddr = kmap_atomic(page); 246 void *kaddr = kmap_atomic(page);
250 memcpy(dst, kaddr + (vaddr & ~PAGE_MAS 247 memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
251 kunmap_atomic(kaddr); 248 kunmap_atomic(kaddr);
252 } 249 }
253 250
254 static void copy_to_page(struct page *page, un 251 static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
255 { 252 {
256 void *kaddr = kmap_atomic(page); 253 void *kaddr = kmap_atomic(page);
257 memcpy(kaddr + (vaddr & ~PAGE_MASK), s 254 memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
258 kunmap_atomic(kaddr); 255 kunmap_atomic(kaddr);
259 } 256 }
260 257
261 static int verify_opcode(struct page *page, un 258 static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
262 { 259 {
263 uprobe_opcode_t old_opcode; 260 uprobe_opcode_t old_opcode;
264 bool is_swbp; 261 bool is_swbp;
265 262
266 /* 263 /*
267 * Note: We only check if the old_opco 264 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
268 * We do not check if it is any other 265 * We do not check if it is any other 'trap variant' which could
269 * be conditional trap instruction suc 266 * be conditional trap instruction such as the one powerpc supports.
270 * 267 *
271 * The logic is that we do not care if 268 * The logic is that we do not care if the underlying instruction
272 * is a trap variant; uprobes always w 269 * is a trap variant; uprobes always wins over any other (gdb)
273 * breakpoint. 270 * breakpoint.
274 */ 271 */
275 copy_from_page(page, vaddr, &old_opcod 272 copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
276 is_swbp = is_swbp_insn(&old_opcode); 273 is_swbp = is_swbp_insn(&old_opcode);
277 274
278 if (is_swbp_insn(new_opcode)) { 275 if (is_swbp_insn(new_opcode)) {
279 if (is_swbp) /* reg 276 if (is_swbp) /* register: already installed? */
280 return 0; 277 return 0;
281 } else { 278 } else {
282 if (!is_swbp) /* unr 279 if (!is_swbp) /* unregister: was it changed by us? */
283 return 0; 280 return 0;
284 } 281 }
285 282
286 return 1; 283 return 1;
287 } 284 }
288 285
289 static struct delayed_uprobe * 286 static struct delayed_uprobe *
290 delayed_uprobe_check(struct uprobe *uprobe, st 287 delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
291 { 288 {
292 struct delayed_uprobe *du; 289 struct delayed_uprobe *du;
293 290
294 list_for_each_entry(du, &delayed_uprob 291 list_for_each_entry(du, &delayed_uprobe_list, list)
295 if (du->uprobe == uprobe && du 292 if (du->uprobe == uprobe && du->mm == mm)
296 return du; 293 return du;
297 return NULL; 294 return NULL;
298 } 295 }
299 296
300 static int delayed_uprobe_add(struct uprobe *u 297 static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
301 { 298 {
302 struct delayed_uprobe *du; 299 struct delayed_uprobe *du;
303 300
304 if (delayed_uprobe_check(uprobe, mm)) 301 if (delayed_uprobe_check(uprobe, mm))
305 return 0; 302 return 0;
306 303
307 du = kzalloc(sizeof(*du), GFP_KERNEL) 304 du = kzalloc(sizeof(*du), GFP_KERNEL);
308 if (!du) 305 if (!du)
309 return -ENOMEM; 306 return -ENOMEM;
310 307
311 du->uprobe = uprobe; 308 du->uprobe = uprobe;
312 du->mm = mm; 309 du->mm = mm;
313 list_add(&du->list, &delayed_uprobe_li 310 list_add(&du->list, &delayed_uprobe_list);
314 return 0; 311 return 0;
315 } 312 }
316 313
317 static void delayed_uprobe_delete(struct delay 314 static void delayed_uprobe_delete(struct delayed_uprobe *du)
318 { 315 {
319 if (WARN_ON(!du)) 316 if (WARN_ON(!du))
320 return; 317 return;
321 list_del(&du->list); 318 list_del(&du->list);
322 kfree(du); 319 kfree(du);
323 } 320 }
324 321
325 static void delayed_uprobe_remove(struct uprob 322 static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
326 { 323 {
327 struct list_head *pos, *q; 324 struct list_head *pos, *q;
328 struct delayed_uprobe *du; 325 struct delayed_uprobe *du;
329 326
330 if (!uprobe && !mm) 327 if (!uprobe && !mm)
331 return; 328 return;
332 329
333 list_for_each_safe(pos, q, &delayed_up 330 list_for_each_safe(pos, q, &delayed_uprobe_list) {
334 du = list_entry(pos, struct de 331 du = list_entry(pos, struct delayed_uprobe, list);
335 332
336 if (uprobe && du->uprobe != up 333 if (uprobe && du->uprobe != uprobe)
337 continue; 334 continue;
338 if (mm && du->mm != mm) 335 if (mm && du->mm != mm)
339 continue; 336 continue;
340 337
341 delayed_uprobe_delete(du); 338 delayed_uprobe_delete(du);
342 } 339 }
343 } 340 }
344 341
345 static bool valid_ref_ctr_vma(struct uprobe *u 342 static bool valid_ref_ctr_vma(struct uprobe *uprobe,
346 struct vm_area_s 343 struct vm_area_struct *vma)
347 { 344 {
348 unsigned long vaddr = offset_to_vaddr( 345 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
349 346
350 return uprobe->ref_ctr_offset && 347 return uprobe->ref_ctr_offset &&
351 vma->vm_file && 348 vma->vm_file &&
352 file_inode(vma->vm_file) == up 349 file_inode(vma->vm_file) == uprobe->inode &&
353 (vma->vm_flags & (VM_WRITE|VM_ 350 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
354 vma->vm_start <= vaddr && 351 vma->vm_start <= vaddr &&
355 vma->vm_end > vaddr; 352 vma->vm_end > vaddr;
356 } 353 }
357 354
358 static struct vm_area_struct * 355 static struct vm_area_struct *
359 find_ref_ctr_vma(struct uprobe *uprobe, struct 356 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
360 { 357 {
361 VMA_ITERATOR(vmi, mm, 0); <<
362 struct vm_area_struct *tmp; 358 struct vm_area_struct *tmp;
363 359
364 for_each_vma(vmi, tmp) !! 360 for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
365 if (valid_ref_ctr_vma(uprobe, 361 if (valid_ref_ctr_vma(uprobe, tmp))
366 return tmp; 362 return tmp;
367 363
368 return NULL; 364 return NULL;
369 } 365 }
370 366
371 static int 367 static int
372 __update_ref_ctr(struct mm_struct *mm, unsigne 368 __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
373 { 369 {
374 void *kaddr; 370 void *kaddr;
375 struct page *page; 371 struct page *page;
>> 372 struct vm_area_struct *vma;
376 int ret; 373 int ret;
377 short *ptr; 374 short *ptr;
378 375
379 if (!vaddr || !d) 376 if (!vaddr || !d)
380 return -EINVAL; 377 return -EINVAL;
381 378
382 ret = get_user_pages_remote(mm, vaddr, 379 ret = get_user_pages_remote(mm, vaddr, 1,
383 FOLL_WRITE !! 380 FOLL_WRITE, &page, &vma, NULL);
384 if (unlikely(ret <= 0)) { 381 if (unlikely(ret <= 0)) {
385 /* 382 /*
386 * We are asking for 1 page. I 383 * We are asking for 1 page. If get_user_pages_remote() fails,
387 * it may return 0, in that ca 384 * it may return 0, in that case we have to return error.
388 */ 385 */
389 return ret == 0 ? -EBUSY : ret 386 return ret == 0 ? -EBUSY : ret;
390 } 387 }
391 388
392 kaddr = kmap_atomic(page); 389 kaddr = kmap_atomic(page);
393 ptr = kaddr + (vaddr & ~PAGE_MASK); 390 ptr = kaddr + (vaddr & ~PAGE_MASK);
394 391
395 if (unlikely(*ptr + d < 0)) { 392 if (unlikely(*ptr + d < 0)) {
396 pr_warn("ref_ctr going negativ 393 pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
397 "curr val: %d, delta: 394 "curr val: %d, delta: %d\n", vaddr, *ptr, d);
398 ret = -EINVAL; 395 ret = -EINVAL;
399 goto out; 396 goto out;
400 } 397 }
401 398
402 *ptr += d; 399 *ptr += d;
403 ret = 0; 400 ret = 0;
404 out: 401 out:
405 kunmap_atomic(kaddr); 402 kunmap_atomic(kaddr);
406 put_page(page); 403 put_page(page);
407 return ret; 404 return ret;
408 } 405 }
409 406
410 static void update_ref_ctr_warn(struct uprobe 407 static void update_ref_ctr_warn(struct uprobe *uprobe,
411 struct mm_stru 408 struct mm_struct *mm, short d)
412 { 409 {
413 pr_warn("ref_ctr %s failed for inode: 410 pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
414 "0x%llx ref_ctr_offset: 0x%llx 411 "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
415 d > 0 ? "increment" : "decreme 412 d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
416 (unsigned long long) uprobe->o 413 (unsigned long long) uprobe->offset,
417 (unsigned long long) uprobe->r 414 (unsigned long long) uprobe->ref_ctr_offset, mm);
418 } 415 }
419 416
420 static int update_ref_ctr(struct uprobe *uprob 417 static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
421 short d) 418 short d)
422 { 419 {
423 struct vm_area_struct *rc_vma; 420 struct vm_area_struct *rc_vma;
424 unsigned long rc_vaddr; 421 unsigned long rc_vaddr;
425 int ret = 0; 422 int ret = 0;
426 423
427 rc_vma = find_ref_ctr_vma(uprobe, mm); 424 rc_vma = find_ref_ctr_vma(uprobe, mm);
428 425
429 if (rc_vma) { 426 if (rc_vma) {
430 rc_vaddr = offset_to_vaddr(rc_ 427 rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
431 ret = __update_ref_ctr(mm, rc_ 428 ret = __update_ref_ctr(mm, rc_vaddr, d);
432 if (ret) 429 if (ret)
433 update_ref_ctr_warn(up 430 update_ref_ctr_warn(uprobe, mm, d);
434 431
435 if (d > 0) 432 if (d > 0)
436 return ret; 433 return ret;
437 } 434 }
438 435
439 mutex_lock(&delayed_uprobe_lock); 436 mutex_lock(&delayed_uprobe_lock);
440 if (d > 0) 437 if (d > 0)
441 ret = delayed_uprobe_add(uprob 438 ret = delayed_uprobe_add(uprobe, mm);
442 else 439 else
443 delayed_uprobe_remove(uprobe, 440 delayed_uprobe_remove(uprobe, mm);
444 mutex_unlock(&delayed_uprobe_lock); 441 mutex_unlock(&delayed_uprobe_lock);
445 442
446 return ret; 443 return ret;
447 } 444 }
448 445
449 /* 446 /*
450 * NOTE: 447 * NOTE:
451 * Expect the breakpoint instruction to be the 448 * Expect the breakpoint instruction to be the smallest size instruction for
452 * the architecture. If an arch has variable l 449 * the architecture. If an arch has variable length instruction and the
453 * breakpoint instruction is not of the smalle 450 * breakpoint instruction is not of the smallest length instruction
454 * supported by that architecture then we need 451 * supported by that architecture then we need to modify is_trap_at_addr and
455 * uprobe_write_opcode accordingly. This would 452 * uprobe_write_opcode accordingly. This would never be a problem for archs
456 * that have fixed length instructions. 453 * that have fixed length instructions.
457 * 454 *
458 * uprobe_write_opcode - write the opcode at a 455 * uprobe_write_opcode - write the opcode at a given virtual address.
459 * @auprobe: arch specific probepoint informat <<
460 * @mm: the probed process address space. 456 * @mm: the probed process address space.
461 * @vaddr: the virtual address to store the op 457 * @vaddr: the virtual address to store the opcode.
462 * @opcode: opcode to be written at @vaddr. 458 * @opcode: opcode to be written at @vaddr.
463 * 459 *
464 * Called with mm->mmap_lock held for read or !! 460 * Called with mm->mmap_lock held for write.
465 * Return 0 (success) or a negative errno. 461 * Return 0 (success) or a negative errno.
466 */ 462 */
467 int uprobe_write_opcode(struct arch_uprobe *au 463 int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
468 unsigned long vaddr, u 464 unsigned long vaddr, uprobe_opcode_t opcode)
469 { 465 {
470 struct uprobe *uprobe; 466 struct uprobe *uprobe;
471 struct page *old_page, *new_page; 467 struct page *old_page, *new_page;
472 struct vm_area_struct *vma; 468 struct vm_area_struct *vma;
473 int ret, is_register, ref_ctr_updated 469 int ret, is_register, ref_ctr_updated = 0;
474 bool orig_page_huge = false; 470 bool orig_page_huge = false;
475 unsigned int gup_flags = FOLL_FORCE; 471 unsigned int gup_flags = FOLL_FORCE;
476 472
477 is_register = is_swbp_insn(&opcode); 473 is_register = is_swbp_insn(&opcode);
478 uprobe = container_of(auprobe, struct 474 uprobe = container_of(auprobe, struct uprobe, arch);
479 475
480 retry: 476 retry:
481 if (is_register) 477 if (is_register)
482 gup_flags |= FOLL_SPLIT_PMD; 478 gup_flags |= FOLL_SPLIT_PMD;
483 /* Read the page with vaddr into memor 479 /* Read the page with vaddr into memory */
484 old_page = get_user_page_vma_remote(mm !! 480 ret = get_user_pages_remote(mm, vaddr, 1, gup_flags,
485 if (IS_ERR(old_page)) !! 481 &old_page, &vma, NULL);
486 return PTR_ERR(old_page); !! 482 if (ret <= 0)
>> 483 return ret;
487 484
488 ret = verify_opcode(old_page, vaddr, & 485 ret = verify_opcode(old_page, vaddr, &opcode);
489 if (ret <= 0) 486 if (ret <= 0)
490 goto put_old; 487 goto put_old;
491 488
492 if (WARN(!is_register && PageCompound( 489 if (WARN(!is_register && PageCompound(old_page),
493 "uprobe unregister should nev 490 "uprobe unregister should never work on compound page\n")) {
494 ret = -EINVAL; 491 ret = -EINVAL;
495 goto put_old; 492 goto put_old;
496 } 493 }
497 494
498 /* We are going to replace instruction 495 /* We are going to replace instruction, update ref_ctr. */
499 if (!ref_ctr_updated && uprobe->ref_ct 496 if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
500 ret = update_ref_ctr(uprobe, m 497 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
501 if (ret) 498 if (ret)
502 goto put_old; 499 goto put_old;
503 500
504 ref_ctr_updated = 1; 501 ref_ctr_updated = 1;
505 } 502 }
506 503
507 ret = 0; 504 ret = 0;
508 if (!is_register && !PageAnon(old_page 505 if (!is_register && !PageAnon(old_page))
509 goto put_old; 506 goto put_old;
510 507
511 ret = anon_vma_prepare(vma); 508 ret = anon_vma_prepare(vma);
512 if (ret) 509 if (ret)
513 goto put_old; 510 goto put_old;
514 511
515 ret = -ENOMEM; 512 ret = -ENOMEM;
516 new_page = alloc_page_vma(GFP_HIGHUSER 513 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
517 if (!new_page) 514 if (!new_page)
518 goto put_old; 515 goto put_old;
519 516
520 __SetPageUptodate(new_page); 517 __SetPageUptodate(new_page);
521 copy_highpage(new_page, old_page); 518 copy_highpage(new_page, old_page);
522 copy_to_page(new_page, vaddr, &opcode, 519 copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
523 520
524 if (!is_register) { 521 if (!is_register) {
525 struct page *orig_page; 522 struct page *orig_page;
526 pgoff_t index; 523 pgoff_t index;
527 524
528 VM_BUG_ON_PAGE(!PageAnon(old_p 525 VM_BUG_ON_PAGE(!PageAnon(old_page), old_page);
529 526
530 index = vaddr_to_offset(vma, v 527 index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT;
531 orig_page = find_get_page(vma- 528 orig_page = find_get_page(vma->vm_file->f_inode->i_mapping,
532 inde 529 index);
533 530
534 if (orig_page) { 531 if (orig_page) {
535 if (PageUptodate(orig_ 532 if (PageUptodate(orig_page) &&
536 pages_identical(ne 533 pages_identical(new_page, orig_page)) {
537 /* let go new_ 534 /* let go new_page */
538 put_page(new_p 535 put_page(new_page);
539 new_page = NUL 536 new_page = NULL;
540 537
541 if (PageCompou 538 if (PageCompound(orig_page))
542 orig_p 539 orig_page_huge = true;
543 } 540 }
544 put_page(orig_page); 541 put_page(orig_page);
545 } 542 }
546 } 543 }
547 544
548 ret = __replace_page(vma, vaddr & PAGE !! 545 ret = __replace_page(vma, vaddr, old_page, new_page);
549 if (new_page) 546 if (new_page)
550 put_page(new_page); 547 put_page(new_page);
551 put_old: 548 put_old:
552 put_page(old_page); 549 put_page(old_page);
553 550
554 if (unlikely(ret == -EAGAIN)) 551 if (unlikely(ret == -EAGAIN))
555 goto retry; 552 goto retry;
556 553
557 /* Revert back reference counter if in 554 /* Revert back reference counter if instruction update failed. */
558 if (ret && is_register && ref_ctr_upda 555 if (ret && is_register && ref_ctr_updated)
559 update_ref_ctr(uprobe, mm, -1) 556 update_ref_ctr(uprobe, mm, -1);
560 557
561 /* try collapse pmd for compound page 558 /* try collapse pmd for compound page */
562 if (!ret && orig_page_huge) 559 if (!ret && orig_page_huge)
563 collapse_pte_mapped_thp(mm, va !! 560 collapse_pte_mapped_thp(mm, vaddr);
564 561
565 return ret; 562 return ret;
566 } 563 }
567 564
568 /** 565 /**
569 * set_swbp - store breakpoint at a given addr 566 * set_swbp - store breakpoint at a given address.
570 * @auprobe: arch specific probepoint informat 567 * @auprobe: arch specific probepoint information.
571 * @mm: the probed process address space. 568 * @mm: the probed process address space.
572 * @vaddr: the virtual address to insert the o 569 * @vaddr: the virtual address to insert the opcode.
573 * 570 *
574 * For mm @mm, store the breakpoint instructio 571 * For mm @mm, store the breakpoint instruction at @vaddr.
575 * Return 0 (success) or a negative errno. 572 * Return 0 (success) or a negative errno.
576 */ 573 */
577 int __weak set_swbp(struct arch_uprobe *auprob 574 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
578 { 575 {
579 return uprobe_write_opcode(auprobe, mm 576 return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);
580 } 577 }
581 578
582 /** 579 /**
583 * set_orig_insn - Restore the original instru 580 * set_orig_insn - Restore the original instruction.
584 * @mm: the probed process address space. 581 * @mm: the probed process address space.
585 * @auprobe: arch specific probepoint informat 582 * @auprobe: arch specific probepoint information.
586 * @vaddr: the virtual address to insert the o 583 * @vaddr: the virtual address to insert the opcode.
587 * 584 *
588 * For mm @mm, restore the original opcode (op 585 * For mm @mm, restore the original opcode (opcode) at @vaddr.
589 * Return 0 (success) or a negative errno. 586 * Return 0 (success) or a negative errno.
590 */ 587 */
591 int __weak 588 int __weak
592 set_orig_insn(struct arch_uprobe *auprobe, str 589 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
593 { 590 {
594 return uprobe_write_opcode(auprobe, mm 591 return uprobe_write_opcode(auprobe, mm, vaddr,
595 *(uprobe_opcode_t *)&a 592 *(uprobe_opcode_t *)&auprobe->insn);
596 } 593 }
597 594
598 /* uprobe should have guaranteed positive refc <<
599 static struct uprobe *get_uprobe(struct uprobe 595 static struct uprobe *get_uprobe(struct uprobe *uprobe)
600 { 596 {
601 refcount_inc(&uprobe->ref); 597 refcount_inc(&uprobe->ref);
602 return uprobe; 598 return uprobe;
603 } 599 }
604 600
605 /* <<
606 * uprobe should have guaranteed lifetime, whi <<
607 * - caller already has refcount taken (and <<
608 * - uprobe is RCU protected and won't be fr <<
609 * - we are holding uprobes_treelock (for re <<
610 */ <<
611 static struct uprobe *try_get_uprobe(struct up <<
612 { <<
613 if (refcount_inc_not_zero(&uprobe->ref <<
614 return uprobe; <<
615 return NULL; <<
616 } <<
617 <<
618 static inline bool uprobe_is_active(struct upr <<
619 { <<
620 return !RB_EMPTY_NODE(&uprobe->rb_node <<
621 } <<
622 <<
623 static void uprobe_free_rcu(struct rcu_head *r <<
624 { <<
625 struct uprobe *uprobe = container_of(r <<
626 <<
627 kfree(uprobe); <<
628 } <<
629 <<
630 static void put_uprobe(struct uprobe *uprobe) 601 static void put_uprobe(struct uprobe *uprobe)
631 { 602 {
632 if (!refcount_dec_and_test(&uprobe->re !! 603 if (refcount_dec_and_test(&uprobe->ref)) {
633 return; !! 604 /*
634 !! 605 * If application munmap(exec_vma) before uprobe_unregister()
635 write_lock(&uprobes_treelock); !! 606 * gets called, we don't get a chance to remove uprobe from
636 !! 607 * delayed_uprobe_list from remove_breakpoint(). Do it here.
637 if (uprobe_is_active(uprobe)) { !! 608 */
638 write_seqcount_begin(&uprobes_ !! 609 mutex_lock(&delayed_uprobe_lock);
639 rb_erase(&uprobe->rb_node, &up !! 610 delayed_uprobe_remove(uprobe, NULL);
640 write_seqcount_end(&uprobes_se !! 611 mutex_unlock(&delayed_uprobe_lock);
>> 612 kfree(uprobe);
641 } 613 }
642 <<
643 write_unlock(&uprobes_treelock); <<
644 <<
645 /* <<
646 * If application munmap(exec_vma) bef <<
647 * gets called, we don't get a chance <<
648 * delayed_uprobe_list from remove_bre <<
649 */ <<
650 mutex_lock(&delayed_uprobe_lock); <<
651 delayed_uprobe_remove(uprobe, NULL); <<
652 mutex_unlock(&delayed_uprobe_lock); <<
653 <<
654 call_srcu(&uprobes_srcu, &uprobe->rcu, <<
655 } 614 }
656 615
657 static __always_inline 616 static __always_inline
658 int uprobe_cmp(const struct inode *l_inode, co 617 int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
659 const struct uprobe *r) 618 const struct uprobe *r)
660 { 619 {
661 if (l_inode < r->inode) 620 if (l_inode < r->inode)
662 return -1; 621 return -1;
663 622
664 if (l_inode > r->inode) 623 if (l_inode > r->inode)
665 return 1; 624 return 1;
666 625
667 if (l_offset < r->offset) 626 if (l_offset < r->offset)
668 return -1; 627 return -1;
669 628
670 if (l_offset > r->offset) 629 if (l_offset > r->offset)
671 return 1; 630 return 1;
672 631
673 return 0; 632 return 0;
674 } 633 }
675 634
676 #define __node_2_uprobe(node) \ 635 #define __node_2_uprobe(node) \
677 rb_entry((node), struct uprobe, rb_nod 636 rb_entry((node), struct uprobe, rb_node)
678 637
679 struct __uprobe_key { 638 struct __uprobe_key {
680 struct inode *inode; 639 struct inode *inode;
681 loff_t offset; 640 loff_t offset;
682 }; 641 };
683 642
684 static inline int __uprobe_cmp_key(const void 643 static inline int __uprobe_cmp_key(const void *key, const struct rb_node *b)
685 { 644 {
686 const struct __uprobe_key *a = key; 645 const struct __uprobe_key *a = key;
687 return uprobe_cmp(a->inode, a->offset, 646 return uprobe_cmp(a->inode, a->offset, __node_2_uprobe(b));
688 } 647 }
689 648
690 static inline int __uprobe_cmp(struct rb_node 649 static inline int __uprobe_cmp(struct rb_node *a, const struct rb_node *b)
691 { 650 {
692 struct uprobe *u = __node_2_uprobe(a); 651 struct uprobe *u = __node_2_uprobe(a);
693 return uprobe_cmp(u->inode, u->offset, 652 return uprobe_cmp(u->inode, u->offset, __node_2_uprobe(b));
694 } 653 }
695 654
696 /* !! 655 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
697 * Assumes being inside RCU protected region. <<
698 * No refcount is taken on returned uprobe. <<
699 */ <<
700 static struct uprobe *find_uprobe_rcu(struct i <<
701 { 656 {
702 struct __uprobe_key key = { 657 struct __uprobe_key key = {
703 .inode = inode, 658 .inode = inode,
704 .offset = offset, 659 .offset = offset,
705 }; 660 };
706 struct rb_node *node; !! 661 struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key);
707 unsigned int seq; <<
708 662
709 lockdep_assert(srcu_read_lock_held(&up !! 663 if (node)
710 !! 664 return get_uprobe(__node_2_uprobe(node));
711 do { <<
712 seq = read_seqcount_begin(&upr <<
713 node = rb_find_rcu(&key, &upro <<
714 /* <<
715 * Lockless RB-tree lookups ca <<
716 * If the element is found, it <<
717 * under RCU protection. If we <<
718 * validate that seqcount didn <<
719 * try again as we might have <<
720 * negative). If seqcount is u <<
721 */ <<
722 if (node) <<
723 return __node_2_uprobe <<
724 } while (read_seqcount_retry(&uprobes_ <<
725 665
726 return NULL; 666 return NULL;
727 } 667 }
728 668
729 /* 669 /*
730 * Attempt to insert a new uprobe into uprobes !! 670 * Find a uprobe corresponding to a given inode:offset
731 * !! 671 * Acquires uprobes_treelock
732 * If uprobe already exists (for given inode+o <<
733 * refcount of previously existing uprobe. <<
734 * <<
735 * If not, a provided new instance of uprobe i <<
736 * assumed initial refcount == 1). <<
737 * <<
738 * In any case, we return a uprobe instance th <<
739 * Caller has to clean up new uprobe instance, <<
740 * inserted into the tree. <<
741 * <<
742 * We assume that uprobes_treelock is held for <<
743 */ 672 */
>> 673 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
>> 674 {
>> 675 struct uprobe *uprobe;
>> 676
>> 677 spin_lock(&uprobes_treelock);
>> 678 uprobe = __find_uprobe(inode, offset);
>> 679 spin_unlock(&uprobes_treelock);
>> 680
>> 681 return uprobe;
>> 682 }
>> 683
744 static struct uprobe *__insert_uprobe(struct u 684 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
745 { 685 {
746 struct rb_node *node; 686 struct rb_node *node;
747 again: <<
748 node = rb_find_add_rcu(&uprobe->rb_nod <<
749 if (node) { <<
750 struct uprobe *u = __node_2_up <<
751 <<
752 if (!try_get_uprobe(u)) { <<
753 rb_erase(node, &uprobe <<
754 RB_CLEAR_NODE(&u->rb_n <<
755 goto again; <<
756 } <<
757 687
758 return u; !! 688 node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
759 } !! 689 if (node)
>> 690 return get_uprobe(__node_2_uprobe(node));
760 691
761 return uprobe; !! 692 /* get access + creation ref */
>> 693 refcount_set(&uprobe->ref, 2);
>> 694 return NULL;
762 } 695 }
763 696
764 /* 697 /*
765 * Acquire uprobes_treelock and insert uprobe !! 698 * Acquire uprobes_treelock.
766 * (or reuse existing one, see __insert_uprobe !! 699 * Matching uprobe already exists in rbtree;
>> 700 * increment (access refcount) and return the matching uprobe.
>> 701 *
>> 702 * No matching uprobe; insert the uprobe in rb_tree;
>> 703 * get a double refcount (access + creation) and return NULL.
767 */ 704 */
768 static struct uprobe *insert_uprobe(struct upr 705 static struct uprobe *insert_uprobe(struct uprobe *uprobe)
769 { 706 {
770 struct uprobe *u; 707 struct uprobe *u;
771 708
772 write_lock(&uprobes_treelock); !! 709 spin_lock(&uprobes_treelock);
773 write_seqcount_begin(&uprobes_seqcount <<
774 u = __insert_uprobe(uprobe); 710 u = __insert_uprobe(uprobe);
775 write_seqcount_end(&uprobes_seqcount); !! 711 spin_unlock(&uprobes_treelock);
776 write_unlock(&uprobes_treelock); <<
777 712
778 return u; 713 return u;
779 } 714 }
780 715
781 static void 716 static void
782 ref_ctr_mismatch_warn(struct uprobe *cur_uprob 717 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
783 { 718 {
784 pr_warn("ref_ctr_offset mismatch. inod 719 pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
785 "ref_ctr_offset(old): 0x%llx r 720 "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
786 uprobe->inode->i_ino, (unsigne 721 uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
787 (unsigned long long) cur_uprob 722 (unsigned long long) cur_uprobe->ref_ctr_offset,
788 (unsigned long long) uprobe->r 723 (unsigned long long) uprobe->ref_ctr_offset);
789 } 724 }
790 725
791 static struct uprobe *alloc_uprobe(struct inod 726 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
792 loff_t ref_ 727 loff_t ref_ctr_offset)
793 { 728 {
794 struct uprobe *uprobe, *cur_uprobe; 729 struct uprobe *uprobe, *cur_uprobe;
795 730
796 uprobe = kzalloc(sizeof(struct uprobe) 731 uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
797 if (!uprobe) 732 if (!uprobe)
798 return ERR_PTR(-ENOMEM); !! 733 return NULL;
799 734
800 uprobe->inode = inode; 735 uprobe->inode = inode;
801 uprobe->offset = offset; 736 uprobe->offset = offset;
802 uprobe->ref_ctr_offset = ref_ctr_offse 737 uprobe->ref_ctr_offset = ref_ctr_offset;
803 INIT_LIST_HEAD(&uprobe->consumers); <<
804 init_rwsem(&uprobe->register_rwsem); 738 init_rwsem(&uprobe->register_rwsem);
805 init_rwsem(&uprobe->consumer_rwsem); 739 init_rwsem(&uprobe->consumer_rwsem);
806 RB_CLEAR_NODE(&uprobe->rb_node); <<
807 refcount_set(&uprobe->ref, 1); <<
808 740
809 /* add to uprobes_tree, sorted on inod 741 /* add to uprobes_tree, sorted on inode:offset */
810 cur_uprobe = insert_uprobe(uprobe); 742 cur_uprobe = insert_uprobe(uprobe);
811 /* a uprobe exists for this inode:offs 743 /* a uprobe exists for this inode:offset combination */
812 if (cur_uprobe != uprobe) { !! 744 if (cur_uprobe) {
813 if (cur_uprobe->ref_ctr_offset 745 if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
814 ref_ctr_mismatch_warn( 746 ref_ctr_mismatch_warn(cur_uprobe, uprobe);
815 put_uprobe(cur_uprobe) 747 put_uprobe(cur_uprobe);
816 kfree(uprobe); 748 kfree(uprobe);
817 return ERR_PTR(-EINVAL 749 return ERR_PTR(-EINVAL);
818 } 750 }
819 kfree(uprobe); 751 kfree(uprobe);
820 uprobe = cur_uprobe; 752 uprobe = cur_uprobe;
821 } 753 }
822 754
823 return uprobe; 755 return uprobe;
824 } 756 }
825 757
826 static void consumer_add(struct uprobe *uprobe 758 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
827 { 759 {
828 down_write(&uprobe->consumer_rwsem); 760 down_write(&uprobe->consumer_rwsem);
829 list_add_rcu(&uc->cons_node, &uprobe-> !! 761 uc->next = uprobe->consumers;
>> 762 uprobe->consumers = uc;
830 up_write(&uprobe->consumer_rwsem); 763 up_write(&uprobe->consumer_rwsem);
831 } 764 }
832 765
833 /* 766 /*
834 * For uprobe @uprobe, delete the consumer @uc 767 * For uprobe @uprobe, delete the consumer @uc.
835 * Should never be called with consumer that's !! 768 * Return true if the @uc is deleted successfully
>> 769 * or return false.
836 */ 770 */
837 static void consumer_del(struct uprobe *uprobe !! 771 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
838 { 772 {
>> 773 struct uprobe_consumer **con;
>> 774 bool ret = false;
>> 775
839 down_write(&uprobe->consumer_rwsem); 776 down_write(&uprobe->consumer_rwsem);
840 list_del_rcu(&uc->cons_node); !! 777 for (con = &uprobe->consumers; *con; con = &(*con)->next) {
>> 778 if (*con == uc) {
>> 779 *con = uc->next;
>> 780 ret = true;
>> 781 break;
>> 782 }
>> 783 }
841 up_write(&uprobe->consumer_rwsem); 784 up_write(&uprobe->consumer_rwsem);
>> 785
>> 786 return ret;
842 } 787 }
843 788
844 static int __copy_insn(struct address_space *m 789 static int __copy_insn(struct address_space *mapping, struct file *filp,
845 void *insn, int nbytes 790 void *insn, int nbytes, loff_t offset)
846 { 791 {
847 struct page *page; 792 struct page *page;
848 /* 793 /*
849 * Ensure that the page that has the o 794 * Ensure that the page that has the original instruction is populated
850 * and in page-cache. If ->read_folio !! 795 * and in page-cache. If ->readpage == NULL it must be shmem_mapping(),
851 * see uprobe_register(). 796 * see uprobe_register().
852 */ 797 */
853 if (mapping->a_ops->read_folio) !! 798 if (mapping->a_ops->readpage)
854 page = read_mapping_page(mappi 799 page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
855 else 800 else
856 page = shmem_read_mapping_page 801 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
857 if (IS_ERR(page)) 802 if (IS_ERR(page))
858 return PTR_ERR(page); 803 return PTR_ERR(page);
859 804
860 copy_from_page(page, offset, insn, nby 805 copy_from_page(page, offset, insn, nbytes);
861 put_page(page); 806 put_page(page);
862 807
863 return 0; 808 return 0;
864 } 809 }
865 810
866 static int copy_insn(struct uprobe *uprobe, st 811 static int copy_insn(struct uprobe *uprobe, struct file *filp)
867 { 812 {
868 struct address_space *mapping = uprobe 813 struct address_space *mapping = uprobe->inode->i_mapping;
869 loff_t offs = uprobe->offset; 814 loff_t offs = uprobe->offset;
870 void *insn = &uprobe->arch.insn; 815 void *insn = &uprobe->arch.insn;
871 int size = sizeof(uprobe->arch.insn); 816 int size = sizeof(uprobe->arch.insn);
872 int len, err = -EIO; 817 int len, err = -EIO;
873 818
874 /* Copy only available bytes, -EIO if 819 /* Copy only available bytes, -EIO if nothing was read */
875 do { 820 do {
876 if (offs >= i_size_read(uprobe 821 if (offs >= i_size_read(uprobe->inode))
877 break; 822 break;
878 823
879 len = min_t(int, size, PAGE_SI 824 len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
880 err = __copy_insn(mapping, fil 825 err = __copy_insn(mapping, filp, insn, len, offs);
881 if (err) 826 if (err)
882 break; 827 break;
883 828
884 insn += len; 829 insn += len;
885 offs += len; 830 offs += len;
886 size -= len; 831 size -= len;
887 } while (size); 832 } while (size);
888 833
889 return err; 834 return err;
890 } 835 }
891 836
892 static int prepare_uprobe(struct uprobe *uprob 837 static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
893 struct mm_stru 838 struct mm_struct *mm, unsigned long vaddr)
894 { 839 {
895 int ret = 0; 840 int ret = 0;
896 841
897 if (test_bit(UPROBE_COPY_INSN, &uprobe 842 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
898 return ret; 843 return ret;
899 844
900 /* TODO: move this into _register, unt 845 /* TODO: move this into _register, until then we abuse this sem. */
901 down_write(&uprobe->consumer_rwsem); 846 down_write(&uprobe->consumer_rwsem);
902 if (test_bit(UPROBE_COPY_INSN, &uprobe 847 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
903 goto out; 848 goto out;
904 849
905 ret = copy_insn(uprobe, file); 850 ret = copy_insn(uprobe, file);
906 if (ret) 851 if (ret)
907 goto out; 852 goto out;
908 853
909 ret = -ENOTSUPP; 854 ret = -ENOTSUPP;
910 if (is_trap_insn((uprobe_opcode_t *)&u 855 if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
911 goto out; 856 goto out;
912 857
913 ret = arch_uprobe_analyze_insn(&uprobe 858 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
914 if (ret) 859 if (ret)
915 goto out; 860 goto out;
916 861
917 smp_wmb(); /* pairs with the smp_rmb() 862 smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
918 set_bit(UPROBE_COPY_INSN, &uprobe->fla 863 set_bit(UPROBE_COPY_INSN, &uprobe->flags);
919 864
920 out: 865 out:
921 up_write(&uprobe->consumer_rwsem); 866 up_write(&uprobe->consumer_rwsem);
922 867
923 return ret; 868 return ret;
924 } 869 }
925 870
926 static inline bool consumer_filter(struct upro !! 871 static inline bool consumer_filter(struct uprobe_consumer *uc,
>> 872 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
927 { 873 {
928 return !uc->filter || uc->filter(uc, m !! 874 return !uc->filter || uc->filter(uc, ctx, mm);
929 } 875 }
930 876
931 static bool filter_chain(struct uprobe *uprobe !! 877 static bool filter_chain(struct uprobe *uprobe,
>> 878 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
932 { 879 {
933 struct uprobe_consumer *uc; 880 struct uprobe_consumer *uc;
934 bool ret = false; 881 bool ret = false;
935 882
936 down_read(&uprobe->consumer_rwsem); 883 down_read(&uprobe->consumer_rwsem);
937 list_for_each_entry_srcu(uc, &uprobe-> !! 884 for (uc = uprobe->consumers; uc; uc = uc->next) {
938 srcu_read_loc !! 885 ret = consumer_filter(uc, ctx, mm);
939 ret = consumer_filter(uc, mm); <<
940 if (ret) 886 if (ret)
941 break; 887 break;
942 } 888 }
943 up_read(&uprobe->consumer_rwsem); 889 up_read(&uprobe->consumer_rwsem);
944 890
945 return ret; 891 return ret;
946 } 892 }
947 893
948 static int 894 static int
949 install_breakpoint(struct uprobe *uprobe, stru 895 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
950 struct vm_area_struct 896 struct vm_area_struct *vma, unsigned long vaddr)
951 { 897 {
952 bool first_uprobe; 898 bool first_uprobe;
953 int ret; 899 int ret;
954 900
955 ret = prepare_uprobe(uprobe, vma->vm_f 901 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
956 if (ret) 902 if (ret)
957 return ret; 903 return ret;
958 904
959 /* 905 /*
960 * set MMF_HAS_UPROBES in advance for 906 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
961 * the task can hit this breakpoint ri 907 * the task can hit this breakpoint right after __replace_page().
962 */ 908 */
963 first_uprobe = !test_bit(MMF_HAS_UPROB 909 first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
964 if (first_uprobe) 910 if (first_uprobe)
965 set_bit(MMF_HAS_UPROBES, &mm-> 911 set_bit(MMF_HAS_UPROBES, &mm->flags);
966 912
967 ret = set_swbp(&uprobe->arch, mm, vadd 913 ret = set_swbp(&uprobe->arch, mm, vaddr);
968 if (!ret) 914 if (!ret)
969 clear_bit(MMF_RECALC_UPROBES, 915 clear_bit(MMF_RECALC_UPROBES, &mm->flags);
970 else if (first_uprobe) 916 else if (first_uprobe)
971 clear_bit(MMF_HAS_UPROBES, &mm 917 clear_bit(MMF_HAS_UPROBES, &mm->flags);
972 918
973 return ret; 919 return ret;
974 } 920 }
975 921
976 static int 922 static int
977 remove_breakpoint(struct uprobe *uprobe, struc 923 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
978 { 924 {
979 set_bit(MMF_RECALC_UPROBES, &mm->flags 925 set_bit(MMF_RECALC_UPROBES, &mm->flags);
980 return set_orig_insn(&uprobe->arch, mm 926 return set_orig_insn(&uprobe->arch, mm, vaddr);
981 } 927 }
982 928
>> 929 static inline bool uprobe_is_active(struct uprobe *uprobe)
>> 930 {
>> 931 return !RB_EMPTY_NODE(&uprobe->rb_node);
>> 932 }
>> 933 /*
>> 934 * There could be threads that have already hit the breakpoint. They
>> 935 * will recheck the current insn and restart if find_uprobe() fails.
>> 936 * See find_active_uprobe().
>> 937 */
>> 938 static void delete_uprobe(struct uprobe *uprobe)
>> 939 {
>> 940 if (WARN_ON(!uprobe_is_active(uprobe)))
>> 941 return;
>> 942
>> 943 spin_lock(&uprobes_treelock);
>> 944 rb_erase(&uprobe->rb_node, &uprobes_tree);
>> 945 spin_unlock(&uprobes_treelock);
>> 946 RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
>> 947 put_uprobe(uprobe);
>> 948 }
>> 949
983 struct map_info { 950 struct map_info {
984 struct map_info *next; 951 struct map_info *next;
985 struct mm_struct *mm; 952 struct mm_struct *mm;
986 unsigned long vaddr; 953 unsigned long vaddr;
987 }; 954 };
988 955
989 static inline struct map_info *free_map_info(s 956 static inline struct map_info *free_map_info(struct map_info *info)
990 { 957 {
991 struct map_info *next = info->next; 958 struct map_info *next = info->next;
992 kfree(info); 959 kfree(info);
993 return next; 960 return next;
994 } 961 }
995 962
996 static struct map_info * 963 static struct map_info *
997 build_map_info(struct address_space *mapping, 964 build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
998 { 965 {
999 unsigned long pgoff = offset >> PAGE_S 966 unsigned long pgoff = offset >> PAGE_SHIFT;
1000 struct vm_area_struct *vma; 967 struct vm_area_struct *vma;
1001 struct map_info *curr = NULL; 968 struct map_info *curr = NULL;
1002 struct map_info *prev = NULL; 969 struct map_info *prev = NULL;
1003 struct map_info *info; 970 struct map_info *info;
1004 int more = 0; 971 int more = 0;
1005 972
1006 again: 973 again:
1007 i_mmap_lock_read(mapping); 974 i_mmap_lock_read(mapping);
1008 vma_interval_tree_foreach(vma, &mappi 975 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1009 if (!valid_vma(vma, is_regist 976 if (!valid_vma(vma, is_register))
1010 continue; 977 continue;
1011 978
1012 if (!prev && !more) { 979 if (!prev && !more) {
1013 /* 980 /*
1014 * Needs GFP_NOWAIT t 981 * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
1015 * reclaim. This is o 982 * reclaim. This is optimistic, no harm done if it fails.
1016 */ 983 */
1017 prev = kmalloc(sizeof 984 prev = kmalloc(sizeof(struct map_info),
1018 GFP_N 985 GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
1019 if (prev) 986 if (prev)
1020 prev->next = 987 prev->next = NULL;
1021 } 988 }
1022 if (!prev) { 989 if (!prev) {
1023 more++; 990 more++;
1024 continue; 991 continue;
1025 } 992 }
1026 993
1027 if (!mmget_not_zero(vma->vm_m 994 if (!mmget_not_zero(vma->vm_mm))
1028 continue; 995 continue;
1029 996
1030 info = prev; 997 info = prev;
1031 prev = prev->next; 998 prev = prev->next;
1032 info->next = curr; 999 info->next = curr;
1033 curr = info; 1000 curr = info;
1034 1001
1035 info->mm = vma->vm_mm; 1002 info->mm = vma->vm_mm;
1036 info->vaddr = offset_to_vaddr 1003 info->vaddr = offset_to_vaddr(vma, offset);
1037 } 1004 }
1038 i_mmap_unlock_read(mapping); 1005 i_mmap_unlock_read(mapping);
1039 1006
1040 if (!more) 1007 if (!more)
1041 goto out; 1008 goto out;
1042 1009
1043 prev = curr; 1010 prev = curr;
1044 while (curr) { 1011 while (curr) {
1045 mmput(curr->mm); 1012 mmput(curr->mm);
1046 curr = curr->next; 1013 curr = curr->next;
1047 } 1014 }
1048 1015
1049 do { 1016 do {
1050 info = kmalloc(sizeof(struct 1017 info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
1051 if (!info) { 1018 if (!info) {
1052 curr = ERR_PTR(-ENOME 1019 curr = ERR_PTR(-ENOMEM);
1053 goto out; 1020 goto out;
1054 } 1021 }
1055 info->next = prev; 1022 info->next = prev;
1056 prev = info; 1023 prev = info;
1057 } while (--more); 1024 } while (--more);
1058 1025
1059 goto again; 1026 goto again;
1060 out: 1027 out:
1061 while (prev) 1028 while (prev)
1062 prev = free_map_info(prev); 1029 prev = free_map_info(prev);
1063 return curr; 1030 return curr;
1064 } 1031 }
1065 1032
1066 static int 1033 static int
1067 register_for_each_vma(struct uprobe *uprobe, 1034 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
1068 { 1035 {
1069 bool is_register = !!new; 1036 bool is_register = !!new;
1070 struct map_info *info; 1037 struct map_info *info;
1071 int err = 0; 1038 int err = 0;
1072 1039
1073 percpu_down_write(&dup_mmap_sem); 1040 percpu_down_write(&dup_mmap_sem);
1074 info = build_map_info(uprobe->inode-> 1041 info = build_map_info(uprobe->inode->i_mapping,
1075 uprob 1042 uprobe->offset, is_register);
1076 if (IS_ERR(info)) { 1043 if (IS_ERR(info)) {
1077 err = PTR_ERR(info); 1044 err = PTR_ERR(info);
1078 goto out; 1045 goto out;
1079 } 1046 }
1080 1047
1081 while (info) { 1048 while (info) {
1082 struct mm_struct *mm = info-> 1049 struct mm_struct *mm = info->mm;
1083 struct vm_area_struct *vma; 1050 struct vm_area_struct *vma;
1084 1051
1085 if (err && is_register) 1052 if (err && is_register)
1086 goto free; 1053 goto free;
1087 /* !! 1054
1088 * We take mmap_lock for writ <<
1089 * find_active_uprobe_rcu() w <<
1090 * Thus this install_breakpoi <<
1091 * is_trap_at_addr() true rig <<
1092 * returns NULL in find_activ <<
1093 */ <<
1094 mmap_write_lock(mm); 1055 mmap_write_lock(mm);
1095 vma = find_vma(mm, info->vadd 1056 vma = find_vma(mm, info->vaddr);
1096 if (!vma || !valid_vma(vma, i 1057 if (!vma || !valid_vma(vma, is_register) ||
1097 file_inode(vma->vm_file) 1058 file_inode(vma->vm_file) != uprobe->inode)
1098 goto unlock; 1059 goto unlock;
1099 1060
1100 if (vma->vm_start > info->vad 1061 if (vma->vm_start > info->vaddr ||
1101 vaddr_to_offset(vma, info 1062 vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
1102 goto unlock; 1063 goto unlock;
1103 1064
1104 if (is_register) { 1065 if (is_register) {
1105 /* consult only the " 1066 /* consult only the "caller", new consumer. */
1106 if (consumer_filter(n !! 1067 if (consumer_filter(new,
>> 1068 UPROBE_FILTER_REGISTER, mm))
1107 err = install 1069 err = install_breakpoint(uprobe, mm, vma, info->vaddr);
1108 } else if (test_bit(MMF_HAS_U 1070 } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
1109 if (!filter_chain(upr !! 1071 if (!filter_chain(uprobe,
>> 1072 UPROBE_FILTER_UNREGISTER, mm))
1110 err |= remove 1073 err |= remove_breakpoint(uprobe, mm, info->vaddr);
1111 } 1074 }
1112 1075
1113 unlock: 1076 unlock:
1114 mmap_write_unlock(mm); 1077 mmap_write_unlock(mm);
1115 free: 1078 free:
1116 mmput(mm); 1079 mmput(mm);
1117 info = free_map_info(info); 1080 info = free_map_info(info);
1118 } 1081 }
1119 out: 1082 out:
1120 percpu_up_write(&dup_mmap_sem); 1083 percpu_up_write(&dup_mmap_sem);
1121 return err; 1084 return err;
1122 } 1085 }
1123 1086
1124 /** !! 1087 static void
1125 * uprobe_unregister_nosync - unregister an a !! 1088 __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
1126 * @uprobe: uprobe to remove <<
1127 * @uc: identify which probe if multiple prob <<
1128 */ <<
1129 void uprobe_unregister_nosync(struct uprobe * <<
1130 { 1089 {
1131 int err; 1090 int err;
1132 1091
1133 down_write(&uprobe->register_rwsem); !! 1092 if (WARN_ON(!consumer_del(uprobe, uc)))
1134 consumer_del(uprobe, uc); <<
1135 err = register_for_each_vma(uprobe, N <<
1136 up_write(&uprobe->register_rwsem); <<
1137 <<
1138 /* TODO : cant unregister? schedule a <<
1139 if (unlikely(err)) { <<
1140 uprobe_warn(current, "unregis <<
1141 return; 1093 return;
1142 } <<
1143 1094
1144 put_uprobe(uprobe); !! 1095 err = register_for_each_vma(uprobe, NULL);
>> 1096 /* TODO : cant unregister? schedule a worker thread */
>> 1097 if (!uprobe->consumers && !err)
>> 1098 delete_uprobe(uprobe);
1145 } 1099 }
1146 EXPORT_SYMBOL_GPL(uprobe_unregister_nosync); <<
1147 1100
1148 void uprobe_unregister_sync(void) !! 1101 /*
>> 1102 * uprobe_unregister - unregister an already registered probe.
>> 1103 * @inode: the file in which the probe has to be removed.
>> 1104 * @offset: offset from the start of the file.
>> 1105 * @uc: identify which probe if multiple probes are colocated.
>> 1106 */
>> 1107 void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
1149 { 1108 {
1150 /* !! 1109 struct uprobe *uprobe;
1151 * Now that handler_chain() and handl !! 1110
1152 * uprobe->consumers list under RCU p !! 1111 uprobe = find_uprobe(inode, offset);
1153 * uprobe->register_rwsem, we need to !! 1112 if (WARN_ON(!uprobe))
1154 * make sure that we can't call into !! 1113 return;
1155 * uprobe_consumer's callbacks anymor !! 1114
1156 * unlucky enough caller can free con !! 1115 down_write(&uprobe->register_rwsem);
1157 * handler_chain() or handle_uretprob !! 1116 __uprobe_unregister(uprobe, uc);
1158 */ !! 1117 up_write(&uprobe->register_rwsem);
1159 synchronize_srcu(&uprobes_srcu); !! 1118 put_uprobe(uprobe);
1160 } 1119 }
1161 EXPORT_SYMBOL_GPL(uprobe_unregister_sync); !! 1120 EXPORT_SYMBOL_GPL(uprobe_unregister);
1162 1121
1163 /** !! 1122 /*
1164 * uprobe_register - register a probe !! 1123 * __uprobe_register - register a probe
1165 * @inode: the file in which the probe has to 1124 * @inode: the file in which the probe has to be placed.
1166 * @offset: offset from the start of the file 1125 * @offset: offset from the start of the file.
1167 * @ref_ctr_offset: offset of SDT marker / re <<
1168 * @uc: information on howto handle the probe 1126 * @uc: information on howto handle the probe..
1169 * 1127 *
1170 * Apart from the access refcount, uprobe_reg !! 1128 * Apart from the access refcount, __uprobe_register() takes a creation
1171 * refcount (thro alloc_uprobe) if and only i 1129 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
1172 * inserted into the rbtree (i.e first consum 1130 * inserted into the rbtree (i.e first consumer for a @inode:@offset
1173 * tuple). Creation refcount stops uprobe_un 1131 * tuple). Creation refcount stops uprobe_unregister from freeing the
1174 * @uprobe even before the register operation 1132 * @uprobe even before the register operation is complete. Creation
1175 * refcount is released when the last @uc for 1133 * refcount is released when the last @uc for the @uprobe
1176 * unregisters. Caller of uprobe_register() i !! 1134 * unregisters. Caller of __uprobe_register() is required to keep @inode
1177 * (and the containing mount) referenced. 1135 * (and the containing mount) referenced.
1178 * 1136 *
1179 * Return: pointer to the new uprobe on succe !! 1137 * Return errno if it cannot successully install probes
>> 1138 * else return 0 (success)
1180 */ 1139 */
1181 struct uprobe *uprobe_register(struct inode * !! 1140 static int __uprobe_register(struct inode *inode, loff_t offset,
1182 loff_t offset !! 1141 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1183 struct uprobe <<
1184 { 1142 {
1185 struct uprobe *uprobe; 1143 struct uprobe *uprobe;
1186 int ret; 1144 int ret;
1187 1145
1188 /* Uprobe must have at least one set 1146 /* Uprobe must have at least one set consumer */
1189 if (!uc->handler && !uc->ret_handler) 1147 if (!uc->handler && !uc->ret_handler)
1190 return ERR_PTR(-EINVAL); !! 1148 return -EINVAL;
1191 1149
1192 /* copy_insn() uses read_mapping_page 1150 /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
1193 if (!inode->i_mapping->a_ops->read_fo !! 1151 if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
1194 !shmem_mapping(inode->i_mapping)) !! 1152 return -EIO;
1195 return ERR_PTR(-EIO); <<
1196 /* Racy, just to catch the obvious mi 1153 /* Racy, just to catch the obvious mistakes */
1197 if (offset > i_size_read(inode)) 1154 if (offset > i_size_read(inode))
1198 return ERR_PTR(-EINVAL); !! 1155 return -EINVAL;
1199 1156
1200 /* 1157 /*
1201 * This ensures that copy_from_page() 1158 * This ensures that copy_from_page(), copy_to_page() and
1202 * __update_ref_ctr() can't cross pag 1159 * __update_ref_ctr() can't cross page boundary.
1203 */ 1160 */
1204 if (!IS_ALIGNED(offset, UPROBE_SWBP_I 1161 if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
1205 return ERR_PTR(-EINVAL); !! 1162 return -EINVAL;
1206 if (!IS_ALIGNED(ref_ctr_offset, sizeo 1163 if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
1207 return ERR_PTR(-EINVAL); !! 1164 return -EINVAL;
1208 1165
>> 1166 retry:
1209 uprobe = alloc_uprobe(inode, offset, 1167 uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
>> 1168 if (!uprobe)
>> 1169 return -ENOMEM;
1210 if (IS_ERR(uprobe)) 1170 if (IS_ERR(uprobe))
1211 return uprobe; !! 1171 return PTR_ERR(uprobe);
1212 1172
>> 1173 /*
>> 1174 * We can race with uprobe_unregister()->delete_uprobe().
>> 1175 * Check uprobe_is_active() and retry if it is false.
>> 1176 */
1213 down_write(&uprobe->register_rwsem); 1177 down_write(&uprobe->register_rwsem);
1214 consumer_add(uprobe, uc); !! 1178 ret = -EAGAIN;
1215 ret = register_for_each_vma(uprobe, u !! 1179 if (likely(uprobe_is_active(uprobe))) {
>> 1180 consumer_add(uprobe, uc);
>> 1181 ret = register_for_each_vma(uprobe, uc);
>> 1182 if (ret)
>> 1183 __uprobe_unregister(uprobe, uc);
>> 1184 }
1216 up_write(&uprobe->register_rwsem); 1185 up_write(&uprobe->register_rwsem);
>> 1186 put_uprobe(uprobe);
1217 1187
1218 if (ret) { !! 1188 if (unlikely(ret == -EAGAIN))
1219 uprobe_unregister_nosync(upro !! 1189 goto retry;
1220 /* !! 1190 return ret;
1221 * Registration might have pa !! 1191 }
1222 * this consumer being called <<
1223 * sync here. It's ok, it's u <<
1224 */ <<
1225 uprobe_unregister_sync(); <<
1226 return ERR_PTR(ret); <<
1227 } <<
1228 1192
1229 return uprobe; !! 1193 int uprobe_register(struct inode *inode, loff_t offset,
>> 1194 struct uprobe_consumer *uc)
>> 1195 {
>> 1196 return __uprobe_register(inode, offset, 0, uc);
1230 } 1197 }
1231 EXPORT_SYMBOL_GPL(uprobe_register); 1198 EXPORT_SYMBOL_GPL(uprobe_register);
1232 1199
1233 /** !! 1200 int uprobe_register_refctr(struct inode *inode, loff_t offset,
1234 * uprobe_apply - add or remove the breakpoin !! 1201 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1235 * @uprobe: uprobe which "owns" the breakpoin !! 1202 {
>> 1203 return __uprobe_register(inode, offset, ref_ctr_offset, uc);
>> 1204 }
>> 1205 EXPORT_SYMBOL_GPL(uprobe_register_refctr);
>> 1206
>> 1207 /*
>> 1208 * uprobe_apply - unregister an already registered probe.
>> 1209 * @inode: the file in which the probe has to be removed.
>> 1210 * @offset: offset from the start of the file.
1236 * @uc: consumer which wants to add more or r 1211 * @uc: consumer which wants to add more or remove some breakpoints
1237 * @add: add or remove the breakpoints 1212 * @add: add or remove the breakpoints
1238 * Return: 0 on success or negative error cod <<
1239 */ 1213 */
1240 int uprobe_apply(struct uprobe *uprobe, struc !! 1214 int uprobe_apply(struct inode *inode, loff_t offset,
>> 1215 struct uprobe_consumer *uc, bool add)
1241 { 1216 {
>> 1217 struct uprobe *uprobe;
1242 struct uprobe_consumer *con; 1218 struct uprobe_consumer *con;
1243 int ret = -ENOENT, srcu_idx; !! 1219 int ret = -ENOENT;
1244 <<
1245 down_write(&uprobe->register_rwsem); <<
1246 1220
1247 srcu_idx = srcu_read_lock(&uprobes_sr !! 1221 uprobe = find_uprobe(inode, offset);
1248 list_for_each_entry_srcu(con, &uprobe !! 1222 if (WARN_ON(!uprobe))
1249 srcu_read_lo !! 1223 return ret;
1250 if (con == uc) { <<
1251 ret = register_for_ea <<
1252 break; <<
1253 } <<
1254 } <<
1255 srcu_read_unlock(&uprobes_srcu, srcu_ <<
1256 1224
>> 1225 down_write(&uprobe->register_rwsem);
>> 1226 for (con = uprobe->consumers; con && con != uc ; con = con->next)
>> 1227 ;
>> 1228 if (con)
>> 1229 ret = register_for_each_vma(uprobe, add ? uc : NULL);
1257 up_write(&uprobe->register_rwsem); 1230 up_write(&uprobe->register_rwsem);
>> 1231 put_uprobe(uprobe);
1258 1232
1259 return ret; 1233 return ret;
1260 } 1234 }
1261 1235
1262 static int unapply_uprobe(struct uprobe *upro 1236 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
1263 { 1237 {
1264 VMA_ITERATOR(vmi, mm, 0); <<
1265 struct vm_area_struct *vma; 1238 struct vm_area_struct *vma;
1266 int err = 0; 1239 int err = 0;
1267 1240
1268 mmap_read_lock(mm); 1241 mmap_read_lock(mm);
1269 for_each_vma(vmi, vma) { !! 1242 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1270 unsigned long vaddr; 1243 unsigned long vaddr;
1271 loff_t offset; 1244 loff_t offset;
1272 1245
1273 if (!valid_vma(vma, false) || 1246 if (!valid_vma(vma, false) ||
1274 file_inode(vma->vm_file) 1247 file_inode(vma->vm_file) != uprobe->inode)
1275 continue; 1248 continue;
1276 1249
1277 offset = (loff_t)vma->vm_pgof 1250 offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
1278 if (uprobe->offset < offset 1251 if (uprobe->offset < offset ||
1279 uprobe->offset >= offset 1252 uprobe->offset >= offset + vma->vm_end - vma->vm_start)
1280 continue; 1253 continue;
1281 1254
1282 vaddr = offset_to_vaddr(vma, 1255 vaddr = offset_to_vaddr(vma, uprobe->offset);
1283 err |= remove_breakpoint(upro 1256 err |= remove_breakpoint(uprobe, mm, vaddr);
1284 } 1257 }
1285 mmap_read_unlock(mm); 1258 mmap_read_unlock(mm);
1286 1259
1287 return err; 1260 return err;
1288 } 1261 }
1289 1262
1290 static struct rb_node * 1263 static struct rb_node *
1291 find_node_in_range(struct inode *inode, loff_ 1264 find_node_in_range(struct inode *inode, loff_t min, loff_t max)
1292 { 1265 {
1293 struct rb_node *n = uprobes_tree.rb_n 1266 struct rb_node *n = uprobes_tree.rb_node;
1294 1267
1295 while (n) { 1268 while (n) {
1296 struct uprobe *u = rb_entry(n 1269 struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
1297 1270
1298 if (inode < u->inode) { 1271 if (inode < u->inode) {
1299 n = n->rb_left; 1272 n = n->rb_left;
1300 } else if (inode > u->inode) 1273 } else if (inode > u->inode) {
1301 n = n->rb_right; 1274 n = n->rb_right;
1302 } else { 1275 } else {
1303 if (max < u->offset) 1276 if (max < u->offset)
1304 n = n->rb_lef 1277 n = n->rb_left;
1305 else if (min > u->off 1278 else if (min > u->offset)
1306 n = n->rb_rig 1279 n = n->rb_right;
1307 else 1280 else
1308 break; 1281 break;
1309 } 1282 }
1310 } 1283 }
1311 1284
1312 return n; 1285 return n;
1313 } 1286 }
1314 1287
1315 /* 1288 /*
1316 * For a given range in vma, build a list of 1289 * For a given range in vma, build a list of probes that need to be inserted.
1317 */ 1290 */
1318 static void build_probe_list(struct inode *in 1291 static void build_probe_list(struct inode *inode,
1319 struct vm_are 1292 struct vm_area_struct *vma,
1320 unsigned long 1293 unsigned long start, unsigned long end,
1321 struct list_h 1294 struct list_head *head)
1322 { 1295 {
1323 loff_t min, max; 1296 loff_t min, max;
1324 struct rb_node *n, *t; 1297 struct rb_node *n, *t;
1325 struct uprobe *u; 1298 struct uprobe *u;
1326 1299
1327 INIT_LIST_HEAD(head); 1300 INIT_LIST_HEAD(head);
1328 min = vaddr_to_offset(vma, start); 1301 min = vaddr_to_offset(vma, start);
1329 max = min + (end - start) - 1; 1302 max = min + (end - start) - 1;
1330 1303
1331 read_lock(&uprobes_treelock); !! 1304 spin_lock(&uprobes_treelock);
1332 n = find_node_in_range(inode, min, ma 1305 n = find_node_in_range(inode, min, max);
1333 if (n) { 1306 if (n) {
1334 for (t = n; t; t = rb_prev(t) 1307 for (t = n; t; t = rb_prev(t)) {
1335 u = rb_entry(t, struc 1308 u = rb_entry(t, struct uprobe, rb_node);
1336 if (u->inode != inode 1309 if (u->inode != inode || u->offset < min)
1337 break; 1310 break;
1338 /* if uprobe went awa !! 1311 list_add(&u->pending_list, head);
1339 if (try_get_uprobe(u) !! 1312 get_uprobe(u);
1340 list_add(&u-> <<
1341 } 1313 }
1342 for (t = n; (t = rb_next(t)); 1314 for (t = n; (t = rb_next(t)); ) {
1343 u = rb_entry(t, struc 1315 u = rb_entry(t, struct uprobe, rb_node);
1344 if (u->inode != inode 1316 if (u->inode != inode || u->offset > max)
1345 break; 1317 break;
1346 /* if uprobe went awa !! 1318 list_add(&u->pending_list, head);
1347 if (try_get_uprobe(u) !! 1319 get_uprobe(u);
1348 list_add(&u-> <<
1349 } 1320 }
1350 } 1321 }
1351 read_unlock(&uprobes_treelock); !! 1322 spin_unlock(&uprobes_treelock);
1352 } 1323 }
1353 1324
1354 /* @vma contains reference counter, not the p 1325 /* @vma contains reference counter, not the probed instruction. */
1355 static int delayed_ref_ctr_inc(struct vm_area 1326 static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
1356 { 1327 {
1357 struct list_head *pos, *q; 1328 struct list_head *pos, *q;
1358 struct delayed_uprobe *du; 1329 struct delayed_uprobe *du;
1359 unsigned long vaddr; 1330 unsigned long vaddr;
1360 int ret = 0, err = 0; 1331 int ret = 0, err = 0;
1361 1332
1362 mutex_lock(&delayed_uprobe_lock); 1333 mutex_lock(&delayed_uprobe_lock);
1363 list_for_each_safe(pos, q, &delayed_u 1334 list_for_each_safe(pos, q, &delayed_uprobe_list) {
1364 du = list_entry(pos, struct d 1335 du = list_entry(pos, struct delayed_uprobe, list);
1365 1336
1366 if (du->mm != vma->vm_mm || 1337 if (du->mm != vma->vm_mm ||
1367 !valid_ref_ctr_vma(du->up 1338 !valid_ref_ctr_vma(du->uprobe, vma))
1368 continue; 1339 continue;
1369 1340
1370 vaddr = offset_to_vaddr(vma, 1341 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
1371 ret = __update_ref_ctr(vma->v 1342 ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
1372 if (ret) { 1343 if (ret) {
1373 update_ref_ctr_warn(d 1344 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
1374 if (!err) 1345 if (!err)
1375 err = ret; 1346 err = ret;
1376 } 1347 }
1377 delayed_uprobe_delete(du); 1348 delayed_uprobe_delete(du);
1378 } 1349 }
1379 mutex_unlock(&delayed_uprobe_lock); 1350 mutex_unlock(&delayed_uprobe_lock);
1380 return err; 1351 return err;
1381 } 1352 }
1382 1353
1383 /* 1354 /*
1384 * Called from mmap_region/vma_merge with mm- !! 1355 * Called from mmap_region/vma_adjust with mm->mmap_lock acquired.
1385 * 1356 *
1386 * Currently we ignore all errors and always 1357 * Currently we ignore all errors and always return 0, the callers
1387 * can't handle the failure anyway. 1358 * can't handle the failure anyway.
1388 */ 1359 */
1389 int uprobe_mmap(struct vm_area_struct *vma) 1360 int uprobe_mmap(struct vm_area_struct *vma)
1390 { 1361 {
1391 struct list_head tmp_list; 1362 struct list_head tmp_list;
1392 struct uprobe *uprobe, *u; 1363 struct uprobe *uprobe, *u;
1393 struct inode *inode; 1364 struct inode *inode;
1394 1365
1395 if (no_uprobe_events()) 1366 if (no_uprobe_events())
1396 return 0; 1367 return 0;
1397 1368
1398 if (vma->vm_file && 1369 if (vma->vm_file &&
1399 (vma->vm_flags & (VM_WRITE|VM_SHA 1370 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
1400 test_bit(MMF_HAS_UPROBES, &vma->v 1371 test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
1401 delayed_ref_ctr_inc(vma); 1372 delayed_ref_ctr_inc(vma);
1402 1373
1403 if (!valid_vma(vma, true)) 1374 if (!valid_vma(vma, true))
1404 return 0; 1375 return 0;
1405 1376
1406 inode = file_inode(vma->vm_file); 1377 inode = file_inode(vma->vm_file);
1407 if (!inode) 1378 if (!inode)
1408 return 0; 1379 return 0;
1409 1380
1410 mutex_lock(uprobes_mmap_hash(inode)); 1381 mutex_lock(uprobes_mmap_hash(inode));
1411 build_probe_list(inode, vma, vma->vm_ 1382 build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
1412 /* 1383 /*
1413 * We can race with uprobe_unregister 1384 * We can race with uprobe_unregister(), this uprobe can be already
1414 * removed. But in this case filter_c 1385 * removed. But in this case filter_chain() must return false, all
1415 * consumers have gone away. 1386 * consumers have gone away.
1416 */ 1387 */
1417 list_for_each_entry_safe(uprobe, u, & 1388 list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
1418 if (!fatal_signal_pending(cur 1389 if (!fatal_signal_pending(current) &&
1419 filter_chain(uprobe, vma- !! 1390 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
1420 unsigned long vaddr = 1391 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
1421 install_breakpoint(up 1392 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
1422 } 1393 }
1423 put_uprobe(uprobe); 1394 put_uprobe(uprobe);
1424 } 1395 }
1425 mutex_unlock(uprobes_mmap_hash(inode) 1396 mutex_unlock(uprobes_mmap_hash(inode));
1426 1397
1427 return 0; 1398 return 0;
1428 } 1399 }
1429 1400
1430 static bool 1401 static bool
1431 vma_has_uprobes(struct vm_area_struct *vma, u 1402 vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1432 { 1403 {
1433 loff_t min, max; 1404 loff_t min, max;
1434 struct inode *inode; 1405 struct inode *inode;
1435 struct rb_node *n; 1406 struct rb_node *n;
1436 1407
1437 inode = file_inode(vma->vm_file); 1408 inode = file_inode(vma->vm_file);
1438 1409
1439 min = vaddr_to_offset(vma, start); 1410 min = vaddr_to_offset(vma, start);
1440 max = min + (end - start) - 1; 1411 max = min + (end - start) - 1;
1441 1412
1442 read_lock(&uprobes_treelock); !! 1413 spin_lock(&uprobes_treelock);
1443 n = find_node_in_range(inode, min, ma 1414 n = find_node_in_range(inode, min, max);
1444 read_unlock(&uprobes_treelock); !! 1415 spin_unlock(&uprobes_treelock);
1445 1416
1446 return !!n; 1417 return !!n;
1447 } 1418 }
1448 1419
1449 /* 1420 /*
1450 * Called in context of a munmap of a vma. 1421 * Called in context of a munmap of a vma.
1451 */ 1422 */
1452 void uprobe_munmap(struct vm_area_struct *vma 1423 void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1453 { 1424 {
1454 if (no_uprobe_events() || !valid_vma( 1425 if (no_uprobe_events() || !valid_vma(vma, false))
1455 return; 1426 return;
1456 1427
1457 if (!atomic_read(&vma->vm_mm->mm_user 1428 if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
1458 return; 1429 return;
1459 1430
1460 if (!test_bit(MMF_HAS_UPROBES, &vma-> 1431 if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
1461 test_bit(MMF_RECALC_UPROBES, &vm 1432 test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
1462 return; 1433 return;
1463 1434
1464 if (vma_has_uprobes(vma, start, end)) 1435 if (vma_has_uprobes(vma, start, end))
1465 set_bit(MMF_RECALC_UPROBES, & 1436 set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
1466 } 1437 }
1467 1438
1468 static vm_fault_t xol_fault(const struct vm_s <<
1469 struct vm_area_st <<
1470 { <<
1471 struct xol_area *area = vma->vm_mm->u <<
1472 <<
1473 vmf->page = area->page; <<
1474 get_page(vmf->page); <<
1475 return 0; <<
1476 } <<
1477 <<
1478 static const struct vm_special_mapping xol_ma <<
1479 .name = "[uprobes]", <<
1480 .fault = xol_fault, <<
1481 }; <<
1482 <<
1483 /* Slot allocation for XOL */ 1439 /* Slot allocation for XOL */
1484 static int xol_add_vma(struct mm_struct *mm, 1440 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1485 { 1441 {
1486 struct vm_area_struct *vma; 1442 struct vm_area_struct *vma;
1487 int ret; 1443 int ret;
1488 1444
1489 if (mmap_write_lock_killable(mm)) 1445 if (mmap_write_lock_killable(mm))
1490 return -EINTR; 1446 return -EINTR;
1491 1447
1492 if (mm->uprobes_state.xol_area) { 1448 if (mm->uprobes_state.xol_area) {
1493 ret = -EALREADY; 1449 ret = -EALREADY;
1494 goto fail; 1450 goto fail;
1495 } 1451 }
1496 1452
1497 if (!area->vaddr) { 1453 if (!area->vaddr) {
1498 /* Try to map as high as poss 1454 /* Try to map as high as possible, this is only a hint. */
1499 area->vaddr = get_unmapped_ar 1455 area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1500 1456 PAGE_SIZE, 0, 0);
1501 if (IS_ERR_VALUE(area->vaddr) 1457 if (IS_ERR_VALUE(area->vaddr)) {
1502 ret = area->vaddr; 1458 ret = area->vaddr;
1503 goto fail; 1459 goto fail;
1504 } 1460 }
1505 } 1461 }
1506 1462
1507 vma = _install_special_mapping(mm, ar 1463 vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1508 VM_EXEC|VM_MA 1464 VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
1509 &xol_mapping) !! 1465 &area->xol_mapping);
1510 if (IS_ERR(vma)) { 1466 if (IS_ERR(vma)) {
1511 ret = PTR_ERR(vma); 1467 ret = PTR_ERR(vma);
1512 goto fail; 1468 goto fail;
1513 } 1469 }
1514 1470
1515 ret = 0; 1471 ret = 0;
1516 /* pairs with get_xol_area() */ 1472 /* pairs with get_xol_area() */
1517 smp_store_release(&mm->uprobes_state. 1473 smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
1518 fail: 1474 fail:
1519 mmap_write_unlock(mm); 1475 mmap_write_unlock(mm);
1520 1476
1521 return ret; 1477 return ret;
1522 } 1478 }
1523 1479
1524 void * __weak arch_uprobe_trampoline(unsigned <<
1525 { <<
1526 static uprobe_opcode_t insn = UPROBE_ <<
1527 <<
1528 *psize = UPROBE_SWBP_INSN_SIZE; <<
1529 return &insn; <<
1530 } <<
1531 <<
1532 static struct xol_area *__create_xol_area(uns 1480 static struct xol_area *__create_xol_area(unsigned long vaddr)
1533 { 1481 {
1534 struct mm_struct *mm = current->mm; 1482 struct mm_struct *mm = current->mm;
1535 unsigned long insns_size; !! 1483 uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1536 struct xol_area *area; 1484 struct xol_area *area;
1537 void *insns; <<
1538 1485
1539 area = kzalloc(sizeof(*area), GFP_KER !! 1486 area = kmalloc(sizeof(*area), GFP_KERNEL);
1540 if (unlikely(!area)) 1487 if (unlikely(!area))
1541 goto out; 1488 goto out;
1542 1489
1543 area->bitmap = kcalloc(BITS_TO_LONGS( 1490 area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
1544 GFP_KERNEL); 1491 GFP_KERNEL);
1545 if (!area->bitmap) 1492 if (!area->bitmap)
1546 goto free_area; 1493 goto free_area;
1547 1494
1548 area->page = alloc_page(GFP_HIGHUSER !! 1495 area->xol_mapping.name = "[uprobes]";
1549 if (!area->page) !! 1496 area->xol_mapping.fault = NULL;
>> 1497 area->xol_mapping.pages = area->pages;
>> 1498 area->pages[0] = alloc_page(GFP_HIGHUSER);
>> 1499 if (!area->pages[0])
1550 goto free_bitmap; 1500 goto free_bitmap;
>> 1501 area->pages[1] = NULL;
1551 1502
1552 area->vaddr = vaddr; 1503 area->vaddr = vaddr;
1553 init_waitqueue_head(&area->wq); 1504 init_waitqueue_head(&area->wq);
1554 /* Reserve the 1st slot for get_tramp 1505 /* Reserve the 1st slot for get_trampoline_vaddr() */
1555 set_bit(0, area->bitmap); 1506 set_bit(0, area->bitmap);
1556 atomic_set(&area->slot_count, 1); 1507 atomic_set(&area->slot_count, 1);
1557 insns = arch_uprobe_trampoline(&insns !! 1508 arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
1558 arch_uprobe_copy_ixol(area->page, 0, <<
1559 1509
1560 if (!xol_add_vma(mm, area)) 1510 if (!xol_add_vma(mm, area))
1561 return area; 1511 return area;
1562 1512
1563 __free_page(area->page); !! 1513 __free_page(area->pages[0]);
1564 free_bitmap: 1514 free_bitmap:
1565 kfree(area->bitmap); 1515 kfree(area->bitmap);
1566 free_area: 1516 free_area:
1567 kfree(area); 1517 kfree(area);
1568 out: 1518 out:
1569 return NULL; 1519 return NULL;
1570 } 1520 }
1571 1521
1572 /* 1522 /*
1573 * get_xol_area - Allocate process's xol_area 1523 * get_xol_area - Allocate process's xol_area if necessary.
1574 * This area will be used for storing instruc 1524 * This area will be used for storing instructions for execution out of line.
1575 * 1525 *
1576 * Returns the allocated area or NULL. 1526 * Returns the allocated area or NULL.
1577 */ 1527 */
1578 static struct xol_area *get_xol_area(void) 1528 static struct xol_area *get_xol_area(void)
1579 { 1529 {
1580 struct mm_struct *mm = current->mm; 1530 struct mm_struct *mm = current->mm;
1581 struct xol_area *area; 1531 struct xol_area *area;
1582 1532
1583 if (!mm->uprobes_state.xol_area) 1533 if (!mm->uprobes_state.xol_area)
1584 __create_xol_area(0); 1534 __create_xol_area(0);
1585 1535
1586 /* Pairs with xol_add_vma() smp_store 1536 /* Pairs with xol_add_vma() smp_store_release() */
1587 area = READ_ONCE(mm->uprobes_state.xo 1537 area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
1588 return area; 1538 return area;
1589 } 1539 }
1590 1540
1591 /* 1541 /*
1592 * uprobe_clear_state - Free the area allocat 1542 * uprobe_clear_state - Free the area allocated for slots.
1593 */ 1543 */
1594 void uprobe_clear_state(struct mm_struct *mm) 1544 void uprobe_clear_state(struct mm_struct *mm)
1595 { 1545 {
1596 struct xol_area *area = mm->uprobes_s 1546 struct xol_area *area = mm->uprobes_state.xol_area;
1597 1547
1598 mutex_lock(&delayed_uprobe_lock); 1548 mutex_lock(&delayed_uprobe_lock);
1599 delayed_uprobe_remove(NULL, mm); 1549 delayed_uprobe_remove(NULL, mm);
1600 mutex_unlock(&delayed_uprobe_lock); 1550 mutex_unlock(&delayed_uprobe_lock);
1601 1551
1602 if (!area) 1552 if (!area)
1603 return; 1553 return;
1604 1554
1605 put_page(area->page); !! 1555 put_page(area->pages[0]);
1606 kfree(area->bitmap); 1556 kfree(area->bitmap);
1607 kfree(area); 1557 kfree(area);
1608 } 1558 }
1609 1559
1610 void uprobe_start_dup_mmap(void) 1560 void uprobe_start_dup_mmap(void)
1611 { 1561 {
1612 percpu_down_read(&dup_mmap_sem); 1562 percpu_down_read(&dup_mmap_sem);
1613 } 1563 }
1614 1564
1615 void uprobe_end_dup_mmap(void) 1565 void uprobe_end_dup_mmap(void)
1616 { 1566 {
1617 percpu_up_read(&dup_mmap_sem); 1567 percpu_up_read(&dup_mmap_sem);
1618 } 1568 }
1619 1569
1620 void uprobe_dup_mmap(struct mm_struct *oldmm, 1570 void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
1621 { 1571 {
1622 if (test_bit(MMF_HAS_UPROBES, &oldmm- 1572 if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
1623 set_bit(MMF_HAS_UPROBES, &new 1573 set_bit(MMF_HAS_UPROBES, &newmm->flags);
1624 /* unconditionally, dup_mmap( 1574 /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
1625 set_bit(MMF_RECALC_UPROBES, & 1575 set_bit(MMF_RECALC_UPROBES, &newmm->flags);
1626 } 1576 }
1627 } 1577 }
1628 1578
1629 /* 1579 /*
1630 * - search for a free slot. 1580 * - search for a free slot.
1631 */ 1581 */
1632 static unsigned long xol_take_insn_slot(struc 1582 static unsigned long xol_take_insn_slot(struct xol_area *area)
1633 { 1583 {
1634 unsigned long slot_addr; 1584 unsigned long slot_addr;
1635 int slot_nr; 1585 int slot_nr;
1636 1586
1637 do { 1587 do {
1638 slot_nr = find_first_zero_bit 1588 slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1639 if (slot_nr < UINSNS_PER_PAGE 1589 if (slot_nr < UINSNS_PER_PAGE) {
1640 if (!test_and_set_bit 1590 if (!test_and_set_bit(slot_nr, area->bitmap))
1641 break; 1591 break;
1642 1592
1643 slot_nr = UINSNS_PER_ 1593 slot_nr = UINSNS_PER_PAGE;
1644 continue; 1594 continue;
1645 } 1595 }
1646 wait_event(area->wq, (atomic_ 1596 wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1647 } while (slot_nr >= UINSNS_PER_PAGE); 1597 } while (slot_nr >= UINSNS_PER_PAGE);
1648 1598
1649 slot_addr = area->vaddr + (slot_nr * 1599 slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1650 atomic_inc(&area->slot_count); 1600 atomic_inc(&area->slot_count);
1651 1601
1652 return slot_addr; 1602 return slot_addr;
1653 } 1603 }
1654 1604
1655 /* 1605 /*
1656 * xol_get_insn_slot - allocate a slot for xo 1606 * xol_get_insn_slot - allocate a slot for xol.
1657 * Returns the allocated slot address or 0. 1607 * Returns the allocated slot address or 0.
1658 */ 1608 */
1659 static unsigned long xol_get_insn_slot(struct 1609 static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
1660 { 1610 {
1661 struct xol_area *area; 1611 struct xol_area *area;
1662 unsigned long xol_vaddr; 1612 unsigned long xol_vaddr;
1663 1613
1664 area = get_xol_area(); 1614 area = get_xol_area();
1665 if (!area) 1615 if (!area)
1666 return 0; 1616 return 0;
1667 1617
1668 xol_vaddr = xol_take_insn_slot(area); 1618 xol_vaddr = xol_take_insn_slot(area);
1669 if (unlikely(!xol_vaddr)) 1619 if (unlikely(!xol_vaddr))
1670 return 0; 1620 return 0;
1671 1621
1672 arch_uprobe_copy_ixol(area->page, xol !! 1622 arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
1673 &uprobe->arch.i 1623 &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1674 1624
1675 return xol_vaddr; 1625 return xol_vaddr;
1676 } 1626 }
1677 1627
1678 /* 1628 /*
1679 * xol_free_insn_slot - If slot was earlier a 1629 * xol_free_insn_slot - If slot was earlier allocated by
1680 * @xol_get_insn_slot(), make the slot availa 1630 * @xol_get_insn_slot(), make the slot available for
1681 * subsequent requests. 1631 * subsequent requests.
1682 */ 1632 */
1683 static void xol_free_insn_slot(struct task_st 1633 static void xol_free_insn_slot(struct task_struct *tsk)
1684 { 1634 {
1685 struct xol_area *area; 1635 struct xol_area *area;
1686 unsigned long vma_end; 1636 unsigned long vma_end;
1687 unsigned long slot_addr; 1637 unsigned long slot_addr;
1688 1638
1689 if (!tsk->mm || !tsk->mm->uprobes_sta 1639 if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
1690 return; 1640 return;
1691 1641
1692 slot_addr = tsk->utask->xol_vaddr; 1642 slot_addr = tsk->utask->xol_vaddr;
1693 if (unlikely(!slot_addr)) 1643 if (unlikely(!slot_addr))
1694 return; 1644 return;
1695 1645
1696 area = tsk->mm->uprobes_state.xol_are 1646 area = tsk->mm->uprobes_state.xol_area;
1697 vma_end = area->vaddr + PAGE_SIZE; 1647 vma_end = area->vaddr + PAGE_SIZE;
1698 if (area->vaddr <= slot_addr && slot_ 1648 if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1699 unsigned long offset; 1649 unsigned long offset;
1700 int slot_nr; 1650 int slot_nr;
1701 1651
1702 offset = slot_addr - area->va 1652 offset = slot_addr - area->vaddr;
1703 slot_nr = offset / UPROBE_XOL 1653 slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1704 if (slot_nr >= UINSNS_PER_PAG 1654 if (slot_nr >= UINSNS_PER_PAGE)
1705 return; 1655 return;
1706 1656
1707 clear_bit(slot_nr, area->bitm 1657 clear_bit(slot_nr, area->bitmap);
1708 atomic_dec(&area->slot_count) 1658 atomic_dec(&area->slot_count);
1709 smp_mb__after_atomic(); /* pa 1659 smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1710 if (waitqueue_active(&area->w 1660 if (waitqueue_active(&area->wq))
1711 wake_up(&area->wq); 1661 wake_up(&area->wq);
1712 1662
1713 tsk->utask->xol_vaddr = 0; 1663 tsk->utask->xol_vaddr = 0;
1714 } 1664 }
1715 } 1665 }
1716 1666
1717 void __weak arch_uprobe_copy_ixol(struct page 1667 void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
1718 void *src, 1668 void *src, unsigned long len)
1719 { 1669 {
1720 /* Initialize the slot */ 1670 /* Initialize the slot */
1721 copy_to_page(page, vaddr, src, len); 1671 copy_to_page(page, vaddr, src, len);
1722 1672
1723 /* 1673 /*
1724 * We probably need flush_icache_user 1674 * We probably need flush_icache_user_page() but it needs vma.
1725 * This should work on most of archit 1675 * This should work on most of architectures by default. If
1726 * architecture needs to do something 1676 * architecture needs to do something different it can define
1727 * its own version of the function. 1677 * its own version of the function.
1728 */ 1678 */
1729 flush_dcache_page(page); 1679 flush_dcache_page(page);
1730 } 1680 }
1731 1681
1732 /** 1682 /**
1733 * uprobe_get_swbp_addr - compute address of 1683 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1734 * @regs: Reflects the saved state of the tas 1684 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1735 * instruction. 1685 * instruction.
1736 * Return the address of the breakpoint instr 1686 * Return the address of the breakpoint instruction.
1737 */ 1687 */
1738 unsigned long __weak uprobe_get_swbp_addr(str 1688 unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1739 { 1689 {
1740 return instruction_pointer(regs) - UP 1690 return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1741 } 1691 }
1742 1692
1743 unsigned long uprobe_get_trap_addr(struct pt_ 1693 unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
1744 { 1694 {
1745 struct uprobe_task *utask = current-> 1695 struct uprobe_task *utask = current->utask;
1746 1696
1747 if (unlikely(utask && utask->active_u 1697 if (unlikely(utask && utask->active_uprobe))
1748 return utask->vaddr; 1698 return utask->vaddr;
1749 1699
1750 return instruction_pointer(regs); 1700 return instruction_pointer(regs);
1751 } 1701 }
1752 1702
1753 static struct return_instance *free_ret_insta 1703 static struct return_instance *free_ret_instance(struct return_instance *ri)
1754 { 1704 {
1755 struct return_instance *next = ri->ne 1705 struct return_instance *next = ri->next;
1756 put_uprobe(ri->uprobe); 1706 put_uprobe(ri->uprobe);
1757 kfree(ri); 1707 kfree(ri);
1758 return next; 1708 return next;
1759 } 1709 }
1760 1710
1761 /* 1711 /*
1762 * Called with no locks held. 1712 * Called with no locks held.
1763 * Called in context of an exiting or an exec 1713 * Called in context of an exiting or an exec-ing thread.
1764 */ 1714 */
1765 void uprobe_free_utask(struct task_struct *t) 1715 void uprobe_free_utask(struct task_struct *t)
1766 { 1716 {
1767 struct uprobe_task *utask = t->utask; 1717 struct uprobe_task *utask = t->utask;
1768 struct return_instance *ri; 1718 struct return_instance *ri;
1769 1719
1770 if (!utask) 1720 if (!utask)
1771 return; 1721 return;
1772 1722
1773 if (utask->active_uprobe) 1723 if (utask->active_uprobe)
1774 put_uprobe(utask->active_upro 1724 put_uprobe(utask->active_uprobe);
1775 1725
1776 ri = utask->return_instances; 1726 ri = utask->return_instances;
1777 while (ri) 1727 while (ri)
1778 ri = free_ret_instance(ri); 1728 ri = free_ret_instance(ri);
1779 1729
1780 xol_free_insn_slot(t); 1730 xol_free_insn_slot(t);
1781 kfree(utask); 1731 kfree(utask);
1782 t->utask = NULL; 1732 t->utask = NULL;
1783 } 1733 }
1784 1734
1785 /* 1735 /*
1786 * Allocate a uprobe_task object for the task 1736 * Allocate a uprobe_task object for the task if necessary.
1787 * Called when the thread hits a breakpoint. 1737 * Called when the thread hits a breakpoint.
1788 * 1738 *
1789 * Returns: 1739 * Returns:
1790 * - pointer to new uprobe_task on success 1740 * - pointer to new uprobe_task on success
1791 * - NULL otherwise 1741 * - NULL otherwise
1792 */ 1742 */
1793 static struct uprobe_task *get_utask(void) 1743 static struct uprobe_task *get_utask(void)
1794 { 1744 {
1795 if (!current->utask) 1745 if (!current->utask)
1796 current->utask = kzalloc(size 1746 current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1797 return current->utask; 1747 return current->utask;
1798 } 1748 }
1799 1749
1800 static int dup_utask(struct task_struct *t, s 1750 static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
1801 { 1751 {
1802 struct uprobe_task *n_utask; 1752 struct uprobe_task *n_utask;
1803 struct return_instance **p, *o, *n; 1753 struct return_instance **p, *o, *n;
1804 1754
1805 n_utask = kzalloc(sizeof(struct uprob 1755 n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1806 if (!n_utask) 1756 if (!n_utask)
1807 return -ENOMEM; 1757 return -ENOMEM;
1808 t->utask = n_utask; 1758 t->utask = n_utask;
1809 1759
1810 p = &n_utask->return_instances; 1760 p = &n_utask->return_instances;
1811 for (o = o_utask->return_instances; o 1761 for (o = o_utask->return_instances; o; o = o->next) {
1812 n = kmalloc(sizeof(struct ret 1762 n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1813 if (!n) 1763 if (!n)
1814 return -ENOMEM; 1764 return -ENOMEM;
1815 1765
1816 *n = *o; 1766 *n = *o;
1817 /* <<
1818 * uprobe's refcnt has to be <<
1819 * utask->return_instances it <<
1820 * removed right now, as task <<
1821 * get_uprobe() is safe to us <<
1822 */ <<
1823 get_uprobe(n->uprobe); 1767 get_uprobe(n->uprobe);
1824 n->next = NULL; 1768 n->next = NULL;
1825 1769
1826 *p = n; 1770 *p = n;
1827 p = &n->next; 1771 p = &n->next;
1828 n_utask->depth++; 1772 n_utask->depth++;
1829 } 1773 }
1830 1774
1831 return 0; 1775 return 0;
1832 } 1776 }
1833 1777
>> 1778 static void uprobe_warn(struct task_struct *t, const char *msg)
>> 1779 {
>> 1780 pr_warn("uprobe: %s:%d failed to %s\n",
>> 1781 current->comm, current->pid, msg);
>> 1782 }
>> 1783
1834 static void dup_xol_work(struct callback_head 1784 static void dup_xol_work(struct callback_head *work)
1835 { 1785 {
1836 if (current->flags & PF_EXITING) 1786 if (current->flags & PF_EXITING)
1837 return; 1787 return;
1838 1788
1839 if (!__create_xol_area(current->utask 1789 if (!__create_xol_area(current->utask->dup_xol_addr) &&
1840 !fatal_signal_pending 1790 !fatal_signal_pending(current))
1841 uprobe_warn(current, "dup xol 1791 uprobe_warn(current, "dup xol area");
1842 } 1792 }
1843 1793
1844 /* 1794 /*
1845 * Called in context of a new clone/fork from 1795 * Called in context of a new clone/fork from copy_process.
1846 */ 1796 */
1847 void uprobe_copy_process(struct task_struct * 1797 void uprobe_copy_process(struct task_struct *t, unsigned long flags)
1848 { 1798 {
1849 struct uprobe_task *utask = current-> 1799 struct uprobe_task *utask = current->utask;
1850 struct mm_struct *mm = current->mm; 1800 struct mm_struct *mm = current->mm;
1851 struct xol_area *area; 1801 struct xol_area *area;
1852 1802
1853 t->utask = NULL; 1803 t->utask = NULL;
1854 1804
1855 if (!utask || !utask->return_instance 1805 if (!utask || !utask->return_instances)
1856 return; 1806 return;
1857 1807
1858 if (mm == t->mm && !(flags & CLONE_VF 1808 if (mm == t->mm && !(flags & CLONE_VFORK))
1859 return; 1809 return;
1860 1810
1861 if (dup_utask(t, utask)) 1811 if (dup_utask(t, utask))
1862 return uprobe_warn(t, "dup re 1812 return uprobe_warn(t, "dup ret instances");
1863 1813
1864 /* The task can fork() after dup_xol_ 1814 /* The task can fork() after dup_xol_work() fails */
1865 area = mm->uprobes_state.xol_area; 1815 area = mm->uprobes_state.xol_area;
1866 if (!area) 1816 if (!area)
1867 return uprobe_warn(t, "dup xo 1817 return uprobe_warn(t, "dup xol area");
1868 1818
1869 if (mm == t->mm) 1819 if (mm == t->mm)
1870 return; 1820 return;
1871 1821
1872 t->utask->dup_xol_addr = area->vaddr; 1822 t->utask->dup_xol_addr = area->vaddr;
1873 init_task_work(&t->utask->dup_xol_wor 1823 init_task_work(&t->utask->dup_xol_work, dup_xol_work);
1874 task_work_add(t, &t->utask->dup_xol_w 1824 task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME);
1875 } 1825 }
1876 1826
1877 /* 1827 /*
1878 * Current area->vaddr notion assume the tram 1828 * Current area->vaddr notion assume the trampoline address is always
1879 * equal area->vaddr. 1829 * equal area->vaddr.
1880 * 1830 *
1881 * Returns -1 in case the xol_area is not all 1831 * Returns -1 in case the xol_area is not allocated.
1882 */ 1832 */
1883 unsigned long uprobe_get_trampoline_vaddr(voi !! 1833 static unsigned long get_trampoline_vaddr(void)
1884 { 1834 {
1885 struct xol_area *area; 1835 struct xol_area *area;
1886 unsigned long trampoline_vaddr = -1; 1836 unsigned long trampoline_vaddr = -1;
1887 1837
1888 /* Pairs with xol_add_vma() smp_store 1838 /* Pairs with xol_add_vma() smp_store_release() */
1889 area = READ_ONCE(current->mm->uprobes 1839 area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
1890 if (area) 1840 if (area)
1891 trampoline_vaddr = area->vadd 1841 trampoline_vaddr = area->vaddr;
1892 1842
1893 return trampoline_vaddr; 1843 return trampoline_vaddr;
1894 } 1844 }
1895 1845
1896 static void cleanup_return_instances(struct u 1846 static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
1897 struc 1847 struct pt_regs *regs)
1898 { 1848 {
1899 struct return_instance *ri = utask->r 1849 struct return_instance *ri = utask->return_instances;
1900 enum rp_check ctx = chained ? RP_CHEC 1850 enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
1901 1851
1902 while (ri && !arch_uretprobe_is_alive 1852 while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
1903 ri = free_ret_instance(ri); 1853 ri = free_ret_instance(ri);
1904 utask->depth--; 1854 utask->depth--;
1905 } 1855 }
1906 utask->return_instances = ri; 1856 utask->return_instances = ri;
1907 } 1857 }
1908 1858
1909 static void prepare_uretprobe(struct uprobe * 1859 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
1910 { 1860 {
1911 struct return_instance *ri; 1861 struct return_instance *ri;
1912 struct uprobe_task *utask; 1862 struct uprobe_task *utask;
1913 unsigned long orig_ret_vaddr, trampol 1863 unsigned long orig_ret_vaddr, trampoline_vaddr;
1914 bool chained; 1864 bool chained;
1915 1865
1916 if (!get_xol_area()) 1866 if (!get_xol_area())
1917 return; 1867 return;
1918 1868
1919 utask = get_utask(); 1869 utask = get_utask();
1920 if (!utask) 1870 if (!utask)
1921 return; 1871 return;
1922 1872
1923 if (utask->depth >= MAX_URETPROBE_DEP 1873 if (utask->depth >= MAX_URETPROBE_DEPTH) {
1924 printk_ratelimited(KERN_INFO 1874 printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
1925 " nestedness 1875 " nestedness limit pid/tgid=%d/%d\n",
1926 current->pid, 1876 current->pid, current->tgid);
1927 return; 1877 return;
1928 } 1878 }
1929 1879
1930 /* we need to bump refcount to store <<
1931 if (!try_get_uprobe(uprobe)) <<
1932 return; <<
1933 <<
1934 ri = kmalloc(sizeof(struct return_ins 1880 ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1935 if (!ri) 1881 if (!ri)
1936 goto fail; !! 1882 return;
1937 1883
1938 trampoline_vaddr = uprobe_get_trampol !! 1884 trampoline_vaddr = get_trampoline_vaddr();
1939 orig_ret_vaddr = arch_uretprobe_hijac 1885 orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1940 if (orig_ret_vaddr == -1) 1886 if (orig_ret_vaddr == -1)
1941 goto fail; 1887 goto fail;
1942 1888
1943 /* drop the entries invalidated by lo 1889 /* drop the entries invalidated by longjmp() */
1944 chained = (orig_ret_vaddr == trampoli 1890 chained = (orig_ret_vaddr == trampoline_vaddr);
1945 cleanup_return_instances(utask, chain 1891 cleanup_return_instances(utask, chained, regs);
1946 1892
1947 /* 1893 /*
1948 * We don't want to keep trampoline a 1894 * We don't want to keep trampoline address in stack, rather keep the
1949 * original return address of first c 1895 * original return address of first caller thru all the consequent
1950 * instances. This also makes breakpo 1896 * instances. This also makes breakpoint unwrapping easier.
1951 */ 1897 */
1952 if (chained) { 1898 if (chained) {
1953 if (!utask->return_instances) 1899 if (!utask->return_instances) {
1954 /* 1900 /*
1955 * This situation is 1901 * This situation is not possible. Likely we have an
1956 * attack from user-s 1902 * attack from user-space.
1957 */ 1903 */
1958 uprobe_warn(current, 1904 uprobe_warn(current, "handle tail call");
1959 goto fail; 1905 goto fail;
1960 } 1906 }
1961 orig_ret_vaddr = utask->retur 1907 orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
1962 } 1908 }
1963 ri->uprobe = uprobe; !! 1909
>> 1910 ri->uprobe = get_uprobe(uprobe);
1964 ri->func = instruction_pointer(regs); 1911 ri->func = instruction_pointer(regs);
1965 ri->stack = user_stack_pointer(regs); 1912 ri->stack = user_stack_pointer(regs);
1966 ri->orig_ret_vaddr = orig_ret_vaddr; 1913 ri->orig_ret_vaddr = orig_ret_vaddr;
1967 ri->chained = chained; 1914 ri->chained = chained;
1968 1915
1969 utask->depth++; 1916 utask->depth++;
1970 ri->next = utask->return_instances; 1917 ri->next = utask->return_instances;
1971 utask->return_instances = ri; 1918 utask->return_instances = ri;
1972 1919
1973 return; 1920 return;
1974 fail: !! 1921 fail:
1975 kfree(ri); 1922 kfree(ri);
1976 put_uprobe(uprobe); <<
1977 } 1923 }
1978 1924
1979 /* Prepare to single-step probed instruction 1925 /* Prepare to single-step probed instruction out of line. */
1980 static int 1926 static int
1981 pre_ssout(struct uprobe *uprobe, struct pt_re 1927 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
1982 { 1928 {
1983 struct uprobe_task *utask; 1929 struct uprobe_task *utask;
1984 unsigned long xol_vaddr; 1930 unsigned long xol_vaddr;
1985 int err; 1931 int err;
1986 1932
1987 utask = get_utask(); 1933 utask = get_utask();
1988 if (!utask) 1934 if (!utask)
1989 return -ENOMEM; 1935 return -ENOMEM;
1990 1936
1991 if (!try_get_uprobe(uprobe)) <<
1992 return -EINVAL; <<
1993 <<
1994 xol_vaddr = xol_get_insn_slot(uprobe) 1937 xol_vaddr = xol_get_insn_slot(uprobe);
1995 if (!xol_vaddr) { !! 1938 if (!xol_vaddr)
1996 err = -ENOMEM; !! 1939 return -ENOMEM;
1997 goto err_out; <<
1998 } <<
1999 1940
2000 utask->xol_vaddr = xol_vaddr; 1941 utask->xol_vaddr = xol_vaddr;
2001 utask->vaddr = bp_vaddr; 1942 utask->vaddr = bp_vaddr;
2002 1943
2003 err = arch_uprobe_pre_xol(&uprobe->ar 1944 err = arch_uprobe_pre_xol(&uprobe->arch, regs);
2004 if (unlikely(err)) { 1945 if (unlikely(err)) {
2005 xol_free_insn_slot(current); 1946 xol_free_insn_slot(current);
2006 goto err_out; !! 1947 return err;
2007 } 1948 }
2008 1949
2009 utask->active_uprobe = uprobe; 1950 utask->active_uprobe = uprobe;
2010 utask->state = UTASK_SSTEP; 1951 utask->state = UTASK_SSTEP;
2011 return 0; 1952 return 0;
2012 err_out: <<
2013 put_uprobe(uprobe); <<
2014 return err; <<
2015 } 1953 }
2016 1954
2017 /* 1955 /*
2018 * If we are singlestepping, then ensure this 1956 * If we are singlestepping, then ensure this thread is not connected to
2019 * non-fatal signals until completion of sing 1957 * non-fatal signals until completion of singlestep. When xol insn itself
2020 * triggers the signal, restart the original 1958 * triggers the signal, restart the original insn even if the task is
2021 * already SIGKILL'ed (since coredump should 1959 * already SIGKILL'ed (since coredump should report the correct ip). This
2022 * is even more important if the task has a h 1960 * is even more important if the task has a handler for SIGSEGV/etc, The
2023 * _same_ instruction should be repeated agai 1961 * _same_ instruction should be repeated again after return from the signal
2024 * handler, and SSTEP can never finish in thi 1962 * handler, and SSTEP can never finish in this case.
2025 */ 1963 */
2026 bool uprobe_deny_signal(void) 1964 bool uprobe_deny_signal(void)
2027 { 1965 {
2028 struct task_struct *t = current; 1966 struct task_struct *t = current;
2029 struct uprobe_task *utask = t->utask; 1967 struct uprobe_task *utask = t->utask;
2030 1968
2031 if (likely(!utask || !utask->active_u 1969 if (likely(!utask || !utask->active_uprobe))
2032 return false; 1970 return false;
2033 1971
2034 WARN_ON_ONCE(utask->state != UTASK_SS 1972 WARN_ON_ONCE(utask->state != UTASK_SSTEP);
2035 1973
2036 if (task_sigpending(t)) { 1974 if (task_sigpending(t)) {
2037 spin_lock_irq(&t->sighand->si 1975 spin_lock_irq(&t->sighand->siglock);
2038 clear_tsk_thread_flag(t, TIF_ 1976 clear_tsk_thread_flag(t, TIF_SIGPENDING);
2039 spin_unlock_irq(&t->sighand-> 1977 spin_unlock_irq(&t->sighand->siglock);
2040 1978
2041 if (__fatal_signal_pending(t) 1979 if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
2042 utask->state = UTASK_ 1980 utask->state = UTASK_SSTEP_TRAPPED;
2043 set_tsk_thread_flag(t 1981 set_tsk_thread_flag(t, TIF_UPROBE);
2044 } 1982 }
2045 } 1983 }
2046 1984
2047 return true; 1985 return true;
2048 } 1986 }
2049 1987
2050 static void mmf_recalc_uprobes(struct mm_stru 1988 static void mmf_recalc_uprobes(struct mm_struct *mm)
2051 { 1989 {
2052 VMA_ITERATOR(vmi, mm, 0); <<
2053 struct vm_area_struct *vma; 1990 struct vm_area_struct *vma;
2054 1991
2055 for_each_vma(vmi, vma) { !! 1992 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2056 if (!valid_vma(vma, false)) 1993 if (!valid_vma(vma, false))
2057 continue; 1994 continue;
2058 /* 1995 /*
2059 * This is not strictly accur 1996 * This is not strictly accurate, we can race with
2060 * uprobe_unregister() and se 1997 * uprobe_unregister() and see the already removed
2061 * uprobe if delete_uprobe() 1998 * uprobe if delete_uprobe() was not yet called.
2062 * Or this uprobe can be filt 1999 * Or this uprobe can be filtered out.
2063 */ 2000 */
2064 if (vma_has_uprobes(vma, vma- 2001 if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
2065 return; 2002 return;
2066 } 2003 }
2067 2004
2068 clear_bit(MMF_HAS_UPROBES, &mm->flags 2005 clear_bit(MMF_HAS_UPROBES, &mm->flags);
2069 } 2006 }
2070 2007
2071 static int is_trap_at_addr(struct mm_struct * 2008 static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
2072 { 2009 {
2073 struct page *page; 2010 struct page *page;
2074 uprobe_opcode_t opcode; 2011 uprobe_opcode_t opcode;
2075 int result; 2012 int result;
2076 2013
2077 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, U 2014 if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE)))
2078 return -EINVAL; 2015 return -EINVAL;
2079 2016
2080 pagefault_disable(); 2017 pagefault_disable();
2081 result = __get_user(opcode, (uprobe_o 2018 result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
2082 pagefault_enable(); 2019 pagefault_enable();
2083 2020
2084 if (likely(result == 0)) 2021 if (likely(result == 0))
2085 goto out; 2022 goto out;
2086 2023
2087 result = get_user_pages(vaddr, 1, FOL !! 2024 /*
>> 2025 * The NULL 'tsk' here ensures that any faults that occur here
>> 2026 * will not be accounted to the task. 'mm' *is* current->mm,
>> 2027 * but we treat this as a 'remote' access since it is
>> 2028 * essentially a kernel access to the memory.
>> 2029 */
>> 2030 result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page,
>> 2031 NULL, NULL);
2088 if (result < 0) 2032 if (result < 0)
2089 return result; 2033 return result;
2090 2034
2091 copy_from_page(page, vaddr, &opcode, 2035 copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
2092 put_page(page); 2036 put_page(page);
2093 out: 2037 out:
2094 /* This needs to return true for any 2038 /* This needs to return true for any variant of the trap insn */
2095 return is_trap_insn(&opcode); 2039 return is_trap_insn(&opcode);
2096 } 2040 }
2097 2041
2098 /* assumes being inside RCU protected region !! 2042 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
2099 static struct uprobe *find_active_uprobe_rcu( <<
2100 { 2043 {
2101 struct mm_struct *mm = current->mm; 2044 struct mm_struct *mm = current->mm;
2102 struct uprobe *uprobe = NULL; 2045 struct uprobe *uprobe = NULL;
2103 struct vm_area_struct *vma; 2046 struct vm_area_struct *vma;
2104 2047
2105 mmap_read_lock(mm); 2048 mmap_read_lock(mm);
2106 vma = vma_lookup(mm, bp_vaddr); !! 2049 vma = find_vma(mm, bp_vaddr);
2107 if (vma) { !! 2050 if (vma && vma->vm_start <= bp_vaddr) {
2108 if (valid_vma(vma, false)) { 2051 if (valid_vma(vma, false)) {
2109 struct inode *inode = 2052 struct inode *inode = file_inode(vma->vm_file);
2110 loff_t offset = vaddr 2053 loff_t offset = vaddr_to_offset(vma, bp_vaddr);
2111 2054
2112 uprobe = find_uprobe_ !! 2055 uprobe = find_uprobe(inode, offset);
2113 } 2056 }
2114 2057
2115 if (!uprobe) 2058 if (!uprobe)
2116 *is_swbp = is_trap_at 2059 *is_swbp = is_trap_at_addr(mm, bp_vaddr);
2117 } else { 2060 } else {
2118 *is_swbp = -EFAULT; 2061 *is_swbp = -EFAULT;
2119 } 2062 }
2120 2063
2121 if (!uprobe && test_and_clear_bit(MMF 2064 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
2122 mmf_recalc_uprobes(mm); 2065 mmf_recalc_uprobes(mm);
2123 mmap_read_unlock(mm); 2066 mmap_read_unlock(mm);
2124 2067
2125 return uprobe; 2068 return uprobe;
2126 } 2069 }
2127 2070
2128 static void handler_chain(struct uprobe *upro 2071 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
2129 { 2072 {
2130 struct uprobe_consumer *uc; 2073 struct uprobe_consumer *uc;
2131 int remove = UPROBE_HANDLER_REMOVE; 2074 int remove = UPROBE_HANDLER_REMOVE;
2132 bool need_prep = false; /* prepare re 2075 bool need_prep = false; /* prepare return uprobe, when needed */
2133 bool has_consumers = false; <<
2134 <<
2135 current->utask->auprobe = &uprobe->ar <<
2136 2076
2137 list_for_each_entry_srcu(uc, &uprobe- !! 2077 down_read(&uprobe->register_rwsem);
2138 srcu_read_lo !! 2078 for (uc = uprobe->consumers; uc; uc = uc->next) {
2139 int rc = 0; 2079 int rc = 0;
2140 2080
2141 if (uc->handler) { 2081 if (uc->handler) {
2142 rc = uc->handler(uc, 2082 rc = uc->handler(uc, regs);
2143 WARN(rc & ~UPROBE_HAN 2083 WARN(rc & ~UPROBE_HANDLER_MASK,
2144 "bad rc=0x%x 2084 "bad rc=0x%x from %ps()\n", rc, uc->handler);
2145 } 2085 }
2146 2086
2147 if (uc->ret_handler) 2087 if (uc->ret_handler)
2148 need_prep = true; 2088 need_prep = true;
2149 2089
2150 remove &= rc; 2090 remove &= rc;
2151 has_consumers = true; <<
2152 } 2091 }
2153 current->utask->auprobe = NULL; <<
2154 2092
2155 if (need_prep && !remove) 2093 if (need_prep && !remove)
2156 prepare_uretprobe(uprobe, reg 2094 prepare_uretprobe(uprobe, regs); /* put bp at return */
2157 2095
2158 if (remove && has_consumers) { !! 2096 if (remove && uprobe->consumers) {
2159 down_read(&uprobe->register_r !! 2097 WARN_ON(!uprobe_is_active(uprobe));
2160 !! 2098 unapply_uprobe(uprobe, current->mm);
2161 /* re-check that removal is s <<
2162 if (!filter_chain(uprobe, cur <<
2163 WARN_ON(!uprobe_is_ac <<
2164 unapply_uprobe(uprobe <<
2165 } <<
2166 <<
2167 up_read(&uprobe->register_rws <<
2168 } 2099 }
>> 2100 up_read(&uprobe->register_rwsem);
2169 } 2101 }
2170 2102
2171 static void 2103 static void
2172 handle_uretprobe_chain(struct return_instance 2104 handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
2173 { 2105 {
2174 struct uprobe *uprobe = ri->uprobe; 2106 struct uprobe *uprobe = ri->uprobe;
2175 struct uprobe_consumer *uc; 2107 struct uprobe_consumer *uc;
2176 int srcu_idx; <<
2177 2108
2178 srcu_idx = srcu_read_lock(&uprobes_sr !! 2109 down_read(&uprobe->register_rwsem);
2179 list_for_each_entry_srcu(uc, &uprobe- !! 2110 for (uc = uprobe->consumers; uc; uc = uc->next) {
2180 srcu_read_lo <<
2181 if (uc->ret_handler) 2111 if (uc->ret_handler)
2182 uc->ret_handler(uc, r 2112 uc->ret_handler(uc, ri->func, regs);
2183 } 2113 }
2184 srcu_read_unlock(&uprobes_srcu, srcu_ !! 2114 up_read(&uprobe->register_rwsem);
2185 } 2115 }
2186 2116
2187 static struct return_instance *find_next_ret_ 2117 static struct return_instance *find_next_ret_chain(struct return_instance *ri)
2188 { 2118 {
2189 bool chained; 2119 bool chained;
2190 2120
2191 do { 2121 do {
2192 chained = ri->chained; 2122 chained = ri->chained;
2193 ri = ri->next; /* can't be N 2123 ri = ri->next; /* can't be NULL if chained */
2194 } while (chained); 2124 } while (chained);
2195 2125
2196 return ri; 2126 return ri;
2197 } 2127 }
2198 2128
2199 void uprobe_handle_trampoline(struct pt_regs !! 2129 static void handle_trampoline(struct pt_regs *regs)
2200 { 2130 {
2201 struct uprobe_task *utask; 2131 struct uprobe_task *utask;
2202 struct return_instance *ri, *next; 2132 struct return_instance *ri, *next;
2203 bool valid; 2133 bool valid;
2204 2134
2205 utask = current->utask; 2135 utask = current->utask;
2206 if (!utask) 2136 if (!utask)
2207 goto sigill; 2137 goto sigill;
2208 2138
2209 ri = utask->return_instances; 2139 ri = utask->return_instances;
2210 if (!ri) 2140 if (!ri)
2211 goto sigill; 2141 goto sigill;
2212 2142
2213 do { 2143 do {
2214 /* 2144 /*
2215 * We should throw out the fr 2145 * We should throw out the frames invalidated by longjmp().
2216 * If this chain is valid, th 2146 * If this chain is valid, then the next one should be alive
2217 * or NULL; the latter case m 2147 * or NULL; the latter case means that nobody but ri->func
2218 * could hit this trampoline 2148 * could hit this trampoline on return. TODO: sigaltstack().
2219 */ 2149 */
2220 next = find_next_ret_chain(ri 2150 next = find_next_ret_chain(ri);
2221 valid = !next || arch_uretpro 2151 valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
2222 2152
2223 instruction_pointer_set(regs, 2153 instruction_pointer_set(regs, ri->orig_ret_vaddr);
2224 do { 2154 do {
2225 /* pop current instan <<
2226 * as it's not pendin <<
2227 * instruction pointe <<
2228 * this allows fixup_ <<
2229 * captured stack tra <<
2230 * trampoline address <<
2231 * original return ad <<
2232 */ <<
2233 utask->return_instanc <<
2234 if (valid) 2155 if (valid)
2235 handle_uretpr 2156 handle_uretprobe_chain(ri, regs);
2236 ri = free_ret_instanc 2157 ri = free_ret_instance(ri);
2237 utask->depth--; 2158 utask->depth--;
2238 } while (ri != next); 2159 } while (ri != next);
2239 } while (!valid); 2160 } while (!valid);
2240 2161
2241 utask->return_instances = ri; 2162 utask->return_instances = ri;
2242 return; 2163 return;
2243 2164
2244 sigill: 2165 sigill:
2245 uprobe_warn(current, "handle uretprob 2166 uprobe_warn(current, "handle uretprobe, sending SIGILL.");
2246 force_sig(SIGILL); 2167 force_sig(SIGILL);
2247 2168
2248 } 2169 }
2249 2170
2250 bool __weak arch_uprobe_ignore(struct arch_up 2171 bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
2251 { 2172 {
2252 return false; 2173 return false;
2253 } 2174 }
2254 2175
2255 bool __weak arch_uretprobe_is_alive(struct re 2176 bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
2256 struc 2177 struct pt_regs *regs)
2257 { 2178 {
2258 return true; 2179 return true;
2259 } 2180 }
2260 2181
2261 /* 2182 /*
2262 * Run handler and ask thread to singlestep. 2183 * Run handler and ask thread to singlestep.
2263 * Ensure all non-fatal signals cannot interr 2184 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
2264 */ 2185 */
2265 static void handle_swbp(struct pt_regs *regs) 2186 static void handle_swbp(struct pt_regs *regs)
2266 { 2187 {
2267 struct uprobe *uprobe; 2188 struct uprobe *uprobe;
2268 unsigned long bp_vaddr; 2189 unsigned long bp_vaddr;
2269 int is_swbp, srcu_idx; !! 2190 int is_swbp;
2270 2191
2271 bp_vaddr = uprobe_get_swbp_addr(regs) 2192 bp_vaddr = uprobe_get_swbp_addr(regs);
2272 if (bp_vaddr == uprobe_get_trampoline !! 2193 if (bp_vaddr == get_trampoline_vaddr())
2273 return uprobe_handle_trampoli !! 2194 return handle_trampoline(regs);
2274 <<
2275 srcu_idx = srcu_read_lock(&uprobes_sr <<
2276 2195
2277 uprobe = find_active_uprobe_rcu(bp_va !! 2196 uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
2278 if (!uprobe) { 2197 if (!uprobe) {
2279 if (is_swbp > 0) { 2198 if (is_swbp > 0) {
2280 /* No matching uprobe 2199 /* No matching uprobe; signal SIGTRAP. */
2281 force_sig(SIGTRAP); 2200 force_sig(SIGTRAP);
2282 } else { 2201 } else {
2283 /* 2202 /*
2284 * Either we raced wi 2203 * Either we raced with uprobe_unregister() or we can't
2285 * access this memory 2204 * access this memory. The latter is only possible if
2286 * another thread pla 2205 * another thread plays with our ->mm. In both cases
2287 * we can simply rest 2206 * we can simply restart. If this vma was unmapped we
2288 * can pretend this i 2207 * can pretend this insn was not executed yet and get
2289 * the (correct) SIGS 2208 * the (correct) SIGSEGV after restart.
2290 */ 2209 */
2291 instruction_pointer_s 2210 instruction_pointer_set(regs, bp_vaddr);
2292 } 2211 }
2293 goto out; !! 2212 return;
2294 } 2213 }
2295 2214
2296 /* change it in advance for ->handler 2215 /* change it in advance for ->handler() and restart */
2297 instruction_pointer_set(regs, bp_vadd 2216 instruction_pointer_set(regs, bp_vaddr);
2298 2217
2299 /* 2218 /*
2300 * TODO: move copy_insn/etc into _reg 2219 * TODO: move copy_insn/etc into _register and remove this hack.
2301 * After we hit the bp, _unregister + 2220 * After we hit the bp, _unregister + _register can install the
2302 * new and not-yet-analyzed uprobe at 2221 * new and not-yet-analyzed uprobe at the same address, restart.
2303 */ 2222 */
2304 if (unlikely(!test_bit(UPROBE_COPY_IN 2223 if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
2305 goto out; 2224 goto out;
2306 2225
2307 /* 2226 /*
2308 * Pairs with the smp_wmb() in prepar 2227 * Pairs with the smp_wmb() in prepare_uprobe().
2309 * 2228 *
2310 * Guarantees that if we see the UPRO 2229 * Guarantees that if we see the UPROBE_COPY_INSN bit set, then
2311 * we must also see the stores to &up 2230 * we must also see the stores to &uprobe->arch performed by the
2312 * prepare_uprobe() call. 2231 * prepare_uprobe() call.
2313 */ 2232 */
2314 smp_rmb(); 2233 smp_rmb();
2315 2234
2316 /* Tracing handlers use ->utask to co 2235 /* Tracing handlers use ->utask to communicate with fetch methods */
2317 if (!get_utask()) 2236 if (!get_utask())
2318 goto out; 2237 goto out;
2319 2238
2320 if (arch_uprobe_ignore(&uprobe->arch, 2239 if (arch_uprobe_ignore(&uprobe->arch, regs))
2321 goto out; 2240 goto out;
2322 2241
2323 handler_chain(uprobe, regs); 2242 handler_chain(uprobe, regs);
2324 2243
2325 if (arch_uprobe_skip_sstep(&uprobe->a 2244 if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
2326 goto out; 2245 goto out;
2327 2246
2328 if (pre_ssout(uprobe, regs, bp_vaddr) !! 2247 if (!pre_ssout(uprobe, regs, bp_vaddr))
2329 goto out; !! 2248 return;
2330 2249
2331 out: <<
2332 /* arch_uprobe_skip_sstep() succeeded 2250 /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
2333 srcu_read_unlock(&uprobes_srcu, srcu_ !! 2251 out:
>> 2252 put_uprobe(uprobe);
2334 } 2253 }
2335 2254
2336 /* 2255 /*
2337 * Perform required fix-ups and disable singl 2256 * Perform required fix-ups and disable singlestep.
2338 * Allow pending signals to take effect. 2257 * Allow pending signals to take effect.
2339 */ 2258 */
2340 static void handle_singlestep(struct uprobe_t 2259 static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
2341 { 2260 {
2342 struct uprobe *uprobe; 2261 struct uprobe *uprobe;
2343 int err = 0; 2262 int err = 0;
2344 2263
2345 uprobe = utask->active_uprobe; 2264 uprobe = utask->active_uprobe;
2346 if (utask->state == UTASK_SSTEP_ACK) 2265 if (utask->state == UTASK_SSTEP_ACK)
2347 err = arch_uprobe_post_xol(&u 2266 err = arch_uprobe_post_xol(&uprobe->arch, regs);
2348 else if (utask->state == UTASK_SSTEP_ 2267 else if (utask->state == UTASK_SSTEP_TRAPPED)
2349 arch_uprobe_abort_xol(&uprobe 2268 arch_uprobe_abort_xol(&uprobe->arch, regs);
2350 else 2269 else
2351 WARN_ON_ONCE(1); 2270 WARN_ON_ONCE(1);
2352 2271
2353 put_uprobe(uprobe); 2272 put_uprobe(uprobe);
2354 utask->active_uprobe = NULL; 2273 utask->active_uprobe = NULL;
2355 utask->state = UTASK_RUNNING; 2274 utask->state = UTASK_RUNNING;
2356 xol_free_insn_slot(current); 2275 xol_free_insn_slot(current);
2357 2276
2358 spin_lock_irq(¤t->sighand->sigl 2277 spin_lock_irq(¤t->sighand->siglock);
2359 recalc_sigpending(); /* see uprobe_de 2278 recalc_sigpending(); /* see uprobe_deny_signal() */
2360 spin_unlock_irq(¤t->sighand->si 2279 spin_unlock_irq(¤t->sighand->siglock);
2361 2280
2362 if (unlikely(err)) { 2281 if (unlikely(err)) {
2363 uprobe_warn(current, "execute 2282 uprobe_warn(current, "execute the probed insn, sending SIGILL.");
2364 force_sig(SIGILL); 2283 force_sig(SIGILL);
2365 } 2284 }
2366 } 2285 }
2367 2286
2368 /* 2287 /*
2369 * On breakpoint hit, breakpoint notifier set 2288 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
2370 * allows the thread to return from interrupt 2289 * allows the thread to return from interrupt. After that handle_swbp()
2371 * sets utask->active_uprobe. 2290 * sets utask->active_uprobe.
2372 * 2291 *
2373 * On singlestep exception, singlestep notifi 2292 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
2374 * and allows the thread to return from inter 2293 * and allows the thread to return from interrupt.
2375 * 2294 *
2376 * While returning to userspace, thread notic 2295 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
2377 * uprobe_notify_resume(). 2296 * uprobe_notify_resume().
2378 */ 2297 */
2379 void uprobe_notify_resume(struct pt_regs *reg 2298 void uprobe_notify_resume(struct pt_regs *regs)
2380 { 2299 {
2381 struct uprobe_task *utask; 2300 struct uprobe_task *utask;
2382 2301
2383 clear_thread_flag(TIF_UPROBE); 2302 clear_thread_flag(TIF_UPROBE);
2384 2303
2385 utask = current->utask; 2304 utask = current->utask;
2386 if (utask && utask->active_uprobe) 2305 if (utask && utask->active_uprobe)
2387 handle_singlestep(utask, regs 2306 handle_singlestep(utask, regs);
2388 else 2307 else
2389 handle_swbp(regs); 2308 handle_swbp(regs);
2390 } 2309 }
2391 2310
2392 /* 2311 /*
2393 * uprobe_pre_sstep_notifier gets called from 2312 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
2394 * notifier mechanism. Set TIF_UPROBE flag an 2313 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
2395 */ 2314 */
2396 int uprobe_pre_sstep_notifier(struct pt_regs 2315 int uprobe_pre_sstep_notifier(struct pt_regs *regs)
2397 { 2316 {
2398 if (!current->mm) 2317 if (!current->mm)
2399 return 0; 2318 return 0;
2400 2319
2401 if (!test_bit(MMF_HAS_UPROBES, &curre 2320 if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) &&
2402 (!current->utask || !current->uta 2321 (!current->utask || !current->utask->return_instances))
2403 return 0; 2322 return 0;
2404 2323
2405 set_thread_flag(TIF_UPROBE); 2324 set_thread_flag(TIF_UPROBE);
2406 return 1; 2325 return 1;
2407 } 2326 }
2408 2327
2409 /* 2328 /*
2410 * uprobe_post_sstep_notifier gets called in 2329 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
2411 * mechanism. Set TIF_UPROBE flag and indicat 2330 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
2412 */ 2331 */
2413 int uprobe_post_sstep_notifier(struct pt_regs 2332 int uprobe_post_sstep_notifier(struct pt_regs *regs)
2414 { 2333 {
2415 struct uprobe_task *utask = current-> 2334 struct uprobe_task *utask = current->utask;
2416 2335
2417 if (!current->mm || !utask || !utask- 2336 if (!current->mm || !utask || !utask->active_uprobe)
2418 /* task is currently not upro 2337 /* task is currently not uprobed */
2419 return 0; 2338 return 0;
2420 2339
2421 utask->state = UTASK_SSTEP_ACK; 2340 utask->state = UTASK_SSTEP_ACK;
2422 set_thread_flag(TIF_UPROBE); 2341 set_thread_flag(TIF_UPROBE);
2423 return 1; 2342 return 1;
2424 } 2343 }
2425 2344
2426 static struct notifier_block uprobe_exception 2345 static struct notifier_block uprobe_exception_nb = {
2427 .notifier_call = arch_uprobe 2346 .notifier_call = arch_uprobe_exception_notify,
2428 .priority = INT_MAX-1, 2347 .priority = INT_MAX-1, /* notified after kprobes, kgdb */
2429 }; 2348 };
2430 2349
2431 void __init uprobes_init(void) 2350 void __init uprobes_init(void)
2432 { 2351 {
2433 int i; 2352 int i;
2434 2353
2435 for (i = 0; i < UPROBES_HASH_SZ; i++) 2354 for (i = 0; i < UPROBES_HASH_SZ; i++)
2436 mutex_init(&uprobes_mmap_mute 2355 mutex_init(&uprobes_mmap_mutex[i]);
2437 2356
2438 BUG_ON(register_die_notifier(&uprobe_ 2357 BUG_ON(register_die_notifier(&uprobe_exception_nb));
2439 } 2358 }
2440 2359
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.