1 // SPDX-License-Identifier: GPL-2.0-or-later <<
2 /* 1 /*
3 * Copyright (C) 2008-2014 Mathieu Desnoyers !! 2 * Copyright (C) 2008 Mathieu Desnoyers
>> 3 *
>> 4 * This program is free software; you can redistribute it and/or modify
>> 5 * it under the terms of the GNU General Public License as published by
>> 6 * the Free Software Foundation; either version 2 of the License, or
>> 7 * (at your option) any later version.
>> 8 *
>> 9 * This program is distributed in the hope that it will be useful,
>> 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> 12 * GNU General Public License for more details.
>> 13 *
>> 14 * You should have received a copy of the GNU General Public License
>> 15 * along with this program; if not, write to the Free Software
>> 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
4 */ 17 */
5 #include <linux/module.h> 18 #include <linux/module.h>
6 #include <linux/mutex.h> 19 #include <linux/mutex.h>
7 #include <linux/types.h> 20 #include <linux/types.h>
8 #include <linux/jhash.h> 21 #include <linux/jhash.h>
9 #include <linux/list.h> 22 #include <linux/list.h>
10 #include <linux/rcupdate.h> 23 #include <linux/rcupdate.h>
11 #include <linux/tracepoint.h> 24 #include <linux/tracepoint.h>
12 #include <linux/err.h> 25 #include <linux/err.h>
13 #include <linux/slab.h> 26 #include <linux/slab.h>
14 #include <linux/sched/signal.h> !! 27 #include <linux/sched.h>
15 #include <linux/sched/task.h> <<
16 #include <linux/static_key.h> 28 #include <linux/static_key.h>
17 29
18 enum tp_func_state { !! 30 extern struct tracepoint * const __start___tracepoints_ptrs[];
19 TP_FUNC_0, !! 31 extern struct tracepoint * const __stop___tracepoints_ptrs[];
20 TP_FUNC_1, <<
21 TP_FUNC_2, <<
22 TP_FUNC_N, <<
23 }; <<
24 <<
25 extern tracepoint_ptr_t __start___tracepoints_ <<
26 extern tracepoint_ptr_t __stop___tracepoints_p <<
27 <<
28 DEFINE_SRCU(tracepoint_srcu); <<
29 EXPORT_SYMBOL_GPL(tracepoint_srcu); <<
30 <<
31 enum tp_transition_sync { <<
32 TP_TRANSITION_SYNC_1_0_1, <<
33 TP_TRANSITION_SYNC_N_2_1, <<
34 <<
35 _NR_TP_TRANSITION_SYNC, <<
36 }; <<
37 <<
38 struct tp_transition_snapshot { <<
39 unsigned long rcu; <<
40 unsigned long srcu; <<
41 bool ongoing; <<
42 }; <<
43 <<
44 /* Protected by tracepoints_mutex */ <<
45 static struct tp_transition_snapshot tp_transi <<
46 <<
47 static void tp_rcu_get_state(enum tp_transitio <<
48 { <<
49 struct tp_transition_snapshot *snapsho <<
50 <<
51 /* Keep the latest get_state snapshot. <<
52 snapshot->rcu = get_state_synchronize_ <<
53 snapshot->srcu = start_poll_synchroniz <<
54 snapshot->ongoing = true; <<
55 } <<
56 <<
57 static void tp_rcu_cond_sync(enum tp_transitio <<
58 { <<
59 struct tp_transition_snapshot *snapsho <<
60 <<
61 if (!snapshot->ongoing) <<
62 return; <<
63 cond_synchronize_rcu(snapshot->rcu); <<
64 if (!poll_state_synchronize_srcu(&trac <<
65 synchronize_srcu(&tracepoint_s <<
66 snapshot->ongoing = false; <<
67 } <<
68 32
69 /* Set to 1 to enable tracepoint debug output 33 /* Set to 1 to enable tracepoint debug output */
70 static const int tracepoint_debug; 34 static const int tracepoint_debug;
71 35
72 #ifdef CONFIG_MODULES <<
73 /* 36 /*
74 * Tracepoint module list mutex protects the l !! 37 * Tracepoints mutex protects the builtin and module tracepoints and the hash
>> 38 * table, as well as the local module list.
75 */ 39 */
76 static DEFINE_MUTEX(tracepoint_module_list_mut !! 40 static DEFINE_MUTEX(tracepoints_mutex);
77 41
78 /* Local list of struct tp_module */ !! 42 #ifdef CONFIG_MODULES
>> 43 /* Local list of struct module */
79 static LIST_HEAD(tracepoint_module_list); 44 static LIST_HEAD(tracepoint_module_list);
80 #endif /* CONFIG_MODULES */ 45 #endif /* CONFIG_MODULES */
81 46
82 /* 47 /*
83 * tracepoints_mutex protects the builtin and !! 48 * Tracepoint hash table, containing the active tracepoints.
84 * tracepoints_mutex nests inside tracepoint_m !! 49 * Protected by tracepoints_mutex.
85 */ 50 */
86 static DEFINE_MUTEX(tracepoints_mutex); !! 51 #define TRACEPOINT_HASH_BITS 6
87 !! 52 #define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
88 static struct rcu_head *early_probes; !! 53 static struct hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
89 static bool ok_to_free_tracepoints; <<
90 54
91 /* 55 /*
92 * Note about RCU : 56 * Note about RCU :
93 * It is used to delay the free of multiple pr 57 * It is used to delay the free of multiple probes array until a quiescent
94 * state is reached. 58 * state is reached.
>> 59 * Tracepoint entries modifications are protected by the tracepoints_mutex.
95 */ 60 */
96 struct tp_probes { !! 61 struct tracepoint_entry {
97 struct rcu_head rcu; !! 62 struct hlist_node hlist;
98 struct tracepoint_func probes[]; !! 63 struct tracepoint_func *funcs;
>> 64 int refcount; /* Number of times armed. 0 if disarmed. */
>> 65 char name[0];
99 }; 66 };
100 67
101 /* Called in removal of a func but failed to a !! 68 struct tp_probes {
102 static void tp_stub_func(void) !! 69 union {
103 { !! 70 struct rcu_head rcu;
104 return; !! 71 struct list_head list;
105 } !! 72 } u;
>> 73 struct tracepoint_func probes[0];
>> 74 };
106 75
107 static inline void *allocate_probes(int count) 76 static inline void *allocate_probes(int count)
108 { 77 {
109 struct tp_probes *p = kmalloc(struct_ !! 78 struct tp_probes *p = kmalloc(count * sizeof(struct tracepoint_func)
110 GFP_KER !! 79 + sizeof(struct tp_probes), GFP_KERNEL);
111 return p == NULL ? NULL : p->probes; 80 return p == NULL ? NULL : p->probes;
112 } 81 }
113 82
114 static void srcu_free_old_probes(struct rcu_he <<
115 { <<
116 kfree(container_of(head, struct tp_pro <<
117 } <<
118 <<
119 static void rcu_free_old_probes(struct rcu_hea 83 static void rcu_free_old_probes(struct rcu_head *head)
120 { 84 {
121 call_srcu(&tracepoint_srcu, head, srcu !! 85 kfree(container_of(head, struct tp_probes, u.rcu));
122 } <<
123 <<
124 static __init int release_early_probes(void) <<
125 { <<
126 struct rcu_head *tmp; <<
127 <<
128 ok_to_free_tracepoints = true; <<
129 <<
130 while (early_probes) { <<
131 tmp = early_probes; <<
132 early_probes = tmp->next; <<
133 call_rcu(tmp, rcu_free_old_pro <<
134 } <<
135 <<
136 return 0; <<
137 } 86 }
138 87
139 /* SRCU is initialized at core_initcall */ <<
140 postcore_initcall(release_early_probes); <<
141 <<
142 static inline void release_probes(struct trace 88 static inline void release_probes(struct tracepoint_func *old)
143 { 89 {
144 if (old) { 90 if (old) {
145 struct tp_probes *tp_probes = 91 struct tp_probes *tp_probes = container_of(old,
146 struct tp_probes, prob 92 struct tp_probes, probes[0]);
147 !! 93 call_rcu_sched(&tp_probes->u.rcu, rcu_free_old_probes);
148 /* <<
149 * We can't free probes if SRC <<
150 * Postpone the freeing till a <<
151 */ <<
152 if (unlikely(!ok_to_free_trace <<
153 tp_probes->rcu.next = <<
154 early_probes = &tp_pro <<
155 return; <<
156 } <<
157 <<
158 /* <<
159 * Tracepoint probes are prote <<
160 * by calling the SRCU callbac <<
161 * cover both cases. So let us <<
162 * callbacks to wait for both <<
163 */ <<
164 call_rcu(&tp_probes->rcu, rcu_ <<
165 } 94 }
166 } 95 }
167 96
168 static void debug_print_probes(struct tracepoi !! 97 static void debug_print_probes(struct tracepoint_entry *entry)
169 { 98 {
170 int i; 99 int i;
171 100
172 if (!tracepoint_debug || !funcs) !! 101 if (!tracepoint_debug || !entry->funcs)
173 return; 102 return;
174 103
175 for (i = 0; funcs[i].func; i++) !! 104 for (i = 0; entry->funcs[i].func; i++)
176 printk(KERN_DEBUG "Probe %d : !! 105 printk(KERN_DEBUG "Probe %d : %p\n", i, entry->funcs[i].func);
177 } 106 }
178 107
179 static struct tracepoint_func * 108 static struct tracepoint_func *
180 func_add(struct tracepoint_func **funcs, struc !! 109 tracepoint_entry_add_probe(struct tracepoint_entry *entry,
181 int prio) !! 110 void *probe, void *data)
182 { 111 {
>> 112 int nr_probes = 0;
183 struct tracepoint_func *old, *new; 113 struct tracepoint_func *old, *new;
184 int iter_probes; /* Iterate ove <<
185 int nr_probes = 0; /* Counter for <<
186 int pos = -1; /* Insertion p <<
187 114
188 if (WARN_ON(!tp_func->func)) !! 115 if (WARN_ON(!probe))
189 return ERR_PTR(-EINVAL); 116 return ERR_PTR(-EINVAL);
190 117
191 debug_print_probes(*funcs); !! 118 debug_print_probes(entry);
192 old = *funcs; !! 119 old = entry->funcs;
193 if (old) { 120 if (old) {
194 /* (N -> N+1), (N != 0, 1) pro 121 /* (N -> N+1), (N != 0, 1) probes */
195 for (iter_probes = 0; old[iter !! 122 for (nr_probes = 0; old[nr_probes].func; nr_probes++)
196 if (old[iter_probes].f !! 123 if (old[nr_probes].func == probe &&
197 continue; !! 124 old[nr_probes].data == data)
198 if (old[iter_probes].f <<
199 old[iter_probes].d <<
200 return ERR_PTR 125 return ERR_PTR(-EEXIST);
201 nr_probes++; <<
202 } <<
203 } 126 }
204 /* + 2 : one for new probe, one for NU 127 /* + 2 : one for new probe, one for NULL func */
205 new = allocate_probes(nr_probes + 2); 128 new = allocate_probes(nr_probes + 2);
206 if (new == NULL) 129 if (new == NULL)
207 return ERR_PTR(-ENOMEM); 130 return ERR_PTR(-ENOMEM);
208 if (old) { !! 131 if (old)
209 nr_probes = 0; !! 132 memcpy(new, old, nr_probes * sizeof(struct tracepoint_func));
210 for (iter_probes = 0; old[iter !! 133 new[nr_probes].func = probe;
211 if (old[iter_probes].f !! 134 new[nr_probes].data = data;
212 continue; !! 135 new[nr_probes + 1].func = NULL;
213 /* Insert before probe !! 136 entry->refcount = nr_probes + 1;
214 if (pos < 0 && old[ite !! 137 entry->funcs = new;
215 pos = nr_probe !! 138 debug_print_probes(entry);
216 new[nr_probes++] = old <<
217 } <<
218 if (pos < 0) <<
219 pos = nr_probes++; <<
220 /* nr_probes now points to the <<
221 } else { <<
222 pos = 0; <<
223 nr_probes = 1; /* must point a <<
224 } <<
225 new[pos] = *tp_func; <<
226 new[nr_probes].func = NULL; <<
227 *funcs = new; <<
228 debug_print_probes(*funcs); <<
229 return old; 139 return old;
230 } 140 }
231 141
232 static void *func_remove(struct tracepoint_fun !! 142 static void *
233 struct tracepoint_func *tp_fun !! 143 tracepoint_entry_remove_probe(struct tracepoint_entry *entry,
>> 144 void *probe, void *data)
234 { 145 {
235 int nr_probes = 0, nr_del = 0, i; 146 int nr_probes = 0, nr_del = 0, i;
236 struct tracepoint_func *old, *new; 147 struct tracepoint_func *old, *new;
237 148
238 old = *funcs; !! 149 old = entry->funcs;
239 150
240 if (!old) 151 if (!old)
241 return ERR_PTR(-ENOENT); 152 return ERR_PTR(-ENOENT);
242 153
243 debug_print_probes(*funcs); !! 154 debug_print_probes(entry);
244 /* (N -> M), (N > 1, M >= 0) probes */ 155 /* (N -> M), (N > 1, M >= 0) probes */
245 if (tp_func->func) { !! 156 if (probe) {
246 for (nr_probes = 0; old[nr_pro 157 for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
247 if ((old[nr_probes].fu !! 158 if (old[nr_probes].func == probe &&
248 old[nr_probes].da !! 159 old[nr_probes].data == data)
249 old[nr_probes].fun <<
250 nr_del++; 160 nr_del++;
251 } 161 }
252 } 162 }
253 163
254 /* 164 /*
255 * If probe is NULL, then nr_probes = 165 * If probe is NULL, then nr_probes = nr_del = 0, and then the
256 * entire entry will be removed. 166 * entire entry will be removed.
257 */ 167 */
258 if (nr_probes - nr_del == 0) { 168 if (nr_probes - nr_del == 0) {
259 /* N -> 0, (N > 1) */ 169 /* N -> 0, (N > 1) */
260 *funcs = NULL; !! 170 entry->funcs = NULL;
261 debug_print_probes(*funcs); !! 171 entry->refcount = 0;
>> 172 debug_print_probes(entry);
262 return old; 173 return old;
263 } else { 174 } else {
264 int j = 0; 175 int j = 0;
265 /* N -> M, (N > 1, M > 0) */ 176 /* N -> M, (N > 1, M > 0) */
266 /* + 1 for NULL */ 177 /* + 1 for NULL */
267 new = allocate_probes(nr_probe 178 new = allocate_probes(nr_probes - nr_del + 1);
268 if (new) { !! 179 if (new == NULL)
269 for (i = 0; old[i].fun !! 180 return ERR_PTR(-ENOMEM);
270 if ((old[i].fu !! 181 for (i = 0; old[i].func; i++)
271 old[i].da !! 182 if (old[i].func != probe || old[i].data != data)
272 old[i].fun !! 183 new[j++] = old[i];
273 new[j+ !! 184 new[nr_probes - nr_del].func = NULL;
274 } !! 185 entry->refcount = nr_probes - nr_del;
275 new[nr_probes - nr_del !! 186 entry->funcs = new;
276 *funcs = new; <<
277 } else { <<
278 /* <<
279 * Failed to allocate, <<
280 * with calls to tp_st <<
281 */ <<
282 for (i = 0; old[i].fun <<
283 if (old[i].fun <<
284 old[i].dat <<
285 WRITE_ <<
286 } <<
287 *funcs = old; <<
288 } <<
289 } 187 }
290 debug_print_probes(*funcs); !! 188 debug_print_probes(entry);
291 return old; 189 return old;
292 } 190 }
293 191
294 /* 192 /*
295 * Count the number of functions (enum tp_func !! 193 * Get tracepoint if the tracepoint is present in the tracepoint hash table.
>> 194 * Must be called with tracepoints_mutex held.
>> 195 * Returns NULL if not present.
>> 196 */
>> 197 static struct tracepoint_entry *get_tracepoint(const char *name)
>> 198 {
>> 199 struct hlist_head *head;
>> 200 struct tracepoint_entry *e;
>> 201 u32 hash = jhash(name, strlen(name), 0);
>> 202
>> 203 head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
>> 204 hlist_for_each_entry(e, head, hlist) {
>> 205 if (!strcmp(name, e->name))
>> 206 return e;
>> 207 }
>> 208 return NULL;
>> 209 }
>> 210
>> 211 /*
>> 212 * Add the tracepoint to the tracepoint hash table. Must be called with
>> 213 * tracepoints_mutex held.
296 */ 214 */
297 static enum tp_func_state nr_func_state(const !! 215 static struct tracepoint_entry *add_tracepoint(const char *name)
298 { 216 {
299 if (!tp_funcs) !! 217 struct hlist_head *head;
300 return TP_FUNC_0; !! 218 struct tracepoint_entry *e;
301 if (!tp_funcs[1].func) !! 219 size_t name_len = strlen(name) + 1;
302 return TP_FUNC_1; !! 220 u32 hash = jhash(name, name_len-1, 0);
303 if (!tp_funcs[2].func) !! 221
304 return TP_FUNC_2; !! 222 head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
305 return TP_FUNC_N; /* 3 or more * !! 223 hlist_for_each_entry(e, head, hlist) {
>> 224 if (!strcmp(name, e->name)) {
>> 225 printk(KERN_NOTICE
>> 226 "tracepoint %s busy\n", name);
>> 227 return ERR_PTR(-EEXIST); /* Already there */
>> 228 }
>> 229 }
>> 230 /*
>> 231 * Using kmalloc here to allocate a variable length element. Could
>> 232 * cause some memory fragmentation if overused.
>> 233 */
>> 234 e = kmalloc(sizeof(struct tracepoint_entry) + name_len, GFP_KERNEL);
>> 235 if (!e)
>> 236 return ERR_PTR(-ENOMEM);
>> 237 memcpy(&e->name[0], name, name_len);
>> 238 e->funcs = NULL;
>> 239 e->refcount = 0;
>> 240 hlist_add_head(&e->hlist, head);
>> 241 return e;
306 } 242 }
307 243
308 static void tracepoint_update_call(struct trac !! 244 /*
>> 245 * Remove the tracepoint from the tracepoint hash table. Must be called with
>> 246 * mutex_lock held.
>> 247 */
>> 248 static inline void remove_tracepoint(struct tracepoint_entry *e)
309 { 249 {
310 void *func = tp->iterator; !! 250 hlist_del(&e->hlist);
311 !! 251 kfree(e);
312 /* Synthetic events do not have static <<
313 if (!tp->static_call_key) <<
314 return; <<
315 if (nr_func_state(tp_funcs) == TP_FUNC <<
316 func = tp_funcs[0].func; <<
317 __static_call_update(tp->static_call_k <<
318 } 252 }
319 253
320 /* 254 /*
321 * Add the probe function to a tracepoint. !! 255 * Sets the probe callback corresponding to one tracepoint.
322 */ 256 */
323 static int tracepoint_add_func(struct tracepoi !! 257 static void set_tracepoint(struct tracepoint_entry **entry,
324 struct tracepoi !! 258 struct tracepoint *elem, int active)
325 bool warn) <<
326 { 259 {
327 struct tracepoint_func *old, *tp_funcs !! 260 WARN_ON(strcmp((*entry)->name, elem->name) != 0);
328 int ret; <<
329 261
330 if (tp->regfunc && !static_key_enabled !! 262 if (elem->regfunc && !static_key_enabled(&elem->key) && active)
331 ret = tp->regfunc(); !! 263 elem->regfunc();
332 if (ret < 0) !! 264 else if (elem->unregfunc && static_key_enabled(&elem->key) && !active)
333 return ret; !! 265 elem->unregfunc();
334 } <<
335 <<
336 tp_funcs = rcu_dereference_protected(t <<
337 lockdep_is_held(&trace <<
338 old = func_add(&tp_funcs, func, prio); <<
339 if (IS_ERR(old)) { <<
340 WARN_ON_ONCE(warn && PTR_ERR(o <<
341 return PTR_ERR(old); <<
342 } <<
343 266
344 /* 267 /*
345 * rcu_assign_pointer has as smp_store !! 268 * rcu_assign_pointer has a smp_wmb() which makes sure that the new
346 * that the new probe callbacks array !! 269 * probe callbacks array is consistent before setting a pointer to it.
347 * a pointer to it. This array is ref !! 270 * This array is referenced by __DO_TRACE from
348 * include/linux/tracepoint.h using rc !! 271 * include/linux/tracepoints.h. A matching smp_read_barrier_depends()
>> 272 * is used.
349 */ 273 */
350 switch (nr_func_state(tp_funcs)) { !! 274 rcu_assign_pointer(elem->funcs, (*entry)->funcs);
351 case TP_FUNC_1: /* 0->1 */ !! 275 if (active && !static_key_enabled(&elem->key))
352 /* !! 276 static_key_slow_inc(&elem->key);
353 * Make sure new static func n !! 277 else if (!active && static_key_enabled(&elem->key))
354 * 1->0->1 transition sequence !! 278 static_key_slow_dec(&elem->key);
355 */ <<
356 tp_rcu_cond_sync(TP_TRANSITION <<
357 /* Set static call to first fu <<
358 tracepoint_update_call(tp, tp_ <<
359 /* Both iterator and static ca <<
360 rcu_assign_pointer(tp->funcs, <<
361 static_key_enable(&tp->key); <<
362 break; <<
363 case TP_FUNC_2: /* 1->2 */ <<
364 /* Set iterator static call */ <<
365 tracepoint_update_call(tp, tp_ <<
366 /* <<
367 * Iterator callback installed <<
368 * Requires ordering between R <<
369 * static call update/call. <<
370 */ <<
371 fallthrough; <<
372 case TP_FUNC_N: /* N->N+1 (N>1 <<
373 rcu_assign_pointer(tp->funcs, <<
374 /* <<
375 * Make sure static func never <<
376 * N->...->2->1 (N>1) transiti <<
377 */ <<
378 if (tp_funcs[0].data != old[0] <<
379 tp_rcu_get_state(TP_TR <<
380 break; <<
381 default: <<
382 WARN_ON_ONCE(1); <<
383 break; <<
384 } <<
385 <<
386 release_probes(old); <<
387 return 0; <<
388 } 279 }
389 280
390 /* 281 /*
391 * Remove a probe function from a tracepoint. !! 282 * Disable a tracepoint and its probe callback.
392 * Note: only waiting an RCU period after sett 283 * Note: only waiting an RCU period after setting elem->call to the empty
393 * function insures that the original callback 284 * function insures that the original callback is not used anymore. This insured
394 * by preempt_disable around the call site. 285 * by preempt_disable around the call site.
395 */ 286 */
396 static int tracepoint_remove_func(struct trace !! 287 static void disable_tracepoint(struct tracepoint *elem)
397 struct tracepoint_func *func) <<
398 { 288 {
399 struct tracepoint_func *old, *tp_funcs !! 289 if (elem->unregfunc && static_key_enabled(&elem->key))
>> 290 elem->unregfunc();
400 291
401 tp_funcs = rcu_dereference_protected(t !! 292 if (static_key_enabled(&elem->key))
402 lockdep_is_held(&trace !! 293 static_key_slow_dec(&elem->key);
403 old = func_remove(&tp_funcs, func); !! 294 rcu_assign_pointer(elem->funcs, NULL);
404 if (WARN_ON_ONCE(IS_ERR(old))) !! 295 }
405 return PTR_ERR(old); <<
406 296
407 if (tp_funcs == old) !! 297 /**
408 /* Failed allocating new tp_fu !! 298 * tracepoint_update_probe_range - Update a probe range
409 return 0; !! 299 * @begin: beginning of the range
>> 300 * @end: end of the range
>> 301 *
>> 302 * Updates the probe callback corresponding to a range of tracepoints.
>> 303 * Called with tracepoints_mutex held.
>> 304 */
>> 305 static void tracepoint_update_probe_range(struct tracepoint * const *begin,
>> 306 struct tracepoint * const *end)
>> 307 {
>> 308 struct tracepoint * const *iter;
>> 309 struct tracepoint_entry *mark_entry;
410 310
411 switch (nr_func_state(tp_funcs)) { !! 311 if (!begin)
412 case TP_FUNC_0: /* 1->0 */ !! 312 return;
413 /* Removed last function */ !! 313
414 if (tp->unregfunc && static_ke !! 314 for (iter = begin; iter < end; iter++) {
415 tp->unregfunc(); !! 315 mark_entry = get_tracepoint((*iter)->name);
416 !! 316 if (mark_entry) {
417 static_key_disable(&tp->key); !! 317 set_tracepoint(&mark_entry, *iter,
418 /* Set iterator static call */ !! 318 !!mark_entry->refcount);
419 tracepoint_update_call(tp, tp_ !! 319 } else {
420 /* Both iterator and static ca !! 320 disable_tracepoint(*iter);
421 rcu_assign_pointer(tp->funcs, !! 321 }
422 /* <<
423 * Make sure new static func n <<
424 * 1->0->1 transition sequence <<
425 */ <<
426 tp_rcu_get_state(TP_TRANSITION <<
427 break; <<
428 case TP_FUNC_1: /* 2->1 */ <<
429 rcu_assign_pointer(tp->funcs, <<
430 /* <<
431 * Make sure static func never <<
432 * N->...->2->1 (N>2) transiti <<
433 * element's data has changed, <<
434 * to prevent current readers <<
435 * from calling the new functi <<
436 */ <<
437 if (tp_funcs[0].data != old[0] <<
438 tp_rcu_get_state(TP_TR <<
439 tp_rcu_cond_sync(TP_TRANSITION <<
440 /* Set static call to first fu <<
441 tracepoint_update_call(tp, tp_ <<
442 break; <<
443 case TP_FUNC_2: /* N->N-1 (N>2 <<
444 fallthrough; <<
445 case TP_FUNC_N: <<
446 rcu_assign_pointer(tp->funcs, <<
447 /* <<
448 * Make sure static func never <<
449 * N->...->2->1 (N>2) transiti <<
450 */ <<
451 if (tp_funcs[0].data != old[0] <<
452 tp_rcu_get_state(TP_TR <<
453 break; <<
454 default: <<
455 WARN_ON_ONCE(1); <<
456 break; <<
457 } 322 }
458 release_probes(old); !! 323 }
459 return 0; !! 324
>> 325 #ifdef CONFIG_MODULES
>> 326 void module_update_tracepoints(void)
>> 327 {
>> 328 struct tp_module *tp_mod;
>> 329
>> 330 list_for_each_entry(tp_mod, &tracepoint_module_list, list)
>> 331 tracepoint_update_probe_range(tp_mod->tracepoints_ptrs,
>> 332 tp_mod->tracepoints_ptrs + tp_mod->num_tracepoints);
>> 333 }
>> 334 #else /* CONFIG_MODULES */
>> 335 void module_update_tracepoints(void)
>> 336 {
>> 337 }
>> 338 #endif /* CONFIG_MODULES */
>> 339
>> 340
>> 341 /*
>> 342 * Update probes, removing the faulty probes.
>> 343 * Called with tracepoints_mutex held.
>> 344 */
>> 345 static void tracepoint_update_probes(void)
>> 346 {
>> 347 /* Core kernel tracepoints */
>> 348 tracepoint_update_probe_range(__start___tracepoints_ptrs,
>> 349 __stop___tracepoints_ptrs);
>> 350 /* tracepoints in modules. */
>> 351 module_update_tracepoints();
>> 352 }
>> 353
>> 354 static struct tracepoint_func *
>> 355 tracepoint_add_probe(const char *name, void *probe, void *data)
>> 356 {
>> 357 struct tracepoint_entry *entry;
>> 358 struct tracepoint_func *old;
>> 359
>> 360 entry = get_tracepoint(name);
>> 361 if (!entry) {
>> 362 entry = add_tracepoint(name);
>> 363 if (IS_ERR(entry))
>> 364 return (struct tracepoint_func *)entry;
>> 365 }
>> 366 old = tracepoint_entry_add_probe(entry, probe, data);
>> 367 if (IS_ERR(old) && !entry->refcount)
>> 368 remove_tracepoint(entry);
>> 369 return old;
460 } 370 }
461 371
462 /** 372 /**
463 * tracepoint_probe_register_prio_may_exist - !! 373 * tracepoint_probe_register - Connect a probe to a tracepoint
464 * @tp: tracepoint !! 374 * @name: tracepoint name
465 * @probe: probe handler 375 * @probe: probe handler
466 * @data: tracepoint data <<
467 * @prio: priority of this function over other <<
468 * 376 *
469 * Same as tracepoint_probe_register_prio() ex !! 377 * Returns 0 if ok, error value on error.
470 * if the tracepoint is already registered. !! 378 * The probe address must at least be aligned on the architecture pointer size.
471 */ 379 */
472 int tracepoint_probe_register_prio_may_exist(s !! 380 int tracepoint_probe_register(const char *name, void *probe, void *data)
473 v <<
474 { 381 {
475 struct tracepoint_func tp_func; !! 382 struct tracepoint_func *old;
476 int ret; <<
477 383
478 mutex_lock(&tracepoints_mutex); 384 mutex_lock(&tracepoints_mutex);
479 tp_func.func = probe; !! 385 old = tracepoint_add_probe(name, probe, data);
480 tp_func.data = data; !! 386 if (IS_ERR(old)) {
481 tp_func.prio = prio; !! 387 mutex_unlock(&tracepoints_mutex);
482 ret = tracepoint_add_func(tp, &tp_func !! 388 return PTR_ERR(old);
>> 389 }
>> 390 tracepoint_update_probes(); /* may update entry */
483 mutex_unlock(&tracepoints_mutex); 391 mutex_unlock(&tracepoints_mutex);
484 return ret; !! 392 release_probes(old);
>> 393 return 0;
>> 394 }
>> 395 EXPORT_SYMBOL_GPL(tracepoint_probe_register);
>> 396
>> 397 static struct tracepoint_func *
>> 398 tracepoint_remove_probe(const char *name, void *probe, void *data)
>> 399 {
>> 400 struct tracepoint_entry *entry;
>> 401 struct tracepoint_func *old;
>> 402
>> 403 entry = get_tracepoint(name);
>> 404 if (!entry)
>> 405 return ERR_PTR(-ENOENT);
>> 406 old = tracepoint_entry_remove_probe(entry, probe, data);
>> 407 if (IS_ERR(old))
>> 408 return old;
>> 409 if (!entry->refcount)
>> 410 remove_tracepoint(entry);
>> 411 return old;
485 } 412 }
486 EXPORT_SYMBOL_GPL(tracepoint_probe_register_pr <<
487 413
488 /** 414 /**
489 * tracepoint_probe_register_prio - Connect a !! 415 * tracepoint_probe_unregister - Disconnect a probe from a tracepoint
490 * @tp: tracepoint !! 416 * @name: tracepoint name
491 * @probe: probe handler !! 417 * @probe: probe function pointer
492 * @data: tracepoint data <<
493 * @prio: priority of this function over other <<
494 * 418 *
495 * Returns 0 if ok, error value on error. !! 419 * We do not need to call a synchronize_sched to make sure the probes have
496 * Note: if @tp is within a module, the caller !! 420 * finished running before doing a module unload, because the module unload
497 * unregistering the probe before the module i !! 421 * itself uses stop_machine(), which insures that every preempt disabled section
498 * performed either with a tracepoint module g !! 422 * have finished.
499 * within module exit functions. <<
500 */ 423 */
501 int tracepoint_probe_register_prio(struct trac !! 424 int tracepoint_probe_unregister(const char *name, void *probe, void *data)
502 void *data, <<
503 { 425 {
504 struct tracepoint_func tp_func; !! 426 struct tracepoint_func *old;
505 int ret; <<
506 427
507 mutex_lock(&tracepoints_mutex); 428 mutex_lock(&tracepoints_mutex);
508 tp_func.func = probe; !! 429 old = tracepoint_remove_probe(name, probe, data);
509 tp_func.data = data; !! 430 if (IS_ERR(old)) {
510 tp_func.prio = prio; !! 431 mutex_unlock(&tracepoints_mutex);
511 ret = tracepoint_add_func(tp, &tp_func !! 432 return PTR_ERR(old);
>> 433 }
>> 434 tracepoint_update_probes(); /* may update entry */
512 mutex_unlock(&tracepoints_mutex); 435 mutex_unlock(&tracepoints_mutex);
513 return ret; !! 436 release_probes(old);
>> 437 return 0;
>> 438 }
>> 439 EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
>> 440
>> 441 static LIST_HEAD(old_probes);
>> 442 static int need_update;
>> 443
>> 444 static void tracepoint_add_old_probes(void *old)
>> 445 {
>> 446 need_update = 1;
>> 447 if (old) {
>> 448 struct tp_probes *tp_probes = container_of(old,
>> 449 struct tp_probes, probes[0]);
>> 450 list_add(&tp_probes->u.list, &old_probes);
>> 451 }
514 } 452 }
515 EXPORT_SYMBOL_GPL(tracepoint_probe_register_pr <<
516 453
517 /** 454 /**
518 * tracepoint_probe_register - Connect a prob !! 455 * tracepoint_probe_register_noupdate - register a probe but not connect
519 * @tp: tracepoint !! 456 * @name: tracepoint name
520 * @probe: probe handler 457 * @probe: probe handler
521 * @data: tracepoint data <<
522 * 458 *
523 * Returns 0 if ok, error value on error. !! 459 * caller must call tracepoint_probe_update_all()
524 * Note: if @tp is within a module, the caller <<
525 * unregistering the probe before the module i <<
526 * performed either with a tracepoint module g <<
527 * within module exit functions. <<
528 */ 460 */
529 int tracepoint_probe_register(struct tracepoin !! 461 int tracepoint_probe_register_noupdate(const char *name, void *probe,
>> 462 void *data)
530 { 463 {
531 return tracepoint_probe_register_prio( !! 464 struct tracepoint_func *old;
>> 465
>> 466 mutex_lock(&tracepoints_mutex);
>> 467 old = tracepoint_add_probe(name, probe, data);
>> 468 if (IS_ERR(old)) {
>> 469 mutex_unlock(&tracepoints_mutex);
>> 470 return PTR_ERR(old);
>> 471 }
>> 472 tracepoint_add_old_probes(old);
>> 473 mutex_unlock(&tracepoints_mutex);
>> 474 return 0;
532 } 475 }
533 EXPORT_SYMBOL_GPL(tracepoint_probe_register); !! 476 EXPORT_SYMBOL_GPL(tracepoint_probe_register_noupdate);
534 477
535 /** 478 /**
536 * tracepoint_probe_unregister - Disconnect a !! 479 * tracepoint_probe_unregister_noupdate - remove a probe but not disconnect
537 * @tp: tracepoint !! 480 * @name: tracepoint name
538 * @probe: probe function pointer 481 * @probe: probe function pointer
539 * @data: tracepoint data <<
540 * 482 *
541 * Returns 0 if ok, error value on error. !! 483 * caller must call tracepoint_probe_update_all()
542 */ 484 */
543 int tracepoint_probe_unregister(struct tracepo !! 485 int tracepoint_probe_unregister_noupdate(const char *name, void *probe,
>> 486 void *data)
544 { 487 {
545 struct tracepoint_func tp_func; !! 488 struct tracepoint_func *old;
546 int ret; <<
547 489
548 mutex_lock(&tracepoints_mutex); 490 mutex_lock(&tracepoints_mutex);
549 tp_func.func = probe; !! 491 old = tracepoint_remove_probe(name, probe, data);
550 tp_func.data = data; !! 492 if (IS_ERR(old)) {
551 ret = tracepoint_remove_func(tp, &tp_f !! 493 mutex_unlock(&tracepoints_mutex);
>> 494 return PTR_ERR(old);
>> 495 }
>> 496 tracepoint_add_old_probes(old);
552 mutex_unlock(&tracepoints_mutex); 497 mutex_unlock(&tracepoints_mutex);
553 return ret; !! 498 return 0;
554 } 499 }
555 EXPORT_SYMBOL_GPL(tracepoint_probe_unregister) !! 500 EXPORT_SYMBOL_GPL(tracepoint_probe_unregister_noupdate);
556 501
557 static void for_each_tracepoint_range( !! 502 /**
558 tracepoint_ptr_t *begin, trace !! 503 * tracepoint_probe_update_all - update tracepoints
559 void (*fct)(struct tracepoint !! 504 */
560 void *priv) !! 505 void tracepoint_probe_update_all(void)
561 { 506 {
562 tracepoint_ptr_t *iter; !! 507 LIST_HEAD(release_probes);
>> 508 struct tp_probes *pos, *next;
563 509
564 if (!begin) !! 510 mutex_lock(&tracepoints_mutex);
>> 511 if (!need_update) {
>> 512 mutex_unlock(&tracepoints_mutex);
565 return; 513 return;
566 for (iter = begin; iter < end; iter++) !! 514 }
567 fct(tracepoint_ptr_deref(iter) !! 515 if (!list_empty(&old_probes))
>> 516 list_replace_init(&old_probes, &release_probes);
>> 517 need_update = 0;
>> 518 tracepoint_update_probes();
>> 519 mutex_unlock(&tracepoints_mutex);
>> 520 list_for_each_entry_safe(pos, next, &release_probes, u.list) {
>> 521 list_del(&pos->u.list);
>> 522 call_rcu_sched(&pos->u.rcu, rcu_free_old_probes);
>> 523 }
>> 524 }
>> 525 EXPORT_SYMBOL_GPL(tracepoint_probe_update_all);
>> 526
>> 527 /**
>> 528 * tracepoint_get_iter_range - Get a next tracepoint iterator given a range.
>> 529 * @tracepoint: current tracepoints (in), next tracepoint (out)
>> 530 * @begin: beginning of the range
>> 531 * @end: end of the range
>> 532 *
>> 533 * Returns whether a next tracepoint has been found (1) or not (0).
>> 534 * Will return the first tracepoint in the range if the input tracepoint is
>> 535 * NULL.
>> 536 */
>> 537 static int tracepoint_get_iter_range(struct tracepoint * const **tracepoint,
>> 538 struct tracepoint * const *begin, struct tracepoint * const *end)
>> 539 {
>> 540 if (!*tracepoint && begin != end) {
>> 541 *tracepoint = begin;
>> 542 return 1;
>> 543 }
>> 544 if (*tracepoint >= begin && *tracepoint < end)
>> 545 return 1;
>> 546 return 0;
568 } 547 }
569 548
570 #ifdef CONFIG_MODULES 549 #ifdef CONFIG_MODULES
571 bool trace_module_has_bad_taint(struct module !! 550 static void tracepoint_get_iter(struct tracepoint_iter *iter)
572 { 551 {
573 return mod->taints & ~((1 << TAINT_OOT !! 552 int found = 0;
574 (1 << TAINT_UN !! 553 struct tp_module *iter_mod;
575 (1 << TAINT_LI !! 554
>> 555 /* Core kernel tracepoints */
>> 556 if (!iter->module) {
>> 557 found = tracepoint_get_iter_range(&iter->tracepoint,
>> 558 __start___tracepoints_ptrs,
>> 559 __stop___tracepoints_ptrs);
>> 560 if (found)
>> 561 goto end;
>> 562 }
>> 563 /* Tracepoints in modules */
>> 564 mutex_lock(&tracepoints_mutex);
>> 565 list_for_each_entry(iter_mod, &tracepoint_module_list, list) {
>> 566 /*
>> 567 * Sorted module list
>> 568 */
>> 569 if (iter_mod < iter->module)
>> 570 continue;
>> 571 else if (iter_mod > iter->module)
>> 572 iter->tracepoint = NULL;
>> 573 found = tracepoint_get_iter_range(&iter->tracepoint,
>> 574 iter_mod->tracepoints_ptrs,
>> 575 iter_mod->tracepoints_ptrs
>> 576 + iter_mod->num_tracepoints);
>> 577 if (found) {
>> 578 iter->module = iter_mod;
>> 579 break;
>> 580 }
>> 581 }
>> 582 mutex_unlock(&tracepoints_mutex);
>> 583 end:
>> 584 if (!found)
>> 585 tracepoint_iter_reset(iter);
576 } 586 }
>> 587 #else /* CONFIG_MODULES */
>> 588 static void tracepoint_get_iter(struct tracepoint_iter *iter)
>> 589 {
>> 590 int found = 0;
577 591
578 static BLOCKING_NOTIFIER_HEAD(tracepoint_notif !! 592 /* Core kernel tracepoints */
>> 593 found = tracepoint_get_iter_range(&iter->tracepoint,
>> 594 __start___tracepoints_ptrs,
>> 595 __stop___tracepoints_ptrs);
>> 596 if (!found)
>> 597 tracepoint_iter_reset(iter);
>> 598 }
>> 599 #endif /* CONFIG_MODULES */
579 600
580 /** !! 601 void tracepoint_iter_start(struct tracepoint_iter *iter)
581 * register_tracepoint_module_notifier - regis <<
582 * @nb: notifier block <<
583 * <<
584 * Notifiers registered with this function are <<
585 * coming/going with the tracepoint_module_lis <<
586 * The notifier block callback should expect a <<
587 * pointer. <<
588 */ <<
589 int register_tracepoint_module_notifier(struct <<
590 { 602 {
591 struct tp_module *tp_mod; !! 603 tracepoint_get_iter(iter);
592 int ret; <<
593 <<
594 mutex_lock(&tracepoint_module_list_mut <<
595 ret = blocking_notifier_chain_register <<
596 if (ret) <<
597 goto end; <<
598 list_for_each_entry(tp_mod, &tracepoin <<
599 (void) nb->notifier_call(nb, M <<
600 end: <<
601 mutex_unlock(&tracepoint_module_list_m <<
602 return ret; <<
603 } 604 }
604 EXPORT_SYMBOL_GPL(register_tracepoint_module_n !! 605 EXPORT_SYMBOL_GPL(tracepoint_iter_start);
605 606
606 /** !! 607 void tracepoint_iter_next(struct tracepoint_iter *iter)
607 * unregister_tracepoint_module_notifier - unr <<
608 * @nb: notifier block <<
609 * <<
610 * The notifier block callback should expect a <<
611 * pointer. <<
612 */ <<
613 int unregister_tracepoint_module_notifier(stru <<
614 { 608 {
615 struct tp_module *tp_mod; !! 609 iter->tracepoint++;
616 int ret; !! 610 /*
>> 611 * iter->tracepoint may be invalid because we blindly incremented it.
>> 612 * Make sure it is valid by marshalling on the tracepoints, getting the
>> 613 * tracepoints from following modules if necessary.
>> 614 */
>> 615 tracepoint_get_iter(iter);
>> 616 }
>> 617 EXPORT_SYMBOL_GPL(tracepoint_iter_next);
617 618
618 mutex_lock(&tracepoint_module_list_mut !! 619 void tracepoint_iter_stop(struct tracepoint_iter *iter)
619 ret = blocking_notifier_chain_unregist !! 620 {
620 if (ret) !! 621 }
621 goto end; !! 622 EXPORT_SYMBOL_GPL(tracepoint_iter_stop);
622 list_for_each_entry(tp_mod, &tracepoin <<
623 (void) nb->notifier_call(nb, M <<
624 end: <<
625 mutex_unlock(&tracepoint_module_list_m <<
626 return ret; <<
627 623
>> 624 void tracepoint_iter_reset(struct tracepoint_iter *iter)
>> 625 {
>> 626 #ifdef CONFIG_MODULES
>> 627 iter->module = NULL;
>> 628 #endif /* CONFIG_MODULES */
>> 629 iter->tracepoint = NULL;
628 } 630 }
629 EXPORT_SYMBOL_GPL(unregister_tracepoint_module !! 631 EXPORT_SYMBOL_GPL(tracepoint_iter_reset);
630 632
631 /* !! 633 #ifdef CONFIG_MODULES
632 * Ensure the tracer unregistered the module's !! 634 bool trace_module_has_bad_taint(struct module *mod)
633 * teardown is performed. Prevents leaks of pr <<
634 */ <<
635 static void tp_module_going_check_quiescent(st <<
636 { 635 {
637 WARN_ON_ONCE(tp->funcs); !! 636 return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP));
638 } 637 }
639 638
640 static int tracepoint_module_coming(struct mod 639 static int tracepoint_module_coming(struct module *mod)
641 { 640 {
642 struct tp_module *tp_mod; !! 641 struct tp_module *tp_mod, *iter;
>> 642 int ret = 0;
643 643
644 if (!mod->num_tracepoints) 644 if (!mod->num_tracepoints)
645 return 0; 645 return 0;
646 646
647 /* 647 /*
648 * We skip modules that taint the kern 648 * We skip modules that taint the kernel, especially those with different
649 * module headers (for forced load), t 649 * module headers (for forced load), to make sure we don't cause a crash.
650 * Staging, out-of-tree, unsigned GPL, !! 650 * Staging and out-of-tree GPL modules are fine.
651 */ 651 */
652 if (trace_module_has_bad_taint(mod)) 652 if (trace_module_has_bad_taint(mod))
653 return 0; 653 return 0;
654 !! 654 mutex_lock(&tracepoints_mutex);
655 tp_mod = kmalloc(sizeof(struct tp_modu 655 tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
656 if (!tp_mod) !! 656 if (!tp_mod) {
657 return -ENOMEM; !! 657 ret = -ENOMEM;
658 tp_mod->mod = mod; !! 658 goto end;
659 !! 659 }
660 mutex_lock(&tracepoint_module_list_mut !! 660 tp_mod->num_tracepoints = mod->num_tracepoints;
661 list_add_tail(&tp_mod->list, &tracepoi !! 661 tp_mod->tracepoints_ptrs = mod->tracepoints_ptrs;
662 blocking_notifier_call_chain(&tracepoi !! 662
663 MODULE_STATE_COMING, t !! 663 /*
664 mutex_unlock(&tracepoint_module_list_m !! 664 * tracepoint_module_list is kept sorted by struct module pointer
665 return 0; !! 665 * address for iteration on tracepoints from a seq_file that can release
>> 666 * the mutex between calls.
>> 667 */
>> 668 list_for_each_entry_reverse(iter, &tracepoint_module_list, list) {
>> 669 BUG_ON(iter == tp_mod); /* Should never be in the list twice */
>> 670 if (iter < tp_mod) {
>> 671 /* We belong to the location right after iter. */
>> 672 list_add(&tp_mod->list, &iter->list);
>> 673 goto module_added;
>> 674 }
>> 675 }
>> 676 /* We belong to the beginning of the list */
>> 677 list_add(&tp_mod->list, &tracepoint_module_list);
>> 678 module_added:
>> 679 tracepoint_update_probe_range(mod->tracepoints_ptrs,
>> 680 mod->tracepoints_ptrs + mod->num_tracepoints);
>> 681 end:
>> 682 mutex_unlock(&tracepoints_mutex);
>> 683 return ret;
666 } 684 }
667 685
668 static void tracepoint_module_going(struct mod !! 686 static int tracepoint_module_going(struct module *mod)
669 { 687 {
670 struct tp_module *tp_mod; !! 688 struct tp_module *pos;
671 689
672 if (!mod->num_tracepoints) 690 if (!mod->num_tracepoints)
673 return; !! 691 return 0;
674 692
675 mutex_lock(&tracepoint_module_list_mut !! 693 mutex_lock(&tracepoints_mutex);
676 list_for_each_entry(tp_mod, &tracepoin !! 694 tracepoint_update_probe_range(mod->tracepoints_ptrs,
677 if (tp_mod->mod == mod) { !! 695 mod->tracepoints_ptrs + mod->num_tracepoints);
678 blocking_notifier_call !! 696 list_for_each_entry(pos, &tracepoint_module_list, list) {
679 MODULE !! 697 if (pos->tracepoints_ptrs == mod->tracepoints_ptrs) {
680 list_del(&tp_mod->list !! 698 list_del(&pos->list);
681 kfree(tp_mod); !! 699 kfree(pos);
682 /* <<
683 * Called the going no <<
684 * quiescence. <<
685 */ <<
686 for_each_tracepoint_ra <<
687 mod->tracepoin <<
688 tp_module_goin <<
689 break; 700 break;
690 } 701 }
691 } 702 }
692 /* 703 /*
693 * In the case of modules that were ta 704 * In the case of modules that were tainted at "coming", we'll simply
694 * walk through the list without findi 705 * walk through the list without finding it. We cannot use the "tainted"
695 * flag on "going", in case a module t 706 * flag on "going", in case a module taints the kernel only after being
696 * loaded. 707 * loaded.
697 */ 708 */
698 mutex_unlock(&tracepoint_module_list_m !! 709 mutex_unlock(&tracepoints_mutex);
>> 710 return 0;
699 } 711 }
700 712
701 static int tracepoint_module_notify(struct not !! 713 int tracepoint_module_notify(struct notifier_block *self,
702 unsigned long val, void *data) !! 714 unsigned long val, void *data)
703 { 715 {
704 struct module *mod = data; 716 struct module *mod = data;
705 int ret = 0; 717 int ret = 0;
706 718
707 switch (val) { 719 switch (val) {
708 case MODULE_STATE_COMING: 720 case MODULE_STATE_COMING:
709 ret = tracepoint_module_coming 721 ret = tracepoint_module_coming(mod);
710 break; 722 break;
711 case MODULE_STATE_LIVE: 723 case MODULE_STATE_LIVE:
712 break; 724 break;
713 case MODULE_STATE_GOING: 725 case MODULE_STATE_GOING:
714 tracepoint_module_going(mod); !! 726 ret = tracepoint_module_going(mod);
715 break; <<
716 case MODULE_STATE_UNFORMED: <<
717 break; 727 break;
718 } 728 }
719 return notifier_from_errno(ret); !! 729 return ret;
720 } 730 }
721 731
722 static struct notifier_block tracepoint_module !! 732 struct notifier_block tracepoint_module_nb = {
723 .notifier_call = tracepoint_module_not 733 .notifier_call = tracepoint_module_notify,
724 .priority = 0, 734 .priority = 0,
725 }; 735 };
726 736
727 static __init int init_tracepoints(void) !! 737 static int init_tracepoints(void)
728 { 738 {
729 int ret; !! 739 return register_module_notifier(&tracepoint_module_nb);
730 <<
731 ret = register_module_notifier(&tracep <<
732 if (ret) <<
733 pr_warn("Failed to register tr <<
734 <<
735 return ret; <<
736 } 740 }
737 __initcall(init_tracepoints); 741 __initcall(init_tracepoints);
738 #endif /* CONFIG_MODULES */ 742 #endif /* CONFIG_MODULES */
739 743
740 /** <<
741 * for_each_kernel_tracepoint - iteration on a <<
742 * @fct: callback <<
743 * @priv: private data <<
744 */ <<
745 void for_each_kernel_tracepoint(void (*fct)(st <<
746 void *priv) <<
747 { <<
748 for_each_tracepoint_range(__start___tr <<
749 __stop___tracepoints_ptrs, fct <<
750 } <<
751 EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint); <<
752 <<
753 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS 744 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
754 745
755 /* NB: reg/unreg are called while guarded with 746 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
756 static int sys_tracepoint_refcount; 747 static int sys_tracepoint_refcount;
757 748
758 int syscall_regfunc(void) !! 749 void syscall_regfunc(void)
759 { 750 {
760 struct task_struct *p, *t; !! 751 unsigned long flags;
>> 752 struct task_struct *g, *t;
761 753
762 if (!sys_tracepoint_refcount) { 754 if (!sys_tracepoint_refcount) {
763 read_lock(&tasklist_lock); !! 755 read_lock_irqsave(&tasklist_lock, flags);
764 for_each_process_thread(p, t) !! 756 do_each_thread(g, t) {
765 set_task_syscall_work( !! 757 /* Skip kernel threads. */
766 } !! 758 if (t->mm)
767 read_unlock(&tasklist_lock); !! 759 set_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
>> 760 } while_each_thread(g, t);
>> 761 read_unlock_irqrestore(&tasklist_lock, flags);
768 } 762 }
769 sys_tracepoint_refcount++; 763 sys_tracepoint_refcount++;
770 <<
771 return 0; <<
772 } 764 }
773 765
774 void syscall_unregfunc(void) 766 void syscall_unregfunc(void)
775 { 767 {
776 struct task_struct *p, *t; !! 768 unsigned long flags;
>> 769 struct task_struct *g, *t;
777 770
778 sys_tracepoint_refcount--; 771 sys_tracepoint_refcount--;
779 if (!sys_tracepoint_refcount) { 772 if (!sys_tracepoint_refcount) {
780 read_lock(&tasklist_lock); !! 773 read_lock_irqsave(&tasklist_lock, flags);
781 for_each_process_thread(p, t) !! 774 do_each_thread(g, t) {
782 clear_task_syscall_wor !! 775 clear_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
783 } !! 776 } while_each_thread(g, t);
784 read_unlock(&tasklist_lock); !! 777 read_unlock_irqrestore(&tasklist_lock, flags);
785 } 778 }
786 } 779 }
787 #endif 780 #endif
788 781
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.