1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 #define CREATE_TRACE_POINTS 2 #define CREATE_TRACE_POINTS
3 #include <trace/events/mmap_lock.h> 3 #include <trace/events/mmap_lock.h>
4 4
5 #include <linux/mm.h> 5 #include <linux/mm.h>
6 #include <linux/cgroup.h> 6 #include <linux/cgroup.h>
7 #include <linux/memcontrol.h> 7 #include <linux/memcontrol.h>
8 #include <linux/mmap_lock.h> 8 #include <linux/mmap_lock.h>
9 #include <linux/mutex.h> 9 #include <linux/mutex.h>
10 #include <linux/percpu.h> 10 #include <linux/percpu.h>
11 #include <linux/rcupdate.h> 11 #include <linux/rcupdate.h>
12 #include <linux/smp.h> 12 #include <linux/smp.h>
13 #include <linux/trace_events.h> 13 #include <linux/trace_events.h>
14 #include <linux/local_lock.h> 14 #include <linux/local_lock.h>
15 15
16 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locki 16 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking);
17 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_ret 17 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned);
18 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released); 18 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released);
19 19
20 #ifdef CONFIG_MEMCG 20 #ifdef CONFIG_MEMCG
21 21
22 static atomic_t reg_refcount; !! 22 /*
>> 23 * Our various events all share the same buffer (because we don't want or need
>> 24 * to allocate a set of buffers *per event type*), so we need to protect against
>> 25 * concurrent _reg() and _unreg() calls, and count how many _reg() calls have
>> 26 * been made.
>> 27 */
>> 28 static DEFINE_MUTEX(reg_lock);
>> 29 static int reg_refcount; /* Protected by reg_lock. */
23 30
24 /* 31 /*
25 * Size of the buffer for memcg path names. Ig 32 * Size of the buffer for memcg path names. Ignoring stack trace support,
26 * trace_events_hist.c uses MAX_FILTER_STR_VAL 33 * trace_events_hist.c uses MAX_FILTER_STR_VAL for this, so we also use it.
27 */ 34 */
28 #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL 35 #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL
29 36
>> 37 /*
>> 38 * How many contexts our trace events might be called in: normal, softirq, irq,
>> 39 * and NMI.
>> 40 */
>> 41 #define CONTEXT_COUNT 4
>> 42
>> 43 struct memcg_path {
>> 44 local_lock_t lock;
>> 45 char __rcu *buf;
>> 46 local_t buf_idx;
>> 47 };
>> 48 static DEFINE_PER_CPU(struct memcg_path, memcg_paths) = {
>> 49 .lock = INIT_LOCAL_LOCK(lock),
>> 50 .buf_idx = LOCAL_INIT(0),
>> 51 };
>> 52
>> 53 static char **tmp_bufs;
>> 54
>> 55 /* Called with reg_lock held. */
>> 56 static void free_memcg_path_bufs(void)
>> 57 {
>> 58 struct memcg_path *memcg_path;
>> 59 int cpu;
>> 60 char **old = tmp_bufs;
>> 61
>> 62 for_each_possible_cpu(cpu) {
>> 63 memcg_path = per_cpu_ptr(&memcg_paths, cpu);
>> 64 *(old++) = rcu_dereference_protected(memcg_path->buf,
>> 65 lockdep_is_held(®_lock));
>> 66 rcu_assign_pointer(memcg_path->buf, NULL);
>> 67 }
>> 68
>> 69 /* Wait for inflight memcg_path_buf users to finish. */
>> 70 synchronize_rcu();
>> 71
>> 72 old = tmp_bufs;
>> 73 for_each_possible_cpu(cpu) {
>> 74 kfree(*(old++));
>> 75 }
>> 76
>> 77 kfree(tmp_bufs);
>> 78 tmp_bufs = NULL;
>> 79 }
>> 80
30 int trace_mmap_lock_reg(void) 81 int trace_mmap_lock_reg(void)
31 { 82 {
32 atomic_inc(®_refcount); !! 83 int cpu;
>> 84 char *new;
>> 85
>> 86 mutex_lock(®_lock);
>> 87
>> 88 /* If the refcount is going 0->1, proceed with allocating buffers. */
>> 89 if (reg_refcount++)
>> 90 goto out;
>> 91
>> 92 tmp_bufs = kmalloc_array(num_possible_cpus(), sizeof(*tmp_bufs),
>> 93 GFP_KERNEL);
>> 94 if (tmp_bufs == NULL)
>> 95 goto out_fail;
>> 96
>> 97 for_each_possible_cpu(cpu) {
>> 98 new = kmalloc(MEMCG_PATH_BUF_SIZE * CONTEXT_COUNT, GFP_KERNEL);
>> 99 if (new == NULL)
>> 100 goto out_fail_free;
>> 101 rcu_assign_pointer(per_cpu_ptr(&memcg_paths, cpu)->buf, new);
>> 102 /* Don't need to wait for inflights, they'd have gotten NULL. */
>> 103 }
>> 104
>> 105 out:
>> 106 mutex_unlock(®_lock);
33 return 0; 107 return 0;
>> 108
>> 109 out_fail_free:
>> 110 free_memcg_path_bufs();
>> 111 out_fail:
>> 112 /* Since we failed, undo the earlier ref increment. */
>> 113 --reg_refcount;
>> 114
>> 115 mutex_unlock(®_lock);
>> 116 return -ENOMEM;
34 } 117 }
35 118
36 void trace_mmap_lock_unreg(void) 119 void trace_mmap_lock_unreg(void)
37 { 120 {
38 atomic_dec(®_refcount); !! 121 mutex_lock(®_lock);
>> 122
>> 123 /* If the refcount is going 1->0, proceed with freeing buffers. */
>> 124 if (--reg_refcount)
>> 125 goto out;
>> 126
>> 127 free_memcg_path_bufs();
>> 128
>> 129 out:
>> 130 mutex_unlock(®_lock);
>> 131 }
>> 132
>> 133 static inline char *get_memcg_path_buf(void)
>> 134 {
>> 135 struct memcg_path *memcg_path = this_cpu_ptr(&memcg_paths);
>> 136 char *buf;
>> 137 int idx;
>> 138
>> 139 rcu_read_lock();
>> 140 buf = rcu_dereference(memcg_path->buf);
>> 141 if (buf == NULL) {
>> 142 rcu_read_unlock();
>> 143 return NULL;
>> 144 }
>> 145 idx = local_add_return(MEMCG_PATH_BUF_SIZE, &memcg_path->buf_idx) -
>> 146 MEMCG_PATH_BUF_SIZE;
>> 147 return &buf[idx];
39 } 148 }
40 149
41 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) !! 150 static inline void put_memcg_path_buf(void)
42 do { !! 151 {
43 char buf[MEMCG_PATH_BUF_SIZE]; !! 152 local_sub(MEMCG_PATH_BUF_SIZE, &this_cpu_ptr(&memcg_paths)->buf_idx);
44 get_mm_memcg_path(mm, buf, siz !! 153 rcu_read_unlock();
45 trace_mmap_lock_##type(mm, buf !! 154 }
>> 155
>> 156 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
>> 157 do { \
>> 158 const char *memcg_path; \
>> 159 local_lock(&memcg_paths.lock); \
>> 160 memcg_path = get_mm_memcg_path(mm); \
>> 161 trace_mmap_lock_##type(mm, \
>> 162 memcg_path != NULL ? memcg_path : "", \
>> 163 ##__VA_ARGS__); \
>> 164 if (likely(memcg_path != NULL)) \
>> 165 put_memcg_path_buf(); \
>> 166 local_unlock(&memcg_paths.lock); \
46 } while (0) 167 } while (0)
47 168
48 #else /* !CONFIG_MEMCG */ 169 #else /* !CONFIG_MEMCG */
49 170
50 int trace_mmap_lock_reg(void) 171 int trace_mmap_lock_reg(void)
51 { 172 {
52 return 0; 173 return 0;
53 } 174 }
54 175
55 void trace_mmap_lock_unreg(void) 176 void trace_mmap_lock_unreg(void)
56 { 177 {
57 } 178 }
58 179
59 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) 180 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
60 trace_mmap_lock_##type(mm, "", ##__VA_ 181 trace_mmap_lock_##type(mm, "", ##__VA_ARGS__)
61 182
62 #endif /* CONFIG_MEMCG */ 183 #endif /* CONFIG_MEMCG */
63 184
64 #ifdef CONFIG_TRACING 185 #ifdef CONFIG_TRACING
65 #ifdef CONFIG_MEMCG 186 #ifdef CONFIG_MEMCG
66 /* 187 /*
67 * Write the given mm_struct's memcg path to a !! 188 * Write the given mm_struct's memcg path to a percpu buffer, and return a
68 * determined or the trace event is being unre !! 189 * pointer to it. If the path cannot be determined, or no buffer was available
>> 190 * (because the trace event is being unregistered), NULL is returned.
>> 191 *
>> 192 * Note: buffers are allocated per-cpu to avoid locking, so preemption must be
>> 193 * disabled by the caller before calling us, and re-enabled only after the
>> 194 * caller is done with the pointer.
>> 195 *
>> 196 * The caller must call put_memcg_path_buf() once the buffer is no longer
>> 197 * needed. This must be done while preemption is still disabled.
69 */ 198 */
70 static void get_mm_memcg_path(struct mm_struct !! 199 static const char *get_mm_memcg_path(struct mm_struct *mm)
71 { 200 {
72 struct mem_cgroup *memcg; !! 201 char *buf = NULL;
>> 202 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
73 203
74 buf[0] = '\0'; <<
75 /* No need to get path if no trace eve <<
76 if (!atomic_read(®_refcount)) <<
77 return; <<
78 memcg = get_mem_cgroup_from_mm(mm); <<
79 if (memcg == NULL) 204 if (memcg == NULL)
80 return; !! 205 goto out;
81 if (memcg->css.cgroup) !! 206 if (unlikely(memcg->css.cgroup == NULL))
82 cgroup_path(memcg->css.cgroup, !! 207 goto out_put;
>> 208
>> 209 buf = get_memcg_path_buf();
>> 210 if (buf == NULL)
>> 211 goto out_put;
>> 212
>> 213 cgroup_path(memcg->css.cgroup, buf, MEMCG_PATH_BUF_SIZE);
>> 214
>> 215 out_put:
83 css_put(&memcg->css); 216 css_put(&memcg->css);
>> 217 out:
>> 218 return buf;
84 } 219 }
85 220
86 #endif /* CONFIG_MEMCG */ 221 #endif /* CONFIG_MEMCG */
87 222
88 /* 223 /*
89 * Trace calls must be in a separate file, as 224 * Trace calls must be in a separate file, as otherwise there's a circular
90 * dependency between linux/mmap_lock.h and tr 225 * dependency between linux/mmap_lock.h and trace/events/mmap_lock.h.
91 */ 226 */
92 227
93 void __mmap_lock_do_trace_start_locking(struct 228 void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write)
94 { 229 {
95 TRACE_MMAP_LOCK_EVENT(start_locking, m 230 TRACE_MMAP_LOCK_EVENT(start_locking, mm, write);
96 } 231 }
97 EXPORT_SYMBOL(__mmap_lock_do_trace_start_locki 232 EXPORT_SYMBOL(__mmap_lock_do_trace_start_locking);
98 233
99 void __mmap_lock_do_trace_acquire_returned(str 234 void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
100 boo 235 bool success)
101 { 236 {
102 TRACE_MMAP_LOCK_EVENT(acquire_returned 237 TRACE_MMAP_LOCK_EVENT(acquire_returned, mm, write, success);
103 } 238 }
104 EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_ret 239 EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_returned);
105 240
106 void __mmap_lock_do_trace_released(struct mm_s 241 void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write)
107 { 242 {
108 TRACE_MMAP_LOCK_EVENT(released, mm, wr 243 TRACE_MMAP_LOCK_EVENT(released, mm, write);
109 } 244 }
110 EXPORT_SYMBOL(__mmap_lock_do_trace_released); 245 EXPORT_SYMBOL(__mmap_lock_do_trace_released);
111 #endif /* CONFIG_TRACING */ 246 #endif /* CONFIG_TRACING */
112 247
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.