1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 #define CREATE_TRACE_POINTS 2 #define CREATE_TRACE_POINTS
3 #include <trace/events/mmap_lock.h> 3 #include <trace/events/mmap_lock.h>
4 4
5 #include <linux/mm.h> 5 #include <linux/mm.h>
6 #include <linux/cgroup.h> 6 #include <linux/cgroup.h>
7 #include <linux/memcontrol.h> 7 #include <linux/memcontrol.h>
8 #include <linux/mmap_lock.h> 8 #include <linux/mmap_lock.h>
9 #include <linux/mutex.h> 9 #include <linux/mutex.h>
10 #include <linux/percpu.h> 10 #include <linux/percpu.h>
11 #include <linux/rcupdate.h> 11 #include <linux/rcupdate.h>
12 #include <linux/smp.h> 12 #include <linux/smp.h>
13 #include <linux/trace_events.h> 13 #include <linux/trace_events.h>
14 #include <linux/local_lock.h> 14 #include <linux/local_lock.h>
15 15
16 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locki 16 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking);
17 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_ret 17 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned);
18 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released); 18 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released);
19 19
20 #ifdef CONFIG_MEMCG 20 #ifdef CONFIG_MEMCG
21 21
22 static atomic_t reg_refcount; !! 22 /*
>> 23 * Our various events all share the same buffer (because we don't want or need
>> 24 * to allocate a set of buffers *per event type*), so we need to protect against
>> 25 * concurrent _reg() and _unreg() calls, and count how many _reg() calls have
>> 26 * been made.
>> 27 */
>> 28 static DEFINE_MUTEX(reg_lock);
>> 29 static int reg_refcount; /* Protected by reg_lock. */
23 30
24 /* 31 /*
25 * Size of the buffer for memcg path names. Ig 32 * Size of the buffer for memcg path names. Ignoring stack trace support,
26 * trace_events_hist.c uses MAX_FILTER_STR_VAL 33 * trace_events_hist.c uses MAX_FILTER_STR_VAL for this, so we also use it.
27 */ 34 */
28 #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL 35 #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL
29 36
>> 37 /*
>> 38 * How many contexts our trace events might be called in: normal, softirq, irq,
>> 39 * and NMI.
>> 40 */
>> 41 #define CONTEXT_COUNT 4
>> 42
>> 43 struct memcg_path {
>> 44 local_lock_t lock;
>> 45 char __rcu *buf;
>> 46 local_t buf_idx;
>> 47 };
>> 48 static DEFINE_PER_CPU(struct memcg_path, memcg_paths) = {
>> 49 .lock = INIT_LOCAL_LOCK(lock),
>> 50 .buf_idx = LOCAL_INIT(0),
>> 51 };
>> 52
>> 53 static char **tmp_bufs;
>> 54
>> 55 /* Called with reg_lock held. */
>> 56 static void free_memcg_path_bufs(void)
>> 57 {
>> 58 struct memcg_path *memcg_path;
>> 59 int cpu;
>> 60 char **old = tmp_bufs;
>> 61
>> 62 for_each_possible_cpu(cpu) {
>> 63 memcg_path = per_cpu_ptr(&memcg_paths, cpu);
>> 64 *(old++) = rcu_dereference_protected(memcg_path->buf,
>> 65 lockdep_is_held(®_lock));
>> 66 rcu_assign_pointer(memcg_path->buf, NULL);
>> 67 }
>> 68
>> 69 /* Wait for inflight memcg_path_buf users to finish. */
>> 70 synchronize_rcu();
>> 71
>> 72 old = tmp_bufs;
>> 73 for_each_possible_cpu(cpu) {
>> 74 kfree(*(old++));
>> 75 }
>> 76
>> 77 kfree(tmp_bufs);
>> 78 tmp_bufs = NULL;
>> 79 }
>> 80
30 int trace_mmap_lock_reg(void) 81 int trace_mmap_lock_reg(void)
31 { 82 {
32 atomic_inc(®_refcount); !! 83 int cpu;
>> 84 char *new;
>> 85
>> 86 mutex_lock(®_lock);
>> 87
>> 88 /* If the refcount is going 0->1, proceed with allocating buffers. */
>> 89 if (reg_refcount++)
>> 90 goto out;
>> 91
>> 92 tmp_bufs = kmalloc_array(num_possible_cpus(), sizeof(*tmp_bufs),
>> 93 GFP_KERNEL);
>> 94 if (tmp_bufs == NULL)
>> 95 goto out_fail;
>> 96
>> 97 for_each_possible_cpu(cpu) {
>> 98 new = kmalloc(MEMCG_PATH_BUF_SIZE * CONTEXT_COUNT, GFP_KERNEL);
>> 99 if (new == NULL)
>> 100 goto out_fail_free;
>> 101 rcu_assign_pointer(per_cpu_ptr(&memcg_paths, cpu)->buf, new);
>> 102 /* Don't need to wait for inflights, they'd have gotten NULL. */
>> 103 }
>> 104
>> 105 out:
>> 106 mutex_unlock(®_lock);
33 return 0; 107 return 0;
>> 108
>> 109 out_fail_free:
>> 110 free_memcg_path_bufs();
>> 111 out_fail:
>> 112 /* Since we failed, undo the earlier ref increment. */
>> 113 --reg_refcount;
>> 114
>> 115 mutex_unlock(®_lock);
>> 116 return -ENOMEM;
34 } 117 }
35 118
36 void trace_mmap_lock_unreg(void) 119 void trace_mmap_lock_unreg(void)
37 { 120 {
38 atomic_dec(®_refcount); !! 121 mutex_lock(®_lock);
>> 122
>> 123 /* If the refcount is going 1->0, proceed with freeing buffers. */
>> 124 if (--reg_refcount)
>> 125 goto out;
>> 126
>> 127 free_memcg_path_bufs();
>> 128
>> 129 out:
>> 130 mutex_unlock(®_lock);
>> 131 }
>> 132
>> 133 static inline char *get_memcg_path_buf(void)
>> 134 {
>> 135 struct memcg_path *memcg_path = this_cpu_ptr(&memcg_paths);
>> 136 char *buf;
>> 137 int idx;
>> 138
>> 139 rcu_read_lock();
>> 140 buf = rcu_dereference(memcg_path->buf);
>> 141 if (buf == NULL) {
>> 142 rcu_read_unlock();
>> 143 return NULL;
>> 144 }
>> 145 idx = local_add_return(MEMCG_PATH_BUF_SIZE, &memcg_path->buf_idx) -
>> 146 MEMCG_PATH_BUF_SIZE;
>> 147 return &buf[idx];
>> 148 }
>> 149
>> 150 static inline void put_memcg_path_buf(void)
>> 151 {
>> 152 local_sub(MEMCG_PATH_BUF_SIZE, &this_cpu_ptr(&memcg_paths)->buf_idx);
>> 153 rcu_read_unlock();
39 } 154 }
40 155
41 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) !! 156 /*
42 do { !! 157 * Write the given mm_struct's memcg path to a percpu buffer, and return a
43 char buf[MEMCG_PATH_BUF_SIZE]; !! 158 * pointer to it. If the path cannot be determined, or no buffer was available
44 get_mm_memcg_path(mm, buf, siz !! 159 * (because the trace event is being unregistered), NULL is returned.
45 trace_mmap_lock_##type(mm, buf !! 160 *
>> 161 * Note: buffers are allocated per-cpu to avoid locking, so preemption must be
>> 162 * disabled by the caller before calling us, and re-enabled only after the
>> 163 * caller is done with the pointer.
>> 164 *
>> 165 * The caller must call put_memcg_path_buf() once the buffer is no longer
>> 166 * needed. This must be done while preemption is still disabled.
>> 167 */
>> 168 static const char *get_mm_memcg_path(struct mm_struct *mm)
>> 169 {
>> 170 char *buf = NULL;
>> 171 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
>> 172
>> 173 if (memcg == NULL)
>> 174 goto out;
>> 175 if (unlikely(memcg->css.cgroup == NULL))
>> 176 goto out_put;
>> 177
>> 178 buf = get_memcg_path_buf();
>> 179 if (buf == NULL)
>> 180 goto out_put;
>> 181
>> 182 cgroup_path(memcg->css.cgroup, buf, MEMCG_PATH_BUF_SIZE);
>> 183
>> 184 out_put:
>> 185 css_put(&memcg->css);
>> 186 out:
>> 187 return buf;
>> 188 }
>> 189
>> 190 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
>> 191 do { \
>> 192 const char *memcg_path; \
>> 193 local_lock(&memcg_paths.lock); \
>> 194 memcg_path = get_mm_memcg_path(mm); \
>> 195 trace_mmap_lock_##type(mm, \
>> 196 memcg_path != NULL ? memcg_path : "", \
>> 197 ##__VA_ARGS__); \
>> 198 if (likely(memcg_path != NULL)) \
>> 199 put_memcg_path_buf(); \
>> 200 local_unlock(&memcg_paths.lock); \
46 } while (0) 201 } while (0)
47 202
48 #else /* !CONFIG_MEMCG */ 203 #else /* !CONFIG_MEMCG */
49 204
50 int trace_mmap_lock_reg(void) 205 int trace_mmap_lock_reg(void)
51 { 206 {
52 return 0; 207 return 0;
53 } 208 }
54 209
55 void trace_mmap_lock_unreg(void) 210 void trace_mmap_lock_unreg(void)
56 { 211 {
57 } 212 }
58 213
59 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) 214 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
60 trace_mmap_lock_##type(mm, "", ##__VA_ 215 trace_mmap_lock_##type(mm, "", ##__VA_ARGS__)
61 216
62 #endif /* CONFIG_MEMCG */ 217 #endif /* CONFIG_MEMCG */
63 218
64 #ifdef CONFIG_TRACING <<
65 #ifdef CONFIG_MEMCG <<
66 /* <<
67 * Write the given mm_struct's memcg path to a <<
68 * determined or the trace event is being unre <<
69 */ <<
70 static void get_mm_memcg_path(struct mm_struct <<
71 { <<
72 struct mem_cgroup *memcg; <<
73 <<
74 buf[0] = '\0'; <<
75 /* No need to get path if no trace eve <<
76 if (!atomic_read(®_refcount)) <<
77 return; <<
78 memcg = get_mem_cgroup_from_mm(mm); <<
79 if (memcg == NULL) <<
80 return; <<
81 if (memcg->css.cgroup) <<
82 cgroup_path(memcg->css.cgroup, <<
83 css_put(&memcg->css); <<
84 } <<
85 <<
86 #endif /* CONFIG_MEMCG */ <<
87 <<
88 /* 219 /*
89 * Trace calls must be in a separate file, as 220 * Trace calls must be in a separate file, as otherwise there's a circular
90 * dependency between linux/mmap_lock.h and tr 221 * dependency between linux/mmap_lock.h and trace/events/mmap_lock.h.
91 */ 222 */
92 223
93 void __mmap_lock_do_trace_start_locking(struct 224 void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write)
94 { 225 {
95 TRACE_MMAP_LOCK_EVENT(start_locking, m 226 TRACE_MMAP_LOCK_EVENT(start_locking, mm, write);
96 } 227 }
97 EXPORT_SYMBOL(__mmap_lock_do_trace_start_locki 228 EXPORT_SYMBOL(__mmap_lock_do_trace_start_locking);
98 229
99 void __mmap_lock_do_trace_acquire_returned(str 230 void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
100 boo 231 bool success)
101 { 232 {
102 TRACE_MMAP_LOCK_EVENT(acquire_returned 233 TRACE_MMAP_LOCK_EVENT(acquire_returned, mm, write, success);
103 } 234 }
104 EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_ret 235 EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_returned);
105 236
106 void __mmap_lock_do_trace_released(struct mm_s 237 void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write)
107 { 238 {
108 TRACE_MMAP_LOCK_EVENT(released, mm, wr 239 TRACE_MMAP_LOCK_EVENT(released, mm, write);
109 } 240 }
110 EXPORT_SYMBOL(__mmap_lock_do_trace_released); 241 EXPORT_SYMBOL(__mmap_lock_do_trace_released);
111 #endif /* CONFIG_TRACING */ <<
112 242
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.