1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 // Copyright (c) 2022, Huawei
3
4 #include "vmlinux.h"
5 #include <bpf/bpf_helpers.h>
6 #include <bpf/bpf_tracing.h>
7 #include <bpf/bpf_core_read.h>
8
9 /*
10 * This should be in sync with "util/kwork.h"
11 */
12 enum kwork_class_type {
13 KWORK_CLASS_IRQ,
14 KWORK_CLASS_SOFTIRQ,
15 KWORK_CLASS_WORKQUEUE,
16 KWORK_CLASS_SCHED,
17 KWORK_CLASS_MAX,
18 };
19
20 #define MAX_ENTRIES 102400
21 #define MAX_NR_CPUS 2048
22 #define PF_KTHREAD 0x00200000
23 #define MAX_COMMAND_LEN 16
24
25 struct time_data {
26 __u64 timestamp;
27 };
28
29 struct work_data {
30 __u64 runtime;
31 };
32
33 struct task_data {
34 __u32 tgid;
35 __u32 is_kthread;
36 char comm[MAX_COMMAND_LEN];
37 };
38
39 struct work_key {
40 __u32 type;
41 __u32 pid;
42 __u64 task_p;
43 };
44
45 struct task_key {
46 __u32 pid;
47 __u32 cpu;
48 };
49
50 struct {
51 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
52 __uint(map_flags, BPF_F_NO_PREALLOC);
53 __type(key, int);
54 __type(value, struct time_data);
55 } kwork_top_task_time SEC(".maps");
56
57 struct {
58 __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
59 __uint(key_size, sizeof(struct work_key));
60 __uint(value_size, sizeof(struct time_data));
61 __uint(max_entries, MAX_ENTRIES);
62 } kwork_top_irq_time SEC(".maps");
63
64 struct {
65 __uint(type, BPF_MAP_TYPE_HASH);
66 __uint(key_size, sizeof(struct task_key));
67 __uint(value_size, sizeof(struct task_data));
68 __uint(max_entries, MAX_ENTRIES);
69 } kwork_top_tasks SEC(".maps");
70
71 struct {
72 __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
73 __uint(key_size, sizeof(struct work_key));
74 __uint(value_size, sizeof(struct work_data));
75 __uint(max_entries, MAX_ENTRIES);
76 } kwork_top_works SEC(".maps");
77
78 struct {
79 __uint(type, BPF_MAP_TYPE_HASH);
80 __uint(key_size, sizeof(u32));
81 __uint(value_size, sizeof(u8));
82 __uint(max_entries, MAX_NR_CPUS);
83 } kwork_top_cpu_filter SEC(".maps");
84
85 int enabled = 0;
86
87 int has_cpu_filter = 0;
88
89 __u64 from_timestamp = 0;
90 __u64 to_timestamp = 0;
91
92 static __always_inline int cpu_is_filtered(__u32 cpu)
93 {
94 __u8 *cpu_val;
95
96 if (has_cpu_filter) {
97 cpu_val = bpf_map_lookup_elem(&kwork_top_cpu_filter, &cpu);
98 if (!cpu_val)
99 return 1;
100 }
101
102 return 0;
103 }
104
105 static __always_inline void update_task_info(struct task_struct *task, __u32 cpu)
106 {
107 struct task_key key = {
108 .pid = task->pid,
109 .cpu = cpu,
110 };
111
112 if (!bpf_map_lookup_elem(&kwork_top_tasks, &key)) {
113 struct task_data data = {
114 .tgid = task->tgid,
115 .is_kthread = task->flags & PF_KTHREAD ? 1 : 0,
116 };
117 BPF_CORE_READ_STR_INTO(&data.comm, task, comm);
118
119 bpf_map_update_elem(&kwork_top_tasks, &key, &data, BPF_ANY);
120 }
121 }
122
123 static __always_inline void update_work(struct work_key *key, __u64 delta)
124 {
125 struct work_data *data;
126
127 data = bpf_map_lookup_elem(&kwork_top_works, key);
128 if (data) {
129 data->runtime += delta;
130 } else {
131 struct work_data new_data = {
132 .runtime = delta,
133 };
134
135 bpf_map_update_elem(&kwork_top_works, key, &new_data, BPF_ANY);
136 }
137 }
138
139 static void on_sched_out(struct task_struct *task, __u64 ts, __u32 cpu)
140 {
141 __u64 delta;
142 struct time_data *pelem;
143
144 pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL, 0);
145 if (pelem)
146 delta = ts - pelem->timestamp;
147 else
148 delta = ts - from_timestamp;
149
150 struct work_key key = {
151 .type = KWORK_CLASS_SCHED,
152 .pid = task->pid,
153 .task_p = (__u64)task,
154 };
155
156 update_work(&key, delta);
157 update_task_info(task, cpu);
158 }
159
160 static void on_sched_in(struct task_struct *task, __u64 ts)
161 {
162 struct time_data *pelem;
163
164 pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL,
165 BPF_LOCAL_STORAGE_GET_F_CREATE);
166 if (pelem)
167 pelem->timestamp = ts;
168 }
169
170 SEC("tp_btf/sched_switch")
171 int on_switch(u64 *ctx)
172 {
173 struct task_struct *prev, *next;
174
175 prev = (struct task_struct *)ctx[1];
176 next = (struct task_struct *)ctx[2];
177
178 if (!enabled)
179 return 0;
180
181 __u32 cpu = bpf_get_smp_processor_id();
182
183 if (cpu_is_filtered(cpu))
184 return 0;
185
186 __u64 ts = bpf_ktime_get_ns();
187
188 on_sched_out(prev, ts, cpu);
189 on_sched_in(next, ts);
190
191 return 0;
192 }
193
194 SEC("tp_btf/irq_handler_entry")
195 int on_irq_handler_entry(u64 *cxt)
196 {
197 struct task_struct *task;
198
199 if (!enabled)
200 return 0;
201
202 __u32 cpu = bpf_get_smp_processor_id();
203
204 if (cpu_is_filtered(cpu))
205 return 0;
206
207 __u64 ts = bpf_ktime_get_ns();
208
209 task = (struct task_struct *)bpf_get_current_task();
210 if (!task)
211 return 0;
212
213 struct work_key key = {
214 .type = KWORK_CLASS_IRQ,
215 .pid = BPF_CORE_READ(task, pid),
216 .task_p = (__u64)task,
217 };
218
219 struct time_data data = {
220 .timestamp = ts,
221 };
222
223 bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY);
224
225 return 0;
226 }
227
228 SEC("tp_btf/irq_handler_exit")
229 int on_irq_handler_exit(u64 *cxt)
230 {
231 __u64 delta;
232 struct task_struct *task;
233 struct time_data *pelem;
234
235 if (!enabled)
236 return 0;
237
238 __u32 cpu = bpf_get_smp_processor_id();
239
240 if (cpu_is_filtered(cpu))
241 return 0;
242
243 __u64 ts = bpf_ktime_get_ns();
244
245 task = (struct task_struct *)bpf_get_current_task();
246 if (!task)
247 return 0;
248
249 struct work_key key = {
250 .type = KWORK_CLASS_IRQ,
251 .pid = BPF_CORE_READ(task, pid),
252 .task_p = (__u64)task,
253 };
254
255 pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key);
256 if (pelem && pelem->timestamp != 0)
257 delta = ts - pelem->timestamp;
258 else
259 delta = ts - from_timestamp;
260
261 update_work(&key, delta);
262
263 return 0;
264 }
265
266 SEC("tp_btf/softirq_entry")
267 int on_softirq_entry(u64 *cxt)
268 {
269 struct task_struct *task;
270
271 if (!enabled)
272 return 0;
273
274 __u32 cpu = bpf_get_smp_processor_id();
275
276 if (cpu_is_filtered(cpu))
277 return 0;
278
279 __u64 ts = bpf_ktime_get_ns();
280
281 task = (struct task_struct *)bpf_get_current_task();
282 if (!task)
283 return 0;
284
285 struct work_key key = {
286 .type = KWORK_CLASS_SOFTIRQ,
287 .pid = BPF_CORE_READ(task, pid),
288 .task_p = (__u64)task,
289 };
290
291 struct time_data data = {
292 .timestamp = ts,
293 };
294
295 bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY);
296
297 return 0;
298 }
299
300 SEC("tp_btf/softirq_exit")
301 int on_softirq_exit(u64 *cxt)
302 {
303 __u64 delta;
304 struct task_struct *task;
305 struct time_data *pelem;
306
307 if (!enabled)
308 return 0;
309
310 __u32 cpu = bpf_get_smp_processor_id();
311
312 if (cpu_is_filtered(cpu))
313 return 0;
314
315 __u64 ts = bpf_ktime_get_ns();
316
317 task = (struct task_struct *)bpf_get_current_task();
318 if (!task)
319 return 0;
320
321 struct work_key key = {
322 .type = KWORK_CLASS_SOFTIRQ,
323 .pid = BPF_CORE_READ(task, pid),
324 .task_p = (__u64)task,
325 };
326
327 pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key);
328 if (pelem)
329 delta = ts - pelem->timestamp;
330 else
331 delta = ts - from_timestamp;
332
333 update_work(&key, delta);
334
335 return 0;
336 }
337
338 char LICENSE[] SEC("license") = "Dual BSD/GPL";
339
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.