1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2 1
2 /* Copyright (c) 2022 Meta Platforms, Inc. and
3 #ifndef __USDT_BPF_H__
4 #define __USDT_BPF_H__
5
6 #include <linux/errno.h>
7 #include "bpf_helpers.h"
8 #include "bpf_tracing.h"
9
10 /* Below types and maps are internal implement
11 * support and are subjects to change. Also, b
12 * be considered an unstable API as well and m
13 * feedback from using libbpf's USDT support i
14 */
15
16 /* User can override BPF_USDT_MAX_SPEC_CNT to
17 * map that keeps track of USDT argument speci
18 * necessary if there are a lot of USDT attach
19 */
20 #ifndef BPF_USDT_MAX_SPEC_CNT
21 #define BPF_USDT_MAX_SPEC_CNT 256
22 #endif
23 /* User can override BPF_USDT_MAX_IP_CNT to ch
24 * map that keeps track of IP (memory address)
25 * specification.
26 * Note, if kernel supports BPF cookies, this
27 * resized all the way to 1 to save a bit of m
28 */
29 #ifndef BPF_USDT_MAX_IP_CNT
30 #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_
31 #endif
32
33 enum __bpf_usdt_arg_type {
34 BPF_USDT_ARG_CONST,
35 BPF_USDT_ARG_REG,
36 BPF_USDT_ARG_REG_DEREF,
37 };
38
39 struct __bpf_usdt_arg_spec {
40 /* u64 scalar interpreted depending on
41 __u64 val_off;
42 /* arg location case, see bpf_usdt_arg
43 enum __bpf_usdt_arg_type arg_type;
44 /* offset of referenced register withi
45 short reg_off;
46 /* whether arg should be interpreted a
47 bool arg_signed;
48 /* number of bits that need to be clea
49 * sign-extended to cast arguments tha
50 * long into final 8-byte u64/s64 valu
51 */
52 char arg_bitshift;
53 };
54
55 /* should match USDT_MAX_ARG_CNT in usdt.c exa
56 #define BPF_USDT_MAX_ARG_CNT 12
57 struct __bpf_usdt_spec {
58 struct __bpf_usdt_arg_spec args[BPF_US
59 __u64 usdt_cookie;
60 short arg_cnt;
61 };
62
63 struct {
64 __uint(type, BPF_MAP_TYPE_ARRAY);
65 __uint(max_entries, BPF_USDT_MAX_SPEC_
66 __type(key, int);
67 __type(value, struct __bpf_usdt_spec);
68 } __bpf_usdt_specs SEC(".maps") __weak;
69
70 struct {
71 __uint(type, BPF_MAP_TYPE_HASH);
72 __uint(max_entries, BPF_USDT_MAX_IP_CN
73 __type(key, long);
74 __type(value, __u32);
75 } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak
76
77 extern const _Bool LINUX_HAS_BPF_COOKIE __kcon
78
79 static __always_inline
80 int __bpf_usdt_spec_id(struct pt_regs *ctx)
81 {
82 if (!LINUX_HAS_BPF_COOKIE) {
83 long ip = PT_REGS_IP(ctx);
84 int *spec_id_ptr;
85
86 spec_id_ptr = bpf_map_lookup_e
87 return spec_id_ptr ? *spec_id_
88 }
89
90 return bpf_get_attach_cookie(ctx);
91 }
92
93 /* Return number of USDT arguments defined for
94 __weak __hidden
95 int bpf_usdt_arg_cnt(struct pt_regs *ctx)
96 {
97 struct __bpf_usdt_spec *spec;
98 int spec_id;
99
100 spec_id = __bpf_usdt_spec_id(ctx);
101 if (spec_id < 0)
102 return -ESRCH;
103
104 spec = bpf_map_lookup_elem(&__bpf_usdt
105 if (!spec)
106 return -ESRCH;
107
108 return spec->arg_cnt;
109 }
110
111 /* Fetch USDT argument #*arg_num* (zero-indexe
112 * Returns 0 on success; negative error, other
113 * On error *res is guaranteed to be set to ze
114 */
115 __weak __hidden
116 int bpf_usdt_arg(struct pt_regs *ctx, __u64 ar
117 {
118 struct __bpf_usdt_spec *spec;
119 struct __bpf_usdt_arg_spec *arg_spec;
120 unsigned long val;
121 int err, spec_id;
122
123 *res = 0;
124
125 spec_id = __bpf_usdt_spec_id(ctx);
126 if (spec_id < 0)
127 return -ESRCH;
128
129 spec = bpf_map_lookup_elem(&__bpf_usdt
130 if (!spec)
131 return -ESRCH;
132
133 if (arg_num >= BPF_USDT_MAX_ARG_CNT)
134 return -ENOENT;
135 barrier_var(arg_num);
136 if (arg_num >= spec->arg_cnt)
137 return -ENOENT;
138
139 arg_spec = &spec->args[arg_num];
140 switch (arg_spec->arg_type) {
141 case BPF_USDT_ARG_CONST:
142 /* Arg is just a constant ("-4
143 * value is recorded in arg_sp
144 */
145 val = arg_spec->val_off;
146 break;
147 case BPF_USDT_ARG_REG:
148 /* Arg is in a register (e.g,
149 * so we read the contents of
150 * struct pt_regs. To keep thi
151 * record offsetof(struct pt_r
152 */
153 err = bpf_probe_read_kernel(&v
154 if (err)
155 return err;
156 break;
157 case BPF_USDT_ARG_REG_DEREF:
158 /* Arg is in memory addressed
159 * (e.g., "-4@-1204(%rbp)" in
160 * identified like with BPF_US
161 * is in arg_spec->val_off. We
162 * from pt_regs, then do anoth
163 * fetch argument value itself
164 */
165 err = bpf_probe_read_kernel(&v
166 if (err)
167 return err;
168 err = bpf_probe_read_user(&val
169 if (err)
170 return err;
171 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
172 val >>= arg_spec->arg_bitshift
173 #endif
174 break;
175 default:
176 return -EINVAL;
177 }
178
179 /* cast arg from 1, 2, or 4 bytes to f
180 * necessary upper arg_bitshift bits,
181 * is signed
182 */
183 val <<= arg_spec->arg_bitshift;
184 if (arg_spec->arg_signed)
185 val = ((long)val) >> arg_spec-
186 else
187 val = val >> arg_spec->arg_bit
188 *res = val;
189 return 0;
190 }
191
192 /* Retrieve user-specified cookie value provid
193 * bpf_usdt_opts.usdt_cookie. This serves the
194 * returned by bpf_get_attach_cookie(). Libbpf
195 * utilizing BPF cookies internally, so user c
196 * for USDT programs and has to use bpf_usdt_c
197 */
198 __weak __hidden
199 long bpf_usdt_cookie(struct pt_regs *ctx)
200 {
201 struct __bpf_usdt_spec *spec;
202 int spec_id;
203
204 spec_id = __bpf_usdt_spec_id(ctx);
205 if (spec_id < 0)
206 return 0;
207
208 spec = bpf_map_lookup_elem(&__bpf_usdt
209 if (!spec)
210 return 0;
211
212 return spec->usdt_cookie;
213 }
214
215 /* we rely on ___bpf_apply() and ___bpf_narg()
216 #define ___bpf_usdt_args0() ctx
217 #define ___bpf_usdt_args1(x) ___bpf_usdt_args0
218 #define ___bpf_usdt_args2(x, args...) ___bpf_u
219 #define ___bpf_usdt_args3(x, args...) ___bpf_u
220 #define ___bpf_usdt_args4(x, args...) ___bpf_u
221 #define ___bpf_usdt_args5(x, args...) ___bpf_u
222 #define ___bpf_usdt_args6(x, args...) ___bpf_u
223 #define ___bpf_usdt_args7(x, args...) ___bpf_u
224 #define ___bpf_usdt_args8(x, args...) ___bpf_u
225 #define ___bpf_usdt_args9(x, args...) ___bpf_u
226 #define ___bpf_usdt_args10(x, args...) ___bpf_
227 #define ___bpf_usdt_args11(x, args...) ___bpf_
228 #define ___bpf_usdt_args12(x, args...) ___bpf_
229 #define ___bpf_usdt_args(args...) ___bpf_apply
230
231 /*
232 * BPF_USDT serves the same purpose for USDT h
233 * tp_btf/fentry/fexit BPF programs and BPF_KP
234 * Original struct pt_regs * context is preser
235 */
236 #define BPF_USDT(name, args...)
237 name(struct pt_regs *ctx);
238 static __always_inline typeof(name(0))
239 ____##name(struct pt_regs *ctx, ##args);
240 typeof(name(0)) name(struct pt_regs *ctx)
241 {
242 _Pragma("GCC diagnostic push")
243 _Pragma("GCC diagnostic ignored \"-Win
244 return ____##name(___bpf_usdt_args(arg
245 _Pragma("GCC diagnostic pop")
246 }
247 static __always_inline typeof(name(0))
248 ____##name(struct pt_regs *ctx, ##args)
249
250 #endif /* __USDT_BPF_H__ */
251
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.