1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2 1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2 /* Copyright (c) 2022 Meta Platforms, Inc. and 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #ifndef __USDT_BPF_H__ 3 #ifndef __USDT_BPF_H__
4 #define __USDT_BPF_H__ 4 #define __USDT_BPF_H__
5 5
6 #include <linux/errno.h> 6 #include <linux/errno.h>
7 #include "bpf_helpers.h" !! 7 #include <bpf/bpf_helpers.h>
8 #include "bpf_tracing.h" !! 8 #include <bpf/bpf_tracing.h>
>> 9 #include <bpf/bpf_core_read.h>
9 10
10 /* Below types and maps are internal implement 11 /* Below types and maps are internal implementation details of libbpf's USDT
11 * support and are subjects to change. Also, b 12 * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
12 * be considered an unstable API as well and m 13 * be considered an unstable API as well and might be adjusted based on user
13 * feedback from using libbpf's USDT support i 14 * feedback from using libbpf's USDT support in production.
14 */ 15 */
15 16
16 /* User can override BPF_USDT_MAX_SPEC_CNT to 17 /* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
17 * map that keeps track of USDT argument speci 18 * map that keeps track of USDT argument specifications. This might be
18 * necessary if there are a lot of USDT attach 19 * necessary if there are a lot of USDT attachments.
19 */ 20 */
20 #ifndef BPF_USDT_MAX_SPEC_CNT 21 #ifndef BPF_USDT_MAX_SPEC_CNT
21 #define BPF_USDT_MAX_SPEC_CNT 256 22 #define BPF_USDT_MAX_SPEC_CNT 256
22 #endif 23 #endif
23 /* User can override BPF_USDT_MAX_IP_CNT to ch 24 /* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
24 * map that keeps track of IP (memory address) 25 * map that keeps track of IP (memory address) mapping to USDT argument
25 * specification. 26 * specification.
26 * Note, if kernel supports BPF cookies, this 27 * Note, if kernel supports BPF cookies, this map is not used and could be
27 * resized all the way to 1 to save a bit of m 28 * resized all the way to 1 to save a bit of memory.
28 */ 29 */
29 #ifndef BPF_USDT_MAX_IP_CNT 30 #ifndef BPF_USDT_MAX_IP_CNT
30 #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_ 31 #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
31 #endif 32 #endif
>> 33 /* We use BPF CO-RE to detect support for BPF cookie from BPF side. This is
>> 34 * the only dependency on CO-RE, so if it's undesirable, user can override
>> 35 * BPF_USDT_HAS_BPF_COOKIE to specify whether to BPF cookie is supported or not.
>> 36 */
>> 37 #ifndef BPF_USDT_HAS_BPF_COOKIE
>> 38 #define BPF_USDT_HAS_BPF_COOKIE \
>> 39 bpf_core_enum_value_exists(enum bpf_func_id___usdt, BPF_FUNC_get_attach_cookie___usdt)
>> 40 #endif
32 41
33 enum __bpf_usdt_arg_type { 42 enum __bpf_usdt_arg_type {
34 BPF_USDT_ARG_CONST, 43 BPF_USDT_ARG_CONST,
35 BPF_USDT_ARG_REG, 44 BPF_USDT_ARG_REG,
36 BPF_USDT_ARG_REG_DEREF, 45 BPF_USDT_ARG_REG_DEREF,
37 }; 46 };
38 47
39 struct __bpf_usdt_arg_spec { 48 struct __bpf_usdt_arg_spec {
40 /* u64 scalar interpreted depending on 49 /* u64 scalar interpreted depending on arg_type, see below */
41 __u64 val_off; 50 __u64 val_off;
42 /* arg location case, see bpf_usdt_arg !! 51 /* arg location case, see bpf_udst_arg() for details */
43 enum __bpf_usdt_arg_type arg_type; 52 enum __bpf_usdt_arg_type arg_type;
44 /* offset of referenced register withi 53 /* offset of referenced register within struct pt_regs */
45 short reg_off; 54 short reg_off;
46 /* whether arg should be interpreted a 55 /* whether arg should be interpreted as signed value */
47 bool arg_signed; 56 bool arg_signed;
48 /* number of bits that need to be clea 57 /* number of bits that need to be cleared and, optionally,
49 * sign-extended to cast arguments tha 58 * sign-extended to cast arguments that are 1, 2, or 4 bytes
50 * long into final 8-byte u64/s64 valu 59 * long into final 8-byte u64/s64 value returned to user
51 */ 60 */
52 char arg_bitshift; 61 char arg_bitshift;
53 }; 62 };
54 63
55 /* should match USDT_MAX_ARG_CNT in usdt.c exa 64 /* should match USDT_MAX_ARG_CNT in usdt.c exactly */
56 #define BPF_USDT_MAX_ARG_CNT 12 65 #define BPF_USDT_MAX_ARG_CNT 12
57 struct __bpf_usdt_spec { 66 struct __bpf_usdt_spec {
58 struct __bpf_usdt_arg_spec args[BPF_US 67 struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
59 __u64 usdt_cookie; 68 __u64 usdt_cookie;
60 short arg_cnt; 69 short arg_cnt;
61 }; 70 };
62 71
63 struct { 72 struct {
64 __uint(type, BPF_MAP_TYPE_ARRAY); 73 __uint(type, BPF_MAP_TYPE_ARRAY);
65 __uint(max_entries, BPF_USDT_MAX_SPEC_ 74 __uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
66 __type(key, int); 75 __type(key, int);
67 __type(value, struct __bpf_usdt_spec); 76 __type(value, struct __bpf_usdt_spec);
68 } __bpf_usdt_specs SEC(".maps") __weak; 77 } __bpf_usdt_specs SEC(".maps") __weak;
69 78
70 struct { 79 struct {
71 __uint(type, BPF_MAP_TYPE_HASH); 80 __uint(type, BPF_MAP_TYPE_HASH);
72 __uint(max_entries, BPF_USDT_MAX_IP_CN 81 __uint(max_entries, BPF_USDT_MAX_IP_CNT);
73 __type(key, long); 82 __type(key, long);
74 __type(value, __u32); 83 __type(value, __u32);
75 } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak 84 } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
76 85
77 extern const _Bool LINUX_HAS_BPF_COOKIE __kcon !! 86 /* don't rely on user's BPF code to have latest definition of bpf_func_id */
>> 87 enum bpf_func_id___usdt {
>> 88 BPF_FUNC_get_attach_cookie___usdt = 0xBAD, /* value doesn't matter */
>> 89 };
78 90
79 static __always_inline 91 static __always_inline
80 int __bpf_usdt_spec_id(struct pt_regs *ctx) 92 int __bpf_usdt_spec_id(struct pt_regs *ctx)
81 { 93 {
82 if (!LINUX_HAS_BPF_COOKIE) { !! 94 if (!BPF_USDT_HAS_BPF_COOKIE) {
83 long ip = PT_REGS_IP(ctx); 95 long ip = PT_REGS_IP(ctx);
84 int *spec_id_ptr; 96 int *spec_id_ptr;
85 97
86 spec_id_ptr = bpf_map_lookup_e 98 spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
87 return spec_id_ptr ? *spec_id_ 99 return spec_id_ptr ? *spec_id_ptr : -ESRCH;
88 } 100 }
89 101
90 return bpf_get_attach_cookie(ctx); 102 return bpf_get_attach_cookie(ctx);
91 } 103 }
92 104
93 /* Return number of USDT arguments defined for 105 /* Return number of USDT arguments defined for currently traced USDT. */
94 __weak __hidden 106 __weak __hidden
95 int bpf_usdt_arg_cnt(struct pt_regs *ctx) 107 int bpf_usdt_arg_cnt(struct pt_regs *ctx)
96 { 108 {
97 struct __bpf_usdt_spec *spec; 109 struct __bpf_usdt_spec *spec;
98 int spec_id; 110 int spec_id;
99 111
100 spec_id = __bpf_usdt_spec_id(ctx); 112 spec_id = __bpf_usdt_spec_id(ctx);
101 if (spec_id < 0) 113 if (spec_id < 0)
102 return -ESRCH; 114 return -ESRCH;
103 115
104 spec = bpf_map_lookup_elem(&__bpf_usdt 116 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
105 if (!spec) 117 if (!spec)
106 return -ESRCH; 118 return -ESRCH;
107 119
108 return spec->arg_cnt; 120 return spec->arg_cnt;
109 } 121 }
110 122
111 /* Fetch USDT argument #*arg_num* (zero-indexe 123 /* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
112 * Returns 0 on success; negative error, other 124 * Returns 0 on success; negative error, otherwise.
113 * On error *res is guaranteed to be set to ze 125 * On error *res is guaranteed to be set to zero.
114 */ 126 */
115 __weak __hidden 127 __weak __hidden
116 int bpf_usdt_arg(struct pt_regs *ctx, __u64 ar 128 int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)
117 { 129 {
118 struct __bpf_usdt_spec *spec; 130 struct __bpf_usdt_spec *spec;
119 struct __bpf_usdt_arg_spec *arg_spec; 131 struct __bpf_usdt_arg_spec *arg_spec;
120 unsigned long val; 132 unsigned long val;
121 int err, spec_id; 133 int err, spec_id;
122 134
123 *res = 0; 135 *res = 0;
124 136
125 spec_id = __bpf_usdt_spec_id(ctx); 137 spec_id = __bpf_usdt_spec_id(ctx);
126 if (spec_id < 0) 138 if (spec_id < 0)
127 return -ESRCH; 139 return -ESRCH;
128 140
129 spec = bpf_map_lookup_elem(&__bpf_usdt 141 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
130 if (!spec) 142 if (!spec)
131 return -ESRCH; 143 return -ESRCH;
132 144
133 if (arg_num >= BPF_USDT_MAX_ARG_CNT) !! 145 if (arg_num >= BPF_USDT_MAX_ARG_CNT || arg_num >= spec->arg_cnt)
134 return -ENOENT; <<
135 barrier_var(arg_num); <<
136 if (arg_num >= spec->arg_cnt) <<
137 return -ENOENT; 146 return -ENOENT;
138 147
139 arg_spec = &spec->args[arg_num]; 148 arg_spec = &spec->args[arg_num];
140 switch (arg_spec->arg_type) { 149 switch (arg_spec->arg_type) {
141 case BPF_USDT_ARG_CONST: 150 case BPF_USDT_ARG_CONST:
142 /* Arg is just a constant ("-4 151 /* Arg is just a constant ("-4@$-9" in USDT arg spec).
143 * value is recorded in arg_sp 152 * value is recorded in arg_spec->val_off directly.
144 */ 153 */
145 val = arg_spec->val_off; 154 val = arg_spec->val_off;
146 break; 155 break;
147 case BPF_USDT_ARG_REG: 156 case BPF_USDT_ARG_REG:
148 /* Arg is in a register (e.g, 157 /* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
149 * so we read the contents of 158 * so we read the contents of that register directly from
150 * struct pt_regs. To keep thi 159 * struct pt_regs. To keep things simple user-space parts
151 * record offsetof(struct pt_r 160 * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
152 */ 161 */
153 err = bpf_probe_read_kernel(&v 162 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
154 if (err) 163 if (err)
155 return err; 164 return err;
156 break; 165 break;
157 case BPF_USDT_ARG_REG_DEREF: 166 case BPF_USDT_ARG_REG_DEREF:
158 /* Arg is in memory addressed 167 /* Arg is in memory addressed by register, plus some offset
159 * (e.g., "-4@-1204(%rbp)" in 168 * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
160 * identified like with BPF_US 169 * identified like with BPF_USDT_ARG_REG case, and the offset
161 * is in arg_spec->val_off. We 170 * is in arg_spec->val_off. We first fetch register contents
162 * from pt_regs, then do anoth 171 * from pt_regs, then do another user-space probe read to
163 * fetch argument value itself 172 * fetch argument value itself.
164 */ 173 */
165 err = bpf_probe_read_kernel(&v 174 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
166 if (err) 175 if (err)
167 return err; 176 return err;
168 err = bpf_probe_read_user(&val 177 err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
169 if (err) 178 if (err)
170 return err; 179 return err;
171 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 180 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
172 val >>= arg_spec->arg_bitshift 181 val >>= arg_spec->arg_bitshift;
173 #endif 182 #endif
174 break; 183 break;
175 default: 184 default:
176 return -EINVAL; 185 return -EINVAL;
177 } 186 }
178 187
179 /* cast arg from 1, 2, or 4 bytes to f 188 /* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
180 * necessary upper arg_bitshift bits, 189 * necessary upper arg_bitshift bits, with sign extension if argument
181 * is signed 190 * is signed
182 */ 191 */
183 val <<= arg_spec->arg_bitshift; 192 val <<= arg_spec->arg_bitshift;
184 if (arg_spec->arg_signed) 193 if (arg_spec->arg_signed)
185 val = ((long)val) >> arg_spec- 194 val = ((long)val) >> arg_spec->arg_bitshift;
186 else 195 else
187 val = val >> arg_spec->arg_bit 196 val = val >> arg_spec->arg_bitshift;
188 *res = val; 197 *res = val;
189 return 0; 198 return 0;
190 } 199 }
191 200
192 /* Retrieve user-specified cookie value provid 201 /* Retrieve user-specified cookie value provided during attach as
193 * bpf_usdt_opts.usdt_cookie. This serves the 202 * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
194 * returned by bpf_get_attach_cookie(). Libbpf 203 * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
195 * utilizing BPF cookies internally, so user c 204 * utilizing BPF cookies internally, so user can't use BPF cookie directly
196 * for USDT programs and has to use bpf_usdt_c 205 * for USDT programs and has to use bpf_usdt_cookie() API instead.
197 */ 206 */
198 __weak __hidden 207 __weak __hidden
199 long bpf_usdt_cookie(struct pt_regs *ctx) 208 long bpf_usdt_cookie(struct pt_regs *ctx)
200 { 209 {
201 struct __bpf_usdt_spec *spec; 210 struct __bpf_usdt_spec *spec;
202 int spec_id; 211 int spec_id;
203 212
204 spec_id = __bpf_usdt_spec_id(ctx); 213 spec_id = __bpf_usdt_spec_id(ctx);
205 if (spec_id < 0) 214 if (spec_id < 0)
206 return 0; 215 return 0;
207 216
208 spec = bpf_map_lookup_elem(&__bpf_usdt 217 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
209 if (!spec) 218 if (!spec)
210 return 0; 219 return 0;
211 220
212 return spec->usdt_cookie; 221 return spec->usdt_cookie;
213 } 222 }
214 223
215 /* we rely on ___bpf_apply() and ___bpf_narg() 224 /* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
216 #define ___bpf_usdt_args0() ctx 225 #define ___bpf_usdt_args0() ctx
217 #define ___bpf_usdt_args1(x) ___bpf_usdt_args0 !! 226 #define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); (void *)_x; })
218 #define ___bpf_usdt_args2(x, args...) ___bpf_u !! 227 #define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); (void *)_x; })
219 #define ___bpf_usdt_args3(x, args...) ___bpf_u !! 228 #define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); (void *)_x; })
220 #define ___bpf_usdt_args4(x, args...) ___bpf_u !! 229 #define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); (void *)_x; })
221 #define ___bpf_usdt_args5(x, args...) ___bpf_u !! 230 #define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); (void *)_x; })
222 #define ___bpf_usdt_args6(x, args...) ___bpf_u !! 231 #define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); (void *)_x; })
223 #define ___bpf_usdt_args7(x, args...) ___bpf_u !! 232 #define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); (void *)_x; })
224 #define ___bpf_usdt_args8(x, args...) ___bpf_u !! 233 #define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); (void *)_x; })
225 #define ___bpf_usdt_args9(x, args...) ___bpf_u !! 234 #define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); (void *)_x; })
226 #define ___bpf_usdt_args10(x, args...) ___bpf_ !! 235 #define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); (void *)_x; })
227 #define ___bpf_usdt_args11(x, args...) ___bpf_ !! 236 #define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); (void *)_x; })
228 #define ___bpf_usdt_args12(x, args...) ___bpf_ !! 237 #define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); (void *)_x; })
229 #define ___bpf_usdt_args(args...) ___bpf_apply 238 #define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)
230 239
231 /* 240 /*
232 * BPF_USDT serves the same purpose for USDT h 241 * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
233 * tp_btf/fentry/fexit BPF programs and BPF_KP 242 * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
234 * Original struct pt_regs * context is preser 243 * Original struct pt_regs * context is preserved as 'ctx' argument.
235 */ 244 */
236 #define BPF_USDT(name, args...) 245 #define BPF_USDT(name, args...) \
237 name(struct pt_regs *ctx); 246 name(struct pt_regs *ctx); \
238 static __always_inline typeof(name(0)) !! 247 static __attribute__((always_inline)) typeof(name(0)) \
239 ____##name(struct pt_regs *ctx, ##args); 248 ____##name(struct pt_regs *ctx, ##args); \
240 typeof(name(0)) name(struct pt_regs *ctx) 249 typeof(name(0)) name(struct pt_regs *ctx) \
241 { 250 { \
242 _Pragma("GCC diagnostic push") 251 _Pragma("GCC diagnostic push") \
243 _Pragma("GCC diagnostic ignored \"-Win 252 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
244 return ____##name(___bpf_usdt_args(arg 253 return ____##name(___bpf_usdt_args(args)); \
245 _Pragma("GCC diagnostic pop") 254 _Pragma("GCC diagnostic pop") \
246 } 255 } \
247 static __always_inline typeof(name(0)) !! 256 static __attribute__((always_inline)) typeof(name(0)) \
248 ____##name(struct pt_regs *ctx, ##args) 257 ____##name(struct pt_regs *ctx, ##args)
249 258
250 #endif /* __USDT_BPF_H__ */ 259 #endif /* __USDT_BPF_H__ */
251 260
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.