1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * net/psample/psample.c - Netlink channel for packet sampling
4 * Copyright (c) 2017 Yotam Gigi <yotamg@mellanox.com>
5 */
6
7 #include <linux/types.h>
8 #include <linux/kernel.h>
9 #include <linux/skbuff.h>
10 #include <linux/module.h>
11 #include <linux/timekeeping.h>
12 #include <net/net_namespace.h>
13 #include <net/sock.h>
14 #include <net/netlink.h>
15 #include <net/genetlink.h>
16 #include <net/psample.h>
17 #include <linux/spinlock.h>
18 #include <net/ip_tunnels.h>
19 #include <net/dst_metadata.h>
20
21 #define PSAMPLE_MAX_PACKET_SIZE 0xffff
22
23 static LIST_HEAD(psample_groups_list);
24 static DEFINE_SPINLOCK(psample_groups_lock);
25
26 /* multicast groups */
27 enum psample_nl_multicast_groups {
28 PSAMPLE_NL_MCGRP_CONFIG,
29 PSAMPLE_NL_MCGRP_SAMPLE,
30 };
31
32 static const struct genl_multicast_group psample_nl_mcgrps[] = {
33 [PSAMPLE_NL_MCGRP_CONFIG] = { .name = PSAMPLE_NL_MCGRP_CONFIG_NAME },
34 [PSAMPLE_NL_MCGRP_SAMPLE] = { .name = PSAMPLE_NL_MCGRP_SAMPLE_NAME,
35 .flags = GENL_MCAST_CAP_NET_ADMIN, },
36 };
37
38 static struct genl_family psample_nl_family __ro_after_init;
39
40 static int psample_group_nl_fill(struct sk_buff *msg,
41 struct psample_group *group,
42 enum psample_command cmd, u32 portid, u32 seq,
43 int flags)
44 {
45 void *hdr;
46 int ret;
47
48 hdr = genlmsg_put(msg, portid, seq, &psample_nl_family, flags, cmd);
49 if (!hdr)
50 return -EMSGSIZE;
51
52 ret = nla_put_u32(msg, PSAMPLE_ATTR_SAMPLE_GROUP, group->group_num);
53 if (ret < 0)
54 goto error;
55
56 ret = nla_put_u32(msg, PSAMPLE_ATTR_GROUP_REFCOUNT, group->refcount);
57 if (ret < 0)
58 goto error;
59
60 ret = nla_put_u32(msg, PSAMPLE_ATTR_GROUP_SEQ, group->seq);
61 if (ret < 0)
62 goto error;
63
64 genlmsg_end(msg, hdr);
65 return 0;
66
67 error:
68 genlmsg_cancel(msg, hdr);
69 return -EMSGSIZE;
70 }
71
72 static int psample_nl_cmd_get_group_dumpit(struct sk_buff *msg,
73 struct netlink_callback *cb)
74 {
75 struct psample_group *group;
76 int start = cb->args[0];
77 int idx = 0;
78 int err;
79
80 spin_lock_bh(&psample_groups_lock);
81 list_for_each_entry(group, &psample_groups_list, list) {
82 if (!net_eq(group->net, sock_net(msg->sk)))
83 continue;
84 if (idx < start) {
85 idx++;
86 continue;
87 }
88 err = psample_group_nl_fill(msg, group, PSAMPLE_CMD_NEW_GROUP,
89 NETLINK_CB(cb->skb).portid,
90 cb->nlh->nlmsg_seq, NLM_F_MULTI);
91 if (err)
92 break;
93 idx++;
94 }
95
96 spin_unlock_bh(&psample_groups_lock);
97 cb->args[0] = idx;
98 return msg->len;
99 }
100
101 static const struct genl_small_ops psample_nl_ops[] = {
102 {
103 .cmd = PSAMPLE_CMD_GET_GROUP,
104 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
105 .dumpit = psample_nl_cmd_get_group_dumpit,
106 /* can be retrieved by unprivileged users */
107 }
108 };
109
110 static struct genl_family psample_nl_family __ro_after_init = {
111 .name = PSAMPLE_GENL_NAME,
112 .version = PSAMPLE_GENL_VERSION,
113 .maxattr = PSAMPLE_ATTR_MAX,
114 .netnsok = true,
115 .module = THIS_MODULE,
116 .mcgrps = psample_nl_mcgrps,
117 .small_ops = psample_nl_ops,
118 .n_small_ops = ARRAY_SIZE(psample_nl_ops),
119 .resv_start_op = PSAMPLE_CMD_GET_GROUP + 1,
120 .n_mcgrps = ARRAY_SIZE(psample_nl_mcgrps),
121 };
122
123 static void psample_group_notify(struct psample_group *group,
124 enum psample_command cmd)
125 {
126 struct sk_buff *msg;
127 int err;
128
129 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
130 if (!msg)
131 return;
132
133 err = psample_group_nl_fill(msg, group, cmd, 0, 0, NLM_F_MULTI);
134 if (!err)
135 genlmsg_multicast_netns(&psample_nl_family, group->net, msg, 0,
136 PSAMPLE_NL_MCGRP_CONFIG, GFP_ATOMIC);
137 else
138 nlmsg_free(msg);
139 }
140
141 static struct psample_group *psample_group_create(struct net *net,
142 u32 group_num)
143 {
144 struct psample_group *group;
145
146 group = kzalloc(sizeof(*group), GFP_ATOMIC);
147 if (!group)
148 return NULL;
149
150 group->net = net;
151 group->group_num = group_num;
152 list_add_tail(&group->list, &psample_groups_list);
153
154 psample_group_notify(group, PSAMPLE_CMD_NEW_GROUP);
155 return group;
156 }
157
158 static void psample_group_destroy(struct psample_group *group)
159 {
160 psample_group_notify(group, PSAMPLE_CMD_DEL_GROUP);
161 list_del(&group->list);
162 kfree_rcu(group, rcu);
163 }
164
165 static struct psample_group *
166 psample_group_lookup(struct net *net, u32 group_num)
167 {
168 struct psample_group *group;
169
170 list_for_each_entry(group, &psample_groups_list, list)
171 if ((group->group_num == group_num) && (group->net == net))
172 return group;
173 return NULL;
174 }
175
176 struct psample_group *psample_group_get(struct net *net, u32 group_num)
177 {
178 struct psample_group *group;
179
180 spin_lock_bh(&psample_groups_lock);
181
182 group = psample_group_lookup(net, group_num);
183 if (!group) {
184 group = psample_group_create(net, group_num);
185 if (!group)
186 goto out;
187 }
188 group->refcount++;
189
190 out:
191 spin_unlock_bh(&psample_groups_lock);
192 return group;
193 }
194 EXPORT_SYMBOL_GPL(psample_group_get);
195
196 void psample_group_take(struct psample_group *group)
197 {
198 spin_lock_bh(&psample_groups_lock);
199 group->refcount++;
200 spin_unlock_bh(&psample_groups_lock);
201 }
202 EXPORT_SYMBOL_GPL(psample_group_take);
203
204 void psample_group_put(struct psample_group *group)
205 {
206 spin_lock_bh(&psample_groups_lock);
207
208 if (--group->refcount == 0)
209 psample_group_destroy(group);
210
211 spin_unlock_bh(&psample_groups_lock);
212 }
213 EXPORT_SYMBOL_GPL(psample_group_put);
214
215 #ifdef CONFIG_INET
216 static int __psample_ip_tun_to_nlattr(struct sk_buff *skb,
217 struct ip_tunnel_info *tun_info)
218 {
219 unsigned short tun_proto = ip_tunnel_info_af(tun_info);
220 const void *tun_opts = ip_tunnel_info_opts(tun_info);
221 const struct ip_tunnel_key *tun_key = &tun_info->key;
222 int tun_opts_len = tun_info->options_len;
223
224 if (test_bit(IP_TUNNEL_KEY_BIT, tun_key->tun_flags) &&
225 nla_put_be64(skb, PSAMPLE_TUNNEL_KEY_ATTR_ID, tun_key->tun_id,
226 PSAMPLE_TUNNEL_KEY_ATTR_PAD))
227 return -EMSGSIZE;
228
229 if (tun_info->mode & IP_TUNNEL_INFO_BRIDGE &&
230 nla_put_flag(skb, PSAMPLE_TUNNEL_KEY_ATTR_IPV4_INFO_BRIDGE))
231 return -EMSGSIZE;
232
233 switch (tun_proto) {
234 case AF_INET:
235 if (tun_key->u.ipv4.src &&
236 nla_put_in_addr(skb, PSAMPLE_TUNNEL_KEY_ATTR_IPV4_SRC,
237 tun_key->u.ipv4.src))
238 return -EMSGSIZE;
239 if (tun_key->u.ipv4.dst &&
240 nla_put_in_addr(skb, PSAMPLE_TUNNEL_KEY_ATTR_IPV4_DST,
241 tun_key->u.ipv4.dst))
242 return -EMSGSIZE;
243 break;
244 case AF_INET6:
245 if (!ipv6_addr_any(&tun_key->u.ipv6.src) &&
246 nla_put_in6_addr(skb, PSAMPLE_TUNNEL_KEY_ATTR_IPV6_SRC,
247 &tun_key->u.ipv6.src))
248 return -EMSGSIZE;
249 if (!ipv6_addr_any(&tun_key->u.ipv6.dst) &&
250 nla_put_in6_addr(skb, PSAMPLE_TUNNEL_KEY_ATTR_IPV6_DST,
251 &tun_key->u.ipv6.dst))
252 return -EMSGSIZE;
253 break;
254 }
255 if (tun_key->tos &&
256 nla_put_u8(skb, PSAMPLE_TUNNEL_KEY_ATTR_TOS, tun_key->tos))
257 return -EMSGSIZE;
258 if (nla_put_u8(skb, PSAMPLE_TUNNEL_KEY_ATTR_TTL, tun_key->ttl))
259 return -EMSGSIZE;
260 if (test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, tun_key->tun_flags) &&
261 nla_put_flag(skb, PSAMPLE_TUNNEL_KEY_ATTR_DONT_FRAGMENT))
262 return -EMSGSIZE;
263 if (test_bit(IP_TUNNEL_CSUM_BIT, tun_key->tun_flags) &&
264 nla_put_flag(skb, PSAMPLE_TUNNEL_KEY_ATTR_CSUM))
265 return -EMSGSIZE;
266 if (tun_key->tp_src &&
267 nla_put_be16(skb, PSAMPLE_TUNNEL_KEY_ATTR_TP_SRC, tun_key->tp_src))
268 return -EMSGSIZE;
269 if (tun_key->tp_dst &&
270 nla_put_be16(skb, PSAMPLE_TUNNEL_KEY_ATTR_TP_DST, tun_key->tp_dst))
271 return -EMSGSIZE;
272 if (test_bit(IP_TUNNEL_OAM_BIT, tun_key->tun_flags) &&
273 nla_put_flag(skb, PSAMPLE_TUNNEL_KEY_ATTR_OAM))
274 return -EMSGSIZE;
275 if (tun_opts_len) {
276 if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, tun_key->tun_flags) &&
277 nla_put(skb, PSAMPLE_TUNNEL_KEY_ATTR_GENEVE_OPTS,
278 tun_opts_len, tun_opts))
279 return -EMSGSIZE;
280 else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT,
281 tun_key->tun_flags) &&
282 nla_put(skb, PSAMPLE_TUNNEL_KEY_ATTR_ERSPAN_OPTS,
283 tun_opts_len, tun_opts))
284 return -EMSGSIZE;
285 }
286
287 return 0;
288 }
289
290 static int psample_ip_tun_to_nlattr(struct sk_buff *skb,
291 struct ip_tunnel_info *tun_info)
292 {
293 struct nlattr *nla;
294 int err;
295
296 nla = nla_nest_start_noflag(skb, PSAMPLE_ATTR_TUNNEL);
297 if (!nla)
298 return -EMSGSIZE;
299
300 err = __psample_ip_tun_to_nlattr(skb, tun_info);
301 if (err) {
302 nla_nest_cancel(skb, nla);
303 return err;
304 }
305
306 nla_nest_end(skb, nla);
307
308 return 0;
309 }
310
311 static int psample_tunnel_meta_len(struct ip_tunnel_info *tun_info)
312 {
313 unsigned short tun_proto = ip_tunnel_info_af(tun_info);
314 const struct ip_tunnel_key *tun_key = &tun_info->key;
315 int tun_opts_len = tun_info->options_len;
316 int sum = nla_total_size(0); /* PSAMPLE_ATTR_TUNNEL */
317
318 if (test_bit(IP_TUNNEL_KEY_BIT, tun_key->tun_flags))
319 sum += nla_total_size_64bit(sizeof(u64));
320
321 if (tun_info->mode & IP_TUNNEL_INFO_BRIDGE)
322 sum += nla_total_size(0);
323
324 switch (tun_proto) {
325 case AF_INET:
326 if (tun_key->u.ipv4.src)
327 sum += nla_total_size(sizeof(u32));
328 if (tun_key->u.ipv4.dst)
329 sum += nla_total_size(sizeof(u32));
330 break;
331 case AF_INET6:
332 if (!ipv6_addr_any(&tun_key->u.ipv6.src))
333 sum += nla_total_size(sizeof(struct in6_addr));
334 if (!ipv6_addr_any(&tun_key->u.ipv6.dst))
335 sum += nla_total_size(sizeof(struct in6_addr));
336 break;
337 }
338 if (tun_key->tos)
339 sum += nla_total_size(sizeof(u8));
340 sum += nla_total_size(sizeof(u8)); /* TTL */
341 if (test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, tun_key->tun_flags))
342 sum += nla_total_size(0);
343 if (test_bit(IP_TUNNEL_CSUM_BIT, tun_key->tun_flags))
344 sum += nla_total_size(0);
345 if (tun_key->tp_src)
346 sum += nla_total_size(sizeof(u16));
347 if (tun_key->tp_dst)
348 sum += nla_total_size(sizeof(u16));
349 if (test_bit(IP_TUNNEL_OAM_BIT, tun_key->tun_flags))
350 sum += nla_total_size(0);
351 if (tun_opts_len) {
352 if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, tun_key->tun_flags))
353 sum += nla_total_size(tun_opts_len);
354 else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT,
355 tun_key->tun_flags))
356 sum += nla_total_size(tun_opts_len);
357 }
358
359 return sum;
360 }
361 #endif
362
363 void psample_sample_packet(struct psample_group *group,
364 const struct sk_buff *skb, u32 sample_rate,
365 const struct psample_metadata *md)
366 {
367 ktime_t tstamp = ktime_get_real();
368 int out_ifindex = md->out_ifindex;
369 int in_ifindex = md->in_ifindex;
370 u32 trunc_size = md->trunc_size;
371 #ifdef CONFIG_INET
372 struct ip_tunnel_info *tun_info;
373 #endif
374 struct sk_buff *nl_skb;
375 int data_len;
376 int meta_len;
377 void *data;
378 int ret;
379
380 if (!genl_has_listeners(&psample_nl_family, group->net,
381 PSAMPLE_NL_MCGRP_SAMPLE))
382 return;
383
384 meta_len = (in_ifindex ? nla_total_size(sizeof(u16)) : 0) +
385 (out_ifindex ? nla_total_size(sizeof(u16)) : 0) +
386 (md->out_tc_valid ? nla_total_size(sizeof(u16)) : 0) +
387 (md->out_tc_occ_valid ? nla_total_size_64bit(sizeof(u64)) : 0) +
388 (md->latency_valid ? nla_total_size_64bit(sizeof(u64)) : 0) +
389 nla_total_size(sizeof(u32)) + /* sample_rate */
390 nla_total_size(sizeof(u32)) + /* orig_size */
391 nla_total_size(sizeof(u32)) + /* group_num */
392 nla_total_size(sizeof(u32)) + /* seq */
393 nla_total_size_64bit(sizeof(u64)) + /* timestamp */
394 nla_total_size(sizeof(u16)) + /* protocol */
395 (md->user_cookie_len ?
396 nla_total_size(md->user_cookie_len) : 0); /* user cookie */
397
398 #ifdef CONFIG_INET
399 tun_info = skb_tunnel_info(skb);
400 if (tun_info)
401 meta_len += psample_tunnel_meta_len(tun_info);
402 #endif
403
404 data_len = min(skb->len, trunc_size);
405 if (meta_len + nla_total_size(data_len) > PSAMPLE_MAX_PACKET_SIZE)
406 data_len = PSAMPLE_MAX_PACKET_SIZE - meta_len - NLA_HDRLEN
407 - NLA_ALIGNTO;
408
409 nl_skb = genlmsg_new(meta_len + nla_total_size(data_len), GFP_ATOMIC);
410 if (unlikely(!nl_skb))
411 return;
412
413 data = genlmsg_put(nl_skb, 0, 0, &psample_nl_family, 0,
414 PSAMPLE_CMD_SAMPLE);
415 if (unlikely(!data))
416 goto error;
417
418 if (in_ifindex) {
419 ret = nla_put_u16(nl_skb, PSAMPLE_ATTR_IIFINDEX, in_ifindex);
420 if (unlikely(ret < 0))
421 goto error;
422 }
423
424 if (out_ifindex) {
425 ret = nla_put_u16(nl_skb, PSAMPLE_ATTR_OIFINDEX, out_ifindex);
426 if (unlikely(ret < 0))
427 goto error;
428 }
429
430 ret = nla_put_u32(nl_skb, PSAMPLE_ATTR_SAMPLE_RATE, sample_rate);
431 if (unlikely(ret < 0))
432 goto error;
433
434 ret = nla_put_u32(nl_skb, PSAMPLE_ATTR_ORIGSIZE, skb->len);
435 if (unlikely(ret < 0))
436 goto error;
437
438 ret = nla_put_u32(nl_skb, PSAMPLE_ATTR_SAMPLE_GROUP, group->group_num);
439 if (unlikely(ret < 0))
440 goto error;
441
442 ret = nla_put_u32(nl_skb, PSAMPLE_ATTR_GROUP_SEQ, group->seq++);
443 if (unlikely(ret < 0))
444 goto error;
445
446 if (md->out_tc_valid) {
447 ret = nla_put_u16(nl_skb, PSAMPLE_ATTR_OUT_TC, md->out_tc);
448 if (unlikely(ret < 0))
449 goto error;
450 }
451
452 if (md->out_tc_occ_valid) {
453 ret = nla_put_u64_64bit(nl_skb, PSAMPLE_ATTR_OUT_TC_OCC,
454 md->out_tc_occ, PSAMPLE_ATTR_PAD);
455 if (unlikely(ret < 0))
456 goto error;
457 }
458
459 if (md->latency_valid) {
460 ret = nla_put_u64_64bit(nl_skb, PSAMPLE_ATTR_LATENCY,
461 md->latency, PSAMPLE_ATTR_PAD);
462 if (unlikely(ret < 0))
463 goto error;
464 }
465
466 ret = nla_put_u64_64bit(nl_skb, PSAMPLE_ATTR_TIMESTAMP,
467 ktime_to_ns(tstamp), PSAMPLE_ATTR_PAD);
468 if (unlikely(ret < 0))
469 goto error;
470
471 ret = nla_put_u16(nl_skb, PSAMPLE_ATTR_PROTO,
472 be16_to_cpu(skb->protocol));
473 if (unlikely(ret < 0))
474 goto error;
475
476 if (data_len) {
477 int nla_len = nla_total_size(data_len);
478 struct nlattr *nla;
479
480 nla = skb_put(nl_skb, nla_len);
481 nla->nla_type = PSAMPLE_ATTR_DATA;
482 nla->nla_len = nla_attr_size(data_len);
483
484 if (skb_copy_bits(skb, 0, nla_data(nla), data_len))
485 goto error;
486 }
487
488 #ifdef CONFIG_INET
489 if (tun_info) {
490 ret = psample_ip_tun_to_nlattr(nl_skb, tun_info);
491 if (unlikely(ret < 0))
492 goto error;
493 }
494 #endif
495
496 if (md->user_cookie && md->user_cookie_len &&
497 nla_put(nl_skb, PSAMPLE_ATTR_USER_COOKIE, md->user_cookie_len,
498 md->user_cookie))
499 goto error;
500
501 if (md->rate_as_probability)
502 nla_put_flag(nl_skb, PSAMPLE_ATTR_SAMPLE_PROBABILITY);
503
504 genlmsg_end(nl_skb, data);
505 genlmsg_multicast_netns(&psample_nl_family, group->net, nl_skb, 0,
506 PSAMPLE_NL_MCGRP_SAMPLE, GFP_ATOMIC);
507
508 return;
509 error:
510 pr_err_ratelimited("Could not create psample log message\n");
511 nlmsg_free(nl_skb);
512 }
513 EXPORT_SYMBOL_GPL(psample_sample_packet);
514
515 static int __init psample_module_init(void)
516 {
517 return genl_register_family(&psample_nl_family);
518 }
519
520 static void __exit psample_module_exit(void)
521 {
522 genl_unregister_family(&psample_nl_family);
523 }
524
525 module_init(psample_module_init);
526 module_exit(psample_module_exit);
527
528 MODULE_AUTHOR("Yotam Gigi <yotam.gi@gmail.com>");
529 MODULE_DESCRIPTION("netlink channel for packet sampling");
530 MODULE_LICENSE("GPL v2");
531
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.