1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * xfrm_state.c
4 *
5 * Changes:
6 * Mitsuru KANDA @USAGI
7 * Kazunori MIYAZAWA @USAGI
8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9 * IPv6 support
10 * YOSHIFUJI Hideaki @USAGI
11 * Split up af-specific functions
12 * Derek Atkins <derek@ihtfp.com>
13 * Add UDP Encapsulation
14 *
15 */
16
17 #include <linux/compat.h>
18 #include <linux/workqueue.h>
19 #include <net/xfrm.h>
20 #include <linux/pfkeyv2.h>
21 #include <linux/ipsec.h>
22 #include <linux/module.h>
23 #include <linux/cache.h>
24 #include <linux/audit.h>
25 #include <linux/uaccess.h>
26 #include <linux/ktime.h>
27 #include <linux/slab.h>
28 #include <linux/interrupt.h>
29 #include <linux/kernel.h>
30
31 #include <crypto/aead.h>
32
33 #include "xfrm_hash.h"
34
35 #define xfrm_state_deref_prot(table, net) \
36 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37
38 static void xfrm_state_gc_task(struct work_struct *work);
39
40 /* Each xfrm_state may be linked to two tables:
41
42 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
43 2. Hash table by (daddr,family,reqid) to find what SAs exist for given
44 destination/tunnel endpoint. (output)
45 */
46
47 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
48 static struct kmem_cache *xfrm_state_cache __ro_after_init;
49
50 static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task);
51 static HLIST_HEAD(xfrm_state_gc_list);
52 static HLIST_HEAD(xfrm_state_dev_gc_list);
53
54 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)
55 {
56 return refcount_inc_not_zero(&x->refcnt);
57 }
58
59 static inline unsigned int xfrm_dst_hash(struct net *net,
60 const xfrm_address_t *daddr,
61 const xfrm_address_t *saddr,
62 u32 reqid,
63 unsigned short family)
64 {
65 return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);
66 }
67
68 static inline unsigned int xfrm_src_hash(struct net *net,
69 const xfrm_address_t *daddr,
70 const xfrm_address_t *saddr,
71 unsigned short family)
72 {
73 return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);
74 }
75
76 static inline unsigned int
77 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,
78 __be32 spi, u8 proto, unsigned short family)
79 {
80 return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);
81 }
82
83 static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
84 {
85 return __xfrm_seq_hash(seq, net->xfrm.state_hmask);
86 }
87
88 #define XFRM_STATE_INSERT(by, _n, _h, _type) \
89 { \
90 struct xfrm_state *_x = NULL; \
91 \
92 if (_type != XFRM_DEV_OFFLOAD_PACKET) { \
93 hlist_for_each_entry_rcu(_x, _h, by) { \
94 if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
95 continue; \
96 break; \
97 } \
98 } \
99 \
100 if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
101 /* SAD is empty or consist from HW SAs only */ \
102 hlist_add_head_rcu(_n, _h); \
103 else \
104 hlist_add_before_rcu(_n, &_x->by); \
105 }
106
107 static void xfrm_hash_transfer(struct hlist_head *list,
108 struct hlist_head *ndsttable,
109 struct hlist_head *nsrctable,
110 struct hlist_head *nspitable,
111 struct hlist_head *nseqtable,
112 unsigned int nhashmask)
113 {
114 struct hlist_node *tmp;
115 struct xfrm_state *x;
116
117 hlist_for_each_entry_safe(x, tmp, list, bydst) {
118 unsigned int h;
119
120 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
121 x->props.reqid, x->props.family,
122 nhashmask);
123 XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);
124
125 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,
126 x->props.family,
127 nhashmask);
128 XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);
129
130 if (x->id.spi) {
131 h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,
132 x->id.proto, x->props.family,
133 nhashmask);
134 XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,
135 x->xso.type);
136 }
137
138 if (x->km.seq) {
139 h = __xfrm_seq_hash(x->km.seq, nhashmask);
140 XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
141 x->xso.type);
142 }
143 }
144 }
145
146 static unsigned long xfrm_hash_new_size(unsigned int state_hmask)
147 {
148 return ((state_hmask + 1) << 1) * sizeof(struct hlist_head);
149 }
150
151 static void xfrm_hash_resize(struct work_struct *work)
152 {
153 struct net *net = container_of(work, struct net, xfrm.state_hash_work);
154 struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq;
155 unsigned long nsize, osize;
156 unsigned int nhashmask, ohashmask;
157 int i;
158
159 nsize = xfrm_hash_new_size(net->xfrm.state_hmask);
160 ndst = xfrm_hash_alloc(nsize);
161 if (!ndst)
162 return;
163 nsrc = xfrm_hash_alloc(nsize);
164 if (!nsrc) {
165 xfrm_hash_free(ndst, nsize);
166 return;
167 }
168 nspi = xfrm_hash_alloc(nsize);
169 if (!nspi) {
170 xfrm_hash_free(ndst, nsize);
171 xfrm_hash_free(nsrc, nsize);
172 return;
173 }
174 nseq = xfrm_hash_alloc(nsize);
175 if (!nseq) {
176 xfrm_hash_free(ndst, nsize);
177 xfrm_hash_free(nsrc, nsize);
178 xfrm_hash_free(nspi, nsize);
179 return;
180 }
181
182 spin_lock_bh(&net->xfrm.xfrm_state_lock);
183 write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
184
185 nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
186 odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
187 for (i = net->xfrm.state_hmask; i >= 0; i--)
188 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask);
189
190 osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
191 ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
192 oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);
193 ohashmask = net->xfrm.state_hmask;
194
195 rcu_assign_pointer(net->xfrm.state_bydst, ndst);
196 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
197 rcu_assign_pointer(net->xfrm.state_byspi, nspi);
198 rcu_assign_pointer(net->xfrm.state_byseq, nseq);
199 net->xfrm.state_hmask = nhashmask;
200
201 write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
202 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
203
204 osize = (ohashmask + 1) * sizeof(struct hlist_head);
205
206 synchronize_rcu();
207
208 xfrm_hash_free(odst, osize);
209 xfrm_hash_free(osrc, osize);
210 xfrm_hash_free(ospi, osize);
211 xfrm_hash_free(oseq, osize);
212 }
213
214 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
215 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO];
216
217 static DEFINE_SPINLOCK(xfrm_state_gc_lock);
218 static DEFINE_SPINLOCK(xfrm_state_dev_gc_lock);
219
220 int __xfrm_state_delete(struct xfrm_state *x);
221
222 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
223 static bool km_is_alive(const struct km_event *c);
224 void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
225
226 int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
227 {
228 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
229 int err = 0;
230
231 if (!afinfo)
232 return -EAFNOSUPPORT;
233
234 #define X(afi, T, name) do { \
235 WARN_ON((afi)->type_ ## name); \
236 (afi)->type_ ## name = (T); \
237 } while (0)
238
239 switch (type->proto) {
240 case IPPROTO_COMP:
241 X(afinfo, type, comp);
242 break;
243 case IPPROTO_AH:
244 X(afinfo, type, ah);
245 break;
246 case IPPROTO_ESP:
247 X(afinfo, type, esp);
248 break;
249 case IPPROTO_IPIP:
250 X(afinfo, type, ipip);
251 break;
252 case IPPROTO_DSTOPTS:
253 X(afinfo, type, dstopts);
254 break;
255 case IPPROTO_ROUTING:
256 X(afinfo, type, routing);
257 break;
258 case IPPROTO_IPV6:
259 X(afinfo, type, ipip6);
260 break;
261 default:
262 WARN_ON(1);
263 err = -EPROTONOSUPPORT;
264 break;
265 }
266 #undef X
267 rcu_read_unlock();
268 return err;
269 }
270 EXPORT_SYMBOL(xfrm_register_type);
271
272 void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
273 {
274 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
275
276 if (unlikely(afinfo == NULL))
277 return;
278
279 #define X(afi, T, name) do { \
280 WARN_ON((afi)->type_ ## name != (T)); \
281 (afi)->type_ ## name = NULL; \
282 } while (0)
283
284 switch (type->proto) {
285 case IPPROTO_COMP:
286 X(afinfo, type, comp);
287 break;
288 case IPPROTO_AH:
289 X(afinfo, type, ah);
290 break;
291 case IPPROTO_ESP:
292 X(afinfo, type, esp);
293 break;
294 case IPPROTO_IPIP:
295 X(afinfo, type, ipip);
296 break;
297 case IPPROTO_DSTOPTS:
298 X(afinfo, type, dstopts);
299 break;
300 case IPPROTO_ROUTING:
301 X(afinfo, type, routing);
302 break;
303 case IPPROTO_IPV6:
304 X(afinfo, type, ipip6);
305 break;
306 default:
307 WARN_ON(1);
308 break;
309 }
310 #undef X
311 rcu_read_unlock();
312 }
313 EXPORT_SYMBOL(xfrm_unregister_type);
314
315 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
316 {
317 const struct xfrm_type *type = NULL;
318 struct xfrm_state_afinfo *afinfo;
319 int modload_attempted = 0;
320
321 retry:
322 afinfo = xfrm_state_get_afinfo(family);
323 if (unlikely(afinfo == NULL))
324 return NULL;
325
326 switch (proto) {
327 case IPPROTO_COMP:
328 type = afinfo->type_comp;
329 break;
330 case IPPROTO_AH:
331 type = afinfo->type_ah;
332 break;
333 case IPPROTO_ESP:
334 type = afinfo->type_esp;
335 break;
336 case IPPROTO_IPIP:
337 type = afinfo->type_ipip;
338 break;
339 case IPPROTO_DSTOPTS:
340 type = afinfo->type_dstopts;
341 break;
342 case IPPROTO_ROUTING:
343 type = afinfo->type_routing;
344 break;
345 case IPPROTO_IPV6:
346 type = afinfo->type_ipip6;
347 break;
348 default:
349 break;
350 }
351
352 if (unlikely(type && !try_module_get(type->owner)))
353 type = NULL;
354
355 rcu_read_unlock();
356
357 if (!type && !modload_attempted) {
358 request_module("xfrm-type-%d-%d", family, proto);
359 modload_attempted = 1;
360 goto retry;
361 }
362
363 return type;
364 }
365
366 static void xfrm_put_type(const struct xfrm_type *type)
367 {
368 module_put(type->owner);
369 }
370
371 int xfrm_register_type_offload(const struct xfrm_type_offload *type,
372 unsigned short family)
373 {
374 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
375 int err = 0;
376
377 if (unlikely(afinfo == NULL))
378 return -EAFNOSUPPORT;
379
380 switch (type->proto) {
381 case IPPROTO_ESP:
382 WARN_ON(afinfo->type_offload_esp);
383 afinfo->type_offload_esp = type;
384 break;
385 default:
386 WARN_ON(1);
387 err = -EPROTONOSUPPORT;
388 break;
389 }
390
391 rcu_read_unlock();
392 return err;
393 }
394 EXPORT_SYMBOL(xfrm_register_type_offload);
395
396 void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
397 unsigned short family)
398 {
399 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
400
401 if (unlikely(afinfo == NULL))
402 return;
403
404 switch (type->proto) {
405 case IPPROTO_ESP:
406 WARN_ON(afinfo->type_offload_esp != type);
407 afinfo->type_offload_esp = NULL;
408 break;
409 default:
410 WARN_ON(1);
411 break;
412 }
413 rcu_read_unlock();
414 }
415 EXPORT_SYMBOL(xfrm_unregister_type_offload);
416
417 static const struct xfrm_type_offload *
418 xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load)
419 {
420 const struct xfrm_type_offload *type = NULL;
421 struct xfrm_state_afinfo *afinfo;
422
423 retry:
424 afinfo = xfrm_state_get_afinfo(family);
425 if (unlikely(afinfo == NULL))
426 return NULL;
427
428 switch (proto) {
429 case IPPROTO_ESP:
430 type = afinfo->type_offload_esp;
431 break;
432 default:
433 break;
434 }
435
436 if ((type && !try_module_get(type->owner)))
437 type = NULL;
438
439 rcu_read_unlock();
440
441 if (!type && try_load) {
442 request_module("xfrm-offload-%d-%d", family, proto);
443 try_load = false;
444 goto retry;
445 }
446
447 return type;
448 }
449
450 static void xfrm_put_type_offload(const struct xfrm_type_offload *type)
451 {
452 module_put(type->owner);
453 }
454
455 static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
456 [XFRM_MODE_BEET] = {
457 .encap = XFRM_MODE_BEET,
458 .flags = XFRM_MODE_FLAG_TUNNEL,
459 .family = AF_INET,
460 },
461 [XFRM_MODE_TRANSPORT] = {
462 .encap = XFRM_MODE_TRANSPORT,
463 .family = AF_INET,
464 },
465 [XFRM_MODE_TUNNEL] = {
466 .encap = XFRM_MODE_TUNNEL,
467 .flags = XFRM_MODE_FLAG_TUNNEL,
468 .family = AF_INET,
469 },
470 };
471
472 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = {
473 [XFRM_MODE_BEET] = {
474 .encap = XFRM_MODE_BEET,
475 .flags = XFRM_MODE_FLAG_TUNNEL,
476 .family = AF_INET6,
477 },
478 [XFRM_MODE_ROUTEOPTIMIZATION] = {
479 .encap = XFRM_MODE_ROUTEOPTIMIZATION,
480 .family = AF_INET6,
481 },
482 [XFRM_MODE_TRANSPORT] = {
483 .encap = XFRM_MODE_TRANSPORT,
484 .family = AF_INET6,
485 },
486 [XFRM_MODE_TUNNEL] = {
487 .encap = XFRM_MODE_TUNNEL,
488 .flags = XFRM_MODE_FLAG_TUNNEL,
489 .family = AF_INET6,
490 },
491 };
492
493 static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
494 {
495 const struct xfrm_mode *mode;
496
497 if (unlikely(encap >= XFRM_MODE_MAX))
498 return NULL;
499
500 switch (family) {
501 case AF_INET:
502 mode = &xfrm4_mode_map[encap];
503 if (mode->family == family)
504 return mode;
505 break;
506 case AF_INET6:
507 mode = &xfrm6_mode_map[encap];
508 if (mode->family == family)
509 return mode;
510 break;
511 default:
512 break;
513 }
514
515 return NULL;
516 }
517
518 void xfrm_state_free(struct xfrm_state *x)
519 {
520 kmem_cache_free(xfrm_state_cache, x);
521 }
522 EXPORT_SYMBOL(xfrm_state_free);
523
524 static void ___xfrm_state_destroy(struct xfrm_state *x)
525 {
526 hrtimer_cancel(&x->mtimer);
527 del_timer_sync(&x->rtimer);
528 kfree(x->aead);
529 kfree(x->aalg);
530 kfree(x->ealg);
531 kfree(x->calg);
532 kfree(x->encap);
533 kfree(x->coaddr);
534 kfree(x->replay_esn);
535 kfree(x->preplay_esn);
536 if (x->type_offload)
537 xfrm_put_type_offload(x->type_offload);
538 if (x->type) {
539 x->type->destructor(x);
540 xfrm_put_type(x->type);
541 }
542 if (x->xfrag.page)
543 put_page(x->xfrag.page);
544 xfrm_dev_state_free(x);
545 security_xfrm_state_free(x);
546 xfrm_state_free(x);
547 }
548
549 static void xfrm_state_gc_task(struct work_struct *work)
550 {
551 struct xfrm_state *x;
552 struct hlist_node *tmp;
553 struct hlist_head gc_list;
554
555 spin_lock_bh(&xfrm_state_gc_lock);
556 hlist_move_list(&xfrm_state_gc_list, &gc_list);
557 spin_unlock_bh(&xfrm_state_gc_lock);
558
559 synchronize_rcu();
560
561 hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
562 ___xfrm_state_destroy(x);
563 }
564
565 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
566 {
567 struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);
568 enum hrtimer_restart ret = HRTIMER_NORESTART;
569 time64_t now = ktime_get_real_seconds();
570 time64_t next = TIME64_MAX;
571 int warn = 0;
572 int err = 0;
573
574 spin_lock(&x->lock);
575 xfrm_dev_state_update_stats(x);
576
577 if (x->km.state == XFRM_STATE_DEAD)
578 goto out;
579 if (x->km.state == XFRM_STATE_EXPIRED)
580 goto expired;
581 if (x->lft.hard_add_expires_seconds) {
582 time64_t tmo = x->lft.hard_add_expires_seconds +
583 x->curlft.add_time - now;
584 if (tmo <= 0) {
585 if (x->xflags & XFRM_SOFT_EXPIRE) {
586 /* enter hard expire without soft expire first?!
587 * setting a new date could trigger this.
588 * workaround: fix x->curflt.add_time by below:
589 */
590 x->curlft.add_time = now - x->saved_tmo - 1;
591 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;
592 } else
593 goto expired;
594 }
595 if (tmo < next)
596 next = tmo;
597 }
598 if (x->lft.hard_use_expires_seconds) {
599 time64_t tmo = x->lft.hard_use_expires_seconds +
600 (READ_ONCE(x->curlft.use_time) ? : now) - now;
601 if (tmo <= 0)
602 goto expired;
603 if (tmo < next)
604 next = tmo;
605 }
606 if (x->km.dying)
607 goto resched;
608 if (x->lft.soft_add_expires_seconds) {
609 time64_t tmo = x->lft.soft_add_expires_seconds +
610 x->curlft.add_time - now;
611 if (tmo <= 0) {
612 warn = 1;
613 x->xflags &= ~XFRM_SOFT_EXPIRE;
614 } else if (tmo < next) {
615 next = tmo;
616 x->xflags |= XFRM_SOFT_EXPIRE;
617 x->saved_tmo = tmo;
618 }
619 }
620 if (x->lft.soft_use_expires_seconds) {
621 time64_t tmo = x->lft.soft_use_expires_seconds +
622 (READ_ONCE(x->curlft.use_time) ? : now) - now;
623 if (tmo <= 0)
624 warn = 1;
625 else if (tmo < next)
626 next = tmo;
627 }
628
629 x->km.dying = warn;
630 if (warn)
631 km_state_expired(x, 0, 0);
632 resched:
633 if (next != TIME64_MAX) {
634 hrtimer_forward_now(&x->mtimer, ktime_set(next, 0));
635 ret = HRTIMER_RESTART;
636 }
637
638 goto out;
639
640 expired:
641 if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)
642 x->km.state = XFRM_STATE_EXPIRED;
643
644 err = __xfrm_state_delete(x);
645 if (!err)
646 km_state_expired(x, 1, 0);
647
648 xfrm_audit_state_delete(x, err ? 0 : 1, true);
649
650 out:
651 spin_unlock(&x->lock);
652 return ret;
653 }
654
655 static void xfrm_replay_timer_handler(struct timer_list *t);
656
657 struct xfrm_state *xfrm_state_alloc(struct net *net)
658 {
659 struct xfrm_state *x;
660
661 x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC);
662
663 if (x) {
664 write_pnet(&x->xs_net, net);
665 refcount_set(&x->refcnt, 1);
666 atomic_set(&x->tunnel_users, 0);
667 INIT_LIST_HEAD(&x->km.all);
668 INIT_HLIST_NODE(&x->bydst);
669 INIT_HLIST_NODE(&x->bysrc);
670 INIT_HLIST_NODE(&x->byspi);
671 INIT_HLIST_NODE(&x->byseq);
672 hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
673 x->mtimer.function = xfrm_timer_handler;
674 timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
675 x->curlft.add_time = ktime_get_real_seconds();
676 x->lft.soft_byte_limit = XFRM_INF;
677 x->lft.soft_packet_limit = XFRM_INF;
678 x->lft.hard_byte_limit = XFRM_INF;
679 x->lft.hard_packet_limit = XFRM_INF;
680 x->replay_maxage = 0;
681 x->replay_maxdiff = 0;
682 spin_lock_init(&x->lock);
683 }
684 return x;
685 }
686 EXPORT_SYMBOL(xfrm_state_alloc);
687
688 #ifdef CONFIG_XFRM_OFFLOAD
689 void xfrm_dev_state_delete(struct xfrm_state *x)
690 {
691 struct xfrm_dev_offload *xso = &x->xso;
692 struct net_device *dev = READ_ONCE(xso->dev);
693
694 if (dev) {
695 dev->xfrmdev_ops->xdo_dev_state_delete(x);
696 spin_lock_bh(&xfrm_state_dev_gc_lock);
697 hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list);
698 spin_unlock_bh(&xfrm_state_dev_gc_lock);
699 }
700 }
701 EXPORT_SYMBOL_GPL(xfrm_dev_state_delete);
702
703 void xfrm_dev_state_free(struct xfrm_state *x)
704 {
705 struct xfrm_dev_offload *xso = &x->xso;
706 struct net_device *dev = READ_ONCE(xso->dev);
707
708 if (dev && dev->xfrmdev_ops) {
709 spin_lock_bh(&xfrm_state_dev_gc_lock);
710 if (!hlist_unhashed(&x->dev_gclist))
711 hlist_del(&x->dev_gclist);
712 spin_unlock_bh(&xfrm_state_dev_gc_lock);
713
714 if (dev->xfrmdev_ops->xdo_dev_state_free)
715 dev->xfrmdev_ops->xdo_dev_state_free(x);
716 WRITE_ONCE(xso->dev, NULL);
717 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
718 netdev_put(dev, &xso->dev_tracker);
719 }
720 }
721 #endif
722
723 void __xfrm_state_destroy(struct xfrm_state *x, bool sync)
724 {
725 WARN_ON(x->km.state != XFRM_STATE_DEAD);
726
727 if (sync) {
728 synchronize_rcu();
729 ___xfrm_state_destroy(x);
730 } else {
731 spin_lock_bh(&xfrm_state_gc_lock);
732 hlist_add_head(&x->gclist, &xfrm_state_gc_list);
733 spin_unlock_bh(&xfrm_state_gc_lock);
734 schedule_work(&xfrm_state_gc_work);
735 }
736 }
737 EXPORT_SYMBOL(__xfrm_state_destroy);
738
739 int __xfrm_state_delete(struct xfrm_state *x)
740 {
741 struct net *net = xs_net(x);
742 int err = -ESRCH;
743
744 if (x->km.state != XFRM_STATE_DEAD) {
745 x->km.state = XFRM_STATE_DEAD;
746 spin_lock(&net->xfrm.xfrm_state_lock);
747 list_del(&x->km.all);
748 hlist_del_rcu(&x->bydst);
749 hlist_del_rcu(&x->bysrc);
750 if (x->km.seq)
751 hlist_del_rcu(&x->byseq);
752 if (x->id.spi)
753 hlist_del_rcu(&x->byspi);
754 net->xfrm.state_num--;
755 xfrm_nat_keepalive_state_updated(x);
756 spin_unlock(&net->xfrm.xfrm_state_lock);
757
758 if (x->encap_sk)
759 sock_put(rcu_dereference_raw(x->encap_sk));
760
761 xfrm_dev_state_delete(x);
762
763 /* All xfrm_state objects are created by xfrm_state_alloc.
764 * The xfrm_state_alloc call gives a reference, and that
765 * is what we are dropping here.
766 */
767 xfrm_state_put(x);
768 err = 0;
769 }
770
771 return err;
772 }
773 EXPORT_SYMBOL(__xfrm_state_delete);
774
775 int xfrm_state_delete(struct xfrm_state *x)
776 {
777 int err;
778
779 spin_lock_bh(&x->lock);
780 err = __xfrm_state_delete(x);
781 spin_unlock_bh(&x->lock);
782
783 return err;
784 }
785 EXPORT_SYMBOL(xfrm_state_delete);
786
787 #ifdef CONFIG_SECURITY_NETWORK_XFRM
788 static inline int
789 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
790 {
791 int i, err = 0;
792
793 for (i = 0; i <= net->xfrm.state_hmask; i++) {
794 struct xfrm_state *x;
795
796 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
797 if (xfrm_id_proto_match(x->id.proto, proto) &&
798 (err = security_xfrm_state_delete(x)) != 0) {
799 xfrm_audit_state_delete(x, 0, task_valid);
800 return err;
801 }
802 }
803 }
804
805 return err;
806 }
807
808 static inline int
809 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
810 {
811 int i, err = 0;
812
813 for (i = 0; i <= net->xfrm.state_hmask; i++) {
814 struct xfrm_state *x;
815 struct xfrm_dev_offload *xso;
816
817 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
818 xso = &x->xso;
819
820 if (xso->dev == dev &&
821 (err = security_xfrm_state_delete(x)) != 0) {
822 xfrm_audit_state_delete(x, 0, task_valid);
823 return err;
824 }
825 }
826 }
827
828 return err;
829 }
830 #else
831 static inline int
832 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
833 {
834 return 0;
835 }
836
837 static inline int
838 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
839 {
840 return 0;
841 }
842 #endif
843
844 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync)
845 {
846 int i, err = 0, cnt = 0;
847
848 spin_lock_bh(&net->xfrm.xfrm_state_lock);
849 err = xfrm_state_flush_secctx_check(net, proto, task_valid);
850 if (err)
851 goto out;
852
853 err = -ESRCH;
854 for (i = 0; i <= net->xfrm.state_hmask; i++) {
855 struct xfrm_state *x;
856 restart:
857 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
858 if (!xfrm_state_kern(x) &&
859 xfrm_id_proto_match(x->id.proto, proto)) {
860 xfrm_state_hold(x);
861 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
862
863 err = xfrm_state_delete(x);
864 xfrm_audit_state_delete(x, err ? 0 : 1,
865 task_valid);
866 if (sync)
867 xfrm_state_put_sync(x);
868 else
869 xfrm_state_put(x);
870 if (!err)
871 cnt++;
872
873 spin_lock_bh(&net->xfrm.xfrm_state_lock);
874 goto restart;
875 }
876 }
877 }
878 out:
879 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
880 if (cnt)
881 err = 0;
882
883 return err;
884 }
885 EXPORT_SYMBOL(xfrm_state_flush);
886
887 int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid)
888 {
889 struct xfrm_state *x;
890 struct hlist_node *tmp;
891 struct xfrm_dev_offload *xso;
892 int i, err = 0, cnt = 0;
893
894 spin_lock_bh(&net->xfrm.xfrm_state_lock);
895 err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid);
896 if (err)
897 goto out;
898
899 err = -ESRCH;
900 for (i = 0; i <= net->xfrm.state_hmask; i++) {
901 restart:
902 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
903 xso = &x->xso;
904
905 if (!xfrm_state_kern(x) && xso->dev == dev) {
906 xfrm_state_hold(x);
907 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
908
909 err = xfrm_state_delete(x);
910 xfrm_dev_state_free(x);
911
912 xfrm_audit_state_delete(x, err ? 0 : 1,
913 task_valid);
914 xfrm_state_put(x);
915 if (!err)
916 cnt++;
917
918 spin_lock_bh(&net->xfrm.xfrm_state_lock);
919 goto restart;
920 }
921 }
922 }
923 if (cnt)
924 err = 0;
925
926 out:
927 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
928
929 spin_lock_bh(&xfrm_state_dev_gc_lock);
930 restart_gc:
931 hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) {
932 xso = &x->xso;
933
934 if (xso->dev == dev) {
935 spin_unlock_bh(&xfrm_state_dev_gc_lock);
936 xfrm_dev_state_free(x);
937 spin_lock_bh(&xfrm_state_dev_gc_lock);
938 goto restart_gc;
939 }
940
941 }
942 spin_unlock_bh(&xfrm_state_dev_gc_lock);
943
944 xfrm_flush_gc();
945
946 return err;
947 }
948 EXPORT_SYMBOL(xfrm_dev_state_flush);
949
950 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
951 {
952 spin_lock_bh(&net->xfrm.xfrm_state_lock);
953 si->sadcnt = net->xfrm.state_num;
954 si->sadhcnt = net->xfrm.state_hmask + 1;
955 si->sadhmcnt = xfrm_state_hashmax;
956 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
957 }
958 EXPORT_SYMBOL(xfrm_sad_getinfo);
959
960 static void
961 __xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
962 {
963 const struct flowi4 *fl4 = &fl->u.ip4;
964
965 sel->daddr.a4 = fl4->daddr;
966 sel->saddr.a4 = fl4->saddr;
967 sel->dport = xfrm_flowi_dport(fl, &fl4->uli);
968 sel->dport_mask = htons(0xffff);
969 sel->sport = xfrm_flowi_sport(fl, &fl4->uli);
970 sel->sport_mask = htons(0xffff);
971 sel->family = AF_INET;
972 sel->prefixlen_d = 32;
973 sel->prefixlen_s = 32;
974 sel->proto = fl4->flowi4_proto;
975 sel->ifindex = fl4->flowi4_oif;
976 }
977
978 static void
979 __xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
980 {
981 const struct flowi6 *fl6 = &fl->u.ip6;
982
983 /* Initialize temporary selector matching only to current session. */
984 *(struct in6_addr *)&sel->daddr = fl6->daddr;
985 *(struct in6_addr *)&sel->saddr = fl6->saddr;
986 sel->dport = xfrm_flowi_dport(fl, &fl6->uli);
987 sel->dport_mask = htons(0xffff);
988 sel->sport = xfrm_flowi_sport(fl, &fl6->uli);
989 sel->sport_mask = htons(0xffff);
990 sel->family = AF_INET6;
991 sel->prefixlen_d = 128;
992 sel->prefixlen_s = 128;
993 sel->proto = fl6->flowi6_proto;
994 sel->ifindex = fl6->flowi6_oif;
995 }
996
997 static void
998 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
999 const struct xfrm_tmpl *tmpl,
1000 const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1001 unsigned short family)
1002 {
1003 switch (family) {
1004 case AF_INET:
1005 __xfrm4_init_tempsel(&x->sel, fl);
1006 break;
1007 case AF_INET6:
1008 __xfrm6_init_tempsel(&x->sel, fl);
1009 break;
1010 }
1011
1012 x->id = tmpl->id;
1013
1014 switch (tmpl->encap_family) {
1015 case AF_INET:
1016 if (x->id.daddr.a4 == 0)
1017 x->id.daddr.a4 = daddr->a4;
1018 x->props.saddr = tmpl->saddr;
1019 if (x->props.saddr.a4 == 0)
1020 x->props.saddr.a4 = saddr->a4;
1021 break;
1022 case AF_INET6:
1023 if (ipv6_addr_any((struct in6_addr *)&x->id.daddr))
1024 memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));
1025 memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));
1026 if (ipv6_addr_any((struct in6_addr *)&x->props.saddr))
1027 memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));
1028 break;
1029 }
1030
1031 x->props.mode = tmpl->mode;
1032 x->props.reqid = tmpl->reqid;
1033 x->props.family = tmpl->encap_family;
1034 }
1035
1036 static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark,
1037 const xfrm_address_t *daddr,
1038 __be32 spi, u8 proto,
1039 unsigned short family,
1040 struct xfrm_dev_offload *xdo)
1041 {
1042 unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
1043 struct xfrm_state *x;
1044
1045 hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
1046 #ifdef CONFIG_XFRM_OFFLOAD
1047 if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {
1048 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1049 /* HW states are in the head of list, there is
1050 * no need to iterate further.
1051 */
1052 break;
1053
1054 /* Packet offload: both policy and SA should
1055 * have same device.
1056 */
1057 if (xdo->dev != x->xso.dev)
1058 continue;
1059 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1060 /* Skip HW policy for SW lookups */
1061 continue;
1062 #endif
1063 if (x->props.family != family ||
1064 x->id.spi != spi ||
1065 x->id.proto != proto ||
1066 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1067 continue;
1068
1069 if ((mark & x->mark.m) != x->mark.v)
1070 continue;
1071 if (!xfrm_state_hold_rcu(x))
1072 continue;
1073 return x;
1074 }
1075
1076 return NULL;
1077 }
1078
1079 static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,
1080 const xfrm_address_t *daddr,
1081 __be32 spi, u8 proto,
1082 unsigned short family)
1083 {
1084 unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
1085 struct xfrm_state *x;
1086
1087 hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
1088 if (x->props.family != family ||
1089 x->id.spi != spi ||
1090 x->id.proto != proto ||
1091 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1092 continue;
1093
1094 if ((mark & x->mark.m) != x->mark.v)
1095 continue;
1096 if (!xfrm_state_hold_rcu(x))
1097 continue;
1098 return x;
1099 }
1100
1101 return NULL;
1102 }
1103
1104 static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark,
1105 const xfrm_address_t *daddr,
1106 const xfrm_address_t *saddr,
1107 u8 proto, unsigned short family)
1108 {
1109 unsigned int h = xfrm_src_hash(net, daddr, saddr, family);
1110 struct xfrm_state *x;
1111
1112 hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {
1113 if (x->props.family != family ||
1114 x->id.proto != proto ||
1115 !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
1116 !xfrm_addr_equal(&x->props.saddr, saddr, family))
1117 continue;
1118
1119 if ((mark & x->mark.m) != x->mark.v)
1120 continue;
1121 if (!xfrm_state_hold_rcu(x))
1122 continue;
1123 return x;
1124 }
1125
1126 return NULL;
1127 }
1128
1129 static inline struct xfrm_state *
1130 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)
1131 {
1132 struct net *net = xs_net(x);
1133 u32 mark = x->mark.v & x->mark.m;
1134
1135 if (use_spi)
1136 return __xfrm_state_lookup(net, mark, &x->id.daddr,
1137 x->id.spi, x->id.proto, family);
1138 else
1139 return __xfrm_state_lookup_byaddr(net, mark,
1140 &x->id.daddr,
1141 &x->props.saddr,
1142 x->id.proto, family);
1143 }
1144
1145 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
1146 {
1147 if (have_hash_collision &&
1148 (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&
1149 net->xfrm.state_num > net->xfrm.state_hmask)
1150 schedule_work(&net->xfrm.state_hash_work);
1151 }
1152
1153 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
1154 const struct flowi *fl, unsigned short family,
1155 struct xfrm_state **best, int *acq_in_progress,
1156 int *error)
1157 {
1158 /* Resolution logic:
1159 * 1. There is a valid state with matching selector. Done.
1160 * 2. Valid state with inappropriate selector. Skip.
1161 *
1162 * Entering area of "sysdeps".
1163 *
1164 * 3. If state is not valid, selector is temporary, it selects
1165 * only session which triggered previous resolution. Key
1166 * manager will do something to install a state with proper
1167 * selector.
1168 */
1169 if (x->km.state == XFRM_STATE_VALID) {
1170 if ((x->sel.family &&
1171 (x->sel.family != family ||
1172 !xfrm_selector_match(&x->sel, fl, family))) ||
1173 !security_xfrm_state_pol_flow_match(x, pol,
1174 &fl->u.__fl_common))
1175 return;
1176
1177 if (!*best ||
1178 (*best)->km.dying > x->km.dying ||
1179 ((*best)->km.dying == x->km.dying &&
1180 (*best)->curlft.add_time < x->curlft.add_time))
1181 *best = x;
1182 } else if (x->km.state == XFRM_STATE_ACQ) {
1183 *acq_in_progress = 1;
1184 } else if (x->km.state == XFRM_STATE_ERROR ||
1185 x->km.state == XFRM_STATE_EXPIRED) {
1186 if ((!x->sel.family ||
1187 (x->sel.family == family &&
1188 xfrm_selector_match(&x->sel, fl, family))) &&
1189 security_xfrm_state_pol_flow_match(x, pol,
1190 &fl->u.__fl_common))
1191 *error = -ESRCH;
1192 }
1193 }
1194
1195 struct xfrm_state *
1196 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1197 const struct flowi *fl, struct xfrm_tmpl *tmpl,
1198 struct xfrm_policy *pol, int *err,
1199 unsigned short family, u32 if_id)
1200 {
1201 static xfrm_address_t saddr_wildcard = { };
1202 struct net *net = xp_net(pol);
1203 unsigned int h, h_wildcard;
1204 struct xfrm_state *x, *x0, *to_put;
1205 int acquire_in_progress = 0;
1206 int error = 0;
1207 struct xfrm_state *best = NULL;
1208 u32 mark = pol->mark.v & pol->mark.m;
1209 unsigned short encap_family = tmpl->encap_family;
1210 unsigned int sequence;
1211 struct km_event c;
1212
1213 to_put = NULL;
1214
1215 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
1216
1217 rcu_read_lock();
1218 h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);
1219 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {
1220 #ifdef CONFIG_XFRM_OFFLOAD
1221 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1222 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1223 /* HW states are in the head of list, there is
1224 * no need to iterate further.
1225 */
1226 break;
1227
1228 /* Packet offload: both policy and SA should
1229 * have same device.
1230 */
1231 if (pol->xdo.dev != x->xso.dev)
1232 continue;
1233 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1234 /* Skip HW policy for SW lookups */
1235 continue;
1236 #endif
1237 if (x->props.family == encap_family &&
1238 x->props.reqid == tmpl->reqid &&
1239 (mark & x->mark.m) == x->mark.v &&
1240 x->if_id == if_id &&
1241 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1242 xfrm_state_addr_check(x, daddr, saddr, encap_family) &&
1243 tmpl->mode == x->props.mode &&
1244 tmpl->id.proto == x->id.proto &&
1245 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1246 xfrm_state_look_at(pol, x, fl, family,
1247 &best, &acquire_in_progress, &error);
1248 }
1249 if (best || acquire_in_progress)
1250 goto found;
1251
1252 h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family);
1253 hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {
1254 #ifdef CONFIG_XFRM_OFFLOAD
1255 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1256 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1257 /* HW states are in the head of list, there is
1258 * no need to iterate further.
1259 */
1260 break;
1261
1262 /* Packet offload: both policy and SA should
1263 * have same device.
1264 */
1265 if (pol->xdo.dev != x->xso.dev)
1266 continue;
1267 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1268 /* Skip HW policy for SW lookups */
1269 continue;
1270 #endif
1271 if (x->props.family == encap_family &&
1272 x->props.reqid == tmpl->reqid &&
1273 (mark & x->mark.m) == x->mark.v &&
1274 x->if_id == if_id &&
1275 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1276 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&
1277 tmpl->mode == x->props.mode &&
1278 tmpl->id.proto == x->id.proto &&
1279 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1280 xfrm_state_look_at(pol, x, fl, family,
1281 &best, &acquire_in_progress, &error);
1282 }
1283
1284 found:
1285 x = best;
1286 if (!x && !error && !acquire_in_progress) {
1287 if (tmpl->id.spi &&
1288 (x0 = __xfrm_state_lookup_all(net, mark, daddr,
1289 tmpl->id.spi, tmpl->id.proto,
1290 encap_family,
1291 &pol->xdo)) != NULL) {
1292 to_put = x0;
1293 error = -EEXIST;
1294 goto out;
1295 }
1296
1297 c.net = net;
1298 /* If the KMs have no listeners (yet...), avoid allocating an SA
1299 * for each and every packet - garbage collection might not
1300 * handle the flood.
1301 */
1302 if (!km_is_alive(&c)) {
1303 error = -ESRCH;
1304 goto out;
1305 }
1306
1307 x = xfrm_state_alloc(net);
1308 if (x == NULL) {
1309 error = -ENOMEM;
1310 goto out;
1311 }
1312 /* Initialize temporary state matching only
1313 * to current session. */
1314 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);
1315 memcpy(&x->mark, &pol->mark, sizeof(x->mark));
1316 x->if_id = if_id;
1317
1318 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid);
1319 if (error) {
1320 x->km.state = XFRM_STATE_DEAD;
1321 to_put = x;
1322 x = NULL;
1323 goto out;
1324 }
1325 #ifdef CONFIG_XFRM_OFFLOAD
1326 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1327 struct xfrm_dev_offload *xdo = &pol->xdo;
1328 struct xfrm_dev_offload *xso = &x->xso;
1329
1330 xso->type = XFRM_DEV_OFFLOAD_PACKET;
1331 xso->dir = xdo->dir;
1332 xso->dev = xdo->dev;
1333 xso->real_dev = xdo->real_dev;
1334 xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
1335 netdev_hold(xso->dev, &xso->dev_tracker, GFP_ATOMIC);
1336 error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL);
1337 if (error) {
1338 xso->dir = 0;
1339 netdev_put(xso->dev, &xso->dev_tracker);
1340 xso->dev = NULL;
1341 xso->real_dev = NULL;
1342 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
1343 x->km.state = XFRM_STATE_DEAD;
1344 to_put = x;
1345 x = NULL;
1346 goto out;
1347 }
1348 }
1349 #endif
1350 if (km_query(x, tmpl, pol) == 0) {
1351 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1352 x->km.state = XFRM_STATE_ACQ;
1353 x->dir = XFRM_SA_DIR_OUT;
1354 list_add(&x->km.all, &net->xfrm.state_all);
1355 XFRM_STATE_INSERT(bydst, &x->bydst,
1356 net->xfrm.state_bydst + h,
1357 x->xso.type);
1358 h = xfrm_src_hash(net, daddr, saddr, encap_family);
1359 XFRM_STATE_INSERT(bysrc, &x->bysrc,
1360 net->xfrm.state_bysrc + h,
1361 x->xso.type);
1362 if (x->id.spi) {
1363 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);
1364 XFRM_STATE_INSERT(byspi, &x->byspi,
1365 net->xfrm.state_byspi + h,
1366 x->xso.type);
1367 }
1368 if (x->km.seq) {
1369 h = xfrm_seq_hash(net, x->km.seq);
1370 XFRM_STATE_INSERT(byseq, &x->byseq,
1371 net->xfrm.state_byseq + h,
1372 x->xso.type);
1373 }
1374 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1375 hrtimer_start(&x->mtimer,
1376 ktime_set(net->xfrm.sysctl_acq_expires, 0),
1377 HRTIMER_MODE_REL_SOFT);
1378 net->xfrm.state_num++;
1379 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1380 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1381 } else {
1382 #ifdef CONFIG_XFRM_OFFLOAD
1383 struct xfrm_dev_offload *xso = &x->xso;
1384
1385 if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {
1386 xfrm_dev_state_delete(x);
1387 xfrm_dev_state_free(x);
1388 }
1389 #endif
1390 x->km.state = XFRM_STATE_DEAD;
1391 to_put = x;
1392 x = NULL;
1393 error = -ESRCH;
1394 }
1395 }
1396 out:
1397 if (x) {
1398 if (!xfrm_state_hold_rcu(x)) {
1399 *err = -EAGAIN;
1400 x = NULL;
1401 }
1402 } else {
1403 *err = acquire_in_progress ? -EAGAIN : error;
1404 }
1405 rcu_read_unlock();
1406 if (to_put)
1407 xfrm_state_put(to_put);
1408
1409 if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {
1410 *err = -EAGAIN;
1411 if (x) {
1412 xfrm_state_put(x);
1413 x = NULL;
1414 }
1415 }
1416
1417 return x;
1418 }
1419
1420 struct xfrm_state *
1421 xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id,
1422 xfrm_address_t *daddr, xfrm_address_t *saddr,
1423 unsigned short family, u8 mode, u8 proto, u32 reqid)
1424 {
1425 unsigned int h;
1426 struct xfrm_state *rx = NULL, *x = NULL;
1427
1428 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1429 h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1430 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1431 if (x->props.family == family &&
1432 x->props.reqid == reqid &&
1433 (mark & x->mark.m) == x->mark.v &&
1434 x->if_id == if_id &&
1435 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1436 xfrm_state_addr_check(x, daddr, saddr, family) &&
1437 mode == x->props.mode &&
1438 proto == x->id.proto &&
1439 x->km.state == XFRM_STATE_VALID) {
1440 rx = x;
1441 break;
1442 }
1443 }
1444
1445 if (rx)
1446 xfrm_state_hold(rx);
1447 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1448
1449
1450 return rx;
1451 }
1452 EXPORT_SYMBOL(xfrm_stateonly_find);
1453
1454 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
1455 unsigned short family)
1456 {
1457 struct xfrm_state *x;
1458 struct xfrm_state_walk *w;
1459
1460 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1461 list_for_each_entry(w, &net->xfrm.state_all, all) {
1462 x = container_of(w, struct xfrm_state, km);
1463 if (x->props.family != family ||
1464 x->id.spi != spi)
1465 continue;
1466
1467 xfrm_state_hold(x);
1468 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1469 return x;
1470 }
1471 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1472 return NULL;
1473 }
1474 EXPORT_SYMBOL(xfrm_state_lookup_byspi);
1475
1476 static void __xfrm_state_insert(struct xfrm_state *x)
1477 {
1478 struct net *net = xs_net(x);
1479 unsigned int h;
1480
1481 list_add(&x->km.all, &net->xfrm.state_all);
1482
1483 h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
1484 x->props.reqid, x->props.family);
1485 XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
1486 x->xso.type);
1487
1488 h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);
1489 XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
1490 x->xso.type);
1491
1492 if (x->id.spi) {
1493 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,
1494 x->props.family);
1495
1496 XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
1497 x->xso.type);
1498 }
1499
1500 if (x->km.seq) {
1501 h = xfrm_seq_hash(net, x->km.seq);
1502
1503 XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,
1504 x->xso.type);
1505 }
1506
1507 hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
1508 if (x->replay_maxage)
1509 mod_timer(&x->rtimer, jiffies + x->replay_maxage);
1510
1511 net->xfrm.state_num++;
1512
1513 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1514 xfrm_nat_keepalive_state_updated(x);
1515 }
1516
1517 /* net->xfrm.xfrm_state_lock is held */
1518 static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
1519 {
1520 struct net *net = xs_net(xnew);
1521 unsigned short family = xnew->props.family;
1522 u32 reqid = xnew->props.reqid;
1523 struct xfrm_state *x;
1524 unsigned int h;
1525 u32 mark = xnew->mark.v & xnew->mark.m;
1526 u32 if_id = xnew->if_id;
1527
1528 h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);
1529 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1530 if (x->props.family == family &&
1531 x->props.reqid == reqid &&
1532 x->if_id == if_id &&
1533 (mark & x->mark.m) == x->mark.v &&
1534 xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&
1535 xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))
1536 x->genid++;
1537 }
1538 }
1539
1540 void xfrm_state_insert(struct xfrm_state *x)
1541 {
1542 struct net *net = xs_net(x);
1543
1544 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1545 __xfrm_state_bump_genids(x);
1546 __xfrm_state_insert(x);
1547 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1548 }
1549 EXPORT_SYMBOL(xfrm_state_insert);
1550
1551 /* net->xfrm.xfrm_state_lock is held */
1552 static struct xfrm_state *__find_acq_core(struct net *net,
1553 const struct xfrm_mark *m,
1554 unsigned short family, u8 mode,
1555 u32 reqid, u32 if_id, u8 proto,
1556 const xfrm_address_t *daddr,
1557 const xfrm_address_t *saddr,
1558 int create)
1559 {
1560 unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1561 struct xfrm_state *x;
1562 u32 mark = m->v & m->m;
1563
1564 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1565 if (x->props.reqid != reqid ||
1566 x->props.mode != mode ||
1567 x->props.family != family ||
1568 x->km.state != XFRM_STATE_ACQ ||
1569 x->id.spi != 0 ||
1570 x->id.proto != proto ||
1571 (mark & x->mark.m) != x->mark.v ||
1572 !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
1573 !xfrm_addr_equal(&x->props.saddr, saddr, family))
1574 continue;
1575
1576 xfrm_state_hold(x);
1577 return x;
1578 }
1579
1580 if (!create)
1581 return NULL;
1582
1583 x = xfrm_state_alloc(net);
1584 if (likely(x)) {
1585 switch (family) {
1586 case AF_INET:
1587 x->sel.daddr.a4 = daddr->a4;
1588 x->sel.saddr.a4 = saddr->a4;
1589 x->sel.prefixlen_d = 32;
1590 x->sel.prefixlen_s = 32;
1591 x->props.saddr.a4 = saddr->a4;
1592 x->id.daddr.a4 = daddr->a4;
1593 break;
1594
1595 case AF_INET6:
1596 x->sel.daddr.in6 = daddr->in6;
1597 x->sel.saddr.in6 = saddr->in6;
1598 x->sel.prefixlen_d = 128;
1599 x->sel.prefixlen_s = 128;
1600 x->props.saddr.in6 = saddr->in6;
1601 x->id.daddr.in6 = daddr->in6;
1602 break;
1603 }
1604
1605 x->km.state = XFRM_STATE_ACQ;
1606 x->id.proto = proto;
1607 x->props.family = family;
1608 x->props.mode = mode;
1609 x->props.reqid = reqid;
1610 x->if_id = if_id;
1611 x->mark.v = m->v;
1612 x->mark.m = m->m;
1613 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1614 xfrm_state_hold(x);
1615 hrtimer_start(&x->mtimer,
1616 ktime_set(net->xfrm.sysctl_acq_expires, 0),
1617 HRTIMER_MODE_REL_SOFT);
1618 list_add(&x->km.all, &net->xfrm.state_all);
1619 XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
1620 x->xso.type);
1621 h = xfrm_src_hash(net, daddr, saddr, family);
1622 XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
1623 x->xso.type);
1624
1625 net->xfrm.state_num++;
1626
1627 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1628 }
1629
1630 return x;
1631 }
1632
1633 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq);
1634
1635 int xfrm_state_add(struct xfrm_state *x)
1636 {
1637 struct net *net = xs_net(x);
1638 struct xfrm_state *x1, *to_put;
1639 int family;
1640 int err;
1641 u32 mark = x->mark.v & x->mark.m;
1642 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
1643
1644 family = x->props.family;
1645
1646 to_put = NULL;
1647
1648 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1649
1650 x1 = __xfrm_state_locate(x, use_spi, family);
1651 if (x1) {
1652 to_put = x1;
1653 x1 = NULL;
1654 err = -EEXIST;
1655 goto out;
1656 }
1657
1658 if (use_spi && x->km.seq) {
1659 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq);
1660 if (x1 && ((x1->id.proto != x->id.proto) ||
1661 !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) {
1662 to_put = x1;
1663 x1 = NULL;
1664 }
1665 }
1666
1667 if (use_spi && !x1)
1668 x1 = __find_acq_core(net, &x->mark, family, x->props.mode,
1669 x->props.reqid, x->if_id, x->id.proto,
1670 &x->id.daddr, &x->props.saddr, 0);
1671
1672 __xfrm_state_bump_genids(x);
1673 __xfrm_state_insert(x);
1674 err = 0;
1675
1676 out:
1677 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1678
1679 if (x1) {
1680 xfrm_state_delete(x1);
1681 xfrm_state_put(x1);
1682 }
1683
1684 if (to_put)
1685 xfrm_state_put(to_put);
1686
1687 return err;
1688 }
1689 EXPORT_SYMBOL(xfrm_state_add);
1690
1691 #ifdef CONFIG_XFRM_MIGRATE
1692 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)
1693 {
1694 struct xfrm_user_sec_ctx *uctx;
1695 int size = sizeof(*uctx) + security->ctx_len;
1696 int err;
1697
1698 uctx = kmalloc(size, GFP_KERNEL);
1699 if (!uctx)
1700 return -ENOMEM;
1701
1702 uctx->exttype = XFRMA_SEC_CTX;
1703 uctx->len = size;
1704 uctx->ctx_doi = security->ctx_doi;
1705 uctx->ctx_alg = security->ctx_alg;
1706 uctx->ctx_len = security->ctx_len;
1707 memcpy(uctx + 1, security->ctx_str, security->ctx_len);
1708 err = security_xfrm_state_alloc(x, uctx);
1709 kfree(uctx);
1710 if (err)
1711 return err;
1712
1713 return 0;
1714 }
1715
1716 static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
1717 struct xfrm_encap_tmpl *encap)
1718 {
1719 struct net *net = xs_net(orig);
1720 struct xfrm_state *x = xfrm_state_alloc(net);
1721 if (!x)
1722 goto out;
1723
1724 memcpy(&x->id, &orig->id, sizeof(x->id));
1725 memcpy(&x->sel, &orig->sel, sizeof(x->sel));
1726 memcpy(&x->lft, &orig->lft, sizeof(x->lft));
1727 x->props.mode = orig->props.mode;
1728 x->props.replay_window = orig->props.replay_window;
1729 x->props.reqid = orig->props.reqid;
1730 x->props.family = orig->props.family;
1731 x->props.saddr = orig->props.saddr;
1732
1733 if (orig->aalg) {
1734 x->aalg = xfrm_algo_auth_clone(orig->aalg);
1735 if (!x->aalg)
1736 goto error;
1737 }
1738 x->props.aalgo = orig->props.aalgo;
1739
1740 if (orig->aead) {
1741 x->aead = xfrm_algo_aead_clone(orig->aead);
1742 x->geniv = orig->geniv;
1743 if (!x->aead)
1744 goto error;
1745 }
1746 if (orig->ealg) {
1747 x->ealg = xfrm_algo_clone(orig->ealg);
1748 if (!x->ealg)
1749 goto error;
1750 }
1751 x->props.ealgo = orig->props.ealgo;
1752
1753 if (orig->calg) {
1754 x->calg = xfrm_algo_clone(orig->calg);
1755 if (!x->calg)
1756 goto error;
1757 }
1758 x->props.calgo = orig->props.calgo;
1759
1760 if (encap || orig->encap) {
1761 if (encap)
1762 x->encap = kmemdup(encap, sizeof(*x->encap),
1763 GFP_KERNEL);
1764 else
1765 x->encap = kmemdup(orig->encap, sizeof(*x->encap),
1766 GFP_KERNEL);
1767
1768 if (!x->encap)
1769 goto error;
1770 }
1771
1772 if (orig->security)
1773 if (clone_security(x, orig->security))
1774 goto error;
1775
1776 if (orig->coaddr) {
1777 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),
1778 GFP_KERNEL);
1779 if (!x->coaddr)
1780 goto error;
1781 }
1782
1783 if (orig->replay_esn) {
1784 if (xfrm_replay_clone(x, orig))
1785 goto error;
1786 }
1787
1788 memcpy(&x->mark, &orig->mark, sizeof(x->mark));
1789 memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));
1790
1791 x->props.flags = orig->props.flags;
1792 x->props.extra_flags = orig->props.extra_flags;
1793
1794 x->if_id = orig->if_id;
1795 x->tfcpad = orig->tfcpad;
1796 x->replay_maxdiff = orig->replay_maxdiff;
1797 x->replay_maxage = orig->replay_maxage;
1798 memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft));
1799 x->km.state = orig->km.state;
1800 x->km.seq = orig->km.seq;
1801 x->replay = orig->replay;
1802 x->preplay = orig->preplay;
1803 x->mapping_maxage = orig->mapping_maxage;
1804 x->lastused = orig->lastused;
1805 x->new_mapping = 0;
1806 x->new_mapping_sport = 0;
1807 x->dir = orig->dir;
1808
1809 return x;
1810
1811 error:
1812 xfrm_state_put(x);
1813 out:
1814 return NULL;
1815 }
1816
1817 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
1818 u32 if_id)
1819 {
1820 unsigned int h;
1821 struct xfrm_state *x = NULL;
1822
1823 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1824
1825 if (m->reqid) {
1826 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,
1827 m->reqid, m->old_family);
1828 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1829 if (x->props.mode != m->mode ||
1830 x->id.proto != m->proto)
1831 continue;
1832 if (m->reqid && x->props.reqid != m->reqid)
1833 continue;
1834 if (if_id != 0 && x->if_id != if_id)
1835 continue;
1836 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
1837 m->old_family) ||
1838 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
1839 m->old_family))
1840 continue;
1841 xfrm_state_hold(x);
1842 break;
1843 }
1844 } else {
1845 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,
1846 m->old_family);
1847 hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {
1848 if (x->props.mode != m->mode ||
1849 x->id.proto != m->proto)
1850 continue;
1851 if (if_id != 0 && x->if_id != if_id)
1852 continue;
1853 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
1854 m->old_family) ||
1855 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
1856 m->old_family))
1857 continue;
1858 xfrm_state_hold(x);
1859 break;
1860 }
1861 }
1862
1863 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1864
1865 return x;
1866 }
1867 EXPORT_SYMBOL(xfrm_migrate_state_find);
1868
1869 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
1870 struct xfrm_migrate *m,
1871 struct xfrm_encap_tmpl *encap)
1872 {
1873 struct xfrm_state *xc;
1874
1875 xc = xfrm_state_clone(x, encap);
1876 if (!xc)
1877 return NULL;
1878
1879 xc->props.family = m->new_family;
1880
1881 if (xfrm_init_state(xc) < 0)
1882 goto error;
1883
1884 memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
1885 memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
1886
1887 /* add state */
1888 if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {
1889 /* a care is needed when the destination address of the
1890 state is to be updated as it is a part of triplet */
1891 xfrm_state_insert(xc);
1892 } else {
1893 if (xfrm_state_add(xc) < 0)
1894 goto error;
1895 }
1896
1897 return xc;
1898 error:
1899 xfrm_state_put(xc);
1900 return NULL;
1901 }
1902 EXPORT_SYMBOL(xfrm_state_migrate);
1903 #endif
1904
1905 int xfrm_state_update(struct xfrm_state *x)
1906 {
1907 struct xfrm_state *x1, *to_put;
1908 int err;
1909 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
1910 struct net *net = xs_net(x);
1911
1912 to_put = NULL;
1913
1914 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1915 x1 = __xfrm_state_locate(x, use_spi, x->props.family);
1916
1917 err = -ESRCH;
1918 if (!x1)
1919 goto out;
1920
1921 if (xfrm_state_kern(x1)) {
1922 to_put = x1;
1923 err = -EEXIST;
1924 goto out;
1925 }
1926
1927 if (x1->km.state == XFRM_STATE_ACQ) {
1928 if (x->dir && x1->dir != x->dir)
1929 goto out;
1930
1931 __xfrm_state_insert(x);
1932 x = NULL;
1933 } else {
1934 if (x1->dir != x->dir)
1935 goto out;
1936 }
1937 err = 0;
1938
1939 out:
1940 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1941
1942 if (to_put)
1943 xfrm_state_put(to_put);
1944
1945 if (err)
1946 return err;
1947
1948 if (!x) {
1949 xfrm_state_delete(x1);
1950 xfrm_state_put(x1);
1951 return 0;
1952 }
1953
1954 err = -EINVAL;
1955 spin_lock_bh(&x1->lock);
1956 if (likely(x1->km.state == XFRM_STATE_VALID)) {
1957 if (x->encap && x1->encap &&
1958 x->encap->encap_type == x1->encap->encap_type)
1959 memcpy(x1->encap, x->encap, sizeof(*x1->encap));
1960 else if (x->encap || x1->encap)
1961 goto fail;
1962
1963 if (x->coaddr && x1->coaddr) {
1964 memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));
1965 }
1966 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel)))
1967 memcpy(&x1->sel, &x->sel, sizeof(x1->sel));
1968 memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
1969 x1->km.dying = 0;
1970
1971 hrtimer_start(&x1->mtimer, ktime_set(1, 0),
1972 HRTIMER_MODE_REL_SOFT);
1973 if (READ_ONCE(x1->curlft.use_time))
1974 xfrm_state_check_expire(x1);
1975
1976 if (x->props.smark.m || x->props.smark.v || x->if_id) {
1977 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1978
1979 if (x->props.smark.m || x->props.smark.v)
1980 x1->props.smark = x->props.smark;
1981
1982 if (x->if_id)
1983 x1->if_id = x->if_id;
1984
1985 __xfrm_state_bump_genids(x1);
1986 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1987 }
1988
1989 err = 0;
1990 x->km.state = XFRM_STATE_DEAD;
1991 __xfrm_state_put(x);
1992 }
1993
1994 fail:
1995 spin_unlock_bh(&x1->lock);
1996
1997 xfrm_state_put(x1);
1998
1999 return err;
2000 }
2001 EXPORT_SYMBOL(xfrm_state_update);
2002
2003 int xfrm_state_check_expire(struct xfrm_state *x)
2004 {
2005 xfrm_dev_state_update_stats(x);
2006
2007 if (!READ_ONCE(x->curlft.use_time))
2008 WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());
2009
2010 if (x->curlft.bytes >= x->lft.hard_byte_limit ||
2011 x->curlft.packets >= x->lft.hard_packet_limit) {
2012 x->km.state = XFRM_STATE_EXPIRED;
2013 hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT);
2014 return -EINVAL;
2015 }
2016
2017 if (!x->km.dying &&
2018 (x->curlft.bytes >= x->lft.soft_byte_limit ||
2019 x->curlft.packets >= x->lft.soft_packet_limit)) {
2020 x->km.dying = 1;
2021 km_state_expired(x, 0, 0);
2022 }
2023 return 0;
2024 }
2025 EXPORT_SYMBOL(xfrm_state_check_expire);
2026
2027 void xfrm_state_update_stats(struct net *net)
2028 {
2029 struct xfrm_state *x;
2030 int i;
2031
2032 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2033 for (i = 0; i <= net->xfrm.state_hmask; i++) {
2034 hlist_for_each_entry(x, net->xfrm.state_bydst + i, bydst)
2035 xfrm_dev_state_update_stats(x);
2036 }
2037 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2038 }
2039
2040 struct xfrm_state *
2041 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,
2042 u8 proto, unsigned short family)
2043 {
2044 struct xfrm_state *x;
2045
2046 rcu_read_lock();
2047 x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family);
2048 rcu_read_unlock();
2049 return x;
2050 }
2051 EXPORT_SYMBOL(xfrm_state_lookup);
2052
2053 struct xfrm_state *
2054 xfrm_state_lookup_byaddr(struct net *net, u32 mark,
2055 const xfrm_address_t *daddr, const xfrm_address_t *saddr,
2056 u8 proto, unsigned short family)
2057 {
2058 struct xfrm_state *x;
2059
2060 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2061 x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family);
2062 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2063 return x;
2064 }
2065 EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
2066
2067 struct xfrm_state *
2068 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
2069 u32 if_id, u8 proto, const xfrm_address_t *daddr,
2070 const xfrm_address_t *saddr, int create, unsigned short family)
2071 {
2072 struct xfrm_state *x;
2073
2074 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2075 x = __find_acq_core(net, mark, family, mode, reqid, if_id, proto, daddr, saddr, create);
2076 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2077
2078 return x;
2079 }
2080 EXPORT_SYMBOL(xfrm_find_acq);
2081
2082 #ifdef CONFIG_XFRM_SUB_POLICY
2083 #if IS_ENABLED(CONFIG_IPV6)
2084 /* distribution counting sort function for xfrm_state and xfrm_tmpl */
2085 static void
2086 __xfrm6_sort(void **dst, void **src, int n,
2087 int (*cmp)(const void *p), int maxclass)
2088 {
2089 int count[XFRM_MAX_DEPTH] = { };
2090 int class[XFRM_MAX_DEPTH];
2091 int i;
2092
2093 for (i = 0; i < n; i++) {
2094 int c = cmp(src[i]);
2095
2096 class[i] = c;
2097 count[c]++;
2098 }
2099
2100 for (i = 2; i < maxclass; i++)
2101 count[i] += count[i - 1];
2102
2103 for (i = 0; i < n; i++) {
2104 dst[count[class[i] - 1]++] = src[i];
2105 src[i] = NULL;
2106 }
2107 }
2108
2109 /* Rule for xfrm_state:
2110 *
2111 * rule 1: select IPsec transport except AH
2112 * rule 2: select MIPv6 RO or inbound trigger
2113 * rule 3: select IPsec transport AH
2114 * rule 4: select IPsec tunnel
2115 * rule 5: others
2116 */
2117 static int __xfrm6_state_sort_cmp(const void *p)
2118 {
2119 const struct xfrm_state *v = p;
2120
2121 switch (v->props.mode) {
2122 case XFRM_MODE_TRANSPORT:
2123 if (v->id.proto != IPPROTO_AH)
2124 return 1;
2125 else
2126 return 3;
2127 #if IS_ENABLED(CONFIG_IPV6_MIP6)
2128 case XFRM_MODE_ROUTEOPTIMIZATION:
2129 case XFRM_MODE_IN_TRIGGER:
2130 return 2;
2131 #endif
2132 case XFRM_MODE_TUNNEL:
2133 case XFRM_MODE_BEET:
2134 return 4;
2135 }
2136 return 5;
2137 }
2138
2139 /* Rule for xfrm_tmpl:
2140 *
2141 * rule 1: select IPsec transport
2142 * rule 2: select MIPv6 RO or inbound trigger
2143 * rule 3: select IPsec tunnel
2144 * rule 4: others
2145 */
2146 static int __xfrm6_tmpl_sort_cmp(const void *p)
2147 {
2148 const struct xfrm_tmpl *v = p;
2149
2150 switch (v->mode) {
2151 case XFRM_MODE_TRANSPORT:
2152 return 1;
2153 #if IS_ENABLED(CONFIG_IPV6_MIP6)
2154 case XFRM_MODE_ROUTEOPTIMIZATION:
2155 case XFRM_MODE_IN_TRIGGER:
2156 return 2;
2157 #endif
2158 case XFRM_MODE_TUNNEL:
2159 case XFRM_MODE_BEET:
2160 return 3;
2161 }
2162 return 4;
2163 }
2164 #else
2165 static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; }
2166 static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; }
2167
2168 static inline void
2169 __xfrm6_sort(void **dst, void **src, int n,
2170 int (*cmp)(const void *p), int maxclass)
2171 {
2172 int i;
2173
2174 for (i = 0; i < n; i++)
2175 dst[i] = src[i];
2176 }
2177 #endif /* CONFIG_IPV6 */
2178
2179 void
2180 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
2181 unsigned short family)
2182 {
2183 int i;
2184
2185 if (family == AF_INET6)
2186 __xfrm6_sort((void **)dst, (void **)src, n,
2187 __xfrm6_tmpl_sort_cmp, 5);
2188 else
2189 for (i = 0; i < n; i++)
2190 dst[i] = src[i];
2191 }
2192
2193 void
2194 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
2195 unsigned short family)
2196 {
2197 int i;
2198
2199 if (family == AF_INET6)
2200 __xfrm6_sort((void **)dst, (void **)src, n,
2201 __xfrm6_state_sort_cmp, 6);
2202 else
2203 for (i = 0; i < n; i++)
2204 dst[i] = src[i];
2205 }
2206 #endif
2207
2208 /* Silly enough, but I'm lazy to build resolution list */
2209
2210 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
2211 {
2212 unsigned int h = xfrm_seq_hash(net, seq);
2213 struct xfrm_state *x;
2214
2215 hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {
2216 if (x->km.seq == seq &&
2217 (mark & x->mark.m) == x->mark.v &&
2218 x->km.state == XFRM_STATE_ACQ) {
2219 xfrm_state_hold(x);
2220 return x;
2221 }
2222 }
2223
2224 return NULL;
2225 }
2226
2227 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
2228 {
2229 struct xfrm_state *x;
2230
2231 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2232 x = __xfrm_find_acq_byseq(net, mark, seq);
2233 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2234 return x;
2235 }
2236 EXPORT_SYMBOL(xfrm_find_acq_byseq);
2237
2238 u32 xfrm_get_acqseq(void)
2239 {
2240 u32 res;
2241 static atomic_t acqseq;
2242
2243 do {
2244 res = atomic_inc_return(&acqseq);
2245 } while (!res);
2246
2247 return res;
2248 }
2249 EXPORT_SYMBOL(xfrm_get_acqseq);
2250
2251 int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack)
2252 {
2253 switch (proto) {
2254 case IPPROTO_AH:
2255 case IPPROTO_ESP:
2256 break;
2257
2258 case IPPROTO_COMP:
2259 /* IPCOMP spi is 16-bits. */
2260 if (max >= 0x10000) {
2261 NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535");
2262 return -EINVAL;
2263 }
2264 break;
2265
2266 default:
2267 NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP");
2268 return -EINVAL;
2269 }
2270
2271 if (min > max) {
2272 NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max");
2273 return -EINVAL;
2274 }
2275
2276 return 0;
2277 }
2278 EXPORT_SYMBOL(verify_spi_info);
2279
2280 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
2281 struct netlink_ext_ack *extack)
2282 {
2283 struct net *net = xs_net(x);
2284 unsigned int h;
2285 struct xfrm_state *x0;
2286 int err = -ENOENT;
2287 __be32 minspi = htonl(low);
2288 __be32 maxspi = htonl(high);
2289 __be32 newspi = 0;
2290 u32 mark = x->mark.v & x->mark.m;
2291
2292 spin_lock_bh(&x->lock);
2293 if (x->km.state == XFRM_STATE_DEAD) {
2294 NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state");
2295 goto unlock;
2296 }
2297
2298 err = 0;
2299 if (x->id.spi)
2300 goto unlock;
2301
2302 err = -ENOENT;
2303
2304 if (minspi == maxspi) {
2305 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);
2306 if (x0) {
2307 NL_SET_ERR_MSG(extack, "Requested SPI is already in use");
2308 xfrm_state_put(x0);
2309 goto unlock;
2310 }
2311 newspi = minspi;
2312 } else {
2313 u32 spi = 0;
2314 for (h = 0; h < high-low+1; h++) {
2315 spi = get_random_u32_inclusive(low, high);
2316 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);
2317 if (x0 == NULL) {
2318 newspi = htonl(spi);
2319 break;
2320 }
2321 xfrm_state_put(x0);
2322 }
2323 }
2324 if (newspi) {
2325 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2326 x->id.spi = newspi;
2327 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);
2328 XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
2329 x->xso.type);
2330 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2331
2332 err = 0;
2333 } else {
2334 NL_SET_ERR_MSG(extack, "No SPI available in the requested range");
2335 }
2336
2337 unlock:
2338 spin_unlock_bh(&x->lock);
2339
2340 return err;
2341 }
2342 EXPORT_SYMBOL(xfrm_alloc_spi);
2343
2344 static bool __xfrm_state_filter_match(struct xfrm_state *x,
2345 struct xfrm_address_filter *filter)
2346 {
2347 if (filter) {
2348 if ((filter->family == AF_INET ||
2349 filter->family == AF_INET6) &&
2350 x->props.family != filter->family)
2351 return false;
2352
2353 return addr_match(&x->props.saddr, &filter->saddr,
2354 filter->splen) &&
2355 addr_match(&x->id.daddr, &filter->daddr,
2356 filter->dplen);
2357 }
2358 return true;
2359 }
2360
2361 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
2362 int (*func)(struct xfrm_state *, int, void*),
2363 void *data)
2364 {
2365 struct xfrm_state *state;
2366 struct xfrm_state_walk *x;
2367 int err = 0;
2368
2369 if (walk->seq != 0 && list_empty(&walk->all))
2370 return 0;
2371
2372 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2373 if (list_empty(&walk->all))
2374 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);
2375 else
2376 x = list_first_entry(&walk->all, struct xfrm_state_walk, all);
2377 list_for_each_entry_from(x, &net->xfrm.state_all, all) {
2378 if (x->state == XFRM_STATE_DEAD)
2379 continue;
2380 state = container_of(x, struct xfrm_state, km);
2381 if (!xfrm_id_proto_match(state->id.proto, walk->proto))
2382 continue;
2383 if (!__xfrm_state_filter_match(state, walk->filter))
2384 continue;
2385 err = func(state, walk->seq, data);
2386 if (err) {
2387 list_move_tail(&walk->all, &x->all);
2388 goto out;
2389 }
2390 walk->seq++;
2391 }
2392 if (walk->seq == 0) {
2393 err = -ENOENT;
2394 goto out;
2395 }
2396 list_del_init(&walk->all);
2397 out:
2398 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2399 return err;
2400 }
2401 EXPORT_SYMBOL(xfrm_state_walk);
2402
2403 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
2404 struct xfrm_address_filter *filter)
2405 {
2406 INIT_LIST_HEAD(&walk->all);
2407 walk->proto = proto;
2408 walk->state = XFRM_STATE_DEAD;
2409 walk->seq = 0;
2410 walk->filter = filter;
2411 }
2412 EXPORT_SYMBOL(xfrm_state_walk_init);
2413
2414 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)
2415 {
2416 kfree(walk->filter);
2417
2418 if (list_empty(&walk->all))
2419 return;
2420
2421 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2422 list_del(&walk->all);
2423 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2424 }
2425 EXPORT_SYMBOL(xfrm_state_walk_done);
2426
2427 static void xfrm_replay_timer_handler(struct timer_list *t)
2428 {
2429 struct xfrm_state *x = from_timer(x, t, rtimer);
2430
2431 spin_lock(&x->lock);
2432
2433 if (x->km.state == XFRM_STATE_VALID) {
2434 if (xfrm_aevent_is_on(xs_net(x)))
2435 xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
2436 else
2437 x->xflags |= XFRM_TIME_DEFER;
2438 }
2439
2440 spin_unlock(&x->lock);
2441 }
2442
2443 static LIST_HEAD(xfrm_km_list);
2444
2445 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
2446 {
2447 struct xfrm_mgr *km;
2448
2449 rcu_read_lock();
2450 list_for_each_entry_rcu(km, &xfrm_km_list, list)
2451 if (km->notify_policy)
2452 km->notify_policy(xp, dir, c);
2453 rcu_read_unlock();
2454 }
2455
2456 void km_state_notify(struct xfrm_state *x, const struct km_event *c)
2457 {
2458 struct xfrm_mgr *km;
2459 rcu_read_lock();
2460 list_for_each_entry_rcu(km, &xfrm_km_list, list)
2461 if (km->notify)
2462 km->notify(x, c);
2463 rcu_read_unlock();
2464 }
2465
2466 EXPORT_SYMBOL(km_policy_notify);
2467 EXPORT_SYMBOL(km_state_notify);
2468
2469 void km_state_expired(struct xfrm_state *x, int hard, u32 portid)
2470 {
2471 struct km_event c;
2472
2473 c.data.hard = hard;
2474 c.portid = portid;
2475 c.event = XFRM_MSG_EXPIRE;
2476 km_state_notify(x, &c);
2477 }
2478
2479 EXPORT_SYMBOL(km_state_expired);
2480 /*
2481 * We send to all registered managers regardless of failure
2482 * We are happy with one success
2483 */
2484 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
2485 {
2486 int err = -EINVAL, acqret;
2487 struct xfrm_mgr *km;
2488
2489 rcu_read_lock();
2490 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2491 acqret = km->acquire(x, t, pol);
2492 if (!acqret)
2493 err = acqret;
2494 }
2495 rcu_read_unlock();
2496 return err;
2497 }
2498 EXPORT_SYMBOL(km_query);
2499
2500 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2501 {
2502 int err = -EINVAL;
2503 struct xfrm_mgr *km;
2504
2505 rcu_read_lock();
2506 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2507 if (km->new_mapping)
2508 err = km->new_mapping(x, ipaddr, sport);
2509 if (!err)
2510 break;
2511 }
2512 rcu_read_unlock();
2513 return err;
2514 }
2515
2516 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2517 {
2518 int ret = 0;
2519
2520 if (x->mapping_maxage) {
2521 if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||
2522 x->new_mapping_sport != sport) {
2523 x->new_mapping_sport = sport;
2524 x->new_mapping = jiffies / HZ;
2525 ret = __km_new_mapping(x, ipaddr, sport);
2526 }
2527 } else {
2528 ret = __km_new_mapping(x, ipaddr, sport);
2529 }
2530
2531 return ret;
2532 }
2533 EXPORT_SYMBOL(km_new_mapping);
2534
2535 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
2536 {
2537 struct km_event c;
2538
2539 c.data.hard = hard;
2540 c.portid = portid;
2541 c.event = XFRM_MSG_POLEXPIRE;
2542 km_policy_notify(pol, dir, &c);
2543 }
2544 EXPORT_SYMBOL(km_policy_expired);
2545
2546 #ifdef CONFIG_XFRM_MIGRATE
2547 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
2548 const struct xfrm_migrate *m, int num_migrate,
2549 const struct xfrm_kmaddress *k,
2550 const struct xfrm_encap_tmpl *encap)
2551 {
2552 int err = -EINVAL;
2553 int ret;
2554 struct xfrm_mgr *km;
2555
2556 rcu_read_lock();
2557 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2558 if (km->migrate) {
2559 ret = km->migrate(sel, dir, type, m, num_migrate, k,
2560 encap);
2561 if (!ret)
2562 err = ret;
2563 }
2564 }
2565 rcu_read_unlock();
2566 return err;
2567 }
2568 EXPORT_SYMBOL(km_migrate);
2569 #endif
2570
2571 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)
2572 {
2573 int err = -EINVAL;
2574 int ret;
2575 struct xfrm_mgr *km;
2576
2577 rcu_read_lock();
2578 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2579 if (km->report) {
2580 ret = km->report(net, proto, sel, addr);
2581 if (!ret)
2582 err = ret;
2583 }
2584 }
2585 rcu_read_unlock();
2586 return err;
2587 }
2588 EXPORT_SYMBOL(km_report);
2589
2590 static bool km_is_alive(const struct km_event *c)
2591 {
2592 struct xfrm_mgr *km;
2593 bool is_alive = false;
2594
2595 rcu_read_lock();
2596 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2597 if (km->is_alive && km->is_alive(c)) {
2598 is_alive = true;
2599 break;
2600 }
2601 }
2602 rcu_read_unlock();
2603
2604 return is_alive;
2605 }
2606
2607 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2608 static DEFINE_SPINLOCK(xfrm_translator_lock);
2609 static struct xfrm_translator __rcu *xfrm_translator;
2610
2611 struct xfrm_translator *xfrm_get_translator(void)
2612 {
2613 struct xfrm_translator *xtr;
2614
2615 rcu_read_lock();
2616 xtr = rcu_dereference(xfrm_translator);
2617 if (unlikely(!xtr))
2618 goto out;
2619 if (!try_module_get(xtr->owner))
2620 xtr = NULL;
2621 out:
2622 rcu_read_unlock();
2623 return xtr;
2624 }
2625 EXPORT_SYMBOL_GPL(xfrm_get_translator);
2626
2627 void xfrm_put_translator(struct xfrm_translator *xtr)
2628 {
2629 module_put(xtr->owner);
2630 }
2631 EXPORT_SYMBOL_GPL(xfrm_put_translator);
2632
2633 int xfrm_register_translator(struct xfrm_translator *xtr)
2634 {
2635 int err = 0;
2636
2637 spin_lock_bh(&xfrm_translator_lock);
2638 if (unlikely(xfrm_translator != NULL))
2639 err = -EEXIST;
2640 else
2641 rcu_assign_pointer(xfrm_translator, xtr);
2642 spin_unlock_bh(&xfrm_translator_lock);
2643
2644 return err;
2645 }
2646 EXPORT_SYMBOL_GPL(xfrm_register_translator);
2647
2648 int xfrm_unregister_translator(struct xfrm_translator *xtr)
2649 {
2650 int err = 0;
2651
2652 spin_lock_bh(&xfrm_translator_lock);
2653 if (likely(xfrm_translator != NULL)) {
2654 if (rcu_access_pointer(xfrm_translator) != xtr)
2655 err = -EINVAL;
2656 else
2657 RCU_INIT_POINTER(xfrm_translator, NULL);
2658 }
2659 spin_unlock_bh(&xfrm_translator_lock);
2660 synchronize_rcu();
2661
2662 return err;
2663 }
2664 EXPORT_SYMBOL_GPL(xfrm_unregister_translator);
2665 #endif
2666
2667 int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
2668 {
2669 int err;
2670 u8 *data;
2671 struct xfrm_mgr *km;
2672 struct xfrm_policy *pol = NULL;
2673
2674 if (sockptr_is_null(optval) && !optlen) {
2675 xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL);
2676 xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL);
2677 __sk_dst_reset(sk);
2678 return 0;
2679 }
2680
2681 if (optlen <= 0 || optlen > PAGE_SIZE)
2682 return -EMSGSIZE;
2683
2684 data = memdup_sockptr(optval, optlen);
2685 if (IS_ERR(data))
2686 return PTR_ERR(data);
2687
2688 if (in_compat_syscall()) {
2689 struct xfrm_translator *xtr = xfrm_get_translator();
2690
2691 if (!xtr) {
2692 kfree(data);
2693 return -EOPNOTSUPP;
2694 }
2695
2696 err = xtr->xlate_user_policy_sockptr(&data, optlen);
2697 xfrm_put_translator(xtr);
2698 if (err) {
2699 kfree(data);
2700 return err;
2701 }
2702 }
2703
2704 err = -EINVAL;
2705 rcu_read_lock();
2706 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2707 pol = km->compile_policy(sk, optname, data,
2708 optlen, &err);
2709 if (err >= 0)
2710 break;
2711 }
2712 rcu_read_unlock();
2713
2714 if (err >= 0) {
2715 xfrm_sk_policy_insert(sk, err, pol);
2716 xfrm_pol_put(pol);
2717 __sk_dst_reset(sk);
2718 err = 0;
2719 }
2720
2721 kfree(data);
2722 return err;
2723 }
2724 EXPORT_SYMBOL(xfrm_user_policy);
2725
2726 static DEFINE_SPINLOCK(xfrm_km_lock);
2727
2728 void xfrm_register_km(struct xfrm_mgr *km)
2729 {
2730 spin_lock_bh(&xfrm_km_lock);
2731 list_add_tail_rcu(&km->list, &xfrm_km_list);
2732 spin_unlock_bh(&xfrm_km_lock);
2733 }
2734 EXPORT_SYMBOL(xfrm_register_km);
2735
2736 void xfrm_unregister_km(struct xfrm_mgr *km)
2737 {
2738 spin_lock_bh(&xfrm_km_lock);
2739 list_del_rcu(&km->list);
2740 spin_unlock_bh(&xfrm_km_lock);
2741 synchronize_rcu();
2742 }
2743 EXPORT_SYMBOL(xfrm_unregister_km);
2744
2745 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo)
2746 {
2747 int err = 0;
2748
2749 if (WARN_ON(afinfo->family >= NPROTO))
2750 return -EAFNOSUPPORT;
2751
2752 spin_lock_bh(&xfrm_state_afinfo_lock);
2753 if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))
2754 err = -EEXIST;
2755 else
2756 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);
2757 spin_unlock_bh(&xfrm_state_afinfo_lock);
2758 return err;
2759 }
2760 EXPORT_SYMBOL(xfrm_state_register_afinfo);
2761
2762 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo)
2763 {
2764 int err = 0, family = afinfo->family;
2765
2766 if (WARN_ON(family >= NPROTO))
2767 return -EAFNOSUPPORT;
2768
2769 spin_lock_bh(&xfrm_state_afinfo_lock);
2770 if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {
2771 if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo)
2772 err = -EINVAL;
2773 else
2774 RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);
2775 }
2776 spin_unlock_bh(&xfrm_state_afinfo_lock);
2777 synchronize_rcu();
2778 return err;
2779 }
2780 EXPORT_SYMBOL(xfrm_state_unregister_afinfo);
2781
2782 struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family)
2783 {
2784 if (unlikely(family >= NPROTO))
2785 return NULL;
2786
2787 return rcu_dereference(xfrm_state_afinfo[family]);
2788 }
2789 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu);
2790
2791 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
2792 {
2793 struct xfrm_state_afinfo *afinfo;
2794 if (unlikely(family >= NPROTO))
2795 return NULL;
2796 rcu_read_lock();
2797 afinfo = rcu_dereference(xfrm_state_afinfo[family]);
2798 if (unlikely(!afinfo))
2799 rcu_read_unlock();
2800 return afinfo;
2801 }
2802
2803 void xfrm_flush_gc(void)
2804 {
2805 flush_work(&xfrm_state_gc_work);
2806 }
2807 EXPORT_SYMBOL(xfrm_flush_gc);
2808
2809 /* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
2810 void xfrm_state_delete_tunnel(struct xfrm_state *x)
2811 {
2812 if (x->tunnel) {
2813 struct xfrm_state *t = x->tunnel;
2814
2815 if (atomic_read(&t->tunnel_users) == 2)
2816 xfrm_state_delete(t);
2817 atomic_dec(&t->tunnel_users);
2818 xfrm_state_put_sync(t);
2819 x->tunnel = NULL;
2820 }
2821 }
2822 EXPORT_SYMBOL(xfrm_state_delete_tunnel);
2823
2824 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
2825 {
2826 const struct xfrm_type *type = READ_ONCE(x->type);
2827 struct crypto_aead *aead;
2828 u32 blksize, net_adj = 0;
2829
2830 if (x->km.state != XFRM_STATE_VALID ||
2831 !type || type->proto != IPPROTO_ESP)
2832 return mtu - x->props.header_len;
2833
2834 aead = x->data;
2835 blksize = ALIGN(crypto_aead_blocksize(aead), 4);
2836
2837 switch (x->props.mode) {
2838 case XFRM_MODE_TRANSPORT:
2839 case XFRM_MODE_BEET:
2840 if (x->props.family == AF_INET)
2841 net_adj = sizeof(struct iphdr);
2842 else if (x->props.family == AF_INET6)
2843 net_adj = sizeof(struct ipv6hdr);
2844 break;
2845 case XFRM_MODE_TUNNEL:
2846 break;
2847 default:
2848 WARN_ON_ONCE(1);
2849 break;
2850 }
2851
2852 return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -
2853 net_adj) & ~(blksize - 1)) + net_adj - 2;
2854 }
2855 EXPORT_SYMBOL_GPL(xfrm_state_mtu);
2856
2857 int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
2858 struct netlink_ext_ack *extack)
2859 {
2860 const struct xfrm_mode *inner_mode;
2861 const struct xfrm_mode *outer_mode;
2862 int family = x->props.family;
2863 int err;
2864
2865 if (family == AF_INET &&
2866 READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
2867 x->props.flags |= XFRM_STATE_NOPMTUDISC;
2868
2869 err = -EPROTONOSUPPORT;
2870
2871 if (x->sel.family != AF_UNSPEC) {
2872 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);
2873 if (inner_mode == NULL) {
2874 NL_SET_ERR_MSG(extack, "Requested mode not found");
2875 goto error;
2876 }
2877
2878 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
2879 family != x->sel.family) {
2880 NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family");
2881 goto error;
2882 }
2883
2884 x->inner_mode = *inner_mode;
2885 } else {
2886 const struct xfrm_mode *inner_mode_iaf;
2887 int iafamily = AF_INET;
2888
2889 inner_mode = xfrm_get_mode(x->props.mode, x->props.family);
2890 if (inner_mode == NULL) {
2891 NL_SET_ERR_MSG(extack, "Requested mode not found");
2892 goto error;
2893 }
2894
2895 x->inner_mode = *inner_mode;
2896
2897 if (x->props.family == AF_INET)
2898 iafamily = AF_INET6;
2899
2900 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);
2901 if (inner_mode_iaf) {
2902 if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
2903 x->inner_mode_iaf = *inner_mode_iaf;
2904 }
2905 }
2906
2907 x->type = xfrm_get_type(x->id.proto, family);
2908 if (x->type == NULL) {
2909 NL_SET_ERR_MSG(extack, "Requested type not found");
2910 goto error;
2911 }
2912
2913 x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload);
2914
2915 err = x->type->init_state(x, extack);
2916 if (err)
2917 goto error;
2918
2919 outer_mode = xfrm_get_mode(x->props.mode, family);
2920 if (!outer_mode) {
2921 NL_SET_ERR_MSG(extack, "Requested mode not found");
2922 err = -EPROTONOSUPPORT;
2923 goto error;
2924 }
2925
2926 x->outer_mode = *outer_mode;
2927 if (init_replay) {
2928 err = xfrm_init_replay(x, extack);
2929 if (err)
2930 goto error;
2931 }
2932
2933 if (x->nat_keepalive_interval) {
2934 if (x->dir != XFRM_SA_DIR_OUT) {
2935 NL_SET_ERR_MSG(extack, "NAT keepalive is only supported for outbound SAs");
2936 err = -EINVAL;
2937 goto error;
2938 }
2939
2940 if (!x->encap || x->encap->encap_type != UDP_ENCAP_ESPINUDP) {
2941 NL_SET_ERR_MSG(extack,
2942 "NAT keepalive is only supported for UDP encapsulation");
2943 err = -EINVAL;
2944 goto error;
2945 }
2946 }
2947
2948 error:
2949 return err;
2950 }
2951
2952 EXPORT_SYMBOL(__xfrm_init_state);
2953
2954 int xfrm_init_state(struct xfrm_state *x)
2955 {
2956 int err;
2957
2958 err = __xfrm_init_state(x, true, false, NULL);
2959 if (!err)
2960 x->km.state = XFRM_STATE_VALID;
2961
2962 return err;
2963 }
2964
2965 EXPORT_SYMBOL(xfrm_init_state);
2966
2967 int __net_init xfrm_state_init(struct net *net)
2968 {
2969 unsigned int sz;
2970
2971 if (net_eq(net, &init_net))
2972 xfrm_state_cache = KMEM_CACHE(xfrm_state,
2973 SLAB_HWCACHE_ALIGN | SLAB_PANIC);
2974
2975 INIT_LIST_HEAD(&net->xfrm.state_all);
2976
2977 sz = sizeof(struct hlist_head) * 8;
2978
2979 net->xfrm.state_bydst = xfrm_hash_alloc(sz);
2980 if (!net->xfrm.state_bydst)
2981 goto out_bydst;
2982 net->xfrm.state_bysrc = xfrm_hash_alloc(sz);
2983 if (!net->xfrm.state_bysrc)
2984 goto out_bysrc;
2985 net->xfrm.state_byspi = xfrm_hash_alloc(sz);
2986 if (!net->xfrm.state_byspi)
2987 goto out_byspi;
2988 net->xfrm.state_byseq = xfrm_hash_alloc(sz);
2989 if (!net->xfrm.state_byseq)
2990 goto out_byseq;
2991 net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
2992
2993 net->xfrm.state_num = 0;
2994 INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
2995 spin_lock_init(&net->xfrm.xfrm_state_lock);
2996 seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,
2997 &net->xfrm.xfrm_state_lock);
2998 return 0;
2999
3000 out_byseq:
3001 xfrm_hash_free(net->xfrm.state_byspi, sz);
3002 out_byspi:
3003 xfrm_hash_free(net->xfrm.state_bysrc, sz);
3004 out_bysrc:
3005 xfrm_hash_free(net->xfrm.state_bydst, sz);
3006 out_bydst:
3007 return -ENOMEM;
3008 }
3009
3010 void xfrm_state_fini(struct net *net)
3011 {
3012 unsigned int sz;
3013
3014 flush_work(&net->xfrm.state_hash_work);
3015 flush_work(&xfrm_state_gc_work);
3016 xfrm_state_flush(net, 0, false, true);
3017
3018 WARN_ON(!list_empty(&net->xfrm.state_all));
3019
3020 sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
3021 WARN_ON(!hlist_empty(net->xfrm.state_byseq));
3022 xfrm_hash_free(net->xfrm.state_byseq, sz);
3023 WARN_ON(!hlist_empty(net->xfrm.state_byspi));
3024 xfrm_hash_free(net->xfrm.state_byspi, sz);
3025 WARN_ON(!hlist_empty(net->xfrm.state_bysrc));
3026 xfrm_hash_free(net->xfrm.state_bysrc, sz);
3027 WARN_ON(!hlist_empty(net->xfrm.state_bydst));
3028 xfrm_hash_free(net->xfrm.state_bydst, sz);
3029 }
3030
3031 #ifdef CONFIG_AUDITSYSCALL
3032 static void xfrm_audit_helper_sainfo(struct xfrm_state *x,
3033 struct audit_buffer *audit_buf)
3034 {
3035 struct xfrm_sec_ctx *ctx = x->security;
3036 u32 spi = ntohl(x->id.spi);
3037
3038 if (ctx)
3039 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
3040 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
3041
3042 switch (x->props.family) {
3043 case AF_INET:
3044 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
3045 &x->props.saddr.a4, &x->id.daddr.a4);
3046 break;
3047 case AF_INET6:
3048 audit_log_format(audit_buf, " src=%pI6 dst=%pI6",
3049 x->props.saddr.a6, x->id.daddr.a6);
3050 break;
3051 }
3052
3053 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
3054 }
3055
3056 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family,
3057 struct audit_buffer *audit_buf)
3058 {
3059 const struct iphdr *iph4;
3060 const struct ipv6hdr *iph6;
3061
3062 switch (family) {
3063 case AF_INET:
3064 iph4 = ip_hdr(skb);
3065 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
3066 &iph4->saddr, &iph4->daddr);
3067 break;
3068 case AF_INET6:
3069 iph6 = ipv6_hdr(skb);
3070 audit_log_format(audit_buf,
3071 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
3072 &iph6->saddr, &iph6->daddr,
3073 iph6->flow_lbl[0] & 0x0f,
3074 iph6->flow_lbl[1],
3075 iph6->flow_lbl[2]);
3076 break;
3077 }
3078 }
3079
3080 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)
3081 {
3082 struct audit_buffer *audit_buf;
3083
3084 audit_buf = xfrm_audit_start("SAD-add");
3085 if (audit_buf == NULL)
3086 return;
3087 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3088 xfrm_audit_helper_sainfo(x, audit_buf);
3089 audit_log_format(audit_buf, " res=%u", result);
3090 audit_log_end(audit_buf);
3091 }
3092 EXPORT_SYMBOL_GPL(xfrm_audit_state_add);
3093
3094 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)
3095 {
3096 struct audit_buffer *audit_buf;
3097
3098 audit_buf = xfrm_audit_start("SAD-delete");
3099 if (audit_buf == NULL)
3100 return;
3101 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3102 xfrm_audit_helper_sainfo(x, audit_buf);
3103 audit_log_format(audit_buf, " res=%u", result);
3104 audit_log_end(audit_buf);
3105 }
3106 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete);
3107
3108 void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
3109 struct sk_buff *skb)
3110 {
3111 struct audit_buffer *audit_buf;
3112 u32 spi;
3113
3114 audit_buf = xfrm_audit_start("SA-replay-overflow");
3115 if (audit_buf == NULL)
3116 return;
3117 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3118 /* don't record the sequence number because it's inherent in this kind
3119 * of audit message */
3120 spi = ntohl(x->id.spi);
3121 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
3122 audit_log_end(audit_buf);
3123 }
3124 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow);
3125
3126 void xfrm_audit_state_replay(struct xfrm_state *x,
3127 struct sk_buff *skb, __be32 net_seq)
3128 {
3129 struct audit_buffer *audit_buf;
3130 u32 spi;
3131
3132 audit_buf = xfrm_audit_start("SA-replayed-pkt");
3133 if (audit_buf == NULL)
3134 return;
3135 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3136 spi = ntohl(x->id.spi);
3137 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3138 spi, spi, ntohl(net_seq));
3139 audit_log_end(audit_buf);
3140 }
3141 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay);
3142
3143 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family)
3144 {
3145 struct audit_buffer *audit_buf;
3146
3147 audit_buf = xfrm_audit_start("SA-notfound");
3148 if (audit_buf == NULL)
3149 return;
3150 xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3151 audit_log_end(audit_buf);
3152 }
3153 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple);
3154
3155 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
3156 __be32 net_spi, __be32 net_seq)
3157 {
3158 struct audit_buffer *audit_buf;
3159 u32 spi;
3160
3161 audit_buf = xfrm_audit_start("SA-notfound");
3162 if (audit_buf == NULL)
3163 return;
3164 xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3165 spi = ntohl(net_spi);
3166 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3167 spi, spi, ntohl(net_seq));
3168 audit_log_end(audit_buf);
3169 }
3170 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound);
3171
3172 void xfrm_audit_state_icvfail(struct xfrm_state *x,
3173 struct sk_buff *skb, u8 proto)
3174 {
3175 struct audit_buffer *audit_buf;
3176 __be32 net_spi;
3177 __be32 net_seq;
3178
3179 audit_buf = xfrm_audit_start("SA-icv-failure");
3180 if (audit_buf == NULL)
3181 return;
3182 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3183 if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) {
3184 u32 spi = ntohl(net_spi);
3185 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3186 spi, spi, ntohl(net_seq));
3187 }
3188 audit_log_end(audit_buf);
3189 }
3190 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail);
3191 #endif /* CONFIG_AUDITSYSCALL */
3192
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.