1 // SPDX-License-Identifier: GPL-2.0
6 * YOSHIFUJI Hideaki @USAGI
7 * Split up af-specific portion
11 #include <linux/bottom_half.h>
12 #include <linux/interrupt.h>
13 #include <linux/slab.h>
14 #include <linux/module.h>
15 #include <linux/netdevice.h>
16 #include <linux/percpu.h>
20 #include <net/ip_tunnels.h>
21 #include <net/ip6_tunnel.h>
23 struct xfrm_trans_tasklet
{
24 struct tasklet_struct tasklet
;
25 struct sk_buff_head queue
;
28 struct xfrm_trans_cb
{
30 struct inet_skb_parm h4
;
31 #if IS_ENABLED(CONFIG_IPV6)
32 struct inet6_skb_parm h6
;
35 int (*finish
)(struct net
*net
, struct sock
*sk
, struct sk_buff
*skb
);
38 #define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
40 static struct kmem_cache
*secpath_cachep __read_mostly
;
42 static DEFINE_SPINLOCK(xfrm_input_afinfo_lock
);
43 static struct xfrm_input_afinfo
const __rcu
*xfrm_input_afinfo
[AF_INET6
+ 1];
45 static struct gro_cells gro_cells
;
46 static struct net_device xfrm_napi_dev
;
48 static DEFINE_PER_CPU(struct xfrm_trans_tasklet
, xfrm_trans_tasklet
);
50 int xfrm_input_register_afinfo(const struct xfrm_input_afinfo
*afinfo
)
54 if (WARN_ON(afinfo
->family
>= ARRAY_SIZE(xfrm_input_afinfo
)))
57 spin_lock_bh(&xfrm_input_afinfo_lock
);
58 if (unlikely(xfrm_input_afinfo
[afinfo
->family
] != NULL
))
61 rcu_assign_pointer(xfrm_input_afinfo
[afinfo
->family
], afinfo
);
62 spin_unlock_bh(&xfrm_input_afinfo_lock
);
65 EXPORT_SYMBOL(xfrm_input_register_afinfo
);
67 int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo
*afinfo
)
71 spin_lock_bh(&xfrm_input_afinfo_lock
);
72 if (likely(xfrm_input_afinfo
[afinfo
->family
] != NULL
)) {
73 if (unlikely(xfrm_input_afinfo
[afinfo
->family
] != afinfo
))
76 RCU_INIT_POINTER(xfrm_input_afinfo
[afinfo
->family
], NULL
);
78 spin_unlock_bh(&xfrm_input_afinfo_lock
);
82 EXPORT_SYMBOL(xfrm_input_unregister_afinfo
);
84 static const struct xfrm_input_afinfo
*xfrm_input_get_afinfo(unsigned int family
)
86 const struct xfrm_input_afinfo
*afinfo
;
88 if (WARN_ON_ONCE(family
>= ARRAY_SIZE(xfrm_input_afinfo
)))
92 afinfo
= rcu_dereference(xfrm_input_afinfo
[family
]);
93 if (unlikely(!afinfo
))
98 static int xfrm_rcv_cb(struct sk_buff
*skb
, unsigned int family
, u8 protocol
,
102 const struct xfrm_input_afinfo
*afinfo
= xfrm_input_get_afinfo(family
);
105 return -EAFNOSUPPORT
;
107 ret
= afinfo
->callback(skb
, protocol
, err
);
113 void __secpath_destroy(struct sec_path
*sp
)
116 for (i
= 0; i
< sp
->len
; i
++)
117 xfrm_state_put(sp
->xvec
[i
]);
118 kmem_cache_free(secpath_cachep
, sp
);
120 EXPORT_SYMBOL(__secpath_destroy
);
122 struct sec_path
*secpath_dup(struct sec_path
*src
)
126 sp
= kmem_cache_alloc(secpath_cachep
, GFP_ATOMIC
);
133 memset(sp
->ovec
, 0, sizeof(sp
->ovec
[XFRM_MAX_OFFLOAD_DEPTH
]));
138 memcpy(sp
, src
, sizeof(*sp
));
139 for (i
= 0; i
< sp
->len
; i
++)
140 xfrm_state_hold(sp
->xvec
[i
]);
142 refcount_set(&sp
->refcnt
, 1);
145 EXPORT_SYMBOL(secpath_dup
);
147 int secpath_set(struct sk_buff
*skb
)
151 /* Allocate new secpath or COW existing one. */
152 if (!skb
->sp
|| refcount_read(&skb
->sp
->refcnt
) != 1) {
153 sp
= secpath_dup(skb
->sp
);
158 secpath_put(skb
->sp
);
163 EXPORT_SYMBOL(secpath_set
);
165 /* Fetch spi and seq from ipsec header */
167 int xfrm_parse_spi(struct sk_buff
*skb
, u8 nexthdr
, __be32
*spi
, __be32
*seq
)
169 int offset
, offset_seq
;
174 hlen
= sizeof(struct ip_auth_hdr
);
175 offset
= offsetof(struct ip_auth_hdr
, spi
);
176 offset_seq
= offsetof(struct ip_auth_hdr
, seq_no
);
179 hlen
= sizeof(struct ip_esp_hdr
);
180 offset
= offsetof(struct ip_esp_hdr
, spi
);
181 offset_seq
= offsetof(struct ip_esp_hdr
, seq_no
);
184 if (!pskb_may_pull(skb
, sizeof(struct ip_comp_hdr
)))
186 *spi
= htonl(ntohs(*(__be16
*)(skb_transport_header(skb
) + 2)));
193 if (!pskb_may_pull(skb
, hlen
))
196 *spi
= *(__be32
*)(skb_transport_header(skb
) + offset
);
197 *seq
= *(__be32
*)(skb_transport_header(skb
) + offset_seq
);
200 EXPORT_SYMBOL(xfrm_parse_spi
);
202 int xfrm_prepare_input(struct xfrm_state
*x
, struct sk_buff
*skb
)
204 struct xfrm_mode
*inner_mode
= x
->inner_mode
;
207 err
= x
->outer_mode
->afinfo
->extract_input(x
, skb
);
211 if (x
->sel
.family
== AF_UNSPEC
) {
212 inner_mode
= xfrm_ip2inner_mode(x
, XFRM_MODE_SKB_CB(skb
)->protocol
);
213 if (inner_mode
== NULL
)
214 return -EAFNOSUPPORT
;
217 skb
->protocol
= inner_mode
->afinfo
->eth_proto
;
218 return inner_mode
->input2(x
, skb
);
220 EXPORT_SYMBOL(xfrm_prepare_input
);
222 int xfrm_input(struct sk_buff
*skb
, int nexthdr
, __be32 spi
, int encap_type
)
224 struct net
*net
= dev_net(skb
->dev
);
228 struct xfrm_state
*x
= NULL
;
229 xfrm_address_t
*daddr
;
230 struct xfrm_mode
*inner_mode
;
231 u32 mark
= skb
->mark
;
232 unsigned int family
= AF_UNSPEC
;
235 bool xfrm_gro
= false;
236 bool crypto_done
= false;
237 struct xfrm_offload
*xo
= xfrm_offload(skb
);
239 if (encap_type
< 0) {
240 x
= xfrm_input_state(skb
);
242 if (unlikely(x
->km
.state
!= XFRM_STATE_VALID
)) {
243 if (x
->km
.state
== XFRM_STATE_ACQ
)
244 XFRM_INC_STATS(net
, LINUX_MIB_XFRMACQUIREERROR
);
247 LINUX_MIB_XFRMINSTATEINVALID
);
251 family
= x
->outer_mode
->afinfo
->family
;
253 /* An encap_type of -1 indicates async resumption. */
254 if (encap_type
== -1) {
256 seq
= XFRM_SKB_CB(skb
)->seq
.input
.low
;
260 /* encap_type < -1 indicates a GRO call. */
262 seq
= XFRM_SPI_SKB_CB(skb
)->seq
;
264 if (xo
&& (xo
->flags
& CRYPTO_DONE
)) {
266 x
= xfrm_input_state(skb
);
267 family
= XFRM_SPI_SKB_CB(skb
)->family
;
269 if (!(xo
->status
& CRYPTO_SUCCESS
)) {
271 (CRYPTO_TRANSPORT_AH_AUTH_FAILED
|
272 CRYPTO_TRANSPORT_ESP_AUTH_FAILED
|
273 CRYPTO_TUNNEL_AH_AUTH_FAILED
|
274 CRYPTO_TUNNEL_ESP_AUTH_FAILED
)) {
276 xfrm_audit_state_icvfail(x
, skb
,
278 x
->stats
.integrity_failed
++;
279 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEPROTOERROR
);
283 if (xo
->status
& CRYPTO_INVALID_PROTOCOL
) {
284 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEPROTOERROR
);
288 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINBUFFERERROR
);
292 if ((err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
)) != 0) {
293 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
301 family
= XFRM_SPI_SKB_CB(skb
)->family
;
303 /* if tunnel is present override skb->mark value with tunnel i_key */
306 if (XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip4
)
307 mark
= be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip4
->parms
.i_key
);
310 if (XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip6
)
311 mark
= be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip6
->parms
.i_key
);
315 err
= secpath_set(skb
);
317 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINERROR
);
322 if (!spi
&& (err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
)) != 0) {
323 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
327 daddr
= (xfrm_address_t
*)(skb_network_header(skb
) +
328 XFRM_SPI_SKB_CB(skb
)->daddroff
);
330 if (skb
->sp
->len
== XFRM_MAX_DEPTH
) {
331 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINBUFFERERROR
);
335 x
= xfrm_state_lookup(net
, mark
, daddr
, spi
, nexthdr
, family
);
337 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINNOSTATES
);
338 xfrm_audit_state_notfound(skb
, family
, spi
, seq
);
342 skb
->sp
->xvec
[skb
->sp
->len
++] = x
;
346 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINERROR
);
353 if (unlikely(x
->km
.state
!= XFRM_STATE_VALID
)) {
354 if (x
->km
.state
== XFRM_STATE_ACQ
)
355 XFRM_INC_STATS(net
, LINUX_MIB_XFRMACQUIREERROR
);
358 LINUX_MIB_XFRMINSTATEINVALID
);
362 if ((x
->encap
? x
->encap
->encap_type
: 0) != encap_type
) {
363 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMISMATCH
);
367 if (x
->repl
->check(x
, skb
, seq
)) {
368 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATESEQERROR
);
372 if (xfrm_state_check_expire(x
)) {
373 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEEXPIRED
);
377 spin_unlock(&x
->lock
);
379 if (xfrm_tunnel_check(skb
, x
, family
)) {
380 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
384 seq_hi
= htonl(xfrm_replay_seqhi(x
, seq
));
386 XFRM_SKB_CB(skb
)->seq
.input
.low
= seq
;
387 XFRM_SKB_CB(skb
)->seq
.input
.hi
= seq_hi
;
392 nexthdr
= x
->type_offload
->input_tail(x
, skb
);
394 nexthdr
= x
->type
->input(x
, skb
);
396 if (nexthdr
== -EINPROGRESS
)
403 if (nexthdr
== -EBADMSG
) {
404 xfrm_audit_state_icvfail(x
, skb
,
406 x
->stats
.integrity_failed
++;
408 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEPROTOERROR
);
412 /* only the first xfrm gets the encap type */
415 if (async
&& x
->repl
->recheck(x
, skb
, seq
)) {
416 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATESEQERROR
);
420 x
->repl
->advance(x
, seq
);
422 x
->curlft
.bytes
+= skb
->len
;
425 spin_unlock(&x
->lock
);
427 XFRM_MODE_SKB_CB(skb
)->protocol
= nexthdr
;
429 inner_mode
= x
->inner_mode
;
431 if (x
->sel
.family
== AF_UNSPEC
) {
432 inner_mode
= xfrm_ip2inner_mode(x
, XFRM_MODE_SKB_CB(skb
)->protocol
);
433 if (inner_mode
== NULL
) {
434 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
439 if (inner_mode
->input(x
, skb
)) {
440 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
444 if (x
->outer_mode
->flags
& XFRM_MODE_FLAG_TUNNEL
) {
450 * We need the inner address. However, we only get here for
451 * transport mode so the outer address is identical.
453 daddr
= &x
->id
.daddr
;
454 family
= x
->outer_mode
->afinfo
->family
;
456 err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
);
458 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
464 err
= xfrm_rcv_cb(skb
, family
, x
->type
->proto
, 0);
474 gro_cells_receive(&gro_cells
, skb
);
477 xo
= xfrm_offload(skb
);
479 xfrm_gro
= xo
->flags
& XFRM_GRO
;
481 err
= x
->inner_mode
->afinfo
->transport_finish(skb
, xfrm_gro
|| async
);
486 gro_cells_receive(&gro_cells
, skb
);
494 spin_unlock(&x
->lock
);
496 xfrm_rcv_cb(skb
, family
, x
&& x
->type
? x
->type
->proto
: nexthdr
, -1);
500 EXPORT_SYMBOL(xfrm_input
);
502 int xfrm_input_resume(struct sk_buff
*skb
, int nexthdr
)
504 return xfrm_input(skb
, nexthdr
, 0, -1);
506 EXPORT_SYMBOL(xfrm_input_resume
);
508 static void xfrm_trans_reinject(unsigned long data
)
510 struct xfrm_trans_tasklet
*trans
= (void *)data
;
511 struct sk_buff_head queue
;
514 __skb_queue_head_init(&queue
);
515 skb_queue_splice_init(&trans
->queue
, &queue
);
517 while ((skb
= __skb_dequeue(&queue
)))
518 XFRM_TRANS_SKB_CB(skb
)->finish(dev_net(skb
->dev
), NULL
, skb
);
521 int xfrm_trans_queue(struct sk_buff
*skb
,
522 int (*finish
)(struct net
*, struct sock
*,
525 struct xfrm_trans_tasklet
*trans
;
527 trans
= this_cpu_ptr(&xfrm_trans_tasklet
);
529 if (skb_queue_len(&trans
->queue
) >= netdev_max_backlog
)
532 XFRM_TRANS_SKB_CB(skb
)->finish
= finish
;
533 __skb_queue_tail(&trans
->queue
, skb
);
534 tasklet_schedule(&trans
->tasklet
);
537 EXPORT_SYMBOL(xfrm_trans_queue
);
539 void __init
xfrm_input_init(void)
544 init_dummy_netdev(&xfrm_napi_dev
);
545 err
= gro_cells_init(&gro_cells
, &xfrm_napi_dev
);
547 gro_cells
.cells
= NULL
;
549 secpath_cachep
= kmem_cache_create("secpath_cache",
550 sizeof(struct sec_path
),
551 0, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
,
554 for_each_possible_cpu(i
) {
555 struct xfrm_trans_tasklet
*trans
;
557 trans
= &per_cpu(xfrm_trans_tasklet
, i
);
558 __skb_queue_head_init(&trans
->queue
);
559 tasklet_init(&trans
->tasklet
, xfrm_trans_reinject
,
560 (unsigned long)trans
);