2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * The Internet Protocol (IP) output module.
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Donald Becker, <becker@super.org>
11 * Alan Cox, <Alan.Cox@linux.org>
13 * Stefan Becker, <stefanb@yello.ping.de>
14 * Jorge Cwik, <jorge@laser.satlink.net>
15 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
16 * Hirokazu Takahashi, <taka@valinux.co.jp>
18 * See ip_input.c for original log
21 * Alan Cox : Missing nonblock feature in ip_build_xmit.
22 * Mike Kilburn : htons() missing in ip_build_xmit.
23 * Bradford Johnson: Fix faulty handling of some frames when
25 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit
26 * (in case if packet not accepted by
27 * output firewall rules)
28 * Mike McLagan : Routing by source
29 * Alexey Kuznetsov: use new route cache
30 * Andi Kleen: Fix broken PMTU recovery and remove
31 * some redundant tests.
32 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
33 * Andi Kleen : Replace ip_reply with ip_send_reply.
34 * Andi Kleen : Split fast and slow ip_build_xmit path
35 * for decreased register pressure on x86
36 * and more readibility.
37 * Marc Boucher : When call_out_firewall returns FW_QUEUE,
38 * silently drop skb instead of failing with -EPERM.
39 * Detlev Wengorz : Copy protocol for fragments.
40 * Hirokazu Takahashi: HW checksumming for outgoing UDP
42 * Hirokazu Takahashi: sendfile() on UDP works now.
45 #include <asm/uaccess.h>
46 #include <asm/system.h>
47 #include <linux/module.h>
48 #include <linux/types.h>
49 #include <linux/kernel.h>
51 #include <linux/string.h>
52 #include <linux/errno.h>
53 #include <linux/highmem.h>
54 #include <linux/slab.h>
56 #include <linux/socket.h>
57 #include <linux/sockios.h>
59 #include <linux/inet.h>
60 #include <linux/netdevice.h>
61 #include <linux/etherdevice.h>
62 #include <linux/proc_fs.h>
63 #include <linux/stat.h>
64 #include <linux/init.h>
68 #include <net/protocol.h>
69 #include <net/route.h>
71 #include <linux/skbuff.h>
75 #include <net/checksum.h>
76 #include <net/inetpeer.h>
77 #include <linux/igmp.h>
78 #include <linux/netfilter_ipv4.h>
79 #include <linux/netfilter_bridge.h>
80 #include <linux/mroute.h>
81 #include <linux/netlink.h>
82 #include <linux/tcp.h>
84 int sysctl_ip_default_ttl __read_mostly
= IPDEFTTL
;
85 EXPORT_SYMBOL(sysctl_ip_default_ttl
);
87 /* Generate a checksum for an outgoing IP datagram. */
88 __inline__
void ip_send_check(struct iphdr
*iph
)
91 iph
->check
= ip_fast_csum((unsigned char *)iph
, iph
->ihl
);
93 EXPORT_SYMBOL(ip_send_check
);
95 int __ip_local_out(struct sk_buff
*skb
)
97 struct iphdr
*iph
= ip_hdr(skb
);
99 iph
->tot_len
= htons(skb
->len
);
101 return nf_hook(NFPROTO_IPV4
, NF_INET_LOCAL_OUT
, skb
, NULL
,
102 skb_dst(skb
)->dev
, dst_output
);
105 int ip_local_out(struct sk_buff
*skb
)
109 err
= __ip_local_out(skb
);
110 if (likely(err
== 1))
111 err
= dst_output(skb
);
115 EXPORT_SYMBOL_GPL(ip_local_out
);
117 /* dev_loopback_xmit for use with netfilter. */
118 static int ip_dev_loopback_xmit(struct sk_buff
*newskb
)
120 skb_reset_mac_header(newskb
);
121 __skb_pull(newskb
, skb_network_offset(newskb
));
122 newskb
->pkt_type
= PACKET_LOOPBACK
;
123 newskb
->ip_summed
= CHECKSUM_UNNECESSARY
;
124 WARN_ON(!skb_dst(newskb
));
129 static inline int ip_select_ttl(struct inet_sock
*inet
, struct dst_entry
*dst
)
131 int ttl
= inet
->uc_ttl
;
134 ttl
= ip4_dst_hoplimit(dst
);
139 * Add an ip header to a skbuff and send it out.
142 int ip_build_and_send_pkt(struct sk_buff
*skb
, struct sock
*sk
,
143 __be32 saddr
, __be32 daddr
, struct ip_options_rcu
*opt
)
145 struct inet_sock
*inet
= inet_sk(sk
);
146 struct rtable
*rt
= skb_rtable(skb
);
149 /* Build the IP header. */
150 skb_push(skb
, sizeof(struct iphdr
) + (opt
? opt
->opt
.optlen
: 0));
151 skb_reset_network_header(skb
);
155 iph
->tos
= inet
->tos
;
156 if (ip_dont_fragment(sk
, &rt
->dst
))
157 iph
->frag_off
= htons(IP_DF
);
160 iph
->ttl
= ip_select_ttl(inet
, &rt
->dst
);
161 iph
->daddr
= (opt
&& opt
->opt
.srr
? opt
->opt
.faddr
: daddr
);
163 iph
->protocol
= sk
->sk_protocol
;
164 ip_select_ident(iph
, &rt
->dst
, sk
);
166 if (opt
&& opt
->opt
.optlen
) {
167 iph
->ihl
+= opt
->opt
.optlen
>>2;
168 ip_options_build(skb
, &opt
->opt
, daddr
, rt
, 0);
171 skb
->priority
= sk
->sk_priority
;
172 skb
->mark
= sk
->sk_mark
;
175 return ip_local_out(skb
);
177 EXPORT_SYMBOL_GPL(ip_build_and_send_pkt
);
179 static inline int ip_finish_output2(struct sk_buff
*skb
)
181 struct dst_entry
*dst
= skb_dst(skb
);
182 struct rtable
*rt
= (struct rtable
*)dst
;
183 struct net_device
*dev
= dst
->dev
;
184 unsigned int hh_len
= LL_RESERVED_SPACE(dev
);
186 if (rt
->rt_type
== RTN_MULTICAST
) {
187 IP_UPD_PO_STATS(dev_net(dev
), IPSTATS_MIB_OUTMCAST
, skb
->len
);
188 } else if (rt
->rt_type
== RTN_BROADCAST
)
189 IP_UPD_PO_STATS(dev_net(dev
), IPSTATS_MIB_OUTBCAST
, skb
->len
);
191 /* Be paranoid, rather than too clever. */
192 if (unlikely(skb_headroom(skb
) < hh_len
&& dev
->header_ops
)) {
193 struct sk_buff
*skb2
;
195 skb2
= skb_realloc_headroom(skb
, LL_RESERVED_SPACE(dev
));
201 skb_set_owner_w(skb2
, skb
->sk
);
207 return neigh_hh_output(dst
->hh
, skb
);
208 else if (dst
->neighbour
)
209 return dst
->neighbour
->output(skb
);
212 printk(KERN_DEBUG
"ip_finish_output2: No header cache and no neighbour!\n");
217 static inline int ip_skb_dst_mtu(struct sk_buff
*skb
)
219 struct inet_sock
*inet
= skb
->sk
? inet_sk(skb
->sk
) : NULL
;
221 return (inet
&& inet
->pmtudisc
== IP_PMTUDISC_PROBE
) ?
222 skb_dst(skb
)->dev
->mtu
: dst_mtu(skb_dst(skb
));
225 static int ip_finish_output(struct sk_buff
*skb
)
227 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
228 /* Policy lookup after SNAT yielded a new policy */
229 if (skb_dst(skb
)->xfrm
!= NULL
) {
230 IPCB(skb
)->flags
|= IPSKB_REROUTED
;
231 return dst_output(skb
);
234 if (skb
->len
> ip_skb_dst_mtu(skb
) && !skb_is_gso(skb
))
235 return ip_fragment(skb
, ip_finish_output2
);
237 return ip_finish_output2(skb
);
240 int ip_mc_output(struct sk_buff
*skb
)
242 struct sock
*sk
= skb
->sk
;
243 struct rtable
*rt
= skb_rtable(skb
);
244 struct net_device
*dev
= rt
->dst
.dev
;
247 * If the indicated interface is up and running, send the packet.
249 IP_UPD_PO_STATS(dev_net(dev
), IPSTATS_MIB_OUT
, skb
->len
);
252 skb
->protocol
= htons(ETH_P_IP
);
255 * Multicasts are looped back for other local users
258 if (rt
->rt_flags
&RTCF_MULTICAST
) {
260 #ifdef CONFIG_IP_MROUTE
261 /* Small optimization: do not loopback not local frames,
262 which returned after forwarding; they will be dropped
263 by ip_mr_input in any case.
264 Note, that local frames are looped back to be delivered
267 This check is duplicated in ip_mr_input at the moment.
270 ((rt
->rt_flags
& RTCF_LOCAL
) ||
271 !(IPCB(skb
)->flags
& IPSKB_FORWARDED
))
274 struct sk_buff
*newskb
= skb_clone(skb
, GFP_ATOMIC
);
276 NF_HOOK(NFPROTO_IPV4
, NF_INET_POST_ROUTING
,
277 newskb
, NULL
, newskb
->dev
,
278 ip_dev_loopback_xmit
);
281 /* Multicasts with ttl 0 must not go beyond the host */
283 if (ip_hdr(skb
)->ttl
== 0) {
289 if (rt
->rt_flags
&RTCF_BROADCAST
) {
290 struct sk_buff
*newskb
= skb_clone(skb
, GFP_ATOMIC
);
292 NF_HOOK(NFPROTO_IPV4
, NF_INET_POST_ROUTING
, newskb
,
293 NULL
, newskb
->dev
, ip_dev_loopback_xmit
);
296 return NF_HOOK_COND(NFPROTO_IPV4
, NF_INET_POST_ROUTING
, skb
, NULL
,
297 skb
->dev
, ip_finish_output
,
298 !(IPCB(skb
)->flags
& IPSKB_REROUTED
));
301 int ip_output(struct sk_buff
*skb
)
303 struct net_device
*dev
= skb_dst(skb
)->dev
;
305 IP_UPD_PO_STATS(dev_net(dev
), IPSTATS_MIB_OUT
, skb
->len
);
308 skb
->protocol
= htons(ETH_P_IP
);
310 return NF_HOOK_COND(NFPROTO_IPV4
, NF_INET_POST_ROUTING
, skb
, NULL
, dev
,
312 !(IPCB(skb
)->flags
& IPSKB_REROUTED
));
315 int ip_queue_xmit(struct sk_buff
*skb
, struct flowi
*fl
)
317 struct sock
*sk
= skb
->sk
;
318 struct inet_sock
*inet
= inet_sk(sk
);
319 struct ip_options_rcu
*inet_opt
;
325 /* Skip all of this if the packet is already routed,
326 * f.e. by something like SCTP.
329 inet_opt
= rcu_dereference(inet
->inet_opt
);
331 rt
= skb_rtable(skb
);
335 /* Make sure we can route this packet. */
336 rt
= (struct rtable
*)__sk_dst_check(sk
, 0);
340 /* Use correct destination address if we have options. */
341 daddr
= inet
->inet_daddr
;
342 if (inet_opt
&& inet_opt
->opt
.srr
)
343 daddr
= inet_opt
->opt
.faddr
;
345 /* If this fails, retransmit mechanism of transport layer will
346 * keep trying until route appears or the connection times
349 rt
= ip_route_output_ports(sock_net(sk
), fl4
, sk
,
350 daddr
, inet
->inet_saddr
,
355 sk
->sk_bound_dev_if
);
358 sk_setup_caps(sk
, &rt
->dst
);
360 skb_dst_set_noref(skb
, &rt
->dst
);
363 if (inet_opt
&& inet_opt
->opt
.is_strictroute
&& fl4
->daddr
!= rt
->rt_gateway
)
366 /* OK, we know where to send it, allocate and build IP header. */
367 skb_push(skb
, sizeof(struct iphdr
) + (inet_opt
? inet_opt
->opt
.optlen
: 0));
368 skb_reset_network_header(skb
);
370 *((__be16
*)iph
) = htons((4 << 12) | (5 << 8) | (inet
->tos
& 0xff));
371 if (ip_dont_fragment(sk
, &rt
->dst
) && !skb
->local_df
)
372 iph
->frag_off
= htons(IP_DF
);
375 iph
->ttl
= ip_select_ttl(inet
, &rt
->dst
);
376 iph
->protocol
= sk
->sk_protocol
;
377 iph
->saddr
= fl4
->saddr
;
378 iph
->daddr
= fl4
->daddr
;
379 /* Transport layer set skb->h.foo itself. */
381 if (inet_opt
&& inet_opt
->opt
.optlen
) {
382 iph
->ihl
+= inet_opt
->opt
.optlen
>> 2;
383 ip_options_build(skb
, &inet_opt
->opt
, inet
->inet_daddr
, rt
, 0);
386 ip_select_ident_more(iph
, &rt
->dst
, sk
,
387 (skb_shinfo(skb
)->gso_segs
?: 1) - 1);
389 skb
->priority
= sk
->sk_priority
;
390 skb
->mark
= sk
->sk_mark
;
392 res
= ip_local_out(skb
);
398 IP_INC_STATS(sock_net(sk
), IPSTATS_MIB_OUTNOROUTES
);
400 return -EHOSTUNREACH
;
402 EXPORT_SYMBOL(ip_queue_xmit
);
405 static void ip_copy_metadata(struct sk_buff
*to
, struct sk_buff
*from
)
407 to
->pkt_type
= from
->pkt_type
;
408 to
->priority
= from
->priority
;
409 to
->protocol
= from
->protocol
;
411 skb_dst_copy(to
, from
);
413 to
->mark
= from
->mark
;
415 /* Copy the flags to each fragment. */
416 IPCB(to
)->flags
= IPCB(from
)->flags
;
418 #ifdef CONFIG_NET_SCHED
419 to
->tc_index
= from
->tc_index
;
422 #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
423 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
424 to
->nf_trace
= from
->nf_trace
;
426 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
427 to
->ipvs_property
= from
->ipvs_property
;
429 skb_copy_secmark(to
, from
);
433 * This IP datagram is too large to be sent in one piece. Break it up into
434 * smaller pieces (each of size equal to IP header plus
435 * a block of the data of the original IP data part) that will yet fit in a
436 * single device frame, and queue such a frame for sending.
439 int ip_fragment(struct sk_buff
*skb
, int (*output
)(struct sk_buff
*))
443 struct net_device
*dev
;
444 struct sk_buff
*skb2
;
445 unsigned int mtu
, hlen
, left
, len
, ll_rs
;
447 __be16 not_last_frag
;
448 struct rtable
*rt
= skb_rtable(skb
);
454 * Point into the IP datagram header.
459 if (unlikely((iph
->frag_off
& htons(IP_DF
)) && !skb
->local_df
)) {
460 IP_INC_STATS(dev_net(dev
), IPSTATS_MIB_FRAGFAILS
);
461 icmp_send(skb
, ICMP_DEST_UNREACH
, ICMP_FRAG_NEEDED
,
462 htonl(ip_skb_dst_mtu(skb
)));
468 * Setup starting values.
472 mtu
= dst_mtu(&rt
->dst
) - hlen
; /* Size of data space */
473 #ifdef CONFIG_BRIDGE_NETFILTER
475 mtu
-= nf_bridge_mtu_reduction(skb
);
477 IPCB(skb
)->flags
|= IPSKB_FRAG_COMPLETE
;
479 /* When frag_list is given, use it. First, check its validity:
480 * some transformers could create wrong frag_list or break existing
481 * one, it is not prohibited. In this case fall back to copying.
483 * LATER: this step can be merged to real generation of fragments,
484 * we can switch to copy when see the first bad fragment.
486 if (skb_has_frag_list(skb
)) {
487 struct sk_buff
*frag
, *frag2
;
488 int first_len
= skb_pagelen(skb
);
490 if (first_len
- hlen
> mtu
||
491 ((first_len
- hlen
) & 7) ||
492 (iph
->frag_off
& htons(IP_MF
|IP_OFFSET
)) ||
496 skb_walk_frags(skb
, frag
) {
497 /* Correct geometry. */
498 if (frag
->len
> mtu
||
499 ((frag
->len
& 7) && frag
->next
) ||
500 skb_headroom(frag
) < hlen
)
501 goto slow_path_clean
;
503 /* Partially cloned skb? */
504 if (skb_shared(frag
))
505 goto slow_path_clean
;
510 frag
->destructor
= sock_wfree
;
512 skb
->truesize
-= frag
->truesize
;
515 /* Everything is OK. Generate! */
519 frag
= skb_shinfo(skb
)->frag_list
;
520 skb_frag_list_init(skb
);
521 skb
->data_len
= first_len
- skb_headlen(skb
);
522 skb
->len
= first_len
;
523 iph
->tot_len
= htons(first_len
);
524 iph
->frag_off
= htons(IP_MF
);
528 /* Prepare header of the next frame,
529 * before previous one went down. */
531 frag
->ip_summed
= CHECKSUM_NONE
;
532 skb_reset_transport_header(frag
);
533 __skb_push(frag
, hlen
);
534 skb_reset_network_header(frag
);
535 memcpy(skb_network_header(frag
), iph
, hlen
);
537 iph
->tot_len
= htons(frag
->len
);
538 ip_copy_metadata(frag
, skb
);
540 ip_options_fragment(frag
);
541 offset
+= skb
->len
- hlen
;
542 iph
->frag_off
= htons(offset
>>3);
543 if (frag
->next
!= NULL
)
544 iph
->frag_off
|= htons(IP_MF
);
545 /* Ready, complete checksum */
552 IP_INC_STATS(dev_net(dev
), IPSTATS_MIB_FRAGCREATES
);
562 IP_INC_STATS(dev_net(dev
), IPSTATS_MIB_FRAGOKS
);
571 IP_INC_STATS(dev_net(dev
), IPSTATS_MIB_FRAGFAILS
);
575 skb_walk_frags(skb
, frag2
) {
579 frag2
->destructor
= NULL
;
580 skb
->truesize
+= frag2
->truesize
;
585 left
= skb
->len
- hlen
; /* Space per frame */
586 ptr
= hlen
; /* Where to start from */
588 /* for bridged IP traffic encapsulated inside f.e. a vlan header,
589 * we need to make room for the encapsulating header
591 ll_rs
= LL_RESERVED_SPACE_EXTRA(rt
->dst
.dev
, nf_bridge_pad(skb
));
594 * Fragment the datagram.
597 offset
= (ntohs(iph
->frag_off
) & IP_OFFSET
) << 3;
598 not_last_frag
= iph
->frag_off
& htons(IP_MF
);
601 * Keep copying data until we run out.
606 /* IF: it doesn't fit, use 'mtu' - the data space left */
609 /* IF: we are not sending up to and including the packet end
610 then align the next start on an eight byte boundary */
618 if ((skb2
= alloc_skb(len
+hlen
+ll_rs
, GFP_ATOMIC
)) == NULL
) {
619 NETDEBUG(KERN_INFO
"IP: frag: no memory for new fragment!\n");
625 * Set up data on packet
628 ip_copy_metadata(skb2
, skb
);
629 skb_reserve(skb2
, ll_rs
);
630 skb_put(skb2
, len
+ hlen
);
631 skb_reset_network_header(skb2
);
632 skb2
->transport_header
= skb2
->network_header
+ hlen
;
635 * Charge the memory for the fragment to any owner
640 skb_set_owner_w(skb2
, skb
->sk
);
643 * Copy the packet header into the new buffer.
646 skb_copy_from_linear_data(skb
, skb_network_header(skb2
), hlen
);
649 * Copy a block of the IP datagram.
651 if (skb_copy_bits(skb
, ptr
, skb_transport_header(skb2
), len
))
656 * Fill in the new header fields.
659 iph
->frag_off
= htons((offset
>> 3));
661 /* ANK: dirty, but effective trick. Upgrade options only if
662 * the segment to be fragmented was THE FIRST (otherwise,
663 * options are already fixed) and make it ONCE
664 * on the initial skb, so that all the following fragments
665 * will inherit fixed options.
668 ip_options_fragment(skb
);
671 * Added AC : If we are fragmenting a fragment that's not the
672 * last fragment then keep MF on each bit
674 if (left
> 0 || not_last_frag
)
675 iph
->frag_off
|= htons(IP_MF
);
680 * Put this fragment into the sending queue.
682 iph
->tot_len
= htons(len
+ hlen
);
690 IP_INC_STATS(dev_net(dev
), IPSTATS_MIB_FRAGCREATES
);
693 IP_INC_STATS(dev_net(dev
), IPSTATS_MIB_FRAGOKS
);
698 IP_INC_STATS(dev_net(dev
), IPSTATS_MIB_FRAGFAILS
);
701 EXPORT_SYMBOL(ip_fragment
);
704 ip_generic_getfrag(void *from
, char *to
, int offset
, int len
, int odd
, struct sk_buff
*skb
)
706 struct iovec
*iov
= from
;
708 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
709 if (memcpy_fromiovecend(to
, iov
, offset
, len
) < 0)
713 if (csum_partial_copy_fromiovecend(to
, iov
, offset
, len
, &csum
) < 0)
715 skb
->csum
= csum_block_add(skb
->csum
, csum
, odd
);
719 EXPORT_SYMBOL(ip_generic_getfrag
);
722 csum_page(struct page
*page
, int offset
, int copy
)
727 csum
= csum_partial(kaddr
+ offset
, copy
, 0);
732 static inline int ip_ufo_append_data(struct sock
*sk
,
733 struct sk_buff_head
*queue
,
734 int getfrag(void *from
, char *to
, int offset
, int len
,
735 int odd
, struct sk_buff
*skb
),
736 void *from
, int length
, int hh_len
, int fragheaderlen
,
737 int transhdrlen
, int mtu
, unsigned int flags
)
742 /* There is support for UDP fragmentation offload by network
743 * device, so create one single skb packet containing complete
746 if ((skb
= skb_peek_tail(queue
)) == NULL
) {
747 skb
= sock_alloc_send_skb(sk
,
748 hh_len
+ fragheaderlen
+ transhdrlen
+ 20,
749 (flags
& MSG_DONTWAIT
), &err
);
754 /* reserve space for Hardware header */
755 skb_reserve(skb
, hh_len
);
757 /* create space for UDP/IP header */
758 skb_put(skb
, fragheaderlen
+ transhdrlen
);
760 /* initialize network header pointer */
761 skb_reset_network_header(skb
);
763 /* initialize protocol header pointer */
764 skb
->transport_header
= skb
->network_header
+ fragheaderlen
;
766 skb
->ip_summed
= CHECKSUM_PARTIAL
;
769 /* specify the length of each IP datagram fragment */
770 skb_shinfo(skb
)->gso_size
= mtu
- fragheaderlen
;
771 skb_shinfo(skb
)->gso_type
= SKB_GSO_UDP
;
772 __skb_queue_tail(queue
, skb
);
775 return skb_append_datato_frags(sk
, skb
, getfrag
, from
,
776 (length
- transhdrlen
));
779 static int __ip_append_data(struct sock
*sk
, struct sk_buff_head
*queue
,
780 struct inet_cork
*cork
,
781 int getfrag(void *from
, char *to
, int offset
,
782 int len
, int odd
, struct sk_buff
*skb
),
783 void *from
, int length
, int transhdrlen
,
786 struct inet_sock
*inet
= inet_sk(sk
);
789 struct ip_options
*opt
= cork
->opt
;
796 unsigned int maxfraglen
, fragheaderlen
;
797 int csummode
= CHECKSUM_NONE
;
798 struct rtable
*rt
= (struct rtable
*)cork
->dst
;
800 exthdrlen
= transhdrlen
? rt
->dst
.header_len
: 0;
802 transhdrlen
+= exthdrlen
;
803 mtu
= cork
->fragsize
;
805 hh_len
= LL_RESERVED_SPACE(rt
->dst
.dev
);
807 fragheaderlen
= sizeof(struct iphdr
) + (opt
? opt
->optlen
: 0);
808 maxfraglen
= ((mtu
- fragheaderlen
) & ~7) + fragheaderlen
;
810 if (cork
->length
+ length
> 0xFFFF - fragheaderlen
) {
811 ip_local_error(sk
, EMSGSIZE
, rt
->rt_dst
, inet
->inet_dport
,
817 * transhdrlen > 0 means that this is the first fragment and we wish
818 * it won't be fragmented in the future.
821 length
+ fragheaderlen
<= mtu
&&
822 rt
->dst
.dev
->features
& NETIF_F_V4_CSUM
&&
824 csummode
= CHECKSUM_PARTIAL
;
826 skb
= skb_peek_tail(queue
);
828 cork
->length
+= length
;
829 if (((length
> mtu
) || (skb
&& skb_is_gso(skb
))) &&
830 (sk
->sk_protocol
== IPPROTO_UDP
) &&
831 (rt
->dst
.dev
->features
& NETIF_F_UFO
)) {
832 err
= ip_ufo_append_data(sk
, queue
, getfrag
, from
, length
,
833 hh_len
, fragheaderlen
, transhdrlen
,
840 /* So, what's going on in the loop below?
842 * We use calculated fragment length to generate chained skb,
843 * each of segments is IP fragment ready for sending to network after
844 * adding appropriate IP header.
851 /* Check if the remaining data fits into current packet. */
852 copy
= mtu
- skb
->len
;
854 copy
= maxfraglen
- skb
->len
;
857 unsigned int datalen
;
858 unsigned int fraglen
;
859 unsigned int fraggap
;
860 unsigned int alloclen
;
861 struct sk_buff
*skb_prev
;
865 fraggap
= skb_prev
->len
- maxfraglen
;
870 * If remaining data exceeds the mtu,
871 * we know we need more fragment(s).
873 datalen
= length
+ fraggap
;
874 if (datalen
> mtu
- fragheaderlen
)
875 datalen
= maxfraglen
- fragheaderlen
;
876 fraglen
= datalen
+ fragheaderlen
;
878 if ((flags
& MSG_MORE
) &&
879 !(rt
->dst
.dev
->features
&NETIF_F_SG
))
884 /* The last fragment gets additional space at tail.
885 * Note, with MSG_MORE we overallocate on fragments,
886 * because we have no idea what fragment will be
889 if (datalen
== length
+ fraggap
) {
890 alloclen
+= rt
->dst
.trailer_len
;
891 /* make sure mtu is not reached */
892 if (datalen
> mtu
- fragheaderlen
- rt
->dst
.trailer_len
)
893 datalen
-= ALIGN(rt
->dst
.trailer_len
, 8);
896 skb
= sock_alloc_send_skb(sk
,
897 alloclen
+ hh_len
+ 15,
898 (flags
& MSG_DONTWAIT
), &err
);
901 if (atomic_read(&sk
->sk_wmem_alloc
) <=
903 skb
= sock_wmalloc(sk
,
904 alloclen
+ hh_len
+ 15, 1,
906 if (unlikely(skb
== NULL
))
909 /* only the initial fragment is
917 * Fill in the control structures
919 skb
->ip_summed
= csummode
;
921 skb_reserve(skb
, hh_len
);
922 skb_shinfo(skb
)->tx_flags
= cork
->tx_flags
;
925 * Find where to start putting bytes.
927 data
= skb_put(skb
, fraglen
);
928 skb_set_network_header(skb
, exthdrlen
);
929 skb
->transport_header
= (skb
->network_header
+
931 data
+= fragheaderlen
;
934 skb
->csum
= skb_copy_and_csum_bits(
935 skb_prev
, maxfraglen
,
936 data
+ transhdrlen
, fraggap
, 0);
937 skb_prev
->csum
= csum_sub(skb_prev
->csum
,
940 pskb_trim_unique(skb_prev
, maxfraglen
);
943 copy
= datalen
- transhdrlen
- fraggap
;
944 if (copy
> 0 && getfrag(from
, data
+ transhdrlen
, offset
, copy
, fraggap
, skb
) < 0) {
951 length
-= datalen
- fraggap
;
954 csummode
= CHECKSUM_NONE
;
957 * Put the packet on the pending queue.
959 __skb_queue_tail(queue
, skb
);
966 if (!(rt
->dst
.dev
->features
&NETIF_F_SG
)) {
970 if (getfrag(from
, skb_put(skb
, copy
),
971 offset
, copy
, off
, skb
) < 0) {
972 __skb_trim(skb
, off
);
977 int i
= skb_shinfo(skb
)->nr_frags
;
978 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
-1];
979 struct page
*page
= cork
->page
;
983 if (page
&& (left
= PAGE_SIZE
- off
) > 0) {
986 if (page
!= frag
->page
) {
987 if (i
== MAX_SKB_FRAGS
) {
992 skb_fill_page_desc(skb
, i
, page
, off
, 0);
993 frag
= &skb_shinfo(skb
)->frags
[i
];
995 } else if (i
< MAX_SKB_FRAGS
) {
996 if (copy
> PAGE_SIZE
)
998 page
= alloc_pages(sk
->sk_allocation
, 0);
1006 skb_fill_page_desc(skb
, i
, page
, 0, 0);
1007 frag
= &skb_shinfo(skb
)->frags
[i
];
1012 if (getfrag(from
, page_address(frag
->page
)+frag
->page_offset
+frag
->size
, offset
, copy
, skb
->len
, skb
) < 0) {
1019 skb
->data_len
+= copy
;
1020 skb
->truesize
+= copy
;
1021 atomic_add(copy
, &sk
->sk_wmem_alloc
);
1030 cork
->length
-= length
;
1031 IP_INC_STATS(sock_net(sk
), IPSTATS_MIB_OUTDISCARDS
);
1035 static int ip_setup_cork(struct sock
*sk
, struct inet_cork
*cork
,
1036 struct ipcm_cookie
*ipc
, struct rtable
**rtp
)
1038 struct inet_sock
*inet
= inet_sk(sk
);
1039 struct ip_options_rcu
*opt
;
1043 * setup for corking.
1047 if (cork
->opt
== NULL
) {
1048 cork
->opt
= kmalloc(sizeof(struct ip_options
) + 40,
1050 if (unlikely(cork
->opt
== NULL
))
1053 memcpy(cork
->opt
, &opt
->opt
, sizeof(struct ip_options
) + opt
->opt
.optlen
);
1054 cork
->flags
|= IPCORK_OPT
;
1055 cork
->addr
= ipc
->addr
;
1061 * We steal reference to this route, caller should not release it
1064 cork
->fragsize
= inet
->pmtudisc
== IP_PMTUDISC_PROBE
?
1065 rt
->dst
.dev
->mtu
: dst_mtu(rt
->dst
.path
);
1066 cork
->dst
= &rt
->dst
;
1068 cork
->tx_flags
= ipc
->tx_flags
;
1076 * ip_append_data() and ip_append_page() can make one large IP datagram
1077 * from many pieces of data. Each pieces will be holded on the socket
1078 * until ip_push_pending_frames() is called. Each piece can be a page
1081 * Not only UDP, other transport protocols - e.g. raw sockets - can use
1082 * this interface potentially.
1084 * LATER: length must be adjusted by pad at tail, when it is required.
1086 int ip_append_data(struct sock
*sk
,
1087 int getfrag(void *from
, char *to
, int offset
, int len
,
1088 int odd
, struct sk_buff
*skb
),
1089 void *from
, int length
, int transhdrlen
,
1090 struct ipcm_cookie
*ipc
, struct rtable
**rtp
,
1093 struct inet_sock
*inet
= inet_sk(sk
);
1096 if (flags
&MSG_PROBE
)
1099 if (skb_queue_empty(&sk
->sk_write_queue
)) {
1100 err
= ip_setup_cork(sk
, &inet
->cork
.base
, ipc
, rtp
);
1107 return __ip_append_data(sk
, &sk
->sk_write_queue
, &inet
->cork
.base
, getfrag
,
1108 from
, length
, transhdrlen
, flags
);
1111 ssize_t
ip_append_page(struct sock
*sk
, struct page
*page
,
1112 int offset
, size_t size
, int flags
)
1114 struct inet_sock
*inet
= inet_sk(sk
);
1115 struct sk_buff
*skb
;
1117 struct ip_options
*opt
= NULL
;
1118 struct inet_cork
*cork
;
1123 unsigned int maxfraglen
, fragheaderlen
, fraggap
;
1128 if (flags
&MSG_PROBE
)
1131 if (skb_queue_empty(&sk
->sk_write_queue
))
1134 cork
= &inet
->cork
.base
;
1135 rt
= (struct rtable
*)cork
->dst
;
1136 if (cork
->flags
& IPCORK_OPT
)
1139 if (!(rt
->dst
.dev
->features
&NETIF_F_SG
))
1142 hh_len
= LL_RESERVED_SPACE(rt
->dst
.dev
);
1143 mtu
= cork
->fragsize
;
1145 fragheaderlen
= sizeof(struct iphdr
) + (opt
? opt
->optlen
: 0);
1146 maxfraglen
= ((mtu
- fragheaderlen
) & ~7) + fragheaderlen
;
1148 if (cork
->length
+ size
> 0xFFFF - fragheaderlen
) {
1149 ip_local_error(sk
, EMSGSIZE
, rt
->rt_dst
, inet
->inet_dport
, mtu
);
1153 if ((skb
= skb_peek_tail(&sk
->sk_write_queue
)) == NULL
)
1156 cork
->length
+= size
;
1157 if ((size
+ skb
->len
> mtu
) &&
1158 (sk
->sk_protocol
== IPPROTO_UDP
) &&
1159 (rt
->dst
.dev
->features
& NETIF_F_UFO
)) {
1160 skb_shinfo(skb
)->gso_size
= mtu
- fragheaderlen
;
1161 skb_shinfo(skb
)->gso_type
= SKB_GSO_UDP
;
1168 if (skb_is_gso(skb
))
1172 /* Check if the remaining data fits into current packet. */
1173 len
= mtu
- skb
->len
;
1175 len
= maxfraglen
- skb
->len
;
1178 struct sk_buff
*skb_prev
;
1182 fraggap
= skb_prev
->len
- maxfraglen
;
1184 alloclen
= fragheaderlen
+ hh_len
+ fraggap
+ 15;
1185 skb
= sock_wmalloc(sk
, alloclen
, 1, sk
->sk_allocation
);
1186 if (unlikely(!skb
)) {
1192 * Fill in the control structures
1194 skb
->ip_summed
= CHECKSUM_NONE
;
1196 skb_reserve(skb
, hh_len
);
1199 * Find where to start putting bytes.
1201 skb_put(skb
, fragheaderlen
+ fraggap
);
1202 skb_reset_network_header(skb
);
1203 skb
->transport_header
= (skb
->network_header
+
1206 skb
->csum
= skb_copy_and_csum_bits(skb_prev
,
1208 skb_transport_header(skb
),
1210 skb_prev
->csum
= csum_sub(skb_prev
->csum
,
1212 pskb_trim_unique(skb_prev
, maxfraglen
);
1216 * Put the packet on the pending queue.
1218 __skb_queue_tail(&sk
->sk_write_queue
, skb
);
1222 i
= skb_shinfo(skb
)->nr_frags
;
1225 if (skb_can_coalesce(skb
, i
, page
, offset
)) {
1226 skb_shinfo(skb
)->frags
[i
-1].size
+= len
;
1227 } else if (i
< MAX_SKB_FRAGS
) {
1229 skb_fill_page_desc(skb
, i
, page
, offset
, len
);
1235 if (skb
->ip_summed
== CHECKSUM_NONE
) {
1237 csum
= csum_page(page
, offset
, len
);
1238 skb
->csum
= csum_block_add(skb
->csum
, csum
, skb
->len
);
1242 skb
->data_len
+= len
;
1243 skb
->truesize
+= len
;
1244 atomic_add(len
, &sk
->sk_wmem_alloc
);
1251 cork
->length
-= size
;
1252 IP_INC_STATS(sock_net(sk
), IPSTATS_MIB_OUTDISCARDS
);
1256 static void ip_cork_release(struct inet_cork
*cork
)
1258 cork
->flags
&= ~IPCORK_OPT
;
1261 dst_release(cork
->dst
);
1266 * Combined all pending IP fragments on the socket as one IP datagram
1267 * and push them out.
1269 struct sk_buff
*__ip_make_skb(struct sock
*sk
,
1271 struct sk_buff_head
*queue
,
1272 struct inet_cork
*cork
)
1274 struct sk_buff
*skb
, *tmp_skb
;
1275 struct sk_buff
**tail_skb
;
1276 struct inet_sock
*inet
= inet_sk(sk
);
1277 struct net
*net
= sock_net(sk
);
1278 struct ip_options
*opt
= NULL
;
1279 struct rtable
*rt
= (struct rtable
*)cork
->dst
;
1284 if ((skb
= __skb_dequeue(queue
)) == NULL
)
1286 tail_skb
= &(skb_shinfo(skb
)->frag_list
);
1288 /* move skb->data to ip header from ext header */
1289 if (skb
->data
< skb_network_header(skb
))
1290 __skb_pull(skb
, skb_network_offset(skb
));
1291 while ((tmp_skb
= __skb_dequeue(queue
)) != NULL
) {
1292 __skb_pull(tmp_skb
, skb_network_header_len(skb
));
1293 *tail_skb
= tmp_skb
;
1294 tail_skb
= &(tmp_skb
->next
);
1295 skb
->len
+= tmp_skb
->len
;
1296 skb
->data_len
+= tmp_skb
->len
;
1297 skb
->truesize
+= tmp_skb
->truesize
;
1298 tmp_skb
->destructor
= NULL
;
1302 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1303 * to fragment the frame generated here. No matter, what transforms
1304 * how transforms change size of the packet, it will come out.
1306 if (inet
->pmtudisc
< IP_PMTUDISC_DO
)
1309 /* DF bit is set when we want to see DF on outgoing frames.
1310 * If local_df is set too, we still allow to fragment this frame
1312 if (inet
->pmtudisc
>= IP_PMTUDISC_DO
||
1313 (skb
->len
<= dst_mtu(&rt
->dst
) &&
1314 ip_dont_fragment(sk
, &rt
->dst
)))
1317 if (cork
->flags
& IPCORK_OPT
)
1320 if (rt
->rt_type
== RTN_MULTICAST
)
1323 ttl
= ip_select_ttl(inet
, &rt
->dst
);
1325 iph
= (struct iphdr
*)skb
->data
;
1329 iph
->ihl
+= opt
->optlen
>>2;
1330 ip_options_build(skb
, opt
, cork
->addr
, rt
, 0);
1332 iph
->tos
= inet
->tos
;
1334 ip_select_ident(iph
, &rt
->dst
, sk
);
1336 iph
->protocol
= sk
->sk_protocol
;
1337 iph
->saddr
= fl4
->saddr
;
1338 iph
->daddr
= fl4
->daddr
;
1340 skb
->priority
= sk
->sk_priority
;
1341 skb
->mark
= sk
->sk_mark
;
1343 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
1347 skb_dst_set(skb
, &rt
->dst
);
1349 if (iph
->protocol
== IPPROTO_ICMP
)
1350 icmp_out_count(net
, ((struct icmphdr
*)
1351 skb_transport_header(skb
))->type
);
1353 ip_cork_release(cork
);
1358 int ip_send_skb(struct sk_buff
*skb
)
1360 struct net
*net
= sock_net(skb
->sk
);
1363 err
= ip_local_out(skb
);
1366 err
= net_xmit_errno(err
);
1368 IP_INC_STATS(net
, IPSTATS_MIB_OUTDISCARDS
);
1374 int ip_push_pending_frames(struct sock
*sk
, struct flowi4
*fl4
)
1376 struct sk_buff
*skb
;
1378 skb
= ip_finish_skb(sk
, fl4
);
1382 /* Netfilter gets whole the not fragmented skb. */
1383 return ip_send_skb(skb
);
1387 * Throw away all pending data on the socket.
1389 static void __ip_flush_pending_frames(struct sock
*sk
,
1390 struct sk_buff_head
*queue
,
1391 struct inet_cork
*cork
)
1393 struct sk_buff
*skb
;
1395 while ((skb
= __skb_dequeue_tail(queue
)) != NULL
)
1398 ip_cork_release(cork
);
1401 void ip_flush_pending_frames(struct sock
*sk
)
1403 __ip_flush_pending_frames(sk
, &sk
->sk_write_queue
, &inet_sk(sk
)->cork
.base
);
1406 struct sk_buff
*ip_make_skb(struct sock
*sk
,
1408 int getfrag(void *from
, char *to
, int offset
,
1409 int len
, int odd
, struct sk_buff
*skb
),
1410 void *from
, int length
, int transhdrlen
,
1411 struct ipcm_cookie
*ipc
, struct rtable
**rtp
,
1414 struct inet_cork cork
;
1415 struct sk_buff_head queue
;
1418 if (flags
& MSG_PROBE
)
1421 __skb_queue_head_init(&queue
);
1426 err
= ip_setup_cork(sk
, &cork
, ipc
, rtp
);
1428 return ERR_PTR(err
);
1430 err
= __ip_append_data(sk
, &queue
, &cork
, getfrag
,
1431 from
, length
, transhdrlen
, flags
);
1433 __ip_flush_pending_frames(sk
, &queue
, &cork
);
1434 return ERR_PTR(err
);
1437 return __ip_make_skb(sk
, fl4
, &queue
, &cork
);
1441 * Fetch data from kernel space and fill in checksum if needed.
1443 static int ip_reply_glue_bits(void *dptr
, char *to
, int offset
,
1444 int len
, int odd
, struct sk_buff
*skb
)
1448 csum
= csum_partial_copy_nocheck(dptr
+offset
, to
, len
, 0);
1449 skb
->csum
= csum_block_add(skb
->csum
, csum
, odd
);
1454 * Generic function to send a packet as reply to another packet.
1455 * Used to send TCP resets so far. ICMP should use this function too.
1457 * Should run single threaded per socket because it uses the sock
1458 * structure to pass arguments.
1460 void ip_send_reply(struct sock
*sk
, struct sk_buff
*skb
, struct ip_reply_arg
*arg
,
1463 struct inet_sock
*inet
= inet_sk(sk
);
1464 struct ip_options_data replyopts
;
1465 struct ipcm_cookie ipc
;
1468 struct rtable
*rt
= skb_rtable(skb
);
1470 if (ip_options_echo(&replyopts
.opt
.opt
, skb
))
1473 daddr
= ipc
.addr
= rt
->rt_src
;
1477 if (replyopts
.opt
.opt
.optlen
) {
1478 ipc
.opt
= &replyopts
.opt
;
1480 if (replyopts
.opt
.opt
.srr
)
1481 daddr
= replyopts
.opt
.opt
.faddr
;
1484 flowi4_init_output(&fl4
, arg
->bound_dev_if
, 0,
1485 RT_TOS(ip_hdr(skb
)->tos
),
1486 RT_SCOPE_UNIVERSE
, sk
->sk_protocol
,
1487 ip_reply_arg_flowi_flags(arg
),
1488 daddr
, rt
->rt_spec_dst
,
1489 tcp_hdr(skb
)->source
, tcp_hdr(skb
)->dest
);
1490 security_skb_classify_flow(skb
, flowi4_to_flowi(&fl4
));
1491 rt
= ip_route_output_key(sock_net(sk
), &fl4
);
1495 /* And let IP do all the hard work.
1497 This chunk is not reenterable, hence spinlock.
1498 Note that it uses the fact, that this function is called
1499 with locally disabled BH and that sk cannot be already spinlocked.
1502 inet
->tos
= ip_hdr(skb
)->tos
;
1503 sk
->sk_priority
= skb
->priority
;
1504 sk
->sk_protocol
= ip_hdr(skb
)->protocol
;
1505 sk
->sk_bound_dev_if
= arg
->bound_dev_if
;
1506 ip_append_data(sk
, ip_reply_glue_bits
, arg
->iov
->iov_base
, len
, 0,
1507 &ipc
, &rt
, MSG_DONTWAIT
);
1508 if ((skb
= skb_peek(&sk
->sk_write_queue
)) != NULL
) {
1509 if (arg
->csumoffset
>= 0)
1510 *((__sum16
*)skb_transport_header(skb
) +
1511 arg
->csumoffset
) = csum_fold(csum_add(skb
->csum
,
1513 skb
->ip_summed
= CHECKSUM_NONE
;
1514 ip_push_pending_frames(sk
, &fl4
);
1522 void __init
ip_init(void)
1527 #if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1528 igmp_mc_proc_init();