Merge tag 'v3.10.75' into update
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / net / ipv4 / tcp_output.c
CommitLineData
1da177e4
LT
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Implementation of the Transmission Control Protocol(TCP).
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Mark Evans, <evansmp@uhura.aston.ac.uk>
11 * Corey Minyard <wf-rch!minyard@relay.EU.net>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14 * Linus Torvalds, <torvalds@cs.helsinki.fi>
15 * Alan Cox, <gw4pts@gw4pts.ampr.org>
16 * Matthew Dillon, <dillon@apollo.west.oic.com>
17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18 * Jorge Cwik, <jorge@laser.satlink.net>
19 */
20
21/*
22 * Changes: Pedro Roque : Retransmit queue handled by TCP.
23 * : Fragmentation on mtu decrease
24 * : Segment collapse on retransmit
25 * : AF independence
26 *
27 * Linus Torvalds : send_delayed_ack
28 * David S. Miller : Charge memory using the right skb
29 * during syn/ack processing.
30 * David S. Miller : Output engine completely rewritten.
31 * Andrea Arcangeli: SYNACK carry ts_recent in tsecr.
32 * Cacophonix Gaul : draft-minshall-nagle-01
33 * J Hadi Salim : ECN support
34 *
35 */
36
91df42be
JP
37#define pr_fmt(fmt) "TCP: " fmt
38
1da177e4
LT
39#include <net/tcp.h>
40
41#include <linux/compiler.h>
5a0e3ad6 42#include <linux/gfp.h>
1da177e4 43#include <linux/module.h>
1da177e4
LT
44
45/* People can turn this off for buggy TCP's found in printers etc. */
ab32ea5d 46int sysctl_tcp_retrans_collapse __read_mostly = 1;
1da177e4 47
09cb105e 48/* People can turn this on to work with those rare, broken TCPs that
15d99e02
RJ
49 * interpret the window field as a signed quantity.
50 */
ab32ea5d 51int sysctl_tcp_workaround_signed_windows __read_mostly = 0;
15d99e02 52
46d3ceab
ED
53/* Default TSQ limit of two TSO segments */
54int sysctl_tcp_limit_output_bytes __read_mostly = 131072;
55
1da177e4
LT
56/* This limits the percentage of the congestion window which we
57 * will allow a single TSO frame to consume. Building TSO frames
58 * which are too large can cause TCP streams to be bursty.
59 */
ab32ea5d 60int sysctl_tcp_tso_win_divisor __read_mostly = 3;
1da177e4 61
ab32ea5d 62int sysctl_tcp_mtu_probing __read_mostly = 0;
97b1ce25 63int sysctl_tcp_base_mss __read_mostly = TCP_BASE_MSS;
5d424d5a 64
35089bb2 65/* By default, RFC2861 behavior. */
ab32ea5d 66int sysctl_tcp_slow_start_after_idle __read_mostly = 1;
35089bb2 67
46d3ceab
ED
68static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
69 int push_one, gfp_t gfp);
519855c5 70
67edfef7 71/* Account for new data that has been sent to the network. */
cf533ea5 72static void tcp_event_new_data_sent(struct sock *sk, const struct sk_buff *skb)
1da177e4 73{
6ba8a3b1 74 struct inet_connection_sock *icsk = inet_csk(sk);
9e412ba7 75 struct tcp_sock *tp = tcp_sk(sk);
66f5fe62 76 unsigned int prior_packets = tp->packets_out;
9e412ba7 77
fe067e8a 78 tcp_advance_send_head(sk, skb);
1da177e4 79 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
8512430e 80
66f5fe62 81 tp->packets_out += tcp_skb_pcount(skb);
6ba8a3b1 82 if (!prior_packets || icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS ||
6a5dc9e5 83 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
750ea2ba 84 tcp_rearm_rto(sk);
6a5dc9e5 85 }
1da177e4
LT
86}
87
88/* SND.NXT, if window was not shrunk.
89 * If window has been shrunk, what should we make? It is not clear at all.
90 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
91 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
92 * invalid. OK, let's make this for now:
93 */
cf533ea5 94static inline __u32 tcp_acceptable_seq(const struct sock *sk)
1da177e4 95{
cf533ea5 96 const struct tcp_sock *tp = tcp_sk(sk);
9e412ba7 97
90840def 98 if (!before(tcp_wnd_end(tp), tp->snd_nxt))
1da177e4
LT
99 return tp->snd_nxt;
100 else
90840def 101 return tcp_wnd_end(tp);
1da177e4
LT
102}
103
104/* Calculate mss to advertise in SYN segment.
105 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
106 *
107 * 1. It is independent of path mtu.
108 * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
109 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
110 * attached devices, because some buggy hosts are confused by
111 * large MSS.
112 * 4. We do not make 3, we advertise MSS, calculated from first
113 * hop device mtu, but allow to raise it to ip_rt_min_advmss.
114 * This may be overridden via information stored in routing table.
115 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
116 * probably even Jumbo".
117 */
118static __u16 tcp_advertise_mss(struct sock *sk)
119{
120 struct tcp_sock *tp = tcp_sk(sk);
cf533ea5 121 const struct dst_entry *dst = __sk_dst_get(sk);
1da177e4
LT
122 int mss = tp->advmss;
123
0dbaee3b
DM
124 if (dst) {
125 unsigned int metric = dst_metric_advmss(dst);
126
127 if (metric < mss) {
128 mss = metric;
129 tp->advmss = mss;
130 }
1da177e4
LT
131 }
132
133 return (__u16)mss;
134}
135
136/* RFC2861. Reset CWND after idle period longer RTO to "restart window".
137 * This is the first part of cwnd validation mechanism. */
cf533ea5 138static void tcp_cwnd_restart(struct sock *sk, const struct dst_entry *dst)
1da177e4 139{
463c84b9 140 struct tcp_sock *tp = tcp_sk(sk);
1da177e4
LT
141 s32 delta = tcp_time_stamp - tp->lsndtime;
142 u32 restart_cwnd = tcp_init_cwnd(tp, dst);
143 u32 cwnd = tp->snd_cwnd;
144
6687e988 145 tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
1da177e4 146
6687e988 147 tp->snd_ssthresh = tcp_current_ssthresh(sk);
1da177e4
LT
148 restart_cwnd = min(restart_cwnd, cwnd);
149
463c84b9 150 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
1da177e4
LT
151 cwnd >>= 1;
152 tp->snd_cwnd = max(cwnd, restart_cwnd);
153 tp->snd_cwnd_stamp = tcp_time_stamp;
154 tp->snd_cwnd_used = 0;
155}
156
67edfef7 157/* Congestion state accounting after a packet has been sent. */
40efc6fa 158static void tcp_event_data_sent(struct tcp_sock *tp,
cf533ea5 159 struct sock *sk)
1da177e4 160{
463c84b9
ACM
161 struct inet_connection_sock *icsk = inet_csk(sk);
162 const u32 now = tcp_time_stamp;
1da177e4 163
35089bb2
DM
164 if (sysctl_tcp_slow_start_after_idle &&
165 (!tp->packets_out && (s32)(now - tp->lsndtime) > icsk->icsk_rto))
463c84b9 166 tcp_cwnd_restart(sk, __sk_dst_get(sk));
1da177e4
LT
167
168 tp->lsndtime = now;
169
170 /* If it is a reply for ato after last received
171 * packet, enter pingpong mode.
172 */
463c84b9
ACM
173 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
174 icsk->icsk_ack.pingpong = 1;
1da177e4
LT
175}
176
67edfef7 177/* Account for an ACK we sent. */
40efc6fa 178static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts)
1da177e4 179{
463c84b9
ACM
180 tcp_dec_quickack_mode(sk, pkts);
181 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
1da177e4
LT
182}
183
184/* Determine a window scaling and initial window to offer.
185 * Based on the assumption that the given amount of space
186 * will be offered. Store the results in the tp structure.
187 * NOTE: for smooth operation initial space offering should
188 * be a multiple of mss if possible. We assume here that mss >= 1.
189 * This MUST be enforced by all callers.
190 */
191void tcp_select_initial_window(int __space, __u32 mss,
192 __u32 *rcv_wnd, __u32 *window_clamp,
31d12926 193 int wscale_ok, __u8 *rcv_wscale,
194 __u32 init_rcv_wnd)
1da177e4
LT
195{
196 unsigned int space = (__space < 0 ? 0 : __space);
197
198 /* If no clamp set the clamp to the max possible scaled window */
199 if (*window_clamp == 0)
200 (*window_clamp) = (65535 << 14);
201 space = min(*window_clamp, space);
202
203 /* Quantize space offering to a multiple of mss if possible. */
204 if (space > mss)
205 space = (space / mss) * mss;
206
207 /* NOTE: offering an initial window larger than 32767
15d99e02
RJ
208 * will break some buggy TCP stacks. If the admin tells us
209 * it is likely we could be speaking with such a buggy stack
210 * we will truncate our initial window offering to 32K-1
211 * unless the remote has sent us a window scaling option,
212 * which we interpret as a sign the remote TCP is not
213 * misinterpreting the window field as a signed quantity.
1da177e4 214 */
15d99e02
RJ
215 if (sysctl_tcp_workaround_signed_windows)
216 (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
217 else
218 (*rcv_wnd) = space;
219
1da177e4
LT
220 (*rcv_wscale) = 0;
221 if (wscale_ok) {
222 /* Set window scaling on max possible window
e905a9ed 223 * See RFC1323 for an explanation of the limit to 14
1da177e4
LT
224 */
225 space = max_t(u32, sysctl_tcp_rmem[2], sysctl_rmem_max);
316c1592 226 space = min_t(u32, space, *window_clamp);
1da177e4
LT
227 while (space > 65535 && (*rcv_wscale) < 14) {
228 space >>= 1;
229 (*rcv_wscale)++;
230 }
231 }
232
356f0398 233 /* Set initial window to a value enough for senders starting with
6fa3eb70 234 * initial congestion window of sysctl_tcp_default_init_rwnd. Place
356f0398
ND
235 * a limit on the initial window when mss is larger than 1460.
236 */
056834d9 237 if (mss > (1 << *rcv_wscale)) {
6fa3eb70 238 int init_cwnd = sysctl_tcp_default_init_rwnd;
356f0398 239 if (mss > 1460)
6fa3eb70 240 init_cwnd = max_t(u32, (1460 * init_cwnd) / mss, 2);
31d12926 241 /* when initializing use the value from init_rcv_wnd
242 * rather than the default from above
243 */
b1afde60
ND
244 if (init_rcv_wnd)
245 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
246 else
247 *rcv_wnd = min(*rcv_wnd, init_cwnd * mss);
1da177e4
LT
248 }
249
250 /* Set the clamp no higher than max representable value */
251 (*window_clamp) = min(65535U << (*rcv_wscale), *window_clamp);
252}
4bc2f18b 253EXPORT_SYMBOL(tcp_select_initial_window);
1da177e4
LT
254
255/* Chose a new window to advertise, update state in tcp_sock for the
256 * socket, and return result with RFC1323 scaling applied. The return
257 * value can be stuffed directly into th->window for an outgoing
258 * frame.
259 */
40efc6fa 260static u16 tcp_select_window(struct sock *sk)
1da177e4
LT
261{
262 struct tcp_sock *tp = tcp_sk(sk);
263 u32 cur_win = tcp_receive_window(tp);
264 u32 new_win = __tcp_select_window(sk);
265
266 /* Never shrink the offered window */
2de979bd 267 if (new_win < cur_win) {
1da177e4
LT
268 /* Danger Will Robinson!
269 * Don't update rcv_wup/rcv_wnd here or else
270 * we will not be able to advertise a zero
271 * window in time. --DaveM
272 *
273 * Relax Will Robinson.
274 */
607bfbf2 275 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
1da177e4
LT
276 }
277 tp->rcv_wnd = new_win;
278 tp->rcv_wup = tp->rcv_nxt;
279
280 /* Make sure we do not exceed the maximum possible
281 * scaled window.
282 */
15d99e02 283 if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows)
1da177e4
LT
284 new_win = min(new_win, MAX_TCP_WINDOW);
285 else
286 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
287
288 /* RFC1323 scaling applied */
289 new_win >>= tp->rx_opt.rcv_wscale;
290
291 /* If we advertise zero window, disable fast path. */
292 if (new_win == 0)
293 tp->pred_flags = 0;
294
295 return new_win;
296}
297
67edfef7 298/* Packet ECN state for a SYN-ACK */
cf533ea5 299static inline void TCP_ECN_send_synack(const struct tcp_sock *tp, struct sk_buff *skb)
bdf1ee5d 300{
4de075e0 301 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
056834d9 302 if (!(tp->ecn_flags & TCP_ECN_OK))
4de075e0 303 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
bdf1ee5d
IJ
304}
305
67edfef7 306/* Packet ECN state for a SYN. */
bdf1ee5d
IJ
307static inline void TCP_ECN_send_syn(struct sock *sk, struct sk_buff *skb)
308{
309 struct tcp_sock *tp = tcp_sk(sk);
310
311 tp->ecn_flags = 0;
5d134f1c 312 if (sock_net(sk)->ipv4.sysctl_tcp_ecn == 1) {
4de075e0 313 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
bdf1ee5d
IJ
314 tp->ecn_flags = TCP_ECN_OK;
315 }
316}
317
318static __inline__ void
cf533ea5 319TCP_ECN_make_synack(const struct request_sock *req, struct tcphdr *th)
bdf1ee5d
IJ
320{
321 if (inet_rsk(req)->ecn_ok)
322 th->ece = 1;
323}
324
67edfef7
AK
325/* Set up ECN state for a packet on a ESTABLISHED socket that is about to
326 * be sent.
327 */
bdf1ee5d
IJ
328static inline void TCP_ECN_send(struct sock *sk, struct sk_buff *skb,
329 int tcp_header_len)
330{
331 struct tcp_sock *tp = tcp_sk(sk);
332
333 if (tp->ecn_flags & TCP_ECN_OK) {
334 /* Not-retransmitted data segment: set ECT and inject CWR. */
335 if (skb->len != tcp_header_len &&
336 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
337 INET_ECN_xmit(sk);
056834d9 338 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
bdf1ee5d
IJ
339 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
340 tcp_hdr(skb)->cwr = 1;
341 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
342 }
343 } else {
344 /* ACK or retransmitted segment: clear ECT|CE */
345 INET_ECN_dontxmit(sk);
346 }
347 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
348 tcp_hdr(skb)->ece = 1;
349 }
350}
351
e870a8ef
IJ
352/* Constructs common control bits of non-data skb. If SYN/FIN is present,
353 * auto increment end seqno.
354 */
355static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
356{
2e8e18ef 357 skb->ip_summed = CHECKSUM_PARTIAL;
e870a8ef
IJ
358 skb->csum = 0;
359
4de075e0 360 TCP_SKB_CB(skb)->tcp_flags = flags;
e870a8ef
IJ
361 TCP_SKB_CB(skb)->sacked = 0;
362
363 skb_shinfo(skb)->gso_segs = 1;
364 skb_shinfo(skb)->gso_size = 0;
365 skb_shinfo(skb)->gso_type = 0;
366
367 TCP_SKB_CB(skb)->seq = seq;
a3433f35 368 if (flags & (TCPHDR_SYN | TCPHDR_FIN))
e870a8ef
IJ
369 seq++;
370 TCP_SKB_CB(skb)->end_seq = seq;
371}
372
a2a385d6 373static inline bool tcp_urg_mode(const struct tcp_sock *tp)
33f5f57e
IJ
374{
375 return tp->snd_una != tp->snd_up;
376}
377
33ad798c
AL
378#define OPTION_SACK_ADVERTISE (1 << 0)
379#define OPTION_TS (1 << 1)
380#define OPTION_MD5 (1 << 2)
89e95a61 381#define OPTION_WSCALE (1 << 3)
2100c8d2 382#define OPTION_FAST_OPEN_COOKIE (1 << 8)
33ad798c
AL
383
384struct tcp_out_options {
2100c8d2
YC
385 u16 options; /* bit field of OPTION_* */
386 u16 mss; /* 0 to disable */
33ad798c
AL
387 u8 ws; /* window scale, 0 to disable */
388 u8 num_sack_blocks; /* number of SACK blocks to include */
bd0388ae 389 u8 hash_size; /* bytes in hash_location */
bd0388ae 390 __u8 *hash_location; /* temporary pointer, overloaded */
2100c8d2
YC
391 __u32 tsval, tsecr; /* need to include OPTION_TS */
392 struct tcp_fastopen_cookie *fastopen_cookie; /* Fast open cookie */
33ad798c
AL
393};
394
67edfef7
AK
395/* Write previously computed TCP options to the packet.
396 *
397 * Beware: Something in the Internet is very sensitive to the ordering of
fd6149d3
IJ
398 * TCP options, we learned this through the hard way, so be careful here.
399 * Luckily we can at least blame others for their non-compliance but from
400 * inter-operatibility perspective it seems that we're somewhat stuck with
401 * the ordering which we have been using if we want to keep working with
402 * those broken things (not that it currently hurts anybody as there isn't
403 * particular reason why the ordering would need to be changed).
404 *
405 * At least SACK_PERM as the first option is known to lead to a disaster
406 * (but it may well be that other scenarios fail similarly).
407 */
33ad798c 408static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
bd0388ae
WAS
409 struct tcp_out_options *opts)
410{
2100c8d2 411 u16 options = opts->options; /* mungable copy */
bd0388ae 412
bd0388ae 413 if (unlikely(OPTION_MD5 & options)) {
1a2c6181
CP
414 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
415 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
bd0388ae
WAS
416 /* overload cookie hash location */
417 opts->hash_location = (__u8 *)ptr;
33ad798c 418 ptr += 4;
40efc6fa 419 }
33ad798c 420
fd6149d3
IJ
421 if (unlikely(opts->mss)) {
422 *ptr++ = htonl((TCPOPT_MSS << 24) |
423 (TCPOLEN_MSS << 16) |
424 opts->mss);
425 }
426
bd0388ae
WAS
427 if (likely(OPTION_TS & options)) {
428 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
33ad798c
AL
429 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
430 (TCPOLEN_SACK_PERM << 16) |
431 (TCPOPT_TIMESTAMP << 8) |
432 TCPOLEN_TIMESTAMP);
bd0388ae 433 options &= ~OPTION_SACK_ADVERTISE;
33ad798c
AL
434 } else {
435 *ptr++ = htonl((TCPOPT_NOP << 24) |
436 (TCPOPT_NOP << 16) |
437 (TCPOPT_TIMESTAMP << 8) |
438 TCPOLEN_TIMESTAMP);
439 }
440 *ptr++ = htonl(opts->tsval);
441 *ptr++ = htonl(opts->tsecr);
442 }
443
bd0388ae 444 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
33ad798c
AL
445 *ptr++ = htonl((TCPOPT_NOP << 24) |
446 (TCPOPT_NOP << 16) |
447 (TCPOPT_SACK_PERM << 8) |
448 TCPOLEN_SACK_PERM);
449 }
450
bd0388ae 451 if (unlikely(OPTION_WSCALE & options)) {
33ad798c
AL
452 *ptr++ = htonl((TCPOPT_NOP << 24) |
453 (TCPOPT_WINDOW << 16) |
454 (TCPOLEN_WINDOW << 8) |
455 opts->ws);
456 }
457
458 if (unlikely(opts->num_sack_blocks)) {
459 struct tcp_sack_block *sp = tp->rx_opt.dsack ?
460 tp->duplicate_sack : tp->selective_acks;
40efc6fa
SH
461 int this_sack;
462
463 *ptr++ = htonl((TCPOPT_NOP << 24) |
464 (TCPOPT_NOP << 16) |
465 (TCPOPT_SACK << 8) |
33ad798c 466 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
40efc6fa 467 TCPOLEN_SACK_PERBLOCK)));
2de979bd 468
33ad798c
AL
469 for (this_sack = 0; this_sack < opts->num_sack_blocks;
470 ++this_sack) {
40efc6fa
SH
471 *ptr++ = htonl(sp[this_sack].start_seq);
472 *ptr++ = htonl(sp[this_sack].end_seq);
473 }
2de979bd 474
5861f8e5 475 tp->rx_opt.dsack = 0;
40efc6fa 476 }
2100c8d2
YC
477
478 if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
479 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
480
481 *ptr++ = htonl((TCPOPT_EXP << 24) |
482 ((TCPOLEN_EXP_FASTOPEN_BASE + foc->len) << 16) |
483 TCPOPT_FASTOPEN_MAGIC);
484
485 memcpy(ptr, foc->val, foc->len);
486 if ((foc->len & 3) == 2) {
487 u8 *align = ((u8 *)ptr) + foc->len;
488 align[0] = align[1] = TCPOPT_NOP;
489 }
490 ptr += (foc->len + 3) >> 2;
491 }
33ad798c
AL
492}
493
67edfef7
AK
494/* Compute TCP options for SYN packets. This is not the final
495 * network wire format yet.
496 */
95c96174 497static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
33ad798c 498 struct tcp_out_options *opts,
cf533ea5
ED
499 struct tcp_md5sig_key **md5)
500{
33ad798c 501 struct tcp_sock *tp = tcp_sk(sk);
95c96174 502 unsigned int remaining = MAX_TCP_OPTION_SPACE;
783237e8 503 struct tcp_fastopen_request *fastopen = tp->fastopen_req;
33ad798c 504
cfb6eeb4 505#ifdef CONFIG_TCP_MD5SIG
33ad798c
AL
506 *md5 = tp->af_specific->md5_lookup(sk, sk);
507 if (*md5) {
508 opts->options |= OPTION_MD5;
bd0388ae 509 remaining -= TCPOLEN_MD5SIG_ALIGNED;
cfb6eeb4 510 }
33ad798c
AL
511#else
512 *md5 = NULL;
cfb6eeb4 513#endif
33ad798c
AL
514
515 /* We always get an MSS option. The option bytes which will be seen in
516 * normal data packets should timestamps be used, must be in the MSS
517 * advertised. But we subtract them from tp->mss_cache so that
518 * calculations in tcp_sendmsg are simpler etc. So account for this
519 * fact here if necessary. If we don't do this correctly, as a
520 * receiver we won't recognize data packets as being full sized when we
521 * should, and thus we won't abide by the delayed ACK rules correctly.
522 * SACKs don't matter, we never delay an ACK when we have any of those
523 * going out. */
524 opts->mss = tcp_advertise_mss(sk);
bd0388ae 525 remaining -= TCPOLEN_MSS_ALIGNED;
33ad798c 526
bb5b7c11 527 if (likely(sysctl_tcp_timestamps && *md5 == NULL)) {
33ad798c 528 opts->options |= OPTION_TS;
ee684b6f 529 opts->tsval = TCP_SKB_CB(skb)->when + tp->tsoffset;
33ad798c 530 opts->tsecr = tp->rx_opt.ts_recent;
bd0388ae 531 remaining -= TCPOLEN_TSTAMP_ALIGNED;
33ad798c 532 }
bb5b7c11 533 if (likely(sysctl_tcp_window_scaling)) {
33ad798c 534 opts->ws = tp->rx_opt.rcv_wscale;
89e95a61 535 opts->options |= OPTION_WSCALE;
bd0388ae 536 remaining -= TCPOLEN_WSCALE_ALIGNED;
33ad798c 537 }
bb5b7c11 538 if (likely(sysctl_tcp_sack)) {
33ad798c 539 opts->options |= OPTION_SACK_ADVERTISE;
b32d1310 540 if (unlikely(!(OPTION_TS & opts->options)))
bd0388ae 541 remaining -= TCPOLEN_SACKPERM_ALIGNED;
33ad798c
AL
542 }
543
783237e8
YC
544 if (fastopen && fastopen->cookie.len >= 0) {
545 u32 need = TCPOLEN_EXP_FASTOPEN_BASE + fastopen->cookie.len;
546 need = (need + 3) & ~3U; /* Align to 32 bits */
547 if (remaining >= need) {
548 opts->options |= OPTION_FAST_OPEN_COOKIE;
549 opts->fastopen_cookie = &fastopen->cookie;
550 remaining -= need;
551 tp->syn_fastopen = 1;
552 }
553 }
bd0388ae 554
bd0388ae 555 return MAX_TCP_OPTION_SPACE - remaining;
40efc6fa
SH
556}
557
67edfef7 558/* Set up TCP options for SYN-ACKs. */
95c96174 559static unsigned int tcp_synack_options(struct sock *sk,
33ad798c 560 struct request_sock *req,
95c96174 561 unsigned int mss, struct sk_buff *skb,
33ad798c 562 struct tcp_out_options *opts,
4957faad 563 struct tcp_md5sig_key **md5,
8336886f 564 struct tcp_fastopen_cookie *foc)
4957faad 565{
33ad798c 566 struct inet_request_sock *ireq = inet_rsk(req);
95c96174 567 unsigned int remaining = MAX_TCP_OPTION_SPACE;
33ad798c 568
cfb6eeb4 569#ifdef CONFIG_TCP_MD5SIG
33ad798c
AL
570 *md5 = tcp_rsk(req)->af_specific->md5_lookup(sk, req);
571 if (*md5) {
572 opts->options |= OPTION_MD5;
4957faad
WAS
573 remaining -= TCPOLEN_MD5SIG_ALIGNED;
574
575 /* We can't fit any SACK blocks in a packet with MD5 + TS
576 * options. There was discussion about disabling SACK
577 * rather than TS in order to fit in better with old,
578 * buggy kernels, but that was deemed to be unnecessary.
579 */
de213e5e 580 ireq->tstamp_ok &= !ireq->sack_ok;
cfb6eeb4 581 }
33ad798c
AL
582#else
583 *md5 = NULL;
cfb6eeb4 584#endif
33ad798c 585
4957faad 586 /* We always send an MSS option. */
33ad798c 587 opts->mss = mss;
4957faad 588 remaining -= TCPOLEN_MSS_ALIGNED;
33ad798c
AL
589
590 if (likely(ireq->wscale_ok)) {
591 opts->ws = ireq->rcv_wscale;
89e95a61 592 opts->options |= OPTION_WSCALE;
4957faad 593 remaining -= TCPOLEN_WSCALE_ALIGNED;
33ad798c 594 }
de213e5e 595 if (likely(ireq->tstamp_ok)) {
33ad798c
AL
596 opts->options |= OPTION_TS;
597 opts->tsval = TCP_SKB_CB(skb)->when;
598 opts->tsecr = req->ts_recent;
4957faad 599 remaining -= TCPOLEN_TSTAMP_ALIGNED;
33ad798c
AL
600 }
601 if (likely(ireq->sack_ok)) {
602 opts->options |= OPTION_SACK_ADVERTISE;
de213e5e 603 if (unlikely(!ireq->tstamp_ok))
4957faad 604 remaining -= TCPOLEN_SACKPERM_ALIGNED;
33ad798c 605 }
8336886f
JC
606 if (foc != NULL) {
607 u32 need = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
608 need = (need + 3) & ~3U; /* Align to 32 bits */
609 if (remaining >= need) {
610 opts->options |= OPTION_FAST_OPEN_COOKIE;
611 opts->fastopen_cookie = foc;
612 remaining -= need;
613 }
614 }
1a2c6181 615
4957faad 616 return MAX_TCP_OPTION_SPACE - remaining;
33ad798c
AL
617}
618
67edfef7
AK
619/* Compute TCP options for ESTABLISHED sockets. This is not the
620 * final wire format yet.
621 */
95c96174 622static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
33ad798c 623 struct tcp_out_options *opts,
cf533ea5
ED
624 struct tcp_md5sig_key **md5)
625{
33ad798c
AL
626 struct tcp_skb_cb *tcb = skb ? TCP_SKB_CB(skb) : NULL;
627 struct tcp_sock *tp = tcp_sk(sk);
95c96174 628 unsigned int size = 0;
cabeccbd 629 unsigned int eff_sacks;
33ad798c
AL
630
631#ifdef CONFIG_TCP_MD5SIG
632 *md5 = tp->af_specific->md5_lookup(sk, sk);
633 if (unlikely(*md5)) {
634 opts->options |= OPTION_MD5;
635 size += TCPOLEN_MD5SIG_ALIGNED;
636 }
637#else
638 *md5 = NULL;
639#endif
640
641 if (likely(tp->rx_opt.tstamp_ok)) {
642 opts->options |= OPTION_TS;
ee684b6f 643 opts->tsval = tcb ? tcb->when + tp->tsoffset : 0;
33ad798c
AL
644 opts->tsecr = tp->rx_opt.ts_recent;
645 size += TCPOLEN_TSTAMP_ALIGNED;
646 }
647
cabeccbd
IJ
648 eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
649 if (unlikely(eff_sacks)) {
95c96174 650 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
33ad798c 651 opts->num_sack_blocks =
95c96174 652 min_t(unsigned int, eff_sacks,
33ad798c
AL
653 (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
654 TCPOLEN_SACK_PERBLOCK);
655 size += TCPOLEN_SACK_BASE_ALIGNED +
656 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
657 }
658
659 return size;
40efc6fa 660}
1da177e4 661
46d3ceab
ED
662
663/* TCP SMALL QUEUES (TSQ)
664 *
665 * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
666 * to reduce RTT and bufferbloat.
667 * We do this using a special skb destructor (tcp_wfree).
668 *
669 * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
670 * needs to be reallocated in a driver.
671 * The invariant being skb->truesize substracted from sk->sk_wmem_alloc
672 *
673 * Since transmit from skb destructor is forbidden, we use a tasklet
674 * to process all sockets that eventually need to send more skbs.
675 * We use one tasklet per cpu, with its own queue of sockets.
676 */
677struct tsq_tasklet {
678 struct tasklet_struct tasklet;
679 struct list_head head; /* queue of tcp sockets */
680};
681static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
682
6f458dfb
ED
683static void tcp_tsq_handler(struct sock *sk)
684{
685 if ((1 << sk->sk_state) &
686 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
687 TCPF_CLOSE_WAIT | TCPF_LAST_ACK))
94ee16ae
JO
688 tcp_write_xmit(sk, tcp_current_mss(sk), tcp_sk(sk)->nonagle,
689 0, GFP_ATOMIC);
6f458dfb 690}
46d3ceab
ED
691/*
692 * One tasklest per cpu tries to send more skbs.
693 * We run in tasklet context but need to disable irqs when
694 * transfering tsq->head because tcp_wfree() might
695 * interrupt us (non NAPI drivers)
696 */
697static void tcp_tasklet_func(unsigned long data)
698{
699 struct tsq_tasklet *tsq = (struct tsq_tasklet *)data;
700 LIST_HEAD(list);
701 unsigned long flags;
702 struct list_head *q, *n;
703 struct tcp_sock *tp;
704 struct sock *sk;
705
706 local_irq_save(flags);
707 list_splice_init(&tsq->head, &list);
708 local_irq_restore(flags);
709
710 list_for_each_safe(q, n, &list) {
711 tp = list_entry(q, struct tcp_sock, tsq_node);
712 list_del(&tp->tsq_node);
713
714 sk = (struct sock *)tp;
715 bh_lock_sock(sk);
716
717 if (!sock_owned_by_user(sk)) {
6f458dfb 718 tcp_tsq_handler(sk);
46d3ceab
ED
719 } else {
720 /* defer the work to tcp_release_cb() */
6f458dfb 721 set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
46d3ceab
ED
722 }
723 bh_unlock_sock(sk);
724
725 clear_bit(TSQ_QUEUED, &tp->tsq_flags);
726 sk_free(sk);
727 }
728}
729
6f458dfb
ED
730#define TCP_DEFERRED_ALL ((1UL << TCP_TSQ_DEFERRED) | \
731 (1UL << TCP_WRITE_TIMER_DEFERRED) | \
563d34d0
ED
732 (1UL << TCP_DELACK_TIMER_DEFERRED) | \
733 (1UL << TCP_MTU_REDUCED_DEFERRED))
46d3ceab
ED
734/**
735 * tcp_release_cb - tcp release_sock() callback
736 * @sk: socket
737 *
738 * called from release_sock() to perform protocol dependent
739 * actions before socket release.
740 */
741void tcp_release_cb(struct sock *sk)
742{
743 struct tcp_sock *tp = tcp_sk(sk);
6f458dfb 744 unsigned long flags, nflags;
46d3ceab 745
6f458dfb
ED
746 /* perform an atomic operation only if at least one flag is set */
747 do {
748 flags = tp->tsq_flags;
749 if (!(flags & TCP_DEFERRED_ALL))
750 return;
751 nflags = flags & ~TCP_DEFERRED_ALL;
752 } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags);
753
754 if (flags & (1UL << TCP_TSQ_DEFERRED))
755 tcp_tsq_handler(sk);
756
cbbb5a25
ED
757 /* Here begins the tricky part :
758 * We are called from release_sock() with :
759 * 1) BH disabled
760 * 2) sk_lock.slock spinlock held
761 * 3) socket owned by us (sk->sk_lock.owned == 1)
762 *
763 * But following code is meant to be called from BH handlers,
764 * so we should keep BH disabled, but early release socket ownership
765 */
766 sock_release_ownership(sk);
767
144d56e9 768 if (flags & (1UL << TCP_WRITE_TIMER_DEFERRED)) {
6f458dfb 769 tcp_write_timer_handler(sk);
144d56e9
ED
770 __sock_put(sk);
771 }
772 if (flags & (1UL << TCP_DELACK_TIMER_DEFERRED)) {
6f458dfb 773 tcp_delack_timer_handler(sk);
144d56e9
ED
774 __sock_put(sk);
775 }
776 if (flags & (1UL << TCP_MTU_REDUCED_DEFERRED)) {
5f80f4d8 777 inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
144d56e9
ED
778 __sock_put(sk);
779 }
46d3ceab
ED
780}
781EXPORT_SYMBOL(tcp_release_cb);
782
783void __init tcp_tasklet_init(void)
784{
785 int i;
786
787 for_each_possible_cpu(i) {
788 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
789
790 INIT_LIST_HEAD(&tsq->head);
791 tasklet_init(&tsq->tasklet,
792 tcp_tasklet_func,
793 (unsigned long)tsq);
794 }
795}
796
797/*
798 * Write buffer destructor automatically called from kfree_skb.
799 * We cant xmit new skbs from this context, as we might already
800 * hold qdisc lock.
801 */
d6a4a104 802void tcp_wfree(struct sk_buff *skb)
46d3ceab
ED
803{
804 struct sock *sk = skb->sk;
805 struct tcp_sock *tp = tcp_sk(sk);
806
807 if (test_and_clear_bit(TSQ_THROTTLED, &tp->tsq_flags) &&
808 !test_and_set_bit(TSQ_QUEUED, &tp->tsq_flags)) {
809 unsigned long flags;
810 struct tsq_tasklet *tsq;
811
812 /* Keep a ref on socket.
813 * This last ref will be released in tcp_tasklet_func()
814 */
815 atomic_sub(skb->truesize - 1, &sk->sk_wmem_alloc);
816
817 /* queue this socket to tasklet queue */
818 local_irq_save(flags);
819 tsq = &__get_cpu_var(tsq_tasklet);
820 list_add(&tp->tsq_node, &tsq->head);
821 tasklet_schedule(&tsq->tasklet);
822 local_irq_restore(flags);
823 } else {
824 sock_wfree(skb);
825 }
826}
827
1da177e4
LT
828/* This routine actually transmits TCP packets queued in by
829 * tcp_do_sendmsg(). This is used by both the initial
830 * transmission and possible later retransmissions.
831 * All SKB's seen here are completely headerless. It is our
832 * job to build the TCP header, and pass the packet down to
833 * IP so it can do the same plus pass the packet off to the
834 * device.
835 *
836 * We are working here with either a clone of the original
837 * SKB, or a fresh unique copy made by the retransmit engine.
838 */
056834d9
IJ
839static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
840 gfp_t gfp_mask)
1da177e4 841{
dfb4b9dc
DM
842 const struct inet_connection_sock *icsk = inet_csk(sk);
843 struct inet_sock *inet;
844 struct tcp_sock *tp;
845 struct tcp_skb_cb *tcb;
33ad798c 846 struct tcp_out_options opts;
95c96174 847 unsigned int tcp_options_size, tcp_header_size;
cfb6eeb4 848 struct tcp_md5sig_key *md5;
dfb4b9dc 849 struct tcphdr *th;
dfb4b9dc
DM
850 int err;
851
852 BUG_ON(!skb || !tcp_skb_pcount(skb));
853
854 /* If congestion control is doing timestamping, we must
855 * take such a timestamp before we potentially clone/copy.
856 */
164891aa 857 if (icsk->icsk_ca_ops->flags & TCP_CONG_RTT_STAMP)
dfb4b9dc
DM
858 __net_timestamp(skb);
859
860 if (likely(clone_it)) {
0e280af0
ED
861 const struct sk_buff *fclone = skb + 1;
862
863 if (unlikely(skb->fclone == SKB_FCLONE_ORIG &&
864 fclone->fclone == SKB_FCLONE_CLONE))
865 NET_INC_STATS_BH(sock_net(sk),
866 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
867
dfb4b9dc
DM
868 if (unlikely(skb_cloned(skb)))
869 skb = pskb_copy(skb, gfp_mask);
870 else
871 skb = skb_clone(skb, gfp_mask);
872 if (unlikely(!skb))
873 return -ENOBUFS;
874 }
1da177e4 875
dfb4b9dc
DM
876 inet = inet_sk(sk);
877 tp = tcp_sk(sk);
878 tcb = TCP_SKB_CB(skb);
33ad798c 879 memset(&opts, 0, sizeof(opts));
1da177e4 880
4de075e0 881 if (unlikely(tcb->tcp_flags & TCPHDR_SYN))
33ad798c
AL
882 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
883 else
884 tcp_options_size = tcp_established_options(sk, skb, &opts,
885 &md5);
886 tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
e905a9ed 887
547669d4 888 if (tcp_packets_in_flight(tp) == 0)
dfb4b9dc 889 tcp_ca_event(sk, CA_EVENT_TX_START);
547669d4
ED
890
891 /* if no packet is in qdisc/device queue, then allow XPS to select
892 * another queue.
893 */
894 skb->ooo_okay = sk_wmem_alloc_get(sk) == 0;
dfb4b9dc 895
aa8223c7
ACM
896 skb_push(skb, tcp_header_size);
897 skb_reset_transport_header(skb);
46d3ceab
ED
898
899 skb_orphan(skb);
900 skb->sk = sk;
0ae5f47e 901 skb->destructor = tcp_wfree;
46d3ceab 902 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
dfb4b9dc
DM
903
904 /* Build TCP header and checksum it. */
aa8223c7 905 th = tcp_hdr(skb);
c720c7e8
ED
906 th->source = inet->inet_sport;
907 th->dest = inet->inet_dport;
dfb4b9dc
DM
908 th->seq = htonl(tcb->seq);
909 th->ack_seq = htonl(tp->rcv_nxt);
df7a3b07 910 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) |
4de075e0 911 tcb->tcp_flags);
dfb4b9dc 912
4de075e0 913 if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
dfb4b9dc
DM
914 /* RFC1323: The window in SYN & SYN/ACK segments
915 * is never scaled.
916 */
600ff0c2 917 th->window = htons(min(tp->rcv_wnd, 65535U));
dfb4b9dc
DM
918 } else {
919 th->window = htons(tcp_select_window(sk));
920 }
921 th->check = 0;
922 th->urg_ptr = 0;
1da177e4 923
33f5f57e 924 /* The urg_mode check is necessary during a below snd_una win probe */
7691367d
HX
925 if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
926 if (before(tp->snd_up, tcb->seq + 0x10000)) {
927 th->urg_ptr = htons(tp->snd_up - tcb->seq);
928 th->urg = 1;
929 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
0eae88f3 930 th->urg_ptr = htons(0xFFFF);
7691367d
HX
931 th->urg = 1;
932 }
dfb4b9dc 933 }
1da177e4 934
bd0388ae 935 tcp_options_write((__be32 *)(th + 1), tp, &opts);
4de075e0 936 if (likely((tcb->tcp_flags & TCPHDR_SYN) == 0))
9e412ba7 937 TCP_ECN_send(sk, skb, tcp_header_size);
1da177e4 938
cfb6eeb4
YH
939#ifdef CONFIG_TCP_MD5SIG
940 /* Calculate the MD5 hash, as we have all we need now */
941 if (md5) {
a465419b 942 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
bd0388ae 943 tp->af_specific->calc_md5_hash(opts.hash_location,
49a72dfb 944 md5, sk, NULL, skb);
cfb6eeb4
YH
945 }
946#endif
947
bb296246 948 icsk->icsk_af_ops->send_check(sk, skb);
1da177e4 949
4de075e0 950 if (likely(tcb->tcp_flags & TCPHDR_ACK))
dfb4b9dc 951 tcp_event_ack_sent(sk, tcp_skb_pcount(skb));
1da177e4 952
dfb4b9dc 953 if (skb->len != tcp_header_size)
cf533ea5 954 tcp_event_data_sent(tp, sk);
1da177e4 955
bd37a088 956 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
aa2ea058
TH
957 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
958 tcp_skb_pcount(skb));
1da177e4 959
d9d8da80 960 err = icsk->icsk_af_ops->queue_xmit(skb, &inet->cork.fl);
83de47cd 961 if (likely(err <= 0))
dfb4b9dc
DM
962 return err;
963
3cfe3baa 964 tcp_enter_cwr(sk, 1);
dfb4b9dc 965
b9df3cb8 966 return net_xmit_eval(err);
1da177e4
LT
967}
968
67edfef7 969/* This routine just queues the buffer for sending.
1da177e4
LT
970 *
971 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
972 * otherwise socket can stall.
973 */
974static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
975{
976 struct tcp_sock *tp = tcp_sk(sk);
977
978 /* Advance write_seq and place onto the write_queue. */
979 tp->write_seq = TCP_SKB_CB(skb)->end_seq;
980 skb_header_release(skb);
fe067e8a 981 tcp_add_write_queue_tail(sk, skb);
3ab224be
HA
982 sk->sk_wmem_queued += skb->truesize;
983 sk_mem_charge(sk, skb->truesize);
1da177e4
LT
984}
985
67edfef7 986/* Initialize TSO segments for a packet. */
cf533ea5 987static void tcp_set_skb_tso_segs(const struct sock *sk, struct sk_buff *skb,
056834d9 988 unsigned int mss_now)
f6302d1d 989{
b81908e1
ED
990 /* Make sure we own this skb before messing gso_size/gso_segs */
991 WARN_ON_ONCE(skb_cloned(skb));
992
8e5b9dda
HX
993 if (skb->len <= mss_now || !sk_can_gso(sk) ||
994 skb->ip_summed == CHECKSUM_NONE) {
f6302d1d
DM
995 /* Avoid the costly divide in the normal
996 * non-TSO case.
997 */
7967168c
HX
998 skb_shinfo(skb)->gso_segs = 1;
999 skb_shinfo(skb)->gso_size = 0;
c9af6db4 1000 skb_shinfo(skb)->gso_type = 0;
f6302d1d 1001 } else {
356f89e1 1002 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss_now);
7967168c 1003 skb_shinfo(skb)->gso_size = mss_now;
c9af6db4 1004 skb_shinfo(skb)->gso_type = sk->sk_gso_type;
1da177e4
LT
1005 }
1006}
1007
91fed7a1 1008/* When a modification to fackets out becomes necessary, we need to check
68f8353b 1009 * skb is counted to fackets_out or not.
91fed7a1 1010 */
cf533ea5 1011static void tcp_adjust_fackets_out(struct sock *sk, const struct sk_buff *skb,
91fed7a1
IJ
1012 int decr)
1013{
a47e5a98
IJ
1014 struct tcp_sock *tp = tcp_sk(sk);
1015
dc86967b 1016 if (!tp->sacked_out || tcp_is_reno(tp))
91fed7a1
IJ
1017 return;
1018
6859d494 1019 if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq))
91fed7a1 1020 tp->fackets_out -= decr;
91fed7a1
IJ
1021}
1022
797108d1
IJ
1023/* Pcount in the middle of the write queue got changed, we need to do various
1024 * tweaks to fix counters
1025 */
cf533ea5 1026static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
797108d1
IJ
1027{
1028 struct tcp_sock *tp = tcp_sk(sk);
1029
1030 tp->packets_out -= decr;
1031
1032 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1033 tp->sacked_out -= decr;
1034 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1035 tp->retrans_out -= decr;
1036 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1037 tp->lost_out -= decr;
1038
1039 /* Reno case is special. Sigh... */
1040 if (tcp_is_reno(tp) && decr > 0)
1041 tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1042
1043 tcp_adjust_fackets_out(sk, skb, decr);
1044
1045 if (tp->lost_skb_hint &&
1046 before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
52cf3cc8 1047 (tcp_is_fack(tp) || (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)))
797108d1
IJ
1048 tp->lost_cnt_hint -= decr;
1049
1050 tcp_verify_left_out(tp);
1051}
1052
1da177e4
LT
1053/* Function to create two new TCP segments. Shrinks the given segment
1054 * to the specified size and appends a new segment with the rest of the
e905a9ed 1055 * packet to the list. This won't be called frequently, I hope.
1da177e4
LT
1056 * Remember, these are still headerless SKBs at this point.
1057 */
056834d9
IJ
1058int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
1059 unsigned int mss_now)
1da177e4
LT
1060{
1061 struct tcp_sock *tp = tcp_sk(sk);
1062 struct sk_buff *buff;
6475be16 1063 int nsize, old_factor;
b60b49ea 1064 int nlen;
9ce01461 1065 u8 flags;
1da177e4 1066
2fceec13
IJ
1067 if (WARN_ON(len > skb->len))
1068 return -EINVAL;
6a438bbe 1069
1da177e4
LT
1070 nsize = skb_headlen(skb) - len;
1071 if (nsize < 0)
1072 nsize = 0;
1073
b81908e1 1074 if (skb_unclone(skb, GFP_ATOMIC))
1da177e4
LT
1075 return -ENOMEM;
1076
1077 /* Get a new skb... force flag on. */
1078 buff = sk_stream_alloc_skb(sk, nsize, GFP_ATOMIC);
1079 if (buff == NULL)
1080 return -ENOMEM; /* We'll just try again later. */
ef5cb973 1081
3ab224be
HA
1082 sk->sk_wmem_queued += buff->truesize;
1083 sk_mem_charge(sk, buff->truesize);
b60b49ea
HX
1084 nlen = skb->len - len - nsize;
1085 buff->truesize += nlen;
1086 skb->truesize -= nlen;
1da177e4
LT
1087
1088 /* Correct the sequence numbers. */
1089 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1090 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1091 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1092
1093 /* PSH and FIN should only be set in the second packet. */
4de075e0
ED
1094 flags = TCP_SKB_CB(skb)->tcp_flags;
1095 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1096 TCP_SKB_CB(buff)->tcp_flags = flags;
e14c3caf 1097 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1da177e4 1098
84fa7933 1099 if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) {
1da177e4 1100 /* Copy and checksum data tail into the new buffer. */
056834d9
IJ
1101 buff->csum = csum_partial_copy_nocheck(skb->data + len,
1102 skb_put(buff, nsize),
1da177e4
LT
1103 nsize, 0);
1104
1105 skb_trim(skb, len);
1106
1107 skb->csum = csum_block_sub(skb->csum, buff->csum, len);
1108 } else {
84fa7933 1109 skb->ip_summed = CHECKSUM_PARTIAL;
1da177e4
LT
1110 skb_split(skb, buff, len);
1111 }
1112
1113 buff->ip_summed = skb->ip_summed;
1114
1115 /* Looks stupid, but our code really uses when of
1116 * skbs, which it never sent before. --ANK
1117 */
1118 TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when;
a61bbcf2 1119 buff->tstamp = skb->tstamp;
1da177e4 1120
6475be16
DM
1121 old_factor = tcp_skb_pcount(skb);
1122
1da177e4 1123 /* Fix up tso_factor for both original and new SKB. */
846998ae
DM
1124 tcp_set_skb_tso_segs(sk, skb, mss_now);
1125 tcp_set_skb_tso_segs(sk, buff, mss_now);
1da177e4 1126
6475be16
DM
1127 /* If this packet has been sent out already, we must
1128 * adjust the various packet counters.
1129 */
cf0b450c 1130 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
6475be16
DM
1131 int diff = old_factor - tcp_skb_pcount(skb) -
1132 tcp_skb_pcount(buff);
1da177e4 1133
797108d1
IJ
1134 if (diff)
1135 tcp_adjust_pcount(sk, skb, diff);
1da177e4
LT
1136 }
1137
1138 /* Link BUFF into the send queue. */
f44b5271 1139 skb_header_release(buff);
fe067e8a 1140 tcp_insert_write_queue_after(skb, buff, sk);
1da177e4
LT
1141
1142 return 0;
1143}
1144
1145/* This is similar to __pskb_pull_head() (it will go to core/skbuff.c
1146 * eventually). The difference is that pulled data not copied, but
1147 * immediately discarded.
1148 */
f2911969 1149static void __pskb_trim_head(struct sk_buff *skb, int len)
1da177e4
LT
1150{
1151 int i, k, eat;
1152
4fa48bf3
ED
1153 eat = min_t(int, len, skb_headlen(skb));
1154 if (eat) {
1155 __skb_pull(skb, eat);
1156 len -= eat;
1157 if (!len)
1158 return;
1159 }
1da177e4
LT
1160 eat = len;
1161 k = 0;
056834d9 1162 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
9e903e08
ED
1163 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1164
1165 if (size <= eat) {
aff65da0 1166 skb_frag_unref(skb, i);
9e903e08 1167 eat -= size;
1da177e4
LT
1168 } else {
1169 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1170 if (eat) {
1171 skb_shinfo(skb)->frags[k].page_offset += eat;
9e903e08 1172 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1da177e4
LT
1173 eat = 0;
1174 }
1175 k++;
1176 }
1177 }
1178 skb_shinfo(skb)->nr_frags = k;
1179
27a884dc 1180 skb_reset_tail_pointer(skb);
1da177e4
LT
1181 skb->data_len -= len;
1182 skb->len = skb->data_len;
1da177e4
LT
1183}
1184
67edfef7 1185/* Remove acked data from a packet in the transmit queue. */
1da177e4
LT
1186int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1187{
14bbd6a5 1188 if (skb_unclone(skb, GFP_ATOMIC))
1da177e4
LT
1189 return -ENOMEM;
1190
4fa48bf3 1191 __pskb_trim_head(skb, len);
1da177e4
LT
1192
1193 TCP_SKB_CB(skb)->seq += len;
84fa7933 1194 skb->ip_summed = CHECKSUM_PARTIAL;
1da177e4
LT
1195
1196 skb->truesize -= len;
1197 sk->sk_wmem_queued -= len;
3ab224be 1198 sk_mem_uncharge(sk, len);
1da177e4
LT
1199 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1200
5b35e1e6 1201 /* Any change of skb->len requires recalculation of tso factor. */
1da177e4 1202 if (tcp_skb_pcount(skb) > 1)
5b35e1e6 1203 tcp_set_skb_tso_segs(sk, skb, tcp_skb_mss(skb));
1da177e4
LT
1204
1205 return 0;
1206}
1207
1b63edd6
YC
1208/* Calculate MSS not accounting any TCP options. */
1209static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
5d424d5a 1210{
cf533ea5
ED
1211 const struct tcp_sock *tp = tcp_sk(sk);
1212 const struct inet_connection_sock *icsk = inet_csk(sk);
5d424d5a
JH
1213 int mss_now;
1214
1215 /* Calculate base mss without TCP options:
1216 It is MMS_S - sizeof(tcphdr) of rfc1122
1217 */
1218 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1219
67469601
ED
1220 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1221 if (icsk->icsk_af_ops->net_frag_header_len) {
1222 const struct dst_entry *dst = __sk_dst_get(sk);
1223
1224 if (dst && dst_allfrag(dst))
1225 mss_now -= icsk->icsk_af_ops->net_frag_header_len;
1226 }
1227
5d424d5a
JH
1228 /* Clamp it (mss_clamp does not include tcp options) */
1229 if (mss_now > tp->rx_opt.mss_clamp)
1230 mss_now = tp->rx_opt.mss_clamp;
1231
1232 /* Now subtract optional transport overhead */
1233 mss_now -= icsk->icsk_ext_hdr_len;
1234
1235 /* Then reserve room for full set of TCP options and 8 bytes of data */
1236 if (mss_now < 48)
1237 mss_now = 48;
5d424d5a
JH
1238 return mss_now;
1239}
1240
1b63edd6
YC
1241/* Calculate MSS. Not accounting for SACKs here. */
1242int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1243{
1244 /* Subtract TCP options size, not including SACKs */
1245 return __tcp_mtu_to_mss(sk, pmtu) -
1246 (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1247}
1248
5d424d5a 1249/* Inverse of above */
67469601 1250int tcp_mss_to_mtu(struct sock *sk, int mss)
5d424d5a 1251{
cf533ea5
ED
1252 const struct tcp_sock *tp = tcp_sk(sk);
1253 const struct inet_connection_sock *icsk = inet_csk(sk);
5d424d5a
JH
1254 int mtu;
1255
1256 mtu = mss +
1257 tp->tcp_header_len +
1258 icsk->icsk_ext_hdr_len +
1259 icsk->icsk_af_ops->net_header_len;
1260
67469601
ED
1261 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1262 if (icsk->icsk_af_ops->net_frag_header_len) {
1263 const struct dst_entry *dst = __sk_dst_get(sk);
1264
1265 if (dst && dst_allfrag(dst))
1266 mtu += icsk->icsk_af_ops->net_frag_header_len;
1267 }
5d424d5a
JH
1268 return mtu;
1269}
1270
67edfef7 1271/* MTU probing init per socket */
5d424d5a
JH
1272void tcp_mtup_init(struct sock *sk)
1273{
1274 struct tcp_sock *tp = tcp_sk(sk);
1275 struct inet_connection_sock *icsk = inet_csk(sk);
1276
1277 icsk->icsk_mtup.enabled = sysctl_tcp_mtu_probing > 1;
1278 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
e905a9ed 1279 icsk->icsk_af_ops->net_header_len;
5d424d5a
JH
1280 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, sysctl_tcp_base_mss);
1281 icsk->icsk_mtup.probe_size = 0;
1282}
4bc2f18b 1283EXPORT_SYMBOL(tcp_mtup_init);
5d424d5a 1284
1da177e4
LT
1285/* This function synchronize snd mss to current pmtu/exthdr set.
1286
1287 tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1288 for TCP options, but includes only bare TCP header.
1289
1290 tp->rx_opt.mss_clamp is mss negotiated at connection setup.
caa20d9a 1291 It is minimum of user_mss and mss received with SYN.
1da177e4
LT
1292 It also does not include TCP options.
1293
d83d8461 1294 inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1da177e4
LT
1295
1296 tp->mss_cache is current effective sending mss, including
1297 all tcp options except for SACKs. It is evaluated,
1298 taking into account current pmtu, but never exceeds
1299 tp->rx_opt.mss_clamp.
1300
1301 NOTE1. rfc1122 clearly states that advertised MSS
1302 DOES NOT include either tcp or ip options.
1303
d83d8461
ACM
1304 NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1305 are READ ONLY outside this function. --ANK (980731)
1da177e4 1306 */
1da177e4
LT
1307unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1308{
1309 struct tcp_sock *tp = tcp_sk(sk);
d83d8461 1310 struct inet_connection_sock *icsk = inet_csk(sk);
5d424d5a 1311 int mss_now;
1da177e4 1312
5d424d5a
JH
1313 if (icsk->icsk_mtup.search_high > pmtu)
1314 icsk->icsk_mtup.search_high = pmtu;
1da177e4 1315
5d424d5a 1316 mss_now = tcp_mtu_to_mss(sk, pmtu);
409d22b4 1317 mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1da177e4
LT
1318
1319 /* And store cached results */
d83d8461 1320 icsk->icsk_pmtu_cookie = pmtu;
5d424d5a
JH
1321 if (icsk->icsk_mtup.enabled)
1322 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
c1b4a7e6 1323 tp->mss_cache = mss_now;
1da177e4
LT
1324
1325 return mss_now;
1326}
4bc2f18b 1327EXPORT_SYMBOL(tcp_sync_mss);
1da177e4
LT
1328
1329/* Compute the current effective MSS, taking SACKs and IP options,
1330 * and even PMTU discovery events into account.
1da177e4 1331 */
0c54b85f 1332unsigned int tcp_current_mss(struct sock *sk)
1da177e4 1333{
cf533ea5
ED
1334 const struct tcp_sock *tp = tcp_sk(sk);
1335 const struct dst_entry *dst = __sk_dst_get(sk);
c1b4a7e6 1336 u32 mss_now;
95c96174 1337 unsigned int header_len;
33ad798c
AL
1338 struct tcp_out_options opts;
1339 struct tcp_md5sig_key *md5;
c1b4a7e6
DM
1340
1341 mss_now = tp->mss_cache;
1342
1da177e4
LT
1343 if (dst) {
1344 u32 mtu = dst_mtu(dst);
d83d8461 1345 if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1da177e4
LT
1346 mss_now = tcp_sync_mss(sk, mtu);
1347 }
1348
33ad798c
AL
1349 header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1350 sizeof(struct tcphdr);
1351 /* The mss_cache is sized based on tp->tcp_header_len, which assumes
1352 * some common options. If this is an odd packet (because we have SACK
1353 * blocks etc) then our calculated header_len will be different, and
1354 * we have to adjust mss_now correspondingly */
1355 if (header_len != tp->tcp_header_len) {
1356 int delta = (int) header_len - tp->tcp_header_len;
1357 mss_now -= delta;
1358 }
cfb6eeb4 1359
1da177e4
LT
1360 return mss_now;
1361}
1362
a762a980 1363/* Congestion window validation. (RFC2861) */
9e412ba7 1364static void tcp_cwnd_validate(struct sock *sk)
a762a980 1365{
9e412ba7 1366 struct tcp_sock *tp = tcp_sk(sk);
a762a980 1367
d436d686 1368 if (tp->packets_out >= tp->snd_cwnd) {
a762a980
DM
1369 /* Network is feed fully. */
1370 tp->snd_cwnd_used = 0;
1371 tp->snd_cwnd_stamp = tcp_time_stamp;
1372 } else {
1373 /* Network starves. */
1374 if (tp->packets_out > tp->snd_cwnd_used)
1375 tp->snd_cwnd_used = tp->packets_out;
1376
15d33c07
DM
1377 if (sysctl_tcp_slow_start_after_idle &&
1378 (s32)(tcp_time_stamp - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto)
a762a980
DM
1379 tcp_cwnd_application_limited(sk);
1380 }
1381}
1382
0e3a4803
IJ
1383/* Returns the portion of skb which can be sent right away without
1384 * introducing MSS oddities to segment boundaries. In rare cases where
1385 * mss_now != mss_cache, we will request caller to create a small skb
1386 * per input skb which could be mostly avoided here (if desired).
5ea3a748
IJ
1387 *
1388 * We explicitly want to create a request for splitting write queue tail
1389 * to a small skb for Nagle purposes while avoiding unnecessary modulos,
1390 * thus all the complexity (cwnd_len is always MSS multiple which we
1391 * return whenever allowed by the other factors). Basically we need the
1392 * modulo only when the receiver window alone is the limiting factor or
1393 * when we would be allowed to send the split-due-to-Nagle skb fully.
0e3a4803 1394 */
cf533ea5 1395static unsigned int tcp_mss_split_point(const struct sock *sk, const struct sk_buff *skb,
1485348d 1396 unsigned int mss_now, unsigned int max_segs)
c1b4a7e6 1397{
cf533ea5 1398 const struct tcp_sock *tp = tcp_sk(sk);
1485348d 1399 u32 needed, window, max_len;
c1b4a7e6 1400
90840def 1401 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1485348d 1402 max_len = mss_now * max_segs;
0e3a4803 1403
1485348d
BH
1404 if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
1405 return max_len;
0e3a4803 1406
5ea3a748
IJ
1407 needed = min(skb->len, window);
1408
1485348d
BH
1409 if (max_len <= needed)
1410 return max_len;
0e3a4803 1411
0e3a4803 1412 return needed - needed % mss_now;
c1b4a7e6
DM
1413}
1414
1415/* Can at least one segment of SKB be sent right now, according to the
1416 * congestion window rules? If so, return how many segments are allowed.
1417 */
cf533ea5
ED
1418static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
1419 const struct sk_buff *skb)
c1b4a7e6
DM
1420{
1421 u32 in_flight, cwnd;
1422
1423 /* Don't be strict about the congestion window for the final FIN. */
4de075e0
ED
1424 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
1425 tcp_skb_pcount(skb) == 1)
c1b4a7e6
DM
1426 return 1;
1427
1428 in_flight = tcp_packets_in_flight(tp);
1429 cwnd = tp->snd_cwnd;
1430 if (in_flight < cwnd)
1431 return (cwnd - in_flight);
1432
1433 return 0;
1434}
1435
b595076a 1436/* Initialize TSO state of a skb.
67edfef7 1437 * This must be invoked the first time we consider transmitting
c1b4a7e6
DM
1438 * SKB onto the wire.
1439 */
cf533ea5 1440static int tcp_init_tso_segs(const struct sock *sk, struct sk_buff *skb,
056834d9 1441 unsigned int mss_now)
c1b4a7e6
DM
1442{
1443 int tso_segs = tcp_skb_pcount(skb);
1444
f8269a49 1445 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
846998ae 1446 tcp_set_skb_tso_segs(sk, skb, mss_now);
c1b4a7e6
DM
1447 tso_segs = tcp_skb_pcount(skb);
1448 }
1449 return tso_segs;
1450}
1451
67edfef7 1452/* Minshall's variant of the Nagle send check. */
a2a385d6 1453static inline bool tcp_minshall_check(const struct tcp_sock *tp)
c1b4a7e6 1454{
09cb105e 1455 return after(tp->snd_sml, tp->snd_una) &&
c1b4a7e6
DM
1456 !after(tp->snd_sml, tp->snd_nxt);
1457}
1458
a2a385d6 1459/* Return false, if packet can be sent now without violation Nagle's rules:
c1b4a7e6
DM
1460 * 1. It is full sized.
1461 * 2. Or it contains FIN. (already checked by caller)
6d67e9be 1462 * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
c1b4a7e6
DM
1463 * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1464 * With Minshall's modification: all sent small packets are ACKed.
1465 */
a2a385d6 1466static inline bool tcp_nagle_check(const struct tcp_sock *tp,
e905a9ed 1467 const struct sk_buff *skb,
95c96174 1468 unsigned int mss_now, int nonagle)
c1b4a7e6 1469{
a02cec21 1470 return skb->len < mss_now &&
056834d9 1471 ((nonagle & TCP_NAGLE_CORK) ||
a02cec21 1472 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
c1b4a7e6
DM
1473}
1474
a2a385d6 1475/* Return true if the Nagle test allows this packet to be
c1b4a7e6
DM
1476 * sent now.
1477 */
a2a385d6
ED
1478static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
1479 unsigned int cur_mss, int nonagle)
c1b4a7e6
DM
1480{
1481 /* Nagle rule does not apply to frames, which sit in the middle of the
1482 * write_queue (they have no chances to get new data).
1483 *
1484 * This is implemented in the callers, where they modify the 'nonagle'
1485 * argument based upon the location of SKB in the send queue.
1486 */
1487 if (nonagle & TCP_NAGLE_PUSH)
a2a385d6 1488 return true;
c1b4a7e6 1489
9b44190d
YC
1490 /* Don't use the nagle rule for urgent data (or for the final FIN). */
1491 if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
a2a385d6 1492 return true;
c1b4a7e6
DM
1493
1494 if (!tcp_nagle_check(tp, skb, cur_mss, nonagle))
a2a385d6 1495 return true;
c1b4a7e6 1496
a2a385d6 1497 return false;
c1b4a7e6
DM
1498}
1499
1500/* Does at least the first segment of SKB fit into the send window? */
a2a385d6
ED
1501static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
1502 const struct sk_buff *skb,
1503 unsigned int cur_mss)
c1b4a7e6
DM
1504{
1505 u32 end_seq = TCP_SKB_CB(skb)->end_seq;
1506
1507 if (skb->len > cur_mss)
1508 end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
1509
90840def 1510 return !after(end_seq, tcp_wnd_end(tp));
c1b4a7e6
DM
1511}
1512
fe067e8a 1513/* This checks if the data bearing packet SKB (usually tcp_send_head(sk))
c1b4a7e6
DM
1514 * should be put on the wire right now. If so, it returns the number of
1515 * packets allowed by the congestion window.
1516 */
cf533ea5 1517static unsigned int tcp_snd_test(const struct sock *sk, struct sk_buff *skb,
c1b4a7e6
DM
1518 unsigned int cur_mss, int nonagle)
1519{
cf533ea5 1520 const struct tcp_sock *tp = tcp_sk(sk);
c1b4a7e6
DM
1521 unsigned int cwnd_quota;
1522
846998ae 1523 tcp_init_tso_segs(sk, skb, cur_mss);
c1b4a7e6
DM
1524
1525 if (!tcp_nagle_test(tp, skb, cur_mss, nonagle))
1526 return 0;
1527
1528 cwnd_quota = tcp_cwnd_test(tp, skb);
056834d9 1529 if (cwnd_quota && !tcp_snd_wnd_test(tp, skb, cur_mss))
c1b4a7e6
DM
1530 cwnd_quota = 0;
1531
1532 return cwnd_quota;
1533}
1534
67edfef7 1535/* Test if sending is allowed right now. */
a2a385d6 1536bool tcp_may_send_now(struct sock *sk)
c1b4a7e6 1537{
cf533ea5 1538 const struct tcp_sock *tp = tcp_sk(sk);
fe067e8a 1539 struct sk_buff *skb = tcp_send_head(sk);
c1b4a7e6 1540
a02cec21 1541 return skb &&
0c54b85f 1542 tcp_snd_test(sk, skb, tcp_current_mss(sk),
c1b4a7e6 1543 (tcp_skb_is_last(sk, skb) ?
a02cec21 1544 tp->nonagle : TCP_NAGLE_PUSH));
c1b4a7e6
DM
1545}
1546
1547/* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
1548 * which is put after SKB on the list. It is very much like
1549 * tcp_fragment() except that it may make several kinds of assumptions
1550 * in order to speed up the splitting operation. In particular, we
1551 * know that all the data is in scatter-gather pages, and that the
1552 * packet has never been sent out before (and thus is not cloned).
1553 */
056834d9 1554static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
c4ead4c5 1555 unsigned int mss_now, gfp_t gfp)
c1b4a7e6
DM
1556{
1557 struct sk_buff *buff;
1558 int nlen = skb->len - len;
9ce01461 1559 u8 flags;
c1b4a7e6
DM
1560
1561 /* All of a TSO frame must be composed of paged data. */
c8ac3774
HX
1562 if (skb->len != skb->data_len)
1563 return tcp_fragment(sk, skb, len, mss_now);
c1b4a7e6 1564
c4ead4c5 1565 buff = sk_stream_alloc_skb(sk, 0, gfp);
c1b4a7e6
DM
1566 if (unlikely(buff == NULL))
1567 return -ENOMEM;
1568
3ab224be
HA
1569 sk->sk_wmem_queued += buff->truesize;
1570 sk_mem_charge(sk, buff->truesize);
b60b49ea 1571 buff->truesize += nlen;
c1b4a7e6
DM
1572 skb->truesize -= nlen;
1573
1574 /* Correct the sequence numbers. */
1575 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1576 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1577 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1578
1579 /* PSH and FIN should only be set in the second packet. */
4de075e0
ED
1580 flags = TCP_SKB_CB(skb)->tcp_flags;
1581 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1582 TCP_SKB_CB(buff)->tcp_flags = flags;
c1b4a7e6
DM
1583
1584 /* This packet was never sent out yet, so no SACK bits. */
1585 TCP_SKB_CB(buff)->sacked = 0;
1586
84fa7933 1587 buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL;
c1b4a7e6
DM
1588 skb_split(skb, buff, len);
1589
1590 /* Fix up tso_factor for both original and new SKB. */
846998ae
DM
1591 tcp_set_skb_tso_segs(sk, skb, mss_now);
1592 tcp_set_skb_tso_segs(sk, buff, mss_now);
c1b4a7e6
DM
1593
1594 /* Link BUFF into the send queue. */
1595 skb_header_release(buff);
fe067e8a 1596 tcp_insert_write_queue_after(skb, buff, sk);
c1b4a7e6
DM
1597
1598 return 0;
1599}
1600
1601/* Try to defer sending, if possible, in order to minimize the amount
1602 * of TSO splitting we do. View it as a kind of TSO Nagle test.
1603 *
1604 * This algorithm is from John Heffner.
1605 */
a2a385d6 1606static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
c1b4a7e6 1607{
9e412ba7 1608 struct tcp_sock *tp = tcp_sk(sk);
6687e988 1609 const struct inet_connection_sock *icsk = inet_csk(sk);
c1b4a7e6 1610 u32 send_win, cong_win, limit, in_flight;
ad9f4f50 1611 int win_divisor;
c1b4a7e6 1612
4de075e0 1613 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
ae8064ac 1614 goto send_now;
c1b4a7e6 1615
6687e988 1616 if (icsk->icsk_ca_state != TCP_CA_Open)
ae8064ac
JH
1617 goto send_now;
1618
1619 /* Defer for less than two clock ticks. */
bd515c3e 1620 if (tp->tso_deferred &&
a2acde07 1621 (((u32)jiffies << 1) >> 1) - (tp->tso_deferred >> 1) > 1)
ae8064ac 1622 goto send_now;
908a75c1 1623
c1b4a7e6
DM
1624 in_flight = tcp_packets_in_flight(tp);
1625
056834d9 1626 BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));
c1b4a7e6 1627
90840def 1628 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
c1b4a7e6
DM
1629
1630 /* From in_flight test above, we know that cwnd > in_flight. */
1631 cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
1632
1633 limit = min(send_win, cong_win);
1634
ba244fe9 1635 /* If a full-sized TSO skb can be sent, do it. */
1485348d 1636 if (limit >= min_t(unsigned int, sk->sk_gso_max_size,
5e25ba50 1637 tp->xmit_size_goal_segs * tp->mss_cache))
ae8064ac 1638 goto send_now;
ba244fe9 1639
62ad2761
IJ
1640 /* Middle in queue won't get any more data, full sendable already? */
1641 if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
1642 goto send_now;
1643
ad9f4f50
ED
1644 win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor);
1645 if (win_divisor) {
c1b4a7e6
DM
1646 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
1647
1648 /* If at least some fraction of a window is available,
1649 * just use it.
1650 */
ad9f4f50 1651 chunk /= win_divisor;
c1b4a7e6 1652 if (limit >= chunk)
ae8064ac 1653 goto send_now;
c1b4a7e6
DM
1654 } else {
1655 /* Different approach, try not to defer past a single
1656 * ACK. Receiver should ACK every other full sized
1657 * frame, so if we have space for more than 3 frames
1658 * then send now.
1659 */
6b5a5c0d 1660 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
ae8064ac 1661 goto send_now;
c1b4a7e6
DM
1662 }
1663
f4541d60
ED
1664 /* Ok, it looks like it is advisable to defer.
1665 * Do not rearm the timer if already set to not break TCP ACK clocking.
1666 */
1667 if (!tp->tso_deferred)
1668 tp->tso_deferred = 1 | (jiffies << 1);
ae8064ac 1669
a2a385d6 1670 return true;
ae8064ac
JH
1671
1672send_now:
1673 tp->tso_deferred = 0;
a2a385d6 1674 return false;
c1b4a7e6
DM
1675}
1676
5d424d5a 1677/* Create a new MTU probe if we are ready.
67edfef7
AK
1678 * MTU probe is regularly attempting to increase the path MTU by
1679 * deliberately sending larger packets. This discovers routing
1680 * changes resulting in larger path MTUs.
1681 *
5d424d5a
JH
1682 * Returns 0 if we should wait to probe (no cwnd available),
1683 * 1 if a probe was sent,
056834d9
IJ
1684 * -1 otherwise
1685 */
5d424d5a
JH
1686static int tcp_mtu_probe(struct sock *sk)
1687{
1688 struct tcp_sock *tp = tcp_sk(sk);
1689 struct inet_connection_sock *icsk = inet_csk(sk);
1690 struct sk_buff *skb, *nskb, *next;
1691 int len;
1692 int probe_size;
91cc17c0 1693 int size_needed;
5d424d5a
JH
1694 int copy;
1695 int mss_now;
1696
1697 /* Not currently probing/verifying,
1698 * not in recovery,
1699 * have enough cwnd, and
1700 * not SACKing (the variable headers throw things off) */
1701 if (!icsk->icsk_mtup.enabled ||
1702 icsk->icsk_mtup.probe_size ||
1703 inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
1704 tp->snd_cwnd < 11 ||
cabeccbd 1705 tp->rx_opt.num_sacks || tp->rx_opt.dsack)
5d424d5a
JH
1706 return -1;
1707
1708 /* Very simple search strategy: just double the MSS. */
0c54b85f 1709 mss_now = tcp_current_mss(sk);
056834d9 1710 probe_size = 2 * tp->mss_cache;
91cc17c0 1711 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
5d424d5a
JH
1712 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high)) {
1713 /* TODO: set timer for probe_converge_event */
1714 return -1;
1715 }
1716
1717 /* Have enough data in the send queue to probe? */
7f9c33e5 1718 if (tp->write_seq - tp->snd_nxt < size_needed)
5d424d5a
JH
1719 return -1;
1720
91cc17c0
IJ
1721 if (tp->snd_wnd < size_needed)
1722 return -1;
90840def 1723 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
91cc17c0 1724 return 0;
5d424d5a 1725
d67c58e9
IJ
1726 /* Do we need to wait to drain cwnd? With none in flight, don't stall */
1727 if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {
1728 if (!tcp_packets_in_flight(tp))
5d424d5a
JH
1729 return -1;
1730 else
1731 return 0;
1732 }
1733
1734 /* We're allowed to probe. Build it now. */
1735 if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL)
1736 return -1;
3ab224be
HA
1737 sk->sk_wmem_queued += nskb->truesize;
1738 sk_mem_charge(sk, nskb->truesize);
5d424d5a 1739
fe067e8a 1740 skb = tcp_send_head(sk);
5d424d5a
JH
1741
1742 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
1743 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
4de075e0 1744 TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
5d424d5a
JH
1745 TCP_SKB_CB(nskb)->sacked = 0;
1746 nskb->csum = 0;
84fa7933 1747 nskb->ip_summed = skb->ip_summed;
5d424d5a 1748
50c4817e
IJ
1749 tcp_insert_write_queue_before(nskb, skb, sk);
1750
5d424d5a 1751 len = 0;
234b6860 1752 tcp_for_write_queue_from_safe(skb, next, sk) {
5d424d5a
JH
1753 copy = min_t(int, skb->len, probe_size - len);
1754 if (nskb->ip_summed)
1755 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
1756 else
1757 nskb->csum = skb_copy_and_csum_bits(skb, 0,
056834d9
IJ
1758 skb_put(nskb, copy),
1759 copy, nskb->csum);
5d424d5a
JH
1760
1761 if (skb->len <= copy) {
1762 /* We've eaten all the data from this skb.
1763 * Throw it away. */
4de075e0 1764 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
fe067e8a 1765 tcp_unlink_write_queue(skb, sk);
3ab224be 1766 sk_wmem_free_skb(sk, skb);
5d424d5a 1767 } else {
4de075e0 1768 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
a3433f35 1769 ~(TCPHDR_FIN|TCPHDR_PSH);
5d424d5a
JH
1770 if (!skb_shinfo(skb)->nr_frags) {
1771 skb_pull(skb, copy);
84fa7933 1772 if (skb->ip_summed != CHECKSUM_PARTIAL)
056834d9
IJ
1773 skb->csum = csum_partial(skb->data,
1774 skb->len, 0);
5d424d5a
JH
1775 } else {
1776 __pskb_trim_head(skb, copy);
1777 tcp_set_skb_tso_segs(sk, skb, mss_now);
1778 }
1779 TCP_SKB_CB(skb)->seq += copy;
1780 }
1781
1782 len += copy;
234b6860
IJ
1783
1784 if (len >= probe_size)
1785 break;
5d424d5a
JH
1786 }
1787 tcp_init_tso_segs(sk, nskb, nskb->len);
1788
1789 /* We're ready to send. If this fails, the probe will
1790 * be resegmented into mss-sized pieces by tcp_write_xmit(). */
1791 TCP_SKB_CB(nskb)->when = tcp_time_stamp;
1792 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
1793 /* Decrement cwnd here because we are sending
056834d9 1794 * effectively two packets. */
5d424d5a 1795 tp->snd_cwnd--;
66f5fe62 1796 tcp_event_new_data_sent(sk, nskb);
5d424d5a
JH
1797
1798 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
0e7b1368
JH
1799 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
1800 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
5d424d5a
JH
1801
1802 return 1;
1803 }
1804
1805 return -1;
1806}
1807
1da177e4
LT
1808/* This routine writes packets to the network. It advances the
1809 * send_head. This happens as incoming acks open up the remote
1810 * window for us.
1811 *
f8269a49
IJ
1812 * LARGESEND note: !tcp_urg_mode is overkill, only frames between
1813 * snd_up-64k-mss .. snd_up cannot be large. However, taking into
1814 * account rare use of URG, this is not a big flaw.
1815 *
6ba8a3b1
ND
1816 * Send at most one packet when push_one > 0. Temporarily ignore
1817 * cwnd limit to force at most one packet out when push_one == 2.
1818
a2a385d6
ED
1819 * Returns true, if no segments are in flight and we have queued segments,
1820 * but cannot send anything now because of SWS or another problem.
1da177e4 1821 */
a2a385d6
ED
1822static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
1823 int push_one, gfp_t gfp)
1da177e4
LT
1824{
1825 struct tcp_sock *tp = tcp_sk(sk);
92df7b51 1826 struct sk_buff *skb;
c1b4a7e6
DM
1827 unsigned int tso_segs, sent_pkts;
1828 int cwnd_quota;
5d424d5a 1829 int result;
1da177e4 1830
92df7b51 1831 sent_pkts = 0;
5d424d5a 1832
d5dd9175
IJ
1833 if (!push_one) {
1834 /* Do MTU probing. */
1835 result = tcp_mtu_probe(sk);
1836 if (!result) {
a2a385d6 1837 return false;
d5dd9175
IJ
1838 } else if (result > 0) {
1839 sent_pkts = 1;
1840 }
5d424d5a
JH
1841 }
1842
fe067e8a 1843 while ((skb = tcp_send_head(sk))) {
c8ac3774
HX
1844 unsigned int limit;
1845
b68e9f85 1846 tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
c1b4a7e6 1847 BUG_ON(!tso_segs);
aa93466b 1848
ec342325
AV
1849 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE)
1850 goto repair; /* Skip network transmission */
1851
b68e9f85 1852 cwnd_quota = tcp_cwnd_test(tp, skb);
6ba8a3b1
ND
1853 if (!cwnd_quota) {
1854 if (push_one == 2)
1855 /* Force out a loss probe pkt. */
1856 cwnd_quota = 1;
1857 else
1858 break;
1859 }
b68e9f85
HX
1860
1861 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now)))
1862 break;
1863
afd67b68 1864 if (tso_segs == 1 || !sk->sk_gso_max_segs) {
c1b4a7e6
DM
1865 if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
1866 (tcp_skb_is_last(sk, skb) ?
1867 nonagle : TCP_NAGLE_PUSH))))
1868 break;
1869 } else {
d5dd9175 1870 if (!push_one && tcp_tso_should_defer(sk, skb))
c1b4a7e6
DM
1871 break;
1872 }
aa93466b 1873
0ae5f47e
ED
1874 /* TCP Small Queues :
1875 * Control number of packets in qdisc/devices to two packets / or ~1 ms.
1876 * This allows for :
1877 * - better RTT estimation and ACK scheduling
1878 * - faster recovery
1879 * - high rates
6ef30bda
ED
1880 * Alas, some drivers / subsystems require a fair amount
1881 * of queued bytes to ensure line rate.
1882 * One example is wifi aggregation (802.11 AMPDU)
46d3ceab 1883 */
6ef30bda
ED
1884 limit = max_t(unsigned int, sysctl_tcp_limit_output_bytes,
1885 sk->sk_pacing_rate >> 10);
0ae5f47e
ED
1886
1887 if (atomic_read(&sk->sk_wmem_alloc) > limit) {
46d3ceab 1888 set_bit(TSQ_THROTTLED, &tp->tsq_flags);
94ee16ae
JO
1889 /* It is possible TX completion already happened
1890 * before we set TSQ_THROTTLED, so we must
1891 * test again the condition.
1892 * We abuse smp_mb__after_clear_bit() because
1893 * there is no smp_mb__after_set_bit() yet
1894 */
1895 smp_mb__after_clear_bit();
1896 if (atomic_read(&sk->sk_wmem_alloc) > limit)
1897 break;
46d3ceab 1898 }
0ae5f47e 1899
c8ac3774 1900 limit = mss_now;
afd67b68 1901 if (tso_segs > 1 && sk->sk_gso_max_segs && !tcp_urg_mode(tp))
0e3a4803 1902 limit = tcp_mss_split_point(sk, skb, mss_now,
1485348d
BH
1903 min_t(unsigned int,
1904 cwnd_quota,
1905 sk->sk_gso_max_segs));
1da177e4 1906
c8ac3774 1907 if (skb->len > limit &&
c4ead4c5 1908 unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
c8ac3774
HX
1909 break;
1910
92df7b51 1911 TCP_SKB_CB(skb)->when = tcp_time_stamp;
c1b4a7e6 1912
d5dd9175 1913 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
92df7b51 1914 break;
1da177e4 1915
ec342325 1916repair:
92df7b51
DM
1917 /* Advance the send_head. This one is sent out.
1918 * This call will increment packets_out.
1919 */
66f5fe62 1920 tcp_event_new_data_sent(sk, skb);
1da177e4 1921
92df7b51 1922 tcp_minshall_update(tp, mss_now, skb);
a262f0cd 1923 sent_pkts += tcp_skb_pcount(skb);
d5dd9175
IJ
1924
1925 if (push_one)
1926 break;
92df7b51 1927 }
1da177e4 1928
aa93466b 1929 if (likely(sent_pkts)) {
684bad11
YC
1930 if (tcp_in_cwnd_reduction(sk))
1931 tp->prr_out += sent_pkts;
6ba8a3b1
ND
1932
1933 /* Send one loss probe per tail loss episode. */
1934 if (push_one != 2)
1935 tcp_schedule_loss_probe(sk);
9e412ba7 1936 tcp_cwnd_validate(sk);
a2a385d6 1937 return false;
1da177e4 1938 }
6ba8a3b1
ND
1939 return (push_one == 2) || (!tp->packets_out && tcp_send_head(sk));
1940}
1941
1942bool tcp_schedule_loss_probe(struct sock *sk)
1943{
1944 struct inet_connection_sock *icsk = inet_csk(sk);
1945 struct tcp_sock *tp = tcp_sk(sk);
1946 u32 timeout, tlp_time_stamp, rto_time_stamp;
1947 u32 rtt = tp->srtt >> 3;
1948
1949 if (WARN_ON(icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS))
1950 return false;
1951 /* No consecutive loss probes. */
1952 if (WARN_ON(icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)) {
1953 tcp_rearm_rto(sk);
1954 return false;
1955 }
1956 /* Don't do any loss probe on a Fast Open connection before 3WHS
1957 * finishes.
1958 */
1959 if (sk->sk_state == TCP_SYN_RECV)
1960 return false;
1961
1962 /* TLP is only scheduled when next timer event is RTO. */
1963 if (icsk->icsk_pending != ICSK_TIME_RETRANS)
1964 return false;
1965
1966 /* Schedule a loss probe in 2*RTT for SACK capable connections
1967 * in Open state, that are either limited by cwnd or application.
1968 */
1969 if (sysctl_tcp_early_retrans < 3 || !rtt || !tp->packets_out ||
1970 !tcp_is_sack(tp) || inet_csk(sk)->icsk_ca_state != TCP_CA_Open)
1971 return false;
1972
1973 if ((tp->snd_cwnd > tcp_packets_in_flight(tp)) &&
1974 tcp_send_head(sk))
1975 return false;
1976
1977 /* Probe timeout is at least 1.5*rtt + TCP_DELACK_MAX to account
1978 * for delayed ack when there's one outstanding packet.
1979 */
1980 timeout = rtt << 1;
1981 if (tp->packets_out == 1)
1982 timeout = max_t(u32, timeout,
1983 (rtt + (rtt >> 1) + TCP_DELACK_MAX));
1984 timeout = max_t(u32, timeout, msecs_to_jiffies(10));
1985
1986 /* If RTO is shorter, just schedule TLP in its place. */
1987 tlp_time_stamp = tcp_time_stamp + timeout;
1988 rto_time_stamp = (u32)inet_csk(sk)->icsk_timeout;
1989 if ((s32)(tlp_time_stamp - rto_time_stamp) > 0) {
1990 s32 delta = rto_time_stamp - tcp_time_stamp;
1991 if (delta > 0)
1992 timeout = delta;
1993 }
1994
1995 inet_csk_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout,
6fa3eb70 1996 sysctl_tcp_rto_max);
6ba8a3b1
ND
1997 return true;
1998}
1999
2000/* When probe timeout (PTO) fires, send a new segment if one exists, else
2001 * retransmit the last segment.
2002 */
2003void tcp_send_loss_probe(struct sock *sk)
2004{
9b717a8d 2005 struct tcp_sock *tp = tcp_sk(sk);
6ba8a3b1
ND
2006 struct sk_buff *skb;
2007 int pcount;
2008 int mss = tcp_current_mss(sk);
2009 int err = -1;
2010
2011 if (tcp_send_head(sk) != NULL) {
2012 err = tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2013 goto rearm_timer;
2014 }
2015
9b717a8d
ND
2016 /* At most one outstanding TLP retransmission. */
2017 if (tp->tlp_high_seq)
2018 goto rearm_timer;
2019
6ba8a3b1
ND
2020 /* Retransmit last segment. */
2021 skb = tcp_write_queue_tail(sk);
2022 if (WARN_ON(!skb))
2023 goto rearm_timer;
2024
2025 pcount = tcp_skb_pcount(skb);
2026 if (WARN_ON(!pcount))
2027 goto rearm_timer;
2028
2029 if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2030 if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss)))
2031 goto rearm_timer;
2032 skb = tcp_write_queue_tail(sk);
2033 }
2034
2035 if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2036 goto rearm_timer;
2037
dc5a1700 2038 err = __tcp_retransmit_skb(sk, skb);
6ba8a3b1 2039
9b717a8d
ND
2040 /* Record snd_nxt for loss detection. */
2041 if (likely(!err))
2042 tp->tlp_high_seq = tp->snd_nxt;
2043
6ba8a3b1
ND
2044rearm_timer:
2045 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2046 inet_csk(sk)->icsk_rto,
6fa3eb70 2047 sysctl_tcp_rto_max);
6ba8a3b1
ND
2048
2049 if (likely(!err))
2050 NET_INC_STATS_BH(sock_net(sk),
2051 LINUX_MIB_TCPLOSSPROBES);
2052 return;
1da177e4
LT
2053}
2054
a762a980
DM
2055/* Push out any pending frames which were held back due to
2056 * TCP_CORK or attempt at coalescing tiny packets.
2057 * The socket must be locked by the caller.
2058 */
9e412ba7
IJ
2059void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
2060 int nonagle)
a762a980 2061{
726e07a8
IJ
2062 /* If we are closed, the bytes will have to remain here.
2063 * In time closedown will finish, we empty the write queue and
2064 * all will be happy.
2065 */
2066 if (unlikely(sk->sk_state == TCP_CLOSE))
2067 return;
2068
99a1dec7
MG
2069 if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
2070 sk_gfp_atomic(sk, GFP_ATOMIC)))
726e07a8 2071 tcp_check_probe_timer(sk);
a762a980
DM
2072}
2073
c1b4a7e6
DM
2074/* Send _single_ skb sitting at the send head. This function requires
2075 * true push pending frames to setup probe timer etc.
2076 */
2077void tcp_push_one(struct sock *sk, unsigned int mss_now)
2078{
fe067e8a 2079 struct sk_buff *skb = tcp_send_head(sk);
c1b4a7e6
DM
2080
2081 BUG_ON(!skb || skb->len < mss_now);
2082
d5dd9175 2083 tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
c1b4a7e6
DM
2084}
2085
1da177e4
LT
2086/* This function returns the amount that we can raise the
2087 * usable window based on the following constraints
e905a9ed 2088 *
1da177e4
LT
2089 * 1. The window can never be shrunk once it is offered (RFC 793)
2090 * 2. We limit memory per socket
2091 *
2092 * RFC 1122:
2093 * "the suggested [SWS] avoidance algorithm for the receiver is to keep
2094 * RECV.NEXT + RCV.WIN fixed until:
2095 * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
2096 *
2097 * i.e. don't raise the right edge of the window until you can raise
2098 * it at least MSS bytes.
2099 *
2100 * Unfortunately, the recommended algorithm breaks header prediction,
2101 * since header prediction assumes th->window stays fixed.
2102 *
2103 * Strictly speaking, keeping th->window fixed violates the receiver
2104 * side SWS prevention criteria. The problem is that under this rule
2105 * a stream of single byte packets will cause the right side of the
2106 * window to always advance by a single byte.
e905a9ed 2107 *
1da177e4
LT
2108 * Of course, if the sender implements sender side SWS prevention
2109 * then this will not be a problem.
e905a9ed 2110 *
1da177e4 2111 * BSD seems to make the following compromise:
e905a9ed 2112 *
1da177e4
LT
2113 * If the free space is less than the 1/4 of the maximum
2114 * space available and the free space is less than 1/2 mss,
2115 * then set the window to 0.
2116 * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
2117 * Otherwise, just prevent the window from shrinking
2118 * and from being larger than the largest representable value.
2119 *
2120 * This prevents incremental opening of the window in the regime
2121 * where TCP is limited by the speed of the reader side taking
2122 * data out of the TCP receive queue. It does nothing about
2123 * those cases where the window is constrained on the sender side
2124 * because the pipeline is full.
2125 *
2126 * BSD also seems to "accidentally" limit itself to windows that are a
2127 * multiple of MSS, at least until the free space gets quite small.
2128 * This would appear to be a side effect of the mbuf implementation.
2129 * Combining these two algorithms results in the observed behavior
2130 * of having a fixed window size at almost all times.
2131 *
2132 * Below we obtain similar behavior by forcing the offered window to
2133 * a multiple of the mss when it is feasible to do so.
2134 *
2135 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
2136 * Regular options like TIMESTAMP are taken into account.
2137 */
2138u32 __tcp_select_window(struct sock *sk)
2139{
463c84b9 2140 struct inet_connection_sock *icsk = inet_csk(sk);
1da177e4 2141 struct tcp_sock *tp = tcp_sk(sk);
caa20d9a 2142 /* MSS for the peer's data. Previous versions used mss_clamp
1da177e4
LT
2143 * here. I don't know if the value based on our guesses
2144 * of peer's MSS is better for the performance. It's more correct
2145 * but may be worse for the performance because of rcv_mss
2146 * fluctuations. --SAW 1998/11/1
2147 */
463c84b9 2148 int mss = icsk->icsk_ack.rcv_mss;
1da177e4
LT
2149 int free_space = tcp_space(sk);
2150 int full_space = min_t(int, tp->window_clamp, tcp_full_space(sk));
2151 int window;
2152
2153 if (mss > full_space)
e905a9ed 2154 mss = full_space;
1da177e4 2155
b92edbe0 2156 if (free_space < (full_space >> 1)) {
463c84b9 2157 icsk->icsk_ack.quick = 0;
1da177e4 2158
180d8cd9 2159 if (sk_under_memory_pressure(sk))
056834d9
IJ
2160 tp->rcv_ssthresh = min(tp->rcv_ssthresh,
2161 4U * tp->advmss);
1da177e4
LT
2162
2163 if (free_space < mss)
2164 return 0;
2165 }
2166
2167 if (free_space > tp->rcv_ssthresh)
2168 free_space = tp->rcv_ssthresh;
2169
2170 /* Don't do rounding if we are using window scaling, since the
2171 * scaled window will not line up with the MSS boundary anyway.
2172 */
2173 window = tp->rcv_wnd;
2174 if (tp->rx_opt.rcv_wscale) {
2175 window = free_space;
2176
2177 /* Advertise enough space so that it won't get scaled away.
2178 * Import case: prevent zero window announcement if
2179 * 1<<rcv_wscale > mss.
2180 */
2181 if (((window >> tp->rx_opt.rcv_wscale) << tp->rx_opt.rcv_wscale) != window)
2182 window = (((window >> tp->rx_opt.rcv_wscale) + 1)
2183 << tp->rx_opt.rcv_wscale);
2184 } else {
2185 /* Get the largest window that is a nice multiple of mss.
2186 * Window clamp already applied above.
2187 * If our current window offering is within 1 mss of the
2188 * free space we just keep it. This prevents the divide
2189 * and multiply from happening most of the time.
2190 * We also don't do any window rounding when the free space
2191 * is too small.
2192 */
2193 if (window <= free_space - mss || window > free_space)
056834d9 2194 window = (free_space / mss) * mss;
84565070 2195 else if (mss == full_space &&
b92edbe0 2196 free_space > window + (full_space >> 1))
84565070 2197 window = free_space;
1da177e4
LT
2198 }
2199
2200 return window;
2201}
2202
4a17fc3a
IJ
2203/* Collapses two adjacent SKB's during retransmission. */
2204static void tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
1da177e4
LT
2205{
2206 struct tcp_sock *tp = tcp_sk(sk);
fe067e8a 2207 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb);
058dc334 2208 int skb_size, next_skb_size;
1da177e4 2209
058dc334
IJ
2210 skb_size = skb->len;
2211 next_skb_size = next_skb->len;
1da177e4 2212
058dc334 2213 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
a6963a6b 2214
058dc334 2215 tcp_highest_sack_combine(sk, next_skb, skb);
1da177e4 2216
058dc334 2217 tcp_unlink_write_queue(next_skb, sk);
1da177e4 2218
058dc334
IJ
2219 skb_copy_from_linear_data(next_skb, skb_put(skb, next_skb_size),
2220 next_skb_size);
1da177e4 2221
058dc334
IJ
2222 if (next_skb->ip_summed == CHECKSUM_PARTIAL)
2223 skb->ip_summed = CHECKSUM_PARTIAL;
1da177e4 2224
058dc334
IJ
2225 if (skb->ip_summed != CHECKSUM_PARTIAL)
2226 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size);
1da177e4 2227
058dc334
IJ
2228 /* Update sequence range on original skb. */
2229 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
1da177e4 2230
e6c7d085 2231 /* Merge over control information. This moves PSH/FIN etc. over */
4de075e0 2232 TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
058dc334
IJ
2233
2234 /* All done, get rid of second SKB and account for it so
2235 * packet counting does not break.
2236 */
2237 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
058dc334
IJ
2238
2239 /* changed transmit queue under us so clear hints */
ef9da47c
IJ
2240 tcp_clear_retrans_hints_partial(tp);
2241 if (next_skb == tp->retransmit_skb_hint)
2242 tp->retransmit_skb_hint = skb;
058dc334 2243
797108d1
IJ
2244 tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
2245
058dc334 2246 sk_wmem_free_skb(sk, next_skb);
1da177e4
LT
2247}
2248
67edfef7 2249/* Check if coalescing SKBs is legal. */
a2a385d6 2250static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
4a17fc3a
IJ
2251{
2252 if (tcp_skb_pcount(skb) > 1)
a2a385d6 2253 return false;
4a17fc3a
IJ
2254 /* TODO: SACK collapsing could be used to remove this condition */
2255 if (skb_shinfo(skb)->nr_frags != 0)
a2a385d6 2256 return false;
4a17fc3a 2257 if (skb_cloned(skb))
a2a385d6 2258 return false;
4a17fc3a 2259 if (skb == tcp_send_head(sk))
a2a385d6 2260 return false;
4a17fc3a
IJ
2261 /* Some heurestics for collapsing over SACK'd could be invented */
2262 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
a2a385d6 2263 return false;
4a17fc3a 2264
a2a385d6 2265 return true;
4a17fc3a
IJ
2266}
2267
67edfef7
AK
2268/* Collapse packets in the retransmit queue to make to create
2269 * less packets on the wire. This is only done on retransmission.
2270 */
4a17fc3a
IJ
2271static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
2272 int space)
2273{
2274 struct tcp_sock *tp = tcp_sk(sk);
2275 struct sk_buff *skb = to, *tmp;
a2a385d6 2276 bool first = true;
4a17fc3a
IJ
2277
2278 if (!sysctl_tcp_retrans_collapse)
2279 return;
4de075e0 2280 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
4a17fc3a
IJ
2281 return;
2282
2283 tcp_for_write_queue_from_safe(skb, tmp, sk) {
2284 if (!tcp_can_collapse(sk, skb))
2285 break;
2286
2287 space -= skb->len;
2288
2289 if (first) {
a2a385d6 2290 first = false;
4a17fc3a
IJ
2291 continue;
2292 }
2293
2294 if (space < 0)
2295 break;
2296 /* Punt if not enough space exists in the first SKB for
2297 * the data in the second
2298 */
a21d4572 2299 if (skb->len > skb_availroom(to))
4a17fc3a
IJ
2300 break;
2301
2302 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
2303 break;
2304
2305 tcp_collapse_retrans(sk, to);
2306 }
2307}
2308
1da177e4
LT
2309/* This retransmits one SKB. Policy decisions and retransmit queue
2310 * state updates are done by the caller. Returns non-zero if an
2311 * error occurred which prevented the send.
2312 */
93b174ad 2313int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
1da177e4
LT
2314{
2315 struct tcp_sock *tp = tcp_sk(sk);
5d424d5a 2316 struct inet_connection_sock *icsk = inet_csk(sk);
7d227cd2 2317 unsigned int cur_mss;
1da177e4 2318
5d424d5a
JH
2319 /* Inconslusive MTU probe */
2320 if (icsk->icsk_mtup.probe_size) {
2321 icsk->icsk_mtup.probe_size = 0;
2322 }
2323
1da177e4 2324 /* Do not sent more than we queued. 1/4 is reserved for possible
caa20d9a 2325 * copying overhead: fragmentation, tunneling, mangling etc.
1da177e4
LT
2326 */
2327 if (atomic_read(&sk->sk_wmem_alloc) >
2328 min(sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), sk->sk_sndbuf))
2329 return -EAGAIN;
2330
2331 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
2332 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
2333 BUG();
1da177e4
LT
2334 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
2335 return -ENOMEM;
2336 }
2337
7d227cd2
SS
2338 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
2339 return -EHOSTUNREACH; /* Routing failure or similar. */
2340
0c54b85f 2341 cur_mss = tcp_current_mss(sk);
7d227cd2 2342
1da177e4
LT
2343 /* If receiver has shrunk his window, and skb is out of
2344 * new window, do not retransmit it. The exception is the
2345 * case, when window is shrunk to zero. In this case
2346 * our retransmit serves as a zero window probe.
2347 */
9d4fb27d
JP
2348 if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&
2349 TCP_SKB_CB(skb)->seq != tp->snd_una)
1da177e4
LT
2350 return -EAGAIN;
2351
2352 if (skb->len > cur_mss) {
846998ae 2353 if (tcp_fragment(sk, skb, cur_mss, cur_mss))
1da177e4 2354 return -ENOMEM; /* We'll try again later. */
02276f3c 2355 } else {
9eb9362e
IJ
2356 int oldpcount = tcp_skb_pcount(skb);
2357
2358 if (unlikely(oldpcount > 1)) {
b81908e1
ED
2359 if (skb_unclone(skb, GFP_ATOMIC))
2360 return -ENOMEM;
9eb9362e
IJ
2361 tcp_init_tso_segs(sk, skb, cur_mss);
2362 tcp_adjust_pcount(sk, skb, oldpcount - tcp_skb_pcount(skb));
2363 }
1da177e4
LT
2364 }
2365
4a17fc3a 2366 tcp_retrans_try_collapse(sk, skb, cur_mss);
1da177e4 2367
1da177e4
LT
2368 /* Some Solaris stacks overoptimize and ignore the FIN on a
2369 * retransmit when old data is attached. So strip it off
2370 * since it is cheap to do so and saves bytes on the network.
2371 */
2de979bd 2372 if (skb->len > 0 &&
4de075e0 2373 (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
2de979bd 2374 tp->snd_una == (TCP_SKB_CB(skb)->end_seq - 1)) {
1da177e4 2375 if (!pskb_trim(skb, 0)) {
e870a8ef
IJ
2376 /* Reuse, even though it does some unnecessary work */
2377 tcp_init_nondata_skb(skb, TCP_SKB_CB(skb)->end_seq - 1,
4de075e0 2378 TCP_SKB_CB(skb)->tcp_flags);
1da177e4 2379 skb->ip_summed = CHECKSUM_NONE;
1da177e4
LT
2380 }
2381 }
2382
2383 /* Make a copy, if the first transmission SKB clone we made
2384 * is still in somebody's hands, else make a clone.
2385 */
2386 TCP_SKB_CB(skb)->when = tcp_time_stamp;
1da177e4 2387
50bceae9
TG
2388 /* make sure skb->data is aligned on arches that require it
2389 * and check if ack-trimming & collapsing extended the headroom
2390 * beyond what csum_start can cover.
2391 */
2392 if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
2393 skb_headroom(skb) >= 0xFFFF)) {
117632e6
ED
2394 struct sk_buff *nskb = __pskb_copy(skb, MAX_TCP_HEADER,
2395 GFP_ATOMIC);
93b174ad
YC
2396 return nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) :
2397 -ENOBUFS;
117632e6 2398 } else {
93b174ad 2399 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
117632e6 2400 }
93b174ad
YC
2401}
2402
2403int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
2404{
2405 struct tcp_sock *tp = tcp_sk(sk);
2406 int err = __tcp_retransmit_skb(sk, skb);
1da177e4
LT
2407
2408 if (err == 0) {
2409 /* Update global TCP statistics. */
81cc8a75 2410 TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
1da177e4
LT
2411
2412 tp->total_retrans++;
2413
2414#if FASTRETRANS_DEBUG > 0
056834d9 2415 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
e87cc472 2416 net_dbg_ratelimited("retrans_out leaked\n");
1da177e4
LT
2417 }
2418#endif
b08d6cb2
IJ
2419 if (!tp->retrans_out)
2420 tp->lost_retrans_low = tp->snd_nxt;
1da177e4
LT
2421 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
2422 tp->retrans_out += tcp_skb_pcount(skb);
2423
2424 /* Save stamp of the first retransmit. */
2425 if (!tp->retrans_stamp)
2426 tp->retrans_stamp = TCP_SKB_CB(skb)->when;
2427
1da177e4
LT
2428 /* snd_nxt is stored to detect loss of retransmitted segment,
2429 * see tcp_input.c tcp_sacktag_write_queue().
2430 */
2431 TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt;
2432 }
76fd3c89
YC
2433
2434 if (tp->undo_retrans < 0)
2435 tp->undo_retrans = 0;
2436 tp->undo_retrans += tcp_skb_pcount(skb);
1da177e4
LT
2437 return err;
2438}
2439
67edfef7
AK
2440/* Check if we forward retransmits are possible in the current
2441 * window/congestion state.
2442 */
a2a385d6 2443static bool tcp_can_forward_retransmit(struct sock *sk)
b5afe7bc
IJ
2444{
2445 const struct inet_connection_sock *icsk = inet_csk(sk);
cf533ea5 2446 const struct tcp_sock *tp = tcp_sk(sk);
b5afe7bc
IJ
2447
2448 /* Forward retransmissions are possible only during Recovery. */
2449 if (icsk->icsk_ca_state != TCP_CA_Recovery)
a2a385d6 2450 return false;
b5afe7bc
IJ
2451
2452 /* No forward retransmissions in Reno are possible. */
2453 if (tcp_is_reno(tp))
a2a385d6 2454 return false;
b5afe7bc
IJ
2455
2456 /* Yeah, we have to make difficult choice between forward transmission
2457 * and retransmission... Both ways have their merits...
2458 *
2459 * For now we do not retransmit anything, while we have some new
2460 * segments to send. In the other cases, follow rule 3 for
2461 * NextSeg() specified in RFC3517.
2462 */
2463
2464 if (tcp_may_send_now(sk))
a2a385d6 2465 return false;
b5afe7bc 2466
a2a385d6 2467 return true;
b5afe7bc
IJ
2468}
2469
1da177e4
LT
2470/* This gets called after a retransmit timeout, and the initially
2471 * retransmitted data is acknowledged. It tries to continue
2472 * resending the rest of the retransmit queue, until either
2473 * we've sent it all or the congestion window limit is reached.
2474 * If doing SACK, the first ACK which comes back for a timeout
2475 * based retransmit packet might feed us FACK information again.
2476 * If so, we use it to avoid unnecessarily retransmissions.
2477 */
2478void tcp_xmit_retransmit_queue(struct sock *sk)
2479{
6687e988 2480 const struct inet_connection_sock *icsk = inet_csk(sk);
1da177e4
LT
2481 struct tcp_sock *tp = tcp_sk(sk);
2482 struct sk_buff *skb;
0e1c54c2 2483 struct sk_buff *hole = NULL;
618d9f25 2484 u32 last_lost;
61eb55f4 2485 int mib_idx;
0e1c54c2 2486 int fwd_rexmitting = 0;
6a438bbe 2487
45e77d31
IJ
2488 if (!tp->packets_out)
2489 return;
2490
08ebd172
IJ
2491 if (!tp->lost_out)
2492 tp->retransmit_high = tp->snd_una;
2493
618d9f25 2494 if (tp->retransmit_skb_hint) {
6a438bbe 2495 skb = tp->retransmit_skb_hint;
618d9f25
IJ
2496 last_lost = TCP_SKB_CB(skb)->end_seq;
2497 if (after(last_lost, tp->retransmit_high))
2498 last_lost = tp->retransmit_high;
2499 } else {
fe067e8a 2500 skb = tcp_write_queue_head(sk);
618d9f25
IJ
2501 last_lost = tp->snd_una;
2502 }
1da177e4 2503
08ebd172
IJ
2504 tcp_for_write_queue_from(skb, sk) {
2505 __u8 sacked = TCP_SKB_CB(skb)->sacked;
1da177e4 2506
08ebd172
IJ
2507 if (skb == tcp_send_head(sk))
2508 break;
2509 /* we could do better than to assign each time */
0e1c54c2
IJ
2510 if (hole == NULL)
2511 tp->retransmit_skb_hint = skb;
08ebd172
IJ
2512
2513 /* Assume this retransmit will generate
2514 * only one packet for congestion window
2515 * calculation purposes. This works because
2516 * tcp_retransmit_skb() will chop up the
2517 * packet to be MSS sized and all the
2518 * packet counting works out.
2519 */
2520 if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
2521 return;
1da177e4 2522
0e1c54c2
IJ
2523 if (fwd_rexmitting) {
2524begin_fwd:
2525 if (!before(TCP_SKB_CB(skb)->seq, tcp_highest_sack_seq(tp)))
2526 break;
2527 mib_idx = LINUX_MIB_TCPFORWARDRETRANS;
6a438bbe 2528
0e1c54c2 2529 } else if (!before(TCP_SKB_CB(skb)->seq, tp->retransmit_high)) {
618d9f25 2530 tp->retransmit_high = last_lost;
0e1c54c2
IJ
2531 if (!tcp_can_forward_retransmit(sk))
2532 break;
2533 /* Backtrack if necessary to non-L'ed skb */
2534 if (hole != NULL) {
2535 skb = hole;
2536 hole = NULL;
2537 }
2538 fwd_rexmitting = 1;
2539 goto begin_fwd;
1da177e4 2540
0e1c54c2 2541 } else if (!(sacked & TCPCB_LOST)) {
ac11ba75 2542 if (hole == NULL && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
0e1c54c2
IJ
2543 hole = skb;
2544 continue;
1da177e4 2545
0e1c54c2 2546 } else {
618d9f25 2547 last_lost = TCP_SKB_CB(skb)->end_seq;
0e1c54c2
IJ
2548 if (icsk->icsk_ca_state != TCP_CA_Loss)
2549 mib_idx = LINUX_MIB_TCPFASTRETRANS;
2550 else
2551 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
2552 }
1da177e4 2553
0e1c54c2 2554 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
1da177e4
LT
2555 continue;
2556
09e9b813
ED
2557 if (tcp_retransmit_skb(sk, skb)) {
2558 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL);
0e1c54c2 2559 return;
09e9b813 2560 }
0e1c54c2 2561 NET_INC_STATS_BH(sock_net(sk), mib_idx);
1da177e4 2562
684bad11 2563 if (tcp_in_cwnd_reduction(sk))
a262f0cd
ND
2564 tp->prr_out += tcp_skb_pcount(skb);
2565
fe067e8a 2566 if (skb == tcp_write_queue_head(sk))
3f421baa
ACM
2567 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2568 inet_csk(sk)->icsk_rto,
6fa3eb70 2569 sysctl_tcp_rto_max);
1da177e4
LT
2570 }
2571}
2572
1da177e4
LT
2573/* Send a fin. The caller locks the socket for us. This cannot be
2574 * allowed to fail queueing a FIN frame under any circumstances.
2575 */
2576void tcp_send_fin(struct sock *sk)
2577{
e905a9ed 2578 struct tcp_sock *tp = tcp_sk(sk);
fe067e8a 2579 struct sk_buff *skb = tcp_write_queue_tail(sk);
1da177e4 2580 int mss_now;
e905a9ed 2581
1da177e4
LT
2582 /* Optimization, tack on the FIN if we have a queue of
2583 * unsent frames. But be careful about outgoing SACKS
2584 * and IP options.
2585 */
0c54b85f 2586 mss_now = tcp_current_mss(sk);
1da177e4 2587
fe067e8a 2588 if (tcp_send_head(sk) != NULL) {
4de075e0 2589 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_FIN;
1da177e4
LT
2590 TCP_SKB_CB(skb)->end_seq++;
2591 tp->write_seq++;
2592 } else {
2593 /* Socket is locked, keep trying until memory is available. */
2594 for (;;) {
175ff19c 2595 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation);
1da177e4
LT
2596 if (skb)
2597 break;
2598 yield();
2599 }
1da177e4 2600 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
e870a8ef 2601 tcp_init_nondata_skb(skb, tp->write_seq,
a3433f35 2602 TCPHDR_ACK | TCPHDR_FIN);
1da177e4
LT
2603 tcp_queue_skb(sk, skb);
2604 }
9e412ba7 2605 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_OFF);
1da177e4
LT
2606}
2607
2608/* We get here when a process closes a file descriptor (either due to
2609 * an explicit close() or as a byproduct of exit()'ing) and there
2610 * was unread data in the receive queue. This behavior is recommended
65bb723c 2611 * by RFC 2525, section 2.17. -DaveM
1da177e4 2612 */
dd0fc66f 2613void tcp_send_active_reset(struct sock *sk, gfp_t priority)
1da177e4 2614{
1da177e4
LT
2615 struct sk_buff *skb;
2616
2617 /* NOTE: No TCP options attached and we never retransmit this. */
2618 skb = alloc_skb(MAX_TCP_HEADER, priority);
2619 if (!skb) {
4e673444 2620 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
1da177e4
LT
2621 return;
2622 }
2623
2624 /* Reserve space for headers and prepare control bits. */
2625 skb_reserve(skb, MAX_TCP_HEADER);
e870a8ef 2626 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
a3433f35 2627 TCPHDR_ACK | TCPHDR_RST);
1da177e4 2628 /* Send it off. */
1da177e4 2629 TCP_SKB_CB(skb)->when = tcp_time_stamp;
dfb4b9dc 2630 if (tcp_transmit_skb(sk, skb, 0, priority))
4e673444 2631 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
26af65cb 2632
81cc8a75 2633 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
1da177e4
LT
2634}
2635
67edfef7
AK
2636/* Send a crossed SYN-ACK during socket establishment.
2637 * WARNING: This routine must only be called when we have already sent
1da177e4
LT
2638 * a SYN packet that crossed the incoming SYN that caused this routine
2639 * to get called. If this assumption fails then the initial rcv_wnd
2640 * and rcv_wscale values will not be correct.
2641 */
2642int tcp_send_synack(struct sock *sk)
2643{
056834d9 2644 struct sk_buff *skb;
1da177e4 2645
fe067e8a 2646 skb = tcp_write_queue_head(sk);
4de075e0 2647 if (skb == NULL || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
91df42be 2648 pr_debug("%s: wrong queue state\n", __func__);
1da177e4
LT
2649 return -EFAULT;
2650 }
4de075e0 2651 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
1da177e4
LT
2652 if (skb_cloned(skb)) {
2653 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
2654 if (nskb == NULL)
2655 return -ENOMEM;
fe067e8a 2656 tcp_unlink_write_queue(skb, sk);
1da177e4 2657 skb_header_release(nskb);
fe067e8a 2658 __tcp_add_write_queue_head(sk, nskb);
3ab224be
HA
2659 sk_wmem_free_skb(sk, skb);
2660 sk->sk_wmem_queued += nskb->truesize;
2661 sk_mem_charge(sk, nskb->truesize);
1da177e4
LT
2662 skb = nskb;
2663 }
2664
4de075e0 2665 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
1da177e4
LT
2666 TCP_ECN_send_synack(tcp_sk(sk), skb);
2667 }
2668 TCP_SKB_CB(skb)->when = tcp_time_stamp;
dfb4b9dc 2669 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
1da177e4
LT
2670}
2671
4aea39c1
ED
2672/**
2673 * tcp_make_synack - Prepare a SYN-ACK.
2674 * sk: listener socket
2675 * dst: dst entry attached to the SYNACK
2676 * req: request_sock pointer
4aea39c1
ED
2677 *
2678 * Allocate one skb and build a SYNACK packet.
2679 * @dst is consumed : Caller should not use it again.
2680 */
056834d9 2681struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst,
e6b4d113 2682 struct request_sock *req,
8336886f 2683 struct tcp_fastopen_cookie *foc)
1da177e4 2684{
bd0388ae 2685 struct tcp_out_options opts;
2e6599cb 2686 struct inet_request_sock *ireq = inet_rsk(req);
1da177e4
LT
2687 struct tcp_sock *tp = tcp_sk(sk);
2688 struct tcphdr *th;
1da177e4 2689 struct sk_buff *skb;
cfb6eeb4 2690 struct tcp_md5sig_key *md5;
bd0388ae 2691 int tcp_header_size;
f5fff5dc 2692 int mss;
1da177e4 2693
b70a23ab 2694 skb = sock_wmalloc(sk, MAX_TCP_HEADER + 15, 1, GFP_ATOMIC);
4aea39c1
ED
2695 if (unlikely(!skb)) {
2696 dst_release(dst);
1da177e4 2697 return NULL;
4aea39c1 2698 }
1da177e4
LT
2699 /* Reserve space for headers. */
2700 skb_reserve(skb, MAX_TCP_HEADER);
2701
4aea39c1 2702 skb_dst_set(skb, dst);
ca10b9e9 2703 security_skb_owned_by(skb, sk);
1da177e4 2704
0dbaee3b 2705 mss = dst_metric_advmss(dst);
f5fff5dc
TQ
2706 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < mss)
2707 mss = tp->rx_opt.user_mss;
2708
33ad798c
AL
2709 if (req->rcv_wnd == 0) { /* ignored for retransmitted syns */
2710 __u8 rcv_wscale;
2711 /* Set this up on the first call only */
2712 req->window_clamp = tp->window_clamp ? : dst_metric(dst, RTAX_WINDOW);
e88c64f0
HPP
2713
2714 /* limit the window selection if the user enforce a smaller rx buffer */
2715 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
2716 (req->window_clamp > tcp_full_space(sk) || req->window_clamp == 0))
2717 req->window_clamp = tcp_full_space(sk);
2718
33ad798c
AL
2719 /* tcp_full_space because it is guaranteed to be the first packet */
2720 tcp_select_initial_window(tcp_full_space(sk),
f5fff5dc 2721 mss - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0),
33ad798c
AL
2722 &req->rcv_wnd,
2723 &req->window_clamp,
2724 ireq->wscale_ok,
31d12926 2725 &rcv_wscale,
2726 dst_metric(dst, RTAX_INITRWND));
33ad798c
AL
2727 ireq->rcv_wscale = rcv_wscale;
2728 }
2729
2730 memset(&opts, 0, sizeof(opts));
8b5f12d0
FW
2731#ifdef CONFIG_SYN_COOKIES
2732 if (unlikely(req->cookie_ts))
2733 TCP_SKB_CB(skb)->when = cookie_init_timestamp(req);
2734 else
2735#endif
33ad798c 2736 TCP_SKB_CB(skb)->when = tcp_time_stamp;
1a2c6181
CP
2737 tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, &md5,
2738 foc) + sizeof(*th);
cfb6eeb4 2739
aa8223c7
ACM
2740 skb_push(skb, tcp_header_size);
2741 skb_reset_transport_header(skb);
1da177e4 2742
aa8223c7 2743 th = tcp_hdr(skb);
1da177e4
LT
2744 memset(th, 0, sizeof(struct tcphdr));
2745 th->syn = 1;
2746 th->ack = 1;
1da177e4 2747 TCP_ECN_make_synack(req, th);
a3116ac5 2748 th->source = ireq->loc_port;
2e6599cb 2749 th->dest = ireq->rmt_port;
e870a8ef
IJ
2750 /* Setting of flags are superfluous here for callers (and ECE is
2751 * not even correctly set)
2752 */
2753 tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn,
a3433f35 2754 TCPHDR_SYN | TCPHDR_ACK);
4957faad 2755
1da177e4 2756 th->seq = htonl(TCP_SKB_CB(skb)->seq);
8336886f
JC
2757 /* XXX data is queued and acked as is. No buffer/window check */
2758 th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
1da177e4
LT
2759
2760 /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
600ff0c2 2761 th->window = htons(min(req->rcv_wnd, 65535U));
bd0388ae 2762 tcp_options_write((__be32 *)(th + 1), tp, &opts);
1da177e4 2763 th->doff = (tcp_header_size >> 2);
aa2ea058 2764 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS, tcp_skb_pcount(skb));
cfb6eeb4
YH
2765
2766#ifdef CONFIG_TCP_MD5SIG
2767 /* Okay, we have all we need - do the md5 hash if needed */
2768 if (md5) {
bd0388ae 2769 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
49a72dfb 2770 md5, NULL, req, skb);
cfb6eeb4
YH
2771 }
2772#endif
2773
1da177e4
LT
2774 return skb;
2775}
4bc2f18b 2776EXPORT_SYMBOL(tcp_make_synack);
1da177e4 2777
67edfef7 2778/* Do all connect socket setups that can be done AF independent. */
370816ae 2779void tcp_connect_init(struct sock *sk)
1da177e4 2780{
cf533ea5 2781 const struct dst_entry *dst = __sk_dst_get(sk);
1da177e4
LT
2782 struct tcp_sock *tp = tcp_sk(sk);
2783 __u8 rcv_wscale;
2784
2785 /* We'll fix this up when we get a response from the other end.
2786 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
2787 */
2788 tp->tcp_header_len = sizeof(struct tcphdr) +
bb5b7c11 2789 (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0);
1da177e4 2790
cfb6eeb4
YH
2791#ifdef CONFIG_TCP_MD5SIG
2792 if (tp->af_specific->md5_lookup(sk, sk) != NULL)
2793 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
2794#endif
2795
1da177e4
LT
2796 /* If user gave his TCP_MAXSEG, record it to clamp */
2797 if (tp->rx_opt.user_mss)
2798 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
2799 tp->max_window = 0;
5d424d5a 2800 tcp_mtup_init(sk);
1da177e4
LT
2801 tcp_sync_mss(sk, dst_mtu(dst));
2802
2803 if (!tp->window_clamp)
2804 tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
0dbaee3b 2805 tp->advmss = dst_metric_advmss(dst);
f5fff5dc
TQ
2806 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->advmss)
2807 tp->advmss = tp->rx_opt.user_mss;
2808
1da177e4 2809 tcp_initialize_rcv_mss(sk);
1da177e4 2810
e88c64f0
HPP
2811 /* limit the window selection if the user enforce a smaller rx buffer */
2812 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
2813 (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
2814 tp->window_clamp = tcp_full_space(sk);
2815
1da177e4
LT
2816 tcp_select_initial_window(tcp_full_space(sk),
2817 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
2818 &tp->rcv_wnd,
2819 &tp->window_clamp,
bb5b7c11 2820 sysctl_tcp_window_scaling,
31d12926 2821 &rcv_wscale,
2822 dst_metric(dst, RTAX_INITRWND));
1da177e4
LT
2823
2824 tp->rx_opt.rcv_wscale = rcv_wscale;
2825 tp->rcv_ssthresh = tp->rcv_wnd;
2826
2827 sk->sk_err = 0;
2828 sock_reset_flag(sk, SOCK_DONE);
2829 tp->snd_wnd = 0;
ee7537b6 2830 tcp_init_wl(tp, 0);
1da177e4
LT
2831 tp->snd_una = tp->write_seq;
2832 tp->snd_sml = tp->write_seq;
33f5f57e 2833 tp->snd_up = tp->write_seq;
370816ae 2834 tp->snd_nxt = tp->write_seq;
ee995283
PE
2835
2836 if (likely(!tp->repair))
2837 tp->rcv_nxt = 0;
6f198dca
AV
2838 else
2839 tp->rcv_tstamp = tcp_time_stamp;
ee995283
PE
2840 tp->rcv_wup = tp->rcv_nxt;
2841 tp->copied_seq = tp->rcv_nxt;
1da177e4 2842
463c84b9
ACM
2843 inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT;
2844 inet_csk(sk)->icsk_retransmits = 0;
1da177e4
LT
2845 tcp_clear_retrans(tp);
2846}
2847
783237e8
YC
2848static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
2849{
2850 struct tcp_sock *tp = tcp_sk(sk);
2851 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
2852
2853 tcb->end_seq += skb->len;
2854 skb_header_release(skb);
2855 __tcp_add_write_queue_tail(sk, skb);
2856 sk->sk_wmem_queued += skb->truesize;
2857 sk_mem_charge(sk, skb->truesize);
2858 tp->write_seq = tcb->end_seq;
2859 tp->packets_out += tcp_skb_pcount(skb);
2860}
2861
2862/* Build and send a SYN with data and (cached) Fast Open cookie. However,
2863 * queue a data-only packet after the regular SYN, such that regular SYNs
2864 * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
2865 * only the SYN sequence, the data are retransmitted in the first ACK.
2866 * If cookie is not cached or other error occurs, falls back to send a
2867 * regular SYN with Fast Open cookie request option.
2868 */
2869static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
2870{
2871 struct tcp_sock *tp = tcp_sk(sk);
2872 struct tcp_fastopen_request *fo = tp->fastopen_req;
e64a8519 2873 int syn_loss = 0, space, err = 0;
aab48743 2874 unsigned long last_syn_loss = 0;
e64a8519 2875 struct sk_buff *syn_data;
aab48743 2876
67da22d2 2877 tp->rx_opt.mss_clamp = tp->advmss; /* If MSS is not cached */
aab48743
YC
2878 tcp_fastopen_cache_get(sk, &tp->rx_opt.mss_clamp, &fo->cookie,
2879 &syn_loss, &last_syn_loss);
2880 /* Recurring FO SYN losses: revert to regular handshake temporarily */
2881 if (syn_loss > 1 &&
2882 time_before(jiffies, last_syn_loss + (60*HZ << syn_loss))) {
2883 fo->cookie.len = -1;
2884 goto fallback;
2885 }
783237e8 2886
67da22d2
YC
2887 if (sysctl_tcp_fastopen & TFO_CLIENT_NO_COOKIE)
2888 fo->cookie.len = -1;
2889 else if (fo->cookie.len <= 0)
783237e8
YC
2890 goto fallback;
2891
2892 /* MSS for SYN-data is based on cached MSS and bounded by PMTU and
2893 * user-MSS. Reserve maximum option space for middleboxes that add
2894 * private TCP options. The cost is reduced data space in SYN :(
2895 */
2896 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->rx_opt.mss_clamp)
2897 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
1b63edd6 2898 space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) -
783237e8
YC
2899 MAX_TCP_OPTION_SPACE;
2900
fe42b170
ED
2901 space = min_t(size_t, space, fo->size);
2902
2903 /* limit to order-0 allocations */
2904 space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER));
2905
e64a8519
ED
2906 syn_data = sk_stream_alloc_skb(sk, space, sk->sk_allocation);
2907 if (!syn_data)
783237e8 2908 goto fallback;
e64a8519
ED
2909 syn_data->ip_summed = CHECKSUM_PARTIAL;
2910 memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
75a518b0 2911 skb_shinfo(syn_data)->gso_segs = 1;
e64a8519
ED
2912 if (unlikely(memcpy_fromiovecend(skb_put(syn_data, space),
2913 fo->data->msg_iov, 0, space))) {
2914 kfree_skb(syn_data);
2915 goto fallback;
2916 }
783237e8 2917
e64a8519
ED
2918 /* No more data pending in inet_wait_for_connect() */
2919 if (space == fo->size)
2920 fo->data = NULL;
2921 fo->copied = space;
783237e8 2922
e64a8519 2923 tcp_connect_queue_skb(sk, syn_data);
783237e8 2924
e64a8519 2925 err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
783237e8 2926
e64a8519
ED
2927 /* Now full SYN+DATA was cloned and sent (or not),
2928 * remove the SYN from the original skb (syn_data)
2929 * we keep in write queue in case of a retransmit, as we
2930 * also have the SYN packet (with no data) in the same queue.
2931 */
2932 TCP_SKB_CB(syn_data)->seq++;
2933 TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
2934 if (!err) {
67da22d2 2935 tp->syn_data = (fo->copied > 0);
783237e8
YC
2936 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
2937 goto done;
2938 }
783237e8
YC
2939
2940fallback:
2941 /* Send a regular SYN with Fast Open cookie request option */
2942 if (fo->cookie.len > 0)
2943 fo->cookie.len = 0;
2944 err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
2945 if (err)
2946 tp->syn_fastopen = 0;
783237e8
YC
2947done:
2948 fo->cookie.len = -1; /* Exclude Fast Open option for SYN retries */
2949 return err;
2950}
2951
67edfef7 2952/* Build a SYN and send it off. */
1da177e4
LT
2953int tcp_connect(struct sock *sk)
2954{
2955 struct tcp_sock *tp = tcp_sk(sk);
2956 struct sk_buff *buff;
ee586811 2957 int err;
1da177e4
LT
2958
2959 tcp_connect_init(sk);
2960
2b916477
AV
2961 if (unlikely(tp->repair)) {
2962 tcp_finish_connect(sk, NULL);
2963 return 0;
2964 }
2965
e64a8519
ED
2966 buff = sk_stream_alloc_skb(sk, 0, sk->sk_allocation);
2967 if (unlikely(!buff))
1da177e4
LT
2968 return -ENOBUFS;
2969
a3433f35 2970 tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
783237e8
YC
2971 tp->retrans_stamp = TCP_SKB_CB(buff)->when = tcp_time_stamp;
2972 tcp_connect_queue_skb(sk, buff);
e870a8ef 2973 TCP_ECN_send_syn(sk, buff);
1da177e4 2974
783237e8
YC
2975 /* Send off SYN; include data in Fast Open. */
2976 err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
2977 tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
ee586811
EP
2978 if (err == -ECONNREFUSED)
2979 return err;
bd37a088
WY
2980
2981 /* We change tp->snd_nxt after the tcp_transmit_skb() call
2982 * in order to make this packet get counted in tcpOutSegs.
2983 */
2984 tp->snd_nxt = tp->write_seq;
2985 tp->pushed_seq = tp->write_seq;
81cc8a75 2986 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
1da177e4
LT
2987
2988 /* Timer for repeating the SYN until an answer. */
3f421baa 2989 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
6fa3eb70 2990 inet_csk(sk)->icsk_rto, sysctl_tcp_rto_max);
1da177e4
LT
2991 return 0;
2992}
4bc2f18b 2993EXPORT_SYMBOL(tcp_connect);
1da177e4
LT
2994
2995/* Send out a delayed ack, the caller does the policy checking
2996 * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check()
2997 * for details.
2998 */
2999void tcp_send_delayed_ack(struct sock *sk)
3000{
463c84b9
ACM
3001 struct inet_connection_sock *icsk = inet_csk(sk);
3002 int ato = icsk->icsk_ack.ato;
1da177e4
LT
3003 unsigned long timeout;
3004
3005 if (ato > TCP_DELACK_MIN) {
463c84b9 3006 const struct tcp_sock *tp = tcp_sk(sk);
056834d9 3007 int max_ato = HZ / 2;
1da177e4 3008
056834d9
IJ
3009 if (icsk->icsk_ack.pingpong ||
3010 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
1da177e4
LT
3011 max_ato = TCP_DELACK_MAX;
3012
3013 /* Slow path, intersegment interval is "high". */
3014
3015 /* If some rtt estimate is known, use it to bound delayed ack.
463c84b9 3016 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
1da177e4
LT
3017 * directly.
3018 */
3019 if (tp->srtt) {
056834d9 3020 int rtt = max(tp->srtt >> 3, TCP_DELACK_MIN);
1da177e4
LT
3021
3022 if (rtt < max_ato)
3023 max_ato = rtt;
3024 }
3025
3026 ato = min(ato, max_ato);
3027 }
3028
3029 /* Stay within the limit we were given */
3030 timeout = jiffies + ato;
3031
3032 /* Use new timeout only if there wasn't a older one earlier. */
463c84b9 3033 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
1da177e4
LT
3034 /* If delack timer was blocked or is about to expire,
3035 * send ACK now.
3036 */
463c84b9
ACM
3037 if (icsk->icsk_ack.blocked ||
3038 time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
1da177e4
LT
3039 tcp_send_ack(sk);
3040 return;
3041 }
3042
463c84b9
ACM
3043 if (!time_before(timeout, icsk->icsk_ack.timeout))
3044 timeout = icsk->icsk_ack.timeout;
1da177e4 3045 }
463c84b9
ACM
3046 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
3047 icsk->icsk_ack.timeout = timeout;
3048 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
1da177e4
LT
3049}
3050
3051/* This routine sends an ack and also updates the window. */
3052void tcp_send_ack(struct sock *sk)
3053{
058dc334 3054 struct sk_buff *buff;
1da177e4 3055
058dc334
IJ
3056 /* If we have been reset, we may not send again. */
3057 if (sk->sk_state == TCP_CLOSE)
3058 return;
1da177e4 3059
058dc334
IJ
3060 /* We are not putting this on the write queue, so
3061 * tcp_transmit_skb() will set the ownership to this
3062 * sock.
3063 */
99a1dec7 3064 buff = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC));
058dc334
IJ
3065 if (buff == NULL) {
3066 inet_csk_schedule_ack(sk);
3067 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
3068 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
6fa3eb70 3069 TCP_DELACK_MAX, sysctl_tcp_rto_max);
058dc334 3070 return;
1da177e4 3071 }
058dc334
IJ
3072
3073 /* Reserve space for headers and prepare control bits. */
3074 skb_reserve(buff, MAX_TCP_HEADER);
a3433f35 3075 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
058dc334
IJ
3076
3077 /* Send it off, this clears delayed acks for us. */
058dc334 3078 TCP_SKB_CB(buff)->when = tcp_time_stamp;
99a1dec7 3079 tcp_transmit_skb(sk, buff, 0, sk_gfp_atomic(sk, GFP_ATOMIC));
1da177e4
LT
3080}
3081
3082/* This routine sends a packet with an out of date sequence
3083 * number. It assumes the other end will try to ack it.
3084 *
3085 * Question: what should we make while urgent mode?
3086 * 4.4BSD forces sending single byte of data. We cannot send
3087 * out of window data, because we have SND.NXT==SND.MAX...
3088 *
3089 * Current solution: to send TWO zero-length segments in urgent mode:
3090 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
3091 * out-of-date with SND.UNA-1 to probe window.
3092 */
3093static int tcp_xmit_probe_skb(struct sock *sk, int urgent)
3094{
3095 struct tcp_sock *tp = tcp_sk(sk);
3096 struct sk_buff *skb;
3097
3098 /* We don't queue it, tcp_transmit_skb() sets ownership. */
99a1dec7 3099 skb = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC));
e905a9ed 3100 if (skb == NULL)
1da177e4
LT
3101 return -1;
3102
3103 /* Reserve space for headers and set control bits. */
3104 skb_reserve(skb, MAX_TCP_HEADER);
1da177e4
LT
3105 /* Use a previous sequence. This should cause the other
3106 * end to send an ack. Don't queue or clone SKB, just
3107 * send it.
3108 */
a3433f35 3109 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
1da177e4 3110 TCP_SKB_CB(skb)->when = tcp_time_stamp;
dfb4b9dc 3111 return tcp_transmit_skb(sk, skb, 0, GFP_ATOMIC);
1da177e4
LT
3112}
3113
ee995283
PE
3114void tcp_send_window_probe(struct sock *sk)
3115{
3116 if (sk->sk_state == TCP_ESTABLISHED) {
3117 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
3118 tcp_xmit_probe_skb(sk, 0);
3119 }
3120}
3121
67edfef7 3122/* Initiate keepalive or window probe from timer. */
1da177e4
LT
3123int tcp_write_wakeup(struct sock *sk)
3124{
058dc334
IJ
3125 struct tcp_sock *tp = tcp_sk(sk);
3126 struct sk_buff *skb;
1da177e4 3127
058dc334
IJ
3128 if (sk->sk_state == TCP_CLOSE)
3129 return -1;
3130
3131 if ((skb = tcp_send_head(sk)) != NULL &&
3132 before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
3133 int err;
0c54b85f 3134 unsigned int mss = tcp_current_mss(sk);
058dc334
IJ
3135 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3136
3137 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
3138 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
3139
3140 /* We are probing the opening of a window
3141 * but the window size is != 0
3142 * must have been a result SWS avoidance ( sender )
3143 */
3144 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
3145 skb->len > mss) {
3146 seg_size = min(seg_size, mss);
4de075e0 3147 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
058dc334
IJ
3148 if (tcp_fragment(sk, skb, seg_size, mss))
3149 return -1;
3150 } else if (!tcp_skb_pcount(skb))
3151 tcp_set_skb_tso_segs(sk, skb, mss);
3152
4de075e0 3153 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
058dc334
IJ
3154 TCP_SKB_CB(skb)->when = tcp_time_stamp;
3155 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3156 if (!err)
3157 tcp_event_new_data_sent(sk, skb);
3158 return err;
3159 } else {
33f5f57e 3160 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
058dc334
IJ
3161 tcp_xmit_probe_skb(sk, 1);
3162 return tcp_xmit_probe_skb(sk, 0);
1da177e4 3163 }
1da177e4
LT
3164}
3165
3166/* A window probe timeout has occurred. If window is not closed send
3167 * a partial packet else a zero probe.
3168 */
3169void tcp_send_probe0(struct sock *sk)
3170{
463c84b9 3171 struct inet_connection_sock *icsk = inet_csk(sk);
1da177e4
LT
3172 struct tcp_sock *tp = tcp_sk(sk);
3173 int err;
3174
3175 err = tcp_write_wakeup(sk);
3176
fe067e8a 3177 if (tp->packets_out || !tcp_send_head(sk)) {
1da177e4 3178 /* Cancel probe timer, if it is not required. */
6687e988 3179 icsk->icsk_probes_out = 0;
463c84b9 3180 icsk->icsk_backoff = 0;
1da177e4
LT
3181 return;
3182 }
3183
3184 if (err <= 0) {
463c84b9
ACM
3185 if (icsk->icsk_backoff < sysctl_tcp_retries2)
3186 icsk->icsk_backoff++;
6687e988 3187 icsk->icsk_probes_out++;
e905a9ed 3188 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
6fa3eb70
S
3189 min_t(unsigned int, icsk->icsk_rto << icsk->icsk_backoff, sysctl_tcp_rto_max),
3190 sysctl_tcp_rto_max);
1da177e4
LT
3191 } else {
3192 /* If packet was not sent due to local congestion,
6687e988 3193 * do not backoff and do not remember icsk_probes_out.
1da177e4
LT
3194 * Let local senders to fight for local resources.
3195 *
3196 * Use accumulated backoff yet.
3197 */
6687e988
ACM
3198 if (!icsk->icsk_probes_out)
3199 icsk->icsk_probes_out = 1;
e905a9ed 3200 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
463c84b9 3201 min(icsk->icsk_rto << icsk->icsk_backoff,
3f421baa 3202 TCP_RESOURCE_PROBE_INTERVAL),
6fa3eb70 3203 sysctl_tcp_rto_max);
1da177e4
LT
3204 }
3205}