projects / GitHub / LineageOS / android_kernel_samsung_universal7580.git / blob
? search:


c3d204973dbc4779520d70ea8e703f218564510b
[GitHub/LineageOS/android_kernel_samsung_universal7580.git] / net / netfilter / ipvs / ip_vs_xmit.c
1 /*
2 * ip_vs_xmit.c: various packet transmitters for IPVS
3 *
4 * Authors: Wensong Zhang <wensong@linuxvirtualserver.org>
5 * Julian Anastasov <ja@ssi.bg>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 * Changes:
13 *
14 * Description of forwarding methods:
15 * - all transmitters are called from LOCAL_IN (remote clients) and
16 * LOCAL_OUT (local clients) but for ICMP can be called from FORWARD
17 * - not all connections have destination server, for example,
18 * connections in backup server when fwmark is used
19 * - bypass connections use daddr from packet
20 * - we can use dst without ref while sending in RCU section, we use
21 * ref when returning NF_ACCEPT for NAT-ed packet via loopback
22 * LOCAL_OUT rules:
23 * - skb->dev is NULL, skb->protocol is not set (both are set in POST_ROUTING)
24 * - skb->pkt_type is not set yet
25 * - the only place where we can see skb->sk != NULL
26 */
27
28 #define KMSG_COMPONENT "IPVS"
29 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
30
31 #include <linux/kernel.h>
32 #include <linux/slab.h>
33 #include <linux/tcp.h> /* for tcphdr */
34 #include <net/ip.h>
35 #include <net/tcp.h> /* for csum_tcpudp_magic */
36 #include <net/udp.h>
37 #include <net/icmp.h> /* for icmp_send */
38 #include <net/route.h> /* for ip_route_output */
39 #include <net/ipv6.h>
40 #include <net/ip6_route.h>
41 #include <net/addrconf.h>
42 #include <linux/icmpv6.h>
43 #include <linux/netfilter.h>
44 #include <linux/netfilter_ipv4.h>
45
46 #include <net/ip_vs.h>
47
48 enum {
49 IP_VS_RT_MODE_LOCAL = 1, /* Allow local dest */
50 IP_VS_RT_MODE_NON_LOCAL = 2, /* Allow non-local dest */
51 IP_VS_RT_MODE_RDR = 4, /* Allow redirect from remote daddr to
52 * local
53 */
54 IP_VS_RT_MODE_CONNECT = 8, /* Always bind route to saddr */
55 IP_VS_RT_MODE_KNOWN_NH = 16,/* Route via remote addr */
56 IP_VS_RT_MODE_TUNNEL = 32,/* Tunnel mode */
57 };
58
59 static inline struct ip_vs_dest_dst *ip_vs_dest_dst_alloc(void)
60 {
61 return kmalloc(sizeof(struct ip_vs_dest_dst), GFP_ATOMIC);
62 }
63
64 static inline void ip_vs_dest_dst_free(struct ip_vs_dest_dst *dest_dst)
65 {
66 kfree(dest_dst);
67 }
68
69 /*
70 * Destination cache to speed up outgoing route lookup
71 */
72 static inline void
73 __ip_vs_dst_set(struct ip_vs_dest *dest, struct ip_vs_dest_dst *dest_dst,
74 struct dst_entry *dst, u32 dst_cookie)
75 {
76 struct ip_vs_dest_dst *old;
77
78 old = rcu_dereference_protected(dest->dest_dst,
79 lockdep_is_held(&dest->dst_lock));
80
81 if (dest_dst) {
82 dest_dst->dst_cache = dst;
83 dest_dst->dst_cookie = dst_cookie;
84 }
85 rcu_assign_pointer(dest->dest_dst, dest_dst);
86
87 if (old)
88 call_rcu(&old->rcu_head, ip_vs_dest_dst_rcu_free);
89 }
90
91 static inline struct ip_vs_dest_dst *
92 __ip_vs_dst_check(struct ip_vs_dest *dest)
93 {
94 struct ip_vs_dest_dst *dest_dst = rcu_dereference(dest->dest_dst);
95 struct dst_entry *dst;
96
97 if (!dest_dst)
98 return NULL;
99 dst = dest_dst->dst_cache;
100 if (dst->obsolete &&
101 dst->ops->check(dst, dest_dst->dst_cookie) == NULL)
102 return NULL;
103 return dest_dst;
104 }
105
106 static inline bool
107 __mtu_check_toobig_v6(const struct sk_buff *skb, u32 mtu)
108 {
109 if (IP6CB(skb)->frag_max_size) {
110 /* frag_max_size tell us that, this packet have been
111 * defragmented by netfilter IPv6 conntrack module.
112 */
113 if (IP6CB(skb)->frag_max_size > mtu)
114 return true; /* largest fragment violate MTU */
115 }
116 else if (skb->len > mtu && !skb_is_gso(skb)) {
117 return true; /* Packet size violate MTU size */
118 }
119 return false;
120 }
121
122 /* Get route to daddr, update *saddr, optionally bind route to saddr */
123 static struct rtable *do_output_route4(struct net *net, __be32 daddr,
124 int rt_mode, __be32 *saddr)
125 {
126 struct flowi4 fl4;
127 struct rtable *rt;
128 int loop = 0;
129
130 memset(&fl4, 0, sizeof(fl4));
131 fl4.daddr = daddr;
132 fl4.flowi4_flags = (rt_mode & IP_VS_RT_MODE_KNOWN_NH) ?
133 FLOWI_FLAG_KNOWN_NH : 0;
134
135 retry:
136 rt = ip_route_output_key(net, &fl4);
137 if (IS_ERR(rt)) {
138 /* Invalid saddr ? */
139 if (PTR_ERR(rt) == -EINVAL && *saddr &&
140 rt_mode & IP_VS_RT_MODE_CONNECT && !loop) {
141 *saddr = 0;
142 flowi4_update_output(&fl4, 0, 0, daddr, 0);
143 goto retry;
144 }
145 IP_VS_DBG_RL("ip_route_output error, dest: %pI4\n", &daddr);
146 return NULL;
147 } else if (!*saddr && rt_mode & IP_VS_RT_MODE_CONNECT && fl4.saddr) {
148 ip_rt_put(rt);
149 *saddr = fl4.saddr;
150 flowi4_update_output(&fl4, 0, 0, daddr, fl4.saddr);
151 loop++;
152 goto retry;
153 }
154 *saddr = fl4.saddr;
155 return rt;
156 }
157
158 /* Get route to destination or remote server */
159 static int
160 __ip_vs_get_out_rt(struct sk_buff *skb, struct ip_vs_dest *dest,
161 __be32 daddr, int rt_mode, __be32 *ret_saddr)
162 {
163 struct net *net = dev_net(skb_dst(skb)->dev);
164 struct netns_ipvs *ipvs = net_ipvs(net);
165 struct ip_vs_dest_dst *dest_dst;
166 struct rtable *rt; /* Route to the other host */
167 struct rtable *ort; /* Original route */
168 struct iphdr *iph;
169 __be16 df;
170 int mtu;
171 int local, noref = 1;
172
173 if (dest) {
174 dest_dst = __ip_vs_dst_check(dest);
175 if (likely(dest_dst))
176 rt = (struct rtable *) dest_dst->dst_cache;
177 else {
178 dest_dst = ip_vs_dest_dst_alloc();
179 spin_lock_bh(&dest->dst_lock);
180 if (!dest_dst) {
181 __ip_vs_dst_set(dest, NULL, NULL, 0);
182 spin_unlock_bh(&dest->dst_lock);
183 goto err_unreach;
184 }
185 rt = do_output_route4(net, dest->addr.ip, rt_mode,
186 &dest_dst->dst_saddr.ip);
187 if (!rt) {
188 __ip_vs_dst_set(dest, NULL, NULL, 0);
189 spin_unlock_bh(&dest->dst_lock);
190 ip_vs_dest_dst_free(dest_dst);
191 goto err_unreach;
192 }
193 __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0);
194 spin_unlock_bh(&dest->dst_lock);
195 IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n",
196 &dest->addr.ip, &dest_dst->dst_saddr.ip,
197 atomic_read(&rt->dst.__refcnt));
198 }
199 daddr = dest->addr.ip;
200 if (ret_saddr)
201 *ret_saddr = dest_dst->dst_saddr.ip;
202 } else {
203 __be32 saddr = htonl(INADDR_ANY);
204
205 noref = 0;
206
207 /* For such unconfigured boxes avoid many route lookups
208 * for performance reasons because we do not remember saddr
209 */
210 rt_mode &= ~IP_VS_RT_MODE_CONNECT;
211 rt = do_output_route4(net, daddr, rt_mode, &saddr);
212 if (!rt)
213 goto err_unreach;
214 if (ret_saddr)
215 *ret_saddr = saddr;
216 }
217
218 local = (rt->rt_flags & RTCF_LOCAL) ? 1 : 0;
219 if (!((local ? IP_VS_RT_MODE_LOCAL : IP_VS_RT_MODE_NON_LOCAL) &
220 rt_mode)) {
221 IP_VS_DBG_RL("Stopping traffic to %s address, dest: %pI4\n",
222 (rt->rt_flags & RTCF_LOCAL) ?
223 "local":"non-local", &daddr);
224 goto err_put;
225 }
226 iph = ip_hdr(skb);
227 if (likely(!local)) {
228 if (unlikely(ipv4_is_loopback(iph->saddr))) {
229 IP_VS_DBG_RL("Stopping traffic from loopback address "
230 "%pI4 to non-local address, dest: %pI4\n",
231 &iph->saddr, &daddr);
232 goto err_put;
233 }
234 } else {
235 ort = skb_rtable(skb);
236 if (!(rt_mode & IP_VS_RT_MODE_RDR) &&
237 !(ort->rt_flags & RTCF_LOCAL)) {
238 IP_VS_DBG_RL("Redirect from non-local address %pI4 to "
239 "local requires NAT method, dest: %pI4\n",
240 &iph->daddr, &daddr);
241 goto err_put;
242 }
243 /* skb to local stack, preserve old route */
244 if (!noref)
245 ip_rt_put(rt);
246 return local;
247 }
248
249 if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL))) {
250 mtu = dst_mtu(&rt->dst);
251 df = iph->frag_off & htons(IP_DF);
252 } else {
253 struct sock *sk = skb->sk;
254
255 mtu = dst_mtu(&rt->dst) - sizeof(struct iphdr);
256 if (mtu < 68) {
257 IP_VS_DBG_RL("%s(): mtu less than 68\n", __func__);
258 goto err_put;
259 }
260 ort = skb_rtable(skb);
261 if (!skb->dev && sk && sk->sk_state != TCP_TIME_WAIT)
262 ort->dst.ops->update_pmtu(&ort->dst, sk, NULL, mtu);
263 /* MTU check allowed? */
264 df = sysctl_pmtu_disc(ipvs) ? iph->frag_off & htons(IP_DF) : 0;
265 }
266
267 /* MTU checking */
268 if (unlikely(df && skb->len > mtu && !skb_is_gso(skb))) {
269 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
270 IP_VS_DBG(1, "frag needed for %pI4\n", &iph->saddr);
271 goto err_put;
272 }
273
274 skb_dst_drop(skb);
275 if (noref) {
276 if (!local)
277 skb_dst_set_noref_force(skb, &rt->dst);
278 else
279 skb_dst_set(skb, dst_clone(&rt->dst));
280 } else
281 skb_dst_set(skb, &rt->dst);
282
283 return local;
284
285 err_put:
286 if (!noref)
287 ip_rt_put(rt);
288 return -1;
289
290 err_unreach:
291 dst_link_failure(skb);
292 return -1;
293 }
294
295 #ifdef CONFIG_IP_VS_IPV6
296
297 static inline int __ip_vs_is_local_route6(struct rt6_info *rt)
298 {
299 return rt->dst.dev && rt->dst.dev->flags & IFF_LOOPBACK;
300 }
301
302 static struct dst_entry *
303 __ip_vs_route_output_v6(struct net *net, struct in6_addr *daddr,
304 struct in6_addr *ret_saddr, int do_xfrm)
305 {
306 struct dst_entry *dst;
307 struct flowi6 fl6 = {
308 .daddr = *daddr,
309 };
310
311 dst = ip6_route_output(net, NULL, &fl6);
312 if (dst->error)
313 goto out_err;
314 if (!ret_saddr)
315 return dst;
316 if (ipv6_addr_any(&fl6.saddr) &&
317 ipv6_dev_get_saddr(net, ip6_dst_idev(dst)->dev,
318 &fl6.daddr, 0, &fl6.saddr) < 0)
319 goto out_err;
320 if (do_xfrm) {
321 dst = xfrm_lookup(net, dst, flowi6_to_flowi(&fl6), NULL, 0);
322 if (IS_ERR(dst)) {
323 dst = NULL;
324 goto out_err;
325 }
326 }
327 *ret_saddr = fl6.saddr;
328 return dst;
329
330 out_err:
331 dst_release(dst);
332 IP_VS_DBG_RL("ip6_route_output error, dest: %pI6\n", daddr);
333 return NULL;
334 }
335
336 /*
337 * Get route to destination or remote server
338 */
339 static int
340 __ip_vs_get_out_rt_v6(struct sk_buff *skb, struct ip_vs_dest *dest,
341 struct in6_addr *daddr, struct in6_addr *ret_saddr,
342 struct ip_vs_iphdr *ipvsh, int do_xfrm, int rt_mode)
343 {
344 struct net *net = dev_net(skb_dst(skb)->dev);
345 struct ip_vs_dest_dst *dest_dst;
346 struct rt6_info *rt; /* Route to the other host */
347 struct rt6_info *ort; /* Original route */
348 struct dst_entry *dst;
349 int mtu;
350 int local, noref = 1;
351
352 if (dest) {
353 dest_dst = __ip_vs_dst_check(dest);
354 if (likely(dest_dst))
355 rt = (struct rt6_info *) dest_dst->dst_cache;
356 else {
357 u32 cookie;
358
359 dest_dst = ip_vs_dest_dst_alloc();
360 spin_lock_bh(&dest->dst_lock);
361 if (!dest_dst) {
362 __ip_vs_dst_set(dest, NULL, NULL, 0);
363 spin_unlock_bh(&dest->dst_lock);
364 goto err_unreach;
365 }
366 dst = __ip_vs_route_output_v6(net, &dest->addr.in6,
367 &dest_dst->dst_saddr.in6,
368 do_xfrm);
369 if (!dst) {
370 __ip_vs_dst_set(dest, NULL, NULL, 0);
371 spin_unlock_bh(&dest->dst_lock);
372 ip_vs_dest_dst_free(dest_dst);
373 goto err_unreach;
374 }
375 rt = (struct rt6_info *) dst;
376 cookie = rt->rt6i_node ? rt->rt6i_node->fn_sernum : 0;
377 __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie);
378 spin_unlock_bh(&dest->dst_lock);
379 IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n",
380 &dest->addr.in6, &dest_dst->dst_saddr.in6,
381 atomic_read(&rt->dst.__refcnt));
382 }
383 if (ret_saddr)
384 *ret_saddr = dest_dst->dst_saddr.in6;
385 } else {
386 noref = 0;
387 dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm);
388 if (!dst)
389 goto err_unreach;
390 rt = (struct rt6_info *) dst;
391 }
392
393 local = __ip_vs_is_local_route6(rt);
394 if (!((local ? IP_VS_RT_MODE_LOCAL : IP_VS_RT_MODE_NON_LOCAL) &
395 rt_mode)) {
396 IP_VS_DBG_RL("Stopping traffic to %s address, dest: %pI6c\n",
397 local ? "local":"non-local", daddr);
398 goto err_put;
399 }
400 if (likely(!local)) {
401 if (unlikely((!skb->dev || skb->dev->flags & IFF_LOOPBACK) &&
402 ipv6_addr_type(&ipv6_hdr(skb)->saddr) &
403 IPV6_ADDR_LOOPBACK)) {
404 IP_VS_DBG_RL("Stopping traffic from loopback address "
405 "%pI6c to non-local address, "
406 "dest: %pI6c\n",
407 &ipv6_hdr(skb)->saddr, daddr);
408 goto err_put;
409 }
410 } else {
411 ort = (struct rt6_info *) skb_dst(skb);
412 if (!(rt_mode & IP_VS_RT_MODE_RDR) &&
413 !__ip_vs_is_local_route6(ort)) {
414 IP_VS_DBG_RL("Redirect from non-local address %pI6c "
415 "to local requires NAT method, "
416 "dest: %pI6c\n",
417 &ipv6_hdr(skb)->daddr, daddr);
418 goto err_put;
419 }
420 /* skb to local stack, preserve old route */
421 if (!noref)
422 dst_release(&rt->dst);
423 return local;
424 }
425
426 /* MTU checking */
427 if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL)))
428 mtu = dst_mtu(&rt->dst);
429 else {
430 struct sock *sk = skb->sk;
431
432 mtu = dst_mtu(&rt->dst) - sizeof(struct ipv6hdr);
433 if (mtu < IPV6_MIN_MTU) {
434 IP_VS_DBG_RL("%s(): mtu less than %d\n", __func__,
435 IPV6_MIN_MTU);
436 goto err_put;
437 }
438 ort = (struct rt6_info *) skb_dst(skb);
439 if (!skb->dev && sk && sk->sk_state != TCP_TIME_WAIT)
440 ort->dst.ops->update_pmtu(&ort->dst, sk, NULL, mtu);
441 }
442
443 if (unlikely(__mtu_check_toobig_v6(skb, mtu))) {
444 if (!skb->dev)
445 skb->dev = net->loopback_dev;
446 /* only send ICMP too big on first fragment */
447 if (!ipvsh->fragoffs)
448 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
449 IP_VS_DBG(1, "frag needed for %pI6c\n", &ipv6_hdr(skb)->saddr);
450 goto err_put;
451 }
452
453 skb_dst_drop(skb);
454 if (noref) {
455 if (!local)
456 skb_dst_set_noref_force(skb, &rt->dst);
457 else
458 skb_dst_set(skb, dst_clone(&rt->dst));
459 } else
460 skb_dst_set(skb, &rt->dst);
461
462 return local;
463
464 err_put:
465 if (!noref)
466 dst_release(&rt->dst);
467 return -1;
468
469 err_unreach:
470 dst_link_failure(skb);
471 return -1;
472 }
473 #endif
474
475
476 /* return NF_ACCEPT to allow forwarding or other NF_xxx on error */
477 static inline int ip_vs_tunnel_xmit_prepare(struct sk_buff *skb,
478 struct ip_vs_conn *cp)
479 {
480 int ret = NF_ACCEPT;
481
482 skb->ipvs_property = 1;
483 if (unlikely(cp->flags & IP_VS_CONN_F_NFCT))
484 ret = ip_vs_confirm_conntrack(skb);
485 if (ret == NF_ACCEPT) {
486 nf_reset(skb);
487 skb_forward_csum(skb);
488 }
489 return ret;
490 }
491
492 /* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */
493 static inline int ip_vs_nat_send_or_cont(int pf, struct sk_buff *skb,
494 struct ip_vs_conn *cp, int local)
495 {
496 int ret = NF_STOLEN;
497
498 skb->ipvs_property = 1;
499 if (likely(!(cp->flags & IP_VS_CONN_F_NFCT)))
500 ip_vs_notrack(skb);
501 else
502 ip_vs_update_conntrack(skb, cp, 1);
503 if (!local) {
504 skb_forward_csum(skb);
505 NF_HOOK(pf, NF_INET_LOCAL_OUT, skb, NULL, skb_dst(skb)->dev,
506 dst_output);
507 } else
508 ret = NF_ACCEPT;
509 return ret;
510 }
511
512 /* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */
513 static inline int ip_vs_send_or_cont(int pf, struct sk_buff *skb,
514 struct ip_vs_conn *cp, int local)
515 {
516 int ret = NF_STOLEN;
517
518 skb->ipvs_property = 1;
519 if (likely(!(cp->flags & IP_VS_CONN_F_NFCT)))
520 ip_vs_notrack(skb);
521 if (!local) {
522 skb_forward_csum(skb);
523 NF_HOOK(pf, NF_INET_LOCAL_OUT, skb, NULL, skb_dst(skb)->dev,
524 dst_output);
525 } else
526 ret = NF_ACCEPT;
527 return ret;
528 }
529
530
531 /*
532 * NULL transmitter (do nothing except return NF_ACCEPT)
533 */
534 int
535 ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
536 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
537 {
538 /* we do not touch skb and do not need pskb ptr */
539 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
540 }
541
542
543 /*
544 * Bypass transmitter
545 * Let packets bypass the destination when the destination is not
546 * available, it may be only used in transparent cache cluster.
547 */
548 int
549 ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
550 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
551 {
552 struct iphdr *iph = ip_hdr(skb);
553
554 EnterFunction(10);
555
556 rcu_read_lock();
557 if (__ip_vs_get_out_rt(skb, NULL, iph->daddr, IP_VS_RT_MODE_NON_LOCAL,
558 NULL) < 0)
559 goto tx_error;
560
561 ip_send_check(iph);
562
563 /* Another hack: avoid icmp_send in ip_fragment */
564 skb->local_df = 1;
565
566 ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0);
567 rcu_read_unlock();
568
569 LeaveFunction(10);
570 return NF_STOLEN;
571
572 tx_error:
573 kfree_skb(skb);
574 rcu_read_unlock();
575 LeaveFunction(10);
576 return NF_STOLEN;
577 }
578
579 #ifdef CONFIG_IP_VS_IPV6
580 int
581 ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
582 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
583 {
584 EnterFunction(10);
585
586 rcu_read_lock();
587 if (__ip_vs_get_out_rt_v6(skb, NULL, &ipvsh->daddr.in6, NULL,
588 ipvsh, 0, IP_VS_RT_MODE_NON_LOCAL) < 0)
589 goto tx_error;
590
591 /* Another hack: avoid icmp_send in ip_fragment */
592 skb->local_df = 1;
593
594 ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0);
595 rcu_read_unlock();
596
597 LeaveFunction(10);
598 return NF_STOLEN;
599
600 tx_error:
601 kfree_skb(skb);
602 rcu_read_unlock();
603 LeaveFunction(10);
604 return NF_STOLEN;
605 }
606 #endif
607
608 /*
609 * NAT transmitter (only for outside-to-inside nat forwarding)
610 * Not used for related ICMP
611 */
612 int
613 ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
614 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
615 {
616 struct rtable *rt; /* Route to the other host */
617 int local, rc, was_input;
618
619 EnterFunction(10);
620
621 rcu_read_lock();
622 /* check if it is a connection of no-client-port */
623 if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) {
624 __be16 _pt, *p;
625
626 p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt);
627 if (p == NULL)
628 goto tx_error;
629 ip_vs_conn_fill_cport(cp, *p);
630 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
631 }
632
633 was_input = rt_is_input_route(skb_rtable(skb));
634 local = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
635 IP_VS_RT_MODE_LOCAL |
636 IP_VS_RT_MODE_NON_LOCAL |
637 IP_VS_RT_MODE_RDR, NULL);
638 if (local < 0)
639 goto tx_error;
640 rt = skb_rtable(skb);
641 /*
642 * Avoid duplicate tuple in reply direction for NAT traffic
643 * to local address when connection is sync-ed
644 */
645 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
646 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
647 enum ip_conntrack_info ctinfo;
648 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
649
650 if (ct && !nf_ct_is_untracked(ct)) {
651 IP_VS_DBG_RL_PKT(10, AF_INET, pp, skb, 0,
652 "ip_vs_nat_xmit(): "
653 "stopping DNAT to local address");
654 goto tx_error;
655 }
656 }
657 #endif
658
659 /* From world but DNAT to loopback address? */
660 if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) {
661 IP_VS_DBG_RL_PKT(1, AF_INET, pp, skb, 0, "ip_vs_nat_xmit(): "
662 "stopping DNAT to loopback address");
663 goto tx_error;
664 }
665
666 /* copy-on-write the packet before mangling it */
667 if (!skb_make_writable(skb, sizeof(struct iphdr)))
668 goto tx_error;
669
670 if (skb_cow(skb, rt->dst.dev->hard_header_len))
671 goto tx_error;
672
673 /* mangle the packet */
674 if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh))
675 goto tx_error;
676 ip_hdr(skb)->daddr = cp->daddr.ip;
677 ip_send_check(ip_hdr(skb));
678
679 IP_VS_DBG_PKT(10, AF_INET, pp, skb, 0, "After DNAT");
680
681 /* FIXME: when application helper enlarges the packet and the length
682 is larger than the MTU of outgoing device, there will be still
683 MTU problem. */
684
685 /* Another hack: avoid icmp_send in ip_fragment */
686 skb->local_df = 1;
687
688 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local);
689 rcu_read_unlock();
690
691 LeaveFunction(10);
692 return rc;
693
694 tx_error:
695 kfree_skb(skb);
696 rcu_read_unlock();
697 LeaveFunction(10);
698 return NF_STOLEN;
699 }
700
701 #ifdef CONFIG_IP_VS_IPV6
702 int
703 ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
704 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
705 {
706 struct rt6_info *rt; /* Route to the other host */
707 int local, rc;
708
709 EnterFunction(10);
710
711 rcu_read_lock();
712 /* check if it is a connection of no-client-port */
713 if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT && !ipvsh->fragoffs)) {
714 __be16 _pt, *p;
715 p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt);
716 if (p == NULL)
717 goto tx_error;
718 ip_vs_conn_fill_cport(cp, *p);
719 IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
720 }
721
722 local = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
723 ipvsh, 0,
724 IP_VS_RT_MODE_LOCAL |
725 IP_VS_RT_MODE_NON_LOCAL |
726 IP_VS_RT_MODE_RDR);
727 if (local < 0)
728 goto tx_error;
729 rt = (struct rt6_info *) skb_dst(skb);
730 /*
731 * Avoid duplicate tuple in reply direction for NAT traffic
732 * to local address when connection is sync-ed
733 */
734 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
735 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
736 enum ip_conntrack_info ctinfo;
737 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
738
739 if (ct && !nf_ct_is_untracked(ct)) {
740 IP_VS_DBG_RL_PKT(10, AF_INET6, pp, skb, 0,
741 "ip_vs_nat_xmit_v6(): "
742 "stopping DNAT to local address");
743 goto tx_error;
744 }
745 }
746 #endif
747
748 /* From world but DNAT to loopback address? */
749 if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
750 ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LOOPBACK) {
751 IP_VS_DBG_RL_PKT(1, AF_INET6, pp, skb, 0,
752 "ip_vs_nat_xmit_v6(): "
753 "stopping DNAT to loopback address");
754 goto tx_error;
755 }
756
757 /* copy-on-write the packet before mangling it */
758 if (!skb_make_writable(skb, sizeof(struct ipv6hdr)))
759 goto tx_error;
760
761 if (skb_cow(skb, rt->dst.dev->hard_header_len))
762 goto tx_error;
763
764 /* mangle the packet */
765 if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh))
766 goto tx_error;
767 ipv6_hdr(skb)->daddr = cp->daddr.in6;
768
769 IP_VS_DBG_PKT(10, AF_INET6, pp, skb, 0, "After DNAT");
770
771 /* FIXME: when application helper enlarges the packet and the length
772 is larger than the MTU of outgoing device, there will be still
773 MTU problem. */
774
775 /* Another hack: avoid icmp_send in ip_fragment */
776 skb->local_df = 1;
777
778 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local);
779 rcu_read_unlock();
780
781 LeaveFunction(10);
782 return rc;
783
784 tx_error:
785 LeaveFunction(10);
786 kfree_skb(skb);
787 rcu_read_unlock();
788 return NF_STOLEN;
789 }
790 #endif
791
792
793 /*
794 * IP Tunneling transmitter
795 *
796 * This function encapsulates the packet in a new IP packet, its
797 * destination will be set to cp->daddr. Most code of this function
798 * is taken from ipip.c.
799 *
800 * It is used in VS/TUN cluster. The load balancer selects a real
801 * server from a cluster based on a scheduling algorithm,
802 * encapsulates the request packet and forwards it to the selected
803 * server. For example, all real servers are configured with
804 * "ifconfig tunl0 <Virtual IP Address> up". When the server receives
805 * the encapsulated packet, it will decapsulate the packet, processe
806 * the request and return the response packets directly to the client
807 * without passing the load balancer. This can greatly increase the
808 * scalability of virtual server.
809 *
810 * Used for ANY protocol
811 */
812 int
813 ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
814 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
815 {
816 struct netns_ipvs *ipvs = net_ipvs(skb_net(skb));
817 struct rtable *rt; /* Route to the other host */
818 __be32 saddr; /* Source for tunnel */
819 struct net_device *tdev; /* Device to other host */
820 struct iphdr *old_iph = ip_hdr(skb);
821 u8 tos = old_iph->tos;
822 __be16 df;
823 struct iphdr *iph; /* Our new IP header */
824 unsigned int max_headroom; /* The extra header space needed */
825 int ret, local;
826
827 EnterFunction(10);
828
829 rcu_read_lock();
830 local = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
831 IP_VS_RT_MODE_LOCAL |
832 IP_VS_RT_MODE_NON_LOCAL |
833 IP_VS_RT_MODE_CONNECT |
834 IP_VS_RT_MODE_TUNNEL, &saddr);
835 if (local < 0)
836 goto tx_error;
837 if (local) {
838 rcu_read_unlock();
839 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
840 }
841
842 rt = skb_rtable(skb);
843 tdev = rt->dst.dev;
844
845 /* Copy DF, reset fragment offset and MF */
846 df = sysctl_pmtu_disc(ipvs) ? old_iph->frag_off & htons(IP_DF) : 0;
847
848 /*
849 * Okay, now see if we can stuff it in the buffer as-is.
850 */
851 max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct iphdr);
852
853 if (skb_headroom(skb) < max_headroom || skb_cloned(skb)) {
854 struct sk_buff *new_skb =
855 skb_realloc_headroom(skb, max_headroom);
856
857 if (!new_skb)
858 goto tx_error;
859 consume_skb(skb);
860 skb = new_skb;
861 old_iph = ip_hdr(skb);
862 }
863
864 skb->transport_header = skb->network_header;
865
866 /* fix old IP header checksum */
867 ip_send_check(old_iph);
868
869 skb_push(skb, sizeof(struct iphdr));
870 skb_reset_network_header(skb);
871 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
872
873 /*
874 * Push down and install the IPIP header.
875 */
876 iph = ip_hdr(skb);
877 iph->version = 4;
878 iph->ihl = sizeof(struct iphdr)>>2;
879 iph->frag_off = df;
880 iph->protocol = IPPROTO_IPIP;
881 iph->tos = tos;
882 iph->daddr = cp->daddr.ip;
883 iph->saddr = saddr;
884 iph->ttl = old_iph->ttl;
885 ip_select_ident(skb, NULL);
886
887 /* Another hack: avoid icmp_send in ip_fragment */
888 skb->local_df = 1;
889
890 ret = ip_vs_tunnel_xmit_prepare(skb, cp);
891 if (ret == NF_ACCEPT)
892 ip_local_out(skb);
893 else if (ret == NF_DROP)
894 kfree_skb(skb);
895 rcu_read_unlock();
896
897 LeaveFunction(10);
898
899 return NF_STOLEN;
900
901 tx_error:
902 kfree_skb(skb);
903 rcu_read_unlock();
904 LeaveFunction(10);
905 return NF_STOLEN;
906 }
907
908 #ifdef CONFIG_IP_VS_IPV6
909 int
910 ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
911 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
912 {
913 struct rt6_info *rt; /* Route to the other host */
914 struct in6_addr saddr; /* Source for tunnel */
915 struct net_device *tdev; /* Device to other host */
916 struct ipv6hdr *old_iph = ipv6_hdr(skb);
917 struct ipv6hdr *iph; /* Our new IP header */
918 unsigned int max_headroom; /* The extra header space needed */
919 int ret, local;
920
921 EnterFunction(10);
922
923 rcu_read_lock();
924 local = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6,
925 &saddr, ipvsh, 1,
926 IP_VS_RT_MODE_LOCAL |
927 IP_VS_RT_MODE_NON_LOCAL |
928 IP_VS_RT_MODE_TUNNEL);
929 if (local < 0)
930 goto tx_error;
931 if (local) {
932 rcu_read_unlock();
933 return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1);
934 }
935
936 rt = (struct rt6_info *) skb_dst(skb);
937 tdev = rt->dst.dev;
938
939 /*
940 * Okay, now see if we can stuff it in the buffer as-is.
941 */
942 max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct ipv6hdr);
943
944 if (skb_headroom(skb) < max_headroom || skb_cloned(skb)) {
945 struct sk_buff *new_skb =
946 skb_realloc_headroom(skb, max_headroom);
947
948 if (!new_skb)
949 goto tx_error;
950 consume_skb(skb);
951 skb = new_skb;
952 old_iph = ipv6_hdr(skb);
953 }
954
955 skb->transport_header = skb->network_header;
956
957 skb_push(skb, sizeof(struct ipv6hdr));
958 skb_reset_network_header(skb);
959 memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
960
961 /*
962 * Push down and install the IPIP header.
963 */
964 iph = ipv6_hdr(skb);
965 iph->version = 6;
966 iph->nexthdr = IPPROTO_IPV6;
967 iph->payload_len = old_iph->payload_len;
968 be16_add_cpu(&iph->payload_len, sizeof(*old_iph));
969 memset(&iph->flow_lbl, 0, sizeof(iph->flow_lbl));
970 ipv6_change_dsfield(iph, 0, ipv6_get_dsfield(old_iph));
971 iph->daddr = cp->daddr.in6;
972 iph->saddr = saddr;
973 iph->hop_limit = old_iph->hop_limit;
974
975 /* Another hack: avoid icmp_send in ip_fragment */
976 skb->local_df = 1;
977
978 ret = ip_vs_tunnel_xmit_prepare(skb, cp);
979 if (ret == NF_ACCEPT)
980 ip6_local_out(skb);
981 else if (ret == NF_DROP)
982 kfree_skb(skb);
983 rcu_read_unlock();
984
985 LeaveFunction(10);
986
987 return NF_STOLEN;
988
989 tx_error:
990 kfree_skb(skb);
991 rcu_read_unlock();
992 LeaveFunction(10);
993 return NF_STOLEN;
994 }
995 #endif
996
997
998 /*
999 * Direct Routing transmitter
1000 * Used for ANY protocol
1001 */
1002 int
1003 ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1004 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1005 {
1006 int local;
1007
1008 EnterFunction(10);
1009
1010 rcu_read_lock();
1011 local = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
1012 IP_VS_RT_MODE_LOCAL |
1013 IP_VS_RT_MODE_NON_LOCAL |
1014 IP_VS_RT_MODE_KNOWN_NH, NULL);
1015 if (local < 0)
1016 goto tx_error;
1017 if (local) {
1018 rcu_read_unlock();
1019 return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
1020 }
1021
1022 ip_send_check(ip_hdr(skb));
1023
1024 /* Another hack: avoid icmp_send in ip_fragment */
1025 skb->local_df = 1;
1026
1027 ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0);
1028 rcu_read_unlock();
1029
1030 LeaveFunction(10);
1031 return NF_STOLEN;
1032
1033 tx_error:
1034 kfree_skb(skb);
1035 rcu_read_unlock();
1036 LeaveFunction(10);
1037 return NF_STOLEN;
1038 }
1039
1040 #ifdef CONFIG_IP_VS_IPV6
1041 int
1042 ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1043 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1044 {
1045 int local;
1046
1047 EnterFunction(10);
1048
1049 rcu_read_lock();
1050 local = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
1051 ipvsh, 0,
1052 IP_VS_RT_MODE_LOCAL |
1053 IP_VS_RT_MODE_NON_LOCAL);
1054 if (local < 0)
1055 goto tx_error;
1056 if (local) {
1057 rcu_read_unlock();
1058 return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1);
1059 }
1060
1061 /* Another hack: avoid icmp_send in ip_fragment */
1062 skb->local_df = 1;
1063
1064 ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0);
1065 rcu_read_unlock();
1066
1067 LeaveFunction(10);
1068 return NF_STOLEN;
1069
1070 tx_error:
1071 kfree_skb(skb);
1072 rcu_read_unlock();
1073 LeaveFunction(10);
1074 return NF_STOLEN;
1075 }
1076 #endif
1077
1078
1079 /*
1080 * ICMP packet transmitter
1081 * called by the ip_vs_in_icmp
1082 */
1083 int
1084 ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1085 struct ip_vs_protocol *pp, int offset, unsigned int hooknum,
1086 struct ip_vs_iphdr *iph)
1087 {
1088 struct rtable *rt; /* Route to the other host */
1089 int rc;
1090 int local;
1091 int rt_mode, was_input;
1092
1093 EnterFunction(10);
1094
1095 /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1096 forwarded directly here, because there is no need to
1097 translate address/port back */
1098 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1099 if (cp->packet_xmit)
1100 rc = cp->packet_xmit(skb, cp, pp, iph);
1101 else
1102 rc = NF_ACCEPT;
1103 /* do not touch skb anymore */
1104 atomic_inc(&cp->in_pkts);
1105 goto out;
1106 }
1107
1108 /*
1109 * mangle and send the packet here (only for VS/NAT)
1110 */
1111 was_input = rt_is_input_route(skb_rtable(skb));
1112
1113 /* LOCALNODE from FORWARD hook is not supported */
1114 rt_mode = (hooknum != NF_INET_FORWARD) ?
1115 IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1116 IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1117 rcu_read_lock();
1118 local = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip, rt_mode, NULL);
1119 if (local < 0)
1120 goto tx_error;
1121 rt = skb_rtable(skb);
1122
1123 /*
1124 * Avoid duplicate tuple in reply direction for NAT traffic
1125 * to local address when connection is sync-ed
1126 */
1127 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
1128 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1129 enum ip_conntrack_info ctinfo;
1130 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
1131
1132 if (ct && !nf_ct_is_untracked(ct)) {
1133 IP_VS_DBG(10, "%s(): "
1134 "stopping DNAT to local address %pI4\n",
1135 __func__, &cp->daddr.ip);
1136 goto tx_error;
1137 }
1138 }
1139 #endif
1140
1141 /* From world but DNAT to loopback address? */
1142 if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) {
1143 IP_VS_DBG(1, "%s(): "
1144 "stopping DNAT to loopback %pI4\n",
1145 __func__, &cp->daddr.ip);
1146 goto tx_error;
1147 }
1148
1149 /* copy-on-write the packet before mangling it */
1150 if (!skb_make_writable(skb, offset))
1151 goto tx_error;
1152
1153 if (skb_cow(skb, rt->dst.dev->hard_header_len))
1154 goto tx_error;
1155
1156 ip_vs_nat_icmp(skb, pp, cp, 0);
1157
1158 /* Another hack: avoid icmp_send in ip_fragment */
1159 skb->local_df = 1;
1160
1161 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local);
1162 rcu_read_unlock();
1163 goto out;
1164
1165 tx_error:
1166 kfree_skb(skb);
1167 rcu_read_unlock();
1168 rc = NF_STOLEN;
1169 out:
1170 LeaveFunction(10);
1171 return rc;
1172 }
1173
1174 #ifdef CONFIG_IP_VS_IPV6
1175 int
1176 ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1177 struct ip_vs_protocol *pp, int offset, unsigned int hooknum,
1178 struct ip_vs_iphdr *ipvsh)
1179 {
1180 struct rt6_info *rt; /* Route to the other host */
1181 int rc;
1182 int local;
1183 int rt_mode;
1184
1185 EnterFunction(10);
1186
1187 /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1188 forwarded directly here, because there is no need to
1189 translate address/port back */
1190 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1191 if (cp->packet_xmit)
1192 rc = cp->packet_xmit(skb, cp, pp, ipvsh);
1193 else
1194 rc = NF_ACCEPT;
1195 /* do not touch skb anymore */
1196 atomic_inc(&cp->in_pkts);
1197 goto out;
1198 }
1199
1200 /*
1201 * mangle and send the packet here (only for VS/NAT)
1202 */
1203
1204 /* LOCALNODE from FORWARD hook is not supported */
1205 rt_mode = (hooknum != NF_INET_FORWARD) ?
1206 IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1207 IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1208 rcu_read_lock();
1209 local = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
1210 ipvsh, 0, rt_mode);
1211 if (local < 0)
1212 goto tx_error;
1213 rt = (struct rt6_info *) skb_dst(skb);
1214 /*
1215 * Avoid duplicate tuple in reply direction for NAT traffic
1216 * to local address when connection is sync-ed
1217 */
1218 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
1219 if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1220 enum ip_conntrack_info ctinfo;
1221 struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
1222
1223 if (ct && !nf_ct_is_untracked(ct)) {
1224 IP_VS_DBG(10, "%s(): "
1225 "stopping DNAT to local address %pI6\n",
1226 __func__, &cp->daddr.in6);
1227 goto tx_error;
1228 }
1229 }
1230 #endif
1231
1232 /* From world but DNAT to loopback address? */
1233 if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
1234 ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LOOPBACK) {
1235 IP_VS_DBG(1, "%s(): "
1236 "stopping DNAT to loopback %pI6\n",
1237 __func__, &cp->daddr.in6);
1238 goto tx_error;
1239 }
1240
1241 /* copy-on-write the packet before mangling it */
1242 if (!skb_make_writable(skb, offset))
1243 goto tx_error;
1244
1245 if (skb_cow(skb, rt->dst.dev->hard_header_len))
1246 goto tx_error;
1247
1248 ip_vs_nat_icmp_v6(skb, pp, cp, 0);
1249
1250 /* Another hack: avoid icmp_send in ip_fragment */
1251 skb->local_df = 1;
1252
1253 rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local);
1254 rcu_read_unlock();
1255 goto out;
1256
1257 tx_error:
1258 kfree_skb(skb);
1259 rcu_read_unlock();
1260 rc = NF_STOLEN;
1261 out:
1262 LeaveFunction(10);
1263 return rc;
1264 }
1265 #endif