soreuseport: UDP/IPv4 implementation
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / net / ipv6 / inet6_hashtables.c
CommitLineData
5324a040
ACM
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic INET6 transport hashtables
7 *
d8313f5c
ACM
8 * Authors: Lotsa people, from code originally in tcp, generalised here
9 * by Arnaldo Carvalho de Melo <acme@mandriva.com>
5324a040
ACM
10 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version
14 * 2 of the License, or (at your option) any later version.
15 */
16
5324a040 17#include <linux/module.h>
d8313f5c 18#include <linux/random.h>
5324a040
ACM
19
20#include <net/inet_connection_sock.h>
21#include <net/inet_hashtables.h>
22#include <net/inet6_hashtables.h>
6e5714ea 23#include <net/secure_seq.h>
d8313f5c 24#include <net/ip.h>
5324a040 25
9327f705 26int __inet6_hash(struct sock *sk, struct inet_timewait_sock *tw)
b1a7ffcb 27{
39d8cda7 28 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
9327f705 29 int twrefcnt = 0;
b1a7ffcb 30
547b792c 31 WARN_ON(!sk_unhashed(sk));
b1a7ffcb
DV
32
33 if (sk->sk_state == TCP_LISTEN) {
5caea4ea 34 struct inet_listen_hashbucket *ilb;
3ab5aee7 35
5caea4ea
ED
36 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
37 spin_lock(&ilb->lock);
c25eb3bf 38 __sk_nulls_add_node_rcu(sk, &ilb->head);
5caea4ea 39 spin_unlock(&ilb->lock);
b1a7ffcb
DV
40 } else {
41 unsigned int hash;
3ab5aee7 42 struct hlist_nulls_head *list;
9db66bdc 43 spinlock_t *lock;
3ab5aee7 44
b1a7ffcb 45 sk->sk_hash = hash = inet6_sk_ehashfn(sk);
230140cf
ED
46 list = &inet_ehash_bucket(hashinfo, hash)->chain;
47 lock = inet_ehash_lockp(hashinfo, hash);
9db66bdc 48 spin_lock(lock);
3ab5aee7 49 __sk_nulls_add_node_rcu(sk, list);
9327f705
ED
50 if (tw) {
51 WARN_ON(sk->sk_hash != tw->tw_hash);
52 twrefcnt = inet_twsk_unhash(tw);
53 }
9db66bdc 54 spin_unlock(lock);
b1a7ffcb
DV
55 }
56
c29a0bc4 57 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
9327f705 58 return twrefcnt;
b1a7ffcb
DV
59}
60EXPORT_SYMBOL(__inet6_hash);
61
62/*
63 * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
64 * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
65 *
66 * The sockhash lock must be held as a reader here.
67 */
d86e0dac
PE
68struct sock *__inet6_lookup_established(struct net *net,
69 struct inet_hashinfo *hashinfo,
b1a7ffcb 70 const struct in6_addr *saddr,
d2ecd9cc 71 const __be16 sport,
b1a7ffcb
DV
72 const struct in6_addr *daddr,
73 const u16 hnum,
74 const int dif)
75{
76 struct sock *sk;
3ab5aee7 77 const struct hlist_nulls_node *node;
4f765d84 78 const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
b1a7ffcb
DV
79 /* Optimize here for direct hit, only listening connections can
80 * have wildcards anyways.
81 */
33de014c 82 unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
f373b53b 83 unsigned int slot = hash & hashinfo->ehash_mask;
3ab5aee7 84 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
b1a7ffcb 85
3ab5aee7
ED
86
87 rcu_read_lock();
88begin:
89 sk_nulls_for_each_rcu(sk, node, &head->chain) {
ce43b03e
ED
90 if (sk->sk_hash != hash)
91 continue;
92 if (likely(INET6_MATCH(sk, net, saddr, daddr, ports, dif))) {
3ab5aee7
ED
93 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
94 goto begintw;
ce43b03e
ED
95 if (unlikely(!INET6_MATCH(sk, net, saddr, daddr,
96 ports, dif))) {
3ab5aee7
ED
97 sock_put(sk);
98 goto begin;
99 }
100 goto out;
101 }
b1a7ffcb 102 }
3ab5aee7
ED
103 if (get_nulls_value(node) != slot)
104 goto begin;
105
106begintw:
b1a7ffcb 107 /* Must check for a TIME_WAIT'er before going to listener hash. */
3ab5aee7 108 sk_nulls_for_each_rcu(sk, node, &head->twchain) {
ce43b03e
ED
109 if (sk->sk_hash != hash)
110 continue;
111 if (likely(INET6_TW_MATCH(sk, net, saddr, daddr,
112 ports, dif))) {
3ab5aee7
ED
113 if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) {
114 sk = NULL;
115 goto out;
116 }
ce43b03e
ED
117 if (unlikely(!INET6_TW_MATCH(sk, net, saddr, daddr,
118 ports, dif))) {
3ab5aee7
ED
119 sock_put(sk);
120 goto begintw;
121 }
122 goto out;
123 }
b1a7ffcb 124 }
3ab5aee7
ED
125 if (get_nulls_value(node) != slot)
126 goto begintw;
127 sk = NULL;
128out:
129 rcu_read_unlock();
b1a7ffcb
DV
130 return sk;
131}
132EXPORT_SYMBOL(__inet6_lookup_established);
133
42b16b3f 134static inline int compute_score(struct sock *sk, struct net *net,
c25eb3bf
ED
135 const unsigned short hnum,
136 const struct in6_addr *daddr,
137 const int dif)
138{
139 int score = -1;
140
c720c7e8 141 if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
c25eb3bf
ED
142 sk->sk_family == PF_INET6) {
143 const struct ipv6_pinfo *np = inet6_sk(sk);
144
145 score = 1;
146 if (!ipv6_addr_any(&np->rcv_saddr)) {
147 if (!ipv6_addr_equal(&np->rcv_saddr, daddr))
148 return -1;
149 score++;
150 }
151 if (sk->sk_bound_dev_if) {
152 if (sk->sk_bound_dev_if != dif)
153 return -1;
154 score++;
155 }
156 }
157 return score;
158}
159
d86e0dac
PE
160struct sock *inet6_lookup_listener(struct net *net,
161 struct inet_hashinfo *hashinfo, const struct in6_addr *daddr,
162 const unsigned short hnum, const int dif)
5324a040
ACM
163{
164 struct sock *sk;
c25eb3bf
ED
165 const struct hlist_nulls_node *node;
166 struct sock *result;
167 int score, hiscore;
168 unsigned int hash = inet_lhashfn(net, hnum);
169 struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
170
171 rcu_read_lock();
172begin:
173 result = NULL;
174 hiscore = -1;
175 sk_nulls_for_each(sk, node, &ilb->head) {
176 score = compute_score(sk, net, hnum, daddr, dif);
177 if (score > hiscore) {
178 hiscore = score;
179 result = sk;
5324a040
ACM
180 }
181 }
c25eb3bf
ED
182 /*
183 * if the nulls value we got at the end of this lookup is
184 * not the expected one, we must restart lookup.
185 * We probably met an item that was moved to another chain.
186 */
187 if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
188 goto begin;
189 if (result) {
190 if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
191 result = NULL;
192 else if (unlikely(compute_score(result, net, hnum, daddr,
193 dif) < hiscore)) {
194 sock_put(result);
195 goto begin;
196 }
197 }
198 rcu_read_unlock();
5324a040
ACM
199 return result;
200}
201
202EXPORT_SYMBOL_GPL(inet6_lookup_listener);
203
d86e0dac 204struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
d2ecd9cc
AV
205 const struct in6_addr *saddr, const __be16 sport,
206 const struct in6_addr *daddr, const __be16 dport,
5324a040
ACM
207 const int dif)
208{
209 struct sock *sk;
210
211 local_bh_disable();
d86e0dac 212 sk = __inet6_lookup(net, hashinfo, saddr, sport, daddr, ntohs(dport), dif);
5324a040
ACM
213 local_bh_enable();
214
215 return sk;
216}
217
218EXPORT_SYMBOL_GPL(inet6_lookup);
d8313f5c
ACM
219
220static int __inet6_check_established(struct inet_timewait_death_row *death_row,
221 struct sock *sk, const __u16 lport,
222 struct inet_timewait_sock **twp)
223{
224 struct inet_hashinfo *hinfo = death_row->hashinfo;
3759fa9c 225 struct inet_sock *inet = inet_sk(sk);
d8313f5c
ACM
226 const struct ipv6_pinfo *np = inet6_sk(sk);
227 const struct in6_addr *daddr = &np->rcv_saddr;
228 const struct in6_addr *saddr = &np->daddr;
229 const int dif = sk->sk_bound_dev_if;
c720c7e8 230 const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
33de014c
PE
231 struct net *net = sock_net(sk);
232 const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
c720c7e8 233 inet->inet_dport);
d8313f5c 234 struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
9db66bdc 235 spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
d8313f5c 236 struct sock *sk2;
3ab5aee7 237 const struct hlist_nulls_node *node;
d8313f5c 238 struct inet_timewait_sock *tw;
13475a30 239 int twrefcnt = 0;
d8313f5c 240
9db66bdc 241 spin_lock(lock);
d8313f5c
ACM
242
243 /* Check TIME-WAIT sockets first. */
3ab5aee7 244 sk_nulls_for_each(sk2, node, &head->twchain) {
ce43b03e
ED
245 if (sk2->sk_hash != hash)
246 continue;
d8313f5c 247
ce43b03e
ED
248 if (likely(INET6_TW_MATCH(sk2, net, saddr, daddr,
249 ports, dif))) {
250 tw = inet_twsk(sk2);
d8313f5c
ACM
251 if (twsk_unique(sk, sk2, twp))
252 goto unique;
253 else
254 goto not_unique;
255 }
256 }
257 tw = NULL;
258
259 /* And established part... */
3ab5aee7 260 sk_nulls_for_each(sk2, node, &head->chain) {
ce43b03e
ED
261 if (sk2->sk_hash != hash)
262 continue;
263 if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif)))
d8313f5c
ACM
264 goto not_unique;
265 }
266
267unique:
3759fa9c
HX
268 /* Must record num and sport now. Otherwise we will see
269 * in hash table socket with a funny identity. */
c720c7e8
ED
270 inet->inet_num = lport;
271 inet->inet_sport = htons(lport);
13475a30 272 sk->sk_hash = hash;
547b792c 273 WARN_ON(!sk_unhashed(sk));
3ab5aee7 274 __sk_nulls_add_node_rcu(sk, &head->chain);
13475a30
ED
275 if (tw) {
276 twrefcnt = inet_twsk_unhash(tw);
277 NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
278 }
9db66bdc 279 spin_unlock(lock);
13475a30
ED
280 if (twrefcnt)
281 inet_twsk_put(tw);
c29a0bc4 282 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
d8313f5c 283
13475a30 284 if (twp) {
d8313f5c 285 *twp = tw;
13475a30 286 } else if (tw) {
d8313f5c
ACM
287 /* Silly. Should hash-dance instead... */
288 inet_twsk_deschedule(tw, death_row);
d8313f5c
ACM
289
290 inet_twsk_put(tw);
291 }
292 return 0;
293
294not_unique:
9db66bdc 295 spin_unlock(lock);
d8313f5c
ACM
296 return -EADDRNOTAVAIL;
297}
298
299static inline u32 inet6_sk_port_offset(const struct sock *sk)
300{
301 const struct inet_sock *inet = inet_sk(sk);
302 const struct ipv6_pinfo *np = inet6_sk(sk);
303 return secure_ipv6_port_ephemeral(np->rcv_saddr.s6_addr32,
304 np->daddr.s6_addr32,
c720c7e8 305 inet->inet_dport);
d8313f5c
ACM
306}
307
308int inet6_hash_connect(struct inet_timewait_death_row *death_row,
309 struct sock *sk)
310{
5d8c0aa9 311 return __inet_hash_connect(death_row, sk, inet6_sk_port_offset(sk),
5ee31fc1 312 __inet6_check_established, __inet6_hash);
d8313f5c
ACM
313}
314
315EXPORT_SYMBOL_GPL(inet6_hash_connect);