projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'v3.10.106' into update
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
1da177e4
LT		 91
92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
c3ac8a13
DB		 240static struct net_device *packet_cached_dev_get(struct packet_sock *po)
241{
242 struct net_device *dev;
243
244 rcu_read_lock();
245 dev = rcu_dereference(po->cached_dev);
246 if (likely(dev))
247 dev_hold(dev);
248 rcu_read_unlock();
249
250 return dev;
251}
252
253static void packet_cached_dev_assign(struct packet_sock *po,
254 struct net_device *dev)
255{
256 rcu_assign_pointer(po->cached_dev, dev);
257}
258
259static void packet_cached_dev_reset(struct packet_sock *po)
260{
261 RCU_INIT_POINTER(po->cached_dev, NULL);
262}
263
ce06b03e
DM		 264/* register_prot_hook must be invoked with the po->bind_lock held,
265 * or from a context in which asynchronous accesses to the packet
266 * socket is not possible (packet_create()).
267 */
268static void register_prot_hook(struct sock *sk)
269{
270 struct packet_sock *po = pkt_sk(sk);
026bb405 271
ce06b03e 272 if (!po->running) {
c3ac8a13 273 if (po->fanout)
dc99f600 274 __fanout_link(sk, po);
c3ac8a13 275 else
dc99f600 276 dev_add_pack(&po->prot_hook);
026bb405 277
ce06b03e
DM		 278 sock_hold(sk);
279 po->running = 1;
280 }
281}
282
283/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
284 * held. If the sync parameter is true, we will temporarily drop
285 * the po->bind_lock and do a synchronize_net to make sure no
286 * asynchronous packet processing paths still refer to the elements
287 * of po->prot_hook. If the sync parameter is false, it is the
288 * callers responsibility to take care of this.
289 */
290static void __unregister_prot_hook(struct sock *sk, bool sync)
291{
292 struct packet_sock *po = pkt_sk(sk);
293
294 po->running = 0;
c3ac8a13
DB		 295
296 if (po->fanout)
dc99f600 297 __fanout_unlink(sk, po);
c3ac8a13 298 else
dc99f600 299 __dev_remove_pack(&po->prot_hook);
026bb405 300
ce06b03e
DM		 301 __sock_put(sk);
302
303 if (sync) {
304 spin_unlock(&po->bind_lock);
305 synchronize_net();
306 spin_lock(&po->bind_lock);
307 }
308}
309
310static void unregister_prot_hook(struct sock *sk, bool sync)
311{
312 struct packet_sock *po = pkt_sk(sk);
313
314 if (po->running)
315 __unregister_prot_hook(sk, sync);
316}
317
f6dafa95 318static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 319{
320 if (is_vmalloc_addr(addr))
321 return vmalloc_to_page(addr);
322 return virt_to_page(addr);
323}
324
69e3c75f 325static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 326{
184f489e 327 union tpacket_uhdr h;
1da177e4 328
69e3c75f 329 h.raw = frame;
bbd6ef87
PM		 330 switch (po->tp_version) {
331 case TPACKET_V1:
69e3c75f 332 h.h1->tp_status = status;
0af55bb5 333 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 334 break;
335 case TPACKET_V2:
69e3c75f 336 h.h2->tp_status = status;
0af55bb5 337 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 338 break;
f6fb8f10 339 case TPACKET_V3:
69e3c75f 340 default:
f6fb8f10 341 WARN(1, "TPACKET version not supported.\n");
69e3c75f 342 BUG();
bbd6ef87 343 }
69e3c75f
JB		 344
345 smp_wmb();
bbd6ef87
PM		 346}
347
69e3c75f 348static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 349{
184f489e 350 union tpacket_uhdr h;
bbd6ef87 351
69e3c75f
JB		 352 smp_rmb();
353
bbd6ef87
PM		 354 h.raw = frame;
355 switch (po->tp_version) {
356 case TPACKET_V1:
0af55bb5 357 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 358 return h.h1->tp_status;
bbd6ef87 359 case TPACKET_V2:
0af55bb5 360 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 361 return h.h2->tp_status;
f6fb8f10 362 case TPACKET_V3:
69e3c75f 363 default:
f6fb8f10 364 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 365 BUG();
366 return 0;
bbd6ef87 367 }
1da177e4 368}
69e3c75f 369
b9c32fb2
DB		 370static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
371 unsigned int flags)
7a51384c
DB		 372{
373 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
374
375 if (shhwtstamps) {
376 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
377 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 378 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 379 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
380 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 381 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 382 }
383
384 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 385 return TP_STATUS_TS_SOFTWARE;
7a51384c 386
b9c32fb2 387 return 0;
7a51384c
DB		 388}
389
b9c32fb2
DB		 390static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
391 struct sk_buff *skb)
2e31396f
WB		 392{
393 union tpacket_uhdr h;
394 struct timespec ts;
b9c32fb2 395 __u32 ts_status;
2e31396f 396
b9c32fb2
DB		 397 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
398 return 0;
2e31396f
WB		 399
400 h.raw = frame;
401 switch (po->tp_version) {
402 case TPACKET_V1:
403 h.h1->tp_sec = ts.tv_sec;
404 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
405 break;
406 case TPACKET_V2:
407 h.h2->tp_sec = ts.tv_sec;
408 h.h2->tp_nsec = ts.tv_nsec;
409 break;
410 case TPACKET_V3:
411 default:
412 WARN(1, "TPACKET version not supported.\n");
413 BUG();
414 }
415
416 /* one flush is safe, as both fields always lie on the same cacheline */
417 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
418 smp_wmb();
b9c32fb2
DB		 419
420 return ts_status;
2e31396f
WB		 421}
422
69e3c75f
JB		 423static void *packet_lookup_frame(struct packet_sock *po,
424 struct packet_ring_buffer *rb,
425 unsigned int position,
426 int status)
427{
428 unsigned int pg_vec_pos, frame_offset;
184f489e 429 union tpacket_uhdr h;
69e3c75f
JB		 430
431 pg_vec_pos = position / rb->frames_per_block;
432 frame_offset = position % rb->frames_per_block;
433
0e3125c7
NH		 434 h.raw = rb->pg_vec[pg_vec_pos].buffer +
435 (frame_offset * rb->frame_size);
69e3c75f
JB		 436
437 if (status != __packet_get_status(po, h.raw))
438 return NULL;
439
440 return h.raw;
441}
442
eea49cc9 443static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 444 struct packet_ring_buffer *rb,
445 int status)
446{
447 return packet_lookup_frame(po, rb, rb->head, status);
448}
449
bc59ba39 450static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 451{
452 del_timer_sync(&pkc->retire_blk_timer);
453}
454
455static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
456 int tx_ring,
457 struct sk_buff_head *rb_queue)
458{
bc59ba39 459 struct tpacket_kbdq_core *pkc;
f6fb8f10 460
461 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
462
5b9e9be7 463 spin_lock_bh(&rb_queue->lock);
f6fb8f10 464 pkc->delete_blk_timer = 1;
5b9e9be7 465 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 466
467 prb_del_retire_blk_timer(pkc);
468}
469
470static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 471 struct tpacket_kbdq_core *pkc,
f6fb8f10 472 void (*func) (unsigned long))
473{
474 init_timer(&pkc->retire_blk_timer);
475 pkc->retire_blk_timer.data = (long)po;
476 pkc->retire_blk_timer.function = func;
477 pkc->retire_blk_timer.expires = jiffies;
478}
479
480static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
481{
bc59ba39 482 struct tpacket_kbdq_core *pkc;
f6fb8f10 483
484 if (tx_ring)
485 BUG();
486
487 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
488 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
489}
490
491static int prb_calc_retire_blk_tmo(struct packet_sock *po,
492 int blk_size_in_bytes)
493{
494 struct net_device *dev;
495 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 496 struct ethtool_cmd ecmd;
497 int err;
e440cf2c 498 u32 speed;
f6fb8f10 499
4bc71cb9
JP		 500 rtnl_lock();
501 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
502 if (unlikely(!dev)) {
503 rtnl_unlock();
f6fb8f10 504 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 505 }
506 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 507 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 508 rtnl_unlock();
509 if (!err) {
4bc71cb9
JP		 510 /*
511 * If the link speed is so slow you don't really
512 * need to worry about perf anyways
513 */
e440cf2c 514 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 515 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 516 } else {
517 msec = 1;
518 div = speed / 1000;
f6fb8f10 519 }
520 }
521
522 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
523
524 if (div)
525 mbits /= div;
526
527 tmo = mbits * msec;
528
529 if (div)
530 return tmo+1;
531 return tmo;
532}
533
bc59ba39 534static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 535 union tpacket_req_u *req_u)
536{
537 p1->feature_req_word = req_u->req3.tp_feature_req_word;
538}
539
540static void init_prb_bdqc(struct packet_sock *po,
541 struct packet_ring_buffer *rb,
542 struct pgv *pg_vec,
543 union tpacket_req_u *req_u, int tx_ring)
544{
bc59ba39 545 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
546 struct tpacket_block_desc *pbd;
f6fb8f10 547
548 memset(p1, 0x0, sizeof(*p1));
549
550 p1->knxt_seq_num = 1;
551 p1->pkbdq = pg_vec;
bc59ba39 552 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 553 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 554 p1->kblk_size = req_u->req3.tp_block_size;
555 p1->knum_blocks = req_u->req3.tp_block_nr;
556 p1->hdrlen = po->tp_hdrlen;
557 p1->version = po->tp_version;
558 p1->last_kactive_blk_num = 0;
ee80fbf3 559 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 560 if (req_u->req3.tp_retire_blk_tov)
561 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
562 else
563 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
564 req_u->req3.tp_block_size);
565 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
566 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
567
4035ed7b 568 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 569 prb_init_ft_ops(p1, req_u);
570 prb_setup_retire_blk_timer(po, tx_ring);
571 prb_open_block(p1, pbd);
572}
573
574/* Do NOT update the last_blk_num first.
575 * Assumes sk_buff_head lock is held.
576 */
bc59ba39 577static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 578{
579 mod_timer(&pkc->retire_blk_timer,
580 jiffies + pkc->tov_in_jiffies);
581 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
582}
583
584/*
585 * Timer logic:
586 * 1) We refresh the timer only when we open a block.
587 * By doing this we don't waste cycles refreshing the timer
588 * on packet-by-packet basis.
589 *
590 * With a 1MB block-size, on a 1Gbps line, it will take
591 * i) ~8 ms to fill a block + ii) memcpy etc.
592 * In this cut we are not accounting for the memcpy time.
593 *
594 * So, if the user sets the 'tmo' to 10ms then the timer
595 * will never fire while the block is still getting filled
596 * (which is what we want). However, the user could choose
597 * to close a block early and that's fine.
598 *
599 * But when the timer does fire, we check whether or not to refresh it.
600 * Since the tmo granularity is in msecs, it is not too expensive
601 * to refresh the timer, lets say every '8' msecs.
602 * Either the user can set the 'tmo' or we can derive it based on
603 * a) line-speed and b) block-size.
604 * prb_calc_retire_blk_tmo() calculates the tmo.
605 *
606 */
607static void prb_retire_rx_blk_timer_expired(unsigned long data)
608{
609 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 610 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 611 unsigned int frozen;
bc59ba39 612 struct tpacket_block_desc *pbd;
f6fb8f10 613
614 spin_lock(&po->sk.sk_receive_queue.lock);
615
616 frozen = prb_queue_frozen(pkc);
617 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
618
619 if (unlikely(pkc->delete_blk_timer))
620 goto out;
621
622 /* We only need to plug the race when the block is partially filled.
623 * tpacket_rcv:
624 * lock(); increment BLOCK_NUM_PKTS; unlock()
625 * copy_bits() is in progress ...
626 * timer fires on other cpu:
627 * we can't retire the current block because copy_bits
628 * is in progress.
629 *
630 */
631 if (BLOCK_NUM_PKTS(pbd)) {
632 while (atomic_read(&pkc->blk_fill_in_prog)) {
633 /* Waiting for skb_copy_bits to finish... */
634 cpu_relax();
635 }
636 }
637
638 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
639 if (!frozen) {
640 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
641 if (!prb_dispatch_next_block(pkc, po))
642 goto refresh_timer;
643 else
644 goto out;
645 } else {
646 /* Case 1. Queue was frozen because user-space was
647 * lagging behind.
648 */
649 if (prb_curr_blk_in_use(pkc, pbd)) {
650 /*
651 * Ok, user-space is still behind.
652 * So just refresh the timer.
653 */
654 goto refresh_timer;
655 } else {
656 /* Case 2. queue was frozen,user-space caught up,
657 * now the link went idle && the timer fired.
658 * We don't have a block to close.So we open this
659 * block and restart the timer.
660 * opening a block thaws the queue,restarts timer
661 * Thawing/timer-refresh is a side effect.
662 */
663 prb_open_block(pkc, pbd);
664 goto out;
665 }
666 }
667 }
668
669refresh_timer:
670 _prb_refresh_rx_retire_blk_timer(pkc);
671
672out:
673 spin_unlock(&po->sk.sk_receive_queue.lock);
674}
675
eea49cc9 676static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 677 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 678{
679 /* Flush everything minus the block header */
680
681#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
682 u8 *start, *end;
683
684 start = (u8 *)pbd1;
685
686 /* Skip the block header(we know header WILL fit in 4K) */
687 start += PAGE_SIZE;
688
689 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
690 for (; start < end; start += PAGE_SIZE)
691 flush_dcache_page(pgv_to_page(start));
692
693 smp_wmb();
694#endif
695
696 /* Now update the block status. */
697
698 BLOCK_STATUS(pbd1) = status;
699
700 /* Flush the block header */
701
702#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
703 start = (u8 *)pbd1;
704 flush_dcache_page(pgv_to_page(start));
705
706 smp_wmb();
707#endif
708}
709
710/*
711 * Side effect:
712 *
713 * 1) flush the block
714 * 2) Increment active_blk_num
715 *
716 * Note:We DONT refresh the timer on purpose.
717 * Because almost always the next block will be opened.
718 */
bc59ba39 719static void prb_close_block(struct tpacket_kbdq_core *pkc1,
720 struct tpacket_block_desc *pbd1,
f6fb8f10 721 struct packet_sock *po, unsigned int stat)
722{
723 __u32 status = TP_STATUS_USER | stat;
724
725 struct tpacket3_hdr *last_pkt;
bc59ba39 726 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 727
ee80fbf3 728 if (po->stats.stats3.tp_drops)
f6fb8f10 729 status |= TP_STATUS_LOSING;
730
731 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
732 last_pkt->tp_next_offset = 0;
733
734 /* Get the ts of the last pkt */
735 if (BLOCK_NUM_PKTS(pbd1)) {
736 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
737 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
738 } else {
739 /* Ok, we tmo'd - so get the current time */
740 struct timespec ts;
741 getnstimeofday(&ts);
742 h1->ts_last_pkt.ts_sec = ts.tv_sec;
743 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
744 }
745
746 smp_wmb();
747
748 /* Flush the block */
749 prb_flush_block(pkc1, pbd1, status);
750
751 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
752}
753
eea49cc9 754static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 755{
756 pkc->reset_pending_on_curr_blk = 0;
757}
758
759/*
760 * Side effect of opening a block:
761 *
762 * 1) prb_queue is thawed.
763 * 2) retire_blk_timer is refreshed.
764 *
765 */
bc59ba39 766static void prb_open_block(struct tpacket_kbdq_core *pkc1,
767 struct tpacket_block_desc *pbd1)
f6fb8f10 768{
769 struct timespec ts;
bc59ba39 770 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 771
772 smp_rmb();
773
8da3056c
DB		 774 /* We could have just memset this but we will lose the
775 * flexibility of making the priv area sticky
776 */
f6fb8f10 777
8da3056c
DB		 778 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
779 BLOCK_NUM_PKTS(pbd1) = 0;
780 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 781
8da3056c
DB		 782 getnstimeofday(&ts);
783
784 h1->ts_first_pkt.ts_sec = ts.tv_sec;
785 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 786
8da3056c
DB		 787 pkc1->pkblk_start = (char *)pbd1;
788 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
789
790 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
791 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
792
793 pbd1->version = pkc1->version;
794 pkc1->prev = pkc1->nxt_offset;
795 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
796
797 prb_thaw_queue(pkc1);
798 _prb_refresh_rx_retire_blk_timer(pkc1);
799
800 smp_wmb();
f6fb8f10 801}
802
803/*
804 * Queue freeze logic:
805 * 1) Assume tp_block_nr = 8 blocks.
806 * 2) At time 't0', user opens Rx ring.
807 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
808 * 4) user-space is either sleeping or processing block '0'.
809 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
810 * it will close block-7,loop around and try to fill block '0'.
811 * call-flow:
812 * __packet_lookup_frame_in_block
813 * prb_retire_current_block()
814 * prb_dispatch_next_block()
815 * |->(BLOCK_STATUS == USER) evaluates to true
816 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
817 * 6) Now there are two cases:
818 * 6.1) Link goes idle right after the queue is frozen.
819 * But remember, the last open_block() refreshed the timer.
820 * When this timer expires,it will refresh itself so that we can
821 * re-open block-0 in near future.
822 * 6.2) Link is busy and keeps on receiving packets. This is a simple
823 * case and __packet_lookup_frame_in_block will check if block-0
824 * is free and can now be re-used.
825 */
eea49cc9 826static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 827 struct packet_sock *po)
828{
829 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 830 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 831}
832
833#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
834
835/*
836 * If the next block is free then we will dispatch it
837 * and return a good offset.
838 * Else, we will freeze the queue.
839 * So, caller must check the return value.
840 */
bc59ba39 841static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 842 struct packet_sock *po)
843{
bc59ba39 844 struct tpacket_block_desc *pbd;
f6fb8f10 845
846 smp_rmb();
847
848 /* 1. Get current block num */
849 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
850
851 /* 2. If this block is currently in_use then freeze the queue */
852 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
853 prb_freeze_queue(pkc, po);
854 return NULL;
855 }
856
857 /*
858 * 3.
859 * open this block and return the offset where the first packet
860 * needs to get stored.
861 */
862 prb_open_block(pkc, pbd);
863 return (void *)pkc->nxt_offset;
864}
865
bc59ba39 866static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 867 struct packet_sock *po, unsigned int status)
868{
bc59ba39 869 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 870
871 /* retire/close the current block */
872 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
873 /*
874 * Plug the case where copy_bits() is in progress on
875 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
876 * have space to copy the pkt in the current block and
877 * called prb_retire_current_block()
878 *
879 * We don't need to worry about the TMO case because
880 * the timer-handler already handled this case.
881 */
882 if (!(status & TP_STATUS_BLK_TMO)) {
883 while (atomic_read(&pkc->blk_fill_in_prog)) {
884 /* Waiting for skb_copy_bits to finish... */
885 cpu_relax();
886 }
887 }
888 prb_close_block(pkc, pbd, po, status);
889 return;
890 }
f6fb8f10 891}
892
eea49cc9 893static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 894 struct tpacket_block_desc *pbd)
f6fb8f10 895{
896 return TP_STATUS_USER & BLOCK_STATUS(pbd);
897}
898
eea49cc9 899static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 900{
901 return pkc->reset_pending_on_curr_blk;
902}
903
eea49cc9 904static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 905{
bc59ba39 906 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 907 atomic_dec(&pkc->blk_fill_in_prog);
908}
909
eea49cc9 910static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 911 struct tpacket3_hdr *ppd)
912{
913 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
914}
915
eea49cc9 916static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 917 struct tpacket3_hdr *ppd)
918{
919 ppd->hv1.tp_rxhash = 0;
920}
921
eea49cc9 922static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 923 struct tpacket3_hdr *ppd)
924{
925 if (vlan_tx_tag_present(pkc->skb)) {
926 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
927 ppd->tp_status = TP_STATUS_VLAN_VALID;
928 } else {
9e67030a 929 ppd->hv1.tp_vlan_tci = 0;
930 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 931 }
932}
933
bc59ba39 934static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 935 struct tpacket3_hdr *ppd)
936{
937 prb_fill_vlan_info(pkc, ppd);
938
939 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
940 prb_fill_rxhash(pkc, ppd);
941 else
942 prb_clear_rxhash(pkc, ppd);
943}
944
eea49cc9 945static void prb_fill_curr_block(char *curr,
bc59ba39 946 struct tpacket_kbdq_core *pkc,
947 struct tpacket_block_desc *pbd,
f6fb8f10 948 unsigned int len)
949{
950 struct tpacket3_hdr *ppd;
951
952 ppd = (struct tpacket3_hdr *)curr;
953 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
954 pkc->prev = curr;
955 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
956 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
957 BLOCK_NUM_PKTS(pbd) += 1;
958 atomic_inc(&pkc->blk_fill_in_prog);
959 prb_run_all_ft_ops(pkc, ppd);
960}
961
962/* Assumes caller has the sk->rx_queue.lock */
963static void *__packet_lookup_frame_in_block(struct packet_sock *po,
964 struct sk_buff *skb,
965 int status,
966 unsigned int len
967 )
968{
bc59ba39 969 struct tpacket_kbdq_core *pkc;
970 struct tpacket_block_desc *pbd;
f6fb8f10 971 char *curr, *end;
972
e3192690 973 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 974 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
975
976 /* Queue is frozen when user space is lagging behind */
977 if (prb_queue_frozen(pkc)) {
978 /*
979 * Check if that last block which caused the queue to freeze,
980 * is still in_use by user-space.
981 */
982 if (prb_curr_blk_in_use(pkc, pbd)) {
983 /* Can't record this packet */
984 return NULL;
985 } else {
986 /*
987 * Ok, the block was released by user-space.
988 * Now let's open that block.
989 * opening a block also thaws the queue.
990 * Thawing is a side effect.
991 */
992 prb_open_block(pkc, pbd);
993 }
994 }
995
996 smp_mb();
997 curr = pkc->nxt_offset;
998 pkc->skb = skb;
e3192690 999 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1000
1001 /* first try the current block */
1002 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1003 prb_fill_curr_block(curr, pkc, pbd, len);
1004 return (void *)curr;
1005 }
1006
1007 /* Ok, close the current block */
1008 prb_retire_current_block(pkc, po, 0);
1009
1010 /* Now, try to dispatch the next block */
1011 curr = (char *)prb_dispatch_next_block(pkc, po);
1012 if (curr) {
1013 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1014 prb_fill_curr_block(curr, pkc, pbd, len);
1015 return (void *)curr;
1016 }
1017
1018 /*
1019 * No free blocks are available.user_space hasn't caught up yet.
1020 * Queue was just frozen and now this packet will get dropped.
1021 */
1022 return NULL;
1023}
1024
eea49cc9 1025static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1026 struct sk_buff *skb,
1027 int status, unsigned int len)
1028{
1029 char *curr = NULL;
1030 switch (po->tp_version) {
1031 case TPACKET_V1:
1032 case TPACKET_V2:
1033 curr = packet_lookup_frame(po, &po->rx_ring,
1034 po->rx_ring.head, status);
1035 return curr;
1036 case TPACKET_V3:
1037 return __packet_lookup_frame_in_block(po, skb, status, len);
1038 default:
1039 WARN(1, "TPACKET version not supported\n");
1040 BUG();
99aa3473 1041 return NULL;
f6fb8f10 1042 }
1043}
1044
eea49cc9 1045static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1046 struct packet_ring_buffer *rb,
77f65ebd 1047 unsigned int idx,
f6fb8f10 1048 int status)
1049{
bc59ba39 1050 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1051 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1052
1053 if (status != BLOCK_STATUS(pbd))
1054 return NULL;
1055 return pbd;
1056}
1057
eea49cc9 1058static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1059{
1060 unsigned int prev;
1061 if (rb->prb_bdqc.kactive_blk_num)
1062 prev = rb->prb_bdqc.kactive_blk_num-1;
1063 else
1064 prev = rb->prb_bdqc.knum_blocks-1;
1065 return prev;
1066}
1067
1068/* Assumes caller has held the rx_queue.lock */
eea49cc9 1069static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1070 struct packet_ring_buffer *rb,
1071 int status)
1072{
1073 unsigned int previous = prb_previous_blk_num(rb);
1074 return prb_lookup_block(po, rb, previous, status);
1075}
1076
eea49cc9 1077static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1078 struct packet_ring_buffer *rb,
1079 int status)
1080{
1081 if (po->tp_version <= TPACKET_V2)
1082 return packet_previous_frame(po, rb, status);
1083
1084 return __prb_previous_block(po, rb, status);
1085}
1086
eea49cc9 1087static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1088 struct packet_ring_buffer *rb)
1089{
1090 switch (po->tp_version) {
1091 case TPACKET_V1:
1092 case TPACKET_V2:
1093 return packet_increment_head(rb);
1094 case TPACKET_V3:
1095 default:
1096 WARN(1, "TPACKET version not supported.\n");
1097 BUG();
1098 return;
1099 }
1100}
1101
eea49cc9 1102static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1103 struct packet_ring_buffer *rb,
1104 int status)
1105{
1106 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1107 return packet_lookup_frame(po, rb, previous, status);
1108}
1109
eea49cc9 1110static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1111{
1112 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1113}
1114
77f65ebd
WB		 1115static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1116{
1117 struct sock *sk = &po->sk;
1118 bool has_room;
1119
1120 if (po->prot_hook.func != tpacket_rcv)
1121 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1122 <= sk->sk_rcvbuf;
1123
1124 spin_lock(&sk->sk_receive_queue.lock);
1125 if (po->tp_version == TPACKET_V3)
1126 has_room = prb_lookup_block(po, &po->rx_ring,
1127 po->rx_ring.prb_bdqc.kactive_blk_num,
1128 TP_STATUS_KERNEL);
1129 else
1130 has_room = packet_lookup_frame(po, &po->rx_ring,
1131 po->rx_ring.head,
1132 TP_STATUS_KERNEL);
1133 spin_unlock(&sk->sk_receive_queue.lock);
1134
1135 return has_room;
1136}
1137
1da177e4
LT		 1138static void packet_sock_destruct(struct sock *sk)
1139{
ed85b565
RC		 1140 skb_queue_purge(&sk->sk_error_queue);
1141
547b792c
IJ		 1142 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1143 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1144
1145 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1146 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1147 return;
1148 }
1149
17ab56a2 1150 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1151}
1152
77f65ebd
WB		 1153static unsigned int fanout_demux_hash(struct packet_fanout *f,
1154 struct sk_buff *skb,
1155 unsigned int num)
dc99f600 1156{
77f65ebd 1157 return (((u64)skb->rxhash) * num) >> 32;
dc99f600
DM		 1158}
1159
77f65ebd
WB		 1160static unsigned int fanout_demux_lb(struct packet_fanout *f,
1161 struct sk_buff *skb,
1162 unsigned int num)
dc99f600 1163{
4552ddd0 1164 unsigned int val = atomic_inc_return(&f->rr_cur);
dc99f600 1165
4552ddd0 1166 return val % num;
77f65ebd
WB		 1167}
1168
1169static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1170 struct sk_buff *skb,
1171 unsigned int num)
1172{
1173 return smp_processor_id() % num;
dc99f600
DM		 1174}
1175
77f65ebd
WB		 1176static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1177 struct sk_buff *skb,
1178 unsigned int idx, unsigned int skip,
1179 unsigned int num)
95ec3eb4 1180{
77f65ebd 1181 unsigned int i, j;
95ec3eb4 1182
77f65ebd
WB		 1183 i = j = min_t(int, f->next[idx], num - 1);
1184 do {
1185 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1186 if (i != j)
1187 f->next[idx] = i;
1188 return i;
1189 }
1190 if (++i == num)
1191 i = 0;
1192 } while (i != j);
1193
1194 return idx;
1195}
1196
1197static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1198{
1199 return f->flags & (flag >> 8);
95ec3eb4
DM		 1200}
1201
95ec3eb4
DM		 1202static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1203 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1204{
1205 struct packet_fanout *f = pt->af_packet_priv;
7eee92bf 1206 unsigned int num = ACCESS_ONCE(f->num_members);
dc99f600 1207 struct packet_sock *po;
77f65ebd 1208 unsigned int idx;
dc99f600
DM		 1209
1210 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1211 !num) {
1212 kfree_skb(skb);
1213 return 0;
1214 }
1215
95ec3eb4
DM		 1216 switch (f->type) {
1217 case PACKET_FANOUT_HASH:
1218 default:
77f65ebd 1219 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1220 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1221 if (!skb)
1222 return 0;
1223 }
1224 skb_get_rxhash(skb);
77f65ebd 1225 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1226 break;
1227 case PACKET_FANOUT_LB:
77f65ebd 1228 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1229 break;
1230 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1231 idx = fanout_demux_cpu(f, skb, num);
1232 break;
1233 case PACKET_FANOUT_ROLLOVER:
1234 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1235 break;
dc99f600
DM		 1236 }
1237
77f65ebd
WB		 1238 po = pkt_sk(f->arr[idx]);
1239 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1240 unlikely(!packet_rcv_has_room(po, skb))) {
1241 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1242 po = pkt_sk(f->arr[idx]);
1243 }
dc99f600
DM		 1244
1245 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1246}
1247
fff3321d
PE		 1248DEFINE_MUTEX(fanout_mutex);
1249EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1250static LIST_HEAD(fanout_list);
1251
1252static void __fanout_link(struct sock *sk, struct packet_sock *po)
1253{
1254 struct packet_fanout *f = po->fanout;
1255
1256 spin_lock(&f->lock);
1257 f->arr[f->num_members] = sk;
1258 smp_wmb();
1259 f->num_members++;
9aab3592
AS		 1260 if (f->num_members == 1)
1261 dev_add_pack(&f->prot_hook);
dc99f600
DM		 1262 spin_unlock(&f->lock);
1263}
1264
1265static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1266{
1267 struct packet_fanout *f = po->fanout;
1268 int i;
1269
1270 spin_lock(&f->lock);
1271 for (i = 0; i < f->num_members; i++) {
1272 if (f->arr[i] == sk)
1273 break;
1274 }
1275 BUG_ON(i >= f->num_members);
1276 f->arr[i] = f->arr[f->num_members - 1];
1277 f->num_members--;
9aab3592
AS		 1278 if (f->num_members == 0)
1279 __dev_remove_pack(&f->prot_hook);
dc99f600
DM		 1280 spin_unlock(&f->lock);
1281}
1282
a0dfb263 1283static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1284{
1285 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1286 return true;
1287
1288 return false;
1289}
1290
7736d33f 1291static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1292{
1293 struct packet_sock *po = pkt_sk(sk);
1294 struct packet_fanout *f, *match;
7736d33f 1295 u8 type = type_flags & 0xff;
77f65ebd 1296 u8 flags = type_flags >> 8;
dc99f600
DM		 1297 int err;
1298
1299 switch (type) {
77f65ebd
WB		 1300 case PACKET_FANOUT_ROLLOVER:
1301 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1302 return -EINVAL;
dc99f600
DM		 1303 case PACKET_FANOUT_HASH:
1304 case PACKET_FANOUT_LB:
95ec3eb4 1305 case PACKET_FANOUT_CPU:
dc99f600
DM		 1306 break;
1307 default:
1308 return -EINVAL;
1309 }
1310
2a272abc
ED		 1311 mutex_lock(&fanout_mutex);
1312
1313 err = -EINVAL;
dc99f600 1314 if (!po->running)
2a272abc 1315 goto out;
dc99f600 1316
2a272abc 1317 err = -EALREADY;
dc99f600 1318 if (po->fanout)
2a272abc 1319 goto out;
dc99f600 1320
dc99f600
DM		 1321 match = NULL;
1322 list_for_each_entry(f, &fanout_list, list) {
1323 if (f->id == id &&
1324 read_pnet(&f->net) == sock_net(sk)) {
1325 match = f;
1326 break;
1327 }
1328 }
afe62c68 1329 err = -EINVAL;
77f65ebd 1330 if (match && match->flags != flags)
afe62c68 1331 goto out;
dc99f600 1332 if (!match) {
afe62c68 1333 err = -ENOMEM;
dc99f600 1334 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1335 if (!match)
1336 goto out;
1337 write_pnet(&match->net, sock_net(sk));
1338 match->id = id;
1339 match->type = type;
77f65ebd 1340 match->flags = flags;
afe62c68
ED		 1341 atomic_set(&match->rr_cur, 0);
1342 INIT_LIST_HEAD(&match->list);
1343 spin_lock_init(&match->lock);
1344 atomic_set(&match->sk_ref, 0);
1345 match->prot_hook.type = po->prot_hook.type;
1346 match->prot_hook.dev = po->prot_hook.dev;
1347 match->prot_hook.func = packet_rcv_fanout;
1348 match->prot_hook.af_packet_priv = match;
c0de08d0 1349 match->prot_hook.id_match = match_fanout_group;
afe62c68 1350 list_add(&match->list, &fanout_list);
dc99f600 1351 }
afe62c68
ED		 1352 err = -EINVAL;
1353 if (match->type == type &&
1354 match->prot_hook.type == po->prot_hook.type &&
1355 match->prot_hook.dev == po->prot_hook.dev) {
1356 err = -ENOSPC;
1357 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1358 __dev_remove_pack(&po->prot_hook);
1359 po->fanout = match;
1360 atomic_inc(&match->sk_ref);
1361 __fanout_link(sk, po);
1362 err = 0;
dc99f600
DM		 1363 }
1364 }
afe62c68 1365out:
dc99f600
DM		 1366 mutex_unlock(&fanout_mutex);
1367 return err;
1368}
1369
9aab3592
AS		 1370/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1371 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1372 * It is the responsibility of the caller to call fanout_release_data() and
1373 * free the returned packet_fanout (after synchronize_net())
1374 */
1375static struct packet_fanout *fanout_release(struct sock *sk)
dc99f600
DM		 1376{
1377 struct packet_sock *po = pkt_sk(sk);
1378 struct packet_fanout *f;
1379
fff3321d 1380 mutex_lock(&fanout_mutex);
2a272abc
ED		 1381 f = po->fanout;
1382 if (f) {
1383 po->fanout = NULL;
dc99f600 1384
9aab3592 1385 if (atomic_dec_and_test(&f->sk_ref))
2a272abc 1386 list_del(&f->list);
9aab3592
AS		 1387 else
1388 f = NULL;
dc99f600
DM		 1389 }
1390 mutex_unlock(&fanout_mutex);
9aab3592
AS		 1391
1392 return f;
dc99f600 1393}
1da177e4 1394
90ddc4f0 1395static const struct proto_ops packet_ops;
1da177e4 1396
90ddc4f0 1397static const struct proto_ops packet_ops_spkt;
1da177e4 1398
40d4e3df
ED		 1399static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1400 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1401{
1402 struct sock *sk;
1403 struct sockaddr_pkt *spkt;
1404
1405 /*
1406 * When we registered the protocol we saved the socket in the data
1407 * field for just this event.
1408 */
1409
1410 sk = pt->af_packet_priv;
1ce4f28b 1411
1da177e4
LT		 1412 /*
1413 * Yank back the headers [hope the device set this
1414 * right or kerboom...]
1415 *
1416 * Incoming packets have ll header pulled,
1417 * push it back.
1418 *
98e399f8 1419 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1420 * so that this procedure is noop.
1421 */
1422
1423 if (skb->pkt_type == PACKET_LOOPBACK)
1424 goto out;
1425
09ad9bc7 1426 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1427 goto out;
1428
40d4e3df
ED		 1429 skb = skb_share_check(skb, GFP_ATOMIC);
1430 if (skb == NULL)
1da177e4
LT		 1431 goto oom;
1432
1433 /* drop any routing info */
adf30907 1434 skb_dst_drop(skb);
1da177e4 1435
84531c24
PO		 1436 /* drop conntrack reference */
1437 nf_reset(skb);
1438
ffbc6111 1439 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1440
98e399f8 1441 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1442
1443 /*
1444 * The SOCK_PACKET socket receives _all_ frames.
1445 */
1446
1447 spkt->spkt_family = dev->type;
1448 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1449 spkt->spkt_protocol = skb->protocol;
1450
1451 /*
1452 * Charge the memory to the socket. This is done specifically
1453 * to prevent sockets using all the memory up.
1454 */
1455
40d4e3df 1456 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1457 return 0;
1458
1459out:
1460 kfree_skb(skb);
1461oom:
1462 return 0;
1463}
1464
1465
1466/*
1467 * Output a raw packet to a device layer. This bypasses all the other
1468 * protocol layers and you must therefore supply it with a complete frame
1469 */
1ce4f28b 1470
1da177e4
LT		 1471static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1472 struct msghdr *msg, size_t len)
1473{
1474 struct sock *sk = sock->sk;
40d4e3df 1475 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1476 struct sk_buff *skb = NULL;
1da177e4 1477 struct net_device *dev;
40d4e3df 1478 __be16 proto = 0;
1da177e4 1479 int err;
3bdc0eba 1480 int extra_len = 0;
1ce4f28b 1481
1da177e4 1482 /*
1ce4f28b 1483 * Get and verify the address.
1da177e4
LT		 1484 */
1485
40d4e3df 1486 if (saddr) {
1da177e4 1487 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1488 return -EINVAL;
1489 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1490 proto = saddr->spkt_protocol;
1491 } else
1492 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1493
1494 /*
1ce4f28b 1495 * Find the device first to size check it
1da177e4
LT		 1496 */
1497
de74e92a 1498 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1499retry:
654d1f8a
ED		 1500 rcu_read_lock();
1501 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1502 err = -ENODEV;
1503 if (dev == NULL)
1504 goto out_unlock;
1ce4f28b 1505
d5e76b0a
DM		 1506 err = -ENETDOWN;
1507 if (!(dev->flags & IFF_UP))
1508 goto out_unlock;
1509
1da177e4 1510 /*
40d4e3df
ED		 1511 * You may not queue a frame bigger than the mtu. This is the lowest level
1512 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1513 */
1ce4f28b 1514
3bdc0eba
BG		 1515 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1516 if (!netif_supports_nofcs(dev)) {
1517 err = -EPROTONOSUPPORT;
1518 goto out_unlock;
1519 }
1520 extra_len = 4; /* We're doing our own CRC */
1521 }
1522
1da177e4 1523 err = -EMSGSIZE;
3bdc0eba 1524 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1525 goto out_unlock;
1526
1a35ca80
ED		 1527 if (!skb) {
1528 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1529 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1530 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1531
1532 rcu_read_unlock();
4ce40912 1533 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1534 if (skb == NULL)
1535 return -ENOBUFS;
1536 /* FIXME: Save some space for broken drivers that write a hard
1537 * header at transmission time by themselves. PPP is the notable
1538 * one here. This should really be fixed at the driver level.
1539 */
1540 skb_reserve(skb, reserved);
1541 skb_reset_network_header(skb);
1542
1543 /* Try to align data part correctly */
1544 if (hhlen) {
1545 skb->data -= hhlen;
1546 skb->tail -= hhlen;
1547 if (len < hhlen)
1548 skb_reset_network_header(skb);
1549 }
1550 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1551 if (err)
1552 goto out_free;
1553 goto retry;
1da177e4
LT		 1554 }
1555
3bdc0eba 1556 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1557 /* Earlier code assumed this would be a VLAN pkt,
1558 * double-check this now that we have the actual
1559 * packet in hand.
1560 */
1561 struct ethhdr *ehdr;
1562 skb_reset_mac_header(skb);
1563 ehdr = eth_hdr(skb);
1564 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1565 err = -EMSGSIZE;
1566 goto out_unlock;
1567 }
1568 }
1a35ca80 1569
1da177e4
LT		 1570 skb->protocol = proto;
1571 skb->dev = dev;
1572 skb->priority = sk->sk_priority;
2d37a186 1573 skb->mark = sk->sk_mark;
bf84a010
DB		 1574
1575 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1576
3bdc0eba
BG		 1577 if (unlikely(extra_len == 4))
1578 skb->no_fcs = 1;
1579
40893fd0 1580 skb_probe_transport_header(skb, 0);
c1aad275 1581
1da177e4 1582 dev_queue_xmit(skb);
654d1f8a 1583 rcu_read_unlock();
40d4e3df 1584 return len;
1da177e4 1585
1da177e4 1586out_unlock:
654d1f8a 1587 rcu_read_unlock();
1a35ca80
ED		 1588out_free:
1589 kfree_skb(skb);
1da177e4
LT		 1590 return err;
1591}
1da177e4 1592
eea49cc9 1593static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1594 const struct sock *sk,
dbcb5855 1595 unsigned int res)
1da177e4
LT		 1596{
1597 struct sk_filter *filter;
fda9ef5d 1598
80f8f102
ED		 1599 rcu_read_lock();
1600 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1601 if (filter != NULL)
0a14842f 1602 res = SK_RUN_FILTER(filter, skb);
80f8f102 1603 rcu_read_unlock();
1da177e4 1604
dbcb5855 1605 return res;
1da177e4
LT		 1606}
1607
1608/*
62ab0812
ED		 1609 * This function makes lazy skb cloning in hope that most of packets
1610 * are discarded by BPF.
1611 *
1612 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1613 * and skb->cb are mangled. It works because (and until) packets
1614 * falling here are owned by current CPU. Output packets are cloned
1615 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1616 * sequencially, so that if we return skb to original state on exit,
1617 * we will not harm anyone.
1da177e4
LT		 1618 */
1619
40d4e3df
ED		 1620static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1621 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1622{
1623 struct sock *sk;
1624 struct sockaddr_ll *sll;
1625 struct packet_sock *po;
40d4e3df 1626 u8 *skb_head = skb->data;
1da177e4 1627 int skb_len = skb->len;
dbcb5855 1628 unsigned int snaplen, res;
1da177e4
LT		 1629
1630 if (skb->pkt_type == PACKET_LOOPBACK)
1631 goto drop;
1632
1633 sk = pt->af_packet_priv;
1634 po = pkt_sk(sk);
1635
09ad9bc7 1636 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1637 goto drop;
1638
1da177e4
LT		 1639 skb->dev = dev;
1640
3b04ddde 1641 if (dev->header_ops) {
1da177e4 1642 /* The device has an explicit notion of ll header,
62ab0812
ED		 1643 * exported to higher levels.
1644 *
1645 * Otherwise, the device hides details of its frame
1646 * structure, so that corresponding packet head is
1647 * never delivered to user.
1da177e4
LT		 1648 */
1649 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1650 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1651 else if (skb->pkt_type == PACKET_OUTGOING) {
1652 /* Special case: outgoing packets have ll header at head */
bbe735e4 1653 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1654 }
1655 }
1656
1657 snaplen = skb->len;
1658
dbcb5855
DM		 1659 res = run_filter(skb, sk, snaplen);
1660 if (!res)
fda9ef5d 1661 goto drop_n_restore;
dbcb5855
DM		 1662 if (snaplen > res)
1663 snaplen = res;
1da177e4 1664
0fd7bac6 1665 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1666 goto drop_n_acct;
1667
1668 if (skb_shared(skb)) {
1669 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1670 if (nskb == NULL)
1671 goto drop_n_acct;
1672
1673 if (skb_head != skb->data) {
1674 skb->data = skb_head;
1675 skb->len = skb_len;
1676 }
abc4e4fa 1677 consume_skb(skb);
1da177e4
LT		 1678 skb = nskb;
1679 }
1680
ffbc6111
HX		 1681 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1682 sizeof(skb->cb));
1683
1684 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1685 sll->sll_family = AF_PACKET;
1686 sll->sll_hatype = dev->type;
1687 sll->sll_protocol = skb->protocol;
1688 sll->sll_pkttype = skb->pkt_type;
8032b464 1689 if (unlikely(po->origdev))
80feaacb
PWJ		 1690 sll->sll_ifindex = orig_dev->ifindex;
1691 else
1692 sll->sll_ifindex = dev->ifindex;
1da177e4 1693
b95cce35 1694 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1695
ffbc6111 1696 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1697
1da177e4
LT		 1698 if (pskb_trim(skb, snaplen))
1699 goto drop_n_acct;
1700
1701 skb_set_owner_r(skb, sk);
1702 skb->dev = NULL;
adf30907 1703 skb_dst_drop(skb);
1da177e4 1704
84531c24
PO		 1705 /* drop conntrack reference */
1706 nf_reset(skb);
1707
1da177e4 1708 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1709 po->stats.stats1.tp_packets++;
3b885787 1710 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1711 __skb_queue_tail(&sk->sk_receive_queue, skb);
1712 spin_unlock(&sk->sk_receive_queue.lock);
1713 sk->sk_data_ready(sk, skb->len);
1714 return 0;
1715
1716drop_n_acct:
7091fbd8 1717 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1718 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1719 atomic_inc(&sk->sk_drops);
1720 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1721
1722drop_n_restore:
1723 if (skb_head != skb->data && skb_shared(skb)) {
1724 skb->data = skb_head;
1725 skb->len = skb_len;
1726 }
1727drop:
ead2ceb0 1728 consume_skb(skb);
1da177e4
LT		 1729 return 0;
1730}
1731
40d4e3df
ED		 1732static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1733 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1734{
1735 struct sock *sk;
1736 struct packet_sock *po;
1737 struct sockaddr_ll *sll;
184f489e 1738 union tpacket_uhdr h;
40d4e3df 1739 u8 *skb_head = skb->data;
1da177e4 1740 int skb_len = skb->len;
dbcb5855 1741 unsigned int snaplen, res;
f6fb8f10 1742 unsigned long status = TP_STATUS_USER;
bbd6ef87 1743 unsigned short macoff, netoff, hdrlen;
1da177e4 1744 struct sk_buff *copy_skb = NULL;
bbd6ef87 1745 struct timespec ts;
b9c32fb2 1746 __u32 ts_status;
1da177e4
LT		 1747
1748 if (skb->pkt_type == PACKET_LOOPBACK)
1749 goto drop;
1750
1751 sk = pt->af_packet_priv;
1752 po = pkt_sk(sk);
1753
09ad9bc7 1754 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1755 goto drop;
1756
3b04ddde 1757 if (dev->header_ops) {
1da177e4 1758 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1759 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1760 else if (skb->pkt_type == PACKET_OUTGOING) {
1761 /* Special case: outgoing packets have ll header at head */
bbe735e4 1762 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1763 }
1764 }
1765
8dc41944
HX		 1766 if (skb->ip_summed == CHECKSUM_PARTIAL)
1767 status |= TP_STATUS_CSUMNOTREADY;
1768
1da177e4
LT		 1769 snaplen = skb->len;
1770
dbcb5855
DM		 1771 res = run_filter(skb, sk, snaplen);
1772 if (!res)
fda9ef5d 1773 goto drop_n_restore;
dbcb5855
DM		 1774 if (snaplen > res)
1775 snaplen = res;
1da177e4
LT		 1776
1777 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1778 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1779 po->tp_reserve;
1da177e4 1780 } else {
95c96174 1781 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1782 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1783 (maclen < 16 ? 16 : maclen)) +
1784 po->tp_reserve;
1da177e4
LT		 1785 macoff = netoff - maclen;
1786 }
f6fb8f10 1787 if (po->tp_version <= TPACKET_V2) {
1788 if (macoff + snaplen > po->rx_ring.frame_size) {
1789 if (po->copy_thresh &&
0fd7bac6 1790 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1791 if (skb_shared(skb)) {
1792 copy_skb = skb_clone(skb, GFP_ATOMIC);
1793 } else {
1794 copy_skb = skb_get(skb);
1795 skb_head = skb->data;
1796 }
1797 if (copy_skb)
1798 skb_set_owner_r(copy_skb, sk);
1da177e4 1799 }
f6fb8f10 1800 snaplen = po->rx_ring.frame_size - macoff;
1801 if ((int)snaplen < 0)
1802 snaplen = 0;
1da177e4 1803 }
4035ed7b
ED		 1804 } else if (unlikely(macoff + snaplen >
1805 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1806 u32 nval;
1807
1808 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1809 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1810 snaplen, nval, macoff);
1811 snaplen = nval;
1812 if (unlikely((int)snaplen < 0)) {
1813 snaplen = 0;
1814 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1815 }
1da177e4 1816 }
1da177e4 1817 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1818 h.raw = packet_current_rx_frame(po, skb,
1819 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1820 if (!h.raw)
1da177e4 1821 goto ring_is_full;
f6fb8f10 1822 if (po->tp_version <= TPACKET_V2) {
1823 packet_increment_rx_head(po, &po->rx_ring);
1824 /*
1825 * LOSING will be reported till you read the stats,
1826 * because it's COR - Clear On Read.
1827 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1828 * at packet level.
1829 */
ee80fbf3 1830 if (po->stats.stats1.tp_drops)
f6fb8f10 1831 status |= TP_STATUS_LOSING;
1832 }
ee80fbf3 1833 po->stats.stats1.tp_packets++;
1da177e4
LT		 1834 if (copy_skb) {
1835 status |= TP_STATUS_COPY;
1836 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1837 }
1da177e4
LT		 1838 spin_unlock(&sk->sk_receive_queue.lock);
1839
bbd6ef87 1840 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1841
1842 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1843 getnstimeofday(&ts);
1da177e4 1844
b9c32fb2
DB		 1845 status |= ts_status;
1846
bbd6ef87
PM		 1847 switch (po->tp_version) {
1848 case TPACKET_V1:
1849 h.h1->tp_len = skb->len;
1850 h.h1->tp_snaplen = snaplen;
1851 h.h1->tp_mac = macoff;
1852 h.h1->tp_net = netoff;
4b457bdf
DB		 1853 h.h1->tp_sec = ts.tv_sec;
1854 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1855 hdrlen = sizeof(*h.h1);
1856 break;
1857 case TPACKET_V2:
1858 h.h2->tp_len = skb->len;
1859 h.h2->tp_snaplen = snaplen;
1860 h.h2->tp_mac = macoff;
1861 h.h2->tp_net = netoff;
bbd6ef87
PM		 1862 h.h2->tp_sec = ts.tv_sec;
1863 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1864 if (vlan_tx_tag_present(skb)) {
1865 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1866 status |= TP_STATUS_VLAN_VALID;
1867 } else {
1868 h.h2->tp_vlan_tci = 0;
1869 }
13fcb7bd 1870 h.h2->tp_padding = 0;
bbd6ef87
PM		 1871 hdrlen = sizeof(*h.h2);
1872 break;
f6fb8f10 1873 case TPACKET_V3:
1874 /* tp_nxt_offset,vlan are already populated above.
1875 * So DONT clear those fields here
1876 */
1877 h.h3->tp_status |= status;
1878 h.h3->tp_len = skb->len;
1879 h.h3->tp_snaplen = snaplen;
1880 h.h3->tp_mac = macoff;
1881 h.h3->tp_net = netoff;
f6fb8f10 1882 h.h3->tp_sec = ts.tv_sec;
1883 h.h3->tp_nsec = ts.tv_nsec;
1884 hdrlen = sizeof(*h.h3);
1885 break;
bbd6ef87
PM		 1886 default:
1887 BUG();
1888 }
1da177e4 1889
bbd6ef87 1890 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1891 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1892 sll->sll_family = AF_PACKET;
1893 sll->sll_hatype = dev->type;
1894 sll->sll_protocol = skb->protocol;
1895 sll->sll_pkttype = skb->pkt_type;
8032b464 1896 if (unlikely(po->origdev))
80feaacb
PWJ		 1897 sll->sll_ifindex = orig_dev->ifindex;
1898 else
1899 sll->sll_ifindex = dev->ifindex;
1da177e4 1900
e16aa207 1901 smp_mb();
f6dafa95 1902#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1903 {
0af55bb5
CG		 1904 u8 *start, *end;
1905
f6fb8f10 1906 if (po->tp_version <= TPACKET_V2) {
1907 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1908 + macoff + snaplen);
1909 for (start = h.raw; start < end; start += PAGE_SIZE)
1910 flush_dcache_page(pgv_to_page(start));
1911 }
cc9f01b2 1912 smp_wmb();
1da177e4 1913 }
f6dafa95 1914#endif
f6fb8f10 1915 if (po->tp_version <= TPACKET_V2)
1916 __packet_set_status(po, h.raw, status);
1917 else
1918 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1919
1920 sk->sk_data_ready(sk, 0);
1921
1922drop_n_restore:
1923 if (skb_head != skb->data && skb_shared(skb)) {
1924 skb->data = skb_head;
1925 skb->len = skb_len;
1926 }
1927drop:
1ce4f28b 1928 kfree_skb(skb);
1da177e4
LT		 1929 return 0;
1930
1931ring_is_full:
ee80fbf3 1932 po->stats.stats1.tp_drops++;
1da177e4
LT		 1933 spin_unlock(&sk->sk_receive_queue.lock);
1934
1935 sk->sk_data_ready(sk, 0);
acb5d75b 1936 kfree_skb(copy_skb);
1da177e4
LT		 1937 goto drop_n_restore;
1938}
1939
69e3c75f
JB		 1940static void tpacket_destruct_skb(struct sk_buff *skb)
1941{
1942 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1943 void *ph;
1da177e4 1944
69e3c75f 1945 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 1946 __u32 ts;
1947
69e3c75f 1948 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1949 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1950 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 1951
1952 ts = __packet_set_timestamp(po, ph, skb);
1953 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 1954 }
1955
1956 sock_wfree(skb);
1957}
1958
40d4e3df
ED		 1959static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1960 void *frame, struct net_device *dev, int size_max,
ae641949 1961 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1962{
184f489e 1963 union tpacket_uhdr ph;
69e3c75f
JB		 1964 int to_write, offset, len, tp_len, nr_frags, len_max;
1965 struct socket *sock = po->sk.sk_socket;
1966 struct page *page;
1967 void *data;
1968 int err;
1969
1970 ph.raw = frame;
1971
1972 skb->protocol = proto;
1973 skb->dev = dev;
1974 skb->priority = po->sk.sk_priority;
2d37a186 1975 skb->mark = po->sk.sk_mark;
2e31396f 1976 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 1977 skb_shinfo(skb)->destructor_arg = ph.raw;
1978
1979 switch (po->tp_version) {
1980 case TPACKET_V2:
1981 tp_len = ph.h2->tp_len;
1982 break;
1983 default:
1984 tp_len = ph.h1->tp_len;
1985 break;
1986 }
1987 if (unlikely(tp_len > size_max)) {
40d4e3df 1988 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1989 return -EMSGSIZE;
1990 }
1991
ae641949 1992 skb_reserve(skb, hlen);
69e3c75f 1993 skb_reset_network_header(skb);
40893fd0 1994 skb_probe_transport_header(skb, 0);
c1aad275 1995
5920cd3a
PC		 1996 if (po->tp_tx_has_off) {
1997 int off_min, off_max, off;
1998 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1999 off_max = po->tx_ring.frame_size - tp_len;
2000 if (sock->type == SOCK_DGRAM) {
2001 switch (po->tp_version) {
2002 case TPACKET_V2:
2003 off = ph.h2->tp_net;
2004 break;
2005 default:
2006 off = ph.h1->tp_net;
2007 break;
2008 }
2009 } else {
2010 switch (po->tp_version) {
2011 case TPACKET_V2:
2012 off = ph.h2->tp_mac;
2013 break;
2014 default:
2015 off = ph.h1->tp_mac;
2016 break;
2017 }
2018 }
2019 if (unlikely((off < off_min) || (off_max < off)))
2020 return -EINVAL;
2021 data = ph.raw + off;
2022 } else {
2023 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2024 }
69e3c75f
JB		 2025 to_write = tp_len;
2026
2027 if (sock->type == SOCK_DGRAM) {
2028 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2029 NULL, tp_len);
2030 if (unlikely(err < 0))
2031 return -EINVAL;
40d4e3df 2032 } else if (dev->hard_header_len) {
69e3c75f
JB		 2033 /* net device doesn't like empty head */
2034 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2035 pr_err("packet size is too short (%d < %d)\n",
2036 tp_len, dev->hard_header_len);
69e3c75f
JB		 2037 return -EINVAL;
2038 }
2039
2040 skb_push(skb, dev->hard_header_len);
2041 err = skb_store_bits(skb, 0, data,
2042 dev->hard_header_len);
2043 if (unlikely(err))
2044 return err;
2045
2046 data += dev->hard_header_len;
2047 to_write -= dev->hard_header_len;
2048 }
2049
69e3c75f
JB		 2050 offset = offset_in_page(data);
2051 len_max = PAGE_SIZE - offset;
2052 len = ((to_write > len_max) ? len_max : to_write);
2053
2054 skb->data_len = to_write;
2055 skb->len += to_write;
2056 skb->truesize += to_write;
2057 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2058
2059 while (likely(to_write)) {
2060 nr_frags = skb_shinfo(skb)->nr_frags;
2061
2062 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2063 pr_err("Packet exceed the number of skb frags(%lu)\n",
2064 MAX_SKB_FRAGS);
69e3c75f
JB		 2065 return -EFAULT;
2066 }
2067
0af55bb5
CG		 2068 page = pgv_to_page(data);
2069 data += len;
69e3c75f
JB		 2070 flush_dcache_page(page);
2071 get_page(page);
0af55bb5 2072 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2073 to_write -= len;
2074 offset = 0;
2075 len_max = PAGE_SIZE;
2076 len = ((to_write > len_max) ? len_max : to_write);
2077 }
2078
2079 return tp_len;
2080}
2081
2082static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2083{
69e3c75f
JB		 2084 struct sk_buff *skb;
2085 struct net_device *dev;
2086 __be16 proto;
827d9780 2087 int err, reserve = 0;
40d4e3df
ED		 2088 void *ph;
2089 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2090 int tp_len, size_max;
2091 unsigned char *addr;
2092 int len_sum = 0;
9e67030a 2093 int status = TP_STATUS_AVAILABLE;
ae641949 2094 int hlen, tlen;
69e3c75f 2095
69e3c75f
JB		 2096 mutex_lock(&po->pg_vec_lock);
2097
c3ac8a13 2098 if (likely(saddr == NULL)) {
026bb405 2099 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2100 proto = po->num;
2101 addr = NULL;
2102 } else {
2103 err = -EINVAL;
2104 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2105 goto out;
2106 if (msg->msg_namelen < (saddr->sll_halen
2107 + offsetof(struct sockaddr_ll,
2108 sll_addr)))
2109 goto out;
69e3c75f
JB		 2110 proto = saddr->sll_protocol;
2111 addr = saddr->sll_addr;
827d9780 2112 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2113 }
2114
69e3c75f
JB		 2115 err = -ENXIO;
2116 if (unlikely(dev == NULL))
2117 goto out;
69e3c75f
JB		 2118 err = -ENETDOWN;
2119 if (unlikely(!(dev->flags & IFF_UP)))
2120 goto out_put;
2121
026bb405
DB		 2122 reserve = dev->hard_header_len;
2123
69e3c75f 2124 size_max = po->tx_ring.frame_size
b5dd884e 2125 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2126
2127 if (size_max > dev->mtu + reserve)
2128 size_max = dev->mtu + reserve;
2129
2130 do {
2131 ph = packet_current_frame(po, &po->tx_ring,
2132 TP_STATUS_SEND_REQUEST);
2133
2134 if (unlikely(ph == NULL)) {
2135 schedule();
2136 continue;
2137 }
2138
2139 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2140 hlen = LL_RESERVED_SPACE(dev);
2141 tlen = dev->needed_tailroom;
69e3c75f 2142 skb = sock_alloc_send_skb(&po->sk,
ae641949 2143 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2144 0, &err);
2145
2146 if (unlikely(skb == NULL))
2147 goto out_status;
2148
2149 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2150 addr, hlen);
69e3c75f
JB		 2151
2152 if (unlikely(tp_len < 0)) {
2153 if (po->tp_loss) {
2154 __packet_set_status(po, ph,
2155 TP_STATUS_AVAILABLE);
2156 packet_increment_head(&po->tx_ring);
2157 kfree_skb(skb);
2158 continue;
2159 } else {
2160 status = TP_STATUS_WRONG_FORMAT;
2161 err = tp_len;
2162 goto out_status;
2163 }
2164 }
2165
2166 skb->destructor = tpacket_destruct_skb;
2167 __packet_set_status(po, ph, TP_STATUS_SENDING);
2168 atomic_inc(&po->tx_ring.pending);
2169
2170 status = TP_STATUS_SEND_REQUEST;
2171 err = dev_queue_xmit(skb);
eb70df13
JP		 2172 if (unlikely(err > 0)) {
2173 err = net_xmit_errno(err);
2174 if (err && __packet_get_status(po, ph) ==
2175 TP_STATUS_AVAILABLE) {
2176 /* skb was destructed already */
2177 skb = NULL;
2178 goto out_status;
2179 }
2180 /*
2181 * skb was dropped but not destructed yet;
2182 * let's treat it like congestion or err < 0
2183 */
2184 err = 0;
2185 }
69e3c75f
JB		 2186 packet_increment_head(&po->tx_ring);
2187 len_sum += tp_len;
f64f9e71
JP		 2188 } while (likely((ph != NULL) ||
2189 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2190 (atomic_read(&po->tx_ring.pending))))
2191 );
69e3c75f
JB		 2192
2193 err = len_sum;
2194 goto out_put;
2195
69e3c75f
JB		 2196out_status:
2197 __packet_set_status(po, ph, status);
2198 kfree_skb(skb);
2199out_put:
026bb405 2200 dev_put(dev);
69e3c75f
JB		 2201out:
2202 mutex_unlock(&po->pg_vec_lock);
2203 return err;
2204}
69e3c75f 2205
eea49cc9
OJ		 2206static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2207 size_t reserve, size_t len,
2208 size_t linear, int noblock,
2209 int *err)
bfd5f4a3
SS		 2210{
2211 struct sk_buff *skb;
2212
2213 /* Under a page? Don't bother with paged skb. */
2214 if (prepad + len < PAGE_SIZE || !linear)
2215 linear = len;
2216
2217 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2218 err);
2219 if (!skb)
2220 return NULL;
2221
2222 skb_reserve(skb, reserve);
2223 skb_put(skb, linear);
2224 skb->data_len = len - linear;
2225 skb->len += len - linear;
2226
2227 return skb;
2228}
2229
69e3c75f 2230static int packet_snd(struct socket *sock,
1da177e4
LT		 2231 struct msghdr *msg, size_t len)
2232{
2233 struct sock *sk = sock->sk;
40d4e3df 2234 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2235 struct sk_buff *skb;
2236 struct net_device *dev;
0e11c91e 2237 __be16 proto;
1da177e4 2238 unsigned char *addr;
827d9780 2239 int err, reserve = 0;
bfd5f4a3
SS		 2240 struct virtio_net_hdr vnet_hdr = { 0 };
2241 int offset = 0;
2242 int vnet_hdr_len;
2243 struct packet_sock *po = pkt_sk(sk);
2244 unsigned short gso_type = 0;
ae641949 2245 int hlen, tlen;
3bdc0eba 2246 int extra_len = 0;
1da177e4
LT		 2247
2248 /*
1ce4f28b 2249 * Get and verify the address.
1da177e4 2250 */
1ce4f28b 2251
c3ac8a13 2252 if (likely(saddr == NULL)) {
026bb405 2253 dev = packet_cached_dev_get(po);
1da177e4
LT		 2254 proto = po->num;
2255 addr = NULL;
2256 } else {
2257 err = -EINVAL;
2258 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2259 goto out;
0fb375fb
EB		 2260 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2261 goto out;
1da177e4
LT		 2262 proto = saddr->sll_protocol;
2263 addr = saddr->sll_addr;
827d9780 2264 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2265 }
2266
1da177e4 2267 err = -ENXIO;
026bb405 2268 if (unlikely(dev == NULL))
1da177e4 2269 goto out_unlock;
d5e76b0a 2270 err = -ENETDOWN;
026bb405 2271 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2272 goto out_unlock;
2273
026bb405
DB		 2274 if (sock->type == SOCK_RAW)
2275 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2276 if (po->has_vnet_hdr) {
2277 vnet_hdr_len = sizeof(vnet_hdr);
2278
2279 err = -EINVAL;
2280 if (len < vnet_hdr_len)
2281 goto out_unlock;
2282
2283 len -= vnet_hdr_len;
2284
2285 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2286 vnet_hdr_len);
2287 if (err < 0)
2288 goto out_unlock;
2289
2290 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2291 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2292 vnet_hdr.hdr_len))
2293 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2294 vnet_hdr.csum_offset + 2;
2295
2296 err = -EINVAL;
2297 if (vnet_hdr.hdr_len > len)
2298 goto out_unlock;
2299
2300 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2301 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2302 case VIRTIO_NET_HDR_GSO_TCPV4:
2303 gso_type = SKB_GSO_TCPV4;
2304 break;
2305 case VIRTIO_NET_HDR_GSO_TCPV6:
2306 gso_type = SKB_GSO_TCPV6;
2307 break;
2308 case VIRTIO_NET_HDR_GSO_UDP:
2309 gso_type = SKB_GSO_UDP;
2310 break;
2311 default:
2312 goto out_unlock;
2313 }
2314
2315 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2316 gso_type |= SKB_GSO_TCP_ECN;
2317
2318 if (vnet_hdr.gso_size == 0)
2319 goto out_unlock;
2320
2321 }
2322 }
2323
3bdc0eba
BG		 2324 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2325 if (!netif_supports_nofcs(dev)) {
2326 err = -EPROTONOSUPPORT;
2327 goto out_unlock;
2328 }
2329 extra_len = 4; /* We're doing our own CRC */
2330 }
2331
1da177e4 2332 err = -EMSGSIZE;
3bdc0eba 2333 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2334 goto out_unlock;
2335
bfd5f4a3 2336 err = -ENOBUFS;
ae641949
HX		 2337 hlen = LL_RESERVED_SPACE(dev);
2338 tlen = dev->needed_tailroom;
2339 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2340 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2341 if (skb == NULL)
1da177e4
LT		 2342 goto out_unlock;
2343
bfd5f4a3 2344 skb_set_network_header(skb, reserve);
1da177e4 2345
0c4e8581
SH		 2346 err = -EINVAL;
2347 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2348 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2349 goto out_free;
1da177e4
LT		 2350
2351 /* Returns -EFAULT on error */
bfd5f4a3 2352 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2353 if (err)
2354 goto out_free;
bf84a010
DB		 2355
2356 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2357
3bdc0eba 2358 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
57f89bfa
BG		 2359 /* Earlier code assumed this would be a VLAN pkt,
2360 * double-check this now that we have the actual
2361 * packet in hand.
2362 */
2363 struct ethhdr *ehdr;
2364 skb_reset_mac_header(skb);
2365 ehdr = eth_hdr(skb);
2366 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2367 err = -EMSGSIZE;
2368 goto out_free;
2369 }
2370 }
2371
1da177e4
LT		 2372 skb->protocol = proto;
2373 skb->dev = dev;
2374 skb->priority = sk->sk_priority;
2d37a186 2375 skb->mark = sk->sk_mark;
1da177e4 2376
bfd5f4a3
SS		 2377 if (po->has_vnet_hdr) {
2378 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2379 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2380 vnet_hdr.csum_offset)) {
2381 err = -EINVAL;
2382 goto out_free;
2383 }
2384 }
2385
2386 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2387 skb_shinfo(skb)->gso_type = gso_type;
2388
2389 /* Header must be checked, and gso_segs computed. */
2390 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2391 skb_shinfo(skb)->gso_segs = 0;
2392
2393 len += vnet_hdr_len;
2394 }
2395
40893fd0 2396 skb_probe_transport_header(skb, reserve);
c1aad275 2397
3bdc0eba
BG		 2398 if (unlikely(extra_len == 4))
2399 skb->no_fcs = 1;
2400
1da177e4
LT		 2401 /*
2402 * Now send it
2403 */
2404
2405 err = dev_queue_xmit(skb);
2406 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2407 goto out_unlock;
2408
026bb405 2409 dev_put(dev);
1da177e4 2410
40d4e3df 2411 return len;
1da177e4
LT		 2412
2413out_free:
2414 kfree_skb(skb);
2415out_unlock:
026bb405 2416 if (dev)
1da177e4
LT		 2417 dev_put(dev);
2418out:
2419 return err;
2420}
2421
69e3c75f
JB		 2422static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2423 struct msghdr *msg, size_t len)
2424{
69e3c75f
JB		 2425 struct sock *sk = sock->sk;
2426 struct packet_sock *po = pkt_sk(sk);
2427 if (po->tx_ring.pg_vec)
2428 return tpacket_snd(po, msg);
2429 else
69e3c75f
JB		 2430 return packet_snd(sock, msg, len);
2431}
2432
1da177e4
LT		 2433/*
2434 * Close a PACKET socket. This is fairly simple. We immediately go
2435 * to 'closed' state and remove our protocol entry in the device list.
2436 */
2437
2438static int packet_release(struct socket *sock)
2439{
2440 struct sock *sk = sock->sk;
2441 struct packet_sock *po;
9aab3592 2442 struct packet_fanout *f;
d12d01d6 2443 struct net *net;
f6fb8f10 2444 union tpacket_req_u req_u;
1da177e4
LT		 2445
2446 if (!sk)
2447 return 0;
2448
3b1e0a65 2449 net = sock_net(sk);
1da177e4
LT		 2450 po = pkt_sk(sk);
2451
0fa7fa98 2452 mutex_lock(&net->packet.sklist_lock);
808f5114 2453 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2454 mutex_unlock(&net->packet.sklist_lock);
2455
2456 preempt_disable();
920de804 2457 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2458 preempt_enable();
1da177e4 2459
808f5114 2460 spin_lock(&po->bind_lock);
ce06b03e 2461 unregister_prot_hook(sk, false);
c3ac8a13
DB		 2462 packet_cached_dev_reset(po);
2463
160ff18a
BG		 2464 if (po->prot_hook.dev) {
2465 dev_put(po->prot_hook.dev);
2466 po->prot_hook.dev = NULL;
2467 }
808f5114 2468 spin_unlock(&po->bind_lock);
1da177e4 2469
1da177e4 2470 packet_flush_mclist(sk);
1da177e4 2471
9665d5d6
PS		 2472 if (po->rx_ring.pg_vec) {
2473 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2474 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2475 }
69e3c75f 2476
9665d5d6
PS		 2477 if (po->tx_ring.pg_vec) {
2478 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2479 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2480 }
1da177e4 2481
9aab3592 2482 f = fanout_release(sk);
dc99f600 2483
808f5114 2484 synchronize_net();
9aab3592
AS		 2485
2486 if (f) {
2487 kfree(f);
2488 }
1da177e4
LT		 2489 /*
2490 * Now the socket is dead. No more input will appear.
2491 */
1da177e4
LT		 2492 sock_orphan(sk);
2493 sock->sk = NULL;
2494
2495 /* Purge queues */
2496
2497 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2498 sk_refcnt_debug_release(sk);
1da177e4
LT		 2499
2500 sock_put(sk);
2501 return 0;
2502}
2503
2504/*
2505 * Attach a packet hook.
2506 */
2507
0e11c91e 2508static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2509{
2510 struct packet_sock *po = pkt_sk(sk);
dc99f600 2511
aef950b4
WY		 2512 if (po->fanout) {
2513 if (dev)
2514 dev_put(dev);
2515
dc99f600 2516 return -EINVAL;
aef950b4 2517 }
1da177e4
LT		 2518
2519 lock_sock(sk);
2520
2521 spin_lock(&po->bind_lock);
ce06b03e 2522 unregister_prot_hook(sk, true);
c3ac8a13 2523
1da177e4
LT		 2524 po->num = protocol;
2525 po->prot_hook.type = protocol;
160ff18a
BG		 2526 if (po->prot_hook.dev)
2527 dev_put(po->prot_hook.dev);
1da177e4 2528
c3ac8a13 2529 po->prot_hook.dev = dev;
1da177e4
LT		 2530 po->ifindex = dev ? dev->ifindex : 0;
2531
c3ac8a13
DB		 2532 packet_cached_dev_assign(po, dev);
2533
1da177e4
LT		 2534 if (protocol == 0)
2535 goto out_unlock;
2536
be85d4ad 2537 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2538 register_prot_hook(sk);
be85d4ad
UT		 2539 } else {
2540 sk->sk_err = ENETDOWN;
2541 if (!sock_flag(sk, SOCK_DEAD))
2542 sk->sk_error_report(sk);
1da177e4
LT		 2543 }
2544
2545out_unlock:
2546 spin_unlock(&po->bind_lock);
2547 release_sock(sk);
2548 return 0;
2549}
2550
2551/*
2552 * Bind a packet socket to a device
2553 */
2554
40d4e3df
ED		 2555static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2556 int addr_len)
1da177e4 2557{
40d4e3df 2558 struct sock *sk = sock->sk;
1da177e4
LT		 2559 char name[15];
2560 struct net_device *dev;
2561 int err = -ENODEV;
1ce4f28b 2562
1da177e4
LT		 2563 /*
2564 * Check legality
2565 */
1ce4f28b 2566
8ae55f04 2567 if (addr_len != sizeof(struct sockaddr))
1da177e4 2568 return -EINVAL;
40d4e3df 2569 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2570
3b1e0a65 2571 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2572 if (dev)
1da177e4 2573 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2574 return err;
2575}
1da177e4
LT		 2576
2577static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2578{
40d4e3df
ED		 2579 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2580 struct sock *sk = sock->sk;
1da177e4
LT		 2581 struct net_device *dev = NULL;
2582 int err;
2583
2584
2585 /*
2586 * Check legality
2587 */
1ce4f28b 2588
1da177e4
LT		 2589 if (addr_len < sizeof(struct sockaddr_ll))
2590 return -EINVAL;
2591 if (sll->sll_family != AF_PACKET)
2592 return -EINVAL;
2593
2594 if (sll->sll_ifindex) {
2595 err = -ENODEV;
3b1e0a65 2596 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2597 if (dev == NULL)
2598 goto out;
2599 }
2600 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2601
2602out:
2603 return err;
2604}
2605
2606static struct proto packet_proto = {
2607 .name = "PACKET",
2608 .owner = THIS_MODULE,
2609 .obj_size = sizeof(struct packet_sock),
2610};
2611
2612/*
1ce4f28b 2613 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2614 */
2615
3f378b68
EP		 2616static int packet_create(struct net *net, struct socket *sock, int protocol,
2617 int kern)
1da177e4
LT		 2618{
2619 struct sock *sk;
2620 struct packet_sock *po;
0e11c91e 2621 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2622 int err;
2623
df008c91 2624 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2625 return -EPERM;
be02097c
DM		 2626 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2627 sock->type != SOCK_PACKET)
1da177e4
LT		 2628 return -ESOCKTNOSUPPORT;
2629
2630 sock->state = SS_UNCONNECTED;
2631
2632 err = -ENOBUFS;
6257ff21 2633 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2634 if (sk == NULL)
2635 goto out;
2636
2637 sock->ops = &packet_ops;
1da177e4
LT		 2638 if (sock->type == SOCK_PACKET)
2639 sock->ops = &packet_ops_spkt;
be02097c 2640
1da177e4
LT		 2641 sock_init_data(sock, sk);
2642
2643 po = pkt_sk(sk);
2644 sk->sk_family = PF_PACKET;
0e11c91e 2645 po->num = proto;
c3ac8a13
DB		 2646
2647 packet_cached_dev_reset(po);
1da177e4
LT		 2648
2649 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2650 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2651
2652 /*
2653 * Attach a protocol block
2654 */
2655
2656 spin_lock_init(&po->bind_lock);
905db440 2657 mutex_init(&po->pg_vec_lock);
1da177e4 2658 po->prot_hook.func = packet_rcv;
be02097c 2659
1da177e4
LT		 2660 if (sock->type == SOCK_PACKET)
2661 po->prot_hook.func = packet_rcv_spkt;
be02097c 2662
1da177e4
LT		 2663 po->prot_hook.af_packet_priv = sk;
2664
0e11c91e
AV		 2665 if (proto) {
2666 po->prot_hook.type = proto;
ce06b03e 2667 register_prot_hook(sk);
1da177e4
LT		 2668 }
2669
0fa7fa98 2670 mutex_lock(&net->packet.sklist_lock);
808f5114 2671 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2672 mutex_unlock(&net->packet.sklist_lock);
2673
2674 preempt_disable();
3680453c 2675 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2676 preempt_enable();
808f5114 2677
40d4e3df 2678 return 0;
1da177e4
LT		 2679out:
2680 return err;
2681}
2682
ed85b565
RC		 2683static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2684{
2685 struct sock_exterr_skb *serr;
2686 struct sk_buff *skb, *skb2;
2687 int copied, err;
2688
2689 err = -EAGAIN;
2690 skb = skb_dequeue(&sk->sk_error_queue);
2691 if (skb == NULL)
2692 goto out;
2693
2694 copied = skb->len;
2695 if (copied > len) {
2696 msg->msg_flags |= MSG_TRUNC;
2697 copied = len;
2698 }
2699 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2700 if (err)
2701 goto out_free_skb;
2702
2703 sock_recv_timestamp(msg, sk, skb);
2704
2705 serr = SKB_EXT_ERR(skb);
2706 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2707 sizeof(serr->ee), &serr->ee);
2708
2709 msg->msg_flags |= MSG_ERRQUEUE;
2710 err = copied;
2711
2712 /* Reset and regenerate socket error */
2713 spin_lock_bh(&sk->sk_error_queue.lock);
2714 sk->sk_err = 0;
2715 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2716 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2717 spin_unlock_bh(&sk->sk_error_queue.lock);
2718 sk->sk_error_report(sk);
2719 } else
2720 spin_unlock_bh(&sk->sk_error_queue.lock);
2721
2722out_free_skb:
2723 kfree_skb(skb);
2724out:
2725 return err;
2726}
2727
1da177e4
LT		 2728/*
2729 * Pull a packet from our receive queue and hand it to the user.
2730 * If necessary we block.
2731 */
2732
2733static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2734 struct msghdr *msg, size_t len, int flags)
2735{
2736 struct sock *sk = sock->sk;
2737 struct sk_buff *skb;
2738 int copied, err;
bfd5f4a3 2739 int vnet_hdr_len = 0;
1da177e4
LT		 2740
2741 err = -EINVAL;
ed85b565 2742 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2743 goto out;
2744
2745#if 0
2746 /* What error should we return now? EUNATTACH? */
2747 if (pkt_sk(sk)->ifindex < 0)
2748 return -ENODEV;
2749#endif
2750
ed85b565
RC		 2751 if (flags & MSG_ERRQUEUE) {
2752 err = packet_recv_error(sk, msg, len);
2753 goto out;
2754 }
2755
1da177e4
LT		 2756 /*
2757 * Call the generic datagram receiver. This handles all sorts
2758 * of horrible races and re-entrancy so we can forget about it
2759 * in the protocol layers.
2760 *
2761 * Now it will return ENETDOWN, if device have just gone down,
2762 * but then it will block.
2763 */
2764
40d4e3df 2765 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2766
2767 /*
1ce4f28b 2768 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2769 * handles the blocking we don't see and worry about blocking
2770 * retries.
2771 */
2772
8ae55f04 2773 if (skb == NULL)
1da177e4
LT		 2774 goto out;
2775
bfd5f4a3
SS		 2776 if (pkt_sk(sk)->has_vnet_hdr) {
2777 struct virtio_net_hdr vnet_hdr = { 0 };
2778
2779 err = -EINVAL;
2780 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2781 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2782 goto out_free;
2783
1f18b717
MK		 2784 len -= vnet_hdr_len;
2785
bfd5f4a3
SS		 2786 if (skb_is_gso(skb)) {
2787 struct skb_shared_info *sinfo = skb_shinfo(skb);
2788
2789 /* This is a hint as to how much should be linear. */
2790 vnet_hdr.hdr_len = skb_headlen(skb);
2791 vnet_hdr.gso_size = sinfo->gso_size;
2792 if (sinfo->gso_type & SKB_GSO_TCPV4)
2793 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2794 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2795 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2796 else if (sinfo->gso_type & SKB_GSO_UDP)
2797 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2798 else if (sinfo->gso_type & SKB_GSO_FCOE)
2799 goto out_free;
2800 else
2801 BUG();
2802 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2803 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2804 } else
2805 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2806
2807 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2808 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2809 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2810 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2811 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2812 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2813 } /* else everything is zero */
2814
2815 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2816 vnet_hdr_len);
2817 if (err < 0)
2818 goto out_free;
2819 }
2820
2f73d7fd
HFS		 2821 /* You lose any data beyond the buffer you gave. If it worries
2822 * a user program they can ask the device for its MTU
2823 * anyway.
1da177e4 2824 */
1da177e4 2825 copied = skb->len;
40d4e3df
ED		 2826 if (copied > len) {
2827 copied = len;
2828 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2829 }
2830
2831 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2832 if (err)
2833 goto out_free;
2834
3b885787 2835 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2836
2f73d7fd
HFS		 2837 if (msg->msg_name) {
2838 /* If the address length field is there to be filled
2839 * in, we fill it in now.
2840 */
2841 if (sock->type == SOCK_PACKET) {
2842 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2843 } else {
2844 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2845 msg->msg_namelen = sll->sll_halen +
2846 offsetof(struct sockaddr_ll, sll_addr);
2847 }
ffbc6111
HX		 2848 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2849 msg->msg_namelen);
2f73d7fd 2850 }
1da177e4 2851
8dc41944 2852 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2853 struct tpacket_auxdata aux;
2854
2855 aux.tp_status = TP_STATUS_USER;
2856 if (skb->ip_summed == CHECKSUM_PARTIAL)
2857 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2858 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2859 aux.tp_snaplen = skb->len;
2860 aux.tp_mac = 0;
bbe735e4 2861 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2862 if (vlan_tx_tag_present(skb)) {
2863 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2864 aux.tp_status |= TP_STATUS_VLAN_VALID;
2865 } else {
2866 aux.tp_vlan_tci = 0;
2867 }
13fcb7bd 2868 aux.tp_padding = 0;
ffbc6111 2869 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2870 }
2871
1da177e4
LT		 2872 /*
2873 * Free or return the buffer as appropriate. Again this
2874 * hides all the races and re-entrancy issues from us.
2875 */
bfd5f4a3 2876 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2877
2878out_free:
2879 skb_free_datagram(sk, skb);
2880out:
2881 return err;
2882}
2883
1da177e4
LT		 2884static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2885 int *uaddr_len, int peer)
2886{
2887 struct net_device *dev;
2888 struct sock *sk = sock->sk;
2889
2890 if (peer)
2891 return -EOPNOTSUPP;
2892
2893 uaddr->sa_family = AF_PACKET;
2dc85bf3 2894 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2895 rcu_read_lock();
2896 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2897 if (dev)
2dc85bf3 2898 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2899 rcu_read_unlock();
1da177e4
LT		 2900 *uaddr_len = sizeof(*uaddr);
2901
2902 return 0;
2903}
1da177e4
LT		 2904
2905static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2906 int *uaddr_len, int peer)
2907{
2908 struct net_device *dev;
2909 struct sock *sk = sock->sk;
2910 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2911 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2912
2913 if (peer)
2914 return -EOPNOTSUPP;
2915
2916 sll->sll_family = AF_PACKET;
2917 sll->sll_ifindex = po->ifindex;
2918 sll->sll_protocol = po->num;
67286640 2919 sll->sll_pkttype = 0;
654d1f8a
ED		 2920 rcu_read_lock();
2921 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2922 if (dev) {
2923 sll->sll_hatype = dev->type;
2924 sll->sll_halen = dev->addr_len;
2925 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2926 } else {
2927 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2928 sll->sll_halen = 0;
2929 }
654d1f8a 2930 rcu_read_unlock();
0fb375fb 2931 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2932
2933 return 0;
2934}
2935
2aeb0b88
WC		 2936static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2937 int what)
1da177e4
LT		 2938{
2939 switch (i->type) {
2940 case PACKET_MR_MULTICAST:
1162563f
JP		 2941 if (i->alen != dev->addr_len)
2942 return -EINVAL;
1da177e4 2943 if (what > 0)
22bedad3 2944 return dev_mc_add(dev, i->addr);
1da177e4 2945 else
22bedad3 2946 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2947 break;
2948 case PACKET_MR_PROMISC:
2aeb0b88 2949 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2950 break;
2951 case PACKET_MR_ALLMULTI:
2aeb0b88 2952 return dev_set_allmulti(dev, what);
1da177e4 2953 break;
d95ed927 2954 case PACKET_MR_UNICAST:
1162563f
JP		 2955 if (i->alen != dev->addr_len)
2956 return -EINVAL;
d95ed927 2957 if (what > 0)
a748ee24 2958 return dev_uc_add(dev, i->addr);
d95ed927 2959 else
a748ee24 2960 return dev_uc_del(dev, i->addr);
d95ed927 2961 break;
40d4e3df
ED		 2962 default:
2963 break;
1da177e4 2964 }
2aeb0b88 2965 return 0;
1da177e4
LT		 2966}
2967
2968static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2969{
40d4e3df 2970 for ( ; i; i = i->next) {
1da177e4
LT		 2971 if (i->ifindex == dev->ifindex)
2972 packet_dev_mc(dev, i, what);
2973 }
2974}
2975
0fb375fb 2976static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2977{
2978 struct packet_sock *po = pkt_sk(sk);
2979 struct packet_mclist *ml, *i;
2980 struct net_device *dev;
2981 int err;
2982
2983 rtnl_lock();
2984
2985 err = -ENODEV;
3b1e0a65 2986 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2987 if (!dev)
2988 goto done;
2989
2990 err = -EINVAL;
1162563f 2991 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2992 goto done;
2993
2994 err = -ENOBUFS;
8b3a7005 2995 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2996 if (i == NULL)
2997 goto done;
2998
2999 err = 0;
3000 for (ml = po->mclist; ml; ml = ml->next) {
3001 if (ml->ifindex == mreq->mr_ifindex &&
3002 ml->type == mreq->mr_type &&
3003 ml->alen == mreq->mr_alen &&
3004 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3005 ml->count++;
3006 /* Free the new element ... */
3007 kfree(i);
3008 goto done;
3009 }
3010 }
3011
3012 i->type = mreq->mr_type;
3013 i->ifindex = mreq->mr_ifindex;
3014 i->alen = mreq->mr_alen;
3015 memcpy(i->addr, mreq->mr_address, i->alen);
13b89711 3016 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
1da177e4
LT		 3017 i->count = 1;
3018 i->next = po->mclist;
3019 po->mclist = i;
2aeb0b88
WC		 3020 err = packet_dev_mc(dev, i, 1);
3021 if (err) {
3022 po->mclist = i->next;
3023 kfree(i);
3024 }
1da177e4
LT		 3025
3026done:
3027 rtnl_unlock();
3028 return err;
3029}
3030
0fb375fb 3031static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3032{
3033 struct packet_mclist *ml, **mlp;
3034
3035 rtnl_lock();
3036
3037 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3038 if (ml->ifindex == mreq->mr_ifindex &&
3039 ml->type == mreq->mr_type &&
3040 ml->alen == mreq->mr_alen &&
3041 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3042 if (--ml->count == 0) {
3043 struct net_device *dev;
3044 *mlp = ml->next;
ad959e76
ED		 3045 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3046 if (dev)
1da177e4 3047 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3048 kfree(ml);
3049 }
3050 rtnl_unlock();
3051 return 0;
3052 }
3053 }
3054 rtnl_unlock();
3055 return -EADDRNOTAVAIL;
3056}
3057
3058static void packet_flush_mclist(struct sock *sk)
3059{
3060 struct packet_sock *po = pkt_sk(sk);
3061 struct packet_mclist *ml;
3062
3063 if (!po->mclist)
3064 return;
3065
3066 rtnl_lock();
3067 while ((ml = po->mclist) != NULL) {
3068 struct net_device *dev;
3069
3070 po->mclist = ml->next;
ad959e76
ED		 3071 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3072 if (dev != NULL)
1da177e4 3073 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3074 kfree(ml);
3075 }
3076 rtnl_unlock();
3077}
1da177e4
LT		 3078
3079static int
b7058842 3080packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3081{
3082 struct sock *sk = sock->sk;
8dc41944 3083 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3084 int ret;
3085
3086 if (level != SOL_PACKET)
3087 return -ENOPROTOOPT;
3088
69e3c75f 3089 switch (optname) {
1ce4f28b 3090 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3091 case PACKET_DROP_MEMBERSHIP:
3092 {
0fb375fb
EB		 3093 struct packet_mreq_max mreq;
3094 int len = optlen;
3095 memset(&mreq, 0, sizeof(mreq));
3096 if (len < sizeof(struct packet_mreq))
1da177e4 3097 return -EINVAL;
0fb375fb
EB		 3098 if (len > sizeof(mreq))
3099 len = sizeof(mreq);
40d4e3df 3100 if (copy_from_user(&mreq, optval, len))
1da177e4 3101 return -EFAULT;
0fb375fb
EB		 3102 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3103 return -EINVAL;
1da177e4
LT		 3104 if (optname == PACKET_ADD_MEMBERSHIP)
3105 ret = packet_mc_add(sk, &mreq);
3106 else
3107 ret = packet_mc_drop(sk, &mreq);
3108 return ret;
3109 }
a2efcfa0 3110
1da177e4 3111 case PACKET_RX_RING:
69e3c75f 3112 case PACKET_TX_RING:
1da177e4 3113 {
f6fb8f10 3114 union tpacket_req_u req_u;
3115 int len;
1da177e4 3116
f6fb8f10 3117 switch (po->tp_version) {
3118 case TPACKET_V1:
3119 case TPACKET_V2:
3120 len = sizeof(req_u.req);
3121 break;
3122 case TPACKET_V3:
3123 default:
3124 len = sizeof(req_u.req3);
3125 break;
3126 }
3127 if (optlen < len)
1da177e4 3128 return -EINVAL;
bfd5f4a3
SS		 3129 if (pkt_sk(sk)->has_vnet_hdr)
3130 return -EINVAL;
f6fb8f10 3131 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3132 return -EFAULT;
f6fb8f10 3133 return packet_set_ring(sk, &req_u, 0,
3134 optname == PACKET_TX_RING);
1da177e4
LT		 3135 }
3136 case PACKET_COPY_THRESH:
3137 {
3138 int val;
3139
40d4e3df 3140 if (optlen != sizeof(val))
1da177e4 3141 return -EINVAL;
40d4e3df 3142 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3143 return -EFAULT;
3144
3145 pkt_sk(sk)->copy_thresh = val;
3146 return 0;
3147 }
bbd6ef87
PM		 3148 case PACKET_VERSION:
3149 {
3150 int val;
3151
3152 if (optlen != sizeof(val))
3153 return -EINVAL;
bbd6ef87
PM		 3154 if (copy_from_user(&val, optval, sizeof(val)))
3155 return -EFAULT;
3156 switch (val) {
3157 case TPACKET_V1:
3158 case TPACKET_V2:
f6fb8f10 3159 case TPACKET_V3:
4b9e9796 3160 break;
bbd6ef87
PM		 3161 default:
3162 return -EINVAL;
3163 }
4b9e9796
S		 3164 lock_sock(sk);
3165 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3166 ret = -EBUSY;
3167 } else {
3168 po->tp_version = val;
3169 ret = 0;
3170 }
3171 release_sock(sk);
3172 return ret;
bbd6ef87 3173 }
8913336a
PM		 3174 case PACKET_RESERVE:
3175 {
3176 unsigned int val;
3177
3178 if (optlen != sizeof(val))
3179 return -EINVAL;
69e3c75f 3180 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3181 return -EBUSY;
3182 if (copy_from_user(&val, optval, sizeof(val)))
3183 return -EFAULT;
4b9e9796
S		 3184 if (val > INT_MAX)
3185 return -EINVAL;
8913336a
PM		 3186 po->tp_reserve = val;
3187 return 0;
3188 }
69e3c75f
JB		 3189 case PACKET_LOSS:
3190 {
3191 unsigned int val;
3192
3193 if (optlen != sizeof(val))
3194 return -EINVAL;
3195 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3196 return -EBUSY;
3197 if (copy_from_user(&val, optval, sizeof(val)))
3198 return -EFAULT;
3199 po->tp_loss = !!val;
3200 return 0;
3201 }
8dc41944
HX		 3202 case PACKET_AUXDATA:
3203 {
3204 int val;
3205
3206 if (optlen < sizeof(val))
3207 return -EINVAL;
3208 if (copy_from_user(&val, optval, sizeof(val)))
3209 return -EFAULT;
3210
3211 po->auxdata = !!val;
3212 return 0;
3213 }
80feaacb
PWJ		 3214 case PACKET_ORIGDEV:
3215 {
3216 int val;
3217
3218 if (optlen < sizeof(val))
3219 return -EINVAL;
3220 if (copy_from_user(&val, optval, sizeof(val)))
3221 return -EFAULT;
3222
3223 po->origdev = !!val;
3224 return 0;
3225 }
bfd5f4a3
SS		 3226 case PACKET_VNET_HDR:
3227 {
3228 int val;
3229
3230 if (sock->type != SOCK_RAW)
3231 return -EINVAL;
3232 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3233 return -EBUSY;
3234 if (optlen < sizeof(val))
3235 return -EINVAL;
3236 if (copy_from_user(&val, optval, sizeof(val)))
3237 return -EFAULT;
3238
3239 po->has_vnet_hdr = !!val;
3240 return 0;
3241 }
614f60fa
SM		 3242 case PACKET_TIMESTAMP:
3243 {
3244 int val;
3245
3246 if (optlen != sizeof(val))
3247 return -EINVAL;
3248 if (copy_from_user(&val, optval, sizeof(val)))
3249 return -EFAULT;
3250
3251 po->tp_tstamp = val;
3252 return 0;
3253 }
dc99f600
DM		 3254 case PACKET_FANOUT:
3255 {
3256 int val;
3257
3258 if (optlen != sizeof(val))
3259 return -EINVAL;
3260 if (copy_from_user(&val, optval, sizeof(val)))
3261 return -EFAULT;
3262
3263 return fanout_add(sk, val & 0xffff, val >> 16);
3264 }
5920cd3a
PC		 3265 case PACKET_TX_HAS_OFF:
3266 {
3267 unsigned int val;
3268
3269 if (optlen != sizeof(val))
3270 return -EINVAL;
3271 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3272 return -EBUSY;
3273 if (copy_from_user(&val, optval, sizeof(val)))
3274 return -EFAULT;
3275 po->tp_tx_has_off = !!val;
3276 return 0;
3277 }
1da177e4
LT		 3278 default:
3279 return -ENOPROTOOPT;
3280 }
3281}
3282
3283static int packet_getsockopt(struct socket *sock, int level, int optname,
3284 char __user *optval, int __user *optlen)
3285{
3286 int len;
c06fff6e 3287 int val, lv = sizeof(val);
1da177e4
LT		 3288 struct sock *sk = sock->sk;
3289 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3290 void *data = &val;
ee80fbf3 3291 union tpacket_stats_u st;
1da177e4
LT		 3292
3293 if (level != SOL_PACKET)
3294 return -ENOPROTOOPT;
3295
8ae55f04
KK		 3296 if (get_user(len, optlen))
3297 return -EFAULT;
1da177e4
LT		 3298
3299 if (len < 0)
3300 return -EINVAL;
1ce4f28b 3301
69e3c75f 3302 switch (optname) {
1da177e4 3303 case PACKET_STATISTICS:
1da177e4 3304 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3305 memcpy(&st, &po->stats, sizeof(st));
3306 memset(&po->stats, 0, sizeof(po->stats));
3307 spin_unlock_bh(&sk->sk_receive_queue.lock);
3308
f6fb8f10 3309 if (po->tp_version == TPACKET_V3) {
c06fff6e 3310 lv = sizeof(struct tpacket_stats_v3);
fc26e4cf 3311 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3312 data = &st.stats3;
f6fb8f10 3313 } else {
c06fff6e 3314 lv = sizeof(struct tpacket_stats);
fc26e4cf 3315 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3316 data = &st.stats1;
f6fb8f10 3317 }
ee80fbf3 3318
8dc41944
HX		 3319 break;
3320 case PACKET_AUXDATA:
8dc41944 3321 val = po->auxdata;
80feaacb
PWJ		 3322 break;
3323 case PACKET_ORIGDEV:
80feaacb 3324 val = po->origdev;
bfd5f4a3
SS		 3325 break;
3326 case PACKET_VNET_HDR:
bfd5f4a3 3327 val = po->has_vnet_hdr;
1da177e4 3328 break;
bbd6ef87 3329 case PACKET_VERSION:
bbd6ef87 3330 val = po->tp_version;
bbd6ef87
PM		 3331 break;
3332 case PACKET_HDRLEN:
3333 if (len > sizeof(int))
3334 len = sizeof(int);
3335 if (copy_from_user(&val, optval, len))
3336 return -EFAULT;
3337 switch (val) {
3338 case TPACKET_V1:
3339 val = sizeof(struct tpacket_hdr);
3340 break;
3341 case TPACKET_V2:
3342 val = sizeof(struct tpacket2_hdr);
3343 break;
f6fb8f10 3344 case TPACKET_V3:
3345 val = sizeof(struct tpacket3_hdr);
3346 break;
bbd6ef87
PM		 3347 default:
3348 return -EINVAL;
3349 }
bbd6ef87 3350 break;
8913336a 3351 case PACKET_RESERVE:
8913336a 3352 val = po->tp_reserve;
8913336a 3353 break;
69e3c75f 3354 case PACKET_LOSS:
69e3c75f 3355 val = po->tp_loss;
69e3c75f 3356 break;
614f60fa 3357 case PACKET_TIMESTAMP:
614f60fa 3358 val = po->tp_tstamp;
614f60fa 3359 break;
dc99f600 3360 case PACKET_FANOUT:
dc99f600
DM		 3361 val = (po->fanout ?
3362 ((u32)po->fanout->id |
77f65ebd
WB		 3363 ((u32)po->fanout->type << 16) |
3364 ((u32)po->fanout->flags << 24)) :
dc99f600 3365 0);
dc99f600 3366 break;
5920cd3a
PC		 3367 case PACKET_TX_HAS_OFF:
3368 val = po->tp_tx_has_off;
3369 break;
1da177e4
LT		 3370 default:
3371 return -ENOPROTOOPT;
3372 }
3373
c06fff6e
ED		 3374 if (len > lv)
3375 len = lv;
8ae55f04
KK		 3376 if (put_user(len, optlen))
3377 return -EFAULT;
8dc41944
HX		 3378 if (copy_to_user(optval, data, len))
3379 return -EFAULT;
8ae55f04 3380 return 0;
1da177e4
LT		 3381}
3382
3383
3384static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3385{
3386 struct sock *sk;
ad930650 3387 struct net_device *dev = data;
c346dca1 3388 struct net *net = dev_net(dev);
1da177e4 3389
808f5114 3390 rcu_read_lock();
b67bfe0d 3391 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3392 struct packet_sock *po = pkt_sk(sk);
3393
3394 switch (msg) {
3395 case NETDEV_UNREGISTER:
1da177e4
LT		 3396 if (po->mclist)
3397 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3398 /* fallthrough */
3399
1da177e4
LT		 3400 case NETDEV_DOWN:
3401 if (dev->ifindex == po->ifindex) {
3402 spin_lock(&po->bind_lock);
3403 if (po->running) {
ce06b03e 3404 __unregister_prot_hook(sk, false);
1da177e4
LT		 3405 sk->sk_err = ENETDOWN;
3406 if (!sock_flag(sk, SOCK_DEAD))
3407 sk->sk_error_report(sk);
3408 }
3409 if (msg == NETDEV_UNREGISTER) {
c3ac8a13 3410 packet_cached_dev_reset(po);
1da177e4 3411 po->ifindex = -1;
160ff18a
BG		 3412 if (po->prot_hook.dev)
3413 dev_put(po->prot_hook.dev);
1da177e4
LT		 3414 po->prot_hook.dev = NULL;
3415 }
3416 spin_unlock(&po->bind_lock);
3417 }
3418 break;
3419 case NETDEV_UP:
808f5114 3420 if (dev->ifindex == po->ifindex) {
3421 spin_lock(&po->bind_lock);
ce06b03e
DM		 3422 if (po->num)
3423 register_prot_hook(sk);
808f5114 3424 spin_unlock(&po->bind_lock);
1da177e4 3425 }
1da177e4
LT		 3426 break;
3427 }
3428 }
808f5114 3429 rcu_read_unlock();
1da177e4
LT		 3430 return NOTIFY_DONE;
3431}
3432
3433
3434static int packet_ioctl(struct socket *sock, unsigned int cmd,
3435 unsigned long arg)
3436{
3437 struct sock *sk = sock->sk;
3438
69e3c75f 3439 switch (cmd) {
40d4e3df
ED		 3440 case SIOCOUTQ:
3441 {
3442 int amount = sk_wmem_alloc_get(sk);
31e6d363 3443
40d4e3df
ED		 3444 return put_user(amount, (int __user *)arg);
3445 }
3446 case SIOCINQ:
3447 {
3448 struct sk_buff *skb;
3449 int amount = 0;
3450
3451 spin_lock_bh(&sk->sk_receive_queue.lock);
3452 skb = skb_peek(&sk->sk_receive_queue);
3453 if (skb)
3454 amount = skb->len;
3455 spin_unlock_bh(&sk->sk_receive_queue.lock);
3456 return put_user(amount, (int __user *)arg);
3457 }
3458 case SIOCGSTAMP:
3459 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3460 case SIOCGSTAMPNS:
3461 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3462
1da177e4 3463#ifdef CONFIG_INET
40d4e3df
ED		 3464 case SIOCADDRT:
3465 case SIOCDELRT:
3466 case SIOCDARP:
3467 case SIOCGARP:
3468 case SIOCSARP:
3469 case SIOCGIFADDR:
3470 case SIOCSIFADDR:
3471 case SIOCGIFBRDADDR:
3472 case SIOCSIFBRDADDR:
3473 case SIOCGIFNETMASK:
3474 case SIOCSIFNETMASK:
3475 case SIOCGIFDSTADDR:
3476 case SIOCSIFDSTADDR:
3477 case SIOCSIFFLAGS:
40d4e3df 3478 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3479#endif
3480
40d4e3df
ED		 3481 default:
3482 return -ENOIOCTLCMD;
1da177e4
LT		 3483 }
3484 return 0;
3485}
3486
40d4e3df 3487static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3488 poll_table *wait)
3489{
3490 struct sock *sk = sock->sk;
3491 struct packet_sock *po = pkt_sk(sk);
3492 unsigned int mask = datagram_poll(file, sock, wait);
3493
3494 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3495 if (po->rx_ring.pg_vec) {
f6fb8f10 3496 if (!packet_previous_rx_frame(po, &po->rx_ring,
3497 TP_STATUS_KERNEL))
1da177e4
LT		 3498 mask |= POLLIN | POLLRDNORM;
3499 }
3500 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3501 spin_lock_bh(&sk->sk_write_queue.lock);
3502 if (po->tx_ring.pg_vec) {
3503 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3504 mask |= POLLOUT | POLLWRNORM;
3505 }
3506 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3507 return mask;
3508}
3509
3510
3511/* Dirty? Well, I still did not learn better way to account
3512 * for user mmaps.
3513 */
3514
3515static void packet_mm_open(struct vm_area_struct *vma)
3516{
3517 struct file *file = vma->vm_file;
40d4e3df 3518 struct socket *sock = file->private_data;
1da177e4 3519 struct sock *sk = sock->sk;
1ce4f28b 3520
1da177e4
LT		 3521 if (sk)
3522 atomic_inc(&pkt_sk(sk)->mapped);
3523}
3524
3525static void packet_mm_close(struct vm_area_struct *vma)
3526{
3527 struct file *file = vma->vm_file;
40d4e3df 3528 struct socket *sock = file->private_data;
1da177e4 3529 struct sock *sk = sock->sk;
1ce4f28b 3530
1da177e4
LT		 3531 if (sk)
3532 atomic_dec(&pkt_sk(sk)->mapped);
3533}
3534
f0f37e2f 3535static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3536 .open = packet_mm_open,
3537 .close = packet_mm_close,
1da177e4
LT		 3538};
3539
0e3125c7
NH		 3540static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3541 unsigned int len)
1da177e4
LT		 3542{
3543 int i;
3544
4ebf0ae2 3545 for (i = 0; i < len; i++) {
0e3125c7 3546 if (likely(pg_vec[i].buffer)) {
c56b4d90 3547 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3548 vfree(pg_vec[i].buffer);
3549 else
3550 free_pages((unsigned long)pg_vec[i].buffer,
3551 order);
3552 pg_vec[i].buffer = NULL;
3553 }
1da177e4
LT		 3554 }
3555 kfree(pg_vec);
3556}
3557
eea49cc9 3558static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3559{
0e3125c7
NH		 3560 char *buffer = NULL;
3561 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3562 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3563
3564 buffer = (char *) __get_free_pages(gfp_flags, order);
3565
3566 if (buffer)
3567 return buffer;
3568
3569 /*
3570 * __get_free_pages failed, fall back to vmalloc
3571 */
bbce5a59 3572 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3573
0e3125c7
NH		 3574 if (buffer)
3575 return buffer;
3576
3577 /*
3578 * vmalloc failed, lets dig into swap here
3579 */
0e3125c7
NH		 3580 gfp_flags &= ~__GFP_NORETRY;
3581 buffer = (char *)__get_free_pages(gfp_flags, order);
3582 if (buffer)
3583 return buffer;
3584
3585 /*
3586 * complete and utter failure
3587 */
3588 return NULL;
4ebf0ae2
DM		 3589}
3590
0e3125c7 3591static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3592{
3593 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3594 struct pgv *pg_vec;
4ebf0ae2
DM		 3595 int i;
3596
0e3125c7 3597 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3598 if (unlikely(!pg_vec))
3599 goto out;
3600
3601 for (i = 0; i < block_nr; i++) {
c56b4d90 3602 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3603 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3604 goto out_free_pgvec;
3605 }
3606
3607out:
3608 return pg_vec;
3609
3610out_free_pgvec:
3611 free_pg_vec(pg_vec, order, block_nr);
3612 pg_vec = NULL;
3613 goto out;
3614}
1da177e4 3615
f6fb8f10 3616static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3617 int closing, int tx_ring)
1da177e4 3618{
0e3125c7 3619 struct pgv *pg_vec = NULL;
1da177e4 3620 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3621 int was_running, order = 0;
69e3c75f
JB		 3622 struct packet_ring_buffer *rb;
3623 struct sk_buff_head *rb_queue;
0e11c91e 3624 __be16 num;
f6fb8f10 3625 int err = -EINVAL;
3626 /* Added to avoid minimal code churn */
3627 struct tpacket_req *req = &req_u->req;
3628
4b9e9796 3629 lock_sock(sk);
f6fb8f10 3630 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3631 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3632 WARN(1, "Tx-ring is not supported.\n");
3633 goto out;
3634 }
1ce4f28b 3635
69e3c75f
JB		 3636 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3637 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3638
69e3c75f
JB		 3639 err = -EBUSY;
3640 if (!closing) {
3641 if (atomic_read(&po->mapped))
3642 goto out;
3643 if (atomic_read(&rb->pending))
3644 goto out;
3645 }
1da177e4 3646
69e3c75f
JB		 3647 if (req->tp_block_nr) {
3648 /* Sanity tests and some calculations */
3649 err = -EBUSY;
3650 if (unlikely(rb->pg_vec))
3651 goto out;
1da177e4 3652
bbd6ef87
PM		 3653 switch (po->tp_version) {
3654 case TPACKET_V1:
3655 po->tp_hdrlen = TPACKET_HDRLEN;
3656 break;
3657 case TPACKET_V2:
3658 po->tp_hdrlen = TPACKET2_HDRLEN;
3659 break;
f6fb8f10 3660 case TPACKET_V3:
3661 po->tp_hdrlen = TPACKET3_HDRLEN;
3662 break;
bbd6ef87
PM		 3663 }
3664
69e3c75f 3665 err = -EINVAL;
4ebf0ae2 3666 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3667 goto out;
4ebf0ae2 3668 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3669 goto out;
4b9e9796 3670 if (po->tp_version >= TPACKET_V3 &&
4035ed7b
ED		 3671 (int)(req->tp_block_size -
3672 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
4b9e9796 3673 goto out;
8913336a 3674 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3675 po->tp_reserve))
3676 goto out;
4ebf0ae2 3677 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3678 goto out;
1da177e4 3679
69e3c75f
JB		 3680 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3681 if (unlikely(rb->frames_per_block <= 0))
3682 goto out;
4b9e9796
S		 3683 if (unlikely(req->tp_block_size > UINT_MAX / req->tp_block_nr))
3684 goto out;
69e3c75f
JB		 3685 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3686 req->tp_frame_nr))
3687 goto out;
1da177e4
LT		 3688
3689 err = -ENOMEM;
4ebf0ae2
DM		 3690 order = get_order(req->tp_block_size);
3691 pg_vec = alloc_pg_vec(req, order);
3692 if (unlikely(!pg_vec))
1da177e4 3693 goto out;
f6fb8f10 3694 switch (po->tp_version) {
3695 case TPACKET_V3:
3696 /* Transmit path is not supported. We checked
3697 * it above but just being paranoid
3698 */
3699 if (!tx_ring)
3700 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3701 break;
3702 default:
3703 break;
3704 }
69e3c75f
JB		 3705 }
3706 /* Done */
3707 else {
3708 err = -EINVAL;
4ebf0ae2 3709 if (unlikely(req->tp_frame_nr))
69e3c75f 3710 goto out;
1da177e4
LT		 3711 }
3712
1da177e4
LT		 3713
3714 /* Detach socket from network */
3715 spin_lock(&po->bind_lock);
3716 was_running = po->running;
3717 num = po->num;
3718 if (was_running) {
1da177e4 3719 po->num = 0;
ce06b03e 3720 __unregister_prot_hook(sk, false);
1da177e4
LT		 3721 }
3722 spin_unlock(&po->bind_lock);
1ce4f28b 3723
1da177e4
LT		 3724 synchronize_net();
3725
3726 err = -EBUSY;
905db440 3727 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3728 if (closing || atomic_read(&po->mapped) == 0) {
3729 err = 0;
69e3c75f 3730 spin_lock_bh(&rb_queue->lock);
c053fd96 3731 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3732 rb->frame_max = (req->tp_frame_nr - 1);
3733 rb->head = 0;
3734 rb->frame_size = req->tp_frame_size;
3735 spin_unlock_bh(&rb_queue->lock);
3736
c053fd96
CG		 3737 swap(rb->pg_vec_order, order);
3738 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3739
3740 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3741 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3742 tpacket_rcv : packet_rcv;
3743 skb_queue_purge(rb_queue);
1da177e4 3744 if (atomic_read(&po->mapped))
40d4e3df
ED		 3745 pr_err("packet_mmap: vma is busy: %d\n",
3746 atomic_read(&po->mapped));
1da177e4 3747 }
905db440 3748 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3749
3750 spin_lock(&po->bind_lock);
ce06b03e 3751 if (was_running) {
1da177e4 3752 po->num = num;
ce06b03e 3753 register_prot_hook(sk);
1da177e4
LT		 3754 }
3755 spin_unlock(&po->bind_lock);
f6fb8f10 3756 if (closing && (po->tp_version > TPACKET_V2)) {
3757 /* Because we don't support block-based V3 on tx-ring */
3758 if (!tx_ring)
3759 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3760 }
1da177e4 3761
1da177e4
LT		 3762 if (pg_vec)
3763 free_pg_vec(pg_vec, order, req->tp_block_nr);
3764out:
4b9e9796 3765 release_sock(sk);
1da177e4
LT		 3766 return err;
3767}
3768
69e3c75f
JB		 3769static int packet_mmap(struct file *file, struct socket *sock,
3770 struct vm_area_struct *vma)
1da177e4
LT		 3771{
3772 struct sock *sk = sock->sk;
3773 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3774 unsigned long size, expected_size;
3775 struct packet_ring_buffer *rb;
1da177e4
LT		 3776 unsigned long start;
3777 int err = -EINVAL;
3778 int i;
3779
3780 if (vma->vm_pgoff)
3781 return -EINVAL;
3782
905db440 3783 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3784
3785 expected_size = 0;
3786 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3787 if (rb->pg_vec) {
3788 expected_size += rb->pg_vec_len
3789 * rb->pg_vec_pages
3790 * PAGE_SIZE;
3791 }
3792 }
3793
3794 if (expected_size == 0)
1da177e4 3795 goto out;
69e3c75f
JB		 3796
3797 size = vma->vm_end - vma->vm_start;
3798 if (size != expected_size)
1da177e4
LT		 3799 goto out;
3800
1da177e4 3801 start = vma->vm_start;
69e3c75f
JB		 3802 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3803 if (rb->pg_vec == NULL)
3804 continue;
3805
3806 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3807 struct page *page;
3808 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3809 int pg_num;
3810
c56b4d90
CG		 3811 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3812 page = pgv_to_page(kaddr);
69e3c75f
JB		 3813 err = vm_insert_page(vma, start, page);
3814 if (unlikely(err))
3815 goto out;
3816 start += PAGE_SIZE;
0e3125c7 3817 kaddr += PAGE_SIZE;
69e3c75f 3818 }
4ebf0ae2 3819 }
1da177e4 3820 }
69e3c75f 3821
4ebf0ae2 3822 atomic_inc(&po->mapped);
1da177e4
LT		 3823 vma->vm_ops = &packet_mmap_ops;
3824 err = 0;
3825
3826out:
905db440 3827 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3828 return err;
3829}
1da177e4 3830
90ddc4f0 3831static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3832 .family = PF_PACKET,
3833 .owner = THIS_MODULE,
3834 .release = packet_release,
3835 .bind = packet_bind_spkt,
3836 .connect = sock_no_connect,
3837 .socketpair = sock_no_socketpair,
3838 .accept = sock_no_accept,
3839 .getname = packet_getname_spkt,
3840 .poll = datagram_poll,
3841 .ioctl = packet_ioctl,
3842 .listen = sock_no_listen,
3843 .shutdown = sock_no_shutdown,
3844 .setsockopt = sock_no_setsockopt,
3845 .getsockopt = sock_no_getsockopt,
3846 .sendmsg = packet_sendmsg_spkt,
3847 .recvmsg = packet_recvmsg,
3848 .mmap = sock_no_mmap,
3849 .sendpage = sock_no_sendpage,
3850};
1da177e4 3851
90ddc4f0 3852static const struct proto_ops packet_ops = {
1da177e4
LT		 3853 .family = PF_PACKET,
3854 .owner = THIS_MODULE,
3855 .release = packet_release,
3856 .bind = packet_bind,
3857 .connect = sock_no_connect,
3858 .socketpair = sock_no_socketpair,
3859 .accept = sock_no_accept,
1ce4f28b 3860 .getname = packet_getname,
1da177e4
LT		 3861 .poll = packet_poll,
3862 .ioctl = packet_ioctl,
3863 .listen = sock_no_listen,
3864 .shutdown = sock_no_shutdown,
3865 .setsockopt = packet_setsockopt,
3866 .getsockopt = packet_getsockopt,
3867 .sendmsg = packet_sendmsg,
3868 .recvmsg = packet_recvmsg,
3869 .mmap = packet_mmap,
3870 .sendpage = sock_no_sendpage,
3871};
3872
ec1b4cf7 3873static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3874 .family = PF_PACKET,
3875 .create = packet_create,
3876 .owner = THIS_MODULE,
3877};
3878
3879static struct notifier_block packet_netdev_notifier = {
40d4e3df 3880 .notifier_call = packet_notifier,
1da177e4
LT		 3881};
3882
3883#ifdef CONFIG_PROC_FS
1da177e4
LT		 3884
3885static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3886 __acquires(RCU)
1da177e4 3887{
e372c414 3888 struct net *net = seq_file_net(seq);
808f5114 3889
3890 rcu_read_lock();
3891 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3892}
3893
3894static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3895{
1bf40954 3896 struct net *net = seq_file_net(seq);
808f5114 3897 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3898}
3899
3900static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3901 __releases(RCU)
1da177e4 3902{
808f5114 3903 rcu_read_unlock();
1da177e4
LT		 3904}
3905
1ce4f28b 3906static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3907{
3908 if (v == SEQ_START_TOKEN)
3909 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3910 else {
b7ceabd9 3911 struct sock *s = sk_entry(v);
1da177e4
LT		 3912 const struct packet_sock *po = pkt_sk(s);
3913
3914 seq_printf(seq,
71338aa7 3915 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3916 s,
3917 atomic_read(&s->sk_refcnt),
3918 s->sk_type,
3919 ntohs(po->num),
3920 po->ifindex,
3921 po->running,
3922 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3923 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3924 sock_i_ino(s));
1da177e4
LT		 3925 }
3926
3927 return 0;
3928}
3929
56b3d975 3930static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3931 .start = packet_seq_start,
3932 .next = packet_seq_next,
3933 .stop = packet_seq_stop,
3934 .show = packet_seq_show,
3935};
3936
3937static int packet_seq_open(struct inode *inode, struct file *file)
3938{
e372c414
DL		 3939 return seq_open_net(inode, file, &packet_seq_ops,
3940 sizeof(struct seq_net_private));
1da177e4
LT		 3941}
3942
da7071d7 3943static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3944 .owner = THIS_MODULE,
3945 .open = packet_seq_open,
3946 .read = seq_read,
3947 .llseek = seq_lseek,
e372c414 3948 .release = seq_release_net,
1da177e4
LT		 3949};
3950
3951#endif
3952
2c8c1e72 3953static int __net_init packet_net_init(struct net *net)
d12d01d6 3954{
0fa7fa98 3955 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3956 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3957
d4beaa66 3958 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3959 return -ENOMEM;
3960
3961 return 0;
3962}
3963
2c8c1e72 3964static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3965{
ece31ffd 3966 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3967}
3968
3969static struct pernet_operations packet_net_ops = {
3970 .init = packet_net_init,
3971 .exit = packet_net_exit,
3972};
3973
3974
1da177e4
LT		 3975static void __exit packet_exit(void)
3976{
1da177e4 3977 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3978 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3979 sock_unregister(PF_PACKET);
3980 proto_unregister(&packet_proto);
3981}
3982
3983static int __init packet_init(void)
3984{
3985 int rc = proto_register(&packet_proto, 0);
3986
3987 if (rc != 0)
3988 goto out;
3989
3990 sock_register(&packet_family_ops);
d12d01d6 3991 register_pernet_subsys(&packet_net_ops);
1da177e4 3992 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3993out:
3994 return rc;
3995}
3996
3997module_init(packet_init);
3998module_exit(packet_exit);
3999MODULE_LICENSE("GPL");
4000MODULE_ALIAS_NETPROTO(PF_PACKET);
kernel source for telekom puls (alcatel/mediatek)