projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge git://git.kernel.org/pub/scm/linux/kernel/git/jkirsher/net-next
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
76#include <asm/system.h>
77#include <asm/uaccess.h>
78#include <asm/ioctls.h>
79#include <asm/page.h>
a1f8e7f7 80#include <asm/cacheflush.h>
1da177e4
LT		 81#include <asm/io.h>
82#include <linux/proc_fs.h>
83#include <linux/seq_file.h>
84#include <linux/poll.h>
85#include <linux/module.h>
86#include <linux/init.h>
905db440 87#include <linux/mutex.h>
05423b24 88#include <linux/if_vlan.h>
bfd5f4a3 89#include <linux/virtio_net.h>
ed85b565 90#include <linux/errqueue.h>
614f60fa 91#include <linux/net_tstamp.h>
1da177e4
LT		 92
93#ifdef CONFIG_INET
94#include <net/inet_common.h>
95#endif
96
1da177e4
LT		 97/*
98 Assumptions:
99 - if device has no dev->hard_header routine, it adds and removes ll header
100 inside itself. In this case ll header is invisible outside of device,
101 but higher levels still should reserve dev->hard_header_len.
102 Some devices are enough clever to reallocate skb, when header
103 will not fit to reserved space (tunnel), another ones are silly
104 (PPP).
105 - packet socket receives packets with pulled ll header,
106 so that SOCK_RAW should push it back.
107
108On receive:
109-----------
110
111Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 112 mac_header -> ll header
113 data -> data
1da177e4
LT		 114
115Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 116 mac_header -> ll header
117 data -> ll header
1da177e4
LT		 118
119Incoming, dev->hard_header==NULL
b0e380b1
ACM		 120 mac_header -> UNKNOWN position. It is very likely, that it points to ll
121 header. PPP makes it, that is wrong, because introduce
db0c58f9 122 assymetry between rx and tx paths.
b0e380b1 123 data -> data
1da177e4
LT		 124
125Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 126 mac_header -> data. ll header is still not built!
127 data -> data
1da177e4
LT		 128
129Resume
130 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
131
132
133On transmit:
134------------
135
136dev->hard_header != NULL
b0e380b1
ACM		 137 mac_header -> ll header
138 data -> ll header
1da177e4
LT		 139
140dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 141 mac_header -> data
142 data -> data
1da177e4
LT		 143
144 We should set nh.raw on output to correct posistion,
145 packet classifier depends on it.
146 */
147
1da177e4
LT		 148/* Private packet socket structures. */
149
40d4e3df 150struct packet_mclist {
1da177e4
LT		 151 struct packet_mclist *next;
152 int ifindex;
153 int count;
154 unsigned short type;
155 unsigned short alen;
0fb375fb
EB		 156 unsigned char addr[MAX_ADDR_LEN];
157};
158/* identical to struct packet_mreq except it has
159 * a longer address field.
160 */
40d4e3df 161struct packet_mreq_max {
0fb375fb
EB		 162 int mr_ifindex;
163 unsigned short mr_type;
164 unsigned short mr_alen;
165 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 166};
a2efcfa0 167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171
172#define V3_ALIGNMENT (8)
173
bc59ba39 174#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 175
176#define BLK_PLUS_PRIV(sz_of_priv) \
177 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178
179/* kbdq - kernel block descriptor queue */
bc59ba39 180struct tpacket_kbdq_core {
f6fb8f10 181 struct pgv *pkbdq;
182 unsigned int feature_req_word;
183 unsigned int hdrlen;
184 unsigned char reset_pending_on_curr_blk;
185 unsigned char delete_blk_timer;
186 unsigned short kactive_blk_num;
187 unsigned short blk_sizeof_priv;
188
189 /* last_kactive_blk_num:
190 * trick to see if user-space has caught up
191 * in order to avoid refreshing timer when every single pkt arrives.
192 */
193 unsigned short last_kactive_blk_num;
194
195 char *pkblk_start;
196 char *pkblk_end;
197 int kblk_size;
198 unsigned int knum_blocks;
199 uint64_t knxt_seq_num;
200 char *prev;
201 char *nxt_offset;
202 struct sk_buff *skb;
203
204 atomic_t blk_fill_in_prog;
205
206 /* Default is set to 8ms */
207#define DEFAULT_PRB_RETIRE_TOV (8)
208
209 unsigned short retire_blk_tov;
210 unsigned short version;
211 unsigned long tov_in_jiffies;
212
213 /* timer to retire an outstanding block */
214 struct timer_list retire_blk_timer;
215};
216
217#define PGV_FROM_VMALLOC 1
0e3125c7
NH		 218struct pgv {
219 char *buffer;
0e3125c7
NH		 220};
221
69e3c75f 222struct packet_ring_buffer {
0e3125c7 223 struct pgv *pg_vec;
69e3c75f
JB		 224 unsigned int head;
225 unsigned int frames_per_block;
226 unsigned int frame_size;
227 unsigned int frame_max;
228
229 unsigned int pg_vec_order;
230 unsigned int pg_vec_pages;
231 unsigned int pg_vec_len;
232
bc59ba39 233 struct tpacket_kbdq_core prb_bdqc;
69e3c75f
JB		 234 atomic_t pending;
235};
236
f6fb8f10 237#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
238#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
239#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
240#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
241#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
242#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
243#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
244
69e3c75f
JB		 245struct packet_sock;
246static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
1da177e4 247
f6fb8f10 248static void *packet_previous_frame(struct packet_sock *po,
249 struct packet_ring_buffer *rb,
250 int status);
251static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 252static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
253 struct tpacket_block_desc *);
254static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 255 struct packet_sock *);
bc59ba39 256static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 257 struct packet_sock *, unsigned int status);
bc59ba39 258static int prb_queue_frozen(struct tpacket_kbdq_core *);
259static void prb_open_block(struct tpacket_kbdq_core *,
260 struct tpacket_block_desc *);
f6fb8f10 261static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 262static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
263static void prb_init_blk_timer(struct packet_sock *,
264 struct tpacket_kbdq_core *,
265 void (*func) (unsigned long));
266static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
267static void prb_clear_rxhash(struct tpacket_kbdq_core *,
268 struct tpacket3_hdr *);
269static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
270 struct tpacket3_hdr *);
1da177e4
LT		 271static void packet_flush_mclist(struct sock *sk);
272
dc99f600 273struct packet_fanout;
1da177e4
LT		 274struct packet_sock {
275 /* struct sock has to be the first member of packet_sock */
276 struct sock sk;
dc99f600 277 struct packet_fanout *fanout;
1da177e4 278 struct tpacket_stats stats;
f6fb8f10 279 union tpacket_stats_u stats_u;
69e3c75f
JB		 280 struct packet_ring_buffer rx_ring;
281 struct packet_ring_buffer tx_ring;
1da177e4 282 int copy_thresh;
1da177e4 283 spinlock_t bind_lock;
905db440 284 struct mutex pg_vec_lock;
8dc41944 285 unsigned int running:1, /* prot_hook is attached*/
80feaacb 286 auxdata:1,
bfd5f4a3
SS		 287 origdev:1,
288 has_vnet_hdr:1;
1da177e4 289 int ifindex; /* bound device */
0e11c91e 290 __be16 num;
1da177e4 291 struct packet_mclist *mclist;
1da177e4 292 atomic_t mapped;
bbd6ef87
PM		 293 enum tpacket_versions tp_version;
294 unsigned int tp_hdrlen;
8913336a 295 unsigned int tp_reserve;
69e3c75f 296 unsigned int tp_loss:1;
614f60fa 297 unsigned int tp_tstamp;
94b05952 298 struct packet_type prot_hook ____cacheline_aligned_in_smp;
1da177e4
LT		 299};
300
dc99f600
DM		 301#define PACKET_FANOUT_MAX 256
302
303struct packet_fanout {
304#ifdef CONFIG_NET_NS
305 struct net *net;
306#endif
307 unsigned int num_members;
308 u16 id;
309 u8 type;
7736d33f 310 u8 defrag;
dc99f600
DM		 311 atomic_t rr_cur;
312 struct list_head list;
313 struct sock *arr[PACKET_FANOUT_MAX];
314 spinlock_t lock;
315 atomic_t sk_ref;
316 struct packet_type prot_hook ____cacheline_aligned_in_smp;
317};
318
ffbc6111
HX		 319struct packet_skb_cb {
320 unsigned int origlen;
321 union {
322 struct sockaddr_pkt pkt;
323 struct sockaddr_ll ll;
324 } sa;
325};
326
327#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 328
bc59ba39 329#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 330#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 331 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 332#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 333 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 334#define GET_NEXT_PRB_BLK_NUM(x) \
335 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
336 ((x)->kactive_blk_num+1) : 0)
337
ce06b03e
DM		 338static inline struct packet_sock *pkt_sk(struct sock *sk)
339{
340 return (struct packet_sock *)sk;
341}
342
dc99f600
DM		 343static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
344static void __fanout_link(struct sock *sk, struct packet_sock *po);
345
ce06b03e
DM		 346/* register_prot_hook must be invoked with the po->bind_lock held,
347 * or from a context in which asynchronous accesses to the packet
348 * socket is not possible (packet_create()).
349 */
350static void register_prot_hook(struct sock *sk)
351{
352 struct packet_sock *po = pkt_sk(sk);
353 if (!po->running) {
dc99f600
DM		 354 if (po->fanout)
355 __fanout_link(sk, po);
356 else
357 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 358 sock_hold(sk);
359 po->running = 1;
360 }
361}
362
363/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
364 * held. If the sync parameter is true, we will temporarily drop
365 * the po->bind_lock and do a synchronize_net to make sure no
366 * asynchronous packet processing paths still refer to the elements
367 * of po->prot_hook. If the sync parameter is false, it is the
368 * callers responsibility to take care of this.
369 */
370static void __unregister_prot_hook(struct sock *sk, bool sync)
371{
372 struct packet_sock *po = pkt_sk(sk);
373
374 po->running = 0;
dc99f600
DM		 375 if (po->fanout)
376 __fanout_unlink(sk, po);
377 else
378 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 379 __sock_put(sk);
380
381 if (sync) {
382 spin_unlock(&po->bind_lock);
383 synchronize_net();
384 spin_lock(&po->bind_lock);
385 }
386}
387
388static void unregister_prot_hook(struct sock *sk, bool sync)
389{
390 struct packet_sock *po = pkt_sk(sk);
391
392 if (po->running)
393 __unregister_prot_hook(sk, sync);
394}
395
f6dafa95 396static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 397{
398 if (is_vmalloc_addr(addr))
399 return vmalloc_to_page(addr);
400 return virt_to_page(addr);
401}
402
69e3c75f 403static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 404{
bbd6ef87
PM		 405 union {
406 struct tpacket_hdr *h1;
407 struct tpacket2_hdr *h2;
408 void *raw;
409 } h;
1da177e4 410
69e3c75f 411 h.raw = frame;
bbd6ef87
PM		 412 switch (po->tp_version) {
413 case TPACKET_V1:
69e3c75f 414 h.h1->tp_status = status;
0af55bb5 415 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 416 break;
417 case TPACKET_V2:
69e3c75f 418 h.h2->tp_status = status;
0af55bb5 419 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 420 break;
f6fb8f10 421 case TPACKET_V3:
69e3c75f 422 default:
f6fb8f10 423 WARN(1, "TPACKET version not supported.\n");
69e3c75f 424 BUG();
bbd6ef87 425 }
69e3c75f
JB		 426
427 smp_wmb();
bbd6ef87
PM		 428}
429
69e3c75f 430static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87
PM		 431{
432 union {
433 struct tpacket_hdr *h1;
434 struct tpacket2_hdr *h2;
435 void *raw;
436 } h;
437
69e3c75f
JB		 438 smp_rmb();
439
bbd6ef87
PM		 440 h.raw = frame;
441 switch (po->tp_version) {
442 case TPACKET_V1:
0af55bb5 443 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 444 return h.h1->tp_status;
bbd6ef87 445 case TPACKET_V2:
0af55bb5 446 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 447 return h.h2->tp_status;
f6fb8f10 448 case TPACKET_V3:
69e3c75f 449 default:
f6fb8f10 450 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 451 BUG();
452 return 0;
bbd6ef87 453 }
1da177e4 454}
69e3c75f
JB		 455
456static void *packet_lookup_frame(struct packet_sock *po,
457 struct packet_ring_buffer *rb,
458 unsigned int position,
459 int status)
460{
461 unsigned int pg_vec_pos, frame_offset;
462 union {
463 struct tpacket_hdr *h1;
464 struct tpacket2_hdr *h2;
465 void *raw;
466 } h;
467
468 pg_vec_pos = position / rb->frames_per_block;
469 frame_offset = position % rb->frames_per_block;
470
0e3125c7
NH		 471 h.raw = rb->pg_vec[pg_vec_pos].buffer +
472 (frame_offset * rb->frame_size);
69e3c75f
JB		 473
474 if (status != __packet_get_status(po, h.raw))
475 return NULL;
476
477 return h.raw;
478}
479
480static inline void *packet_current_frame(struct packet_sock *po,
481 struct packet_ring_buffer *rb,
482 int status)
483{
484 return packet_lookup_frame(po, rb, rb->head, status);
485}
486
bc59ba39 487static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 488{
489 del_timer_sync(&pkc->retire_blk_timer);
490}
491
492static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
493 int tx_ring,
494 struct sk_buff_head *rb_queue)
495{
bc59ba39 496 struct tpacket_kbdq_core *pkc;
f6fb8f10 497
498 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
499
500 spin_lock(&rb_queue->lock);
501 pkc->delete_blk_timer = 1;
502 spin_unlock(&rb_queue->lock);
503
504 prb_del_retire_blk_timer(pkc);
505}
506
507static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 508 struct tpacket_kbdq_core *pkc,
f6fb8f10 509 void (*func) (unsigned long))
510{
511 init_timer(&pkc->retire_blk_timer);
512 pkc->retire_blk_timer.data = (long)po;
513 pkc->retire_blk_timer.function = func;
514 pkc->retire_blk_timer.expires = jiffies;
515}
516
517static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
518{
bc59ba39 519 struct tpacket_kbdq_core *pkc;
f6fb8f10 520
521 if (tx_ring)
522 BUG();
523
524 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
525 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
526}
527
528static int prb_calc_retire_blk_tmo(struct packet_sock *po,
529 int blk_size_in_bytes)
530{
531 struct net_device *dev;
532 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 533 struct ethtool_cmd ecmd;
534 int err;
f6fb8f10 535
4bc71cb9
JP		 536 rtnl_lock();
537 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
538 if (unlikely(!dev)) {
539 rtnl_unlock();
f6fb8f10 540 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 541 }
542 err = __ethtool_get_settings(dev, &ecmd);
543 rtnl_unlock();
544 if (!err) {
545 switch (ecmd.speed) {
546 case SPEED_10000:
547 msec = 1;
548 div = 10000/1000;
549 break;
550 case SPEED_1000:
551 msec = 1;
552 div = 1000/1000;
553 break;
554 /*
555 * If the link speed is so slow you don't really
556 * need to worry about perf anyways
557 */
558 case SPEED_100:
559 case SPEED_10:
560 default:
561 return DEFAULT_PRB_RETIRE_TOV;
f6fb8f10 562 }
563 }
564
565 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
566
567 if (div)
568 mbits /= div;
569
570 tmo = mbits * msec;
571
572 if (div)
573 return tmo+1;
574 return tmo;
575}
576
bc59ba39 577static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 578 union tpacket_req_u *req_u)
579{
580 p1->feature_req_word = req_u->req3.tp_feature_req_word;
581}
582
583static void init_prb_bdqc(struct packet_sock *po,
584 struct packet_ring_buffer *rb,
585 struct pgv *pg_vec,
586 union tpacket_req_u *req_u, int tx_ring)
587{
bc59ba39 588 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
589 struct tpacket_block_desc *pbd;
f6fb8f10 590
591 memset(p1, 0x0, sizeof(*p1));
592
593 p1->knxt_seq_num = 1;
594 p1->pkbdq = pg_vec;
bc59ba39 595 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
f6fb8f10 596 p1->pkblk_start = (char *)pg_vec[0].buffer;
597 p1->kblk_size = req_u->req3.tp_block_size;
598 p1->knum_blocks = req_u->req3.tp_block_nr;
599 p1->hdrlen = po->tp_hdrlen;
600 p1->version = po->tp_version;
601 p1->last_kactive_blk_num = 0;
602 po->stats_u.stats3.tp_freeze_q_cnt = 0;
603 if (req_u->req3.tp_retire_blk_tov)
604 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
605 else
606 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
607 req_u->req3.tp_block_size);
608 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
609 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
610
611 prb_init_ft_ops(p1, req_u);
612 prb_setup_retire_blk_timer(po, tx_ring);
613 prb_open_block(p1, pbd);
614}
615
616/* Do NOT update the last_blk_num first.
617 * Assumes sk_buff_head lock is held.
618 */
bc59ba39 619static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 620{
621 mod_timer(&pkc->retire_blk_timer,
622 jiffies + pkc->tov_in_jiffies);
623 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
624}
625
626/*
627 * Timer logic:
628 * 1) We refresh the timer only when we open a block.
629 * By doing this we don't waste cycles refreshing the timer
630 * on packet-by-packet basis.
631 *
632 * With a 1MB block-size, on a 1Gbps line, it will take
633 * i) ~8 ms to fill a block + ii) memcpy etc.
634 * In this cut we are not accounting for the memcpy time.
635 *
636 * So, if the user sets the 'tmo' to 10ms then the timer
637 * will never fire while the block is still getting filled
638 * (which is what we want). However, the user could choose
639 * to close a block early and that's fine.
640 *
641 * But when the timer does fire, we check whether or not to refresh it.
642 * Since the tmo granularity is in msecs, it is not too expensive
643 * to refresh the timer, lets say every '8' msecs.
644 * Either the user can set the 'tmo' or we can derive it based on
645 * a) line-speed and b) block-size.
646 * prb_calc_retire_blk_tmo() calculates the tmo.
647 *
648 */
649static void prb_retire_rx_blk_timer_expired(unsigned long data)
650{
651 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 652 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 653 unsigned int frozen;
bc59ba39 654 struct tpacket_block_desc *pbd;
f6fb8f10 655
656 spin_lock(&po->sk.sk_receive_queue.lock);
657
658 frozen = prb_queue_frozen(pkc);
659 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
660
661 if (unlikely(pkc->delete_blk_timer))
662 goto out;
663
664 /* We only need to plug the race when the block is partially filled.
665 * tpacket_rcv:
666 * lock(); increment BLOCK_NUM_PKTS; unlock()
667 * copy_bits() is in progress ...
668 * timer fires on other cpu:
669 * we can't retire the current block because copy_bits
670 * is in progress.
671 *
672 */
673 if (BLOCK_NUM_PKTS(pbd)) {
674 while (atomic_read(&pkc->blk_fill_in_prog)) {
675 /* Waiting for skb_copy_bits to finish... */
676 cpu_relax();
677 }
678 }
679
680 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
681 if (!frozen) {
682 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
683 if (!prb_dispatch_next_block(pkc, po))
684 goto refresh_timer;
685 else
686 goto out;
687 } else {
688 /* Case 1. Queue was frozen because user-space was
689 * lagging behind.
690 */
691 if (prb_curr_blk_in_use(pkc, pbd)) {
692 /*
693 * Ok, user-space is still behind.
694 * So just refresh the timer.
695 */
696 goto refresh_timer;
697 } else {
698 /* Case 2. queue was frozen,user-space caught up,
699 * now the link went idle && the timer fired.
700 * We don't have a block to close.So we open this
701 * block and restart the timer.
702 * opening a block thaws the queue,restarts timer
703 * Thawing/timer-refresh is a side effect.
704 */
705 prb_open_block(pkc, pbd);
706 goto out;
707 }
708 }
709 }
710
711refresh_timer:
712 _prb_refresh_rx_retire_blk_timer(pkc);
713
714out:
715 spin_unlock(&po->sk.sk_receive_queue.lock);
716}
717
bc59ba39 718static inline void prb_flush_block(struct tpacket_kbdq_core *pkc1,
719 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 720{
721 /* Flush everything minus the block header */
722
723#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
724 u8 *start, *end;
725
726 start = (u8 *)pbd1;
727
728 /* Skip the block header(we know header WILL fit in 4K) */
729 start += PAGE_SIZE;
730
731 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
732 for (; start < end; start += PAGE_SIZE)
733 flush_dcache_page(pgv_to_page(start));
734
735 smp_wmb();
736#endif
737
738 /* Now update the block status. */
739
740 BLOCK_STATUS(pbd1) = status;
741
742 /* Flush the block header */
743
744#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
745 start = (u8 *)pbd1;
746 flush_dcache_page(pgv_to_page(start));
747
748 smp_wmb();
749#endif
750}
751
752/*
753 * Side effect:
754 *
755 * 1) flush the block
756 * 2) Increment active_blk_num
757 *
758 * Note:We DONT refresh the timer on purpose.
759 * Because almost always the next block will be opened.
760 */
bc59ba39 761static void prb_close_block(struct tpacket_kbdq_core *pkc1,
762 struct tpacket_block_desc *pbd1,
f6fb8f10 763 struct packet_sock *po, unsigned int stat)
764{
765 __u32 status = TP_STATUS_USER | stat;
766
767 struct tpacket3_hdr *last_pkt;
bc59ba39 768 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 769
770 if (po->stats.tp_drops)
771 status |= TP_STATUS_LOSING;
772
773 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
774 last_pkt->tp_next_offset = 0;
775
776 /* Get the ts of the last pkt */
777 if (BLOCK_NUM_PKTS(pbd1)) {
778 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
779 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
780 } else {
781 /* Ok, we tmo'd - so get the current time */
782 struct timespec ts;
783 getnstimeofday(&ts);
784 h1->ts_last_pkt.ts_sec = ts.tv_sec;
785 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
786 }
787
788 smp_wmb();
789
790 /* Flush the block */
791 prb_flush_block(pkc1, pbd1, status);
792
793 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
794}
795
bc59ba39 796static inline void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 797{
798 pkc->reset_pending_on_curr_blk = 0;
799}
800
801/*
802 * Side effect of opening a block:
803 *
804 * 1) prb_queue is thawed.
805 * 2) retire_blk_timer is refreshed.
806 *
807 */
bc59ba39 808static void prb_open_block(struct tpacket_kbdq_core *pkc1,
809 struct tpacket_block_desc *pbd1)
f6fb8f10 810{
811 struct timespec ts;
bc59ba39 812 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 813
814 smp_rmb();
815
816 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd1))) {
817
818 /* We could have just memset this but we will lose the
819 * flexibility of making the priv area sticky
820 */
821 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
822 BLOCK_NUM_PKTS(pbd1) = 0;
823 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
824 getnstimeofday(&ts);
825 h1->ts_first_pkt.ts_sec = ts.tv_sec;
826 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
827 pkc1->pkblk_start = (char *)pbd1;
828 pkc1->nxt_offset = (char *)(pkc1->pkblk_start +
829 BLK_PLUS_PRIV(pkc1->blk_sizeof_priv));
830 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
831 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
832 pbd1->version = pkc1->version;
833 pkc1->prev = pkc1->nxt_offset;
834 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
835 prb_thaw_queue(pkc1);
836 _prb_refresh_rx_retire_blk_timer(pkc1);
837
838 smp_wmb();
839
840 return;
841 }
842
843 WARN(1, "ERROR block:%p is NOT FREE status:%d kactive_blk_num:%d\n",
844 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num);
845 dump_stack();
846 BUG();
847}
848
849/*
850 * Queue freeze logic:
851 * 1) Assume tp_block_nr = 8 blocks.
852 * 2) At time 't0', user opens Rx ring.
853 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
854 * 4) user-space is either sleeping or processing block '0'.
855 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
856 * it will close block-7,loop around and try to fill block '0'.
857 * call-flow:
858 * __packet_lookup_frame_in_block
859 * prb_retire_current_block()
860 * prb_dispatch_next_block()
861 * |->(BLOCK_STATUS == USER) evaluates to true
862 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
863 * 6) Now there are two cases:
864 * 6.1) Link goes idle right after the queue is frozen.
865 * But remember, the last open_block() refreshed the timer.
866 * When this timer expires,it will refresh itself so that we can
867 * re-open block-0 in near future.
868 * 6.2) Link is busy and keeps on receiving packets. This is a simple
869 * case and __packet_lookup_frame_in_block will check if block-0
870 * is free and can now be re-used.
871 */
bc59ba39 872static inline void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 873 struct packet_sock *po)
874{
875 pkc->reset_pending_on_curr_blk = 1;
876 po->stats_u.stats3.tp_freeze_q_cnt++;
877}
878
879#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
880
881/*
882 * If the next block is free then we will dispatch it
883 * and return a good offset.
884 * Else, we will freeze the queue.
885 * So, caller must check the return value.
886 */
bc59ba39 887static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 888 struct packet_sock *po)
889{
bc59ba39 890 struct tpacket_block_desc *pbd;
f6fb8f10 891
892 smp_rmb();
893
894 /* 1. Get current block num */
895 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
896
897 /* 2. If this block is currently in_use then freeze the queue */
898 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
899 prb_freeze_queue(pkc, po);
900 return NULL;
901 }
902
903 /*
904 * 3.
905 * open this block and return the offset where the first packet
906 * needs to get stored.
907 */
908 prb_open_block(pkc, pbd);
909 return (void *)pkc->nxt_offset;
910}
911
bc59ba39 912static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 913 struct packet_sock *po, unsigned int status)
914{
bc59ba39 915 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 916
917 /* retire/close the current block */
918 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
919 /*
920 * Plug the case where copy_bits() is in progress on
921 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
922 * have space to copy the pkt in the current block and
923 * called prb_retire_current_block()
924 *
925 * We don't need to worry about the TMO case because
926 * the timer-handler already handled this case.
927 */
928 if (!(status & TP_STATUS_BLK_TMO)) {
929 while (atomic_read(&pkc->blk_fill_in_prog)) {
930 /* Waiting for skb_copy_bits to finish... */
931 cpu_relax();
932 }
933 }
934 prb_close_block(pkc, pbd, po, status);
935 return;
936 }
937
938 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd);
939 dump_stack();
940 BUG();
941}
942
bc59ba39 943static inline int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
944 struct tpacket_block_desc *pbd)
f6fb8f10 945{
946 return TP_STATUS_USER & BLOCK_STATUS(pbd);
947}
948
bc59ba39 949static inline int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 950{
951 return pkc->reset_pending_on_curr_blk;
952}
953
954static inline void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
955{
bc59ba39 956 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 957 atomic_dec(&pkc->blk_fill_in_prog);
958}
959
bc59ba39 960static inline void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 961 struct tpacket3_hdr *ppd)
962{
963 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
964}
965
bc59ba39 966static inline void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 967 struct tpacket3_hdr *ppd)
968{
969 ppd->hv1.tp_rxhash = 0;
970}
971
bc59ba39 972static inline void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 973 struct tpacket3_hdr *ppd)
974{
975 if (vlan_tx_tag_present(pkc->skb)) {
976 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
977 ppd->tp_status = TP_STATUS_VLAN_VALID;
978 } else {
979 ppd->hv1.tp_vlan_tci = ppd->tp_status = 0;
980 }
981}
982
bc59ba39 983static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 984 struct tpacket3_hdr *ppd)
985{
986 prb_fill_vlan_info(pkc, ppd);
987
988 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
989 prb_fill_rxhash(pkc, ppd);
990 else
991 prb_clear_rxhash(pkc, ppd);
992}
993
bc59ba39 994static inline void prb_fill_curr_block(char *curr,
995 struct tpacket_kbdq_core *pkc,
996 struct tpacket_block_desc *pbd,
f6fb8f10 997 unsigned int len)
998{
999 struct tpacket3_hdr *ppd;
1000
1001 ppd = (struct tpacket3_hdr *)curr;
1002 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1003 pkc->prev = curr;
1004 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1005 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1006 BLOCK_NUM_PKTS(pbd) += 1;
1007 atomic_inc(&pkc->blk_fill_in_prog);
1008 prb_run_all_ft_ops(pkc, ppd);
1009}
1010
1011/* Assumes caller has the sk->rx_queue.lock */
1012static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1013 struct sk_buff *skb,
1014 int status,
1015 unsigned int len
1016 )
1017{
bc59ba39 1018 struct tpacket_kbdq_core *pkc;
1019 struct tpacket_block_desc *pbd;
f6fb8f10 1020 char *curr, *end;
1021
1022 pkc = GET_PBDQC_FROM_RB(((struct packet_ring_buffer *)&po->rx_ring));
1023 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1024
1025 /* Queue is frozen when user space is lagging behind */
1026 if (prb_queue_frozen(pkc)) {
1027 /*
1028 * Check if that last block which caused the queue to freeze,
1029 * is still in_use by user-space.
1030 */
1031 if (prb_curr_blk_in_use(pkc, pbd)) {
1032 /* Can't record this packet */
1033 return NULL;
1034 } else {
1035 /*
1036 * Ok, the block was released by user-space.
1037 * Now let's open that block.
1038 * opening a block also thaws the queue.
1039 * Thawing is a side effect.
1040 */
1041 prb_open_block(pkc, pbd);
1042 }
1043 }
1044
1045 smp_mb();
1046 curr = pkc->nxt_offset;
1047 pkc->skb = skb;
1048 end = (char *) ((char *)pbd + pkc->kblk_size);
1049
1050 /* first try the current block */
1051 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1052 prb_fill_curr_block(curr, pkc, pbd, len);
1053 return (void *)curr;
1054 }
1055
1056 /* Ok, close the current block */
1057 prb_retire_current_block(pkc, po, 0);
1058
1059 /* Now, try to dispatch the next block */
1060 curr = (char *)prb_dispatch_next_block(pkc, po);
1061 if (curr) {
1062 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1063 prb_fill_curr_block(curr, pkc, pbd, len);
1064 return (void *)curr;
1065 }
1066
1067 /*
1068 * No free blocks are available.user_space hasn't caught up yet.
1069 * Queue was just frozen and now this packet will get dropped.
1070 */
1071 return NULL;
1072}
1073
1074static inline void *packet_current_rx_frame(struct packet_sock *po,
1075 struct sk_buff *skb,
1076 int status, unsigned int len)
1077{
1078 char *curr = NULL;
1079 switch (po->tp_version) {
1080 case TPACKET_V1:
1081 case TPACKET_V2:
1082 curr = packet_lookup_frame(po, &po->rx_ring,
1083 po->rx_ring.head, status);
1084 return curr;
1085 case TPACKET_V3:
1086 return __packet_lookup_frame_in_block(po, skb, status, len);
1087 default:
1088 WARN(1, "TPACKET version not supported\n");
1089 BUG();
1090 return 0;
1091 }
1092}
1093
1094static inline void *prb_lookup_block(struct packet_sock *po,
1095 struct packet_ring_buffer *rb,
1096 unsigned int previous,
1097 int status)
1098{
bc59ba39 1099 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1100 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, previous);
f6fb8f10 1101
1102 if (status != BLOCK_STATUS(pbd))
1103 return NULL;
1104 return pbd;
1105}
1106
1107static inline int prb_previous_blk_num(struct packet_ring_buffer *rb)
1108{
1109 unsigned int prev;
1110 if (rb->prb_bdqc.kactive_blk_num)
1111 prev = rb->prb_bdqc.kactive_blk_num-1;
1112 else
1113 prev = rb->prb_bdqc.knum_blocks-1;
1114 return prev;
1115}
1116
1117/* Assumes caller has held the rx_queue.lock */
1118static inline void *__prb_previous_block(struct packet_sock *po,
1119 struct packet_ring_buffer *rb,
1120 int status)
1121{
1122 unsigned int previous = prb_previous_blk_num(rb);
1123 return prb_lookup_block(po, rb, previous, status);
1124}
1125
1126static inline void *packet_previous_rx_frame(struct packet_sock *po,
1127 struct packet_ring_buffer *rb,
1128 int status)
1129{
1130 if (po->tp_version <= TPACKET_V2)
1131 return packet_previous_frame(po, rb, status);
1132
1133 return __prb_previous_block(po, rb, status);
1134}
1135
1136static inline void packet_increment_rx_head(struct packet_sock *po,
1137 struct packet_ring_buffer *rb)
1138{
1139 switch (po->tp_version) {
1140 case TPACKET_V1:
1141 case TPACKET_V2:
1142 return packet_increment_head(rb);
1143 case TPACKET_V3:
1144 default:
1145 WARN(1, "TPACKET version not supported.\n");
1146 BUG();
1147 return;
1148 }
1149}
1150
69e3c75f
JB		 1151static inline void *packet_previous_frame(struct packet_sock *po,
1152 struct packet_ring_buffer *rb,
1153 int status)
1154{
1155 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1156 return packet_lookup_frame(po, rb, previous, status);
1157}
1158
1159static inline void packet_increment_head(struct packet_ring_buffer *buff)
1160{
1161 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1162}
1163
1da177e4
LT		 1164static void packet_sock_destruct(struct sock *sk)
1165{
ed85b565
RC		 1166 skb_queue_purge(&sk->sk_error_queue);
1167
547b792c
IJ		 1168 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1169 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1170
1171 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1172 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1173 return;
1174 }
1175
17ab56a2 1176 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1177}
1178
dc99f600
DM		 1179static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1180{
1181 int x = atomic_read(&f->rr_cur) + 1;
1182
1183 if (x >= num)
1184 x = 0;
1185
1186 return x;
1187}
1188
1189static struct sock *fanout_demux_hash(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1190{
1191 u32 idx, hash = skb->rxhash;
1192
1193 idx = ((u64)hash * num) >> 32;
1194
1195 return f->arr[idx];
1196}
1197
1198static struct sock *fanout_demux_lb(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1199{
1200 int cur, old;
1201
1202 cur = atomic_read(&f->rr_cur);
1203 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1204 fanout_rr_next(f, num))) != cur)
1205 cur = old;
1206 return f->arr[cur];
1207}
1208
95ec3eb4
DM		 1209static struct sock *fanout_demux_cpu(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1210{
1211 unsigned int cpu = smp_processor_id();
1212
1213 return f->arr[cpu % num];
1214}
1215
7736d33f
DM		 1216static struct sk_buff *fanout_check_defrag(struct sk_buff *skb)
1217{
31817df0 1218#ifdef CONFIG_INET
7736d33f
DM		 1219 const struct iphdr *iph;
1220 u32 len;
1221
1222 if (skb->protocol != htons(ETH_P_IP))
1223 return skb;
1224
1225 if (!pskb_may_pull(skb, sizeof(struct iphdr)))
1226 return skb;
1227
1228 iph = ip_hdr(skb);
1229 if (iph->ihl < 5 || iph->version != 4)
1230 return skb;
1231 if (!pskb_may_pull(skb, iph->ihl*4))
1232 return skb;
1233 iph = ip_hdr(skb);
1234 len = ntohs(iph->tot_len);
1235 if (skb->len < len || len < (iph->ihl * 4))
1236 return skb;
1237
1238 if (ip_is_fragment(ip_hdr(skb))) {
aec27311 1239 skb = skb_share_check(skb, GFP_ATOMIC);
7736d33f
DM		 1240 if (skb) {
1241 if (pskb_trim_rcsum(skb, len))
1242 return skb;
1243 memset(IPCB(skb), 0, sizeof(struct inet_skb_parm));
1244 if (ip_defrag(skb, IP_DEFRAG_AF_PACKET))
1245 return NULL;
1246 skb->rxhash = 0;
1247 }
1248 }
31817df0 1249#endif
7736d33f
DM		 1250 return skb;
1251}
1252
95ec3eb4
DM		 1253static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1254 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1255{
1256 struct packet_fanout *f = pt->af_packet_priv;
1257 unsigned int num = f->num_members;
1258 struct packet_sock *po;
1259 struct sock *sk;
1260
1261 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1262 !num) {
1263 kfree_skb(skb);
1264 return 0;
1265 }
1266
95ec3eb4
DM		 1267 switch (f->type) {
1268 case PACKET_FANOUT_HASH:
1269 default:
1270 if (f->defrag) {
1271 skb = fanout_check_defrag(skb);
1272 if (!skb)
1273 return 0;
1274 }
1275 skb_get_rxhash(skb);
1276 sk = fanout_demux_hash(f, skb, num);
1277 break;
1278 case PACKET_FANOUT_LB:
1279 sk = fanout_demux_lb(f, skb, num);
1280 break;
1281 case PACKET_FANOUT_CPU:
1282 sk = fanout_demux_cpu(f, skb, num);
1283 break;
dc99f600
DM		 1284 }
1285
dc99f600
DM		 1286 po = pkt_sk(sk);
1287
1288 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1289}
1290
1291static DEFINE_MUTEX(fanout_mutex);
1292static LIST_HEAD(fanout_list);
1293
1294static void __fanout_link(struct sock *sk, struct packet_sock *po)
1295{
1296 struct packet_fanout *f = po->fanout;
1297
1298 spin_lock(&f->lock);
1299 f->arr[f->num_members] = sk;
1300 smp_wmb();
1301 f->num_members++;
1302 spin_unlock(&f->lock);
1303}
1304
1305static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1306{
1307 struct packet_fanout *f = po->fanout;
1308 int i;
1309
1310 spin_lock(&f->lock);
1311 for (i = 0; i < f->num_members; i++) {
1312 if (f->arr[i] == sk)
1313 break;
1314 }
1315 BUG_ON(i >= f->num_members);
1316 f->arr[i] = f->arr[f->num_members - 1];
1317 f->num_members--;
1318 spin_unlock(&f->lock);
1319}
1320
7736d33f 1321static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1322{
1323 struct packet_sock *po = pkt_sk(sk);
1324 struct packet_fanout *f, *match;
7736d33f
DM		 1325 u8 type = type_flags & 0xff;
1326 u8 defrag = (type_flags & PACKET_FANOUT_FLAG_DEFRAG) ? 1 : 0;
dc99f600
DM		 1327 int err;
1328
1329 switch (type) {
1330 case PACKET_FANOUT_HASH:
1331 case PACKET_FANOUT_LB:
95ec3eb4 1332 case PACKET_FANOUT_CPU:
dc99f600
DM		 1333 break;
1334 default:
1335 return -EINVAL;
1336 }
1337
1338 if (!po->running)
1339 return -EINVAL;
1340
1341 if (po->fanout)
1342 return -EALREADY;
1343
1344 mutex_lock(&fanout_mutex);
1345 match = NULL;
1346 list_for_each_entry(f, &fanout_list, list) {
1347 if (f->id == id &&
1348 read_pnet(&f->net) == sock_net(sk)) {
1349 match = f;
1350 break;
1351 }
1352 }
afe62c68 1353 err = -EINVAL;
7736d33f 1354 if (match && match->defrag != defrag)
afe62c68 1355 goto out;
dc99f600 1356 if (!match) {
afe62c68 1357 err = -ENOMEM;
dc99f600 1358 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1359 if (!match)
1360 goto out;
1361 write_pnet(&match->net, sock_net(sk));
1362 match->id = id;
1363 match->type = type;
1364 match->defrag = defrag;
1365 atomic_set(&match->rr_cur, 0);
1366 INIT_LIST_HEAD(&match->list);
1367 spin_lock_init(&match->lock);
1368 atomic_set(&match->sk_ref, 0);
1369 match->prot_hook.type = po->prot_hook.type;
1370 match->prot_hook.dev = po->prot_hook.dev;
1371 match->prot_hook.func = packet_rcv_fanout;
1372 match->prot_hook.af_packet_priv = match;
1373 dev_add_pack(&match->prot_hook);
1374 list_add(&match->list, &fanout_list);
dc99f600 1375 }
afe62c68
ED		 1376 err = -EINVAL;
1377 if (match->type == type &&
1378 match->prot_hook.type == po->prot_hook.type &&
1379 match->prot_hook.dev == po->prot_hook.dev) {
1380 err = -ENOSPC;
1381 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1382 __dev_remove_pack(&po->prot_hook);
1383 po->fanout = match;
1384 atomic_inc(&match->sk_ref);
1385 __fanout_link(sk, po);
1386 err = 0;
dc99f600
DM		 1387 }
1388 }
afe62c68 1389out:
dc99f600
DM		 1390 mutex_unlock(&fanout_mutex);
1391 return err;
1392}
1393
1394static void fanout_release(struct sock *sk)
1395{
1396 struct packet_sock *po = pkt_sk(sk);
1397 struct packet_fanout *f;
1398
1399 f = po->fanout;
1400 if (!f)
1401 return;
1402
1403 po->fanout = NULL;
1404
1405 mutex_lock(&fanout_mutex);
1406 if (atomic_dec_and_test(&f->sk_ref)) {
1407 list_del(&f->list);
1408 dev_remove_pack(&f->prot_hook);
1409 kfree(f);
1410 }
1411 mutex_unlock(&fanout_mutex);
1412}
1da177e4 1413
90ddc4f0 1414static const struct proto_ops packet_ops;
1da177e4 1415
90ddc4f0 1416static const struct proto_ops packet_ops_spkt;
1da177e4 1417
40d4e3df
ED		 1418static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1419 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1420{
1421 struct sock *sk;
1422 struct sockaddr_pkt *spkt;
1423
1424 /*
1425 * When we registered the protocol we saved the socket in the data
1426 * field for just this event.
1427 */
1428
1429 sk = pt->af_packet_priv;
1ce4f28b 1430
1da177e4
LT		 1431 /*
1432 * Yank back the headers [hope the device set this
1433 * right or kerboom...]
1434 *
1435 * Incoming packets have ll header pulled,
1436 * push it back.
1437 *
98e399f8 1438 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1439 * so that this procedure is noop.
1440 */
1441
1442 if (skb->pkt_type == PACKET_LOOPBACK)
1443 goto out;
1444
09ad9bc7 1445 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1446 goto out;
1447
40d4e3df
ED		 1448 skb = skb_share_check(skb, GFP_ATOMIC);
1449 if (skb == NULL)
1da177e4
LT		 1450 goto oom;
1451
1452 /* drop any routing info */
adf30907 1453 skb_dst_drop(skb);
1da177e4 1454
84531c24
PO		 1455 /* drop conntrack reference */
1456 nf_reset(skb);
1457
ffbc6111 1458 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1459
98e399f8 1460 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1461
1462 /*
1463 * The SOCK_PACKET socket receives _all_ frames.
1464 */
1465
1466 spkt->spkt_family = dev->type;
1467 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1468 spkt->spkt_protocol = skb->protocol;
1469
1470 /*
1471 * Charge the memory to the socket. This is done specifically
1472 * to prevent sockets using all the memory up.
1473 */
1474
40d4e3df 1475 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1476 return 0;
1477
1478out:
1479 kfree_skb(skb);
1480oom:
1481 return 0;
1482}
1483
1484
1485/*
1486 * Output a raw packet to a device layer. This bypasses all the other
1487 * protocol layers and you must therefore supply it with a complete frame
1488 */
1ce4f28b 1489
1da177e4
LT		 1490static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1491 struct msghdr *msg, size_t len)
1492{
1493 struct sock *sk = sock->sk;
40d4e3df 1494 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1495 struct sk_buff *skb = NULL;
1da177e4 1496 struct net_device *dev;
40d4e3df 1497 __be16 proto = 0;
1da177e4 1498 int err;
1ce4f28b 1499
1da177e4 1500 /*
1ce4f28b 1501 * Get and verify the address.
1da177e4
LT		 1502 */
1503
40d4e3df 1504 if (saddr) {
1da177e4 1505 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1506 return -EINVAL;
1507 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1508 proto = saddr->spkt_protocol;
1509 } else
1510 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1511
1512 /*
1ce4f28b 1513 * Find the device first to size check it
1da177e4
LT		 1514 */
1515
1516 saddr->spkt_device[13] = 0;
1a35ca80 1517retry:
654d1f8a
ED		 1518 rcu_read_lock();
1519 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1520 err = -ENODEV;
1521 if (dev == NULL)
1522 goto out_unlock;
1ce4f28b 1523
d5e76b0a
DM		 1524 err = -ENETDOWN;
1525 if (!(dev->flags & IFF_UP))
1526 goto out_unlock;
1527
1da177e4 1528 /*
40d4e3df
ED		 1529 * You may not queue a frame bigger than the mtu. This is the lowest level
1530 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1531 */
1ce4f28b 1532
1da177e4 1533 err = -EMSGSIZE;
57f89bfa 1534 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN)
1da177e4
LT		 1535 goto out_unlock;
1536
1a35ca80
ED		 1537 if (!skb) {
1538 size_t reserved = LL_RESERVED_SPACE(dev);
1539 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1540
1541 rcu_read_unlock();
1542 skb = sock_wmalloc(sk, len + reserved, 0, GFP_KERNEL);
1543 if (skb == NULL)
1544 return -ENOBUFS;
1545 /* FIXME: Save some space for broken drivers that write a hard
1546 * header at transmission time by themselves. PPP is the notable
1547 * one here. This should really be fixed at the driver level.
1548 */
1549 skb_reserve(skb, reserved);
1550 skb_reset_network_header(skb);
1551
1552 /* Try to align data part correctly */
1553 if (hhlen) {
1554 skb->data -= hhlen;
1555 skb->tail -= hhlen;
1556 if (len < hhlen)
1557 skb_reset_network_header(skb);
1558 }
1559 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1560 if (err)
1561 goto out_free;
1562 goto retry;
1da177e4
LT		 1563 }
1564
57f89bfa
BG		 1565 if (len > (dev->mtu + dev->hard_header_len)) {
1566 /* Earlier code assumed this would be a VLAN pkt,
1567 * double-check this now that we have the actual
1568 * packet in hand.
1569 */
1570 struct ethhdr *ehdr;
1571 skb_reset_mac_header(skb);
1572 ehdr = eth_hdr(skb);
1573 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1574 err = -EMSGSIZE;
1575 goto out_unlock;
1576 }
1577 }
1a35ca80 1578
1da177e4
LT		 1579 skb->protocol = proto;
1580 skb->dev = dev;
1581 skb->priority = sk->sk_priority;
2d37a186 1582 skb->mark = sk->sk_mark;
2244d07b 1583 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 1584 if (err < 0)
1585 goto out_unlock;
1da177e4
LT		 1586
1587 dev_queue_xmit(skb);
654d1f8a 1588 rcu_read_unlock();
40d4e3df 1589 return len;
1da177e4 1590
1da177e4 1591out_unlock:
654d1f8a 1592 rcu_read_unlock();
1a35ca80
ED		 1593out_free:
1594 kfree_skb(skb);
1da177e4
LT		 1595 return err;
1596}
1da177e4 1597
62ab0812
ED		 1598static inline unsigned int run_filter(const struct sk_buff *skb,
1599 const struct sock *sk,
dbcb5855 1600 unsigned int res)
1da177e4
LT		 1601{
1602 struct sk_filter *filter;
fda9ef5d 1603
80f8f102
ED		 1604 rcu_read_lock();
1605 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1606 if (filter != NULL)
0a14842f 1607 res = SK_RUN_FILTER(filter, skb);
80f8f102 1608 rcu_read_unlock();
1da177e4 1609
dbcb5855 1610 return res;
1da177e4
LT		 1611}
1612
1613/*
62ab0812
ED		 1614 * This function makes lazy skb cloning in hope that most of packets
1615 * are discarded by BPF.
1616 *
1617 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1618 * and skb->cb are mangled. It works because (and until) packets
1619 * falling here are owned by current CPU. Output packets are cloned
1620 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1621 * sequencially, so that if we return skb to original state on exit,
1622 * we will not harm anyone.
1da177e4
LT		 1623 */
1624
40d4e3df
ED		 1625static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1626 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1627{
1628 struct sock *sk;
1629 struct sockaddr_ll *sll;
1630 struct packet_sock *po;
40d4e3df 1631 u8 *skb_head = skb->data;
1da177e4 1632 int skb_len = skb->len;
dbcb5855 1633 unsigned int snaplen, res;
1da177e4
LT		 1634
1635 if (skb->pkt_type == PACKET_LOOPBACK)
1636 goto drop;
1637
1638 sk = pt->af_packet_priv;
1639 po = pkt_sk(sk);
1640
09ad9bc7 1641 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1642 goto drop;
1643
1da177e4
LT		 1644 skb->dev = dev;
1645
3b04ddde 1646 if (dev->header_ops) {
1da177e4 1647 /* The device has an explicit notion of ll header,
62ab0812
ED		 1648 * exported to higher levels.
1649 *
1650 * Otherwise, the device hides details of its frame
1651 * structure, so that corresponding packet head is
1652 * never delivered to user.
1da177e4
LT		 1653 */
1654 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1655 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1656 else if (skb->pkt_type == PACKET_OUTGOING) {
1657 /* Special case: outgoing packets have ll header at head */
bbe735e4 1658 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1659 }
1660 }
1661
1662 snaplen = skb->len;
1663
dbcb5855
DM		 1664 res = run_filter(skb, sk, snaplen);
1665 if (!res)
fda9ef5d 1666 goto drop_n_restore;
dbcb5855
DM		 1667 if (snaplen > res)
1668 snaplen = res;
1da177e4
LT		 1669
1670 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
1671 (unsigned)sk->sk_rcvbuf)
1672 goto drop_n_acct;
1673
1674 if (skb_shared(skb)) {
1675 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1676 if (nskb == NULL)
1677 goto drop_n_acct;
1678
1679 if (skb_head != skb->data) {
1680 skb->data = skb_head;
1681 skb->len = skb_len;
1682 }
1683 kfree_skb(skb);
1684 skb = nskb;
1685 }
1686
ffbc6111
HX		 1687 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1688 sizeof(skb->cb));
1689
1690 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1691 sll->sll_family = AF_PACKET;
1692 sll->sll_hatype = dev->type;
1693 sll->sll_protocol = skb->protocol;
1694 sll->sll_pkttype = skb->pkt_type;
8032b464 1695 if (unlikely(po->origdev))
80feaacb
PWJ		 1696 sll->sll_ifindex = orig_dev->ifindex;
1697 else
1698 sll->sll_ifindex = dev->ifindex;
1da177e4 1699
b95cce35 1700 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1701
ffbc6111 1702 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1703
1da177e4
LT		 1704 if (pskb_trim(skb, snaplen))
1705 goto drop_n_acct;
1706
1707 skb_set_owner_r(skb, sk);
1708 skb->dev = NULL;
adf30907 1709 skb_dst_drop(skb);
1da177e4 1710
84531c24
PO		 1711 /* drop conntrack reference */
1712 nf_reset(skb);
1713
1da177e4
LT		 1714 spin_lock(&sk->sk_receive_queue.lock);
1715 po->stats.tp_packets++;
3b885787 1716 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1717 __skb_queue_tail(&sk->sk_receive_queue, skb);
1718 spin_unlock(&sk->sk_receive_queue.lock);
1719 sk->sk_data_ready(sk, skb->len);
1720 return 0;
1721
1722drop_n_acct:
7091fbd8
WB		 1723 spin_lock(&sk->sk_receive_queue.lock);
1724 po->stats.tp_drops++;
1725 atomic_inc(&sk->sk_drops);
1726 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1727
1728drop_n_restore:
1729 if (skb_head != skb->data && skb_shared(skb)) {
1730 skb->data = skb_head;
1731 skb->len = skb_len;
1732 }
1733drop:
ead2ceb0 1734 consume_skb(skb);
1da177e4
LT		 1735 return 0;
1736}
1737
40d4e3df
ED		 1738static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1739 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1740{
1741 struct sock *sk;
1742 struct packet_sock *po;
1743 struct sockaddr_ll *sll;
bbd6ef87
PM		 1744 union {
1745 struct tpacket_hdr *h1;
1746 struct tpacket2_hdr *h2;
f6fb8f10 1747 struct tpacket3_hdr *h3;
bbd6ef87
PM		 1748 void *raw;
1749 } h;
40d4e3df 1750 u8 *skb_head = skb->data;
1da177e4 1751 int skb_len = skb->len;
dbcb5855 1752 unsigned int snaplen, res;
f6fb8f10 1753 unsigned long status = TP_STATUS_USER;
bbd6ef87 1754 unsigned short macoff, netoff, hdrlen;
1da177e4 1755 struct sk_buff *copy_skb = NULL;
b7aa0bf7 1756 struct timeval tv;
bbd6ef87 1757 struct timespec ts;
614f60fa 1758 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1da177e4
LT		 1759
1760 if (skb->pkt_type == PACKET_LOOPBACK)
1761 goto drop;
1762
1763 sk = pt->af_packet_priv;
1764 po = pkt_sk(sk);
1765
09ad9bc7 1766 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1767 goto drop;
1768
3b04ddde 1769 if (dev->header_ops) {
1da177e4 1770 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1771 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1772 else if (skb->pkt_type == PACKET_OUTGOING) {
1773 /* Special case: outgoing packets have ll header at head */
bbe735e4 1774 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1775 }
1776 }
1777
8dc41944
HX		 1778 if (skb->ip_summed == CHECKSUM_PARTIAL)
1779 status |= TP_STATUS_CSUMNOTREADY;
1780
1da177e4
LT		 1781 snaplen = skb->len;
1782
dbcb5855
DM		 1783 res = run_filter(skb, sk, snaplen);
1784 if (!res)
fda9ef5d 1785 goto drop_n_restore;
dbcb5855
DM		 1786 if (snaplen > res)
1787 snaplen = res;
1da177e4
LT		 1788
1789 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1790 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1791 po->tp_reserve;
1da177e4 1792 } else {
bbe735e4 1793 unsigned maclen = skb_network_offset(skb);
bbd6ef87 1794 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1795 (maclen < 16 ? 16 : maclen)) +
1796 po->tp_reserve;
1da177e4
LT		 1797 macoff = netoff - maclen;
1798 }
f6fb8f10 1799 if (po->tp_version <= TPACKET_V2) {
1800 if (macoff + snaplen > po->rx_ring.frame_size) {
1801 if (po->copy_thresh &&
1802 atomic_read(&sk->sk_rmem_alloc) + skb->truesize
1803 < (unsigned)sk->sk_rcvbuf) {
1804 if (skb_shared(skb)) {
1805 copy_skb = skb_clone(skb, GFP_ATOMIC);
1806 } else {
1807 copy_skb = skb_get(skb);
1808 skb_head = skb->data;
1809 }
1810 if (copy_skb)
1811 skb_set_owner_r(copy_skb, sk);
1da177e4 1812 }
f6fb8f10 1813 snaplen = po->rx_ring.frame_size - macoff;
1814 if ((int)snaplen < 0)
1815 snaplen = 0;
1da177e4 1816 }
1da177e4 1817 }
1da177e4 1818 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1819 h.raw = packet_current_rx_frame(po, skb,
1820 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1821 if (!h.raw)
1da177e4 1822 goto ring_is_full;
f6fb8f10 1823 if (po->tp_version <= TPACKET_V2) {
1824 packet_increment_rx_head(po, &po->rx_ring);
1825 /*
1826 * LOSING will be reported till you read the stats,
1827 * because it's COR - Clear On Read.
1828 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1829 * at packet level.
1830 */
1831 if (po->stats.tp_drops)
1832 status |= TP_STATUS_LOSING;
1833 }
1da177e4
LT		 1834 po->stats.tp_packets++;
1835 if (copy_skb) {
1836 status |= TP_STATUS_COPY;
1837 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1838 }
1da177e4
LT		 1839 spin_unlock(&sk->sk_receive_queue.lock);
1840
bbd6ef87 1841 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1da177e4 1842
bbd6ef87
PM		 1843 switch (po->tp_version) {
1844 case TPACKET_V1:
1845 h.h1->tp_len = skb->len;
1846 h.h1->tp_snaplen = snaplen;
1847 h.h1->tp_mac = macoff;
1848 h.h1->tp_net = netoff;
614f60fa
SM		 1849 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1850 && shhwtstamps->syststamp.tv64)
1851 tv = ktime_to_timeval(shhwtstamps->syststamp);
1852 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1853 && shhwtstamps->hwtstamp.tv64)
1854 tv = ktime_to_timeval(shhwtstamps->hwtstamp);
1855 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1856 tv = ktime_to_timeval(skb->tstamp);
1857 else
1858 do_gettimeofday(&tv);
1859 h.h1->tp_sec = tv.tv_sec;
1860 h.h1->tp_usec = tv.tv_usec;
1861 hdrlen = sizeof(*h.h1);
1862 break;
1863 case TPACKET_V2:
1864 h.h2->tp_len = skb->len;
1865 h.h2->tp_snaplen = snaplen;
1866 h.h2->tp_mac = macoff;
1867 h.h2->tp_net = netoff;
614f60fa
SM		 1868 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1869 && shhwtstamps->syststamp.tv64)
1870 ts = ktime_to_timespec(shhwtstamps->syststamp);
1871 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1872 && shhwtstamps->hwtstamp.tv64)
1873 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1874 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1875 ts = ktime_to_timespec(skb->tstamp);
1876 else
1877 getnstimeofday(&ts);
1878 h.h2->tp_sec = ts.tv_sec;
1879 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1880 if (vlan_tx_tag_present(skb)) {
1881 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1882 status |= TP_STATUS_VLAN_VALID;
1883 } else {
1884 h.h2->tp_vlan_tci = 0;
1885 }
13fcb7bd 1886 h.h2->tp_padding = 0;
bbd6ef87
PM		 1887 hdrlen = sizeof(*h.h2);
1888 break;
f6fb8f10 1889 case TPACKET_V3:
1890 /* tp_nxt_offset,vlan are already populated above.
1891 * So DONT clear those fields here
1892 */
1893 h.h3->tp_status |= status;
1894 h.h3->tp_len = skb->len;
1895 h.h3->tp_snaplen = snaplen;
1896 h.h3->tp_mac = macoff;
1897 h.h3->tp_net = netoff;
1898 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1899 && shhwtstamps->syststamp.tv64)
1900 ts = ktime_to_timespec(shhwtstamps->syststamp);
1901 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1902 && shhwtstamps->hwtstamp.tv64)
1903 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1904 else if (skb->tstamp.tv64)
1905 ts = ktime_to_timespec(skb->tstamp);
1906 else
1907 getnstimeofday(&ts);
1908 h.h3->tp_sec = ts.tv_sec;
1909 h.h3->tp_nsec = ts.tv_nsec;
1910 hdrlen = sizeof(*h.h3);
1911 break;
bbd6ef87
PM		 1912 default:
1913 BUG();
1914 }
1da177e4 1915
bbd6ef87 1916 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1917 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1918 sll->sll_family = AF_PACKET;
1919 sll->sll_hatype = dev->type;
1920 sll->sll_protocol = skb->protocol;
1921 sll->sll_pkttype = skb->pkt_type;
8032b464 1922 if (unlikely(po->origdev))
80feaacb
PWJ		 1923 sll->sll_ifindex = orig_dev->ifindex;
1924 else
1925 sll->sll_ifindex = dev->ifindex;
1da177e4 1926
e16aa207 1927 smp_mb();
f6dafa95 1928#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1929 {
0af55bb5
CG		 1930 u8 *start, *end;
1931
f6fb8f10 1932 if (po->tp_version <= TPACKET_V2) {
1933 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1934 + macoff + snaplen);
1935 for (start = h.raw; start < end; start += PAGE_SIZE)
1936 flush_dcache_page(pgv_to_page(start));
1937 }
cc9f01b2 1938 smp_wmb();
1da177e4 1939 }
f6dafa95 1940#endif
f6fb8f10 1941 if (po->tp_version <= TPACKET_V2)
1942 __packet_set_status(po, h.raw, status);
1943 else
1944 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1945
1946 sk->sk_data_ready(sk, 0);
1947
1948drop_n_restore:
1949 if (skb_head != skb->data && skb_shared(skb)) {
1950 skb->data = skb_head;
1951 skb->len = skb_len;
1952 }
1953drop:
1ce4f28b 1954 kfree_skb(skb);
1da177e4
LT		 1955 return 0;
1956
1957ring_is_full:
1958 po->stats.tp_drops++;
1959 spin_unlock(&sk->sk_receive_queue.lock);
1960
1961 sk->sk_data_ready(sk, 0);
acb5d75b 1962 kfree_skb(copy_skb);
1da177e4
LT		 1963 goto drop_n_restore;
1964}
1965
69e3c75f
JB		 1966static void tpacket_destruct_skb(struct sk_buff *skb)
1967{
1968 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1969 void *ph;
1da177e4 1970
69e3c75f
JB		 1971 if (likely(po->tx_ring.pg_vec)) {
1972 ph = skb_shinfo(skb)->destructor_arg;
1973 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING);
1974 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1975 atomic_dec(&po->tx_ring.pending);
1976 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
1977 }
1978
1979 sock_wfree(skb);
1980}
1981
40d4e3df
ED		 1982static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1983 void *frame, struct net_device *dev, int size_max,
1984 __be16 proto, unsigned char *addr)
69e3c75f
JB		 1985{
1986 union {
1987 struct tpacket_hdr *h1;
1988 struct tpacket2_hdr *h2;
1989 void *raw;
1990 } ph;
1991 int to_write, offset, len, tp_len, nr_frags, len_max;
1992 struct socket *sock = po->sk.sk_socket;
1993 struct page *page;
1994 void *data;
1995 int err;
1996
1997 ph.raw = frame;
1998
1999 skb->protocol = proto;
2000 skb->dev = dev;
2001 skb->priority = po->sk.sk_priority;
2d37a186 2002 skb->mark = po->sk.sk_mark;
69e3c75f
JB		 2003 skb_shinfo(skb)->destructor_arg = ph.raw;
2004
2005 switch (po->tp_version) {
2006 case TPACKET_V2:
2007 tp_len = ph.h2->tp_len;
2008 break;
2009 default:
2010 tp_len = ph.h1->tp_len;
2011 break;
2012 }
2013 if (unlikely(tp_len > size_max)) {
40d4e3df 2014 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 2015 return -EMSGSIZE;
2016 }
2017
2018 skb_reserve(skb, LL_RESERVED_SPACE(dev));
2019 skb_reset_network_header(skb);
2020
2021 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2022 to_write = tp_len;
2023
2024 if (sock->type == SOCK_DGRAM) {
2025 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2026 NULL, tp_len);
2027 if (unlikely(err < 0))
2028 return -EINVAL;
40d4e3df 2029 } else if (dev->hard_header_len) {
69e3c75f
JB		 2030 /* net device doesn't like empty head */
2031 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2032 pr_err("packet size is too short (%d < %d)\n",
2033 tp_len, dev->hard_header_len);
69e3c75f
JB		 2034 return -EINVAL;
2035 }
2036
2037 skb_push(skb, dev->hard_header_len);
2038 err = skb_store_bits(skb, 0, data,
2039 dev->hard_header_len);
2040 if (unlikely(err))
2041 return err;
2042
2043 data += dev->hard_header_len;
2044 to_write -= dev->hard_header_len;
2045 }
2046
2047 err = -EFAULT;
69e3c75f
JB		 2048 offset = offset_in_page(data);
2049 len_max = PAGE_SIZE - offset;
2050 len = ((to_write > len_max) ? len_max : to_write);
2051
2052 skb->data_len = to_write;
2053 skb->len += to_write;
2054 skb->truesize += to_write;
2055 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2056
2057 while (likely(to_write)) {
2058 nr_frags = skb_shinfo(skb)->nr_frags;
2059
2060 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2061 pr_err("Packet exceed the number of skb frags(%lu)\n",
2062 MAX_SKB_FRAGS);
69e3c75f
JB		 2063 return -EFAULT;
2064 }
2065
0af55bb5
CG		 2066 page = pgv_to_page(data);
2067 data += len;
69e3c75f
JB		 2068 flush_dcache_page(page);
2069 get_page(page);
0af55bb5 2070 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2071 to_write -= len;
2072 offset = 0;
2073 len_max = PAGE_SIZE;
2074 len = ((to_write > len_max) ? len_max : to_write);
2075 }
2076
2077 return tp_len;
2078}
2079
2080static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2081{
69e3c75f
JB		 2082 struct sk_buff *skb;
2083 struct net_device *dev;
2084 __be16 proto;
827d9780
BG		 2085 bool need_rls_dev = false;
2086 int err, reserve = 0;
40d4e3df
ED		 2087 void *ph;
2088 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2089 int tp_len, size_max;
2090 unsigned char *addr;
2091 int len_sum = 0;
2092 int status = 0;
2093
69e3c75f
JB		 2094 mutex_lock(&po->pg_vec_lock);
2095
2096 err = -EBUSY;
2097 if (saddr == NULL) {
827d9780 2098 dev = po->prot_hook.dev;
69e3c75f
JB		 2099 proto = po->num;
2100 addr = NULL;
2101 } else {
2102 err = -EINVAL;
2103 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2104 goto out;
2105 if (msg->msg_namelen < (saddr->sll_halen
2106 + offsetof(struct sockaddr_ll,
2107 sll_addr)))
2108 goto out;
69e3c75f
JB		 2109 proto = saddr->sll_protocol;
2110 addr = saddr->sll_addr;
827d9780
BG		 2111 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2112 need_rls_dev = true;
69e3c75f
JB		 2113 }
2114
69e3c75f
JB		 2115 err = -ENXIO;
2116 if (unlikely(dev == NULL))
2117 goto out;
2118
2119 reserve = dev->hard_header_len;
2120
2121 err = -ENETDOWN;
2122 if (unlikely(!(dev->flags & IFF_UP)))
2123 goto out_put;
2124
2125 size_max = po->tx_ring.frame_size
b5dd884e 2126 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2127
2128 if (size_max > dev->mtu + reserve)
2129 size_max = dev->mtu + reserve;
2130
2131 do {
2132 ph = packet_current_frame(po, &po->tx_ring,
2133 TP_STATUS_SEND_REQUEST);
2134
2135 if (unlikely(ph == NULL)) {
2136 schedule();
2137 continue;
2138 }
2139
2140 status = TP_STATUS_SEND_REQUEST;
2141 skb = sock_alloc_send_skb(&po->sk,
2142 LL_ALLOCATED_SPACE(dev)
2143 + sizeof(struct sockaddr_ll),
2144 0, &err);
2145
2146 if (unlikely(skb == NULL))
2147 goto out_status;
2148
2149 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2150 addr);
2151
2152 if (unlikely(tp_len < 0)) {
2153 if (po->tp_loss) {
2154 __packet_set_status(po, ph,
2155 TP_STATUS_AVAILABLE);
2156 packet_increment_head(&po->tx_ring);
2157 kfree_skb(skb);
2158 continue;
2159 } else {
2160 status = TP_STATUS_WRONG_FORMAT;
2161 err = tp_len;
2162 goto out_status;
2163 }
2164 }
2165
2166 skb->destructor = tpacket_destruct_skb;
2167 __packet_set_status(po, ph, TP_STATUS_SENDING);
2168 atomic_inc(&po->tx_ring.pending);
2169
2170 status = TP_STATUS_SEND_REQUEST;
2171 err = dev_queue_xmit(skb);
eb70df13
JP		 2172 if (unlikely(err > 0)) {
2173 err = net_xmit_errno(err);
2174 if (err && __packet_get_status(po, ph) ==
2175 TP_STATUS_AVAILABLE) {
2176 /* skb was destructed already */
2177 skb = NULL;
2178 goto out_status;
2179 }
2180 /*
2181 * skb was dropped but not destructed yet;
2182 * let's treat it like congestion or err < 0
2183 */
2184 err = 0;
2185 }
69e3c75f
JB		 2186 packet_increment_head(&po->tx_ring);
2187 len_sum += tp_len;
f64f9e71
JP		 2188 } while (likely((ph != NULL) ||
2189 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2190 (atomic_read(&po->tx_ring.pending))))
2191 );
69e3c75f
JB		 2192
2193 err = len_sum;
2194 goto out_put;
2195
69e3c75f
JB		 2196out_status:
2197 __packet_set_status(po, ph, status);
2198 kfree_skb(skb);
2199out_put:
827d9780
BG		 2200 if (need_rls_dev)
2201 dev_put(dev);
69e3c75f
JB		 2202out:
2203 mutex_unlock(&po->pg_vec_lock);
2204 return err;
2205}
69e3c75f 2206
bfd5f4a3
SS		 2207static inline struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2208 size_t reserve, size_t len,
2209 size_t linear, int noblock,
2210 int *err)
2211{
2212 struct sk_buff *skb;
2213
2214 /* Under a page? Don't bother with paged skb. */
2215 if (prepad + len < PAGE_SIZE || !linear)
2216 linear = len;
2217
2218 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2219 err);
2220 if (!skb)
2221 return NULL;
2222
2223 skb_reserve(skb, reserve);
2224 skb_put(skb, linear);
2225 skb->data_len = len - linear;
2226 skb->len += len - linear;
2227
2228 return skb;
2229}
2230
69e3c75f 2231static int packet_snd(struct socket *sock,
1da177e4
LT		 2232 struct msghdr *msg, size_t len)
2233{
2234 struct sock *sk = sock->sk;
40d4e3df 2235 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2236 struct sk_buff *skb;
2237 struct net_device *dev;
0e11c91e 2238 __be16 proto;
827d9780 2239 bool need_rls_dev = false;
1da177e4 2240 unsigned char *addr;
827d9780 2241 int err, reserve = 0;
bfd5f4a3
SS		 2242 struct virtio_net_hdr vnet_hdr = { 0 };
2243 int offset = 0;
2244 int vnet_hdr_len;
2245 struct packet_sock *po = pkt_sk(sk);
2246 unsigned short gso_type = 0;
1da177e4
LT		 2247
2248 /*
1ce4f28b 2249 * Get and verify the address.
1da177e4 2250 */
1ce4f28b 2251
1da177e4 2252 if (saddr == NULL) {
827d9780 2253 dev = po->prot_hook.dev;
1da177e4
LT		 2254 proto = po->num;
2255 addr = NULL;
2256 } else {
2257 err = -EINVAL;
2258 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2259 goto out;
0fb375fb
EB		 2260 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2261 goto out;
1da177e4
LT		 2262 proto = saddr->sll_protocol;
2263 addr = saddr->sll_addr;
827d9780
BG		 2264 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2265 need_rls_dev = true;
1da177e4
LT		 2266 }
2267
1da177e4
LT		 2268 err = -ENXIO;
2269 if (dev == NULL)
2270 goto out_unlock;
2271 if (sock->type == SOCK_RAW)
2272 reserve = dev->hard_header_len;
2273
d5e76b0a
DM		 2274 err = -ENETDOWN;
2275 if (!(dev->flags & IFF_UP))
2276 goto out_unlock;
2277
bfd5f4a3
SS		 2278 if (po->has_vnet_hdr) {
2279 vnet_hdr_len = sizeof(vnet_hdr);
2280
2281 err = -EINVAL;
2282 if (len < vnet_hdr_len)
2283 goto out_unlock;
2284
2285 len -= vnet_hdr_len;
2286
2287 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2288 vnet_hdr_len);
2289 if (err < 0)
2290 goto out_unlock;
2291
2292 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2293 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2294 vnet_hdr.hdr_len))
2295 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2296 vnet_hdr.csum_offset + 2;
2297
2298 err = -EINVAL;
2299 if (vnet_hdr.hdr_len > len)
2300 goto out_unlock;
2301
2302 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2303 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2304 case VIRTIO_NET_HDR_GSO_TCPV4:
2305 gso_type = SKB_GSO_TCPV4;
2306 break;
2307 case VIRTIO_NET_HDR_GSO_TCPV6:
2308 gso_type = SKB_GSO_TCPV6;
2309 break;
2310 case VIRTIO_NET_HDR_GSO_UDP:
2311 gso_type = SKB_GSO_UDP;
2312 break;
2313 default:
2314 goto out_unlock;
2315 }
2316
2317 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2318 gso_type |= SKB_GSO_TCP_ECN;
2319
2320 if (vnet_hdr.gso_size == 0)
2321 goto out_unlock;
2322
2323 }
2324 }
2325
1da177e4 2326 err = -EMSGSIZE;
57f89bfa 2327 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN))
1da177e4
LT		 2328 goto out_unlock;
2329
bfd5f4a3
SS		 2330 err = -ENOBUFS;
2331 skb = packet_alloc_skb(sk, LL_ALLOCATED_SPACE(dev),
2332 LL_RESERVED_SPACE(dev), len, vnet_hdr.hdr_len,
2333 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2334 if (skb == NULL)
1da177e4
LT		 2335 goto out_unlock;
2336
bfd5f4a3 2337 skb_set_network_header(skb, reserve);
1da177e4 2338
0c4e8581
SH		 2339 err = -EINVAL;
2340 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2341 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2342 goto out_free;
1da177e4
LT		 2343
2344 /* Returns -EFAULT on error */
bfd5f4a3 2345 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2346 if (err)
2347 goto out_free;
2244d07b 2348 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 2349 if (err < 0)
2350 goto out_free;
1da177e4 2351
57f89bfa
BG		 2352 if (!gso_type && (len > dev->mtu + reserve)) {
2353 /* Earlier code assumed this would be a VLAN pkt,
2354 * double-check this now that we have the actual
2355 * packet in hand.
2356 */
2357 struct ethhdr *ehdr;
2358 skb_reset_mac_header(skb);
2359 ehdr = eth_hdr(skb);
2360 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2361 err = -EMSGSIZE;
2362 goto out_free;
2363 }
2364 }
2365
1da177e4
LT		 2366 skb->protocol = proto;
2367 skb->dev = dev;
2368 skb->priority = sk->sk_priority;
2d37a186 2369 skb->mark = sk->sk_mark;
1da177e4 2370
bfd5f4a3
SS		 2371 if (po->has_vnet_hdr) {
2372 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2373 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2374 vnet_hdr.csum_offset)) {
2375 err = -EINVAL;
2376 goto out_free;
2377 }
2378 }
2379
2380 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2381 skb_shinfo(skb)->gso_type = gso_type;
2382
2383 /* Header must be checked, and gso_segs computed. */
2384 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2385 skb_shinfo(skb)->gso_segs = 0;
2386
2387 len += vnet_hdr_len;
2388 }
2389
1da177e4
LT		 2390 /*
2391 * Now send it
2392 */
2393
2394 err = dev_queue_xmit(skb);
2395 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2396 goto out_unlock;
2397
827d9780
BG		 2398 if (need_rls_dev)
2399 dev_put(dev);
1da177e4 2400
40d4e3df 2401 return len;
1da177e4
LT		 2402
2403out_free:
2404 kfree_skb(skb);
2405out_unlock:
827d9780 2406 if (dev && need_rls_dev)
1da177e4
LT		 2407 dev_put(dev);
2408out:
2409 return err;
2410}
2411
69e3c75f
JB		 2412static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2413 struct msghdr *msg, size_t len)
2414{
69e3c75f
JB		 2415 struct sock *sk = sock->sk;
2416 struct packet_sock *po = pkt_sk(sk);
2417 if (po->tx_ring.pg_vec)
2418 return tpacket_snd(po, msg);
2419 else
69e3c75f
JB		 2420 return packet_snd(sock, msg, len);
2421}
2422
1da177e4
LT		 2423/*
2424 * Close a PACKET socket. This is fairly simple. We immediately go
2425 * to 'closed' state and remove our protocol entry in the device list.
2426 */
2427
2428static int packet_release(struct socket *sock)
2429{
2430 struct sock *sk = sock->sk;
2431 struct packet_sock *po;
d12d01d6 2432 struct net *net;
f6fb8f10 2433 union tpacket_req_u req_u;
1da177e4
LT		 2434
2435 if (!sk)
2436 return 0;
2437
3b1e0a65 2438 net = sock_net(sk);
1da177e4
LT		 2439 po = pkt_sk(sk);
2440
808f5114 2441 spin_lock_bh(&net->packet.sklist_lock);
2442 sk_del_node_init_rcu(sk);
920de804 2443 sock_prot_inuse_add(net, sk->sk_prot, -1);
808f5114 2444 spin_unlock_bh(&net->packet.sklist_lock);
1da177e4 2445
808f5114 2446 spin_lock(&po->bind_lock);
ce06b03e 2447 unregister_prot_hook(sk, false);
160ff18a
BG		 2448 if (po->prot_hook.dev) {
2449 dev_put(po->prot_hook.dev);
2450 po->prot_hook.dev = NULL;
2451 }
808f5114 2452 spin_unlock(&po->bind_lock);
1da177e4 2453
1da177e4 2454 packet_flush_mclist(sk);
1da177e4 2455
f6fb8f10 2456 memset(&req_u, 0, sizeof(req_u));
69e3c75f
JB		 2457
2458 if (po->rx_ring.pg_vec)
f6fb8f10 2459 packet_set_ring(sk, &req_u, 1, 0);
69e3c75f
JB		 2460
2461 if (po->tx_ring.pg_vec)
f6fb8f10 2462 packet_set_ring(sk, &req_u, 1, 1);
1da177e4 2463
dc99f600
DM		 2464 fanout_release(sk);
2465
808f5114 2466 synchronize_net();
1da177e4
LT		 2467 /*
2468 * Now the socket is dead. No more input will appear.
2469 */
1da177e4
LT		 2470 sock_orphan(sk);
2471 sock->sk = NULL;
2472
2473 /* Purge queues */
2474
2475 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2476 sk_refcnt_debug_release(sk);
1da177e4
LT		 2477
2478 sock_put(sk);
2479 return 0;
2480}
2481
2482/*
2483 * Attach a packet hook.
2484 */
2485
0e11c91e 2486static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2487{
2488 struct packet_sock *po = pkt_sk(sk);
dc99f600
DM		 2489
2490 if (po->fanout)
2491 return -EINVAL;
1da177e4
LT		 2492
2493 lock_sock(sk);
2494
2495 spin_lock(&po->bind_lock);
ce06b03e 2496 unregister_prot_hook(sk, true);
1da177e4
LT		 2497 po->num = protocol;
2498 po->prot_hook.type = protocol;
160ff18a
BG		 2499 if (po->prot_hook.dev)
2500 dev_put(po->prot_hook.dev);
1da177e4
LT		 2501 po->prot_hook.dev = dev;
2502
2503 po->ifindex = dev ? dev->ifindex : 0;
2504
2505 if (protocol == 0)
2506 goto out_unlock;
2507
be85d4ad 2508 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2509 register_prot_hook(sk);
be85d4ad
UT		 2510 } else {
2511 sk->sk_err = ENETDOWN;
2512 if (!sock_flag(sk, SOCK_DEAD))
2513 sk->sk_error_report(sk);
1da177e4
LT		 2514 }
2515
2516out_unlock:
2517 spin_unlock(&po->bind_lock);
2518 release_sock(sk);
2519 return 0;
2520}
2521
2522/*
2523 * Bind a packet socket to a device
2524 */
2525
40d4e3df
ED		 2526static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2527 int addr_len)
1da177e4 2528{
40d4e3df 2529 struct sock *sk = sock->sk;
1da177e4
LT		 2530 char name[15];
2531 struct net_device *dev;
2532 int err = -ENODEV;
1ce4f28b 2533
1da177e4
LT		 2534 /*
2535 * Check legality
2536 */
1ce4f28b 2537
8ae55f04 2538 if (addr_len != sizeof(struct sockaddr))
1da177e4 2539 return -EINVAL;
40d4e3df 2540 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2541
3b1e0a65 2542 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2543 if (dev)
1da177e4 2544 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2545 return err;
2546}
1da177e4
LT		 2547
2548static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2549{
40d4e3df
ED		 2550 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2551 struct sock *sk = sock->sk;
1da177e4
LT		 2552 struct net_device *dev = NULL;
2553 int err;
2554
2555
2556 /*
2557 * Check legality
2558 */
1ce4f28b 2559
1da177e4
LT		 2560 if (addr_len < sizeof(struct sockaddr_ll))
2561 return -EINVAL;
2562 if (sll->sll_family != AF_PACKET)
2563 return -EINVAL;
2564
2565 if (sll->sll_ifindex) {
2566 err = -ENODEV;
3b1e0a65 2567 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2568 if (dev == NULL)
2569 goto out;
2570 }
2571 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2572
2573out:
2574 return err;
2575}
2576
2577static struct proto packet_proto = {
2578 .name = "PACKET",
2579 .owner = THIS_MODULE,
2580 .obj_size = sizeof(struct packet_sock),
2581};
2582
2583/*
1ce4f28b 2584 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2585 */
2586
3f378b68
EP		 2587static int packet_create(struct net *net, struct socket *sock, int protocol,
2588 int kern)
1da177e4
LT		 2589{
2590 struct sock *sk;
2591 struct packet_sock *po;
0e11c91e 2592 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2593 int err;
2594
2595 if (!capable(CAP_NET_RAW))
2596 return -EPERM;
be02097c
DM		 2597 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2598 sock->type != SOCK_PACKET)
1da177e4
LT		 2599 return -ESOCKTNOSUPPORT;
2600
2601 sock->state = SS_UNCONNECTED;
2602
2603 err = -ENOBUFS;
6257ff21 2604 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2605 if (sk == NULL)
2606 goto out;
2607
2608 sock->ops = &packet_ops;
1da177e4
LT		 2609 if (sock->type == SOCK_PACKET)
2610 sock->ops = &packet_ops_spkt;
be02097c 2611
1da177e4
LT		 2612 sock_init_data(sock, sk);
2613
2614 po = pkt_sk(sk);
2615 sk->sk_family = PF_PACKET;
0e11c91e 2616 po->num = proto;
1da177e4
LT		 2617
2618 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2619 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2620
2621 /*
2622 * Attach a protocol block
2623 */
2624
2625 spin_lock_init(&po->bind_lock);
905db440 2626 mutex_init(&po->pg_vec_lock);
1da177e4 2627 po->prot_hook.func = packet_rcv;
be02097c 2628
1da177e4
LT		 2629 if (sock->type == SOCK_PACKET)
2630 po->prot_hook.func = packet_rcv_spkt;
be02097c 2631
1da177e4
LT		 2632 po->prot_hook.af_packet_priv = sk;
2633
0e11c91e
AV		 2634 if (proto) {
2635 po->prot_hook.type = proto;
ce06b03e 2636 register_prot_hook(sk);
1da177e4
LT		 2637 }
2638
808f5114 2639 spin_lock_bh(&net->packet.sklist_lock);
2640 sk_add_node_rcu(sk, &net->packet.sklist);
3680453c 2641 sock_prot_inuse_add(net, &packet_proto, 1);
808f5114 2642 spin_unlock_bh(&net->packet.sklist_lock);
2643
40d4e3df 2644 return 0;
1da177e4
LT		 2645out:
2646 return err;
2647}
2648
ed85b565
RC		 2649static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2650{
2651 struct sock_exterr_skb *serr;
2652 struct sk_buff *skb, *skb2;
2653 int copied, err;
2654
2655 err = -EAGAIN;
2656 skb = skb_dequeue(&sk->sk_error_queue);
2657 if (skb == NULL)
2658 goto out;
2659
2660 copied = skb->len;
2661 if (copied > len) {
2662 msg->msg_flags |= MSG_TRUNC;
2663 copied = len;
2664 }
2665 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2666 if (err)
2667 goto out_free_skb;
2668
2669 sock_recv_timestamp(msg, sk, skb);
2670
2671 serr = SKB_EXT_ERR(skb);
2672 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2673 sizeof(serr->ee), &serr->ee);
2674
2675 msg->msg_flags |= MSG_ERRQUEUE;
2676 err = copied;
2677
2678 /* Reset and regenerate socket error */
2679 spin_lock_bh(&sk->sk_error_queue.lock);
2680 sk->sk_err = 0;
2681 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2682 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2683 spin_unlock_bh(&sk->sk_error_queue.lock);
2684 sk->sk_error_report(sk);
2685 } else
2686 spin_unlock_bh(&sk->sk_error_queue.lock);
2687
2688out_free_skb:
2689 kfree_skb(skb);
2690out:
2691 return err;
2692}
2693
1da177e4
LT		 2694/*
2695 * Pull a packet from our receive queue and hand it to the user.
2696 * If necessary we block.
2697 */
2698
2699static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2700 struct msghdr *msg, size_t len, int flags)
2701{
2702 struct sock *sk = sock->sk;
2703 struct sk_buff *skb;
2704 int copied, err;
0fb375fb 2705 struct sockaddr_ll *sll;
bfd5f4a3 2706 int vnet_hdr_len = 0;
1da177e4
LT		 2707
2708 err = -EINVAL;
ed85b565 2709 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2710 goto out;
2711
2712#if 0
2713 /* What error should we return now? EUNATTACH? */
2714 if (pkt_sk(sk)->ifindex < 0)
2715 return -ENODEV;
2716#endif
2717
ed85b565
RC		 2718 if (flags & MSG_ERRQUEUE) {
2719 err = packet_recv_error(sk, msg, len);
2720 goto out;
2721 }
2722
1da177e4
LT		 2723 /*
2724 * Call the generic datagram receiver. This handles all sorts
2725 * of horrible races and re-entrancy so we can forget about it
2726 * in the protocol layers.
2727 *
2728 * Now it will return ENETDOWN, if device have just gone down,
2729 * but then it will block.
2730 */
2731
40d4e3df 2732 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2733
2734 /*
1ce4f28b 2735 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2736 * handles the blocking we don't see and worry about blocking
2737 * retries.
2738 */
2739
8ae55f04 2740 if (skb == NULL)
1da177e4
LT		 2741 goto out;
2742
bfd5f4a3
SS		 2743 if (pkt_sk(sk)->has_vnet_hdr) {
2744 struct virtio_net_hdr vnet_hdr = { 0 };
2745
2746 err = -EINVAL;
2747 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2748 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2749 goto out_free;
2750
1f18b717
MK		 2751 len -= vnet_hdr_len;
2752
bfd5f4a3
SS		 2753 if (skb_is_gso(skb)) {
2754 struct skb_shared_info *sinfo = skb_shinfo(skb);
2755
2756 /* This is a hint as to how much should be linear. */
2757 vnet_hdr.hdr_len = skb_headlen(skb);
2758 vnet_hdr.gso_size = sinfo->gso_size;
2759 if (sinfo->gso_type & SKB_GSO_TCPV4)
2760 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2761 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2762 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2763 else if (sinfo->gso_type & SKB_GSO_UDP)
2764 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2765 else if (sinfo->gso_type & SKB_GSO_FCOE)
2766 goto out_free;
2767 else
2768 BUG();
2769 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2770 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2771 } else
2772 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2773
2774 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2775 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2776 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2777 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2778 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2779 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2780 } /* else everything is zero */
2781
2782 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2783 vnet_hdr_len);
2784 if (err < 0)
2785 goto out_free;
2786 }
2787
0fb375fb
EB		 2788 /*
2789 * If the address length field is there to be filled in, we fill
2790 * it in now.
2791 */
2792
ffbc6111 2793 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2794 if (sock->type == SOCK_PACKET)
2795 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2796 else
2797 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2798
1da177e4
LT		 2799 /*
2800 * You lose any data beyond the buffer you gave. If it worries a
2801 * user program they can ask the device for its MTU anyway.
2802 */
2803
2804 copied = skb->len;
40d4e3df
ED		 2805 if (copied > len) {
2806 copied = len;
2807 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2808 }
2809
2810 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2811 if (err)
2812 goto out_free;
2813
3b885787 2814 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2815
2816 if (msg->msg_name)
ffbc6111
HX		 2817 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2818 msg->msg_namelen);
1da177e4 2819
8dc41944 2820 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2821 struct tpacket_auxdata aux;
2822
2823 aux.tp_status = TP_STATUS_USER;
2824 if (skb->ip_summed == CHECKSUM_PARTIAL)
2825 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2826 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2827 aux.tp_snaplen = skb->len;
2828 aux.tp_mac = 0;
bbe735e4 2829 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2830 if (vlan_tx_tag_present(skb)) {
2831 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2832 aux.tp_status |= TP_STATUS_VLAN_VALID;
2833 } else {
2834 aux.tp_vlan_tci = 0;
2835 }
13fcb7bd 2836 aux.tp_padding = 0;
ffbc6111 2837 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2838 }
2839
1da177e4
LT		 2840 /*
2841 * Free or return the buffer as appropriate. Again this
2842 * hides all the races and re-entrancy issues from us.
2843 */
bfd5f4a3 2844 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2845
2846out_free:
2847 skb_free_datagram(sk, skb);
2848out:
2849 return err;
2850}
2851
1da177e4
LT		 2852static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2853 int *uaddr_len, int peer)
2854{
2855 struct net_device *dev;
2856 struct sock *sk = sock->sk;
2857
2858 if (peer)
2859 return -EOPNOTSUPP;
2860
2861 uaddr->sa_family = AF_PACKET;
654d1f8a
ED		 2862 rcu_read_lock();
2863 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2864 if (dev)
67286640 2865 strncpy(uaddr->sa_data, dev->name, 14);
654d1f8a 2866 else
1da177e4 2867 memset(uaddr->sa_data, 0, 14);
654d1f8a 2868 rcu_read_unlock();
1da177e4
LT		 2869 *uaddr_len = sizeof(*uaddr);
2870
2871 return 0;
2872}
1da177e4
LT		 2873
2874static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2875 int *uaddr_len, int peer)
2876{
2877 struct net_device *dev;
2878 struct sock *sk = sock->sk;
2879 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2880 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2881
2882 if (peer)
2883 return -EOPNOTSUPP;
2884
2885 sll->sll_family = AF_PACKET;
2886 sll->sll_ifindex = po->ifindex;
2887 sll->sll_protocol = po->num;
67286640 2888 sll->sll_pkttype = 0;
654d1f8a
ED		 2889 rcu_read_lock();
2890 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2891 if (dev) {
2892 sll->sll_hatype = dev->type;
2893 sll->sll_halen = dev->addr_len;
2894 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2895 } else {
2896 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2897 sll->sll_halen = 0;
2898 }
654d1f8a 2899 rcu_read_unlock();
0fb375fb 2900 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2901
2902 return 0;
2903}
2904
2aeb0b88
WC		 2905static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2906 int what)
1da177e4
LT		 2907{
2908 switch (i->type) {
2909 case PACKET_MR_MULTICAST:
1162563f
JP		 2910 if (i->alen != dev->addr_len)
2911 return -EINVAL;
1da177e4 2912 if (what > 0)
22bedad3 2913 return dev_mc_add(dev, i->addr);
1da177e4 2914 else
22bedad3 2915 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2916 break;
2917 case PACKET_MR_PROMISC:
2aeb0b88 2918 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2919 break;
2920 case PACKET_MR_ALLMULTI:
2aeb0b88 2921 return dev_set_allmulti(dev, what);
1da177e4 2922 break;
d95ed927 2923 case PACKET_MR_UNICAST:
1162563f
JP		 2924 if (i->alen != dev->addr_len)
2925 return -EINVAL;
d95ed927 2926 if (what > 0)
a748ee24 2927 return dev_uc_add(dev, i->addr);
d95ed927 2928 else
a748ee24 2929 return dev_uc_del(dev, i->addr);
d95ed927 2930 break;
40d4e3df
ED		 2931 default:
2932 break;
1da177e4 2933 }
2aeb0b88 2934 return 0;
1da177e4
LT		 2935}
2936
2937static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2938{
40d4e3df 2939 for ( ; i; i = i->next) {
1da177e4
LT		 2940 if (i->ifindex == dev->ifindex)
2941 packet_dev_mc(dev, i, what);
2942 }
2943}
2944
0fb375fb 2945static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2946{
2947 struct packet_sock *po = pkt_sk(sk);
2948 struct packet_mclist *ml, *i;
2949 struct net_device *dev;
2950 int err;
2951
2952 rtnl_lock();
2953
2954 err = -ENODEV;
3b1e0a65 2955 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2956 if (!dev)
2957 goto done;
2958
2959 err = -EINVAL;
1162563f 2960 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2961 goto done;
2962
2963 err = -ENOBUFS;
8b3a7005 2964 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2965 if (i == NULL)
2966 goto done;
2967
2968 err = 0;
2969 for (ml = po->mclist; ml; ml = ml->next) {
2970 if (ml->ifindex == mreq->mr_ifindex &&
2971 ml->type == mreq->mr_type &&
2972 ml->alen == mreq->mr_alen &&
2973 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2974 ml->count++;
2975 /* Free the new element ... */
2976 kfree(i);
2977 goto done;
2978 }
2979 }
2980
2981 i->type = mreq->mr_type;
2982 i->ifindex = mreq->mr_ifindex;
2983 i->alen = mreq->mr_alen;
2984 memcpy(i->addr, mreq->mr_address, i->alen);
2985 i->count = 1;
2986 i->next = po->mclist;
2987 po->mclist = i;
2aeb0b88
WC		 2988 err = packet_dev_mc(dev, i, 1);
2989 if (err) {
2990 po->mclist = i->next;
2991 kfree(i);
2992 }
1da177e4
LT		 2993
2994done:
2995 rtnl_unlock();
2996 return err;
2997}
2998
0fb375fb 2999static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3000{
3001 struct packet_mclist *ml, **mlp;
3002
3003 rtnl_lock();
3004
3005 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3006 if (ml->ifindex == mreq->mr_ifindex &&
3007 ml->type == mreq->mr_type &&
3008 ml->alen == mreq->mr_alen &&
3009 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3010 if (--ml->count == 0) {
3011 struct net_device *dev;
3012 *mlp = ml->next;
ad959e76
ED		 3013 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3014 if (dev)
1da177e4 3015 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3016 kfree(ml);
3017 }
3018 rtnl_unlock();
3019 return 0;
3020 }
3021 }
3022 rtnl_unlock();
3023 return -EADDRNOTAVAIL;
3024}
3025
3026static void packet_flush_mclist(struct sock *sk)
3027{
3028 struct packet_sock *po = pkt_sk(sk);
3029 struct packet_mclist *ml;
3030
3031 if (!po->mclist)
3032 return;
3033
3034 rtnl_lock();
3035 while ((ml = po->mclist) != NULL) {
3036 struct net_device *dev;
3037
3038 po->mclist = ml->next;
ad959e76
ED		 3039 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3040 if (dev != NULL)
1da177e4 3041 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3042 kfree(ml);
3043 }
3044 rtnl_unlock();
3045}
1da177e4
LT		 3046
3047static int
b7058842 3048packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3049{
3050 struct sock *sk = sock->sk;
8dc41944 3051 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3052 int ret;
3053
3054 if (level != SOL_PACKET)
3055 return -ENOPROTOOPT;
3056
69e3c75f 3057 switch (optname) {
1ce4f28b 3058 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3059 case PACKET_DROP_MEMBERSHIP:
3060 {
0fb375fb
EB		 3061 struct packet_mreq_max mreq;
3062 int len = optlen;
3063 memset(&mreq, 0, sizeof(mreq));
3064 if (len < sizeof(struct packet_mreq))
1da177e4 3065 return -EINVAL;
0fb375fb
EB		 3066 if (len > sizeof(mreq))
3067 len = sizeof(mreq);
40d4e3df 3068 if (copy_from_user(&mreq, optval, len))
1da177e4 3069 return -EFAULT;
0fb375fb
EB		 3070 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3071 return -EINVAL;
1da177e4
LT		 3072 if (optname == PACKET_ADD_MEMBERSHIP)
3073 ret = packet_mc_add(sk, &mreq);
3074 else
3075 ret = packet_mc_drop(sk, &mreq);
3076 return ret;
3077 }
a2efcfa0 3078
1da177e4 3079 case PACKET_RX_RING:
69e3c75f 3080 case PACKET_TX_RING:
1da177e4 3081 {
f6fb8f10 3082 union tpacket_req_u req_u;
3083 int len;
1da177e4 3084
f6fb8f10 3085 switch (po->tp_version) {
3086 case TPACKET_V1:
3087 case TPACKET_V2:
3088 len = sizeof(req_u.req);
3089 break;
3090 case TPACKET_V3:
3091 default:
3092 len = sizeof(req_u.req3);
3093 break;
3094 }
3095 if (optlen < len)
1da177e4 3096 return -EINVAL;
bfd5f4a3
SS		 3097 if (pkt_sk(sk)->has_vnet_hdr)
3098 return -EINVAL;
f6fb8f10 3099 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3100 return -EFAULT;
f6fb8f10 3101 return packet_set_ring(sk, &req_u, 0,
3102 optname == PACKET_TX_RING);
1da177e4
LT		 3103 }
3104 case PACKET_COPY_THRESH:
3105 {
3106 int val;
3107
40d4e3df 3108 if (optlen != sizeof(val))
1da177e4 3109 return -EINVAL;
40d4e3df 3110 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3111 return -EFAULT;
3112
3113 pkt_sk(sk)->copy_thresh = val;
3114 return 0;
3115 }
bbd6ef87
PM		 3116 case PACKET_VERSION:
3117 {
3118 int val;
3119
3120 if (optlen != sizeof(val))
3121 return -EINVAL;
69e3c75f 3122 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3123 return -EBUSY;
3124 if (copy_from_user(&val, optval, sizeof(val)))
3125 return -EFAULT;
3126 switch (val) {
3127 case TPACKET_V1:
3128 case TPACKET_V2:
f6fb8f10 3129 case TPACKET_V3:
bbd6ef87
PM		 3130 po->tp_version = val;
3131 return 0;
3132 default:
3133 return -EINVAL;
3134 }
3135 }
8913336a
PM		 3136 case PACKET_RESERVE:
3137 {
3138 unsigned int val;
3139
3140 if (optlen != sizeof(val))
3141 return -EINVAL;
69e3c75f 3142 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3143 return -EBUSY;
3144 if (copy_from_user(&val, optval, sizeof(val)))
3145 return -EFAULT;
3146 po->tp_reserve = val;
3147 return 0;
3148 }
69e3c75f
JB		 3149 case PACKET_LOSS:
3150 {
3151 unsigned int val;
3152
3153 if (optlen != sizeof(val))
3154 return -EINVAL;
3155 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3156 return -EBUSY;
3157 if (copy_from_user(&val, optval, sizeof(val)))
3158 return -EFAULT;
3159 po->tp_loss = !!val;
3160 return 0;
3161 }
8dc41944
HX		 3162 case PACKET_AUXDATA:
3163 {
3164 int val;
3165
3166 if (optlen < sizeof(val))
3167 return -EINVAL;
3168 if (copy_from_user(&val, optval, sizeof(val)))
3169 return -EFAULT;
3170
3171 po->auxdata = !!val;
3172 return 0;
3173 }
80feaacb
PWJ		 3174 case PACKET_ORIGDEV:
3175 {
3176 int val;
3177
3178 if (optlen < sizeof(val))
3179 return -EINVAL;
3180 if (copy_from_user(&val, optval, sizeof(val)))
3181 return -EFAULT;
3182
3183 po->origdev = !!val;
3184 return 0;
3185 }
bfd5f4a3
SS		 3186 case PACKET_VNET_HDR:
3187 {
3188 int val;
3189
3190 if (sock->type != SOCK_RAW)
3191 return -EINVAL;
3192 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3193 return -EBUSY;
3194 if (optlen < sizeof(val))
3195 return -EINVAL;
3196 if (copy_from_user(&val, optval, sizeof(val)))
3197 return -EFAULT;
3198
3199 po->has_vnet_hdr = !!val;
3200 return 0;
3201 }
614f60fa
SM		 3202 case PACKET_TIMESTAMP:
3203 {
3204 int val;
3205
3206 if (optlen != sizeof(val))
3207 return -EINVAL;
3208 if (copy_from_user(&val, optval, sizeof(val)))
3209 return -EFAULT;
3210
3211 po->tp_tstamp = val;
3212 return 0;
3213 }
dc99f600
DM		 3214 case PACKET_FANOUT:
3215 {
3216 int val;
3217
3218 if (optlen != sizeof(val))
3219 return -EINVAL;
3220 if (copy_from_user(&val, optval, sizeof(val)))
3221 return -EFAULT;
3222
3223 return fanout_add(sk, val & 0xffff, val >> 16);
3224 }
1da177e4
LT		 3225 default:
3226 return -ENOPROTOOPT;
3227 }
3228}
3229
3230static int packet_getsockopt(struct socket *sock, int level, int optname,
3231 char __user *optval, int __user *optlen)
3232{
3233 int len;
8dc41944 3234 int val;
1da177e4
LT		 3235 struct sock *sk = sock->sk;
3236 struct packet_sock *po = pkt_sk(sk);
8dc41944
HX		 3237 void *data;
3238 struct tpacket_stats st;
f6fb8f10 3239 union tpacket_stats_u st_u;
1da177e4
LT		 3240
3241 if (level != SOL_PACKET)
3242 return -ENOPROTOOPT;
3243
8ae55f04
KK		 3244 if (get_user(len, optlen))
3245 return -EFAULT;
1da177e4
LT		 3246
3247 if (len < 0)
3248 return -EINVAL;
1ce4f28b 3249
69e3c75f 3250 switch (optname) {
1da177e4 3251 case PACKET_STATISTICS:
f6fb8f10 3252 if (po->tp_version == TPACKET_V3) {
3253 len = sizeof(struct tpacket_stats_v3);
3254 } else {
3255 if (len > sizeof(struct tpacket_stats))
3256 len = sizeof(struct tpacket_stats);
3257 }
1da177e4 3258 spin_lock_bh(&sk->sk_receive_queue.lock);
f6fb8f10 3259 if (po->tp_version == TPACKET_V3) {
3260 memcpy(&st_u.stats3, &po->stats,
3261 sizeof(struct tpacket_stats));
3262 st_u.stats3.tp_freeze_q_cnt =
3263 po->stats_u.stats3.tp_freeze_q_cnt;
3264 st_u.stats3.tp_packets += po->stats.tp_drops;
3265 data = &st_u.stats3;
3266 } else {
3267 st = po->stats;
3268 st.tp_packets += st.tp_drops;
3269 data = &st;
3270 }
1da177e4
LT		 3271 memset(&po->stats, 0, sizeof(st));
3272 spin_unlock_bh(&sk->sk_receive_queue.lock);
8dc41944
HX		 3273 break;
3274 case PACKET_AUXDATA:
3275 if (len > sizeof(int))
3276 len = sizeof(int);
3277 val = po->auxdata;
3278
80feaacb
PWJ		 3279 data = &val;
3280 break;
3281 case PACKET_ORIGDEV:
3282 if (len > sizeof(int))
3283 len = sizeof(int);
3284 val = po->origdev;
3285
bfd5f4a3
SS		 3286 data = &val;
3287 break;
3288 case PACKET_VNET_HDR:
3289 if (len > sizeof(int))
3290 len = sizeof(int);
3291 val = po->has_vnet_hdr;
3292
8dc41944 3293 data = &val;
1da177e4 3294 break;
bbd6ef87
PM		 3295 case PACKET_VERSION:
3296 if (len > sizeof(int))
3297 len = sizeof(int);
3298 val = po->tp_version;
3299 data = &val;
3300 break;
3301 case PACKET_HDRLEN:
3302 if (len > sizeof(int))
3303 len = sizeof(int);
3304 if (copy_from_user(&val, optval, len))
3305 return -EFAULT;
3306 switch (val) {
3307 case TPACKET_V1:
3308 val = sizeof(struct tpacket_hdr);
3309 break;
3310 case TPACKET_V2:
3311 val = sizeof(struct tpacket2_hdr);
3312 break;
f6fb8f10 3313 case TPACKET_V3:
3314 val = sizeof(struct tpacket3_hdr);
3315 break;
bbd6ef87
PM		 3316 default:
3317 return -EINVAL;
3318 }
3319 data = &val;
3320 break;
8913336a
PM		 3321 case PACKET_RESERVE:
3322 if (len > sizeof(unsigned int))
3323 len = sizeof(unsigned int);
3324 val = po->tp_reserve;
3325 data = &val;
3326 break;
69e3c75f
JB		 3327 case PACKET_LOSS:
3328 if (len > sizeof(unsigned int))
3329 len = sizeof(unsigned int);
3330 val = po->tp_loss;
3331 data = &val;
3332 break;
614f60fa
SM		 3333 case PACKET_TIMESTAMP:
3334 if (len > sizeof(int))
3335 len = sizeof(int);
3336 val = po->tp_tstamp;
3337 data = &val;
3338 break;
dc99f600
DM		 3339 case PACKET_FANOUT:
3340 if (len > sizeof(int))
3341 len = sizeof(int);
3342 val = (po->fanout ?
3343 ((u32)po->fanout->id |
3344 ((u32)po->fanout->type << 16)) :
3345 0);
3346 data = &val;
3347 break;
1da177e4
LT		 3348 default:
3349 return -ENOPROTOOPT;
3350 }
3351
8ae55f04
KK		 3352 if (put_user(len, optlen))
3353 return -EFAULT;
8dc41944
HX		 3354 if (copy_to_user(optval, data, len))
3355 return -EFAULT;
8ae55f04 3356 return 0;
1da177e4
LT		 3357}
3358
3359
3360static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3361{
3362 struct sock *sk;
3363 struct hlist_node *node;
ad930650 3364 struct net_device *dev = data;
c346dca1 3365 struct net *net = dev_net(dev);
1da177e4 3366
808f5114 3367 rcu_read_lock();
3368 sk_for_each_rcu(sk, node, &net->packet.sklist) {
1da177e4
LT		 3369 struct packet_sock *po = pkt_sk(sk);
3370
3371 switch (msg) {
3372 case NETDEV_UNREGISTER:
1da177e4
LT		 3373 if (po->mclist)
3374 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3375 /* fallthrough */
3376
1da177e4
LT		 3377 case NETDEV_DOWN:
3378 if (dev->ifindex == po->ifindex) {
3379 spin_lock(&po->bind_lock);
3380 if (po->running) {
ce06b03e 3381 __unregister_prot_hook(sk, false);
1da177e4
LT		 3382 sk->sk_err = ENETDOWN;
3383 if (!sock_flag(sk, SOCK_DEAD))
3384 sk->sk_error_report(sk);
3385 }
3386 if (msg == NETDEV_UNREGISTER) {
3387 po->ifindex = -1;
160ff18a
BG		 3388 if (po->prot_hook.dev)
3389 dev_put(po->prot_hook.dev);
1da177e4
LT		 3390 po->prot_hook.dev = NULL;
3391 }
3392 spin_unlock(&po->bind_lock);
3393 }
3394 break;
3395 case NETDEV_UP:
808f5114 3396 if (dev->ifindex == po->ifindex) {
3397 spin_lock(&po->bind_lock);
ce06b03e
DM		 3398 if (po->num)
3399 register_prot_hook(sk);
808f5114 3400 spin_unlock(&po->bind_lock);
1da177e4 3401 }
1da177e4
LT		 3402 break;
3403 }
3404 }
808f5114 3405 rcu_read_unlock();
1da177e4
LT		 3406 return NOTIFY_DONE;
3407}
3408
3409
3410static int packet_ioctl(struct socket *sock, unsigned int cmd,
3411 unsigned long arg)
3412{
3413 struct sock *sk = sock->sk;
3414
69e3c75f 3415 switch (cmd) {
40d4e3df
ED		 3416 case SIOCOUTQ:
3417 {
3418 int amount = sk_wmem_alloc_get(sk);
31e6d363 3419
40d4e3df
ED		 3420 return put_user(amount, (int __user *)arg);
3421 }
3422 case SIOCINQ:
3423 {
3424 struct sk_buff *skb;
3425 int amount = 0;
3426
3427 spin_lock_bh(&sk->sk_receive_queue.lock);
3428 skb = skb_peek(&sk->sk_receive_queue);
3429 if (skb)
3430 amount = skb->len;
3431 spin_unlock_bh(&sk->sk_receive_queue.lock);
3432 return put_user(amount, (int __user *)arg);
3433 }
3434 case SIOCGSTAMP:
3435 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3436 case SIOCGSTAMPNS:
3437 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3438
1da177e4 3439#ifdef CONFIG_INET
40d4e3df
ED		 3440 case SIOCADDRT:
3441 case SIOCDELRT:
3442 case SIOCDARP:
3443 case SIOCGARP:
3444 case SIOCSARP:
3445 case SIOCGIFADDR:
3446 case SIOCSIFADDR:
3447 case SIOCGIFBRDADDR:
3448 case SIOCSIFBRDADDR:
3449 case SIOCGIFNETMASK:
3450 case SIOCSIFNETMASK:
3451 case SIOCGIFDSTADDR:
3452 case SIOCSIFDSTADDR:
3453 case SIOCSIFFLAGS:
40d4e3df 3454 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3455#endif
3456
40d4e3df
ED		 3457 default:
3458 return -ENOIOCTLCMD;
1da177e4
LT		 3459 }
3460 return 0;
3461}
3462
40d4e3df 3463static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3464 poll_table *wait)
3465{
3466 struct sock *sk = sock->sk;
3467 struct packet_sock *po = pkt_sk(sk);
3468 unsigned int mask = datagram_poll(file, sock, wait);
3469
3470 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3471 if (po->rx_ring.pg_vec) {
f6fb8f10 3472 if (!packet_previous_rx_frame(po, &po->rx_ring,
3473 TP_STATUS_KERNEL))
1da177e4
LT		 3474 mask |= POLLIN | POLLRDNORM;
3475 }
3476 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3477 spin_lock_bh(&sk->sk_write_queue.lock);
3478 if (po->tx_ring.pg_vec) {
3479 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3480 mask |= POLLOUT | POLLWRNORM;
3481 }
3482 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3483 return mask;
3484}
3485
3486
3487/* Dirty? Well, I still did not learn better way to account
3488 * for user mmaps.
3489 */
3490
3491static void packet_mm_open(struct vm_area_struct *vma)
3492{
3493 struct file *file = vma->vm_file;
40d4e3df 3494 struct socket *sock = file->private_data;
1da177e4 3495 struct sock *sk = sock->sk;
1ce4f28b 3496
1da177e4
LT		 3497 if (sk)
3498 atomic_inc(&pkt_sk(sk)->mapped);
3499}
3500
3501static void packet_mm_close(struct vm_area_struct *vma)
3502{
3503 struct file *file = vma->vm_file;
40d4e3df 3504 struct socket *sock = file->private_data;
1da177e4 3505 struct sock *sk = sock->sk;
1ce4f28b 3506
1da177e4
LT		 3507 if (sk)
3508 atomic_dec(&pkt_sk(sk)->mapped);
3509}
3510
f0f37e2f 3511static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3512 .open = packet_mm_open,
3513 .close = packet_mm_close,
1da177e4
LT		 3514};
3515
0e3125c7
NH		 3516static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3517 unsigned int len)
1da177e4
LT		 3518{
3519 int i;
3520
4ebf0ae2 3521 for (i = 0; i < len; i++) {
0e3125c7 3522 if (likely(pg_vec[i].buffer)) {
c56b4d90 3523 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3524 vfree(pg_vec[i].buffer);
3525 else
3526 free_pages((unsigned long)pg_vec[i].buffer,
3527 order);
3528 pg_vec[i].buffer = NULL;
3529 }
1da177e4
LT		 3530 }
3531 kfree(pg_vec);
3532}
3533
c56b4d90 3534static inline char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3535{
0e3125c7
NH		 3536 char *buffer = NULL;
3537 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3538 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3539
3540 buffer = (char *) __get_free_pages(gfp_flags, order);
3541
3542 if (buffer)
3543 return buffer;
3544
3545 /*
3546 * __get_free_pages failed, fall back to vmalloc
3547 */
bbce5a59 3548 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3549
0e3125c7
NH		 3550 if (buffer)
3551 return buffer;
3552
3553 /*
3554 * vmalloc failed, lets dig into swap here
3555 */
0e3125c7
NH		 3556 gfp_flags &= ~__GFP_NORETRY;
3557 buffer = (char *)__get_free_pages(gfp_flags, order);
3558 if (buffer)
3559 return buffer;
3560
3561 /*
3562 * complete and utter failure
3563 */
3564 return NULL;
4ebf0ae2
DM		 3565}
3566
0e3125c7 3567static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3568{
3569 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3570 struct pgv *pg_vec;
4ebf0ae2
DM		 3571 int i;
3572
0e3125c7 3573 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3574 if (unlikely(!pg_vec))
3575 goto out;
3576
3577 for (i = 0; i < block_nr; i++) {
c56b4d90 3578 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3579 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3580 goto out_free_pgvec;
3581 }
3582
3583out:
3584 return pg_vec;
3585
3586out_free_pgvec:
3587 free_pg_vec(pg_vec, order, block_nr);
3588 pg_vec = NULL;
3589 goto out;
3590}
1da177e4 3591
f6fb8f10 3592static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3593 int closing, int tx_ring)
1da177e4 3594{
0e3125c7 3595 struct pgv *pg_vec = NULL;
1da177e4 3596 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3597 int was_running, order = 0;
69e3c75f
JB		 3598 struct packet_ring_buffer *rb;
3599 struct sk_buff_head *rb_queue;
0e11c91e 3600 __be16 num;
f6fb8f10 3601 int err = -EINVAL;
3602 /* Added to avoid minimal code churn */
3603 struct tpacket_req *req = &req_u->req;
3604
3605 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3606 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3607 WARN(1, "Tx-ring is not supported.\n");
3608 goto out;
3609 }
1ce4f28b 3610
69e3c75f
JB		 3611 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3612 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3613
69e3c75f
JB		 3614 err = -EBUSY;
3615 if (!closing) {
3616 if (atomic_read(&po->mapped))
3617 goto out;
3618 if (atomic_read(&rb->pending))
3619 goto out;
3620 }
1da177e4 3621
69e3c75f
JB		 3622 if (req->tp_block_nr) {
3623 /* Sanity tests and some calculations */
3624 err = -EBUSY;
3625 if (unlikely(rb->pg_vec))
3626 goto out;
1da177e4 3627
bbd6ef87
PM		 3628 switch (po->tp_version) {
3629 case TPACKET_V1:
3630 po->tp_hdrlen = TPACKET_HDRLEN;
3631 break;
3632 case TPACKET_V2:
3633 po->tp_hdrlen = TPACKET2_HDRLEN;
3634 break;
f6fb8f10 3635 case TPACKET_V3:
3636 po->tp_hdrlen = TPACKET3_HDRLEN;
3637 break;
bbd6ef87
PM		 3638 }
3639
69e3c75f 3640 err = -EINVAL;
4ebf0ae2 3641 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3642 goto out;
4ebf0ae2 3643 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3644 goto out;
8913336a 3645 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3646 po->tp_reserve))
3647 goto out;
4ebf0ae2 3648 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3649 goto out;
1da177e4 3650
69e3c75f
JB		 3651 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3652 if (unlikely(rb->frames_per_block <= 0))
3653 goto out;
3654 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3655 req->tp_frame_nr))
3656 goto out;
1da177e4
LT		 3657
3658 err = -ENOMEM;
4ebf0ae2
DM		 3659 order = get_order(req->tp_block_size);
3660 pg_vec = alloc_pg_vec(req, order);
3661 if (unlikely(!pg_vec))
1da177e4 3662 goto out;
f6fb8f10 3663 switch (po->tp_version) {
3664 case TPACKET_V3:
3665 /* Transmit path is not supported. We checked
3666 * it above but just being paranoid
3667 */
3668 if (!tx_ring)
3669 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3670 break;
3671 default:
3672 break;
3673 }
69e3c75f
JB		 3674 }
3675 /* Done */
3676 else {
3677 err = -EINVAL;
4ebf0ae2 3678 if (unlikely(req->tp_frame_nr))
69e3c75f 3679 goto out;
1da177e4
LT		 3680 }
3681
3682 lock_sock(sk);
3683
3684 /* Detach socket from network */
3685 spin_lock(&po->bind_lock);
3686 was_running = po->running;
3687 num = po->num;
3688 if (was_running) {
1da177e4 3689 po->num = 0;
ce06b03e 3690 __unregister_prot_hook(sk, false);
1da177e4
LT		 3691 }
3692 spin_unlock(&po->bind_lock);
1ce4f28b 3693
1da177e4
LT		 3694 synchronize_net();
3695
3696 err = -EBUSY;
905db440 3697 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3698 if (closing || atomic_read(&po->mapped) == 0) {
3699 err = 0;
69e3c75f 3700 spin_lock_bh(&rb_queue->lock);
c053fd96 3701 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3702 rb->frame_max = (req->tp_frame_nr - 1);
3703 rb->head = 0;
3704 rb->frame_size = req->tp_frame_size;
3705 spin_unlock_bh(&rb_queue->lock);
3706
c053fd96
CG		 3707 swap(rb->pg_vec_order, order);
3708 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3709
3710 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3711 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3712 tpacket_rcv : packet_rcv;
3713 skb_queue_purge(rb_queue);
1da177e4 3714 if (atomic_read(&po->mapped))
40d4e3df
ED		 3715 pr_err("packet_mmap: vma is busy: %d\n",
3716 atomic_read(&po->mapped));
1da177e4 3717 }
905db440 3718 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3719
3720 spin_lock(&po->bind_lock);
ce06b03e 3721 if (was_running) {
1da177e4 3722 po->num = num;
ce06b03e 3723 register_prot_hook(sk);
1da177e4
LT		 3724 }
3725 spin_unlock(&po->bind_lock);
f6fb8f10 3726 if (closing && (po->tp_version > TPACKET_V2)) {
3727 /* Because we don't support block-based V3 on tx-ring */
3728 if (!tx_ring)
3729 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3730 }
1da177e4
LT		 3731 release_sock(sk);
3732
1da177e4
LT		 3733 if (pg_vec)
3734 free_pg_vec(pg_vec, order, req->tp_block_nr);
3735out:
3736 return err;
3737}
3738
69e3c75f
JB		 3739static int packet_mmap(struct file *file, struct socket *sock,
3740 struct vm_area_struct *vma)
1da177e4
LT		 3741{
3742 struct sock *sk = sock->sk;
3743 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3744 unsigned long size, expected_size;
3745 struct packet_ring_buffer *rb;
1da177e4
LT		 3746 unsigned long start;
3747 int err = -EINVAL;
3748 int i;
3749
3750 if (vma->vm_pgoff)
3751 return -EINVAL;
3752
905db440 3753 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3754
3755 expected_size = 0;
3756 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3757 if (rb->pg_vec) {
3758 expected_size += rb->pg_vec_len
3759 * rb->pg_vec_pages
3760 * PAGE_SIZE;
3761 }
3762 }
3763
3764 if (expected_size == 0)
1da177e4 3765 goto out;
69e3c75f
JB		 3766
3767 size = vma->vm_end - vma->vm_start;
3768 if (size != expected_size)
1da177e4
LT		 3769 goto out;
3770
1da177e4 3771 start = vma->vm_start;
69e3c75f
JB		 3772 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3773 if (rb->pg_vec == NULL)
3774 continue;
3775
3776 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3777 struct page *page;
3778 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3779 int pg_num;
3780
c56b4d90
CG		 3781 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3782 page = pgv_to_page(kaddr);
69e3c75f
JB		 3783 err = vm_insert_page(vma, start, page);
3784 if (unlikely(err))
3785 goto out;
3786 start += PAGE_SIZE;
0e3125c7 3787 kaddr += PAGE_SIZE;
69e3c75f 3788 }
4ebf0ae2 3789 }
1da177e4 3790 }
69e3c75f 3791
4ebf0ae2 3792 atomic_inc(&po->mapped);
1da177e4
LT		 3793 vma->vm_ops = &packet_mmap_ops;
3794 err = 0;
3795
3796out:
905db440 3797 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3798 return err;
3799}
1da177e4 3800
90ddc4f0 3801static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3802 .family = PF_PACKET,
3803 .owner = THIS_MODULE,
3804 .release = packet_release,
3805 .bind = packet_bind_spkt,
3806 .connect = sock_no_connect,
3807 .socketpair = sock_no_socketpair,
3808 .accept = sock_no_accept,
3809 .getname = packet_getname_spkt,
3810 .poll = datagram_poll,
3811 .ioctl = packet_ioctl,
3812 .listen = sock_no_listen,
3813 .shutdown = sock_no_shutdown,
3814 .setsockopt = sock_no_setsockopt,
3815 .getsockopt = sock_no_getsockopt,
3816 .sendmsg = packet_sendmsg_spkt,
3817 .recvmsg = packet_recvmsg,
3818 .mmap = sock_no_mmap,
3819 .sendpage = sock_no_sendpage,
3820};
1da177e4 3821
90ddc4f0 3822static const struct proto_ops packet_ops = {
1da177e4
LT		 3823 .family = PF_PACKET,
3824 .owner = THIS_MODULE,
3825 .release = packet_release,
3826 .bind = packet_bind,
3827 .connect = sock_no_connect,
3828 .socketpair = sock_no_socketpair,
3829 .accept = sock_no_accept,
1ce4f28b 3830 .getname = packet_getname,
1da177e4
LT		 3831 .poll = packet_poll,
3832 .ioctl = packet_ioctl,
3833 .listen = sock_no_listen,
3834 .shutdown = sock_no_shutdown,
3835 .setsockopt = packet_setsockopt,
3836 .getsockopt = packet_getsockopt,
3837 .sendmsg = packet_sendmsg,
3838 .recvmsg = packet_recvmsg,
3839 .mmap = packet_mmap,
3840 .sendpage = sock_no_sendpage,
3841};
3842
ec1b4cf7 3843static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3844 .family = PF_PACKET,
3845 .create = packet_create,
3846 .owner = THIS_MODULE,
3847};
3848
3849static struct notifier_block packet_netdev_notifier = {
40d4e3df 3850 .notifier_call = packet_notifier,
1da177e4
LT		 3851};
3852
3853#ifdef CONFIG_PROC_FS
1da177e4
LT		 3854
3855static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3856 __acquires(RCU)
1da177e4 3857{
e372c414 3858 struct net *net = seq_file_net(seq);
808f5114 3859
3860 rcu_read_lock();
3861 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3862}
3863
3864static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3865{
1bf40954 3866 struct net *net = seq_file_net(seq);
808f5114 3867 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3868}
3869
3870static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3871 __releases(RCU)
1da177e4 3872{
808f5114 3873 rcu_read_unlock();
1da177e4
LT		 3874}
3875
1ce4f28b 3876static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3877{
3878 if (v == SEQ_START_TOKEN)
3879 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3880 else {
b7ceabd9 3881 struct sock *s = sk_entry(v);
1da177e4
LT		 3882 const struct packet_sock *po = pkt_sk(s);
3883
3884 seq_printf(seq,
71338aa7 3885 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3886 s,
3887 atomic_read(&s->sk_refcnt),
3888 s->sk_type,
3889 ntohs(po->num),
3890 po->ifindex,
3891 po->running,
3892 atomic_read(&s->sk_rmem_alloc),
3893 sock_i_uid(s),
40d4e3df 3894 sock_i_ino(s));
1da177e4
LT		 3895 }
3896
3897 return 0;
3898}
3899
56b3d975 3900static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3901 .start = packet_seq_start,
3902 .next = packet_seq_next,
3903 .stop = packet_seq_stop,
3904 .show = packet_seq_show,
3905};
3906
3907static int packet_seq_open(struct inode *inode, struct file *file)
3908{
e372c414
DL		 3909 return seq_open_net(inode, file, &packet_seq_ops,
3910 sizeof(struct seq_net_private));
1da177e4
LT		 3911}
3912
da7071d7 3913static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3914 .owner = THIS_MODULE,
3915 .open = packet_seq_open,
3916 .read = seq_read,
3917 .llseek = seq_lseek,
e372c414 3918 .release = seq_release_net,
1da177e4
LT		 3919};
3920
3921#endif
3922
2c8c1e72 3923static int __net_init packet_net_init(struct net *net)
d12d01d6 3924{
808f5114 3925 spin_lock_init(&net->packet.sklist_lock);
2aaef4e4 3926 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6
DL		 3927
3928 if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
3929 return -ENOMEM;
3930
3931 return 0;
3932}
3933
2c8c1e72 3934static void __net_exit packet_net_exit(struct net *net)
d12d01d6
DL		 3935{
3936 proc_net_remove(net, "packet");
3937}
3938
3939static struct pernet_operations packet_net_ops = {
3940 .init = packet_net_init,
3941 .exit = packet_net_exit,
3942};
3943
3944
1da177e4
LT		 3945static void __exit packet_exit(void)
3946{
1da177e4 3947 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3948 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3949 sock_unregister(PF_PACKET);
3950 proto_unregister(&packet_proto);
3951}
3952
3953static int __init packet_init(void)
3954{
3955 int rc = proto_register(&packet_proto, 0);
3956
3957 if (rc != 0)
3958 goto out;
3959
3960 sock_register(&packet_family_ops);
d12d01d6 3961 register_pernet_subsys(&packet_net_ops);
1da177e4 3962 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3963out:
3964 return rc;
3965}
3966
3967module_init(packet_init);
3968module_exit(packet_exit);
3969MODULE_LICENSE("GPL");
3970MODULE_ALIAS_NETPROTO(PF_PACKET);
kernel source for telekom puls (alcatel/mediatek)