projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'v3.10.107' into update
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
1da177e4
LT		 91
92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
c3ac8a13
DB		 240static struct net_device *packet_cached_dev_get(struct packet_sock *po)
241{
242 struct net_device *dev;
243
244 rcu_read_lock();
245 dev = rcu_dereference(po->cached_dev);
246 if (likely(dev))
247 dev_hold(dev);
248 rcu_read_unlock();
249
250 return dev;
251}
252
253static void packet_cached_dev_assign(struct packet_sock *po,
254 struct net_device *dev)
255{
256 rcu_assign_pointer(po->cached_dev, dev);
257}
258
259static void packet_cached_dev_reset(struct packet_sock *po)
260{
261 RCU_INIT_POINTER(po->cached_dev, NULL);
262}
263
ce06b03e
DM		 264/* register_prot_hook must be invoked with the po->bind_lock held,
265 * or from a context in which asynchronous accesses to the packet
266 * socket is not possible (packet_create()).
267 */
268static void register_prot_hook(struct sock *sk)
269{
270 struct packet_sock *po = pkt_sk(sk);
026bb405 271
ce06b03e 272 if (!po->running) {
c3ac8a13 273 if (po->fanout)
dc99f600 274 __fanout_link(sk, po);
c3ac8a13 275 else
dc99f600 276 dev_add_pack(&po->prot_hook);
026bb405 277
ce06b03e
DM		 278 sock_hold(sk);
279 po->running = 1;
280 }
281}
282
283/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
284 * held. If the sync parameter is true, we will temporarily drop
285 * the po->bind_lock and do a synchronize_net to make sure no
286 * asynchronous packet processing paths still refer to the elements
287 * of po->prot_hook. If the sync parameter is false, it is the
288 * callers responsibility to take care of this.
289 */
290static void __unregister_prot_hook(struct sock *sk, bool sync)
291{
292 struct packet_sock *po = pkt_sk(sk);
293
294 po->running = 0;
c3ac8a13
DB		 295
296 if (po->fanout)
dc99f600 297 __fanout_unlink(sk, po);
c3ac8a13 298 else
dc99f600 299 __dev_remove_pack(&po->prot_hook);
026bb405 300
ce06b03e
DM		 301 __sock_put(sk);
302
303 if (sync) {
304 spin_unlock(&po->bind_lock);
305 synchronize_net();
306 spin_lock(&po->bind_lock);
307 }
308}
309
310static void unregister_prot_hook(struct sock *sk, bool sync)
311{
312 struct packet_sock *po = pkt_sk(sk);
313
314 if (po->running)
315 __unregister_prot_hook(sk, sync);
316}
317
f6dafa95 318static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 319{
320 if (is_vmalloc_addr(addr))
321 return vmalloc_to_page(addr);
322 return virt_to_page(addr);
323}
324
69e3c75f 325static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 326{
184f489e 327 union tpacket_uhdr h;
1da177e4 328
69e3c75f 329 h.raw = frame;
bbd6ef87
PM		 330 switch (po->tp_version) {
331 case TPACKET_V1:
69e3c75f 332 h.h1->tp_status = status;
0af55bb5 333 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 334 break;
335 case TPACKET_V2:
69e3c75f 336 h.h2->tp_status = status;
0af55bb5 337 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 338 break;
f6fb8f10 339 case TPACKET_V3:
69e3c75f 340 default:
f6fb8f10 341 WARN(1, "TPACKET version not supported.\n");
69e3c75f 342 BUG();
bbd6ef87 343 }
69e3c75f
JB		 344
345 smp_wmb();
bbd6ef87
PM		 346}
347
69e3c75f 348static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 349{
184f489e 350 union tpacket_uhdr h;
bbd6ef87 351
69e3c75f
JB		 352 smp_rmb();
353
bbd6ef87
PM		 354 h.raw = frame;
355 switch (po->tp_version) {
356 case TPACKET_V1:
0af55bb5 357 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 358 return h.h1->tp_status;
bbd6ef87 359 case TPACKET_V2:
0af55bb5 360 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 361 return h.h2->tp_status;
f6fb8f10 362 case TPACKET_V3:
69e3c75f 363 default:
f6fb8f10 364 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 365 BUG();
366 return 0;
bbd6ef87 367 }
1da177e4 368}
69e3c75f 369
b9c32fb2
DB		 370static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
371 unsigned int flags)
7a51384c
DB		 372{
373 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
374
375 if (shhwtstamps) {
376 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
377 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 378 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 379 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
380 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 381 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 382 }
383
384 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 385 return TP_STATUS_TS_SOFTWARE;
7a51384c 386
b9c32fb2 387 return 0;
7a51384c
DB		 388}
389
b9c32fb2
DB		 390static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
391 struct sk_buff *skb)
2e31396f
WB		 392{
393 union tpacket_uhdr h;
394 struct timespec ts;
b9c32fb2 395 __u32 ts_status;
2e31396f 396
b9c32fb2
DB		 397 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
398 return 0;
2e31396f
WB		 399
400 h.raw = frame;
401 switch (po->tp_version) {
402 case TPACKET_V1:
403 h.h1->tp_sec = ts.tv_sec;
404 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
405 break;
406 case TPACKET_V2:
407 h.h2->tp_sec = ts.tv_sec;
408 h.h2->tp_nsec = ts.tv_nsec;
409 break;
410 case TPACKET_V3:
411 default:
412 WARN(1, "TPACKET version not supported.\n");
413 BUG();
414 }
415
416 /* one flush is safe, as both fields always lie on the same cacheline */
417 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
418 smp_wmb();
b9c32fb2
DB		 419
420 return ts_status;
2e31396f
WB		 421}
422
69e3c75f
JB		 423static void *packet_lookup_frame(struct packet_sock *po,
424 struct packet_ring_buffer *rb,
425 unsigned int position,
426 int status)
427{
428 unsigned int pg_vec_pos, frame_offset;
184f489e 429 union tpacket_uhdr h;
69e3c75f
JB		 430
431 pg_vec_pos = position / rb->frames_per_block;
432 frame_offset = position % rb->frames_per_block;
433
0e3125c7
NH		 434 h.raw = rb->pg_vec[pg_vec_pos].buffer +
435 (frame_offset * rb->frame_size);
69e3c75f
JB		 436
437 if (status != __packet_get_status(po, h.raw))
438 return NULL;
439
440 return h.raw;
441}
442
eea49cc9 443static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 444 struct packet_ring_buffer *rb,
445 int status)
446{
447 return packet_lookup_frame(po, rb, rb->head, status);
448}
449
bc59ba39 450static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 451{
452 del_timer_sync(&pkc->retire_blk_timer);
453}
454
455static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
456 int tx_ring,
457 struct sk_buff_head *rb_queue)
458{
bc59ba39 459 struct tpacket_kbdq_core *pkc;
f6fb8f10 460
461 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
462
5b9e9be7 463 spin_lock_bh(&rb_queue->lock);
f6fb8f10 464 pkc->delete_blk_timer = 1;
5b9e9be7 465 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 466
467 prb_del_retire_blk_timer(pkc);
468}
469
470static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 471 struct tpacket_kbdq_core *pkc,
f6fb8f10 472 void (*func) (unsigned long))
473{
474 init_timer(&pkc->retire_blk_timer);
475 pkc->retire_blk_timer.data = (long)po;
476 pkc->retire_blk_timer.function = func;
477 pkc->retire_blk_timer.expires = jiffies;
478}
479
480static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
481{
bc59ba39 482 struct tpacket_kbdq_core *pkc;
f6fb8f10 483
484 if (tx_ring)
485 BUG();
486
487 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
488 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
489}
490
491static int prb_calc_retire_blk_tmo(struct packet_sock *po,
492 int blk_size_in_bytes)
493{
494 struct net_device *dev;
495 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 496 struct ethtool_cmd ecmd;
497 int err;
e440cf2c 498 u32 speed;
f6fb8f10 499
4bc71cb9
JP		 500 rtnl_lock();
501 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
502 if (unlikely(!dev)) {
503 rtnl_unlock();
f6fb8f10 504 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 505 }
506 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 507 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 508 rtnl_unlock();
509 if (!err) {
4bc71cb9
JP		 510 /*
511 * If the link speed is so slow you don't really
512 * need to worry about perf anyways
513 */
e440cf2c 514 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 515 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 516 } else {
517 msec = 1;
518 div = speed / 1000;
f6fb8f10 519 }
520 }
521
522 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
523
524 if (div)
525 mbits /= div;
526
527 tmo = mbits * msec;
528
529 if (div)
530 return tmo+1;
531 return tmo;
532}
533
bc59ba39 534static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 535 union tpacket_req_u *req_u)
536{
537 p1->feature_req_word = req_u->req3.tp_feature_req_word;
538}
539
540static void init_prb_bdqc(struct packet_sock *po,
541 struct packet_ring_buffer *rb,
542 struct pgv *pg_vec,
543 union tpacket_req_u *req_u, int tx_ring)
544{
bc59ba39 545 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
546 struct tpacket_block_desc *pbd;
f6fb8f10 547
548 memset(p1, 0x0, sizeof(*p1));
549
550 p1->knxt_seq_num = 1;
551 p1->pkbdq = pg_vec;
bc59ba39 552 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 553 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 554 p1->kblk_size = req_u->req3.tp_block_size;
555 p1->knum_blocks = req_u->req3.tp_block_nr;
556 p1->hdrlen = po->tp_hdrlen;
557 p1->version = po->tp_version;
558 p1->last_kactive_blk_num = 0;
ee80fbf3 559 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 560 if (req_u->req3.tp_retire_blk_tov)
561 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
562 else
563 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
564 req_u->req3.tp_block_size);
565 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
566 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
567
4035ed7b 568 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 569 prb_init_ft_ops(p1, req_u);
570 prb_setup_retire_blk_timer(po, tx_ring);
571 prb_open_block(p1, pbd);
572}
573
574/* Do NOT update the last_blk_num first.
575 * Assumes sk_buff_head lock is held.
576 */
bc59ba39 577static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 578{
579 mod_timer(&pkc->retire_blk_timer,
580 jiffies + pkc->tov_in_jiffies);
581 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
582}
583
584/*
585 * Timer logic:
586 * 1) We refresh the timer only when we open a block.
587 * By doing this we don't waste cycles refreshing the timer
588 * on packet-by-packet basis.
589 *
590 * With a 1MB block-size, on a 1Gbps line, it will take
591 * i) ~8 ms to fill a block + ii) memcpy etc.
592 * In this cut we are not accounting for the memcpy time.
593 *
594 * So, if the user sets the 'tmo' to 10ms then the timer
595 * will never fire while the block is still getting filled
596 * (which is what we want). However, the user could choose
597 * to close a block early and that's fine.
598 *
599 * But when the timer does fire, we check whether or not to refresh it.
600 * Since the tmo granularity is in msecs, it is not too expensive
601 * to refresh the timer, lets say every '8' msecs.
602 * Either the user can set the 'tmo' or we can derive it based on
603 * a) line-speed and b) block-size.
604 * prb_calc_retire_blk_tmo() calculates the tmo.
605 *
606 */
607static void prb_retire_rx_blk_timer_expired(unsigned long data)
608{
609 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 610 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 611 unsigned int frozen;
bc59ba39 612 struct tpacket_block_desc *pbd;
f6fb8f10 613
614 spin_lock(&po->sk.sk_receive_queue.lock);
615
616 frozen = prb_queue_frozen(pkc);
617 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
618
619 if (unlikely(pkc->delete_blk_timer))
620 goto out;
621
622 /* We only need to plug the race when the block is partially filled.
623 * tpacket_rcv:
624 * lock(); increment BLOCK_NUM_PKTS; unlock()
625 * copy_bits() is in progress ...
626 * timer fires on other cpu:
627 * we can't retire the current block because copy_bits
628 * is in progress.
629 *
630 */
631 if (BLOCK_NUM_PKTS(pbd)) {
632 while (atomic_read(&pkc->blk_fill_in_prog)) {
633 /* Waiting for skb_copy_bits to finish... */
634 cpu_relax();
635 }
636 }
637
638 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
639 if (!frozen) {
640 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
641 if (!prb_dispatch_next_block(pkc, po))
642 goto refresh_timer;
643 else
644 goto out;
645 } else {
646 /* Case 1. Queue was frozen because user-space was
647 * lagging behind.
648 */
649 if (prb_curr_blk_in_use(pkc, pbd)) {
650 /*
651 * Ok, user-space is still behind.
652 * So just refresh the timer.
653 */
654 goto refresh_timer;
655 } else {
656 /* Case 2. queue was frozen,user-space caught up,
657 * now the link went idle && the timer fired.
658 * We don't have a block to close.So we open this
659 * block and restart the timer.
660 * opening a block thaws the queue,restarts timer
661 * Thawing/timer-refresh is a side effect.
662 */
663 prb_open_block(pkc, pbd);
664 goto out;
665 }
666 }
667 }
668
669refresh_timer:
670 _prb_refresh_rx_retire_blk_timer(pkc);
671
672out:
673 spin_unlock(&po->sk.sk_receive_queue.lock);
674}
675
eea49cc9 676static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 677 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 678{
679 /* Flush everything minus the block header */
680
681#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
682 u8 *start, *end;
683
684 start = (u8 *)pbd1;
685
686 /* Skip the block header(we know header WILL fit in 4K) */
687 start += PAGE_SIZE;
688
689 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
690 for (; start < end; start += PAGE_SIZE)
691 flush_dcache_page(pgv_to_page(start));
692
693 smp_wmb();
694#endif
695
696 /* Now update the block status. */
697
698 BLOCK_STATUS(pbd1) = status;
699
700 /* Flush the block header */
701
702#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
703 start = (u8 *)pbd1;
704 flush_dcache_page(pgv_to_page(start));
705
706 smp_wmb();
707#endif
708}
709
710/*
711 * Side effect:
712 *
713 * 1) flush the block
714 * 2) Increment active_blk_num
715 *
716 * Note:We DONT refresh the timer on purpose.
717 * Because almost always the next block will be opened.
718 */
bc59ba39 719static void prb_close_block(struct tpacket_kbdq_core *pkc1,
720 struct tpacket_block_desc *pbd1,
f6fb8f10 721 struct packet_sock *po, unsigned int stat)
722{
723 __u32 status = TP_STATUS_USER | stat;
724
725 struct tpacket3_hdr *last_pkt;
bc59ba39 726 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 727
ee80fbf3 728 if (po->stats.stats3.tp_drops)
f6fb8f10 729 status |= TP_STATUS_LOSING;
730
731 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
732 last_pkt->tp_next_offset = 0;
733
734 /* Get the ts of the last pkt */
735 if (BLOCK_NUM_PKTS(pbd1)) {
736 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
737 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
738 } else {
739 /* Ok, we tmo'd - so get the current time */
740 struct timespec ts;
741 getnstimeofday(&ts);
742 h1->ts_last_pkt.ts_sec = ts.tv_sec;
743 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
744 }
745
746 smp_wmb();
747
748 /* Flush the block */
749 prb_flush_block(pkc1, pbd1, status);
750
751 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
752}
753
eea49cc9 754static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 755{
756 pkc->reset_pending_on_curr_blk = 0;
757}
758
759/*
760 * Side effect of opening a block:
761 *
762 * 1) prb_queue is thawed.
763 * 2) retire_blk_timer is refreshed.
764 *
765 */
bc59ba39 766static void prb_open_block(struct tpacket_kbdq_core *pkc1,
767 struct tpacket_block_desc *pbd1)
f6fb8f10 768{
769 struct timespec ts;
bc59ba39 770 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 771
772 smp_rmb();
773
8da3056c
DB		 774 /* We could have just memset this but we will lose the
775 * flexibility of making the priv area sticky
776 */
f6fb8f10 777
8da3056c
DB		 778 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
779 BLOCK_NUM_PKTS(pbd1) = 0;
780 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 781
8da3056c
DB		 782 getnstimeofday(&ts);
783
784 h1->ts_first_pkt.ts_sec = ts.tv_sec;
785 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 786
8da3056c
DB		 787 pkc1->pkblk_start = (char *)pbd1;
788 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
789
790 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
791 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
792
793 pbd1->version = pkc1->version;
794 pkc1->prev = pkc1->nxt_offset;
795 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
796
797 prb_thaw_queue(pkc1);
798 _prb_refresh_rx_retire_blk_timer(pkc1);
799
800 smp_wmb();
f6fb8f10 801}
802
803/*
804 * Queue freeze logic:
805 * 1) Assume tp_block_nr = 8 blocks.
806 * 2) At time 't0', user opens Rx ring.
807 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
808 * 4) user-space is either sleeping or processing block '0'.
809 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
810 * it will close block-7,loop around and try to fill block '0'.
811 * call-flow:
812 * __packet_lookup_frame_in_block
813 * prb_retire_current_block()
814 * prb_dispatch_next_block()
815 * |->(BLOCK_STATUS == USER) evaluates to true
816 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
817 * 6) Now there are two cases:
818 * 6.1) Link goes idle right after the queue is frozen.
819 * But remember, the last open_block() refreshed the timer.
820 * When this timer expires,it will refresh itself so that we can
821 * re-open block-0 in near future.
822 * 6.2) Link is busy and keeps on receiving packets. This is a simple
823 * case and __packet_lookup_frame_in_block will check if block-0
824 * is free and can now be re-used.
825 */
eea49cc9 826static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 827 struct packet_sock *po)
828{
829 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 830 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 831}
832
833#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
834
835/*
836 * If the next block is free then we will dispatch it
837 * and return a good offset.
838 * Else, we will freeze the queue.
839 * So, caller must check the return value.
840 */
bc59ba39 841static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 842 struct packet_sock *po)
843{
bc59ba39 844 struct tpacket_block_desc *pbd;
f6fb8f10 845
846 smp_rmb();
847
848 /* 1. Get current block num */
849 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
850
851 /* 2. If this block is currently in_use then freeze the queue */
852 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
853 prb_freeze_queue(pkc, po);
854 return NULL;
855 }
856
857 /*
858 * 3.
859 * open this block and return the offset where the first packet
860 * needs to get stored.
861 */
862 prb_open_block(pkc, pbd);
863 return (void *)pkc->nxt_offset;
864}
865
bc59ba39 866static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 867 struct packet_sock *po, unsigned int status)
868{
bc59ba39 869 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 870
871 /* retire/close the current block */
872 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
873 /*
874 * Plug the case where copy_bits() is in progress on
875 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
876 * have space to copy the pkt in the current block and
877 * called prb_retire_current_block()
878 *
879 * We don't need to worry about the TMO case because
880 * the timer-handler already handled this case.
881 */
882 if (!(status & TP_STATUS_BLK_TMO)) {
883 while (atomic_read(&pkc->blk_fill_in_prog)) {
884 /* Waiting for skb_copy_bits to finish... */
885 cpu_relax();
886 }
887 }
888 prb_close_block(pkc, pbd, po, status);
889 return;
890 }
f6fb8f10 891}
892
eea49cc9 893static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 894 struct tpacket_block_desc *pbd)
f6fb8f10 895{
896 return TP_STATUS_USER & BLOCK_STATUS(pbd);
897}
898
eea49cc9 899static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 900{
901 return pkc->reset_pending_on_curr_blk;
902}
903
eea49cc9 904static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 905{
bc59ba39 906 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 907 atomic_dec(&pkc->blk_fill_in_prog);
908}
909
eea49cc9 910static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 911 struct tpacket3_hdr *ppd)
912{
913 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
914}
915
eea49cc9 916static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 917 struct tpacket3_hdr *ppd)
918{
919 ppd->hv1.tp_rxhash = 0;
920}
921
eea49cc9 922static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 923 struct tpacket3_hdr *ppd)
924{
925 if (vlan_tx_tag_present(pkc->skb)) {
926 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
927 ppd->tp_status = TP_STATUS_VLAN_VALID;
928 } else {
9e67030a 929 ppd->hv1.tp_vlan_tci = 0;
930 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 931 }
932}
933
bc59ba39 934static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 935 struct tpacket3_hdr *ppd)
936{
937 prb_fill_vlan_info(pkc, ppd);
938
939 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
940 prb_fill_rxhash(pkc, ppd);
941 else
942 prb_clear_rxhash(pkc, ppd);
943}
944
eea49cc9 945static void prb_fill_curr_block(char *curr,
bc59ba39 946 struct tpacket_kbdq_core *pkc,
947 struct tpacket_block_desc *pbd,
f6fb8f10 948 unsigned int len)
949{
950 struct tpacket3_hdr *ppd;
951
952 ppd = (struct tpacket3_hdr *)curr;
953 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
954 pkc->prev = curr;
955 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
956 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
957 BLOCK_NUM_PKTS(pbd) += 1;
958 atomic_inc(&pkc->blk_fill_in_prog);
959 prb_run_all_ft_ops(pkc, ppd);
960}
961
962/* Assumes caller has the sk->rx_queue.lock */
963static void *__packet_lookup_frame_in_block(struct packet_sock *po,
964 struct sk_buff *skb,
965 int status,
966 unsigned int len
967 )
968{
bc59ba39 969 struct tpacket_kbdq_core *pkc;
970 struct tpacket_block_desc *pbd;
f6fb8f10 971 char *curr, *end;
972
e3192690 973 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 974 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
975
976 /* Queue is frozen when user space is lagging behind */
977 if (prb_queue_frozen(pkc)) {
978 /*
979 * Check if that last block which caused the queue to freeze,
980 * is still in_use by user-space.
981 */
982 if (prb_curr_blk_in_use(pkc, pbd)) {
983 /* Can't record this packet */
984 return NULL;
985 } else {
986 /*
987 * Ok, the block was released by user-space.
988 * Now let's open that block.
989 * opening a block also thaws the queue.
990 * Thawing is a side effect.
991 */
992 prb_open_block(pkc, pbd);
993 }
994 }
995
996 smp_mb();
997 curr = pkc->nxt_offset;
998 pkc->skb = skb;
e3192690 999 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1000
1001 /* first try the current block */
1002 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1003 prb_fill_curr_block(curr, pkc, pbd, len);
1004 return (void *)curr;
1005 }
1006
1007 /* Ok, close the current block */
1008 prb_retire_current_block(pkc, po, 0);
1009
1010 /* Now, try to dispatch the next block */
1011 curr = (char *)prb_dispatch_next_block(pkc, po);
1012 if (curr) {
1013 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1014 prb_fill_curr_block(curr, pkc, pbd, len);
1015 return (void *)curr;
1016 }
1017
1018 /*
1019 * No free blocks are available.user_space hasn't caught up yet.
1020 * Queue was just frozen and now this packet will get dropped.
1021 */
1022 return NULL;
1023}
1024
eea49cc9 1025static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1026 struct sk_buff *skb,
1027 int status, unsigned int len)
1028{
1029 char *curr = NULL;
1030 switch (po->tp_version) {
1031 case TPACKET_V1:
1032 case TPACKET_V2:
1033 curr = packet_lookup_frame(po, &po->rx_ring,
1034 po->rx_ring.head, status);
1035 return curr;
1036 case TPACKET_V3:
1037 return __packet_lookup_frame_in_block(po, skb, status, len);
1038 default:
1039 WARN(1, "TPACKET version not supported\n");
1040 BUG();
99aa3473 1041 return NULL;
f6fb8f10 1042 }
1043}
1044
eea49cc9 1045static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1046 struct packet_ring_buffer *rb,
77f65ebd 1047 unsigned int idx,
f6fb8f10 1048 int status)
1049{
bc59ba39 1050 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1051 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1052
1053 if (status != BLOCK_STATUS(pbd))
1054 return NULL;
1055 return pbd;
1056}
1057
eea49cc9 1058static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1059{
1060 unsigned int prev;
1061 if (rb->prb_bdqc.kactive_blk_num)
1062 prev = rb->prb_bdqc.kactive_blk_num-1;
1063 else
1064 prev = rb->prb_bdqc.knum_blocks-1;
1065 return prev;
1066}
1067
1068/* Assumes caller has held the rx_queue.lock */
eea49cc9 1069static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1070 struct packet_ring_buffer *rb,
1071 int status)
1072{
1073 unsigned int previous = prb_previous_blk_num(rb);
1074 return prb_lookup_block(po, rb, previous, status);
1075}
1076
eea49cc9 1077static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1078 struct packet_ring_buffer *rb,
1079 int status)
1080{
1081 if (po->tp_version <= TPACKET_V2)
1082 return packet_previous_frame(po, rb, status);
1083
1084 return __prb_previous_block(po, rb, status);
1085}
1086
eea49cc9 1087static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1088 struct packet_ring_buffer *rb)
1089{
1090 switch (po->tp_version) {
1091 case TPACKET_V1:
1092 case TPACKET_V2:
1093 return packet_increment_head(rb);
1094 case TPACKET_V3:
1095 default:
1096 WARN(1, "TPACKET version not supported.\n");
1097 BUG();
1098 return;
1099 }
1100}
1101
eea49cc9 1102static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1103 struct packet_ring_buffer *rb,
1104 int status)
1105{
1106 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1107 return packet_lookup_frame(po, rb, previous, status);
1108}
1109
eea49cc9 1110static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1111{
1112 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1113}
1114
77f65ebd
WB		 1115static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1116{
1117 struct sock *sk = &po->sk;
1118 bool has_room;
1119
1120 if (po->prot_hook.func != tpacket_rcv)
1121 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1122 <= sk->sk_rcvbuf;
1123
1124 spin_lock(&sk->sk_receive_queue.lock);
1125 if (po->tp_version == TPACKET_V3)
1126 has_room = prb_lookup_block(po, &po->rx_ring,
1127 po->rx_ring.prb_bdqc.kactive_blk_num,
1128 TP_STATUS_KERNEL);
1129 else
1130 has_room = packet_lookup_frame(po, &po->rx_ring,
1131 po->rx_ring.head,
1132 TP_STATUS_KERNEL);
1133 spin_unlock(&sk->sk_receive_queue.lock);
1134
1135 return has_room;
1136}
1137
1da177e4
LT		 1138static void packet_sock_destruct(struct sock *sk)
1139{
ed85b565
RC		 1140 skb_queue_purge(&sk->sk_error_queue);
1141
547b792c
IJ		 1142 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1143 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1144
1145 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1146 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1147 return;
1148 }
1149
17ab56a2 1150 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1151}
1152
77f65ebd
WB		 1153static unsigned int fanout_demux_hash(struct packet_fanout *f,
1154 struct sk_buff *skb,
1155 unsigned int num)
dc99f600 1156{
77f65ebd 1157 return (((u64)skb->rxhash) * num) >> 32;
dc99f600
DM		 1158}
1159
77f65ebd
WB		 1160static unsigned int fanout_demux_lb(struct packet_fanout *f,
1161 struct sk_buff *skb,
1162 unsigned int num)
dc99f600 1163{
4552ddd0 1164 unsigned int val = atomic_inc_return(&f->rr_cur);
dc99f600 1165
4552ddd0 1166 return val % num;
77f65ebd
WB		 1167}
1168
1169static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1170 struct sk_buff *skb,
1171 unsigned int num)
1172{
1173 return smp_processor_id() % num;
dc99f600
DM		 1174}
1175
77f65ebd
WB		 1176static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1177 struct sk_buff *skb,
1178 unsigned int idx, unsigned int skip,
1179 unsigned int num)
95ec3eb4 1180{
77f65ebd 1181 unsigned int i, j;
95ec3eb4 1182
77f65ebd
WB		 1183 i = j = min_t(int, f->next[idx], num - 1);
1184 do {
1185 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1186 if (i != j)
1187 f->next[idx] = i;
1188 return i;
1189 }
1190 if (++i == num)
1191 i = 0;
1192 } while (i != j);
1193
1194 return idx;
1195}
1196
1197static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1198{
1199 return f->flags & (flag >> 8);
95ec3eb4
DM		 1200}
1201
95ec3eb4
DM		 1202static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1203 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1204{
1205 struct packet_fanout *f = pt->af_packet_priv;
7eee92bf 1206 unsigned int num = ACCESS_ONCE(f->num_members);
dc99f600 1207 struct packet_sock *po;
77f65ebd 1208 unsigned int idx;
dc99f600
DM		 1209
1210 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1211 !num) {
1212 kfree_skb(skb);
1213 return 0;
1214 }
1215
95ec3eb4
DM		 1216 switch (f->type) {
1217 case PACKET_FANOUT_HASH:
1218 default:
77f65ebd 1219 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1220 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1221 if (!skb)
1222 return 0;
1223 }
1224 skb_get_rxhash(skb);
77f65ebd 1225 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1226 break;
1227 case PACKET_FANOUT_LB:
77f65ebd 1228 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1229 break;
1230 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1231 idx = fanout_demux_cpu(f, skb, num);
1232 break;
1233 case PACKET_FANOUT_ROLLOVER:
1234 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1235 break;
dc99f600
DM		 1236 }
1237
77f65ebd
WB		 1238 po = pkt_sk(f->arr[idx]);
1239 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1240 unlikely(!packet_rcv_has_room(po, skb))) {
1241 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1242 po = pkt_sk(f->arr[idx]);
1243 }
dc99f600
DM		 1244
1245 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1246}
1247
fff3321d
PE		 1248DEFINE_MUTEX(fanout_mutex);
1249EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1250static LIST_HEAD(fanout_list);
1251
1252static void __fanout_link(struct sock *sk, struct packet_sock *po)
1253{
1254 struct packet_fanout *f = po->fanout;
1255
1256 spin_lock(&f->lock);
1257 f->arr[f->num_members] = sk;
1258 smp_wmb();
1259 f->num_members++;
9aab3592
AS		 1260 if (f->num_members == 1)
1261 dev_add_pack(&f->prot_hook);
dc99f600
DM		 1262 spin_unlock(&f->lock);
1263}
1264
1265static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1266{
1267 struct packet_fanout *f = po->fanout;
1268 int i;
1269
1270 spin_lock(&f->lock);
1271 for (i = 0; i < f->num_members; i++) {
1272 if (f->arr[i] == sk)
1273 break;
1274 }
1275 BUG_ON(i >= f->num_members);
1276 f->arr[i] = f->arr[f->num_members - 1];
1277 f->num_members--;
9aab3592
AS		 1278 if (f->num_members == 0)
1279 __dev_remove_pack(&f->prot_hook);
dc99f600
DM		 1280 spin_unlock(&f->lock);
1281}
1282
a0dfb263 1283static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1284{
1285 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1286 return true;
1287
1288 return false;
1289}
1290
7736d33f 1291static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1292{
1293 struct packet_sock *po = pkt_sk(sk);
1294 struct packet_fanout *f, *match;
7736d33f 1295 u8 type = type_flags & 0xff;
77f65ebd 1296 u8 flags = type_flags >> 8;
dc99f600
DM		 1297 int err;
1298
1299 switch (type) {
77f65ebd
WB		 1300 case PACKET_FANOUT_ROLLOVER:
1301 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1302 return -EINVAL;
dc99f600
DM		 1303 case PACKET_FANOUT_HASH:
1304 case PACKET_FANOUT_LB:
95ec3eb4 1305 case PACKET_FANOUT_CPU:
dc99f600
DM		 1306 break;
1307 default:
1308 return -EINVAL;
1309 }
1310
2a272abc
ED		 1311 mutex_lock(&fanout_mutex);
1312
1313 err = -EINVAL;
dc99f600 1314 if (!po->running)
2a272abc 1315 goto out;
dc99f600 1316
2a272abc 1317 err = -EALREADY;
dc99f600 1318 if (po->fanout)
2a272abc 1319 goto out;
dc99f600 1320
dc99f600
DM		 1321 match = NULL;
1322 list_for_each_entry(f, &fanout_list, list) {
1323 if (f->id == id &&
1324 read_pnet(&f->net) == sock_net(sk)) {
1325 match = f;
1326 break;
1327 }
1328 }
afe62c68 1329 err = -EINVAL;
77f65ebd 1330 if (match && match->flags != flags)
afe62c68 1331 goto out;
dc99f600 1332 if (!match) {
afe62c68 1333 err = -ENOMEM;
dc99f600 1334 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1335 if (!match)
1336 goto out;
1337 write_pnet(&match->net, sock_net(sk));
1338 match->id = id;
1339 match->type = type;
77f65ebd 1340 match->flags = flags;
afe62c68
ED		 1341 atomic_set(&match->rr_cur, 0);
1342 INIT_LIST_HEAD(&match->list);
1343 spin_lock_init(&match->lock);
1344 atomic_set(&match->sk_ref, 0);
1345 match->prot_hook.type = po->prot_hook.type;
1346 match->prot_hook.dev = po->prot_hook.dev;
1347 match->prot_hook.func = packet_rcv_fanout;
1348 match->prot_hook.af_packet_priv = match;
c0de08d0 1349 match->prot_hook.id_match = match_fanout_group;
afe62c68 1350 list_add(&match->list, &fanout_list);
dc99f600 1351 }
afe62c68
ED		 1352 err = -EINVAL;
1353 if (match->type == type &&
1354 match->prot_hook.type == po->prot_hook.type &&
1355 match->prot_hook.dev == po->prot_hook.dev) {
1356 err = -ENOSPC;
1357 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1358 __dev_remove_pack(&po->prot_hook);
1359 po->fanout = match;
1360 atomic_inc(&match->sk_ref);
1361 __fanout_link(sk, po);
1362 err = 0;
dc99f600
DM		 1363 }
1364 }
afe62c68 1365out:
dc99f600
DM		 1366 mutex_unlock(&fanout_mutex);
1367 return err;
1368}
1369
9aab3592
AS		 1370/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1371 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1372 * It is the responsibility of the caller to call fanout_release_data() and
1373 * free the returned packet_fanout (after synchronize_net())
1374 */
1375static struct packet_fanout *fanout_release(struct sock *sk)
dc99f600
DM		 1376{
1377 struct packet_sock *po = pkt_sk(sk);
1378 struct packet_fanout *f;
1379
fff3321d 1380 mutex_lock(&fanout_mutex);
2a272abc
ED		 1381 f = po->fanout;
1382 if (f) {
1383 po->fanout = NULL;
dc99f600 1384
9aab3592 1385 if (atomic_dec_and_test(&f->sk_ref))
2a272abc 1386 list_del(&f->list);
9aab3592
AS		 1387 else
1388 f = NULL;
dc99f600
DM		 1389 }
1390 mutex_unlock(&fanout_mutex);
9aab3592
AS		 1391
1392 return f;
dc99f600 1393}
1da177e4 1394
90ddc4f0 1395static const struct proto_ops packet_ops;
1da177e4 1396
90ddc4f0 1397static const struct proto_ops packet_ops_spkt;
1da177e4 1398
40d4e3df
ED		 1399static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1400 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1401{
1402 struct sock *sk;
1403 struct sockaddr_pkt *spkt;
1404
1405 /*
1406 * When we registered the protocol we saved the socket in the data
1407 * field for just this event.
1408 */
1409
1410 sk = pt->af_packet_priv;
1ce4f28b 1411
1da177e4
LT		 1412 /*
1413 * Yank back the headers [hope the device set this
1414 * right or kerboom...]
1415 *
1416 * Incoming packets have ll header pulled,
1417 * push it back.
1418 *
98e399f8 1419 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1420 * so that this procedure is noop.
1421 */
1422
1423 if (skb->pkt_type == PACKET_LOOPBACK)
1424 goto out;
1425
09ad9bc7 1426 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1427 goto out;
1428
40d4e3df
ED		 1429 skb = skb_share_check(skb, GFP_ATOMIC);
1430 if (skb == NULL)
1da177e4
LT		 1431 goto oom;
1432
1433 /* drop any routing info */
adf30907 1434 skb_dst_drop(skb);
1da177e4 1435
84531c24
PO		 1436 /* drop conntrack reference */
1437 nf_reset(skb);
1438
ffbc6111 1439 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1440
98e399f8 1441 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1442
1443 /*
1444 * The SOCK_PACKET socket receives _all_ frames.
1445 */
1446
1447 spkt->spkt_family = dev->type;
1448 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1449 spkt->spkt_protocol = skb->protocol;
1450
1451 /*
1452 * Charge the memory to the socket. This is done specifically
1453 * to prevent sockets using all the memory up.
1454 */
1455
40d4e3df 1456 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1457 return 0;
1458
1459out:
1460 kfree_skb(skb);
1461oom:
1462 return 0;
1463}
1464
1465
1466/*
1467 * Output a raw packet to a device layer. This bypasses all the other
1468 * protocol layers and you must therefore supply it with a complete frame
1469 */
1ce4f28b 1470
1da177e4
LT		 1471static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1472 struct msghdr *msg, size_t len)
1473{
1474 struct sock *sk = sock->sk;
40d4e3df 1475 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1476 struct sk_buff *skb = NULL;
1da177e4 1477 struct net_device *dev;
40d4e3df 1478 __be16 proto = 0;
1da177e4 1479 int err;
3bdc0eba 1480 int extra_len = 0;
1ce4f28b 1481
1da177e4 1482 /*
1ce4f28b 1483 * Get and verify the address.
1da177e4
LT		 1484 */
1485
40d4e3df 1486 if (saddr) {
1da177e4 1487 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1488 return -EINVAL;
1489 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1490 proto = saddr->spkt_protocol;
1491 } else
1492 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1493
1494 /*
1ce4f28b 1495 * Find the device first to size check it
1da177e4
LT		 1496 */
1497
de74e92a 1498 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1499retry:
654d1f8a
ED		 1500 rcu_read_lock();
1501 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1502 err = -ENODEV;
1503 if (dev == NULL)
1504 goto out_unlock;
1ce4f28b 1505
d5e76b0a
DM		 1506 err = -ENETDOWN;
1507 if (!(dev->flags & IFF_UP))
1508 goto out_unlock;
1509
1da177e4 1510 /*
40d4e3df
ED		 1511 * You may not queue a frame bigger than the mtu. This is the lowest level
1512 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1513 */
1ce4f28b 1514
3bdc0eba
BG		 1515 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1516 if (!netif_supports_nofcs(dev)) {
1517 err = -EPROTONOSUPPORT;
1518 goto out_unlock;
1519 }
1520 extra_len = 4; /* We're doing our own CRC */
1521 }
1522
1da177e4 1523 err = -EMSGSIZE;
3bdc0eba 1524 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1525 goto out_unlock;
1526
1a35ca80
ED		 1527 if (!skb) {
1528 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1529 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1530 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1531
1532 rcu_read_unlock();
4ce40912 1533 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1534 if (skb == NULL)
1535 return -ENOBUFS;
1536 /* FIXME: Save some space for broken drivers that write a hard
1537 * header at transmission time by themselves. PPP is the notable
1538 * one here. This should really be fixed at the driver level.
1539 */
1540 skb_reserve(skb, reserved);
1541 skb_reset_network_header(skb);
1542
1543 /* Try to align data part correctly */
1544 if (hhlen) {
1545 skb->data -= hhlen;
1546 skb->tail -= hhlen;
1547 if (len < hhlen)
1548 skb_reset_network_header(skb);
1549 }
1550 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1551 if (err)
1552 goto out_free;
1553 goto retry;
1da177e4
LT		 1554 }
1555
3bdc0eba 1556 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1557 /* Earlier code assumed this would be a VLAN pkt,
1558 * double-check this now that we have the actual
1559 * packet in hand.
1560 */
1561 struct ethhdr *ehdr;
1562 skb_reset_mac_header(skb);
1563 ehdr = eth_hdr(skb);
1564 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1565 err = -EMSGSIZE;
1566 goto out_unlock;
1567 }
1568 }
1a35ca80 1569
1da177e4
LT		 1570 skb->protocol = proto;
1571 skb->dev = dev;
1572 skb->priority = sk->sk_priority;
2d37a186 1573 skb->mark = sk->sk_mark;
bf84a010
DB		 1574
1575 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1576
3bdc0eba
BG		 1577 if (unlikely(extra_len == 4))
1578 skb->no_fcs = 1;
1579
40893fd0 1580 skb_probe_transport_header(skb, 0);
c1aad275 1581
1da177e4 1582 dev_queue_xmit(skb);
654d1f8a 1583 rcu_read_unlock();
40d4e3df 1584 return len;
1da177e4 1585
1da177e4 1586out_unlock:
654d1f8a 1587 rcu_read_unlock();
1a35ca80
ED		 1588out_free:
1589 kfree_skb(skb);
1da177e4
LT		 1590 return err;
1591}
1da177e4 1592
eea49cc9 1593static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1594 const struct sock *sk,
dbcb5855 1595 unsigned int res)
1da177e4
LT		 1596{
1597 struct sk_filter *filter;
fda9ef5d 1598
80f8f102
ED		 1599 rcu_read_lock();
1600 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1601 if (filter != NULL)
0a14842f 1602 res = SK_RUN_FILTER(filter, skb);
80f8f102 1603 rcu_read_unlock();
1da177e4 1604
dbcb5855 1605 return res;
1da177e4
LT		 1606}
1607
1608/*
62ab0812
ED		 1609 * This function makes lazy skb cloning in hope that most of packets
1610 * are discarded by BPF.
1611 *
1612 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1613 * and skb->cb are mangled. It works because (and until) packets
1614 * falling here are owned by current CPU. Output packets are cloned
1615 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1616 * sequencially, so that if we return skb to original state on exit,
1617 * we will not harm anyone.
1da177e4
LT		 1618 */
1619
40d4e3df
ED		 1620static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1621 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1622{
1623 struct sock *sk;
1624 struct sockaddr_ll *sll;
1625 struct packet_sock *po;
40d4e3df 1626 u8 *skb_head = skb->data;
1da177e4 1627 int skb_len = skb->len;
dbcb5855 1628 unsigned int snaplen, res;
1da177e4
LT		 1629
1630 if (skb->pkt_type == PACKET_LOOPBACK)
1631 goto drop;
1632
1633 sk = pt->af_packet_priv;
1634 po = pkt_sk(sk);
1635
09ad9bc7 1636 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1637 goto drop;
1638
1da177e4
LT		 1639 skb->dev = dev;
1640
3b04ddde 1641 if (dev->header_ops) {
1da177e4 1642 /* The device has an explicit notion of ll header,
62ab0812
ED		 1643 * exported to higher levels.
1644 *
1645 * Otherwise, the device hides details of its frame
1646 * structure, so that corresponding packet head is
1647 * never delivered to user.
1da177e4
LT		 1648 */
1649 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1650 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1651 else if (skb->pkt_type == PACKET_OUTGOING) {
1652 /* Special case: outgoing packets have ll header at head */
bbe735e4 1653 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1654 }
1655 }
1656
1657 snaplen = skb->len;
1658
dbcb5855
DM		 1659 res = run_filter(skb, sk, snaplen);
1660 if (!res)
fda9ef5d 1661 goto drop_n_restore;
dbcb5855
DM		 1662 if (snaplen > res)
1663 snaplen = res;
1da177e4 1664
0fd7bac6 1665 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1666 goto drop_n_acct;
1667
1668 if (skb_shared(skb)) {
1669 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1670 if (nskb == NULL)
1671 goto drop_n_acct;
1672
1673 if (skb_head != skb->data) {
1674 skb->data = skb_head;
1675 skb->len = skb_len;
1676 }
abc4e4fa 1677 consume_skb(skb);
1da177e4
LT		 1678 skb = nskb;
1679 }
1680
ffbc6111
HX		 1681 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1682 sizeof(skb->cb));
1683
1684 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1685 sll->sll_family = AF_PACKET;
1686 sll->sll_hatype = dev->type;
1687 sll->sll_protocol = skb->protocol;
1688 sll->sll_pkttype = skb->pkt_type;
8032b464 1689 if (unlikely(po->origdev))
80feaacb
PWJ		 1690 sll->sll_ifindex = orig_dev->ifindex;
1691 else
1692 sll->sll_ifindex = dev->ifindex;
1da177e4 1693
b95cce35 1694 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1695
ffbc6111 1696 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1697
1da177e4
LT		 1698 if (pskb_trim(skb, snaplen))
1699 goto drop_n_acct;
1700
1701 skb_set_owner_r(skb, sk);
1702 skb->dev = NULL;
adf30907 1703 skb_dst_drop(skb);
1da177e4 1704
84531c24
PO		 1705 /* drop conntrack reference */
1706 nf_reset(skb);
1707
1da177e4 1708 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1709 po->stats.stats1.tp_packets++;
3b885787 1710 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1711 __skb_queue_tail(&sk->sk_receive_queue, skb);
1712 spin_unlock(&sk->sk_receive_queue.lock);
1713 sk->sk_data_ready(sk, skb->len);
1714 return 0;
1715
1716drop_n_acct:
7091fbd8 1717 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1718 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1719 atomic_inc(&sk->sk_drops);
1720 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1721
1722drop_n_restore:
1723 if (skb_head != skb->data && skb_shared(skb)) {
1724 skb->data = skb_head;
1725 skb->len = skb_len;
1726 }
1727drop:
ead2ceb0 1728 consume_skb(skb);
1da177e4
LT		 1729 return 0;
1730}
1731
40d4e3df
ED		 1732static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1733 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1734{
1735 struct sock *sk;
1736 struct packet_sock *po;
1737 struct sockaddr_ll *sll;
184f489e 1738 union tpacket_uhdr h;
40d4e3df 1739 u8 *skb_head = skb->data;
1da177e4 1740 int skb_len = skb->len;
dbcb5855 1741 unsigned int snaplen, res;
f6fb8f10 1742 unsigned long status = TP_STATUS_USER;
bbd6ef87 1743 unsigned short macoff, netoff, hdrlen;
1da177e4 1744 struct sk_buff *copy_skb = NULL;
bbd6ef87 1745 struct timespec ts;
b9c32fb2 1746 __u32 ts_status;
1da177e4
LT		 1747
1748 if (skb->pkt_type == PACKET_LOOPBACK)
1749 goto drop;
1750
1751 sk = pt->af_packet_priv;
1752 po = pkt_sk(sk);
1753
09ad9bc7 1754 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1755 goto drop;
1756
3b04ddde 1757 if (dev->header_ops) {
1da177e4 1758 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1759 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1760 else if (skb->pkt_type == PACKET_OUTGOING) {
1761 /* Special case: outgoing packets have ll header at head */
bbe735e4 1762 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1763 }
1764 }
1765
8dc41944
HX		 1766 if (skb->ip_summed == CHECKSUM_PARTIAL)
1767 status |= TP_STATUS_CSUMNOTREADY;
1768
1da177e4
LT		 1769 snaplen = skb->len;
1770
dbcb5855
DM		 1771 res = run_filter(skb, sk, snaplen);
1772 if (!res)
fda9ef5d 1773 goto drop_n_restore;
dbcb5855
DM		 1774 if (snaplen > res)
1775 snaplen = res;
1da177e4
LT		 1776
1777 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1778 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1779 po->tp_reserve;
1da177e4 1780 } else {
95c96174 1781 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1782 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1783 (maclen < 16 ? 16 : maclen)) +
1784 po->tp_reserve;
1da177e4
LT		 1785 macoff = netoff - maclen;
1786 }
f6fb8f10 1787 if (po->tp_version <= TPACKET_V2) {
1788 if (macoff + snaplen > po->rx_ring.frame_size) {
1789 if (po->copy_thresh &&
0fd7bac6 1790 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1791 if (skb_shared(skb)) {
1792 copy_skb = skb_clone(skb, GFP_ATOMIC);
1793 } else {
1794 copy_skb = skb_get(skb);
1795 skb_head = skb->data;
1796 }
1797 if (copy_skb)
1798 skb_set_owner_r(copy_skb, sk);
1da177e4 1799 }
f6fb8f10 1800 snaplen = po->rx_ring.frame_size - macoff;
1801 if ((int)snaplen < 0)
1802 snaplen = 0;
1da177e4 1803 }
4035ed7b
ED		 1804 } else if (unlikely(macoff + snaplen >
1805 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1806 u32 nval;
1807
1808 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1809 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1810 snaplen, nval, macoff);
1811 snaplen = nval;
1812 if (unlikely((int)snaplen < 0)) {
1813 snaplen = 0;
1814 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1815 }
1da177e4 1816 }
1da177e4 1817 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1818 h.raw = packet_current_rx_frame(po, skb,
1819 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1820 if (!h.raw)
1da177e4 1821 goto ring_is_full;
f6fb8f10 1822 if (po->tp_version <= TPACKET_V2) {
1823 packet_increment_rx_head(po, &po->rx_ring);
1824 /*
1825 * LOSING will be reported till you read the stats,
1826 * because it's COR - Clear On Read.
1827 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1828 * at packet level.
1829 */
ee80fbf3 1830 if (po->stats.stats1.tp_drops)
f6fb8f10 1831 status |= TP_STATUS_LOSING;
1832 }
ee80fbf3 1833 po->stats.stats1.tp_packets++;
1da177e4
LT		 1834 if (copy_skb) {
1835 status |= TP_STATUS_COPY;
1836 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1837 }
1da177e4
LT		 1838 spin_unlock(&sk->sk_receive_queue.lock);
1839
bbd6ef87 1840 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1841
1842 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1843 getnstimeofday(&ts);
1da177e4 1844
b9c32fb2
DB		 1845 status |= ts_status;
1846
bbd6ef87
PM		 1847 switch (po->tp_version) {
1848 case TPACKET_V1:
1849 h.h1->tp_len = skb->len;
1850 h.h1->tp_snaplen = snaplen;
1851 h.h1->tp_mac = macoff;
1852 h.h1->tp_net = netoff;
4b457bdf
DB		 1853 h.h1->tp_sec = ts.tv_sec;
1854 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1855 hdrlen = sizeof(*h.h1);
1856 break;
1857 case TPACKET_V2:
1858 h.h2->tp_len = skb->len;
1859 h.h2->tp_snaplen = snaplen;
1860 h.h2->tp_mac = macoff;
1861 h.h2->tp_net = netoff;
bbd6ef87
PM		 1862 h.h2->tp_sec = ts.tv_sec;
1863 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1864 if (vlan_tx_tag_present(skb)) {
1865 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1866 status |= TP_STATUS_VLAN_VALID;
1867 } else {
1868 h.h2->tp_vlan_tci = 0;
1869 }
13fcb7bd 1870 h.h2->tp_padding = 0;
bbd6ef87
PM		 1871 hdrlen = sizeof(*h.h2);
1872 break;
f6fb8f10 1873 case TPACKET_V3:
1874 /* tp_nxt_offset,vlan are already populated above.
1875 * So DONT clear those fields here
1876 */
1877 h.h3->tp_status |= status;
1878 h.h3->tp_len = skb->len;
1879 h.h3->tp_snaplen = snaplen;
1880 h.h3->tp_mac = macoff;
1881 h.h3->tp_net = netoff;
f6fb8f10 1882 h.h3->tp_sec = ts.tv_sec;
1883 h.h3->tp_nsec = ts.tv_nsec;
1884 hdrlen = sizeof(*h.h3);
1885 break;
bbd6ef87
PM		 1886 default:
1887 BUG();
1888 }
1da177e4 1889
bbd6ef87 1890 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1891 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1892 sll->sll_family = AF_PACKET;
1893 sll->sll_hatype = dev->type;
1894 sll->sll_protocol = skb->protocol;
1895 sll->sll_pkttype = skb->pkt_type;
8032b464 1896 if (unlikely(po->origdev))
80feaacb
PWJ		 1897 sll->sll_ifindex = orig_dev->ifindex;
1898 else
1899 sll->sll_ifindex = dev->ifindex;
1da177e4 1900
e16aa207 1901 smp_mb();
f6dafa95 1902#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1903 {
0af55bb5
CG		 1904 u8 *start, *end;
1905
f6fb8f10 1906 if (po->tp_version <= TPACKET_V2) {
1907 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1908 + macoff + snaplen);
1909 for (start = h.raw; start < end; start += PAGE_SIZE)
1910 flush_dcache_page(pgv_to_page(start));
1911 }
cc9f01b2 1912 smp_wmb();
1da177e4 1913 }
f6dafa95 1914#endif
f6fb8f10 1915 if (po->tp_version <= TPACKET_V2)
1916 __packet_set_status(po, h.raw, status);
1917 else
1918 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1919
1920 sk->sk_data_ready(sk, 0);
1921
1922drop_n_restore:
1923 if (skb_head != skb->data && skb_shared(skb)) {
1924 skb->data = skb_head;
1925 skb->len = skb_len;
1926 }
1927drop:
1ce4f28b 1928 kfree_skb(skb);
1da177e4
LT		 1929 return 0;
1930
1931ring_is_full:
ee80fbf3 1932 po->stats.stats1.tp_drops++;
1da177e4
LT		 1933 spin_unlock(&sk->sk_receive_queue.lock);
1934
1935 sk->sk_data_ready(sk, 0);
acb5d75b 1936 kfree_skb(copy_skb);
1da177e4
LT		 1937 goto drop_n_restore;
1938}
1939
69e3c75f
JB		 1940static void tpacket_destruct_skb(struct sk_buff *skb)
1941{
1942 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1943 void *ph;
1da177e4 1944
69e3c75f 1945 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 1946 __u32 ts;
1947
69e3c75f 1948 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1949 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1950 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 1951
1952 ts = __packet_set_timestamp(po, ph, skb);
1953 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 1954 }
1955
1956 sock_wfree(skb);
1957}
1958
40d4e3df
ED		 1959static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1960 void *frame, struct net_device *dev, int size_max,
ae641949 1961 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1962{
184f489e 1963 union tpacket_uhdr ph;
69e3c75f
JB		 1964 int to_write, offset, len, tp_len, nr_frags, len_max;
1965 struct socket *sock = po->sk.sk_socket;
1966 struct page *page;
1967 void *data;
1968 int err;
1969
1970 ph.raw = frame;
1971
1972 skb->protocol = proto;
1973 skb->dev = dev;
1974 skb->priority = po->sk.sk_priority;
2d37a186 1975 skb->mark = po->sk.sk_mark;
2e31396f 1976 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 1977 skb_shinfo(skb)->destructor_arg = ph.raw;
1978
1979 switch (po->tp_version) {
1980 case TPACKET_V2:
1981 tp_len = ph.h2->tp_len;
1982 break;
1983 default:
1984 tp_len = ph.h1->tp_len;
1985 break;
1986 }
1987 if (unlikely(tp_len > size_max)) {
40d4e3df 1988 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1989 return -EMSGSIZE;
1990 }
1991
ae641949 1992 skb_reserve(skb, hlen);
69e3c75f 1993 skb_reset_network_header(skb);
40893fd0 1994 skb_probe_transport_header(skb, 0);
c1aad275 1995
5920cd3a
PC		 1996 if (po->tp_tx_has_off) {
1997 int off_min, off_max, off;
1998 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1999 off_max = po->tx_ring.frame_size - tp_len;
2000 if (sock->type == SOCK_DGRAM) {
2001 switch (po->tp_version) {
2002 case TPACKET_V2:
2003 off = ph.h2->tp_net;
2004 break;
2005 default:
2006 off = ph.h1->tp_net;
2007 break;
2008 }
2009 } else {
2010 switch (po->tp_version) {
2011 case TPACKET_V2:
2012 off = ph.h2->tp_mac;
2013 break;
2014 default:
2015 off = ph.h1->tp_mac;
2016 break;
2017 }
2018 }
2019 if (unlikely((off < off_min) || (off_max < off)))
2020 return -EINVAL;
2021 data = ph.raw + off;
2022 } else {
2023 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2024 }
69e3c75f
JB		 2025 to_write = tp_len;
2026
2027 if (sock->type == SOCK_DGRAM) {
2028 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2029 NULL, tp_len);
2030 if (unlikely(err < 0))
2031 return -EINVAL;
40d4e3df 2032 } else if (dev->hard_header_len) {
69e3c75f
JB		 2033 /* net device doesn't like empty head */
2034 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2035 pr_err("packet size is too short (%d < %d)\n",
2036 tp_len, dev->hard_header_len);
69e3c75f
JB		 2037 return -EINVAL;
2038 }
2039
2040 skb_push(skb, dev->hard_header_len);
2041 err = skb_store_bits(skb, 0, data,
2042 dev->hard_header_len);
2043 if (unlikely(err))
2044 return err;
2045
2046 data += dev->hard_header_len;
2047 to_write -= dev->hard_header_len;
2048 }
2049
69e3c75f
JB		 2050 offset = offset_in_page(data);
2051 len_max = PAGE_SIZE - offset;
2052 len = ((to_write > len_max) ? len_max : to_write);
2053
2054 skb->data_len = to_write;
2055 skb->len += to_write;
2056 skb->truesize += to_write;
2057 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2058
2059 while (likely(to_write)) {
2060 nr_frags = skb_shinfo(skb)->nr_frags;
2061
2062 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2063 pr_err("Packet exceed the number of skb frags(%lu)\n",
2064 MAX_SKB_FRAGS);
69e3c75f
JB		 2065 return -EFAULT;
2066 }
2067
0af55bb5
CG		 2068 page = pgv_to_page(data);
2069 data += len;
69e3c75f
JB		 2070 flush_dcache_page(page);
2071 get_page(page);
0af55bb5 2072 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2073 to_write -= len;
2074 offset = 0;
2075 len_max = PAGE_SIZE;
2076 len = ((to_write > len_max) ? len_max : to_write);
2077 }
2078
2079 return tp_len;
2080}
2081
2082static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2083{
69e3c75f
JB		 2084 struct sk_buff *skb;
2085 struct net_device *dev;
2086 __be16 proto;
827d9780 2087 int err, reserve = 0;
40d4e3df
ED		 2088 void *ph;
2089 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2090 int tp_len, size_max;
2091 unsigned char *addr;
2092 int len_sum = 0;
9e67030a 2093 int status = TP_STATUS_AVAILABLE;
ae641949 2094 int hlen, tlen;
69e3c75f 2095
69e3c75f
JB		 2096 mutex_lock(&po->pg_vec_lock);
2097
c3ac8a13 2098 if (likely(saddr == NULL)) {
026bb405 2099 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2100 proto = po->num;
2101 addr = NULL;
2102 } else {
2103 err = -EINVAL;
2104 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2105 goto out;
2106 if (msg->msg_namelen < (saddr->sll_halen
2107 + offsetof(struct sockaddr_ll,
2108 sll_addr)))
2109 goto out;
69e3c75f
JB		 2110 proto = saddr->sll_protocol;
2111 addr = saddr->sll_addr;
827d9780 2112 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2113 }
2114
69e3c75f
JB		 2115 err = -ENXIO;
2116 if (unlikely(dev == NULL))
2117 goto out;
69e3c75f
JB		 2118 err = -ENETDOWN;
2119 if (unlikely(!(dev->flags & IFF_UP)))
2120 goto out_put;
2121
026bb405
DB		 2122 reserve = dev->hard_header_len;
2123
69e3c75f 2124 size_max = po->tx_ring.frame_size
b5dd884e 2125 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2126
2127 if (size_max > dev->mtu + reserve)
2128 size_max = dev->mtu + reserve;
2129
2130 do {
2131 ph = packet_current_frame(po, &po->tx_ring,
2132 TP_STATUS_SEND_REQUEST);
2133
2134 if (unlikely(ph == NULL)) {
2135 schedule();
2136 continue;
2137 }
2138
2139 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2140 hlen = LL_RESERVED_SPACE(dev);
2141 tlen = dev->needed_tailroom;
69e3c75f 2142 skb = sock_alloc_send_skb(&po->sk,
ae641949 2143 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2144 0, &err);
2145
2146 if (unlikely(skb == NULL))
2147 goto out_status;
2148
2149 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2150 addr, hlen);
69e3c75f
JB		 2151
2152 if (unlikely(tp_len < 0)) {
2153 if (po->tp_loss) {
2154 __packet_set_status(po, ph,
2155 TP_STATUS_AVAILABLE);
2156 packet_increment_head(&po->tx_ring);
2157 kfree_skb(skb);
2158 continue;
2159 } else {
2160 status = TP_STATUS_WRONG_FORMAT;
2161 err = tp_len;
2162 goto out_status;
2163 }
2164 }
2165
2166 skb->destructor = tpacket_destruct_skb;
2167 __packet_set_status(po, ph, TP_STATUS_SENDING);
2168 atomic_inc(&po->tx_ring.pending);
2169
2170 status = TP_STATUS_SEND_REQUEST;
2171 err = dev_queue_xmit(skb);
eb70df13
JP		 2172 if (unlikely(err > 0)) {
2173 err = net_xmit_errno(err);
2174 if (err && __packet_get_status(po, ph) ==
2175 TP_STATUS_AVAILABLE) {
2176 /* skb was destructed already */
2177 skb = NULL;
2178 goto out_status;
2179 }
2180 /*
2181 * skb was dropped but not destructed yet;
2182 * let's treat it like congestion or err < 0
2183 */
2184 err = 0;
2185 }
69e3c75f
JB		 2186 packet_increment_head(&po->tx_ring);
2187 len_sum += tp_len;
f64f9e71
JP		 2188 } while (likely((ph != NULL) ||
2189 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2190 (atomic_read(&po->tx_ring.pending))))
2191 );
69e3c75f
JB		 2192
2193 err = len_sum;
2194 goto out_put;
2195
69e3c75f
JB		 2196out_status:
2197 __packet_set_status(po, ph, status);
2198 kfree_skb(skb);
2199out_put:
026bb405 2200 dev_put(dev);
69e3c75f
JB		 2201out:
2202 mutex_unlock(&po->pg_vec_lock);
2203 return err;
2204}
69e3c75f 2205
eea49cc9
OJ		 2206static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2207 size_t reserve, size_t len,
2208 size_t linear, int noblock,
2209 int *err)
bfd5f4a3
SS		 2210{
2211 struct sk_buff *skb;
2212
2213 /* Under a page? Don't bother with paged skb. */
2214 if (prepad + len < PAGE_SIZE || !linear)
2215 linear = len;
2216
2217 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2218 err);
2219 if (!skb)
2220 return NULL;
2221
2222 skb_reserve(skb, reserve);
2223 skb_put(skb, linear);
2224 skb->data_len = len - linear;
2225 skb->len += len - linear;
2226
2227 return skb;
2228}
2229
69e3c75f 2230static int packet_snd(struct socket *sock,
1da177e4
LT		 2231 struct msghdr *msg, size_t len)
2232{
2233 struct sock *sk = sock->sk;
40d4e3df 2234 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2235 struct sk_buff *skb;
2236 struct net_device *dev;
0e11c91e 2237 __be16 proto;
1da177e4 2238 unsigned char *addr;
827d9780 2239 int err, reserve = 0;
bfd5f4a3
SS		 2240 struct virtio_net_hdr vnet_hdr = { 0 };
2241 int offset = 0;
2242 int vnet_hdr_len;
2243 struct packet_sock *po = pkt_sk(sk);
2244 unsigned short gso_type = 0;
c6cc07d4 2245 int hlen, tlen, linear;
3bdc0eba 2246 int extra_len = 0;
1da177e4
LT		 2247
2248 /*
1ce4f28b 2249 * Get and verify the address.
1da177e4 2250 */
1ce4f28b 2251
c3ac8a13 2252 if (likely(saddr == NULL)) {
026bb405 2253 dev = packet_cached_dev_get(po);
1da177e4
LT		 2254 proto = po->num;
2255 addr = NULL;
2256 } else {
2257 err = -EINVAL;
2258 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2259 goto out;
0fb375fb
EB		 2260 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2261 goto out;
1da177e4
LT		 2262 proto = saddr->sll_protocol;
2263 addr = saddr->sll_addr;
827d9780 2264 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2265 }
2266
1da177e4 2267 err = -ENXIO;
026bb405 2268 if (unlikely(dev == NULL))
1da177e4 2269 goto out_unlock;
d5e76b0a 2270 err = -ENETDOWN;
026bb405 2271 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2272 goto out_unlock;
2273
026bb405
DB		 2274 if (sock->type == SOCK_RAW)
2275 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2276 if (po->has_vnet_hdr) {
2277 vnet_hdr_len = sizeof(vnet_hdr);
2278
2279 err = -EINVAL;
2280 if (len < vnet_hdr_len)
2281 goto out_unlock;
2282
2283 len -= vnet_hdr_len;
2284
2285 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2286 vnet_hdr_len);
2287 if (err < 0)
2288 goto out_unlock;
2289
2290 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2291 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2292 vnet_hdr.hdr_len))
2293 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2294 vnet_hdr.csum_offset + 2;
2295
2296 err = -EINVAL;
2297 if (vnet_hdr.hdr_len > len)
2298 goto out_unlock;
2299
2300 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2301 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2302 case VIRTIO_NET_HDR_GSO_TCPV4:
2303 gso_type = SKB_GSO_TCPV4;
2304 break;
2305 case VIRTIO_NET_HDR_GSO_TCPV6:
2306 gso_type = SKB_GSO_TCPV6;
2307 break;
2308 case VIRTIO_NET_HDR_GSO_UDP:
2309 gso_type = SKB_GSO_UDP;
2310 break;
2311 default:
2312 goto out_unlock;
2313 }
2314
2315 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2316 gso_type |= SKB_GSO_TCP_ECN;
2317
2318 if (vnet_hdr.gso_size == 0)
2319 goto out_unlock;
2320
2321 }
2322 }
2323
3bdc0eba
BG		 2324 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2325 if (!netif_supports_nofcs(dev)) {
2326 err = -EPROTONOSUPPORT;
2327 goto out_unlock;
2328 }
2329 extra_len = 4; /* We're doing our own CRC */
2330 }
2331
1da177e4 2332 err = -EMSGSIZE;
3bdc0eba 2333 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2334 goto out_unlock;
2335
bfd5f4a3 2336 err = -ENOBUFS;
ae641949
HX		 2337 hlen = LL_RESERVED_SPACE(dev);
2338 tlen = dev->needed_tailroom;
c6cc07d4
WB		 2339 linear = vnet_hdr.hdr_len;
2340 linear = max(linear, min_t(int, len, dev->hard_header_len));
2341 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
bfd5f4a3 2342 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2343 if (skb == NULL)
1da177e4
LT		 2344 goto out_unlock;
2345
bfd5f4a3 2346 skb_set_network_header(skb, reserve);
1da177e4 2347
0c4e8581
SH		 2348 err = -EINVAL;
2349 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2350 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2351 goto out_free;
1da177e4
LT		 2352
2353 /* Returns -EFAULT on error */
bfd5f4a3 2354 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2355 if (err)
2356 goto out_free;
bf84a010
DB		 2357
2358 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2359
3bdc0eba 2360 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
57f89bfa
BG		 2361 /* Earlier code assumed this would be a VLAN pkt,
2362 * double-check this now that we have the actual
2363 * packet in hand.
2364 */
2365 struct ethhdr *ehdr;
2366 skb_reset_mac_header(skb);
2367 ehdr = eth_hdr(skb);
2368 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2369 err = -EMSGSIZE;
2370 goto out_free;
2371 }
2372 }
2373
1da177e4
LT		 2374 skb->protocol = proto;
2375 skb->dev = dev;
2376 skb->priority = sk->sk_priority;
2d37a186 2377 skb->mark = sk->sk_mark;
1da177e4 2378
bfd5f4a3
SS		 2379 if (po->has_vnet_hdr) {
2380 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2381 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2382 vnet_hdr.csum_offset)) {
2383 err = -EINVAL;
2384 goto out_free;
2385 }
2386 }
2387
2388 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2389 skb_shinfo(skb)->gso_type = gso_type;
2390
2391 /* Header must be checked, and gso_segs computed. */
2392 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2393 skb_shinfo(skb)->gso_segs = 0;
2394
2395 len += vnet_hdr_len;
2396 }
2397
40893fd0 2398 skb_probe_transport_header(skb, reserve);
c1aad275 2399
3bdc0eba
BG		 2400 if (unlikely(extra_len == 4))
2401 skb->no_fcs = 1;
2402
1da177e4
LT		 2403 /*
2404 * Now send it
2405 */
2406
2407 err = dev_queue_xmit(skb);
2408 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2409 goto out_unlock;
2410
026bb405 2411 dev_put(dev);
1da177e4 2412
40d4e3df 2413 return len;
1da177e4
LT		 2414
2415out_free:
2416 kfree_skb(skb);
2417out_unlock:
026bb405 2418 if (dev)
1da177e4
LT		 2419 dev_put(dev);
2420out:
2421 return err;
2422}
2423
69e3c75f
JB		 2424static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2425 struct msghdr *msg, size_t len)
2426{
69e3c75f
JB		 2427 struct sock *sk = sock->sk;
2428 struct packet_sock *po = pkt_sk(sk);
2429 if (po->tx_ring.pg_vec)
2430 return tpacket_snd(po, msg);
2431 else
69e3c75f
JB		 2432 return packet_snd(sock, msg, len);
2433}
2434
1da177e4
LT		 2435/*
2436 * Close a PACKET socket. This is fairly simple. We immediately go
2437 * to 'closed' state and remove our protocol entry in the device list.
2438 */
2439
2440static int packet_release(struct socket *sock)
2441{
2442 struct sock *sk = sock->sk;
2443 struct packet_sock *po;
9aab3592 2444 struct packet_fanout *f;
d12d01d6 2445 struct net *net;
f6fb8f10 2446 union tpacket_req_u req_u;
1da177e4
LT		 2447
2448 if (!sk)
2449 return 0;
2450
3b1e0a65 2451 net = sock_net(sk);
1da177e4
LT		 2452 po = pkt_sk(sk);
2453
0fa7fa98 2454 mutex_lock(&net->packet.sklist_lock);
808f5114 2455 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2456 mutex_unlock(&net->packet.sklist_lock);
2457
2458 preempt_disable();
920de804 2459 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2460 preempt_enable();
1da177e4 2461
808f5114 2462 spin_lock(&po->bind_lock);
ce06b03e 2463 unregister_prot_hook(sk, false);
c3ac8a13
DB		 2464 packet_cached_dev_reset(po);
2465
160ff18a
BG		 2466 if (po->prot_hook.dev) {
2467 dev_put(po->prot_hook.dev);
2468 po->prot_hook.dev = NULL;
2469 }
808f5114 2470 spin_unlock(&po->bind_lock);
1da177e4 2471
1da177e4 2472 packet_flush_mclist(sk);
1da177e4 2473
9665d5d6
PS		 2474 if (po->rx_ring.pg_vec) {
2475 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2476 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2477 }
69e3c75f 2478
9665d5d6
PS		 2479 if (po->tx_ring.pg_vec) {
2480 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2481 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2482 }
1da177e4 2483
9aab3592 2484 f = fanout_release(sk);
dc99f600 2485
808f5114 2486 synchronize_net();
9aab3592
AS		 2487
2488 if (f) {
2489 kfree(f);
2490 }
1da177e4
LT		 2491 /*
2492 * Now the socket is dead. No more input will appear.
2493 */
1da177e4
LT		 2494 sock_orphan(sk);
2495 sock->sk = NULL;
2496
2497 /* Purge queues */
2498
2499 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2500 sk_refcnt_debug_release(sk);
1da177e4
LT		 2501
2502 sock_put(sk);
2503 return 0;
2504}
2505
2506/*
2507 * Attach a packet hook.
2508 */
2509
0e11c91e 2510static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2511{
2512 struct packet_sock *po = pkt_sk(sk);
dc99f600 2513
aef950b4
WY		 2514 if (po->fanout) {
2515 if (dev)
2516 dev_put(dev);
2517
dc99f600 2518 return -EINVAL;
aef950b4 2519 }
1da177e4
LT		 2520
2521 lock_sock(sk);
2522
2523 spin_lock(&po->bind_lock);
ce06b03e 2524 unregister_prot_hook(sk, true);
c3ac8a13 2525
1da177e4
LT		 2526 po->num = protocol;
2527 po->prot_hook.type = protocol;
160ff18a
BG		 2528 if (po->prot_hook.dev)
2529 dev_put(po->prot_hook.dev);
1da177e4 2530
c3ac8a13 2531 po->prot_hook.dev = dev;
1da177e4
LT		 2532 po->ifindex = dev ? dev->ifindex : 0;
2533
c3ac8a13
DB		 2534 packet_cached_dev_assign(po, dev);
2535
1da177e4
LT		 2536 if (protocol == 0)
2537 goto out_unlock;
2538
be85d4ad 2539 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2540 register_prot_hook(sk);
be85d4ad
UT		 2541 } else {
2542 sk->sk_err = ENETDOWN;
2543 if (!sock_flag(sk, SOCK_DEAD))
2544 sk->sk_error_report(sk);
1da177e4
LT		 2545 }
2546
2547out_unlock:
2548 spin_unlock(&po->bind_lock);
2549 release_sock(sk);
2550 return 0;
2551}
2552
2553/*
2554 * Bind a packet socket to a device
2555 */
2556
40d4e3df
ED		 2557static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2558 int addr_len)
1da177e4 2559{
40d4e3df 2560 struct sock *sk = sock->sk;
bc00602b 2561 char name[sizeof(uaddr->sa_data) + 1];
1da177e4
LT		 2562 struct net_device *dev;
2563 int err = -ENODEV;
1ce4f28b 2564
1da177e4
LT		 2565 /*
2566 * Check legality
2567 */
1ce4f28b 2568
8ae55f04 2569 if (addr_len != sizeof(struct sockaddr))
1da177e4 2570 return -EINVAL;
bc00602b
AP		 2571 /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
2572 * zero-terminated.
2573 */
2574 memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
2575 name[sizeof(uaddr->sa_data)] = 0;
1da177e4 2576
3b1e0a65 2577 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2578 if (dev)
1da177e4 2579 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2580 return err;
2581}
1da177e4
LT		 2582
2583static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2584{
40d4e3df
ED		 2585 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2586 struct sock *sk = sock->sk;
1da177e4
LT		 2587 struct net_device *dev = NULL;
2588 int err;
2589
2590
2591 /*
2592 * Check legality
2593 */
1ce4f28b 2594
1da177e4
LT		 2595 if (addr_len < sizeof(struct sockaddr_ll))
2596 return -EINVAL;
2597 if (sll->sll_family != AF_PACKET)
2598 return -EINVAL;
2599
2600 if (sll->sll_ifindex) {
2601 err = -ENODEV;
3b1e0a65 2602 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2603 if (dev == NULL)
2604 goto out;
2605 }
2606 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2607
2608out:
2609 return err;
2610}
2611
2612static struct proto packet_proto = {
2613 .name = "PACKET",
2614 .owner = THIS_MODULE,
2615 .obj_size = sizeof(struct packet_sock),
2616};
2617
2618/*
1ce4f28b 2619 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2620 */
2621
3f378b68
EP		 2622static int packet_create(struct net *net, struct socket *sock, int protocol,
2623 int kern)
1da177e4
LT		 2624{
2625 struct sock *sk;
2626 struct packet_sock *po;
0e11c91e 2627 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2628 int err;
2629
df008c91 2630 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2631 return -EPERM;
be02097c
DM		 2632 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2633 sock->type != SOCK_PACKET)
1da177e4
LT		 2634 return -ESOCKTNOSUPPORT;
2635
2636 sock->state = SS_UNCONNECTED;
2637
2638 err = -ENOBUFS;
6257ff21 2639 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2640 if (sk == NULL)
2641 goto out;
2642
2643 sock->ops = &packet_ops;
1da177e4
LT		 2644 if (sock->type == SOCK_PACKET)
2645 sock->ops = &packet_ops_spkt;
be02097c 2646
1da177e4
LT		 2647 sock_init_data(sock, sk);
2648
2649 po = pkt_sk(sk);
2650 sk->sk_family = PF_PACKET;
0e11c91e 2651 po->num = proto;
c3ac8a13
DB		 2652
2653 packet_cached_dev_reset(po);
1da177e4
LT		 2654
2655 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2656 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2657
2658 /*
2659 * Attach a protocol block
2660 */
2661
2662 spin_lock_init(&po->bind_lock);
905db440 2663 mutex_init(&po->pg_vec_lock);
1da177e4 2664 po->prot_hook.func = packet_rcv;
be02097c 2665
1da177e4
LT		 2666 if (sock->type == SOCK_PACKET)
2667 po->prot_hook.func = packet_rcv_spkt;
be02097c 2668
1da177e4
LT		 2669 po->prot_hook.af_packet_priv = sk;
2670
0e11c91e
AV		 2671 if (proto) {
2672 po->prot_hook.type = proto;
ce06b03e 2673 register_prot_hook(sk);
1da177e4
LT		 2674 }
2675
0fa7fa98 2676 mutex_lock(&net->packet.sklist_lock);
808f5114 2677 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2678 mutex_unlock(&net->packet.sklist_lock);
2679
2680 preempt_disable();
3680453c 2681 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2682 preempt_enable();
808f5114 2683
40d4e3df 2684 return 0;
1da177e4
LT		 2685out:
2686 return err;
2687}
2688
ed85b565
RC		 2689static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2690{
2691 struct sock_exterr_skb *serr;
2692 struct sk_buff *skb, *skb2;
2693 int copied, err;
2694
2695 err = -EAGAIN;
2696 skb = skb_dequeue(&sk->sk_error_queue);
2697 if (skb == NULL)
2698 goto out;
2699
2700 copied = skb->len;
2701 if (copied > len) {
2702 msg->msg_flags |= MSG_TRUNC;
2703 copied = len;
2704 }
2705 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2706 if (err)
2707 goto out_free_skb;
2708
2709 sock_recv_timestamp(msg, sk, skb);
2710
2711 serr = SKB_EXT_ERR(skb);
2712 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2713 sizeof(serr->ee), &serr->ee);
2714
2715 msg->msg_flags |= MSG_ERRQUEUE;
2716 err = copied;
2717
2718 /* Reset and regenerate socket error */
2719 spin_lock_bh(&sk->sk_error_queue.lock);
2720 sk->sk_err = 0;
2721 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2722 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2723 spin_unlock_bh(&sk->sk_error_queue.lock);
2724 sk->sk_error_report(sk);
2725 } else
2726 spin_unlock_bh(&sk->sk_error_queue.lock);
2727
2728out_free_skb:
2729 kfree_skb(skb);
2730out:
2731 return err;
2732}
2733
1da177e4
LT		 2734/*
2735 * Pull a packet from our receive queue and hand it to the user.
2736 * If necessary we block.
2737 */
2738
2739static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2740 struct msghdr *msg, size_t len, int flags)
2741{
2742 struct sock *sk = sock->sk;
2743 struct sk_buff *skb;
2744 int copied, err;
bfd5f4a3 2745 int vnet_hdr_len = 0;
1da177e4
LT		 2746
2747 err = -EINVAL;
ed85b565 2748 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2749 goto out;
2750
2751#if 0
2752 /* What error should we return now? EUNATTACH? */
2753 if (pkt_sk(sk)->ifindex < 0)
2754 return -ENODEV;
2755#endif
2756
ed85b565
RC		 2757 if (flags & MSG_ERRQUEUE) {
2758 err = packet_recv_error(sk, msg, len);
2759 goto out;
2760 }
2761
1da177e4
LT		 2762 /*
2763 * Call the generic datagram receiver. This handles all sorts
2764 * of horrible races and re-entrancy so we can forget about it
2765 * in the protocol layers.
2766 *
2767 * Now it will return ENETDOWN, if device have just gone down,
2768 * but then it will block.
2769 */
2770
40d4e3df 2771 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2772
2773 /*
1ce4f28b 2774 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2775 * handles the blocking we don't see and worry about blocking
2776 * retries.
2777 */
2778
8ae55f04 2779 if (skb == NULL)
1da177e4
LT		 2780 goto out;
2781
bfd5f4a3
SS		 2782 if (pkt_sk(sk)->has_vnet_hdr) {
2783 struct virtio_net_hdr vnet_hdr = { 0 };
2784
2785 err = -EINVAL;
2786 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2787 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2788 goto out_free;
2789
1f18b717
MK		 2790 len -= vnet_hdr_len;
2791
bfd5f4a3
SS		 2792 if (skb_is_gso(skb)) {
2793 struct skb_shared_info *sinfo = skb_shinfo(skb);
2794
2795 /* This is a hint as to how much should be linear. */
2796 vnet_hdr.hdr_len = skb_headlen(skb);
2797 vnet_hdr.gso_size = sinfo->gso_size;
2798 if (sinfo->gso_type & SKB_GSO_TCPV4)
2799 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2800 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2801 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2802 else if (sinfo->gso_type & SKB_GSO_UDP)
2803 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2804 else if (sinfo->gso_type & SKB_GSO_FCOE)
2805 goto out_free;
2806 else
2807 BUG();
2808 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2809 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2810 } else
2811 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2812
2813 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2814 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2815 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2816 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2817 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2818 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2819 } /* else everything is zero */
2820
2821 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2822 vnet_hdr_len);
2823 if (err < 0)
2824 goto out_free;
2825 }
2826
2f73d7fd
HFS		 2827 /* You lose any data beyond the buffer you gave. If it worries
2828 * a user program they can ask the device for its MTU
2829 * anyway.
1da177e4 2830 */
1da177e4 2831 copied = skb->len;
40d4e3df
ED		 2832 if (copied > len) {
2833 copied = len;
2834 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2835 }
2836
2837 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2838 if (err)
2839 goto out_free;
2840
3b885787 2841 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2842
2f73d7fd
HFS		 2843 if (msg->msg_name) {
2844 /* If the address length field is there to be filled
2845 * in, we fill it in now.
2846 */
2847 if (sock->type == SOCK_PACKET) {
2848 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2849 } else {
2850 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2851 msg->msg_namelen = sll->sll_halen +
2852 offsetof(struct sockaddr_ll, sll_addr);
2853 }
ffbc6111
HX		 2854 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2855 msg->msg_namelen);
2f73d7fd 2856 }
1da177e4 2857
8dc41944 2858 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2859 struct tpacket_auxdata aux;
2860
2861 aux.tp_status = TP_STATUS_USER;
2862 if (skb->ip_summed == CHECKSUM_PARTIAL)
2863 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2864 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2865 aux.tp_snaplen = skb->len;
2866 aux.tp_mac = 0;
bbe735e4 2867 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2868 if (vlan_tx_tag_present(skb)) {
2869 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2870 aux.tp_status |= TP_STATUS_VLAN_VALID;
2871 } else {
2872 aux.tp_vlan_tci = 0;
2873 }
13fcb7bd 2874 aux.tp_padding = 0;
ffbc6111 2875 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2876 }
2877
1da177e4
LT		 2878 /*
2879 * Free or return the buffer as appropriate. Again this
2880 * hides all the races and re-entrancy issues from us.
2881 */
bfd5f4a3 2882 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2883
2884out_free:
2885 skb_free_datagram(sk, skb);
2886out:
2887 return err;
2888}
2889
1da177e4
LT		 2890static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2891 int *uaddr_len, int peer)
2892{
2893 struct net_device *dev;
2894 struct sock *sk = sock->sk;
2895
2896 if (peer)
2897 return -EOPNOTSUPP;
2898
2899 uaddr->sa_family = AF_PACKET;
2dc85bf3 2900 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2901 rcu_read_lock();
2902 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2903 if (dev)
2dc85bf3 2904 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2905 rcu_read_unlock();
1da177e4
LT		 2906 *uaddr_len = sizeof(*uaddr);
2907
2908 return 0;
2909}
1da177e4
LT		 2910
2911static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2912 int *uaddr_len, int peer)
2913{
2914 struct net_device *dev;
2915 struct sock *sk = sock->sk;
2916 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2917 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2918
2919 if (peer)
2920 return -EOPNOTSUPP;
2921
2922 sll->sll_family = AF_PACKET;
2923 sll->sll_ifindex = po->ifindex;
2924 sll->sll_protocol = po->num;
67286640 2925 sll->sll_pkttype = 0;
654d1f8a
ED		 2926 rcu_read_lock();
2927 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2928 if (dev) {
2929 sll->sll_hatype = dev->type;
2930 sll->sll_halen = dev->addr_len;
2931 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2932 } else {
2933 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2934 sll->sll_halen = 0;
2935 }
654d1f8a 2936 rcu_read_unlock();
0fb375fb 2937 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2938
2939 return 0;
2940}
2941
2aeb0b88
WC		 2942static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2943 int what)
1da177e4
LT		 2944{
2945 switch (i->type) {
2946 case PACKET_MR_MULTICAST:
1162563f
JP		 2947 if (i->alen != dev->addr_len)
2948 return -EINVAL;
1da177e4 2949 if (what > 0)
22bedad3 2950 return dev_mc_add(dev, i->addr);
1da177e4 2951 else
22bedad3 2952 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2953 break;
2954 case PACKET_MR_PROMISC:
2aeb0b88 2955 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2956 break;
2957 case PACKET_MR_ALLMULTI:
2aeb0b88 2958 return dev_set_allmulti(dev, what);
1da177e4 2959 break;
d95ed927 2960 case PACKET_MR_UNICAST:
1162563f
JP		 2961 if (i->alen != dev->addr_len)
2962 return -EINVAL;
d95ed927 2963 if (what > 0)
a748ee24 2964 return dev_uc_add(dev, i->addr);
d95ed927 2965 else
a748ee24 2966 return dev_uc_del(dev, i->addr);
d95ed927 2967 break;
40d4e3df
ED		 2968 default:
2969 break;
1da177e4 2970 }
2aeb0b88 2971 return 0;
1da177e4
LT		 2972}
2973
2974static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2975{
40d4e3df 2976 for ( ; i; i = i->next) {
1da177e4
LT		 2977 if (i->ifindex == dev->ifindex)
2978 packet_dev_mc(dev, i, what);
2979 }
2980}
2981
0fb375fb 2982static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2983{
2984 struct packet_sock *po = pkt_sk(sk);
2985 struct packet_mclist *ml, *i;
2986 struct net_device *dev;
2987 int err;
2988
2989 rtnl_lock();
2990
2991 err = -ENODEV;
3b1e0a65 2992 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2993 if (!dev)
2994 goto done;
2995
2996 err = -EINVAL;
1162563f 2997 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2998 goto done;
2999
3000 err = -ENOBUFS;
8b3a7005 3001 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3002 if (i == NULL)
3003 goto done;
3004
3005 err = 0;
3006 for (ml = po->mclist; ml; ml = ml->next) {
3007 if (ml->ifindex == mreq->mr_ifindex &&
3008 ml->type == mreq->mr_type &&
3009 ml->alen == mreq->mr_alen &&
3010 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3011 ml->count++;
3012 /* Free the new element ... */
3013 kfree(i);
3014 goto done;
3015 }
3016 }
3017
3018 i->type = mreq->mr_type;
3019 i->ifindex = mreq->mr_ifindex;
3020 i->alen = mreq->mr_alen;
3021 memcpy(i->addr, mreq->mr_address, i->alen);
13b89711 3022 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
1da177e4
LT		 3023 i->count = 1;
3024 i->next = po->mclist;
3025 po->mclist = i;
2aeb0b88
WC		 3026 err = packet_dev_mc(dev, i, 1);
3027 if (err) {
3028 po->mclist = i->next;
3029 kfree(i);
3030 }
1da177e4
LT		 3031
3032done:
3033 rtnl_unlock();
3034 return err;
3035}
3036
0fb375fb 3037static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3038{
3039 struct packet_mclist *ml, **mlp;
3040
3041 rtnl_lock();
3042
3043 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3044 if (ml->ifindex == mreq->mr_ifindex &&
3045 ml->type == mreq->mr_type &&
3046 ml->alen == mreq->mr_alen &&
3047 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3048 if (--ml->count == 0) {
3049 struct net_device *dev;
3050 *mlp = ml->next;
ad959e76
ED		 3051 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3052 if (dev)
1da177e4 3053 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3054 kfree(ml);
3055 }
3056 rtnl_unlock();
3057 return 0;
3058 }
3059 }
3060 rtnl_unlock();
3061 return -EADDRNOTAVAIL;
3062}
3063
3064static void packet_flush_mclist(struct sock *sk)
3065{
3066 struct packet_sock *po = pkt_sk(sk);
3067 struct packet_mclist *ml;
3068
3069 if (!po->mclist)
3070 return;
3071
3072 rtnl_lock();
3073 while ((ml = po->mclist) != NULL) {
3074 struct net_device *dev;
3075
3076 po->mclist = ml->next;
ad959e76
ED		 3077 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3078 if (dev != NULL)
1da177e4 3079 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3080 kfree(ml);
3081 }
3082 rtnl_unlock();
3083}
1da177e4
LT		 3084
3085static int
b7058842 3086packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3087{
3088 struct sock *sk = sock->sk;
8dc41944 3089 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3090 int ret;
3091
3092 if (level != SOL_PACKET)
3093 return -ENOPROTOOPT;
3094
69e3c75f 3095 switch (optname) {
1ce4f28b 3096 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3097 case PACKET_DROP_MEMBERSHIP:
3098 {
0fb375fb
EB		 3099 struct packet_mreq_max mreq;
3100 int len = optlen;
3101 memset(&mreq, 0, sizeof(mreq));
3102 if (len < sizeof(struct packet_mreq))
1da177e4 3103 return -EINVAL;
0fb375fb
EB		 3104 if (len > sizeof(mreq))
3105 len = sizeof(mreq);
40d4e3df 3106 if (copy_from_user(&mreq, optval, len))
1da177e4 3107 return -EFAULT;
0fb375fb
EB		 3108 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3109 return -EINVAL;
1da177e4
LT		 3110 if (optname == PACKET_ADD_MEMBERSHIP)
3111 ret = packet_mc_add(sk, &mreq);
3112 else
3113 ret = packet_mc_drop(sk, &mreq);
3114 return ret;
3115 }
a2efcfa0 3116
1da177e4 3117 case PACKET_RX_RING:
69e3c75f 3118 case PACKET_TX_RING:
1da177e4 3119 {
f6fb8f10 3120 union tpacket_req_u req_u;
3121 int len;
1da177e4 3122
f6fb8f10 3123 switch (po->tp_version) {
3124 case TPACKET_V1:
3125 case TPACKET_V2:
3126 len = sizeof(req_u.req);
3127 break;
3128 case TPACKET_V3:
3129 default:
3130 len = sizeof(req_u.req3);
3131 break;
3132 }
3133 if (optlen < len)
1da177e4 3134 return -EINVAL;
bfd5f4a3
SS		 3135 if (pkt_sk(sk)->has_vnet_hdr)
3136 return -EINVAL;
f6fb8f10 3137 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3138 return -EFAULT;
f6fb8f10 3139 return packet_set_ring(sk, &req_u, 0,
3140 optname == PACKET_TX_RING);
1da177e4
LT		 3141 }
3142 case PACKET_COPY_THRESH:
3143 {
3144 int val;
3145
40d4e3df 3146 if (optlen != sizeof(val))
1da177e4 3147 return -EINVAL;
40d4e3df 3148 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3149 return -EFAULT;
3150
3151 pkt_sk(sk)->copy_thresh = val;
3152 return 0;
3153 }
bbd6ef87
PM		 3154 case PACKET_VERSION:
3155 {
3156 int val;
3157
3158 if (optlen != sizeof(val))
3159 return -EINVAL;
bbd6ef87
PM		 3160 if (copy_from_user(&val, optval, sizeof(val)))
3161 return -EFAULT;
3162 switch (val) {
3163 case TPACKET_V1:
3164 case TPACKET_V2:
f6fb8f10 3165 case TPACKET_V3:
4bfb6dd7 3166 break;
bbd6ef87
PM		 3167 default:
3168 return -EINVAL;
3169 }
4bfb6dd7
PP		 3170 lock_sock(sk);
3171 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3172 ret = -EBUSY;
3173 } else {
3174 po->tp_version = val;
3175 ret = 0;
3176 }
3177 release_sock(sk);
3178 return ret;
bbd6ef87 3179 }
8913336a
PM		 3180 case PACKET_RESERVE:
3181 {
3182 unsigned int val;
3183
3184 if (optlen != sizeof(val))
3185 return -EINVAL;
69e3c75f 3186 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3187 return -EBUSY;
3188 if (copy_from_user(&val, optval, sizeof(val)))
3189 return -EFAULT;
a8069aee
AK		 3190 if (val > INT_MAX)
3191 return -EINVAL;
8913336a
PM		 3192 po->tp_reserve = val;
3193 return 0;
3194 }
69e3c75f
JB		 3195 case PACKET_LOSS:
3196 {
3197 unsigned int val;
3198
3199 if (optlen != sizeof(val))
3200 return -EINVAL;
3201 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3202 return -EBUSY;
3203 if (copy_from_user(&val, optval, sizeof(val)))
3204 return -EFAULT;
3205 po->tp_loss = !!val;
3206 return 0;
3207 }
8dc41944
HX		 3208 case PACKET_AUXDATA:
3209 {
3210 int val;
3211
3212 if (optlen < sizeof(val))
3213 return -EINVAL;
3214 if (copy_from_user(&val, optval, sizeof(val)))
3215 return -EFAULT;
3216
3217 po->auxdata = !!val;
3218 return 0;
3219 }
80feaacb
PWJ		 3220 case PACKET_ORIGDEV:
3221 {
3222 int val;
3223
3224 if (optlen < sizeof(val))
3225 return -EINVAL;
3226 if (copy_from_user(&val, optval, sizeof(val)))
3227 return -EFAULT;
3228
3229 po->origdev = !!val;
3230 return 0;
3231 }
bfd5f4a3
SS		 3232 case PACKET_VNET_HDR:
3233 {
3234 int val;
3235
3236 if (sock->type != SOCK_RAW)
3237 return -EINVAL;
3238 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3239 return -EBUSY;
3240 if (optlen < sizeof(val))
3241 return -EINVAL;
3242 if (copy_from_user(&val, optval, sizeof(val)))
3243 return -EFAULT;
3244
3245 po->has_vnet_hdr = !!val;
3246 return 0;
3247 }
614f60fa
SM		 3248 case PACKET_TIMESTAMP:
3249 {
3250 int val;
3251
3252 if (optlen != sizeof(val))
3253 return -EINVAL;
3254 if (copy_from_user(&val, optval, sizeof(val)))
3255 return -EFAULT;
3256
3257 po->tp_tstamp = val;
3258 return 0;
3259 }
dc99f600
DM		 3260 case PACKET_FANOUT:
3261 {
3262 int val;
3263
3264 if (optlen != sizeof(val))
3265 return -EINVAL;
3266 if (copy_from_user(&val, optval, sizeof(val)))
3267 return -EFAULT;
3268
3269 return fanout_add(sk, val & 0xffff, val >> 16);
3270 }
5920cd3a
PC		 3271 case PACKET_TX_HAS_OFF:
3272 {
3273 unsigned int val;
3274
3275 if (optlen != sizeof(val))
3276 return -EINVAL;
3277 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3278 return -EBUSY;
3279 if (copy_from_user(&val, optval, sizeof(val)))
3280 return -EFAULT;
3281 po->tp_tx_has_off = !!val;
3282 return 0;
3283 }
1da177e4
LT		 3284 default:
3285 return -ENOPROTOOPT;
3286 }
3287}
3288
3289static int packet_getsockopt(struct socket *sock, int level, int optname,
3290 char __user *optval, int __user *optlen)
3291{
3292 int len;
c06fff6e 3293 int val, lv = sizeof(val);
1da177e4
LT		 3294 struct sock *sk = sock->sk;
3295 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3296 void *data = &val;
ee80fbf3 3297 union tpacket_stats_u st;
1da177e4
LT		 3298
3299 if (level != SOL_PACKET)
3300 return -ENOPROTOOPT;
3301
8ae55f04
KK		 3302 if (get_user(len, optlen))
3303 return -EFAULT;
1da177e4
LT		 3304
3305 if (len < 0)
3306 return -EINVAL;
1ce4f28b 3307
69e3c75f 3308 switch (optname) {
1da177e4 3309 case PACKET_STATISTICS:
1da177e4 3310 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3311 memcpy(&st, &po->stats, sizeof(st));
3312 memset(&po->stats, 0, sizeof(po->stats));
3313 spin_unlock_bh(&sk->sk_receive_queue.lock);
3314
f6fb8f10 3315 if (po->tp_version == TPACKET_V3) {
c06fff6e 3316 lv = sizeof(struct tpacket_stats_v3);
fc26e4cf 3317 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3318 data = &st.stats3;
f6fb8f10 3319 } else {
c06fff6e 3320 lv = sizeof(struct tpacket_stats);
fc26e4cf 3321 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3322 data = &st.stats1;
f6fb8f10 3323 }
ee80fbf3 3324
8dc41944
HX		 3325 break;
3326 case PACKET_AUXDATA:
8dc41944 3327 val = po->auxdata;
80feaacb
PWJ		 3328 break;
3329 case PACKET_ORIGDEV:
80feaacb 3330 val = po->origdev;
bfd5f4a3
SS		 3331 break;
3332 case PACKET_VNET_HDR:
bfd5f4a3 3333 val = po->has_vnet_hdr;
1da177e4 3334 break;
bbd6ef87 3335 case PACKET_VERSION:
bbd6ef87 3336 val = po->tp_version;
bbd6ef87
PM		 3337 break;
3338 case PACKET_HDRLEN:
3339 if (len > sizeof(int))
3340 len = sizeof(int);
3341 if (copy_from_user(&val, optval, len))
3342 return -EFAULT;
3343 switch (val) {
3344 case TPACKET_V1:
3345 val = sizeof(struct tpacket_hdr);
3346 break;
3347 case TPACKET_V2:
3348 val = sizeof(struct tpacket2_hdr);
3349 break;
f6fb8f10 3350 case TPACKET_V3:
3351 val = sizeof(struct tpacket3_hdr);
3352 break;
bbd6ef87
PM		 3353 default:
3354 return -EINVAL;
3355 }
bbd6ef87 3356 break;
8913336a 3357 case PACKET_RESERVE:
8913336a 3358 val = po->tp_reserve;
8913336a 3359 break;
69e3c75f 3360 case PACKET_LOSS:
69e3c75f 3361 val = po->tp_loss;
69e3c75f 3362 break;
614f60fa 3363 case PACKET_TIMESTAMP:
614f60fa 3364 val = po->tp_tstamp;
614f60fa 3365 break;
dc99f600 3366 case PACKET_FANOUT:
dc99f600
DM		 3367 val = (po->fanout ?
3368 ((u32)po->fanout->id |
77f65ebd
WB		 3369 ((u32)po->fanout->type << 16) |
3370 ((u32)po->fanout->flags << 24)) :
dc99f600 3371 0);
dc99f600 3372 break;
5920cd3a
PC		 3373 case PACKET_TX_HAS_OFF:
3374 val = po->tp_tx_has_off;
3375 break;
1da177e4
LT		 3376 default:
3377 return -ENOPROTOOPT;
3378 }
3379
c06fff6e
ED		 3380 if (len > lv)
3381 len = lv;
8ae55f04
KK		 3382 if (put_user(len, optlen))
3383 return -EFAULT;
8dc41944
HX		 3384 if (copy_to_user(optval, data, len))
3385 return -EFAULT;
8ae55f04 3386 return 0;
1da177e4
LT		 3387}
3388
3389
3390static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3391{
3392 struct sock *sk;
ad930650 3393 struct net_device *dev = data;
c346dca1 3394 struct net *net = dev_net(dev);
1da177e4 3395
808f5114 3396 rcu_read_lock();
b67bfe0d 3397 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3398 struct packet_sock *po = pkt_sk(sk);
3399
3400 switch (msg) {
3401 case NETDEV_UNREGISTER:
1da177e4
LT		 3402 if (po->mclist)
3403 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3404 /* fallthrough */
3405
1da177e4
LT		 3406 case NETDEV_DOWN:
3407 if (dev->ifindex == po->ifindex) {
3408 spin_lock(&po->bind_lock);
3409 if (po->running) {
ce06b03e 3410 __unregister_prot_hook(sk, false);
1da177e4
LT		 3411 sk->sk_err = ENETDOWN;
3412 if (!sock_flag(sk, SOCK_DEAD))
3413 sk->sk_error_report(sk);
3414 }
3415 if (msg == NETDEV_UNREGISTER) {
c3ac8a13 3416 packet_cached_dev_reset(po);
1da177e4 3417 po->ifindex = -1;
160ff18a
BG		 3418 if (po->prot_hook.dev)
3419 dev_put(po->prot_hook.dev);
1da177e4
LT		 3420 po->prot_hook.dev = NULL;
3421 }
3422 spin_unlock(&po->bind_lock);
3423 }
3424 break;
3425 case NETDEV_UP:
808f5114 3426 if (dev->ifindex == po->ifindex) {
3427 spin_lock(&po->bind_lock);
ce06b03e
DM		 3428 if (po->num)
3429 register_prot_hook(sk);
808f5114 3430 spin_unlock(&po->bind_lock);
1da177e4 3431 }
1da177e4
LT		 3432 break;
3433 }
3434 }
808f5114 3435 rcu_read_unlock();
1da177e4
LT		 3436 return NOTIFY_DONE;
3437}
3438
3439
3440static int packet_ioctl(struct socket *sock, unsigned int cmd,
3441 unsigned long arg)
3442{
3443 struct sock *sk = sock->sk;
3444
69e3c75f 3445 switch (cmd) {
40d4e3df
ED		 3446 case SIOCOUTQ:
3447 {
3448 int amount = sk_wmem_alloc_get(sk);
31e6d363 3449
40d4e3df
ED		 3450 return put_user(amount, (int __user *)arg);
3451 }
3452 case SIOCINQ:
3453 {
3454 struct sk_buff *skb;
3455 int amount = 0;
3456
3457 spin_lock_bh(&sk->sk_receive_queue.lock);
3458 skb = skb_peek(&sk->sk_receive_queue);
3459 if (skb)
3460 amount = skb->len;
3461 spin_unlock_bh(&sk->sk_receive_queue.lock);
3462 return put_user(amount, (int __user *)arg);
3463 }
3464 case SIOCGSTAMP:
3465 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3466 case SIOCGSTAMPNS:
3467 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3468
1da177e4 3469#ifdef CONFIG_INET
40d4e3df
ED		 3470 case SIOCADDRT:
3471 case SIOCDELRT:
3472 case SIOCDARP:
3473 case SIOCGARP:
3474 case SIOCSARP:
3475 case SIOCGIFADDR:
3476 case SIOCSIFADDR:
3477 case SIOCGIFBRDADDR:
3478 case SIOCSIFBRDADDR:
3479 case SIOCGIFNETMASK:
3480 case SIOCSIFNETMASK:
3481 case SIOCGIFDSTADDR:
3482 case SIOCSIFDSTADDR:
3483 case SIOCSIFFLAGS:
40d4e3df 3484 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3485#endif
3486
40d4e3df
ED		 3487 default:
3488 return -ENOIOCTLCMD;
1da177e4
LT		 3489 }
3490 return 0;
3491}
3492
40d4e3df 3493static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3494 poll_table *wait)
3495{
3496 struct sock *sk = sock->sk;
3497 struct packet_sock *po = pkt_sk(sk);
3498 unsigned int mask = datagram_poll(file, sock, wait);
3499
3500 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3501 if (po->rx_ring.pg_vec) {
f6fb8f10 3502 if (!packet_previous_rx_frame(po, &po->rx_ring,
3503 TP_STATUS_KERNEL))
1da177e4
LT		 3504 mask |= POLLIN | POLLRDNORM;
3505 }
3506 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3507 spin_lock_bh(&sk->sk_write_queue.lock);
3508 if (po->tx_ring.pg_vec) {
3509 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3510 mask |= POLLOUT | POLLWRNORM;
3511 }
3512 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3513 return mask;
3514}
3515
3516
3517/* Dirty? Well, I still did not learn better way to account
3518 * for user mmaps.
3519 */
3520
3521static void packet_mm_open(struct vm_area_struct *vma)
3522{
3523 struct file *file = vma->vm_file;
40d4e3df 3524 struct socket *sock = file->private_data;
1da177e4 3525 struct sock *sk = sock->sk;
1ce4f28b 3526
1da177e4
LT		 3527 if (sk)
3528 atomic_inc(&pkt_sk(sk)->mapped);
3529}
3530
3531static void packet_mm_close(struct vm_area_struct *vma)
3532{
3533 struct file *file = vma->vm_file;
40d4e3df 3534 struct socket *sock = file->private_data;
1da177e4 3535 struct sock *sk = sock->sk;
1ce4f28b 3536
1da177e4
LT		 3537 if (sk)
3538 atomic_dec(&pkt_sk(sk)->mapped);
3539}
3540
f0f37e2f 3541static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3542 .open = packet_mm_open,
3543 .close = packet_mm_close,
1da177e4
LT		 3544};
3545
0e3125c7
NH		 3546static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3547 unsigned int len)
1da177e4
LT		 3548{
3549 int i;
3550
4ebf0ae2 3551 for (i = 0; i < len; i++) {
0e3125c7 3552 if (likely(pg_vec[i].buffer)) {
c56b4d90 3553 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3554 vfree(pg_vec[i].buffer);
3555 else
3556 free_pages((unsigned long)pg_vec[i].buffer,
3557 order);
3558 pg_vec[i].buffer = NULL;
3559 }
1da177e4
LT		 3560 }
3561 kfree(pg_vec);
3562}
3563
eea49cc9 3564static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3565{
0e3125c7
NH		 3566 char *buffer = NULL;
3567 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3568 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3569
3570 buffer = (char *) __get_free_pages(gfp_flags, order);
3571
3572 if (buffer)
3573 return buffer;
3574
3575 /*
3576 * __get_free_pages failed, fall back to vmalloc
3577 */
bbce5a59 3578 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3579
0e3125c7
NH		 3580 if (buffer)
3581 return buffer;
3582
3583 /*
3584 * vmalloc failed, lets dig into swap here
3585 */
0e3125c7
NH		 3586 gfp_flags &= ~__GFP_NORETRY;
3587 buffer = (char *)__get_free_pages(gfp_flags, order);
3588 if (buffer)
3589 return buffer;
3590
3591 /*
3592 * complete and utter failure
3593 */
3594 return NULL;
4ebf0ae2
DM		 3595}
3596
0e3125c7 3597static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3598{
3599 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3600 struct pgv *pg_vec;
4ebf0ae2
DM		 3601 int i;
3602
0e3125c7 3603 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3604 if (unlikely(!pg_vec))
3605 goto out;
3606
3607 for (i = 0; i < block_nr; i++) {
c56b4d90 3608 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3609 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3610 goto out_free_pgvec;
3611 }
3612
3613out:
3614 return pg_vec;
3615
3616out_free_pgvec:
3617 free_pg_vec(pg_vec, order, block_nr);
3618 pg_vec = NULL;
3619 goto out;
3620}
1da177e4 3621
f6fb8f10 3622static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3623 int closing, int tx_ring)
1da177e4 3624{
0e3125c7 3625 struct pgv *pg_vec = NULL;
1da177e4 3626 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3627 int was_running, order = 0;
69e3c75f
JB		 3628 struct packet_ring_buffer *rb;
3629 struct sk_buff_head *rb_queue;
0e11c91e 3630 __be16 num;
f6fb8f10 3631 int err = -EINVAL;
3632 /* Added to avoid minimal code churn */
3633 struct tpacket_req *req = &req_u->req;
3634
4bfb6dd7 3635 lock_sock(sk);
f6fb8f10 3636 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3637 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3638 WARN(1, "Tx-ring is not supported.\n");
3639 goto out;
3640 }
1ce4f28b 3641
69e3c75f
JB		 3642 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3643 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3644
69e3c75f
JB		 3645 err = -EBUSY;
3646 if (!closing) {
3647 if (atomic_read(&po->mapped))
3648 goto out;
3649 if (atomic_read(&rb->pending))
3650 goto out;
3651 }
1da177e4 3652
69e3c75f
JB		 3653 if (req->tp_block_nr) {
3654 /* Sanity tests and some calculations */
3655 err = -EBUSY;
3656 if (unlikely(rb->pg_vec))
3657 goto out;
1da177e4 3658
bbd6ef87
PM		 3659 switch (po->tp_version) {
3660 case TPACKET_V1:
3661 po->tp_hdrlen = TPACKET_HDRLEN;
3662 break;
3663 case TPACKET_V2:
3664 po->tp_hdrlen = TPACKET2_HDRLEN;
3665 break;
f6fb8f10 3666 case TPACKET_V3:
3667 po->tp_hdrlen = TPACKET3_HDRLEN;
3668 break;
bbd6ef87
PM		 3669 }
3670
69e3c75f 3671 err = -EINVAL;
4ebf0ae2 3672 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3673 goto out;
4ebf0ae2 3674 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3675 goto out;
4035ed7b 3676 if (po->tp_version >= TPACKET_V3 &&
56bc42d0
AK		 3677 req->tp_block_size <=
3678 BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv))
4035ed7b 3679 goto out;
8913336a 3680 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3681 po->tp_reserve))
3682 goto out;
4ebf0ae2 3683 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3684 goto out;
1da177e4 3685
69e3c75f
JB		 3686 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3687 if (unlikely(rb->frames_per_block <= 0))
3688 goto out;
492980ca
AK		 3689 if (unlikely(req->tp_block_size > UINT_MAX / req->tp_block_nr))
3690 goto out;
69e3c75f
JB		 3691 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3692 req->tp_frame_nr))
3693 goto out;
1da177e4
LT		 3694
3695 err = -ENOMEM;
4ebf0ae2
DM		 3696 order = get_order(req->tp_block_size);
3697 pg_vec = alloc_pg_vec(req, order);
3698 if (unlikely(!pg_vec))
1da177e4 3699 goto out;
f6fb8f10 3700 switch (po->tp_version) {
3701 case TPACKET_V3:
3702 /* Transmit path is not supported. We checked
3703 * it above but just being paranoid
3704 */
3705 if (!tx_ring)
3706 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
c0c294a3 3707 break;
f6fb8f10 3708 default:
3709 break;
3710 }
69e3c75f
JB		 3711 }
3712 /* Done */
3713 else {
3714 err = -EINVAL;
4ebf0ae2 3715 if (unlikely(req->tp_frame_nr))
69e3c75f 3716 goto out;
1da177e4
LT		 3717 }
3718
1da177e4
LT		 3719
3720 /* Detach socket from network */
3721 spin_lock(&po->bind_lock);
3722 was_running = po->running;
3723 num = po->num;
3724 if (was_running) {
1da177e4 3725 po->num = 0;
ce06b03e 3726 __unregister_prot_hook(sk, false);
1da177e4
LT		 3727 }
3728 spin_unlock(&po->bind_lock);
1ce4f28b 3729
1da177e4
LT		 3730 synchronize_net();
3731
3732 err = -EBUSY;
905db440 3733 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3734 if (closing || atomic_read(&po->mapped) == 0) {
3735 err = 0;
69e3c75f 3736 spin_lock_bh(&rb_queue->lock);
c053fd96 3737 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3738 rb->frame_max = (req->tp_frame_nr - 1);
3739 rb->head = 0;
3740 rb->frame_size = req->tp_frame_size;
3741 spin_unlock_bh(&rb_queue->lock);
3742
c053fd96
CG		 3743 swap(rb->pg_vec_order, order);
3744 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3745
3746 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3747 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3748 tpacket_rcv : packet_rcv;
3749 skb_queue_purge(rb_queue);
1da177e4 3750 if (atomic_read(&po->mapped))
40d4e3df
ED		 3751 pr_err("packet_mmap: vma is busy: %d\n",
3752 atomic_read(&po->mapped));
1da177e4 3753 }
905db440 3754 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3755
3756 spin_lock(&po->bind_lock);
ce06b03e 3757 if (was_running) {
1da177e4 3758 po->num = num;
ce06b03e 3759 register_prot_hook(sk);
1da177e4
LT		 3760 }
3761 spin_unlock(&po->bind_lock);
f6fb8f10 3762 if (closing && (po->tp_version > TPACKET_V2)) {
3763 /* Because we don't support block-based V3 on tx-ring */
3764 if (!tx_ring)
3765 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3766 }
1da177e4 3767
1da177e4
LT		 3768 if (pg_vec)
3769 free_pg_vec(pg_vec, order, req->tp_block_nr);
3770out:
4bfb6dd7 3771 release_sock(sk);
1da177e4
LT		 3772 return err;
3773}
3774
69e3c75f
JB		 3775static int packet_mmap(struct file *file, struct socket *sock,
3776 struct vm_area_struct *vma)
1da177e4
LT		 3777{
3778 struct sock *sk = sock->sk;
3779 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3780 unsigned long size, expected_size;
3781 struct packet_ring_buffer *rb;
1da177e4
LT		 3782 unsigned long start;
3783 int err = -EINVAL;
3784 int i;
3785
3786 if (vma->vm_pgoff)
3787 return -EINVAL;
3788
905db440 3789 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3790
3791 expected_size = 0;
3792 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3793 if (rb->pg_vec) {
3794 expected_size += rb->pg_vec_len
3795 * rb->pg_vec_pages
3796 * PAGE_SIZE;
3797 }
3798 }
3799
3800 if (expected_size == 0)
1da177e4 3801 goto out;
69e3c75f
JB		 3802
3803 size = vma->vm_end - vma->vm_start;
3804 if (size != expected_size)
1da177e4
LT		 3805 goto out;
3806
1da177e4 3807 start = vma->vm_start;
69e3c75f
JB		 3808 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3809 if (rb->pg_vec == NULL)
3810 continue;
3811
3812 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3813 struct page *page;
3814 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3815 int pg_num;
3816
c56b4d90
CG		 3817 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3818 page = pgv_to_page(kaddr);
69e3c75f
JB		 3819 err = vm_insert_page(vma, start, page);
3820 if (unlikely(err))
3821 goto out;
3822 start += PAGE_SIZE;
0e3125c7 3823 kaddr += PAGE_SIZE;
69e3c75f 3824 }
4ebf0ae2 3825 }
1da177e4 3826 }
69e3c75f 3827
4ebf0ae2 3828 atomic_inc(&po->mapped);
1da177e4
LT		 3829 vma->vm_ops = &packet_mmap_ops;
3830 err = 0;
3831
3832out:
905db440 3833 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3834 return err;
3835}
1da177e4 3836
90ddc4f0 3837static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3838 .family = PF_PACKET,
3839 .owner = THIS_MODULE,
3840 .release = packet_release,
3841 .bind = packet_bind_spkt,
3842 .connect = sock_no_connect,
3843 .socketpair = sock_no_socketpair,
3844 .accept = sock_no_accept,
3845 .getname = packet_getname_spkt,
3846 .poll = datagram_poll,
3847 .ioctl = packet_ioctl,
3848 .listen = sock_no_listen,
3849 .shutdown = sock_no_shutdown,
3850 .setsockopt = sock_no_setsockopt,
3851 .getsockopt = sock_no_getsockopt,
3852 .sendmsg = packet_sendmsg_spkt,
3853 .recvmsg = packet_recvmsg,
3854 .mmap = sock_no_mmap,
3855 .sendpage = sock_no_sendpage,
3856};
1da177e4 3857
90ddc4f0 3858static const struct proto_ops packet_ops = {
1da177e4
LT		 3859 .family = PF_PACKET,
3860 .owner = THIS_MODULE,
3861 .release = packet_release,
3862 .bind = packet_bind,
3863 .connect = sock_no_connect,
3864 .socketpair = sock_no_socketpair,
3865 .accept = sock_no_accept,
1ce4f28b 3866 .getname = packet_getname,
1da177e4
LT		 3867 .poll = packet_poll,
3868 .ioctl = packet_ioctl,
3869 .listen = sock_no_listen,
3870 .shutdown = sock_no_shutdown,
3871 .setsockopt = packet_setsockopt,
3872 .getsockopt = packet_getsockopt,
3873 .sendmsg = packet_sendmsg,
3874 .recvmsg = packet_recvmsg,
3875 .mmap = packet_mmap,
3876 .sendpage = sock_no_sendpage,
3877};
3878
ec1b4cf7 3879static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3880 .family = PF_PACKET,
3881 .create = packet_create,
3882 .owner = THIS_MODULE,
3883};
3884
3885static struct notifier_block packet_netdev_notifier = {
40d4e3df 3886 .notifier_call = packet_notifier,
1da177e4
LT		 3887};
3888
3889#ifdef CONFIG_PROC_FS
1da177e4
LT		 3890
3891static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3892 __acquires(RCU)
1da177e4 3893{
e372c414 3894 struct net *net = seq_file_net(seq);
808f5114 3895
3896 rcu_read_lock();
3897 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3898}
3899
3900static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3901{
1bf40954 3902 struct net *net = seq_file_net(seq);
808f5114 3903 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3904}
3905
3906static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3907 __releases(RCU)
1da177e4 3908{
808f5114 3909 rcu_read_unlock();
1da177e4
LT		 3910}
3911
1ce4f28b 3912static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3913{
3914 if (v == SEQ_START_TOKEN)
3915 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3916 else {
b7ceabd9 3917 struct sock *s = sk_entry(v);
1da177e4
LT		 3918 const struct packet_sock *po = pkt_sk(s);
3919
3920 seq_printf(seq,
71338aa7 3921 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3922 s,
3923 atomic_read(&s->sk_refcnt),
3924 s->sk_type,
3925 ntohs(po->num),
3926 po->ifindex,
3927 po->running,
3928 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3929 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3930 sock_i_ino(s));
1da177e4
LT		 3931 }
3932
3933 return 0;
3934}
3935
56b3d975 3936static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3937 .start = packet_seq_start,
3938 .next = packet_seq_next,
3939 .stop = packet_seq_stop,
3940 .show = packet_seq_show,
3941};
3942
3943static int packet_seq_open(struct inode *inode, struct file *file)
3944{
e372c414
DL		 3945 return seq_open_net(inode, file, &packet_seq_ops,
3946 sizeof(struct seq_net_private));
1da177e4
LT		 3947}
3948
da7071d7 3949static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3950 .owner = THIS_MODULE,
3951 .open = packet_seq_open,
3952 .read = seq_read,
3953 .llseek = seq_lseek,
e372c414 3954 .release = seq_release_net,
1da177e4
LT		 3955};
3956
3957#endif
3958
2c8c1e72 3959static int __net_init packet_net_init(struct net *net)
d12d01d6 3960{
0fa7fa98 3961 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3962 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3963
d4beaa66 3964 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3965 return -ENOMEM;
3966
3967 return 0;
3968}
3969
2c8c1e72 3970static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3971{
ece31ffd 3972 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3973}
3974
3975static struct pernet_operations packet_net_ops = {
3976 .init = packet_net_init,
3977 .exit = packet_net_exit,
3978};
3979
3980
1da177e4
LT		 3981static void __exit packet_exit(void)
3982{
1da177e4 3983 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3984 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3985 sock_unregister(PF_PACKET);
3986 proto_unregister(&packet_proto);
3987}
3988
3989static int __init packet_init(void)
3990{
3991 int rc = proto_register(&packet_proto, 0);
3992
3993 if (rc != 0)
3994 goto out;
3995
3996 sock_register(&packet_family_ops);
d12d01d6 3997 register_pernet_subsys(&packet_net_ops);
1da177e4 3998 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3999out:
4000 return rc;
4001}
4002
4003module_init(packet_init);
4004module_exit(packet_exit);
4005MODULE_LICENSE("GPL");
4006MODULE_ALIAS_NETPROTO(PF_PACKET);
kernel source for telekom puls (alcatel/mediatek)