projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: core: introduce skb_probe_transport_header()
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
c1aad275 91#include <net/flow_keys.h>
1da177e4
LT		 92
93#ifdef CONFIG_INET
94#include <net/inet_common.h>
95#endif
96
2787b04b
PE		 97#include "internal.h"
98
1da177e4
LT		 99/*
100 Assumptions:
101 - if device has no dev->hard_header routine, it adds and removes ll header
102 inside itself. In this case ll header is invisible outside of device,
103 but higher levels still should reserve dev->hard_header_len.
104 Some devices are enough clever to reallocate skb, when header
105 will not fit to reserved space (tunnel), another ones are silly
106 (PPP).
107 - packet socket receives packets with pulled ll header,
108 so that SOCK_RAW should push it back.
109
110On receive:
111-----------
112
113Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 114 mac_header -> ll header
115 data -> data
1da177e4
LT		 116
117Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 118 mac_header -> ll header
119 data -> ll header
1da177e4
LT		 120
121Incoming, dev->hard_header==NULL
b0e380b1
ACM		 122 mac_header -> UNKNOWN position. It is very likely, that it points to ll
123 header. PPP makes it, that is wrong, because introduce
db0c58f9 124 assymetry between rx and tx paths.
b0e380b1 125 data -> data
1da177e4
LT		 126
127Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 128 mac_header -> data. ll header is still not built!
129 data -> data
1da177e4
LT		 130
131Resume
132 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
133
134
135On transmit:
136------------
137
138dev->hard_header != NULL
b0e380b1
ACM		 139 mac_header -> ll header
140 data -> ll header
1da177e4
LT		 141
142dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 143 mac_header -> data
144 data -> data
1da177e4
LT		 145
146 We should set nh.raw on output to correct posistion,
147 packet classifier depends on it.
148 */
149
1da177e4
LT		 150/* Private packet socket structures. */
151
0fb375fb
EB		 152/* identical to struct packet_mreq except it has
153 * a longer address field.
154 */
40d4e3df 155struct packet_mreq_max {
0fb375fb
EB		 156 int mr_ifindex;
157 unsigned short mr_type;
158 unsigned short mr_alen;
159 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 160};
a2efcfa0 161
f6fb8f10 162static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 163 int closing, int tx_ring);
164
f6fb8f10 165
166#define V3_ALIGNMENT (8)
167
bc59ba39 168#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 169
170#define BLK_PLUS_PRIV(sz_of_priv) \
171 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
172
f6fb8f10 173#define PGV_FROM_VMALLOC 1
69e3c75f 174
f6fb8f10 175#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
176#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
177#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
178#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
179#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
180#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
181#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
182
69e3c75f
JB		 183struct packet_sock;
184static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 185static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
186 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 187
f6fb8f10 188static void *packet_previous_frame(struct packet_sock *po,
189 struct packet_ring_buffer *rb,
190 int status);
191static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 192static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
193 struct tpacket_block_desc *);
194static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 195 struct packet_sock *);
bc59ba39 196static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 197 struct packet_sock *, unsigned int status);
bc59ba39 198static int prb_queue_frozen(struct tpacket_kbdq_core *);
199static void prb_open_block(struct tpacket_kbdq_core *,
200 struct tpacket_block_desc *);
f6fb8f10 201static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 202static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
203static void prb_init_blk_timer(struct packet_sock *,
204 struct tpacket_kbdq_core *,
205 void (*func) (unsigned long));
206static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
207static void prb_clear_rxhash(struct tpacket_kbdq_core *,
208 struct tpacket3_hdr *);
209static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
210 struct tpacket3_hdr *);
1da177e4
LT		 211static void packet_flush_mclist(struct sock *sk);
212
ffbc6111
HX		 213struct packet_skb_cb {
214 unsigned int origlen;
215 union {
216 struct sockaddr_pkt pkt;
217 struct sockaddr_ll ll;
218 } sa;
219};
220
221#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 222
bc59ba39 223#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 224#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 225 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 226#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 227 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 228#define GET_NEXT_PRB_BLK_NUM(x) \
229 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
230 ((x)->kactive_blk_num+1) : 0)
231
dc99f600
DM		 232static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
233static void __fanout_link(struct sock *sk, struct packet_sock *po);
234
ce06b03e
DM		 235/* register_prot_hook must be invoked with the po->bind_lock held,
236 * or from a context in which asynchronous accesses to the packet
237 * socket is not possible (packet_create()).
238 */
239static void register_prot_hook(struct sock *sk)
240{
241 struct packet_sock *po = pkt_sk(sk);
242 if (!po->running) {
dc99f600
DM		 243 if (po->fanout)
244 __fanout_link(sk, po);
245 else
246 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 247 sock_hold(sk);
248 po->running = 1;
249 }
250}
251
252/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
253 * held. If the sync parameter is true, we will temporarily drop
254 * the po->bind_lock and do a synchronize_net to make sure no
255 * asynchronous packet processing paths still refer to the elements
256 * of po->prot_hook. If the sync parameter is false, it is the
257 * callers responsibility to take care of this.
258 */
259static void __unregister_prot_hook(struct sock *sk, bool sync)
260{
261 struct packet_sock *po = pkt_sk(sk);
262
263 po->running = 0;
dc99f600
DM		 264 if (po->fanout)
265 __fanout_unlink(sk, po);
266 else
267 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 268 __sock_put(sk);
269
270 if (sync) {
271 spin_unlock(&po->bind_lock);
272 synchronize_net();
273 spin_lock(&po->bind_lock);
274 }
275}
276
277static void unregister_prot_hook(struct sock *sk, bool sync)
278{
279 struct packet_sock *po = pkt_sk(sk);
280
281 if (po->running)
282 __unregister_prot_hook(sk, sync);
283}
284
f6dafa95 285static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 286{
287 if (is_vmalloc_addr(addr))
288 return vmalloc_to_page(addr);
289 return virt_to_page(addr);
290}
291
69e3c75f 292static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 293{
bbd6ef87
PM		 294 union {
295 struct tpacket_hdr *h1;
296 struct tpacket2_hdr *h2;
297 void *raw;
298 } h;
1da177e4 299
69e3c75f 300 h.raw = frame;
bbd6ef87
PM		 301 switch (po->tp_version) {
302 case TPACKET_V1:
69e3c75f 303 h.h1->tp_status = status;
0af55bb5 304 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 305 break;
306 case TPACKET_V2:
69e3c75f 307 h.h2->tp_status = status;
0af55bb5 308 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 309 break;
f6fb8f10 310 case TPACKET_V3:
69e3c75f 311 default:
f6fb8f10 312 WARN(1, "TPACKET version not supported.\n");
69e3c75f 313 BUG();
bbd6ef87 314 }
69e3c75f
JB		 315
316 smp_wmb();
bbd6ef87
PM		 317}
318
69e3c75f 319static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87
PM		 320{
321 union {
322 struct tpacket_hdr *h1;
323 struct tpacket2_hdr *h2;
324 void *raw;
325 } h;
326
69e3c75f
JB		 327 smp_rmb();
328
bbd6ef87
PM		 329 h.raw = frame;
330 switch (po->tp_version) {
331 case TPACKET_V1:
0af55bb5 332 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 333 return h.h1->tp_status;
bbd6ef87 334 case TPACKET_V2:
0af55bb5 335 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 336 return h.h2->tp_status;
f6fb8f10 337 case TPACKET_V3:
69e3c75f 338 default:
f6fb8f10 339 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 340 BUG();
341 return 0;
bbd6ef87 342 }
1da177e4 343}
69e3c75f
JB		 344
345static void *packet_lookup_frame(struct packet_sock *po,
346 struct packet_ring_buffer *rb,
347 unsigned int position,
348 int status)
349{
350 unsigned int pg_vec_pos, frame_offset;
351 union {
352 struct tpacket_hdr *h1;
353 struct tpacket2_hdr *h2;
354 void *raw;
355 } h;
356
357 pg_vec_pos = position / rb->frames_per_block;
358 frame_offset = position % rb->frames_per_block;
359
0e3125c7
NH		 360 h.raw = rb->pg_vec[pg_vec_pos].buffer +
361 (frame_offset * rb->frame_size);
69e3c75f
JB		 362
363 if (status != __packet_get_status(po, h.raw))
364 return NULL;
365
366 return h.raw;
367}
368
eea49cc9 369static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 370 struct packet_ring_buffer *rb,
371 int status)
372{
373 return packet_lookup_frame(po, rb, rb->head, status);
374}
375
bc59ba39 376static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 377{
378 del_timer_sync(&pkc->retire_blk_timer);
379}
380
381static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
382 int tx_ring,
383 struct sk_buff_head *rb_queue)
384{
bc59ba39 385 struct tpacket_kbdq_core *pkc;
f6fb8f10 386
387 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
388
389 spin_lock(&rb_queue->lock);
390 pkc->delete_blk_timer = 1;
391 spin_unlock(&rb_queue->lock);
392
393 prb_del_retire_blk_timer(pkc);
394}
395
396static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 397 struct tpacket_kbdq_core *pkc,
f6fb8f10 398 void (*func) (unsigned long))
399{
400 init_timer(&pkc->retire_blk_timer);
401 pkc->retire_blk_timer.data = (long)po;
402 pkc->retire_blk_timer.function = func;
403 pkc->retire_blk_timer.expires = jiffies;
404}
405
406static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
407{
bc59ba39 408 struct tpacket_kbdq_core *pkc;
f6fb8f10 409
410 if (tx_ring)
411 BUG();
412
413 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
414 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
415}
416
417static int prb_calc_retire_blk_tmo(struct packet_sock *po,
418 int blk_size_in_bytes)
419{
420 struct net_device *dev;
421 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 422 struct ethtool_cmd ecmd;
423 int err;
e440cf2c 424 u32 speed;
f6fb8f10 425
4bc71cb9
JP		 426 rtnl_lock();
427 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
428 if (unlikely(!dev)) {
429 rtnl_unlock();
f6fb8f10 430 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 431 }
432 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 433 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 434 rtnl_unlock();
435 if (!err) {
4bc71cb9
JP		 436 /*
437 * If the link speed is so slow you don't really
438 * need to worry about perf anyways
439 */
e440cf2c 440 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 441 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 442 } else {
443 msec = 1;
444 div = speed / 1000;
f6fb8f10 445 }
446 }
447
448 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
449
450 if (div)
451 mbits /= div;
452
453 tmo = mbits * msec;
454
455 if (div)
456 return tmo+1;
457 return tmo;
458}
459
bc59ba39 460static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 461 union tpacket_req_u *req_u)
462{
463 p1->feature_req_word = req_u->req3.tp_feature_req_word;
464}
465
466static void init_prb_bdqc(struct packet_sock *po,
467 struct packet_ring_buffer *rb,
468 struct pgv *pg_vec,
469 union tpacket_req_u *req_u, int tx_ring)
470{
bc59ba39 471 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
472 struct tpacket_block_desc *pbd;
f6fb8f10 473
474 memset(p1, 0x0, sizeof(*p1));
475
476 p1->knxt_seq_num = 1;
477 p1->pkbdq = pg_vec;
bc59ba39 478 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 479 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 480 p1->kblk_size = req_u->req3.tp_block_size;
481 p1->knum_blocks = req_u->req3.tp_block_nr;
482 p1->hdrlen = po->tp_hdrlen;
483 p1->version = po->tp_version;
484 p1->last_kactive_blk_num = 0;
485 po->stats_u.stats3.tp_freeze_q_cnt = 0;
486 if (req_u->req3.tp_retire_blk_tov)
487 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
488 else
489 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
490 req_u->req3.tp_block_size);
491 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
492 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
493
494 prb_init_ft_ops(p1, req_u);
495 prb_setup_retire_blk_timer(po, tx_ring);
496 prb_open_block(p1, pbd);
497}
498
499/* Do NOT update the last_blk_num first.
500 * Assumes sk_buff_head lock is held.
501 */
bc59ba39 502static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 503{
504 mod_timer(&pkc->retire_blk_timer,
505 jiffies + pkc->tov_in_jiffies);
506 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
507}
508
509/*
510 * Timer logic:
511 * 1) We refresh the timer only when we open a block.
512 * By doing this we don't waste cycles refreshing the timer
513 * on packet-by-packet basis.
514 *
515 * With a 1MB block-size, on a 1Gbps line, it will take
516 * i) ~8 ms to fill a block + ii) memcpy etc.
517 * In this cut we are not accounting for the memcpy time.
518 *
519 * So, if the user sets the 'tmo' to 10ms then the timer
520 * will never fire while the block is still getting filled
521 * (which is what we want). However, the user could choose
522 * to close a block early and that's fine.
523 *
524 * But when the timer does fire, we check whether or not to refresh it.
525 * Since the tmo granularity is in msecs, it is not too expensive
526 * to refresh the timer, lets say every '8' msecs.
527 * Either the user can set the 'tmo' or we can derive it based on
528 * a) line-speed and b) block-size.
529 * prb_calc_retire_blk_tmo() calculates the tmo.
530 *
531 */
532static void prb_retire_rx_blk_timer_expired(unsigned long data)
533{
534 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 535 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 536 unsigned int frozen;
bc59ba39 537 struct tpacket_block_desc *pbd;
f6fb8f10 538
539 spin_lock(&po->sk.sk_receive_queue.lock);
540
541 frozen = prb_queue_frozen(pkc);
542 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
543
544 if (unlikely(pkc->delete_blk_timer))
545 goto out;
546
547 /* We only need to plug the race when the block is partially filled.
548 * tpacket_rcv:
549 * lock(); increment BLOCK_NUM_PKTS; unlock()
550 * copy_bits() is in progress ...
551 * timer fires on other cpu:
552 * we can't retire the current block because copy_bits
553 * is in progress.
554 *
555 */
556 if (BLOCK_NUM_PKTS(pbd)) {
557 while (atomic_read(&pkc->blk_fill_in_prog)) {
558 /* Waiting for skb_copy_bits to finish... */
559 cpu_relax();
560 }
561 }
562
563 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
564 if (!frozen) {
565 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
566 if (!prb_dispatch_next_block(pkc, po))
567 goto refresh_timer;
568 else
569 goto out;
570 } else {
571 /* Case 1. Queue was frozen because user-space was
572 * lagging behind.
573 */
574 if (prb_curr_blk_in_use(pkc, pbd)) {
575 /*
576 * Ok, user-space is still behind.
577 * So just refresh the timer.
578 */
579 goto refresh_timer;
580 } else {
581 /* Case 2. queue was frozen,user-space caught up,
582 * now the link went idle && the timer fired.
583 * We don't have a block to close.So we open this
584 * block and restart the timer.
585 * opening a block thaws the queue,restarts timer
586 * Thawing/timer-refresh is a side effect.
587 */
588 prb_open_block(pkc, pbd);
589 goto out;
590 }
591 }
592 }
593
594refresh_timer:
595 _prb_refresh_rx_retire_blk_timer(pkc);
596
597out:
598 spin_unlock(&po->sk.sk_receive_queue.lock);
599}
600
eea49cc9 601static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 602 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 603{
604 /* Flush everything minus the block header */
605
606#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
607 u8 *start, *end;
608
609 start = (u8 *)pbd1;
610
611 /* Skip the block header(we know header WILL fit in 4K) */
612 start += PAGE_SIZE;
613
614 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
615 for (; start < end; start += PAGE_SIZE)
616 flush_dcache_page(pgv_to_page(start));
617
618 smp_wmb();
619#endif
620
621 /* Now update the block status. */
622
623 BLOCK_STATUS(pbd1) = status;
624
625 /* Flush the block header */
626
627#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
628 start = (u8 *)pbd1;
629 flush_dcache_page(pgv_to_page(start));
630
631 smp_wmb();
632#endif
633}
634
635/*
636 * Side effect:
637 *
638 * 1) flush the block
639 * 2) Increment active_blk_num
640 *
641 * Note:We DONT refresh the timer on purpose.
642 * Because almost always the next block will be opened.
643 */
bc59ba39 644static void prb_close_block(struct tpacket_kbdq_core *pkc1,
645 struct tpacket_block_desc *pbd1,
f6fb8f10 646 struct packet_sock *po, unsigned int stat)
647{
648 __u32 status = TP_STATUS_USER | stat;
649
650 struct tpacket3_hdr *last_pkt;
bc59ba39 651 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 652
653 if (po->stats.tp_drops)
654 status |= TP_STATUS_LOSING;
655
656 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
657 last_pkt->tp_next_offset = 0;
658
659 /* Get the ts of the last pkt */
660 if (BLOCK_NUM_PKTS(pbd1)) {
661 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
662 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
663 } else {
664 /* Ok, we tmo'd - so get the current time */
665 struct timespec ts;
666 getnstimeofday(&ts);
667 h1->ts_last_pkt.ts_sec = ts.tv_sec;
668 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
669 }
670
671 smp_wmb();
672
673 /* Flush the block */
674 prb_flush_block(pkc1, pbd1, status);
675
676 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
677}
678
eea49cc9 679static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 680{
681 pkc->reset_pending_on_curr_blk = 0;
682}
683
684/*
685 * Side effect of opening a block:
686 *
687 * 1) prb_queue is thawed.
688 * 2) retire_blk_timer is refreshed.
689 *
690 */
bc59ba39 691static void prb_open_block(struct tpacket_kbdq_core *pkc1,
692 struct tpacket_block_desc *pbd1)
f6fb8f10 693{
694 struct timespec ts;
bc59ba39 695 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 696
697 smp_rmb();
698
699 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd1))) {
700
701 /* We could have just memset this but we will lose the
702 * flexibility of making the priv area sticky
703 */
704 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
705 BLOCK_NUM_PKTS(pbd1) = 0;
706 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
707 getnstimeofday(&ts);
708 h1->ts_first_pkt.ts_sec = ts.tv_sec;
709 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
710 pkc1->pkblk_start = (char *)pbd1;
e3192690 711 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 712 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
713 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
714 pbd1->version = pkc1->version;
715 pkc1->prev = pkc1->nxt_offset;
716 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
717 prb_thaw_queue(pkc1);
718 _prb_refresh_rx_retire_blk_timer(pkc1);
719
720 smp_wmb();
721
722 return;
723 }
724
725 WARN(1, "ERROR block:%p is NOT FREE status:%d kactive_blk_num:%d\n",
726 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num);
727 dump_stack();
728 BUG();
729}
730
731/*
732 * Queue freeze logic:
733 * 1) Assume tp_block_nr = 8 blocks.
734 * 2) At time 't0', user opens Rx ring.
735 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
736 * 4) user-space is either sleeping or processing block '0'.
737 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
738 * it will close block-7,loop around and try to fill block '0'.
739 * call-flow:
740 * __packet_lookup_frame_in_block
741 * prb_retire_current_block()
742 * prb_dispatch_next_block()
743 * |->(BLOCK_STATUS == USER) evaluates to true
744 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
745 * 6) Now there are two cases:
746 * 6.1) Link goes idle right after the queue is frozen.
747 * But remember, the last open_block() refreshed the timer.
748 * When this timer expires,it will refresh itself so that we can
749 * re-open block-0 in near future.
750 * 6.2) Link is busy and keeps on receiving packets. This is a simple
751 * case and __packet_lookup_frame_in_block will check if block-0
752 * is free and can now be re-used.
753 */
eea49cc9 754static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 755 struct packet_sock *po)
756{
757 pkc->reset_pending_on_curr_blk = 1;
758 po->stats_u.stats3.tp_freeze_q_cnt++;
759}
760
761#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
762
763/*
764 * If the next block is free then we will dispatch it
765 * and return a good offset.
766 * Else, we will freeze the queue.
767 * So, caller must check the return value.
768 */
bc59ba39 769static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 770 struct packet_sock *po)
771{
bc59ba39 772 struct tpacket_block_desc *pbd;
f6fb8f10 773
774 smp_rmb();
775
776 /* 1. Get current block num */
777 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
778
779 /* 2. If this block is currently in_use then freeze the queue */
780 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
781 prb_freeze_queue(pkc, po);
782 return NULL;
783 }
784
785 /*
786 * 3.
787 * open this block and return the offset where the first packet
788 * needs to get stored.
789 */
790 prb_open_block(pkc, pbd);
791 return (void *)pkc->nxt_offset;
792}
793
bc59ba39 794static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 795 struct packet_sock *po, unsigned int status)
796{
bc59ba39 797 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 798
799 /* retire/close the current block */
800 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
801 /*
802 * Plug the case where copy_bits() is in progress on
803 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
804 * have space to copy the pkt in the current block and
805 * called prb_retire_current_block()
806 *
807 * We don't need to worry about the TMO case because
808 * the timer-handler already handled this case.
809 */
810 if (!(status & TP_STATUS_BLK_TMO)) {
811 while (atomic_read(&pkc->blk_fill_in_prog)) {
812 /* Waiting for skb_copy_bits to finish... */
813 cpu_relax();
814 }
815 }
816 prb_close_block(pkc, pbd, po, status);
817 return;
818 }
819
820 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd);
821 dump_stack();
822 BUG();
823}
824
eea49cc9 825static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 826 struct tpacket_block_desc *pbd)
f6fb8f10 827{
828 return TP_STATUS_USER & BLOCK_STATUS(pbd);
829}
830
eea49cc9 831static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 832{
833 return pkc->reset_pending_on_curr_blk;
834}
835
eea49cc9 836static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 837{
bc59ba39 838 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 839 atomic_dec(&pkc->blk_fill_in_prog);
840}
841
eea49cc9 842static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 843 struct tpacket3_hdr *ppd)
844{
845 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
846}
847
eea49cc9 848static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 849 struct tpacket3_hdr *ppd)
850{
851 ppd->hv1.tp_rxhash = 0;
852}
853
eea49cc9 854static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 855 struct tpacket3_hdr *ppd)
856{
857 if (vlan_tx_tag_present(pkc->skb)) {
858 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
859 ppd->tp_status = TP_STATUS_VLAN_VALID;
860 } else {
9e67030a 861 ppd->hv1.tp_vlan_tci = 0;
862 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 863 }
864}
865
bc59ba39 866static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 867 struct tpacket3_hdr *ppd)
868{
869 prb_fill_vlan_info(pkc, ppd);
870
871 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
872 prb_fill_rxhash(pkc, ppd);
873 else
874 prb_clear_rxhash(pkc, ppd);
875}
876
eea49cc9 877static void prb_fill_curr_block(char *curr,
bc59ba39 878 struct tpacket_kbdq_core *pkc,
879 struct tpacket_block_desc *pbd,
f6fb8f10 880 unsigned int len)
881{
882 struct tpacket3_hdr *ppd;
883
884 ppd = (struct tpacket3_hdr *)curr;
885 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
886 pkc->prev = curr;
887 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
888 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
889 BLOCK_NUM_PKTS(pbd) += 1;
890 atomic_inc(&pkc->blk_fill_in_prog);
891 prb_run_all_ft_ops(pkc, ppd);
892}
893
894/* Assumes caller has the sk->rx_queue.lock */
895static void *__packet_lookup_frame_in_block(struct packet_sock *po,
896 struct sk_buff *skb,
897 int status,
898 unsigned int len
899 )
900{
bc59ba39 901 struct tpacket_kbdq_core *pkc;
902 struct tpacket_block_desc *pbd;
f6fb8f10 903 char *curr, *end;
904
e3192690 905 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 906 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
907
908 /* Queue is frozen when user space is lagging behind */
909 if (prb_queue_frozen(pkc)) {
910 /*
911 * Check if that last block which caused the queue to freeze,
912 * is still in_use by user-space.
913 */
914 if (prb_curr_blk_in_use(pkc, pbd)) {
915 /* Can't record this packet */
916 return NULL;
917 } else {
918 /*
919 * Ok, the block was released by user-space.
920 * Now let's open that block.
921 * opening a block also thaws the queue.
922 * Thawing is a side effect.
923 */
924 prb_open_block(pkc, pbd);
925 }
926 }
927
928 smp_mb();
929 curr = pkc->nxt_offset;
930 pkc->skb = skb;
e3192690 931 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 932
933 /* first try the current block */
934 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
935 prb_fill_curr_block(curr, pkc, pbd, len);
936 return (void *)curr;
937 }
938
939 /* Ok, close the current block */
940 prb_retire_current_block(pkc, po, 0);
941
942 /* Now, try to dispatch the next block */
943 curr = (char *)prb_dispatch_next_block(pkc, po);
944 if (curr) {
945 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
946 prb_fill_curr_block(curr, pkc, pbd, len);
947 return (void *)curr;
948 }
949
950 /*
951 * No free blocks are available.user_space hasn't caught up yet.
952 * Queue was just frozen and now this packet will get dropped.
953 */
954 return NULL;
955}
956
eea49cc9 957static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 958 struct sk_buff *skb,
959 int status, unsigned int len)
960{
961 char *curr = NULL;
962 switch (po->tp_version) {
963 case TPACKET_V1:
964 case TPACKET_V2:
965 curr = packet_lookup_frame(po, &po->rx_ring,
966 po->rx_ring.head, status);
967 return curr;
968 case TPACKET_V3:
969 return __packet_lookup_frame_in_block(po, skb, status, len);
970 default:
971 WARN(1, "TPACKET version not supported\n");
972 BUG();
99aa3473 973 return NULL;
f6fb8f10 974 }
975}
976
eea49cc9 977static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 978 struct packet_ring_buffer *rb,
77f65ebd 979 unsigned int idx,
f6fb8f10 980 int status)
981{
bc59ba39 982 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 983 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 984
985 if (status != BLOCK_STATUS(pbd))
986 return NULL;
987 return pbd;
988}
989
eea49cc9 990static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 991{
992 unsigned int prev;
993 if (rb->prb_bdqc.kactive_blk_num)
994 prev = rb->prb_bdqc.kactive_blk_num-1;
995 else
996 prev = rb->prb_bdqc.knum_blocks-1;
997 return prev;
998}
999
1000/* Assumes caller has held the rx_queue.lock */
eea49cc9 1001static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1002 struct packet_ring_buffer *rb,
1003 int status)
1004{
1005 unsigned int previous = prb_previous_blk_num(rb);
1006 return prb_lookup_block(po, rb, previous, status);
1007}
1008
eea49cc9 1009static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1010 struct packet_ring_buffer *rb,
1011 int status)
1012{
1013 if (po->tp_version <= TPACKET_V2)
1014 return packet_previous_frame(po, rb, status);
1015
1016 return __prb_previous_block(po, rb, status);
1017}
1018
eea49cc9 1019static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1020 struct packet_ring_buffer *rb)
1021{
1022 switch (po->tp_version) {
1023 case TPACKET_V1:
1024 case TPACKET_V2:
1025 return packet_increment_head(rb);
1026 case TPACKET_V3:
1027 default:
1028 WARN(1, "TPACKET version not supported.\n");
1029 BUG();
1030 return;
1031 }
1032}
1033
eea49cc9 1034static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1035 struct packet_ring_buffer *rb,
1036 int status)
1037{
1038 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1039 return packet_lookup_frame(po, rb, previous, status);
1040}
1041
eea49cc9 1042static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1043{
1044 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1045}
1046
77f65ebd
WB		 1047static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1048{
1049 struct sock *sk = &po->sk;
1050 bool has_room;
1051
1052 if (po->prot_hook.func != tpacket_rcv)
1053 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1054 <= sk->sk_rcvbuf;
1055
1056 spin_lock(&sk->sk_receive_queue.lock);
1057 if (po->tp_version == TPACKET_V3)
1058 has_room = prb_lookup_block(po, &po->rx_ring,
1059 po->rx_ring.prb_bdqc.kactive_blk_num,
1060 TP_STATUS_KERNEL);
1061 else
1062 has_room = packet_lookup_frame(po, &po->rx_ring,
1063 po->rx_ring.head,
1064 TP_STATUS_KERNEL);
1065 spin_unlock(&sk->sk_receive_queue.lock);
1066
1067 return has_room;
1068}
1069
1da177e4
LT		 1070static void packet_sock_destruct(struct sock *sk)
1071{
ed85b565
RC		 1072 skb_queue_purge(&sk->sk_error_queue);
1073
547b792c
IJ		 1074 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1075 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1076
1077 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1078 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1079 return;
1080 }
1081
17ab56a2 1082 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1083}
1084
dc99f600
DM		 1085static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1086{
1087 int x = atomic_read(&f->rr_cur) + 1;
1088
1089 if (x >= num)
1090 x = 0;
1091
1092 return x;
1093}
1094
77f65ebd
WB		 1095static unsigned int fanout_demux_hash(struct packet_fanout *f,
1096 struct sk_buff *skb,
1097 unsigned int num)
dc99f600 1098{
77f65ebd 1099 return (((u64)skb->rxhash) * num) >> 32;
dc99f600
DM		 1100}
1101
77f65ebd
WB		 1102static unsigned int fanout_demux_lb(struct packet_fanout *f,
1103 struct sk_buff *skb,
1104 unsigned int num)
dc99f600
DM		 1105{
1106 int cur, old;
1107
1108 cur = atomic_read(&f->rr_cur);
1109 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1110 fanout_rr_next(f, num))) != cur)
1111 cur = old;
77f65ebd
WB		 1112 return cur;
1113}
1114
1115static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1116 struct sk_buff *skb,
1117 unsigned int num)
1118{
1119 return smp_processor_id() % num;
dc99f600
DM		 1120}
1121
77f65ebd
WB		 1122static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1123 struct sk_buff *skb,
1124 unsigned int idx, unsigned int skip,
1125 unsigned int num)
95ec3eb4 1126{
77f65ebd 1127 unsigned int i, j;
95ec3eb4 1128
77f65ebd
WB		 1129 i = j = min_t(int, f->next[idx], num - 1);
1130 do {
1131 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1132 if (i != j)
1133 f->next[idx] = i;
1134 return i;
1135 }
1136 if (++i == num)
1137 i = 0;
1138 } while (i != j);
1139
1140 return idx;
1141}
1142
1143static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1144{
1145 return f->flags & (flag >> 8);
95ec3eb4
DM		 1146}
1147
95ec3eb4
DM		 1148static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1149 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1150{
1151 struct packet_fanout *f = pt->af_packet_priv;
1152 unsigned int num = f->num_members;
1153 struct packet_sock *po;
77f65ebd 1154 unsigned int idx;
dc99f600
DM		 1155
1156 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1157 !num) {
1158 kfree_skb(skb);
1159 return 0;
1160 }
1161
95ec3eb4
DM		 1162 switch (f->type) {
1163 case PACKET_FANOUT_HASH:
1164 default:
77f65ebd 1165 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1166 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1167 if (!skb)
1168 return 0;
1169 }
1170 skb_get_rxhash(skb);
77f65ebd 1171 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1172 break;
1173 case PACKET_FANOUT_LB:
77f65ebd 1174 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1175 break;
1176 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1177 idx = fanout_demux_cpu(f, skb, num);
1178 break;
1179 case PACKET_FANOUT_ROLLOVER:
1180 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1181 break;
dc99f600
DM		 1182 }
1183
77f65ebd
WB		 1184 po = pkt_sk(f->arr[idx]);
1185 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1186 unlikely(!packet_rcv_has_room(po, skb))) {
1187 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1188 po = pkt_sk(f->arr[idx]);
1189 }
dc99f600
DM		 1190
1191 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1192}
1193
fff3321d
PE		 1194DEFINE_MUTEX(fanout_mutex);
1195EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1196static LIST_HEAD(fanout_list);
1197
1198static void __fanout_link(struct sock *sk, struct packet_sock *po)
1199{
1200 struct packet_fanout *f = po->fanout;
1201
1202 spin_lock(&f->lock);
1203 f->arr[f->num_members] = sk;
1204 smp_wmb();
1205 f->num_members++;
1206 spin_unlock(&f->lock);
1207}
1208
1209static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1210{
1211 struct packet_fanout *f = po->fanout;
1212 int i;
1213
1214 spin_lock(&f->lock);
1215 for (i = 0; i < f->num_members; i++) {
1216 if (f->arr[i] == sk)
1217 break;
1218 }
1219 BUG_ON(i >= f->num_members);
1220 f->arr[i] = f->arr[f->num_members - 1];
1221 f->num_members--;
1222 spin_unlock(&f->lock);
1223}
1224
a0dfb263 1225static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1226{
1227 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1228 return true;
1229
1230 return false;
1231}
1232
7736d33f 1233static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1234{
1235 struct packet_sock *po = pkt_sk(sk);
1236 struct packet_fanout *f, *match;
7736d33f 1237 u8 type = type_flags & 0xff;
77f65ebd 1238 u8 flags = type_flags >> 8;
dc99f600
DM		 1239 int err;
1240
1241 switch (type) {
77f65ebd
WB		 1242 case PACKET_FANOUT_ROLLOVER:
1243 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1244 return -EINVAL;
dc99f600
DM		 1245 case PACKET_FANOUT_HASH:
1246 case PACKET_FANOUT_LB:
95ec3eb4 1247 case PACKET_FANOUT_CPU:
dc99f600
DM		 1248 break;
1249 default:
1250 return -EINVAL;
1251 }
1252
1253 if (!po->running)
1254 return -EINVAL;
1255
1256 if (po->fanout)
1257 return -EALREADY;
1258
1259 mutex_lock(&fanout_mutex);
1260 match = NULL;
1261 list_for_each_entry(f, &fanout_list, list) {
1262 if (f->id == id &&
1263 read_pnet(&f->net) == sock_net(sk)) {
1264 match = f;
1265 break;
1266 }
1267 }
afe62c68 1268 err = -EINVAL;
77f65ebd 1269 if (match && match->flags != flags)
afe62c68 1270 goto out;
dc99f600 1271 if (!match) {
afe62c68 1272 err = -ENOMEM;
dc99f600 1273 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1274 if (!match)
1275 goto out;
1276 write_pnet(&match->net, sock_net(sk));
1277 match->id = id;
1278 match->type = type;
77f65ebd 1279 match->flags = flags;
afe62c68
ED		 1280 atomic_set(&match->rr_cur, 0);
1281 INIT_LIST_HEAD(&match->list);
1282 spin_lock_init(&match->lock);
1283 atomic_set(&match->sk_ref, 0);
1284 match->prot_hook.type = po->prot_hook.type;
1285 match->prot_hook.dev = po->prot_hook.dev;
1286 match->prot_hook.func = packet_rcv_fanout;
1287 match->prot_hook.af_packet_priv = match;
c0de08d0 1288 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1289 dev_add_pack(&match->prot_hook);
1290 list_add(&match->list, &fanout_list);
dc99f600 1291 }
afe62c68
ED		 1292 err = -EINVAL;
1293 if (match->type == type &&
1294 match->prot_hook.type == po->prot_hook.type &&
1295 match->prot_hook.dev == po->prot_hook.dev) {
1296 err = -ENOSPC;
1297 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1298 __dev_remove_pack(&po->prot_hook);
1299 po->fanout = match;
1300 atomic_inc(&match->sk_ref);
1301 __fanout_link(sk, po);
1302 err = 0;
dc99f600
DM		 1303 }
1304 }
afe62c68 1305out:
dc99f600
DM		 1306 mutex_unlock(&fanout_mutex);
1307 return err;
1308}
1309
1310static void fanout_release(struct sock *sk)
1311{
1312 struct packet_sock *po = pkt_sk(sk);
1313 struct packet_fanout *f;
1314
1315 f = po->fanout;
1316 if (!f)
1317 return;
1318
fff3321d 1319 mutex_lock(&fanout_mutex);
dc99f600
DM		 1320 po->fanout = NULL;
1321
dc99f600
DM		 1322 if (atomic_dec_and_test(&f->sk_ref)) {
1323 list_del(&f->list);
1324 dev_remove_pack(&f->prot_hook);
1325 kfree(f);
1326 }
1327 mutex_unlock(&fanout_mutex);
1328}
1da177e4 1329
90ddc4f0 1330static const struct proto_ops packet_ops;
1da177e4 1331
90ddc4f0 1332static const struct proto_ops packet_ops_spkt;
1da177e4 1333
40d4e3df
ED		 1334static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1335 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1336{
1337 struct sock *sk;
1338 struct sockaddr_pkt *spkt;
1339
1340 /*
1341 * When we registered the protocol we saved the socket in the data
1342 * field for just this event.
1343 */
1344
1345 sk = pt->af_packet_priv;
1ce4f28b 1346
1da177e4
LT		 1347 /*
1348 * Yank back the headers [hope the device set this
1349 * right or kerboom...]
1350 *
1351 * Incoming packets have ll header pulled,
1352 * push it back.
1353 *
98e399f8 1354 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1355 * so that this procedure is noop.
1356 */
1357
1358 if (skb->pkt_type == PACKET_LOOPBACK)
1359 goto out;
1360
09ad9bc7 1361 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1362 goto out;
1363
40d4e3df
ED		 1364 skb = skb_share_check(skb, GFP_ATOMIC);
1365 if (skb == NULL)
1da177e4
LT		 1366 goto oom;
1367
1368 /* drop any routing info */
adf30907 1369 skb_dst_drop(skb);
1da177e4 1370
84531c24
PO		 1371 /* drop conntrack reference */
1372 nf_reset(skb);
1373
ffbc6111 1374 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1375
98e399f8 1376 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1377
1378 /*
1379 * The SOCK_PACKET socket receives _all_ frames.
1380 */
1381
1382 spkt->spkt_family = dev->type;
1383 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1384 spkt->spkt_protocol = skb->protocol;
1385
1386 /*
1387 * Charge the memory to the socket. This is done specifically
1388 * to prevent sockets using all the memory up.
1389 */
1390
40d4e3df 1391 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1392 return 0;
1393
1394out:
1395 kfree_skb(skb);
1396oom:
1397 return 0;
1398}
1399
1400
1401/*
1402 * Output a raw packet to a device layer. This bypasses all the other
1403 * protocol layers and you must therefore supply it with a complete frame
1404 */
1ce4f28b 1405
1da177e4
LT		 1406static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1407 struct msghdr *msg, size_t len)
1408{
1409 struct sock *sk = sock->sk;
40d4e3df 1410 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1411 struct sk_buff *skb = NULL;
1da177e4 1412 struct net_device *dev;
40d4e3df 1413 __be16 proto = 0;
1da177e4 1414 int err;
3bdc0eba 1415 int extra_len = 0;
c1aad275 1416 struct flow_keys keys;
1ce4f28b 1417
1da177e4 1418 /*
1ce4f28b 1419 * Get and verify the address.
1da177e4
LT		 1420 */
1421
40d4e3df 1422 if (saddr) {
1da177e4 1423 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1424 return -EINVAL;
1425 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1426 proto = saddr->spkt_protocol;
1427 } else
1428 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1429
1430 /*
1ce4f28b 1431 * Find the device first to size check it
1da177e4
LT		 1432 */
1433
de74e92a 1434 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1435retry:
654d1f8a
ED		 1436 rcu_read_lock();
1437 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1438 err = -ENODEV;
1439 if (dev == NULL)
1440 goto out_unlock;
1ce4f28b 1441
d5e76b0a
DM		 1442 err = -ENETDOWN;
1443 if (!(dev->flags & IFF_UP))
1444 goto out_unlock;
1445
1da177e4 1446 /*
40d4e3df
ED		 1447 * You may not queue a frame bigger than the mtu. This is the lowest level
1448 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1449 */
1ce4f28b 1450
3bdc0eba
BG		 1451 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1452 if (!netif_supports_nofcs(dev)) {
1453 err = -EPROTONOSUPPORT;
1454 goto out_unlock;
1455 }
1456 extra_len = 4; /* We're doing our own CRC */
1457 }
1458
1da177e4 1459 err = -EMSGSIZE;
3bdc0eba 1460 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1461 goto out_unlock;
1462
1a35ca80
ED		 1463 if (!skb) {
1464 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1465 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1466 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1467
1468 rcu_read_unlock();
4ce40912 1469 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1470 if (skb == NULL)
1471 return -ENOBUFS;
1472 /* FIXME: Save some space for broken drivers that write a hard
1473 * header at transmission time by themselves. PPP is the notable
1474 * one here. This should really be fixed at the driver level.
1475 */
1476 skb_reserve(skb, reserved);
1477 skb_reset_network_header(skb);
1478
1479 /* Try to align data part correctly */
1480 if (hhlen) {
1481 skb->data -= hhlen;
1482 skb->tail -= hhlen;
1483 if (len < hhlen)
1484 skb_reset_network_header(skb);
1485 }
1486 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1487 if (err)
1488 goto out_free;
1489 goto retry;
1da177e4
LT		 1490 }
1491
3bdc0eba 1492 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1493 /* Earlier code assumed this would be a VLAN pkt,
1494 * double-check this now that we have the actual
1495 * packet in hand.
1496 */
1497 struct ethhdr *ehdr;
1498 skb_reset_mac_header(skb);
1499 ehdr = eth_hdr(skb);
1500 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1501 err = -EMSGSIZE;
1502 goto out_unlock;
1503 }
1504 }
1a35ca80 1505
1da177e4
LT		 1506 skb->protocol = proto;
1507 skb->dev = dev;
1508 skb->priority = sk->sk_priority;
2d37a186 1509 skb->mark = sk->sk_mark;
2244d07b 1510 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 1511 if (err < 0)
1512 goto out_unlock;
1da177e4 1513
3bdc0eba
BG		 1514 if (unlikely(extra_len == 4))
1515 skb->no_fcs = 1;
1516
c1aad275
JW		 1517 if (skb_flow_dissect(skb, &keys))
1518 skb_set_transport_header(skb, keys.thoff);
1519 else
1520 skb_reset_transport_header(skb);
1521
1da177e4 1522 dev_queue_xmit(skb);
654d1f8a 1523 rcu_read_unlock();
40d4e3df 1524 return len;
1da177e4 1525
1da177e4 1526out_unlock:
654d1f8a 1527 rcu_read_unlock();
1a35ca80
ED		 1528out_free:
1529 kfree_skb(skb);
1da177e4
LT		 1530 return err;
1531}
1da177e4 1532
eea49cc9 1533static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1534 const struct sock *sk,
dbcb5855 1535 unsigned int res)
1da177e4
LT		 1536{
1537 struct sk_filter *filter;
fda9ef5d 1538
80f8f102
ED		 1539 rcu_read_lock();
1540 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1541 if (filter != NULL)
0a14842f 1542 res = SK_RUN_FILTER(filter, skb);
80f8f102 1543 rcu_read_unlock();
1da177e4 1544
dbcb5855 1545 return res;
1da177e4
LT		 1546}
1547
1548/*
62ab0812
ED		 1549 * This function makes lazy skb cloning in hope that most of packets
1550 * are discarded by BPF.
1551 *
1552 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1553 * and skb->cb are mangled. It works because (and until) packets
1554 * falling here are owned by current CPU. Output packets are cloned
1555 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1556 * sequencially, so that if we return skb to original state on exit,
1557 * we will not harm anyone.
1da177e4
LT		 1558 */
1559
40d4e3df
ED		 1560static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1561 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1562{
1563 struct sock *sk;
1564 struct sockaddr_ll *sll;
1565 struct packet_sock *po;
40d4e3df 1566 u8 *skb_head = skb->data;
1da177e4 1567 int skb_len = skb->len;
dbcb5855 1568 unsigned int snaplen, res;
1da177e4
LT		 1569
1570 if (skb->pkt_type == PACKET_LOOPBACK)
1571 goto drop;
1572
1573 sk = pt->af_packet_priv;
1574 po = pkt_sk(sk);
1575
09ad9bc7 1576 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1577 goto drop;
1578
1da177e4
LT		 1579 skb->dev = dev;
1580
3b04ddde 1581 if (dev->header_ops) {
1da177e4 1582 /* The device has an explicit notion of ll header,
62ab0812
ED		 1583 * exported to higher levels.
1584 *
1585 * Otherwise, the device hides details of its frame
1586 * structure, so that corresponding packet head is
1587 * never delivered to user.
1da177e4
LT		 1588 */
1589 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1590 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1591 else if (skb->pkt_type == PACKET_OUTGOING) {
1592 /* Special case: outgoing packets have ll header at head */
bbe735e4 1593 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1594 }
1595 }
1596
1597 snaplen = skb->len;
1598
dbcb5855
DM		 1599 res = run_filter(skb, sk, snaplen);
1600 if (!res)
fda9ef5d 1601 goto drop_n_restore;
dbcb5855
DM		 1602 if (snaplen > res)
1603 snaplen = res;
1da177e4 1604
0fd7bac6 1605 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1606 goto drop_n_acct;
1607
1608 if (skb_shared(skb)) {
1609 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1610 if (nskb == NULL)
1611 goto drop_n_acct;
1612
1613 if (skb_head != skb->data) {
1614 skb->data = skb_head;
1615 skb->len = skb_len;
1616 }
abc4e4fa 1617 consume_skb(skb);
1da177e4
LT		 1618 skb = nskb;
1619 }
1620
ffbc6111
HX		 1621 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1622 sizeof(skb->cb));
1623
1624 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1625 sll->sll_family = AF_PACKET;
1626 sll->sll_hatype = dev->type;
1627 sll->sll_protocol = skb->protocol;
1628 sll->sll_pkttype = skb->pkt_type;
8032b464 1629 if (unlikely(po->origdev))
80feaacb
PWJ		 1630 sll->sll_ifindex = orig_dev->ifindex;
1631 else
1632 sll->sll_ifindex = dev->ifindex;
1da177e4 1633
b95cce35 1634 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1635
ffbc6111 1636 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1637
1da177e4
LT		 1638 if (pskb_trim(skb, snaplen))
1639 goto drop_n_acct;
1640
1641 skb_set_owner_r(skb, sk);
1642 skb->dev = NULL;
adf30907 1643 skb_dst_drop(skb);
1da177e4 1644
84531c24
PO		 1645 /* drop conntrack reference */
1646 nf_reset(skb);
1647
1da177e4
LT		 1648 spin_lock(&sk->sk_receive_queue.lock);
1649 po->stats.tp_packets++;
3b885787 1650 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1651 __skb_queue_tail(&sk->sk_receive_queue, skb);
1652 spin_unlock(&sk->sk_receive_queue.lock);
1653 sk->sk_data_ready(sk, skb->len);
1654 return 0;
1655
1656drop_n_acct:
7091fbd8
WB		 1657 spin_lock(&sk->sk_receive_queue.lock);
1658 po->stats.tp_drops++;
1659 atomic_inc(&sk->sk_drops);
1660 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1661
1662drop_n_restore:
1663 if (skb_head != skb->data && skb_shared(skb)) {
1664 skb->data = skb_head;
1665 skb->len = skb_len;
1666 }
1667drop:
ead2ceb0 1668 consume_skb(skb);
1da177e4
LT		 1669 return 0;
1670}
1671
40d4e3df
ED		 1672static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1673 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1674{
1675 struct sock *sk;
1676 struct packet_sock *po;
1677 struct sockaddr_ll *sll;
bbd6ef87
PM		 1678 union {
1679 struct tpacket_hdr *h1;
1680 struct tpacket2_hdr *h2;
f6fb8f10 1681 struct tpacket3_hdr *h3;
bbd6ef87
PM		 1682 void *raw;
1683 } h;
40d4e3df 1684 u8 *skb_head = skb->data;
1da177e4 1685 int skb_len = skb->len;
dbcb5855 1686 unsigned int snaplen, res;
f6fb8f10 1687 unsigned long status = TP_STATUS_USER;
bbd6ef87 1688 unsigned short macoff, netoff, hdrlen;
1da177e4 1689 struct sk_buff *copy_skb = NULL;
b7aa0bf7 1690 struct timeval tv;
bbd6ef87 1691 struct timespec ts;
614f60fa 1692 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1da177e4
LT		 1693
1694 if (skb->pkt_type == PACKET_LOOPBACK)
1695 goto drop;
1696
1697 sk = pt->af_packet_priv;
1698 po = pkt_sk(sk);
1699
09ad9bc7 1700 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1701 goto drop;
1702
3b04ddde 1703 if (dev->header_ops) {
1da177e4 1704 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1705 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1706 else if (skb->pkt_type == PACKET_OUTGOING) {
1707 /* Special case: outgoing packets have ll header at head */
bbe735e4 1708 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1709 }
1710 }
1711
8dc41944
HX		 1712 if (skb->ip_summed == CHECKSUM_PARTIAL)
1713 status |= TP_STATUS_CSUMNOTREADY;
1714
1da177e4
LT		 1715 snaplen = skb->len;
1716
dbcb5855
DM		 1717 res = run_filter(skb, sk, snaplen);
1718 if (!res)
fda9ef5d 1719 goto drop_n_restore;
dbcb5855
DM		 1720 if (snaplen > res)
1721 snaplen = res;
1da177e4
LT		 1722
1723 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1724 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1725 po->tp_reserve;
1da177e4 1726 } else {
95c96174 1727 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1728 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1729 (maclen < 16 ? 16 : maclen)) +
1730 po->tp_reserve;
1da177e4
LT		 1731 macoff = netoff - maclen;
1732 }
f6fb8f10 1733 if (po->tp_version <= TPACKET_V2) {
1734 if (macoff + snaplen > po->rx_ring.frame_size) {
1735 if (po->copy_thresh &&
0fd7bac6 1736 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1737 if (skb_shared(skb)) {
1738 copy_skb = skb_clone(skb, GFP_ATOMIC);
1739 } else {
1740 copy_skb = skb_get(skb);
1741 skb_head = skb->data;
1742 }
1743 if (copy_skb)
1744 skb_set_owner_r(copy_skb, sk);
1da177e4 1745 }
f6fb8f10 1746 snaplen = po->rx_ring.frame_size - macoff;
1747 if ((int)snaplen < 0)
1748 snaplen = 0;
1da177e4 1749 }
1da177e4 1750 }
1da177e4 1751 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1752 h.raw = packet_current_rx_frame(po, skb,
1753 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1754 if (!h.raw)
1da177e4 1755 goto ring_is_full;
f6fb8f10 1756 if (po->tp_version <= TPACKET_V2) {
1757 packet_increment_rx_head(po, &po->rx_ring);
1758 /*
1759 * LOSING will be reported till you read the stats,
1760 * because it's COR - Clear On Read.
1761 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1762 * at packet level.
1763 */
1764 if (po->stats.tp_drops)
1765 status |= TP_STATUS_LOSING;
1766 }
1da177e4
LT		 1767 po->stats.tp_packets++;
1768 if (copy_skb) {
1769 status |= TP_STATUS_COPY;
1770 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1771 }
1da177e4
LT		 1772 spin_unlock(&sk->sk_receive_queue.lock);
1773
bbd6ef87 1774 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1da177e4 1775
bbd6ef87
PM		 1776 switch (po->tp_version) {
1777 case TPACKET_V1:
1778 h.h1->tp_len = skb->len;
1779 h.h1->tp_snaplen = snaplen;
1780 h.h1->tp_mac = macoff;
1781 h.h1->tp_net = netoff;
614f60fa
SM		 1782 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1783 && shhwtstamps->syststamp.tv64)
1784 tv = ktime_to_timeval(shhwtstamps->syststamp);
1785 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1786 && shhwtstamps->hwtstamp.tv64)
1787 tv = ktime_to_timeval(shhwtstamps->hwtstamp);
1788 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1789 tv = ktime_to_timeval(skb->tstamp);
1790 else
1791 do_gettimeofday(&tv);
1792 h.h1->tp_sec = tv.tv_sec;
1793 h.h1->tp_usec = tv.tv_usec;
1794 hdrlen = sizeof(*h.h1);
1795 break;
1796 case TPACKET_V2:
1797 h.h2->tp_len = skb->len;
1798 h.h2->tp_snaplen = snaplen;
1799 h.h2->tp_mac = macoff;
1800 h.h2->tp_net = netoff;
614f60fa
SM		 1801 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1802 && shhwtstamps->syststamp.tv64)
1803 ts = ktime_to_timespec(shhwtstamps->syststamp);
1804 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1805 && shhwtstamps->hwtstamp.tv64)
1806 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1807 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1808 ts = ktime_to_timespec(skb->tstamp);
1809 else
1810 getnstimeofday(&ts);
1811 h.h2->tp_sec = ts.tv_sec;
1812 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1813 if (vlan_tx_tag_present(skb)) {
1814 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1815 status |= TP_STATUS_VLAN_VALID;
1816 } else {
1817 h.h2->tp_vlan_tci = 0;
1818 }
13fcb7bd 1819 h.h2->tp_padding = 0;
bbd6ef87
PM		 1820 hdrlen = sizeof(*h.h2);
1821 break;
f6fb8f10 1822 case TPACKET_V3:
1823 /* tp_nxt_offset,vlan are already populated above.
1824 * So DONT clear those fields here
1825 */
1826 h.h3->tp_status |= status;
1827 h.h3->tp_len = skb->len;
1828 h.h3->tp_snaplen = snaplen;
1829 h.h3->tp_mac = macoff;
1830 h.h3->tp_net = netoff;
1831 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1832 && shhwtstamps->syststamp.tv64)
1833 ts = ktime_to_timespec(shhwtstamps->syststamp);
1834 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1835 && shhwtstamps->hwtstamp.tv64)
1836 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1837 else if (skb->tstamp.tv64)
1838 ts = ktime_to_timespec(skb->tstamp);
1839 else
1840 getnstimeofday(&ts);
1841 h.h3->tp_sec = ts.tv_sec;
1842 h.h3->tp_nsec = ts.tv_nsec;
1843 hdrlen = sizeof(*h.h3);
1844 break;
bbd6ef87
PM		 1845 default:
1846 BUG();
1847 }
1da177e4 1848
bbd6ef87 1849 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1850 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1851 sll->sll_family = AF_PACKET;
1852 sll->sll_hatype = dev->type;
1853 sll->sll_protocol = skb->protocol;
1854 sll->sll_pkttype = skb->pkt_type;
8032b464 1855 if (unlikely(po->origdev))
80feaacb
PWJ		 1856 sll->sll_ifindex = orig_dev->ifindex;
1857 else
1858 sll->sll_ifindex = dev->ifindex;
1da177e4 1859
e16aa207 1860 smp_mb();
f6dafa95 1861#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1862 {
0af55bb5
CG		 1863 u8 *start, *end;
1864
f6fb8f10 1865 if (po->tp_version <= TPACKET_V2) {
1866 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1867 + macoff + snaplen);
1868 for (start = h.raw; start < end; start += PAGE_SIZE)
1869 flush_dcache_page(pgv_to_page(start));
1870 }
cc9f01b2 1871 smp_wmb();
1da177e4 1872 }
f6dafa95 1873#endif
f6fb8f10 1874 if (po->tp_version <= TPACKET_V2)
1875 __packet_set_status(po, h.raw, status);
1876 else
1877 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1878
1879 sk->sk_data_ready(sk, 0);
1880
1881drop_n_restore:
1882 if (skb_head != skb->data && skb_shared(skb)) {
1883 skb->data = skb_head;
1884 skb->len = skb_len;
1885 }
1886drop:
1ce4f28b 1887 kfree_skb(skb);
1da177e4
LT		 1888 return 0;
1889
1890ring_is_full:
1891 po->stats.tp_drops++;
1892 spin_unlock(&sk->sk_receive_queue.lock);
1893
1894 sk->sk_data_ready(sk, 0);
acb5d75b 1895 kfree_skb(copy_skb);
1da177e4
LT		 1896 goto drop_n_restore;
1897}
1898
69e3c75f
JB		 1899static void tpacket_destruct_skb(struct sk_buff *skb)
1900{
1901 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1902 void *ph;
1da177e4 1903
69e3c75f
JB		 1904 if (likely(po->tx_ring.pg_vec)) {
1905 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1906 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1907 atomic_dec(&po->tx_ring.pending);
1908 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
1909 }
1910
1911 sock_wfree(skb);
1912}
1913
40d4e3df
ED		 1914static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1915 void *frame, struct net_device *dev, int size_max,
ae641949 1916 __be16 proto, unsigned char *addr, int hlen)
69e3c75f
JB		 1917{
1918 union {
1919 struct tpacket_hdr *h1;
1920 struct tpacket2_hdr *h2;
1921 void *raw;
1922 } ph;
1923 int to_write, offset, len, tp_len, nr_frags, len_max;
1924 struct socket *sock = po->sk.sk_socket;
1925 struct page *page;
1926 void *data;
1927 int err;
c1aad275 1928 struct flow_keys keys;
69e3c75f
JB		 1929
1930 ph.raw = frame;
1931
1932 skb->protocol = proto;
1933 skb->dev = dev;
1934 skb->priority = po->sk.sk_priority;
2d37a186 1935 skb->mark = po->sk.sk_mark;
69e3c75f
JB		 1936 skb_shinfo(skb)->destructor_arg = ph.raw;
1937
1938 switch (po->tp_version) {
1939 case TPACKET_V2:
1940 tp_len = ph.h2->tp_len;
1941 break;
1942 default:
1943 tp_len = ph.h1->tp_len;
1944 break;
1945 }
1946 if (unlikely(tp_len > size_max)) {
40d4e3df 1947 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1948 return -EMSGSIZE;
1949 }
1950
ae641949 1951 skb_reserve(skb, hlen);
69e3c75f
JB		 1952 skb_reset_network_header(skb);
1953
c1aad275
JW		 1954 if (skb_flow_dissect(skb, &keys))
1955 skb_set_transport_header(skb, keys.thoff);
1956 else
1957 skb_reset_transport_header(skb);
1958
5920cd3a
PC		 1959 if (po->tp_tx_has_off) {
1960 int off_min, off_max, off;
1961 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1962 off_max = po->tx_ring.frame_size - tp_len;
1963 if (sock->type == SOCK_DGRAM) {
1964 switch (po->tp_version) {
1965 case TPACKET_V2:
1966 off = ph.h2->tp_net;
1967 break;
1968 default:
1969 off = ph.h1->tp_net;
1970 break;
1971 }
1972 } else {
1973 switch (po->tp_version) {
1974 case TPACKET_V2:
1975 off = ph.h2->tp_mac;
1976 break;
1977 default:
1978 off = ph.h1->tp_mac;
1979 break;
1980 }
1981 }
1982 if (unlikely((off < off_min) || (off_max < off)))
1983 return -EINVAL;
1984 data = ph.raw + off;
1985 } else {
1986 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1987 }
69e3c75f
JB		 1988 to_write = tp_len;
1989
1990 if (sock->type == SOCK_DGRAM) {
1991 err = dev_hard_header(skb, dev, ntohs(proto), addr,
1992 NULL, tp_len);
1993 if (unlikely(err < 0))
1994 return -EINVAL;
40d4e3df 1995 } else if (dev->hard_header_len) {
69e3c75f
JB		 1996 /* net device doesn't like empty head */
1997 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 1998 pr_err("packet size is too short (%d < %d)\n",
1999 tp_len, dev->hard_header_len);
69e3c75f
JB		 2000 return -EINVAL;
2001 }
2002
2003 skb_push(skb, dev->hard_header_len);
2004 err = skb_store_bits(skb, 0, data,
2005 dev->hard_header_len);
2006 if (unlikely(err))
2007 return err;
2008
2009 data += dev->hard_header_len;
2010 to_write -= dev->hard_header_len;
2011 }
2012
69e3c75f
JB		 2013 offset = offset_in_page(data);
2014 len_max = PAGE_SIZE - offset;
2015 len = ((to_write > len_max) ? len_max : to_write);
2016
2017 skb->data_len = to_write;
2018 skb->len += to_write;
2019 skb->truesize += to_write;
2020 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2021
2022 while (likely(to_write)) {
2023 nr_frags = skb_shinfo(skb)->nr_frags;
2024
2025 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2026 pr_err("Packet exceed the number of skb frags(%lu)\n",
2027 MAX_SKB_FRAGS);
69e3c75f
JB		 2028 return -EFAULT;
2029 }
2030
0af55bb5
CG		 2031 page = pgv_to_page(data);
2032 data += len;
69e3c75f
JB		 2033 flush_dcache_page(page);
2034 get_page(page);
0af55bb5 2035 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2036 to_write -= len;
2037 offset = 0;
2038 len_max = PAGE_SIZE;
2039 len = ((to_write > len_max) ? len_max : to_write);
2040 }
2041
2042 return tp_len;
2043}
2044
2045static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2046{
69e3c75f
JB		 2047 struct sk_buff *skb;
2048 struct net_device *dev;
2049 __be16 proto;
827d9780
BG		 2050 bool need_rls_dev = false;
2051 int err, reserve = 0;
40d4e3df
ED		 2052 void *ph;
2053 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2054 int tp_len, size_max;
2055 unsigned char *addr;
2056 int len_sum = 0;
9e67030a 2057 int status = TP_STATUS_AVAILABLE;
ae641949 2058 int hlen, tlen;
69e3c75f 2059
69e3c75f
JB		 2060 mutex_lock(&po->pg_vec_lock);
2061
69e3c75f 2062 if (saddr == NULL) {
827d9780 2063 dev = po->prot_hook.dev;
69e3c75f
JB		 2064 proto = po->num;
2065 addr = NULL;
2066 } else {
2067 err = -EINVAL;
2068 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2069 goto out;
2070 if (msg->msg_namelen < (saddr->sll_halen
2071 + offsetof(struct sockaddr_ll,
2072 sll_addr)))
2073 goto out;
69e3c75f
JB		 2074 proto = saddr->sll_protocol;
2075 addr = saddr->sll_addr;
827d9780
BG		 2076 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2077 need_rls_dev = true;
69e3c75f
JB		 2078 }
2079
69e3c75f
JB		 2080 err = -ENXIO;
2081 if (unlikely(dev == NULL))
2082 goto out;
2083
2084 reserve = dev->hard_header_len;
2085
2086 err = -ENETDOWN;
2087 if (unlikely(!(dev->flags & IFF_UP)))
2088 goto out_put;
2089
2090 size_max = po->tx_ring.frame_size
b5dd884e 2091 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2092
2093 if (size_max > dev->mtu + reserve)
2094 size_max = dev->mtu + reserve;
2095
2096 do {
2097 ph = packet_current_frame(po, &po->tx_ring,
2098 TP_STATUS_SEND_REQUEST);
2099
2100 if (unlikely(ph == NULL)) {
2101 schedule();
2102 continue;
2103 }
2104
2105 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2106 hlen = LL_RESERVED_SPACE(dev);
2107 tlen = dev->needed_tailroom;
69e3c75f 2108 skb = sock_alloc_send_skb(&po->sk,
ae641949 2109 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2110 0, &err);
2111
2112 if (unlikely(skb == NULL))
2113 goto out_status;
2114
2115 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2116 addr, hlen);
69e3c75f
JB		 2117
2118 if (unlikely(tp_len < 0)) {
2119 if (po->tp_loss) {
2120 __packet_set_status(po, ph,
2121 TP_STATUS_AVAILABLE);
2122 packet_increment_head(&po->tx_ring);
2123 kfree_skb(skb);
2124 continue;
2125 } else {
2126 status = TP_STATUS_WRONG_FORMAT;
2127 err = tp_len;
2128 goto out_status;
2129 }
2130 }
2131
2132 skb->destructor = tpacket_destruct_skb;
2133 __packet_set_status(po, ph, TP_STATUS_SENDING);
2134 atomic_inc(&po->tx_ring.pending);
2135
2136 status = TP_STATUS_SEND_REQUEST;
2137 err = dev_queue_xmit(skb);
eb70df13
JP		 2138 if (unlikely(err > 0)) {
2139 err = net_xmit_errno(err);
2140 if (err && __packet_get_status(po, ph) ==
2141 TP_STATUS_AVAILABLE) {
2142 /* skb was destructed already */
2143 skb = NULL;
2144 goto out_status;
2145 }
2146 /*
2147 * skb was dropped but not destructed yet;
2148 * let's treat it like congestion or err < 0
2149 */
2150 err = 0;
2151 }
69e3c75f
JB		 2152 packet_increment_head(&po->tx_ring);
2153 len_sum += tp_len;
f64f9e71
JP		 2154 } while (likely((ph != NULL) ||
2155 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2156 (atomic_read(&po->tx_ring.pending))))
2157 );
69e3c75f
JB		 2158
2159 err = len_sum;
2160 goto out_put;
2161
69e3c75f
JB		 2162out_status:
2163 __packet_set_status(po, ph, status);
2164 kfree_skb(skb);
2165out_put:
827d9780
BG		 2166 if (need_rls_dev)
2167 dev_put(dev);
69e3c75f
JB		 2168out:
2169 mutex_unlock(&po->pg_vec_lock);
2170 return err;
2171}
69e3c75f 2172
eea49cc9
OJ		 2173static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2174 size_t reserve, size_t len,
2175 size_t linear, int noblock,
2176 int *err)
bfd5f4a3
SS		 2177{
2178 struct sk_buff *skb;
2179
2180 /* Under a page? Don't bother with paged skb. */
2181 if (prepad + len < PAGE_SIZE || !linear)
2182 linear = len;
2183
2184 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2185 err);
2186 if (!skb)
2187 return NULL;
2188
2189 skb_reserve(skb, reserve);
2190 skb_put(skb, linear);
2191 skb->data_len = len - linear;
2192 skb->len += len - linear;
2193
2194 return skb;
2195}
2196
69e3c75f 2197static int packet_snd(struct socket *sock,
1da177e4
LT		 2198 struct msghdr *msg, size_t len)
2199{
2200 struct sock *sk = sock->sk;
40d4e3df 2201 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2202 struct sk_buff *skb;
2203 struct net_device *dev;
0e11c91e 2204 __be16 proto;
827d9780 2205 bool need_rls_dev = false;
1da177e4 2206 unsigned char *addr;
827d9780 2207 int err, reserve = 0;
bfd5f4a3
SS		 2208 struct virtio_net_hdr vnet_hdr = { 0 };
2209 int offset = 0;
2210 int vnet_hdr_len;
2211 struct packet_sock *po = pkt_sk(sk);
2212 unsigned short gso_type = 0;
ae641949 2213 int hlen, tlen;
3bdc0eba 2214 int extra_len = 0;
c1aad275 2215 struct flow_keys keys;
1da177e4
LT		 2216
2217 /*
1ce4f28b 2218 * Get and verify the address.
1da177e4 2219 */
1ce4f28b 2220
1da177e4 2221 if (saddr == NULL) {
827d9780 2222 dev = po->prot_hook.dev;
1da177e4
LT		 2223 proto = po->num;
2224 addr = NULL;
2225 } else {
2226 err = -EINVAL;
2227 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2228 goto out;
0fb375fb
EB		 2229 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2230 goto out;
1da177e4
LT		 2231 proto = saddr->sll_protocol;
2232 addr = saddr->sll_addr;
827d9780
BG		 2233 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2234 need_rls_dev = true;
1da177e4
LT		 2235 }
2236
1da177e4
LT		 2237 err = -ENXIO;
2238 if (dev == NULL)
2239 goto out_unlock;
2240 if (sock->type == SOCK_RAW)
2241 reserve = dev->hard_header_len;
2242
d5e76b0a
DM		 2243 err = -ENETDOWN;
2244 if (!(dev->flags & IFF_UP))
2245 goto out_unlock;
2246
bfd5f4a3
SS		 2247 if (po->has_vnet_hdr) {
2248 vnet_hdr_len = sizeof(vnet_hdr);
2249
2250 err = -EINVAL;
2251 if (len < vnet_hdr_len)
2252 goto out_unlock;
2253
2254 len -= vnet_hdr_len;
2255
2256 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2257 vnet_hdr_len);
2258 if (err < 0)
2259 goto out_unlock;
2260
2261 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2262 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2263 vnet_hdr.hdr_len))
2264 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2265 vnet_hdr.csum_offset + 2;
2266
2267 err = -EINVAL;
2268 if (vnet_hdr.hdr_len > len)
2269 goto out_unlock;
2270
2271 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2272 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2273 case VIRTIO_NET_HDR_GSO_TCPV4:
2274 gso_type = SKB_GSO_TCPV4;
2275 break;
2276 case VIRTIO_NET_HDR_GSO_TCPV6:
2277 gso_type = SKB_GSO_TCPV6;
2278 break;
2279 case VIRTIO_NET_HDR_GSO_UDP:
2280 gso_type = SKB_GSO_UDP;
2281 break;
2282 default:
2283 goto out_unlock;
2284 }
2285
2286 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2287 gso_type |= SKB_GSO_TCP_ECN;
2288
2289 if (vnet_hdr.gso_size == 0)
2290 goto out_unlock;
2291
2292 }
2293 }
2294
3bdc0eba
BG		 2295 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2296 if (!netif_supports_nofcs(dev)) {
2297 err = -EPROTONOSUPPORT;
2298 goto out_unlock;
2299 }
2300 extra_len = 4; /* We're doing our own CRC */
2301 }
2302
1da177e4 2303 err = -EMSGSIZE;
3bdc0eba 2304 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2305 goto out_unlock;
2306
bfd5f4a3 2307 err = -ENOBUFS;
ae641949
HX		 2308 hlen = LL_RESERVED_SPACE(dev);
2309 tlen = dev->needed_tailroom;
2310 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2311 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2312 if (skb == NULL)
1da177e4
LT		 2313 goto out_unlock;
2314
bfd5f4a3 2315 skb_set_network_header(skb, reserve);
1da177e4 2316
0c4e8581
SH		 2317 err = -EINVAL;
2318 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2319 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2320 goto out_free;
1da177e4
LT		 2321
2322 /* Returns -EFAULT on error */
bfd5f4a3 2323 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2324 if (err)
2325 goto out_free;
2244d07b 2326 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 2327 if (err < 0)
2328 goto out_free;
1da177e4 2329
3bdc0eba 2330 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
57f89bfa
BG		 2331 /* Earlier code assumed this would be a VLAN pkt,
2332 * double-check this now that we have the actual
2333 * packet in hand.
2334 */
2335 struct ethhdr *ehdr;
2336 skb_reset_mac_header(skb);
2337 ehdr = eth_hdr(skb);
2338 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2339 err = -EMSGSIZE;
2340 goto out_free;
2341 }
2342 }
2343
1da177e4
LT		 2344 skb->protocol = proto;
2345 skb->dev = dev;
2346 skb->priority = sk->sk_priority;
2d37a186 2347 skb->mark = sk->sk_mark;
1da177e4 2348
bfd5f4a3
SS		 2349 if (po->has_vnet_hdr) {
2350 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2351 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2352 vnet_hdr.csum_offset)) {
2353 err = -EINVAL;
2354 goto out_free;
2355 }
2356 }
2357
2358 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2359 skb_shinfo(skb)->gso_type = gso_type;
2360
2361 /* Header must be checked, and gso_segs computed. */
2362 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2363 skb_shinfo(skb)->gso_segs = 0;
2364
2365 len += vnet_hdr_len;
2366 }
2367
c1aad275
JW		 2368 if (skb->ip_summed == CHECKSUM_PARTIAL)
2369 skb_set_transport_header(skb, skb_checksum_start_offset(skb));
2370 else if (skb_flow_dissect(skb, &keys))
2371 skb_set_transport_header(skb, keys.thoff);
2372 else
2373 skb_set_transport_header(skb, reserve);
2374
3bdc0eba
BG		 2375 if (unlikely(extra_len == 4))
2376 skb->no_fcs = 1;
2377
1da177e4
LT		 2378 /*
2379 * Now send it
2380 */
2381
2382 err = dev_queue_xmit(skb);
2383 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2384 goto out_unlock;
2385
827d9780
BG		 2386 if (need_rls_dev)
2387 dev_put(dev);
1da177e4 2388
40d4e3df 2389 return len;
1da177e4
LT		 2390
2391out_free:
2392 kfree_skb(skb);
2393out_unlock:
827d9780 2394 if (dev && need_rls_dev)
1da177e4
LT		 2395 dev_put(dev);
2396out:
2397 return err;
2398}
2399
69e3c75f
JB		 2400static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2401 struct msghdr *msg, size_t len)
2402{
69e3c75f
JB		 2403 struct sock *sk = sock->sk;
2404 struct packet_sock *po = pkt_sk(sk);
2405 if (po->tx_ring.pg_vec)
2406 return tpacket_snd(po, msg);
2407 else
69e3c75f
JB		 2408 return packet_snd(sock, msg, len);
2409}
2410
1da177e4
LT		 2411/*
2412 * Close a PACKET socket. This is fairly simple. We immediately go
2413 * to 'closed' state and remove our protocol entry in the device list.
2414 */
2415
2416static int packet_release(struct socket *sock)
2417{
2418 struct sock *sk = sock->sk;
2419 struct packet_sock *po;
d12d01d6 2420 struct net *net;
f6fb8f10 2421 union tpacket_req_u req_u;
1da177e4
LT		 2422
2423 if (!sk)
2424 return 0;
2425
3b1e0a65 2426 net = sock_net(sk);
1da177e4
LT		 2427 po = pkt_sk(sk);
2428
0fa7fa98 2429 mutex_lock(&net->packet.sklist_lock);
808f5114 2430 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2431 mutex_unlock(&net->packet.sklist_lock);
2432
2433 preempt_disable();
920de804 2434 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2435 preempt_enable();
1da177e4 2436
808f5114 2437 spin_lock(&po->bind_lock);
ce06b03e 2438 unregister_prot_hook(sk, false);
160ff18a
BG		 2439 if (po->prot_hook.dev) {
2440 dev_put(po->prot_hook.dev);
2441 po->prot_hook.dev = NULL;
2442 }
808f5114 2443 spin_unlock(&po->bind_lock);
1da177e4 2444
1da177e4 2445 packet_flush_mclist(sk);
1da177e4 2446
9665d5d6
PS		 2447 if (po->rx_ring.pg_vec) {
2448 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2449 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2450 }
69e3c75f 2451
9665d5d6
PS		 2452 if (po->tx_ring.pg_vec) {
2453 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2454 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2455 }
1da177e4 2456
dc99f600
DM		 2457 fanout_release(sk);
2458
808f5114 2459 synchronize_net();
1da177e4
LT		 2460 /*
2461 * Now the socket is dead. No more input will appear.
2462 */
1da177e4
LT		 2463 sock_orphan(sk);
2464 sock->sk = NULL;
2465
2466 /* Purge queues */
2467
2468 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2469 sk_refcnt_debug_release(sk);
1da177e4
LT		 2470
2471 sock_put(sk);
2472 return 0;
2473}
2474
2475/*
2476 * Attach a packet hook.
2477 */
2478
0e11c91e 2479static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2480{
2481 struct packet_sock *po = pkt_sk(sk);
dc99f600 2482
aef950b4
WY		 2483 if (po->fanout) {
2484 if (dev)
2485 dev_put(dev);
2486
dc99f600 2487 return -EINVAL;
aef950b4 2488 }
1da177e4
LT		 2489
2490 lock_sock(sk);
2491
2492 spin_lock(&po->bind_lock);
ce06b03e 2493 unregister_prot_hook(sk, true);
1da177e4
LT		 2494 po->num = protocol;
2495 po->prot_hook.type = protocol;
160ff18a
BG		 2496 if (po->prot_hook.dev)
2497 dev_put(po->prot_hook.dev);
1da177e4
LT		 2498 po->prot_hook.dev = dev;
2499
2500 po->ifindex = dev ? dev->ifindex : 0;
2501
2502 if (protocol == 0)
2503 goto out_unlock;
2504
be85d4ad 2505 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2506 register_prot_hook(sk);
be85d4ad
UT		 2507 } else {
2508 sk->sk_err = ENETDOWN;
2509 if (!sock_flag(sk, SOCK_DEAD))
2510 sk->sk_error_report(sk);
1da177e4
LT		 2511 }
2512
2513out_unlock:
2514 spin_unlock(&po->bind_lock);
2515 release_sock(sk);
2516 return 0;
2517}
2518
2519/*
2520 * Bind a packet socket to a device
2521 */
2522
40d4e3df
ED		 2523static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2524 int addr_len)
1da177e4 2525{
40d4e3df 2526 struct sock *sk = sock->sk;
1da177e4
LT		 2527 char name[15];
2528 struct net_device *dev;
2529 int err = -ENODEV;
1ce4f28b 2530
1da177e4
LT		 2531 /*
2532 * Check legality
2533 */
1ce4f28b 2534
8ae55f04 2535 if (addr_len != sizeof(struct sockaddr))
1da177e4 2536 return -EINVAL;
40d4e3df 2537 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2538
3b1e0a65 2539 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2540 if (dev)
1da177e4 2541 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2542 return err;
2543}
1da177e4
LT		 2544
2545static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2546{
40d4e3df
ED		 2547 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2548 struct sock *sk = sock->sk;
1da177e4
LT		 2549 struct net_device *dev = NULL;
2550 int err;
2551
2552
2553 /*
2554 * Check legality
2555 */
1ce4f28b 2556
1da177e4
LT		 2557 if (addr_len < sizeof(struct sockaddr_ll))
2558 return -EINVAL;
2559 if (sll->sll_family != AF_PACKET)
2560 return -EINVAL;
2561
2562 if (sll->sll_ifindex) {
2563 err = -ENODEV;
3b1e0a65 2564 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2565 if (dev == NULL)
2566 goto out;
2567 }
2568 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2569
2570out:
2571 return err;
2572}
2573
2574static struct proto packet_proto = {
2575 .name = "PACKET",
2576 .owner = THIS_MODULE,
2577 .obj_size = sizeof(struct packet_sock),
2578};
2579
2580/*
1ce4f28b 2581 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2582 */
2583
3f378b68
EP		 2584static int packet_create(struct net *net, struct socket *sock, int protocol,
2585 int kern)
1da177e4
LT		 2586{
2587 struct sock *sk;
2588 struct packet_sock *po;
0e11c91e 2589 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2590 int err;
2591
df008c91 2592 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2593 return -EPERM;
be02097c
DM		 2594 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2595 sock->type != SOCK_PACKET)
1da177e4
LT		 2596 return -ESOCKTNOSUPPORT;
2597
2598 sock->state = SS_UNCONNECTED;
2599
2600 err = -ENOBUFS;
6257ff21 2601 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2602 if (sk == NULL)
2603 goto out;
2604
2605 sock->ops = &packet_ops;
1da177e4
LT		 2606 if (sock->type == SOCK_PACKET)
2607 sock->ops = &packet_ops_spkt;
be02097c 2608
1da177e4
LT		 2609 sock_init_data(sock, sk);
2610
2611 po = pkt_sk(sk);
2612 sk->sk_family = PF_PACKET;
0e11c91e 2613 po->num = proto;
1da177e4
LT		 2614
2615 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2616 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2617
2618 /*
2619 * Attach a protocol block
2620 */
2621
2622 spin_lock_init(&po->bind_lock);
905db440 2623 mutex_init(&po->pg_vec_lock);
1da177e4 2624 po->prot_hook.func = packet_rcv;
be02097c 2625
1da177e4
LT		 2626 if (sock->type == SOCK_PACKET)
2627 po->prot_hook.func = packet_rcv_spkt;
be02097c 2628
1da177e4
LT		 2629 po->prot_hook.af_packet_priv = sk;
2630
0e11c91e
AV		 2631 if (proto) {
2632 po->prot_hook.type = proto;
ce06b03e 2633 register_prot_hook(sk);
1da177e4
LT		 2634 }
2635
0fa7fa98 2636 mutex_lock(&net->packet.sklist_lock);
808f5114 2637 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2638 mutex_unlock(&net->packet.sklist_lock);
2639
2640 preempt_disable();
3680453c 2641 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2642 preempt_enable();
808f5114 2643
40d4e3df 2644 return 0;
1da177e4
LT		 2645out:
2646 return err;
2647}
2648
ed85b565
RC		 2649static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2650{
2651 struct sock_exterr_skb *serr;
2652 struct sk_buff *skb, *skb2;
2653 int copied, err;
2654
2655 err = -EAGAIN;
2656 skb = skb_dequeue(&sk->sk_error_queue);
2657 if (skb == NULL)
2658 goto out;
2659
2660 copied = skb->len;
2661 if (copied > len) {
2662 msg->msg_flags |= MSG_TRUNC;
2663 copied = len;
2664 }
2665 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2666 if (err)
2667 goto out_free_skb;
2668
2669 sock_recv_timestamp(msg, sk, skb);
2670
2671 serr = SKB_EXT_ERR(skb);
2672 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2673 sizeof(serr->ee), &serr->ee);
2674
2675 msg->msg_flags |= MSG_ERRQUEUE;
2676 err = copied;
2677
2678 /* Reset and regenerate socket error */
2679 spin_lock_bh(&sk->sk_error_queue.lock);
2680 sk->sk_err = 0;
2681 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2682 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2683 spin_unlock_bh(&sk->sk_error_queue.lock);
2684 sk->sk_error_report(sk);
2685 } else
2686 spin_unlock_bh(&sk->sk_error_queue.lock);
2687
2688out_free_skb:
2689 kfree_skb(skb);
2690out:
2691 return err;
2692}
2693
1da177e4
LT		 2694/*
2695 * Pull a packet from our receive queue and hand it to the user.
2696 * If necessary we block.
2697 */
2698
2699static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2700 struct msghdr *msg, size_t len, int flags)
2701{
2702 struct sock *sk = sock->sk;
2703 struct sk_buff *skb;
2704 int copied, err;
0fb375fb 2705 struct sockaddr_ll *sll;
bfd5f4a3 2706 int vnet_hdr_len = 0;
1da177e4
LT		 2707
2708 err = -EINVAL;
ed85b565 2709 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2710 goto out;
2711
2712#if 0
2713 /* What error should we return now? EUNATTACH? */
2714 if (pkt_sk(sk)->ifindex < 0)
2715 return -ENODEV;
2716#endif
2717
ed85b565
RC		 2718 if (flags & MSG_ERRQUEUE) {
2719 err = packet_recv_error(sk, msg, len);
2720 goto out;
2721 }
2722
1da177e4
LT		 2723 /*
2724 * Call the generic datagram receiver. This handles all sorts
2725 * of horrible races and re-entrancy so we can forget about it
2726 * in the protocol layers.
2727 *
2728 * Now it will return ENETDOWN, if device have just gone down,
2729 * but then it will block.
2730 */
2731
40d4e3df 2732 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2733
2734 /*
1ce4f28b 2735 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2736 * handles the blocking we don't see and worry about blocking
2737 * retries.
2738 */
2739
8ae55f04 2740 if (skb == NULL)
1da177e4
LT		 2741 goto out;
2742
bfd5f4a3
SS		 2743 if (pkt_sk(sk)->has_vnet_hdr) {
2744 struct virtio_net_hdr vnet_hdr = { 0 };
2745
2746 err = -EINVAL;
2747 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2748 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2749 goto out_free;
2750
1f18b717
MK		 2751 len -= vnet_hdr_len;
2752
bfd5f4a3
SS		 2753 if (skb_is_gso(skb)) {
2754 struct skb_shared_info *sinfo = skb_shinfo(skb);
2755
2756 /* This is a hint as to how much should be linear. */
2757 vnet_hdr.hdr_len = skb_headlen(skb);
2758 vnet_hdr.gso_size = sinfo->gso_size;
2759 if (sinfo->gso_type & SKB_GSO_TCPV4)
2760 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2761 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2762 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2763 else if (sinfo->gso_type & SKB_GSO_UDP)
2764 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2765 else if (sinfo->gso_type & SKB_GSO_FCOE)
2766 goto out_free;
2767 else
2768 BUG();
2769 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2770 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2771 } else
2772 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2773
2774 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2775 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2776 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2777 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2778 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2779 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2780 } /* else everything is zero */
2781
2782 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2783 vnet_hdr_len);
2784 if (err < 0)
2785 goto out_free;
2786 }
2787
0fb375fb
EB		 2788 /*
2789 * If the address length field is there to be filled in, we fill
2790 * it in now.
2791 */
2792
ffbc6111 2793 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2794 if (sock->type == SOCK_PACKET)
2795 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2796 else
2797 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2798
1da177e4
LT		 2799 /*
2800 * You lose any data beyond the buffer you gave. If it worries a
2801 * user program they can ask the device for its MTU anyway.
2802 */
2803
2804 copied = skb->len;
40d4e3df
ED		 2805 if (copied > len) {
2806 copied = len;
2807 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2808 }
2809
2810 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2811 if (err)
2812 goto out_free;
2813
3b885787 2814 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2815
2816 if (msg->msg_name)
ffbc6111
HX		 2817 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2818 msg->msg_namelen);
1da177e4 2819
8dc41944 2820 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2821 struct tpacket_auxdata aux;
2822
2823 aux.tp_status = TP_STATUS_USER;
2824 if (skb->ip_summed == CHECKSUM_PARTIAL)
2825 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2826 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2827 aux.tp_snaplen = skb->len;
2828 aux.tp_mac = 0;
bbe735e4 2829 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2830 if (vlan_tx_tag_present(skb)) {
2831 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2832 aux.tp_status |= TP_STATUS_VLAN_VALID;
2833 } else {
2834 aux.tp_vlan_tci = 0;
2835 }
13fcb7bd 2836 aux.tp_padding = 0;
ffbc6111 2837 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2838 }
2839
1da177e4
LT		 2840 /*
2841 * Free or return the buffer as appropriate. Again this
2842 * hides all the races and re-entrancy issues from us.
2843 */
bfd5f4a3 2844 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2845
2846out_free:
2847 skb_free_datagram(sk, skb);
2848out:
2849 return err;
2850}
2851
1da177e4
LT		 2852static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2853 int *uaddr_len, int peer)
2854{
2855 struct net_device *dev;
2856 struct sock *sk = sock->sk;
2857
2858 if (peer)
2859 return -EOPNOTSUPP;
2860
2861 uaddr->sa_family = AF_PACKET;
654d1f8a
ED		 2862 rcu_read_lock();
2863 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2864 if (dev)
67286640 2865 strncpy(uaddr->sa_data, dev->name, 14);
654d1f8a 2866 else
1da177e4 2867 memset(uaddr->sa_data, 0, 14);
654d1f8a 2868 rcu_read_unlock();
1da177e4
LT		 2869 *uaddr_len = sizeof(*uaddr);
2870
2871 return 0;
2872}
1da177e4
LT		 2873
2874static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2875 int *uaddr_len, int peer)
2876{
2877 struct net_device *dev;
2878 struct sock *sk = sock->sk;
2879 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2880 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2881
2882 if (peer)
2883 return -EOPNOTSUPP;
2884
2885 sll->sll_family = AF_PACKET;
2886 sll->sll_ifindex = po->ifindex;
2887 sll->sll_protocol = po->num;
67286640 2888 sll->sll_pkttype = 0;
654d1f8a
ED		 2889 rcu_read_lock();
2890 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2891 if (dev) {
2892 sll->sll_hatype = dev->type;
2893 sll->sll_halen = dev->addr_len;
2894 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2895 } else {
2896 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2897 sll->sll_halen = 0;
2898 }
654d1f8a 2899 rcu_read_unlock();
0fb375fb 2900 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2901
2902 return 0;
2903}
2904
2aeb0b88
WC		 2905static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2906 int what)
1da177e4
LT		 2907{
2908 switch (i->type) {
2909 case PACKET_MR_MULTICAST:
1162563f
JP		 2910 if (i->alen != dev->addr_len)
2911 return -EINVAL;
1da177e4 2912 if (what > 0)
22bedad3 2913 return dev_mc_add(dev, i->addr);
1da177e4 2914 else
22bedad3 2915 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2916 break;
2917 case PACKET_MR_PROMISC:
2aeb0b88 2918 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2919 break;
2920 case PACKET_MR_ALLMULTI:
2aeb0b88 2921 return dev_set_allmulti(dev, what);
1da177e4 2922 break;
d95ed927 2923 case PACKET_MR_UNICAST:
1162563f
JP		 2924 if (i->alen != dev->addr_len)
2925 return -EINVAL;
d95ed927 2926 if (what > 0)
a748ee24 2927 return dev_uc_add(dev, i->addr);
d95ed927 2928 else
a748ee24 2929 return dev_uc_del(dev, i->addr);
d95ed927 2930 break;
40d4e3df
ED		 2931 default:
2932 break;
1da177e4 2933 }
2aeb0b88 2934 return 0;
1da177e4
LT		 2935}
2936
2937static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2938{
40d4e3df 2939 for ( ; i; i = i->next) {
1da177e4
LT		 2940 if (i->ifindex == dev->ifindex)
2941 packet_dev_mc(dev, i, what);
2942 }
2943}
2944
0fb375fb 2945static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2946{
2947 struct packet_sock *po = pkt_sk(sk);
2948 struct packet_mclist *ml, *i;
2949 struct net_device *dev;
2950 int err;
2951
2952 rtnl_lock();
2953
2954 err = -ENODEV;
3b1e0a65 2955 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2956 if (!dev)
2957 goto done;
2958
2959 err = -EINVAL;
1162563f 2960 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2961 goto done;
2962
2963 err = -ENOBUFS;
8b3a7005 2964 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2965 if (i == NULL)
2966 goto done;
2967
2968 err = 0;
2969 for (ml = po->mclist; ml; ml = ml->next) {
2970 if (ml->ifindex == mreq->mr_ifindex &&
2971 ml->type == mreq->mr_type &&
2972 ml->alen == mreq->mr_alen &&
2973 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2974 ml->count++;
2975 /* Free the new element ... */
2976 kfree(i);
2977 goto done;
2978 }
2979 }
2980
2981 i->type = mreq->mr_type;
2982 i->ifindex = mreq->mr_ifindex;
2983 i->alen = mreq->mr_alen;
2984 memcpy(i->addr, mreq->mr_address, i->alen);
2985 i->count = 1;
2986 i->next = po->mclist;
2987 po->mclist = i;
2aeb0b88
WC		 2988 err = packet_dev_mc(dev, i, 1);
2989 if (err) {
2990 po->mclist = i->next;
2991 kfree(i);
2992 }
1da177e4
LT		 2993
2994done:
2995 rtnl_unlock();
2996 return err;
2997}
2998
0fb375fb 2999static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3000{
3001 struct packet_mclist *ml, **mlp;
3002
3003 rtnl_lock();
3004
3005 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3006 if (ml->ifindex == mreq->mr_ifindex &&
3007 ml->type == mreq->mr_type &&
3008 ml->alen == mreq->mr_alen &&
3009 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3010 if (--ml->count == 0) {
3011 struct net_device *dev;
3012 *mlp = ml->next;
ad959e76
ED		 3013 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3014 if (dev)
1da177e4 3015 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3016 kfree(ml);
3017 }
3018 rtnl_unlock();
3019 return 0;
3020 }
3021 }
3022 rtnl_unlock();
3023 return -EADDRNOTAVAIL;
3024}
3025
3026static void packet_flush_mclist(struct sock *sk)
3027{
3028 struct packet_sock *po = pkt_sk(sk);
3029 struct packet_mclist *ml;
3030
3031 if (!po->mclist)
3032 return;
3033
3034 rtnl_lock();
3035 while ((ml = po->mclist) != NULL) {
3036 struct net_device *dev;
3037
3038 po->mclist = ml->next;
ad959e76
ED		 3039 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3040 if (dev != NULL)
1da177e4 3041 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3042 kfree(ml);
3043 }
3044 rtnl_unlock();
3045}
1da177e4
LT		 3046
3047static int
b7058842 3048packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3049{
3050 struct sock *sk = sock->sk;
8dc41944 3051 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3052 int ret;
3053
3054 if (level != SOL_PACKET)
3055 return -ENOPROTOOPT;
3056
69e3c75f 3057 switch (optname) {
1ce4f28b 3058 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3059 case PACKET_DROP_MEMBERSHIP:
3060 {
0fb375fb
EB		 3061 struct packet_mreq_max mreq;
3062 int len = optlen;
3063 memset(&mreq, 0, sizeof(mreq));
3064 if (len < sizeof(struct packet_mreq))
1da177e4 3065 return -EINVAL;
0fb375fb
EB		 3066 if (len > sizeof(mreq))
3067 len = sizeof(mreq);
40d4e3df 3068 if (copy_from_user(&mreq, optval, len))
1da177e4 3069 return -EFAULT;
0fb375fb
EB		 3070 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3071 return -EINVAL;
1da177e4
LT		 3072 if (optname == PACKET_ADD_MEMBERSHIP)
3073 ret = packet_mc_add(sk, &mreq);
3074 else
3075 ret = packet_mc_drop(sk, &mreq);
3076 return ret;
3077 }
a2efcfa0 3078
1da177e4 3079 case PACKET_RX_RING:
69e3c75f 3080 case PACKET_TX_RING:
1da177e4 3081 {
f6fb8f10 3082 union tpacket_req_u req_u;
3083 int len;
1da177e4 3084
f6fb8f10 3085 switch (po->tp_version) {
3086 case TPACKET_V1:
3087 case TPACKET_V2:
3088 len = sizeof(req_u.req);
3089 break;
3090 case TPACKET_V3:
3091 default:
3092 len = sizeof(req_u.req3);
3093 break;
3094 }
3095 if (optlen < len)
1da177e4 3096 return -EINVAL;
bfd5f4a3
SS		 3097 if (pkt_sk(sk)->has_vnet_hdr)
3098 return -EINVAL;
f6fb8f10 3099 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3100 return -EFAULT;
f6fb8f10 3101 return packet_set_ring(sk, &req_u, 0,
3102 optname == PACKET_TX_RING);
1da177e4
LT		 3103 }
3104 case PACKET_COPY_THRESH:
3105 {
3106 int val;
3107
40d4e3df 3108 if (optlen != sizeof(val))
1da177e4 3109 return -EINVAL;
40d4e3df 3110 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3111 return -EFAULT;
3112
3113 pkt_sk(sk)->copy_thresh = val;
3114 return 0;
3115 }
bbd6ef87
PM		 3116 case PACKET_VERSION:
3117 {
3118 int val;
3119
3120 if (optlen != sizeof(val))
3121 return -EINVAL;
69e3c75f 3122 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3123 return -EBUSY;
3124 if (copy_from_user(&val, optval, sizeof(val)))
3125 return -EFAULT;
3126 switch (val) {
3127 case TPACKET_V1:
3128 case TPACKET_V2:
f6fb8f10 3129 case TPACKET_V3:
bbd6ef87
PM		 3130 po->tp_version = val;
3131 return 0;
3132 default:
3133 return -EINVAL;
3134 }
3135 }
8913336a
PM		 3136 case PACKET_RESERVE:
3137 {
3138 unsigned int val;
3139
3140 if (optlen != sizeof(val))
3141 return -EINVAL;
69e3c75f 3142 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3143 return -EBUSY;
3144 if (copy_from_user(&val, optval, sizeof(val)))
3145 return -EFAULT;
3146 po->tp_reserve = val;
3147 return 0;
3148 }
69e3c75f
JB		 3149 case PACKET_LOSS:
3150 {
3151 unsigned int val;
3152
3153 if (optlen != sizeof(val))
3154 return -EINVAL;
3155 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3156 return -EBUSY;
3157 if (copy_from_user(&val, optval, sizeof(val)))
3158 return -EFAULT;
3159 po->tp_loss = !!val;
3160 return 0;
3161 }
8dc41944
HX		 3162 case PACKET_AUXDATA:
3163 {
3164 int val;
3165
3166 if (optlen < sizeof(val))
3167 return -EINVAL;
3168 if (copy_from_user(&val, optval, sizeof(val)))
3169 return -EFAULT;
3170
3171 po->auxdata = !!val;
3172 return 0;
3173 }
80feaacb
PWJ		 3174 case PACKET_ORIGDEV:
3175 {
3176 int val;
3177
3178 if (optlen < sizeof(val))
3179 return -EINVAL;
3180 if (copy_from_user(&val, optval, sizeof(val)))
3181 return -EFAULT;
3182
3183 po->origdev = !!val;
3184 return 0;
3185 }
bfd5f4a3
SS		 3186 case PACKET_VNET_HDR:
3187 {
3188 int val;
3189
3190 if (sock->type != SOCK_RAW)
3191 return -EINVAL;
3192 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3193 return -EBUSY;
3194 if (optlen < sizeof(val))
3195 return -EINVAL;
3196 if (copy_from_user(&val, optval, sizeof(val)))
3197 return -EFAULT;
3198
3199 po->has_vnet_hdr = !!val;
3200 return 0;
3201 }
614f60fa
SM		 3202 case PACKET_TIMESTAMP:
3203 {
3204 int val;
3205
3206 if (optlen != sizeof(val))
3207 return -EINVAL;
3208 if (copy_from_user(&val, optval, sizeof(val)))
3209 return -EFAULT;
3210
3211 po->tp_tstamp = val;
3212 return 0;
3213 }
dc99f600
DM		 3214 case PACKET_FANOUT:
3215 {
3216 int val;
3217
3218 if (optlen != sizeof(val))
3219 return -EINVAL;
3220 if (copy_from_user(&val, optval, sizeof(val)))
3221 return -EFAULT;
3222
3223 return fanout_add(sk, val & 0xffff, val >> 16);
3224 }
5920cd3a
PC		 3225 case PACKET_TX_HAS_OFF:
3226 {
3227 unsigned int val;
3228
3229 if (optlen != sizeof(val))
3230 return -EINVAL;
3231 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3232 return -EBUSY;
3233 if (copy_from_user(&val, optval, sizeof(val)))
3234 return -EFAULT;
3235 po->tp_tx_has_off = !!val;
3236 return 0;
3237 }
1da177e4
LT		 3238 default:
3239 return -ENOPROTOOPT;
3240 }
3241}
3242
3243static int packet_getsockopt(struct socket *sock, int level, int optname,
3244 char __user *optval, int __user *optlen)
3245{
3246 int len;
c06fff6e 3247 int val, lv = sizeof(val);
1da177e4
LT		 3248 struct sock *sk = sock->sk;
3249 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3250 void *data = &val;
8dc41944 3251 struct tpacket_stats st;
f6fb8f10 3252 union tpacket_stats_u st_u;
1da177e4
LT		 3253
3254 if (level != SOL_PACKET)
3255 return -ENOPROTOOPT;
3256
8ae55f04
KK		 3257 if (get_user(len, optlen))
3258 return -EFAULT;
1da177e4
LT		 3259
3260 if (len < 0)
3261 return -EINVAL;
1ce4f28b 3262
69e3c75f 3263 switch (optname) {
1da177e4 3264 case PACKET_STATISTICS:
1da177e4 3265 spin_lock_bh(&sk->sk_receive_queue.lock);
f6fb8f10 3266 if (po->tp_version == TPACKET_V3) {
c06fff6e 3267 lv = sizeof(struct tpacket_stats_v3);
f6fb8f10 3268 memcpy(&st_u.stats3, &po->stats,
c06fff6e 3269 sizeof(struct tpacket_stats));
f6fb8f10 3270 st_u.stats3.tp_freeze_q_cnt =
c06fff6e 3271 po->stats_u.stats3.tp_freeze_q_cnt;
f6fb8f10 3272 st_u.stats3.tp_packets += po->stats.tp_drops;
3273 data = &st_u.stats3;
3274 } else {
c06fff6e 3275 lv = sizeof(struct tpacket_stats);
f6fb8f10 3276 st = po->stats;
3277 st.tp_packets += st.tp_drops;
3278 data = &st;
3279 }
1da177e4
LT		 3280 memset(&po->stats, 0, sizeof(st));
3281 spin_unlock_bh(&sk->sk_receive_queue.lock);
8dc41944
HX		 3282 break;
3283 case PACKET_AUXDATA:
8dc41944 3284 val = po->auxdata;
80feaacb
PWJ		 3285 break;
3286 case PACKET_ORIGDEV:
80feaacb 3287 val = po->origdev;
bfd5f4a3
SS		 3288 break;
3289 case PACKET_VNET_HDR:
bfd5f4a3 3290 val = po->has_vnet_hdr;
1da177e4 3291 break;
bbd6ef87 3292 case PACKET_VERSION:
bbd6ef87 3293 val = po->tp_version;
bbd6ef87
PM		 3294 break;
3295 case PACKET_HDRLEN:
3296 if (len > sizeof(int))
3297 len = sizeof(int);
3298 if (copy_from_user(&val, optval, len))
3299 return -EFAULT;
3300 switch (val) {
3301 case TPACKET_V1:
3302 val = sizeof(struct tpacket_hdr);
3303 break;
3304 case TPACKET_V2:
3305 val = sizeof(struct tpacket2_hdr);
3306 break;
f6fb8f10 3307 case TPACKET_V3:
3308 val = sizeof(struct tpacket3_hdr);
3309 break;
bbd6ef87
PM		 3310 default:
3311 return -EINVAL;
3312 }
bbd6ef87 3313 break;
8913336a 3314 case PACKET_RESERVE:
8913336a 3315 val = po->tp_reserve;
8913336a 3316 break;
69e3c75f 3317 case PACKET_LOSS:
69e3c75f 3318 val = po->tp_loss;
69e3c75f 3319 break;
614f60fa 3320 case PACKET_TIMESTAMP:
614f60fa 3321 val = po->tp_tstamp;
614f60fa 3322 break;
dc99f600 3323 case PACKET_FANOUT:
dc99f600
DM		 3324 val = (po->fanout ?
3325 ((u32)po->fanout->id |
77f65ebd
WB		 3326 ((u32)po->fanout->type << 16) |
3327 ((u32)po->fanout->flags << 24)) :
dc99f600 3328 0);
dc99f600 3329 break;
5920cd3a
PC		 3330 case PACKET_TX_HAS_OFF:
3331 val = po->tp_tx_has_off;
3332 break;
1da177e4
LT		 3333 default:
3334 return -ENOPROTOOPT;
3335 }
3336
c06fff6e
ED		 3337 if (len > lv)
3338 len = lv;
8ae55f04
KK		 3339 if (put_user(len, optlen))
3340 return -EFAULT;
8dc41944
HX		 3341 if (copy_to_user(optval, data, len))
3342 return -EFAULT;
8ae55f04 3343 return 0;
1da177e4
LT		 3344}
3345
3346
3347static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3348{
3349 struct sock *sk;
ad930650 3350 struct net_device *dev = data;
c346dca1 3351 struct net *net = dev_net(dev);
1da177e4 3352
808f5114 3353 rcu_read_lock();
b67bfe0d 3354 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3355 struct packet_sock *po = pkt_sk(sk);
3356
3357 switch (msg) {
3358 case NETDEV_UNREGISTER:
1da177e4
LT		 3359 if (po->mclist)
3360 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3361 /* fallthrough */
3362
1da177e4
LT		 3363 case NETDEV_DOWN:
3364 if (dev->ifindex == po->ifindex) {
3365 spin_lock(&po->bind_lock);
3366 if (po->running) {
ce06b03e 3367 __unregister_prot_hook(sk, false);
1da177e4
LT		 3368 sk->sk_err = ENETDOWN;
3369 if (!sock_flag(sk, SOCK_DEAD))
3370 sk->sk_error_report(sk);
3371 }
3372 if (msg == NETDEV_UNREGISTER) {
3373 po->ifindex = -1;
160ff18a
BG		 3374 if (po->prot_hook.dev)
3375 dev_put(po->prot_hook.dev);
1da177e4
LT		 3376 po->prot_hook.dev = NULL;
3377 }
3378 spin_unlock(&po->bind_lock);
3379 }
3380 break;
3381 case NETDEV_UP:
808f5114 3382 if (dev->ifindex == po->ifindex) {
3383 spin_lock(&po->bind_lock);
ce06b03e
DM		 3384 if (po->num)
3385 register_prot_hook(sk);
808f5114 3386 spin_unlock(&po->bind_lock);
1da177e4 3387 }
1da177e4
LT		 3388 break;
3389 }
3390 }
808f5114 3391 rcu_read_unlock();
1da177e4
LT		 3392 return NOTIFY_DONE;
3393}
3394
3395
3396static int packet_ioctl(struct socket *sock, unsigned int cmd,
3397 unsigned long arg)
3398{
3399 struct sock *sk = sock->sk;
3400
69e3c75f 3401 switch (cmd) {
40d4e3df
ED		 3402 case SIOCOUTQ:
3403 {
3404 int amount = sk_wmem_alloc_get(sk);
31e6d363 3405
40d4e3df
ED		 3406 return put_user(amount, (int __user *)arg);
3407 }
3408 case SIOCINQ:
3409 {
3410 struct sk_buff *skb;
3411 int amount = 0;
3412
3413 spin_lock_bh(&sk->sk_receive_queue.lock);
3414 skb = skb_peek(&sk->sk_receive_queue);
3415 if (skb)
3416 amount = skb->len;
3417 spin_unlock_bh(&sk->sk_receive_queue.lock);
3418 return put_user(amount, (int __user *)arg);
3419 }
3420 case SIOCGSTAMP:
3421 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3422 case SIOCGSTAMPNS:
3423 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3424
1da177e4 3425#ifdef CONFIG_INET
40d4e3df
ED		 3426 case SIOCADDRT:
3427 case SIOCDELRT:
3428 case SIOCDARP:
3429 case SIOCGARP:
3430 case SIOCSARP:
3431 case SIOCGIFADDR:
3432 case SIOCSIFADDR:
3433 case SIOCGIFBRDADDR:
3434 case SIOCSIFBRDADDR:
3435 case SIOCGIFNETMASK:
3436 case SIOCSIFNETMASK:
3437 case SIOCGIFDSTADDR:
3438 case SIOCSIFDSTADDR:
3439 case SIOCSIFFLAGS:
40d4e3df 3440 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3441#endif
3442
40d4e3df
ED		 3443 default:
3444 return -ENOIOCTLCMD;
1da177e4
LT		 3445 }
3446 return 0;
3447}
3448
40d4e3df 3449static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3450 poll_table *wait)
3451{
3452 struct sock *sk = sock->sk;
3453 struct packet_sock *po = pkt_sk(sk);
3454 unsigned int mask = datagram_poll(file, sock, wait);
3455
3456 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3457 if (po->rx_ring.pg_vec) {
f6fb8f10 3458 if (!packet_previous_rx_frame(po, &po->rx_ring,
3459 TP_STATUS_KERNEL))
1da177e4
LT		 3460 mask |= POLLIN | POLLRDNORM;
3461 }
3462 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3463 spin_lock_bh(&sk->sk_write_queue.lock);
3464 if (po->tx_ring.pg_vec) {
3465 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3466 mask |= POLLOUT | POLLWRNORM;
3467 }
3468 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3469 return mask;
3470}
3471
3472
3473/* Dirty? Well, I still did not learn better way to account
3474 * for user mmaps.
3475 */
3476
3477static void packet_mm_open(struct vm_area_struct *vma)
3478{
3479 struct file *file = vma->vm_file;
40d4e3df 3480 struct socket *sock = file->private_data;
1da177e4 3481 struct sock *sk = sock->sk;
1ce4f28b 3482
1da177e4
LT		 3483 if (sk)
3484 atomic_inc(&pkt_sk(sk)->mapped);
3485}
3486
3487static void packet_mm_close(struct vm_area_struct *vma)
3488{
3489 struct file *file = vma->vm_file;
40d4e3df 3490 struct socket *sock = file->private_data;
1da177e4 3491 struct sock *sk = sock->sk;
1ce4f28b 3492
1da177e4
LT		 3493 if (sk)
3494 atomic_dec(&pkt_sk(sk)->mapped);
3495}
3496
f0f37e2f 3497static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3498 .open = packet_mm_open,
3499 .close = packet_mm_close,
1da177e4
LT		 3500};
3501
0e3125c7
NH		 3502static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3503 unsigned int len)
1da177e4
LT		 3504{
3505 int i;
3506
4ebf0ae2 3507 for (i = 0; i < len; i++) {
0e3125c7 3508 if (likely(pg_vec[i].buffer)) {
c56b4d90 3509 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3510 vfree(pg_vec[i].buffer);
3511 else
3512 free_pages((unsigned long)pg_vec[i].buffer,
3513 order);
3514 pg_vec[i].buffer = NULL;
3515 }
1da177e4
LT		 3516 }
3517 kfree(pg_vec);
3518}
3519
eea49cc9 3520static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3521{
0e3125c7
NH		 3522 char *buffer = NULL;
3523 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3524 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3525
3526 buffer = (char *) __get_free_pages(gfp_flags, order);
3527
3528 if (buffer)
3529 return buffer;
3530
3531 /*
3532 * __get_free_pages failed, fall back to vmalloc
3533 */
bbce5a59 3534 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3535
0e3125c7
NH		 3536 if (buffer)
3537 return buffer;
3538
3539 /*
3540 * vmalloc failed, lets dig into swap here
3541 */
0e3125c7
NH		 3542 gfp_flags &= ~__GFP_NORETRY;
3543 buffer = (char *)__get_free_pages(gfp_flags, order);
3544 if (buffer)
3545 return buffer;
3546
3547 /*
3548 * complete and utter failure
3549 */
3550 return NULL;
4ebf0ae2
DM		 3551}
3552
0e3125c7 3553static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3554{
3555 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3556 struct pgv *pg_vec;
4ebf0ae2
DM		 3557 int i;
3558
0e3125c7 3559 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3560 if (unlikely(!pg_vec))
3561 goto out;
3562
3563 for (i = 0; i < block_nr; i++) {
c56b4d90 3564 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3565 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3566 goto out_free_pgvec;
3567 }
3568
3569out:
3570 return pg_vec;
3571
3572out_free_pgvec:
3573 free_pg_vec(pg_vec, order, block_nr);
3574 pg_vec = NULL;
3575 goto out;
3576}
1da177e4 3577
f6fb8f10 3578static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3579 int closing, int tx_ring)
1da177e4 3580{
0e3125c7 3581 struct pgv *pg_vec = NULL;
1da177e4 3582 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3583 int was_running, order = 0;
69e3c75f
JB		 3584 struct packet_ring_buffer *rb;
3585 struct sk_buff_head *rb_queue;
0e11c91e 3586 __be16 num;
f6fb8f10 3587 int err = -EINVAL;
3588 /* Added to avoid minimal code churn */
3589 struct tpacket_req *req = &req_u->req;
3590
3591 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3592 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3593 WARN(1, "Tx-ring is not supported.\n");
3594 goto out;
3595 }
1ce4f28b 3596
69e3c75f
JB		 3597 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3598 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3599
69e3c75f
JB		 3600 err = -EBUSY;
3601 if (!closing) {
3602 if (atomic_read(&po->mapped))
3603 goto out;
3604 if (atomic_read(&rb->pending))
3605 goto out;
3606 }
1da177e4 3607
69e3c75f
JB		 3608 if (req->tp_block_nr) {
3609 /* Sanity tests and some calculations */
3610 err = -EBUSY;
3611 if (unlikely(rb->pg_vec))
3612 goto out;
1da177e4 3613
bbd6ef87
PM		 3614 switch (po->tp_version) {
3615 case TPACKET_V1:
3616 po->tp_hdrlen = TPACKET_HDRLEN;
3617 break;
3618 case TPACKET_V2:
3619 po->tp_hdrlen = TPACKET2_HDRLEN;
3620 break;
f6fb8f10 3621 case TPACKET_V3:
3622 po->tp_hdrlen = TPACKET3_HDRLEN;
3623 break;
bbd6ef87
PM		 3624 }
3625
69e3c75f 3626 err = -EINVAL;
4ebf0ae2 3627 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3628 goto out;
4ebf0ae2 3629 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3630 goto out;
8913336a 3631 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3632 po->tp_reserve))
3633 goto out;
4ebf0ae2 3634 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3635 goto out;
1da177e4 3636
69e3c75f
JB		 3637 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3638 if (unlikely(rb->frames_per_block <= 0))
3639 goto out;
3640 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3641 req->tp_frame_nr))
3642 goto out;
1da177e4
LT		 3643
3644 err = -ENOMEM;
4ebf0ae2
DM		 3645 order = get_order(req->tp_block_size);
3646 pg_vec = alloc_pg_vec(req, order);
3647 if (unlikely(!pg_vec))
1da177e4 3648 goto out;
f6fb8f10 3649 switch (po->tp_version) {
3650 case TPACKET_V3:
3651 /* Transmit path is not supported. We checked
3652 * it above but just being paranoid
3653 */
3654 if (!tx_ring)
3655 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3656 break;
3657 default:
3658 break;
3659 }
69e3c75f
JB		 3660 }
3661 /* Done */
3662 else {
3663 err = -EINVAL;
4ebf0ae2 3664 if (unlikely(req->tp_frame_nr))
69e3c75f 3665 goto out;
1da177e4
LT		 3666 }
3667
3668 lock_sock(sk);
3669
3670 /* Detach socket from network */
3671 spin_lock(&po->bind_lock);
3672 was_running = po->running;
3673 num = po->num;
3674 if (was_running) {
1da177e4 3675 po->num = 0;
ce06b03e 3676 __unregister_prot_hook(sk, false);
1da177e4
LT		 3677 }
3678 spin_unlock(&po->bind_lock);
1ce4f28b 3679
1da177e4
LT		 3680 synchronize_net();
3681
3682 err = -EBUSY;
905db440 3683 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3684 if (closing || atomic_read(&po->mapped) == 0) {
3685 err = 0;
69e3c75f 3686 spin_lock_bh(&rb_queue->lock);
c053fd96 3687 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3688 rb->frame_max = (req->tp_frame_nr - 1);
3689 rb->head = 0;
3690 rb->frame_size = req->tp_frame_size;
3691 spin_unlock_bh(&rb_queue->lock);
3692
c053fd96
CG		 3693 swap(rb->pg_vec_order, order);
3694 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3695
3696 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3697 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3698 tpacket_rcv : packet_rcv;
3699 skb_queue_purge(rb_queue);
1da177e4 3700 if (atomic_read(&po->mapped))
40d4e3df
ED		 3701 pr_err("packet_mmap: vma is busy: %d\n",
3702 atomic_read(&po->mapped));
1da177e4 3703 }
905db440 3704 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3705
3706 spin_lock(&po->bind_lock);
ce06b03e 3707 if (was_running) {
1da177e4 3708 po->num = num;
ce06b03e 3709 register_prot_hook(sk);
1da177e4
LT		 3710 }
3711 spin_unlock(&po->bind_lock);
f6fb8f10 3712 if (closing && (po->tp_version > TPACKET_V2)) {
3713 /* Because we don't support block-based V3 on tx-ring */
3714 if (!tx_ring)
3715 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3716 }
1da177e4
LT		 3717 release_sock(sk);
3718
1da177e4
LT		 3719 if (pg_vec)
3720 free_pg_vec(pg_vec, order, req->tp_block_nr);
3721out:
3722 return err;
3723}
3724
69e3c75f
JB		 3725static int packet_mmap(struct file *file, struct socket *sock,
3726 struct vm_area_struct *vma)
1da177e4
LT		 3727{
3728 struct sock *sk = sock->sk;
3729 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3730 unsigned long size, expected_size;
3731 struct packet_ring_buffer *rb;
1da177e4
LT		 3732 unsigned long start;
3733 int err = -EINVAL;
3734 int i;
3735
3736 if (vma->vm_pgoff)
3737 return -EINVAL;
3738
905db440 3739 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3740
3741 expected_size = 0;
3742 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3743 if (rb->pg_vec) {
3744 expected_size += rb->pg_vec_len
3745 * rb->pg_vec_pages
3746 * PAGE_SIZE;
3747 }
3748 }
3749
3750 if (expected_size == 0)
1da177e4 3751 goto out;
69e3c75f
JB		 3752
3753 size = vma->vm_end - vma->vm_start;
3754 if (size != expected_size)
1da177e4
LT		 3755 goto out;
3756
1da177e4 3757 start = vma->vm_start;
69e3c75f
JB		 3758 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3759 if (rb->pg_vec == NULL)
3760 continue;
3761
3762 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3763 struct page *page;
3764 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3765 int pg_num;
3766
c56b4d90
CG		 3767 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3768 page = pgv_to_page(kaddr);
69e3c75f
JB		 3769 err = vm_insert_page(vma, start, page);
3770 if (unlikely(err))
3771 goto out;
3772 start += PAGE_SIZE;
0e3125c7 3773 kaddr += PAGE_SIZE;
69e3c75f 3774 }
4ebf0ae2 3775 }
1da177e4 3776 }
69e3c75f 3777
4ebf0ae2 3778 atomic_inc(&po->mapped);
1da177e4
LT		 3779 vma->vm_ops = &packet_mmap_ops;
3780 err = 0;
3781
3782out:
905db440 3783 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3784 return err;
3785}
1da177e4 3786
90ddc4f0 3787static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3788 .family = PF_PACKET,
3789 .owner = THIS_MODULE,
3790 .release = packet_release,
3791 .bind = packet_bind_spkt,
3792 .connect = sock_no_connect,
3793 .socketpair = sock_no_socketpair,
3794 .accept = sock_no_accept,
3795 .getname = packet_getname_spkt,
3796 .poll = datagram_poll,
3797 .ioctl = packet_ioctl,
3798 .listen = sock_no_listen,
3799 .shutdown = sock_no_shutdown,
3800 .setsockopt = sock_no_setsockopt,
3801 .getsockopt = sock_no_getsockopt,
3802 .sendmsg = packet_sendmsg_spkt,
3803 .recvmsg = packet_recvmsg,
3804 .mmap = sock_no_mmap,
3805 .sendpage = sock_no_sendpage,
3806};
1da177e4 3807
90ddc4f0 3808static const struct proto_ops packet_ops = {
1da177e4
LT		 3809 .family = PF_PACKET,
3810 .owner = THIS_MODULE,
3811 .release = packet_release,
3812 .bind = packet_bind,
3813 .connect = sock_no_connect,
3814 .socketpair = sock_no_socketpair,
3815 .accept = sock_no_accept,
1ce4f28b 3816 .getname = packet_getname,
1da177e4
LT		 3817 .poll = packet_poll,
3818 .ioctl = packet_ioctl,
3819 .listen = sock_no_listen,
3820 .shutdown = sock_no_shutdown,
3821 .setsockopt = packet_setsockopt,
3822 .getsockopt = packet_getsockopt,
3823 .sendmsg = packet_sendmsg,
3824 .recvmsg = packet_recvmsg,
3825 .mmap = packet_mmap,
3826 .sendpage = sock_no_sendpage,
3827};
3828
ec1b4cf7 3829static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3830 .family = PF_PACKET,
3831 .create = packet_create,
3832 .owner = THIS_MODULE,
3833};
3834
3835static struct notifier_block packet_netdev_notifier = {
40d4e3df 3836 .notifier_call = packet_notifier,
1da177e4
LT		 3837};
3838
3839#ifdef CONFIG_PROC_FS
1da177e4
LT		 3840
3841static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3842 __acquires(RCU)
1da177e4 3843{
e372c414 3844 struct net *net = seq_file_net(seq);
808f5114 3845
3846 rcu_read_lock();
3847 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3848}
3849
3850static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3851{
1bf40954 3852 struct net *net = seq_file_net(seq);
808f5114 3853 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3854}
3855
3856static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3857 __releases(RCU)
1da177e4 3858{
808f5114 3859 rcu_read_unlock();
1da177e4
LT		 3860}
3861
1ce4f28b 3862static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3863{
3864 if (v == SEQ_START_TOKEN)
3865 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3866 else {
b7ceabd9 3867 struct sock *s = sk_entry(v);
1da177e4
LT		 3868 const struct packet_sock *po = pkt_sk(s);
3869
3870 seq_printf(seq,
71338aa7 3871 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3872 s,
3873 atomic_read(&s->sk_refcnt),
3874 s->sk_type,
3875 ntohs(po->num),
3876 po->ifindex,
3877 po->running,
3878 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3879 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3880 sock_i_ino(s));
1da177e4
LT		 3881 }
3882
3883 return 0;
3884}
3885
56b3d975 3886static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3887 .start = packet_seq_start,
3888 .next = packet_seq_next,
3889 .stop = packet_seq_stop,
3890 .show = packet_seq_show,
3891};
3892
3893static int packet_seq_open(struct inode *inode, struct file *file)
3894{
e372c414
DL		 3895 return seq_open_net(inode, file, &packet_seq_ops,
3896 sizeof(struct seq_net_private));
1da177e4
LT		 3897}
3898
da7071d7 3899static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3900 .owner = THIS_MODULE,
3901 .open = packet_seq_open,
3902 .read = seq_read,
3903 .llseek = seq_lseek,
e372c414 3904 .release = seq_release_net,
1da177e4
LT		 3905};
3906
3907#endif
3908
2c8c1e72 3909static int __net_init packet_net_init(struct net *net)
d12d01d6 3910{
0fa7fa98 3911 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3912 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3913
d4beaa66 3914 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3915 return -ENOMEM;
3916
3917 return 0;
3918}
3919
2c8c1e72 3920static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3921{
ece31ffd 3922 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3923}
3924
3925static struct pernet_operations packet_net_ops = {
3926 .init = packet_net_init,
3927 .exit = packet_net_exit,
3928};
3929
3930
1da177e4
LT		 3931static void __exit packet_exit(void)
3932{
1da177e4 3933 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3934 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3935 sock_unregister(PF_PACKET);
3936 proto_unregister(&packet_proto);
3937}
3938
3939static int __init packet_init(void)
3940{
3941 int rc = proto_register(&packet_proto, 0);
3942
3943 if (rc != 0)
3944 goto out;
3945
3946 sock_register(&packet_family_ops);
d12d01d6 3947 register_pernet_subsys(&packet_net_ops);
1da177e4 3948 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3949out:
3950 return rc;
3951}
3952
3953module_init(packet_init);
3954module_exit(packet_exit);
3955MODULE_LICENSE("GPL");
3956MODULE_ALIAS_NETPROTO(PF_PACKET);
kernel source for telekom puls (alcatel/mediatek)