projects / GitHub / exynos8895 / android_kernel_samsung_universal8895.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
usb: gadget: f_mtp: Avoid race between mtp_read and mtp_function_disable
[GitHub/exynos8895/android_kernel_samsung_universal8895.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
47dceb8e 95#include <linux/bpf.h>
1da177e4 96
2787b04b
PE		 97#include "internal.h"
98
1da177e4
LT		 99/*
100 Assumptions:
101 - if device has no dev->hard_header routine, it adds and removes ll header
102 inside itself. In this case ll header is invisible outside of device,
103 but higher levels still should reserve dev->hard_header_len.
104 Some devices are enough clever to reallocate skb, when header
105 will not fit to reserved space (tunnel), another ones are silly
106 (PPP).
107 - packet socket receives packets with pulled ll header,
108 so that SOCK_RAW should push it back.
109
110On receive:
111-----------
112
113Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 114 mac_header -> ll header
115 data -> data
1da177e4
LT		 116
117Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 118 mac_header -> ll header
119 data -> ll header
1da177e4
LT		 120
121Incoming, dev->hard_header==NULL
b0e380b1
ACM		 122 mac_header -> UNKNOWN position. It is very likely, that it points to ll
123 header. PPP makes it, that is wrong, because introduce
db0c58f9 124 assymetry between rx and tx paths.
b0e380b1 125 data -> data
1da177e4
LT		 126
127Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 128 mac_header -> data. ll header is still not built!
129 data -> data
1da177e4
LT		 130
131Resume
132 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
133
134
135On transmit:
136------------
137
138dev->hard_header != NULL
b0e380b1
ACM		 139 mac_header -> ll header
140 data -> ll header
1da177e4
LT		 141
142dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 143 mac_header -> data
144 data -> data
1da177e4
LT		 145
146 We should set nh.raw on output to correct posistion,
147 packet classifier depends on it.
148 */
149
1da177e4
LT		 150/* Private packet socket structures. */
151
0fb375fb
EB		 152/* identical to struct packet_mreq except it has
153 * a longer address field.
154 */
40d4e3df 155struct packet_mreq_max {
0fb375fb
EB		 156 int mr_ifindex;
157 unsigned short mr_type;
158 unsigned short mr_alen;
159 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 160};
a2efcfa0 161
184f489e
DB		 162union tpacket_uhdr {
163 struct tpacket_hdr *h1;
164 struct tpacket2_hdr *h2;
165 struct tpacket3_hdr *h3;
166 void *raw;
167};
168
f6fb8f10 169static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 170 int closing, int tx_ring);
171
f6fb8f10 172#define V3_ALIGNMENT (8)
173
bc59ba39 174#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 175
176#define BLK_PLUS_PRIV(sz_of_priv) \
177 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178
f6fb8f10 179#define PGV_FROM_VMALLOC 1
69e3c75f 180
f6fb8f10 181#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
182#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
183#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
184#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
185#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
186#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
187#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
188
69e3c75f
JB		 189struct packet_sock;
190static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 191static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
192 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 193
f6fb8f10 194static void *packet_previous_frame(struct packet_sock *po,
195 struct packet_ring_buffer *rb,
196 int status);
197static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 198static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
199 struct tpacket_block_desc *);
200static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 201 struct packet_sock *);
bc59ba39 202static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 203 struct packet_sock *, unsigned int status);
bc59ba39 204static int prb_queue_frozen(struct tpacket_kbdq_core *);
205static void prb_open_block(struct tpacket_kbdq_core *,
206 struct tpacket_block_desc *);
f6fb8f10 207static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 208static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
209static void prb_init_blk_timer(struct packet_sock *,
210 struct tpacket_kbdq_core *,
211 void (*func) (unsigned long));
212static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
213static void prb_clear_rxhash(struct tpacket_kbdq_core *,
214 struct tpacket3_hdr *);
215static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
216 struct tpacket3_hdr *);
1da177e4
LT		 217static void packet_flush_mclist(struct sock *sk);
218
ffbc6111 219struct packet_skb_cb {
ffbc6111
HX		 220 union {
221 struct sockaddr_pkt pkt;
2472d761
EB		 222 union {
223 /* Trick: alias skb original length with
224 * ll.sll_family and ll.protocol in order
225 * to save room.
226 */
227 unsigned int origlen;
228 struct sockaddr_ll ll;
229 };
ffbc6111
HX		 230 } sa;
231};
232
d3869efe
DW		 233#define vio_le() virtio_legacy_is_little_endian()
234
ffbc6111 235#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 236
bc59ba39 237#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 238#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 239 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 240#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 241 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 242#define GET_NEXT_PRB_BLK_NUM(x) \
243 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
244 ((x)->kactive_blk_num+1) : 0)
245
dc99f600
DM		 246static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
247static void __fanout_link(struct sock *sk, struct packet_sock *po);
248
d346a3fa
DB		 249static int packet_direct_xmit(struct sk_buff *skb)
250{
251 struct net_device *dev = skb->dev;
d21daf7f 252 struct sk_buff *orig_skb = skb;
d346a3fa 253 struct netdev_queue *txq;
43279500 254 int ret = NETDEV_TX_BUSY;
d346a3fa
DB		 255
256 if (unlikely(!netif_running(dev) ||
43279500
DB		 257 !netif_carrier_ok(dev)))
258 goto drop;
d346a3fa 259
d21daf7f
WB		 260 skb = validate_xmit_skb_list(skb, dev);
261 if (skb != orig_skb)
43279500 262 goto drop;
d346a3fa 263
10c51b56 264 txq = skb_get_tx_queue(dev, skb);
d346a3fa 265
43279500
DB		 266 local_bh_disable();
267
268 HARD_TX_LOCK(dev, txq, smp_processor_id());
10b3ad8c 269 if (!netif_xmit_frozen_or_drv_stopped(txq))
fa2dbdc2 270 ret = netdev_start_xmit(skb, dev, txq, false);
43279500 271 HARD_TX_UNLOCK(dev, txq);
d346a3fa 272
43279500
DB		 273 local_bh_enable();
274
275 if (!dev_xmit_complete(ret))
d346a3fa 276 kfree_skb(skb);
43279500 277
d346a3fa 278 return ret;
43279500 279drop:
0f97ede4 280 atomic_long_inc(&dev->tx_dropped);
d21daf7f 281 kfree_skb_list(skb);
43279500 282 return NET_XMIT_DROP;
d346a3fa
DB		 283}
284
66e56cd4
DB		 285static struct net_device *packet_cached_dev_get(struct packet_sock *po)
286{
287 struct net_device *dev;
288
289 rcu_read_lock();
290 dev = rcu_dereference(po->cached_dev);
291 if (likely(dev))
292 dev_hold(dev);
293 rcu_read_unlock();
294
295 return dev;
296}
297
298static void packet_cached_dev_assign(struct packet_sock *po,
299 struct net_device *dev)
300{
301 rcu_assign_pointer(po->cached_dev, dev);
302}
303
304static void packet_cached_dev_reset(struct packet_sock *po)
305{
306 RCU_INIT_POINTER(po->cached_dev, NULL);
307}
308
d346a3fa
DB		 309static bool packet_use_direct_xmit(const struct packet_sock *po)
310{
311 return po->xmit == packet_direct_xmit;
312}
313
0fd5d57b 314static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 315{
1cbac010 316 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 317}
318
0fd5d57b
DB		 319static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
320{
321 const struct net_device_ops *ops = dev->netdev_ops;
322 u16 queue_index;
323
324 if (ops->ndo_select_queue) {
325 queue_index = ops->ndo_select_queue(dev, skb, NULL,
326 __packet_pick_tx_queue);
327 queue_index = netdev_cap_txqueue(dev, queue_index);
328 } else {
329 queue_index = __packet_pick_tx_queue(dev, skb);
330 }
331
332 skb_set_queue_mapping(skb, queue_index);
333}
334
ce06b03e
DM		 335/* register_prot_hook must be invoked with the po->bind_lock held,
336 * or from a context in which asynchronous accesses to the packet
337 * socket is not possible (packet_create()).
338 */
339static void register_prot_hook(struct sock *sk)
340{
341 struct packet_sock *po = pkt_sk(sk);
e40526cb 342
ce06b03e 343 if (!po->running) {
66e56cd4 344 if (po->fanout)
dc99f600 345 __fanout_link(sk, po);
66e56cd4 346 else
dc99f600 347 dev_add_pack(&po->prot_hook);
e40526cb 348
ce06b03e
DM		 349 sock_hold(sk);
350 po->running = 1;
351 }
352}
353
354/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
355 * held. If the sync parameter is true, we will temporarily drop
356 * the po->bind_lock and do a synchronize_net to make sure no
357 * asynchronous packet processing paths still refer to the elements
358 * of po->prot_hook. If the sync parameter is false, it is the
359 * callers responsibility to take care of this.
360 */
361static void __unregister_prot_hook(struct sock *sk, bool sync)
362{
363 struct packet_sock *po = pkt_sk(sk);
364
365 po->running = 0;
66e56cd4
DB		 366
367 if (po->fanout)
dc99f600 368 __fanout_unlink(sk, po);
66e56cd4 369 else
dc99f600 370 __dev_remove_pack(&po->prot_hook);
e40526cb 371
ce06b03e
DM		 372 __sock_put(sk);
373
374 if (sync) {
375 spin_unlock(&po->bind_lock);
376 synchronize_net();
377 spin_lock(&po->bind_lock);
378 }
379}
380
381static void unregister_prot_hook(struct sock *sk, bool sync)
382{
383 struct packet_sock *po = pkt_sk(sk);
384
385 if (po->running)
386 __unregister_prot_hook(sk, sync);
387}
388
6e58040b 389static inline struct page * __pure pgv_to_page(void *addr)
0af55bb5
CG		 390{
391 if (is_vmalloc_addr(addr))
392 return vmalloc_to_page(addr);
393 return virt_to_page(addr);
394}
395
69e3c75f 396static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 397{
184f489e 398 union tpacket_uhdr h;
1da177e4 399
69e3c75f 400 h.raw = frame;
bbd6ef87
PM		 401 switch (po->tp_version) {
402 case TPACKET_V1:
69e3c75f 403 h.h1->tp_status = status;
0af55bb5 404 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 405 break;
406 case TPACKET_V2:
69e3c75f 407 h.h2->tp_status = status;
0af55bb5 408 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 409 break;
f6fb8f10 410 case TPACKET_V3:
69e3c75f 411 default:
f6fb8f10 412 WARN(1, "TPACKET version not supported.\n");
69e3c75f 413 BUG();
bbd6ef87 414 }
69e3c75f
JB		 415
416 smp_wmb();
bbd6ef87
PM		 417}
418
69e3c75f 419static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 420{
184f489e 421 union tpacket_uhdr h;
bbd6ef87 422
69e3c75f
JB		 423 smp_rmb();
424
bbd6ef87
PM		 425 h.raw = frame;
426 switch (po->tp_version) {
427 case TPACKET_V1:
0af55bb5 428 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 429 return h.h1->tp_status;
bbd6ef87 430 case TPACKET_V2:
0af55bb5 431 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 432 return h.h2->tp_status;
f6fb8f10 433 case TPACKET_V3:
69e3c75f 434 default:
f6fb8f10 435 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 436 BUG();
437 return 0;
bbd6ef87 438 }
1da177e4 439}
69e3c75f 440
b9c32fb2
DB		 441static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
442 unsigned int flags)
7a51384c
DB		 443{
444 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
445
68a360e8
WB		 446 if (shhwtstamps &&
447 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
448 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
449 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 450
451 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 452 return TP_STATUS_TS_SOFTWARE;
7a51384c 453
b9c32fb2 454 return 0;
7a51384c
DB		 455}
456
b9c32fb2
DB		 457static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
458 struct sk_buff *skb)
2e31396f
WB		 459{
460 union tpacket_uhdr h;
461 struct timespec ts;
b9c32fb2 462 __u32 ts_status;
2e31396f 463
b9c32fb2
DB		 464 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
465 return 0;
2e31396f
WB		 466
467 h.raw = frame;
468 switch (po->tp_version) {
469 case TPACKET_V1:
470 h.h1->tp_sec = ts.tv_sec;
471 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
472 break;
473 case TPACKET_V2:
474 h.h2->tp_sec = ts.tv_sec;
475 h.h2->tp_nsec = ts.tv_nsec;
476 break;
477 case TPACKET_V3:
478 default:
479 WARN(1, "TPACKET version not supported.\n");
480 BUG();
481 }
482
483 /* one flush is safe, as both fields always lie on the same cacheline */
484 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
485 smp_wmb();
b9c32fb2
DB		 486
487 return ts_status;
2e31396f
WB		 488}
489
69e3c75f
JB		 490static void *packet_lookup_frame(struct packet_sock *po,
491 struct packet_ring_buffer *rb,
492 unsigned int position,
493 int status)
494{
495 unsigned int pg_vec_pos, frame_offset;
184f489e 496 union tpacket_uhdr h;
69e3c75f
JB		 497
498 pg_vec_pos = position / rb->frames_per_block;
499 frame_offset = position % rb->frames_per_block;
500
0e3125c7
NH		 501 h.raw = rb->pg_vec[pg_vec_pos].buffer +
502 (frame_offset * rb->frame_size);
69e3c75f
JB		 503
504 if (status != __packet_get_status(po, h.raw))
505 return NULL;
506
507 return h.raw;
508}
509
eea49cc9 510static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 511 struct packet_ring_buffer *rb,
512 int status)
513{
514 return packet_lookup_frame(po, rb, rb->head, status);
515}
516
bc59ba39 517static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 518{
519 del_timer_sync(&pkc->retire_blk_timer);
520}
521
522static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
f6fb8f10 523 struct sk_buff_head *rb_queue)
524{
bc59ba39 525 struct tpacket_kbdq_core *pkc;
f6fb8f10 526
73d0fcf2 527 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 528
ec6f809f 529 spin_lock_bh(&rb_queue->lock);
f6fb8f10 530 pkc->delete_blk_timer = 1;
ec6f809f 531 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 532
533 prb_del_retire_blk_timer(pkc);
534}
535
536static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 537 struct tpacket_kbdq_core *pkc,
f6fb8f10 538 void (*func) (unsigned long))
539{
540 init_timer(&pkc->retire_blk_timer);
541 pkc->retire_blk_timer.data = (long)po;
542 pkc->retire_blk_timer.function = func;
543 pkc->retire_blk_timer.expires = jiffies;
544}
545
e8e85cc5 546static void prb_setup_retire_blk_timer(struct packet_sock *po)
f6fb8f10 547{
bc59ba39 548 struct tpacket_kbdq_core *pkc;
f6fb8f10 549
e8e85cc5 550 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 551 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
552}
553
554static int prb_calc_retire_blk_tmo(struct packet_sock *po,
555 int blk_size_in_bytes)
556{
557 struct net_device *dev;
558 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 559 struct ethtool_cmd ecmd;
560 int err;
e440cf2c 561 u32 speed;
f6fb8f10 562
4bc71cb9
JP		 563 rtnl_lock();
564 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
565 if (unlikely(!dev)) {
566 rtnl_unlock();
f6fb8f10 567 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 568 }
569 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 570 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 571 rtnl_unlock();
572 if (!err) {
4bc71cb9
JP		 573 /*
574 * If the link speed is so slow you don't really
575 * need to worry about perf anyways
576 */
e440cf2c 577 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 578 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 579 } else {
580 msec = 1;
581 div = speed / 1000;
f6fb8f10 582 }
583 }
584
585 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
586
587 if (div)
588 mbits /= div;
589
590 tmo = mbits * msec;
591
592 if (div)
593 return tmo+1;
594 return tmo;
595}
596
bc59ba39 597static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 598 union tpacket_req_u *req_u)
599{
600 p1->feature_req_word = req_u->req3.tp_feature_req_word;
601}
602
603static void init_prb_bdqc(struct packet_sock *po,
604 struct packet_ring_buffer *rb,
605 struct pgv *pg_vec,
e8e85cc5 606 union tpacket_req_u *req_u)
f6fb8f10 607{
22781a5b 608 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 609 struct tpacket_block_desc *pbd;
f6fb8f10 610
611 memset(p1, 0x0, sizeof(*p1));
612
613 p1->knxt_seq_num = 1;
614 p1->pkbdq = pg_vec;
bc59ba39 615 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 616 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 617 p1->kblk_size = req_u->req3.tp_block_size;
618 p1->knum_blocks = req_u->req3.tp_block_nr;
619 p1->hdrlen = po->tp_hdrlen;
620 p1->version = po->tp_version;
621 p1->last_kactive_blk_num = 0;
ee80fbf3 622 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 623 if (req_u->req3.tp_retire_blk_tov)
624 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
625 else
626 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
627 req_u->req3.tp_block_size);
628 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
629 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
630
dc808110 631 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 632 prb_init_ft_ops(p1, req_u);
e8e85cc5 633 prb_setup_retire_blk_timer(po);
f6fb8f10 634 prb_open_block(p1, pbd);
635}
636
637/* Do NOT update the last_blk_num first.
638 * Assumes sk_buff_head lock is held.
639 */
bc59ba39 640static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 641{
642 mod_timer(&pkc->retire_blk_timer,
643 jiffies + pkc->tov_in_jiffies);
644 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
645}
646
647/*
648 * Timer logic:
649 * 1) We refresh the timer only when we open a block.
650 * By doing this we don't waste cycles refreshing the timer
651 * on packet-by-packet basis.
652 *
653 * With a 1MB block-size, on a 1Gbps line, it will take
654 * i) ~8 ms to fill a block + ii) memcpy etc.
655 * In this cut we are not accounting for the memcpy time.
656 *
657 * So, if the user sets the 'tmo' to 10ms then the timer
658 * will never fire while the block is still getting filled
659 * (which is what we want). However, the user could choose
660 * to close a block early and that's fine.
661 *
662 * But when the timer does fire, we check whether or not to refresh it.
663 * Since the tmo granularity is in msecs, it is not too expensive
664 * to refresh the timer, lets say every '8' msecs.
665 * Either the user can set the 'tmo' or we can derive it based on
666 * a) line-speed and b) block-size.
667 * prb_calc_retire_blk_tmo() calculates the tmo.
668 *
669 */
670static void prb_retire_rx_blk_timer_expired(unsigned long data)
671{
672 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 673 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 674 unsigned int frozen;
bc59ba39 675 struct tpacket_block_desc *pbd;
f6fb8f10 676
677 spin_lock(&po->sk.sk_receive_queue.lock);
678
679 frozen = prb_queue_frozen(pkc);
680 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
681
682 if (unlikely(pkc->delete_blk_timer))
683 goto out;
684
685 /* We only need to plug the race when the block is partially filled.
686 * tpacket_rcv:
687 * lock(); increment BLOCK_NUM_PKTS; unlock()
688 * copy_bits() is in progress ...
689 * timer fires on other cpu:
690 * we can't retire the current block because copy_bits
691 * is in progress.
692 *
693 */
694 if (BLOCK_NUM_PKTS(pbd)) {
695 while (atomic_read(&pkc->blk_fill_in_prog)) {
696 /* Waiting for skb_copy_bits to finish... */
697 cpu_relax();
698 }
699 }
700
701 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
702 if (!frozen) {
41a50d62
AD		 703 if (!BLOCK_NUM_PKTS(pbd)) {
704 /* An empty block. Just refresh the timer. */
705 goto refresh_timer;
706 }
f6fb8f10 707 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
708 if (!prb_dispatch_next_block(pkc, po))
709 goto refresh_timer;
710 else
711 goto out;
712 } else {
713 /* Case 1. Queue was frozen because user-space was
714 * lagging behind.
715 */
716 if (prb_curr_blk_in_use(pkc, pbd)) {
717 /*
718 * Ok, user-space is still behind.
719 * So just refresh the timer.
720 */
721 goto refresh_timer;
722 } else {
723 /* Case 2. queue was frozen,user-space caught up,
724 * now the link went idle && the timer fired.
725 * We don't have a block to close.So we open this
726 * block and restart the timer.
727 * opening a block thaws the queue,restarts timer
728 * Thawing/timer-refresh is a side effect.
729 */
730 prb_open_block(pkc, pbd);
731 goto out;
732 }
733 }
734 }
735
736refresh_timer:
737 _prb_refresh_rx_retire_blk_timer(pkc);
738
739out:
740 spin_unlock(&po->sk.sk_receive_queue.lock);
741}
742
eea49cc9 743static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 744 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 745{
746 /* Flush everything minus the block header */
747
748#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
749 u8 *start, *end;
750
751 start = (u8 *)pbd1;
752
753 /* Skip the block header(we know header WILL fit in 4K) */
754 start += PAGE_SIZE;
755
756 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
757 for (; start < end; start += PAGE_SIZE)
758 flush_dcache_page(pgv_to_page(start));
759
760 smp_wmb();
761#endif
762
763 /* Now update the block status. */
764
765 BLOCK_STATUS(pbd1) = status;
766
767 /* Flush the block header */
768
769#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
770 start = (u8 *)pbd1;
771 flush_dcache_page(pgv_to_page(start));
772
773 smp_wmb();
774#endif
775}
776
777/*
778 * Side effect:
779 *
780 * 1) flush the block
781 * 2) Increment active_blk_num
782 *
783 * Note:We DONT refresh the timer on purpose.
784 * Because almost always the next block will be opened.
785 */
bc59ba39 786static void prb_close_block(struct tpacket_kbdq_core *pkc1,
787 struct tpacket_block_desc *pbd1,
f6fb8f10 788 struct packet_sock *po, unsigned int stat)
789{
790 __u32 status = TP_STATUS_USER | stat;
791
792 struct tpacket3_hdr *last_pkt;
bc59ba39 793 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
da413eec 794 struct sock *sk = &po->sk;
f6fb8f10 795
ee80fbf3 796 if (po->stats.stats3.tp_drops)
f6fb8f10 797 status |= TP_STATUS_LOSING;
798
799 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
800 last_pkt->tp_next_offset = 0;
801
802 /* Get the ts of the last pkt */
803 if (BLOCK_NUM_PKTS(pbd1)) {
804 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
805 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
806 } else {
41a50d62
AD		 807 /* Ok, we tmo'd - so get the current time.
808 *
809 * It shouldn't really happen as we don't close empty
810 * blocks. See prb_retire_rx_blk_timer_expired().
811 */
f6fb8f10 812 struct timespec ts;
813 getnstimeofday(&ts);
814 h1->ts_last_pkt.ts_sec = ts.tv_sec;
815 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
816 }
817
818 smp_wmb();
819
820 /* Flush the block */
821 prb_flush_block(pkc1, pbd1, status);
822
da413eec
DC		 823 sk->sk_data_ready(sk);
824
f6fb8f10 825 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
826}
827
eea49cc9 828static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 829{
830 pkc->reset_pending_on_curr_blk = 0;
831}
832
833/*
834 * Side effect of opening a block:
835 *
836 * 1) prb_queue is thawed.
837 * 2) retire_blk_timer is refreshed.
838 *
839 */
bc59ba39 840static void prb_open_block(struct tpacket_kbdq_core *pkc1,
841 struct tpacket_block_desc *pbd1)
f6fb8f10 842{
843 struct timespec ts;
bc59ba39 844 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 845
846 smp_rmb();
847
8da3056c
DB		 848 /* We could have just memset this but we will lose the
849 * flexibility of making the priv area sticky
850 */
f6fb8f10 851
8da3056c
DB		 852 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
853 BLOCK_NUM_PKTS(pbd1) = 0;
854 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 855
8da3056c
DB		 856 getnstimeofday(&ts);
857
858 h1->ts_first_pkt.ts_sec = ts.tv_sec;
859 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 860
8da3056c
DB		 861 pkc1->pkblk_start = (char *)pbd1;
862 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
863
864 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
865 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
866
867 pbd1->version = pkc1->version;
868 pkc1->prev = pkc1->nxt_offset;
869 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
870
871 prb_thaw_queue(pkc1);
872 _prb_refresh_rx_retire_blk_timer(pkc1);
873
874 smp_wmb();
f6fb8f10 875}
876
877/*
878 * Queue freeze logic:
879 * 1) Assume tp_block_nr = 8 blocks.
880 * 2) At time 't0', user opens Rx ring.
881 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
882 * 4) user-space is either sleeping or processing block '0'.
883 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
884 * it will close block-7,loop around and try to fill block '0'.
885 * call-flow:
886 * __packet_lookup_frame_in_block
887 * prb_retire_current_block()
888 * prb_dispatch_next_block()
889 * |->(BLOCK_STATUS == USER) evaluates to true
890 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
891 * 6) Now there are two cases:
892 * 6.1) Link goes idle right after the queue is frozen.
893 * But remember, the last open_block() refreshed the timer.
894 * When this timer expires,it will refresh itself so that we can
895 * re-open block-0 in near future.
896 * 6.2) Link is busy and keeps on receiving packets. This is a simple
897 * case and __packet_lookup_frame_in_block will check if block-0
898 * is free and can now be re-used.
899 */
eea49cc9 900static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 901 struct packet_sock *po)
902{
903 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 904 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 905}
906
907#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
908
909/*
910 * If the next block is free then we will dispatch it
911 * and return a good offset.
912 * Else, we will freeze the queue.
913 * So, caller must check the return value.
914 */
bc59ba39 915static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 916 struct packet_sock *po)
917{
bc59ba39 918 struct tpacket_block_desc *pbd;
f6fb8f10 919
920 smp_rmb();
921
922 /* 1. Get current block num */
923 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
924
925 /* 2. If this block is currently in_use then freeze the queue */
926 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
927 prb_freeze_queue(pkc, po);
928 return NULL;
929 }
930
931 /*
932 * 3.
933 * open this block and return the offset where the first packet
934 * needs to get stored.
935 */
936 prb_open_block(pkc, pbd);
937 return (void *)pkc->nxt_offset;
938}
939
bc59ba39 940static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 941 struct packet_sock *po, unsigned int status)
942{
bc59ba39 943 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 944
945 /* retire/close the current block */
946 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
947 /*
948 * Plug the case where copy_bits() is in progress on
949 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
950 * have space to copy the pkt in the current block and
951 * called prb_retire_current_block()
952 *
953 * We don't need to worry about the TMO case because
954 * the timer-handler already handled this case.
955 */
956 if (!(status & TP_STATUS_BLK_TMO)) {
957 while (atomic_read(&pkc->blk_fill_in_prog)) {
958 /* Waiting for skb_copy_bits to finish... */
959 cpu_relax();
960 }
961 }
962 prb_close_block(pkc, pbd, po, status);
963 return;
964 }
f6fb8f10 965}
966
eea49cc9 967static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 968 struct tpacket_block_desc *pbd)
f6fb8f10 969{
970 return TP_STATUS_USER & BLOCK_STATUS(pbd);
971}
972
eea49cc9 973static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 974{
975 return pkc->reset_pending_on_curr_blk;
976}
977
eea49cc9 978static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 979{
bc59ba39 980 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 981 atomic_dec(&pkc->blk_fill_in_prog);
982}
983
eea49cc9 984static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 985 struct tpacket3_hdr *ppd)
986{
3958afa1 987 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 988}
989
eea49cc9 990static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 991 struct tpacket3_hdr *ppd)
992{
993 ppd->hv1.tp_rxhash = 0;
994}
995
eea49cc9 996static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 997 struct tpacket3_hdr *ppd)
998{
df8a39de
JP		 999 if (skb_vlan_tag_present(pkc->skb)) {
1000 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
a0cdfcf3
AW		 1001 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1002 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 1003 } else {
9e67030a 1004 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1005 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1006 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1007 }
1008}
1009
bc59ba39 1010static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1011 struct tpacket3_hdr *ppd)
1012{
a0cdfcf3 1013 ppd->hv1.tp_padding = 0;
f6fb8f10 1014 prb_fill_vlan_info(pkc, ppd);
1015
1016 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1017 prb_fill_rxhash(pkc, ppd);
1018 else
1019 prb_clear_rxhash(pkc, ppd);
1020}
1021
eea49cc9 1022static void prb_fill_curr_block(char *curr,
bc59ba39 1023 struct tpacket_kbdq_core *pkc,
1024 struct tpacket_block_desc *pbd,
f6fb8f10 1025 unsigned int len)
1026{
1027 struct tpacket3_hdr *ppd;
1028
1029 ppd = (struct tpacket3_hdr *)curr;
1030 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1031 pkc->prev = curr;
1032 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1033 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1034 BLOCK_NUM_PKTS(pbd) += 1;
1035 atomic_inc(&pkc->blk_fill_in_prog);
1036 prb_run_all_ft_ops(pkc, ppd);
1037}
1038
1039/* Assumes caller has the sk->rx_queue.lock */
1040static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1041 struct sk_buff *skb,
1042 int status,
1043 unsigned int len
1044 )
1045{
bc59ba39 1046 struct tpacket_kbdq_core *pkc;
1047 struct tpacket_block_desc *pbd;
f6fb8f10 1048 char *curr, *end;
1049
e3192690 1050 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1051 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1052
1053 /* Queue is frozen when user space is lagging behind */
1054 if (prb_queue_frozen(pkc)) {
1055 /*
1056 * Check if that last block which caused the queue to freeze,
1057 * is still in_use by user-space.
1058 */
1059 if (prb_curr_blk_in_use(pkc, pbd)) {
1060 /* Can't record this packet */
1061 return NULL;
1062 } else {
1063 /*
1064 * Ok, the block was released by user-space.
1065 * Now let's open that block.
1066 * opening a block also thaws the queue.
1067 * Thawing is a side effect.
1068 */
1069 prb_open_block(pkc, pbd);
1070 }
1071 }
1072
1073 smp_mb();
1074 curr = pkc->nxt_offset;
1075 pkc->skb = skb;
e3192690 1076 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1077
1078 /* first try the current block */
1079 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1080 prb_fill_curr_block(curr, pkc, pbd, len);
1081 return (void *)curr;
1082 }
1083
1084 /* Ok, close the current block */
1085 prb_retire_current_block(pkc, po, 0);
1086
1087 /* Now, try to dispatch the next block */
1088 curr = (char *)prb_dispatch_next_block(pkc, po);
1089 if (curr) {
1090 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1091 prb_fill_curr_block(curr, pkc, pbd, len);
1092 return (void *)curr;
1093 }
1094
1095 /*
1096 * No free blocks are available.user_space hasn't caught up yet.
1097 * Queue was just frozen and now this packet will get dropped.
1098 */
1099 return NULL;
1100}
1101
eea49cc9 1102static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1103 struct sk_buff *skb,
1104 int status, unsigned int len)
1105{
1106 char *curr = NULL;
1107 switch (po->tp_version) {
1108 case TPACKET_V1:
1109 case TPACKET_V2:
1110 curr = packet_lookup_frame(po, &po->rx_ring,
1111 po->rx_ring.head, status);
1112 return curr;
1113 case TPACKET_V3:
1114 return __packet_lookup_frame_in_block(po, skb, status, len);
1115 default:
1116 WARN(1, "TPACKET version not supported\n");
1117 BUG();
99aa3473 1118 return NULL;
f6fb8f10 1119 }
1120}
1121
eea49cc9 1122static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1123 struct packet_ring_buffer *rb,
77f65ebd 1124 unsigned int idx,
f6fb8f10 1125 int status)
1126{
bc59ba39 1127 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1128 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1129
1130 if (status != BLOCK_STATUS(pbd))
1131 return NULL;
1132 return pbd;
1133}
1134
eea49cc9 1135static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1136{
1137 unsigned int prev;
1138 if (rb->prb_bdqc.kactive_blk_num)
1139 prev = rb->prb_bdqc.kactive_blk_num-1;
1140 else
1141 prev = rb->prb_bdqc.knum_blocks-1;
1142 return prev;
1143}
1144
1145/* Assumes caller has held the rx_queue.lock */
eea49cc9 1146static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1147 struct packet_ring_buffer *rb,
1148 int status)
1149{
1150 unsigned int previous = prb_previous_blk_num(rb);
1151 return prb_lookup_block(po, rb, previous, status);
1152}
1153
eea49cc9 1154static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1155 struct packet_ring_buffer *rb,
1156 int status)
1157{
1158 if (po->tp_version <= TPACKET_V2)
1159 return packet_previous_frame(po, rb, status);
1160
1161 return __prb_previous_block(po, rb, status);
1162}
1163
eea49cc9 1164static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1165 struct packet_ring_buffer *rb)
1166{
1167 switch (po->tp_version) {
1168 case TPACKET_V1:
1169 case TPACKET_V2:
1170 return packet_increment_head(rb);
1171 case TPACKET_V3:
1172 default:
1173 WARN(1, "TPACKET version not supported.\n");
1174 BUG();
1175 return;
1176 }
1177}
1178
eea49cc9 1179static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1180 struct packet_ring_buffer *rb,
1181 int status)
1182{
1183 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1184 return packet_lookup_frame(po, rb, previous, status);
1185}
1186
eea49cc9 1187static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1188{
1189 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1190}
1191
b0138408
DB		 1192static void packet_inc_pending(struct packet_ring_buffer *rb)
1193{
1194 this_cpu_inc(*rb->pending_refcnt);
1195}
1196
1197static void packet_dec_pending(struct packet_ring_buffer *rb)
1198{
1199 this_cpu_dec(*rb->pending_refcnt);
1200}
1201
1202static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1203{
1204 unsigned int refcnt = 0;
1205 int cpu;
1206
1207 /* We don't use pending refcount in rx_ring. */
1208 if (rb->pending_refcnt == NULL)
1209 return 0;
1210
1211 for_each_possible_cpu(cpu)
1212 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1213
1214 return refcnt;
1215}
1216
1217static int packet_alloc_pending(struct packet_sock *po)
1218{
1219 po->rx_ring.pending_refcnt = NULL;
1220
1221 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1222 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1223 return -ENOBUFS;
1224
1225 return 0;
1226}
1227
1228static void packet_free_pending(struct packet_sock *po)
1229{
1230 free_percpu(po->tx_ring.pending_refcnt);
1231}
1232
9954729b
WB		 1233#define ROOM_POW_OFF 2
1234#define ROOM_NONE 0x0
1235#define ROOM_LOW 0x1
1236#define ROOM_NORMAL 0x2
1237
1238static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
77f65ebd 1239{
9954729b
WB		 1240 int idx, len;
1241
1242 len = po->rx_ring.frame_max + 1;
1243 idx = po->rx_ring.head;
1244 if (pow_off)
1245 idx += len >> pow_off;
1246 if (idx >= len)
1247 idx -= len;
1248 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1249}
1250
1251static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1252{
1253 int idx, len;
1254
1255 len = po->rx_ring.prb_bdqc.knum_blocks;
1256 idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1257 if (pow_off)
1258 idx += len >> pow_off;
1259 if (idx >= len)
1260 idx -= len;
1261 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1262}
77f65ebd 1263
2ccdbaa6 1264static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
9954729b
WB		 1265{
1266 struct sock *sk = &po->sk;
1267 int ret = ROOM_NONE;
1268
1269 if (po->prot_hook.func != tpacket_rcv) {
1270 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
2ccdbaa6 1271 - (skb ? skb->truesize : 0);
9954729b
WB		 1272 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1273 return ROOM_NORMAL;
1274 else if (avail > 0)
1275 return ROOM_LOW;
1276 else
1277 return ROOM_NONE;
1278 }
77f65ebd 1279
9954729b
WB		 1280 if (po->tp_version == TPACKET_V3) {
1281 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1282 ret = ROOM_NORMAL;
1283 else if (__tpacket_v3_has_room(po, 0))
1284 ret = ROOM_LOW;
1285 } else {
1286 if (__tpacket_has_room(po, ROOM_POW_OFF))
1287 ret = ROOM_NORMAL;
1288 else if (__tpacket_has_room(po, 0))
1289 ret = ROOM_LOW;
1290 }
2ccdbaa6
WB		 1291
1292 return ret;
1293}
1294
1295static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1296{
1297 int ret;
1298 bool has_room;
1299
54d7c01d
WB		 1300 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1301 ret = __packet_rcv_has_room(po, skb);
2ccdbaa6
WB		 1302 has_room = ret == ROOM_NORMAL;
1303 if (po->pressure == has_room)
54d7c01d
WB		 1304 po->pressure = !has_room;
1305 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
77f65ebd 1306
9954729b 1307 return ret;
77f65ebd
WB		 1308}
1309
1da177e4
LT		 1310static void packet_sock_destruct(struct sock *sk)
1311{
ed85b565
RC		 1312 skb_queue_purge(&sk->sk_error_queue);
1313
547b792c
IJ		 1314 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1315 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1316
1317 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1318 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1319 return;
1320 }
1321
17ab56a2 1322 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1323}
1324
3b3a5b0a
WB		 1325static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1326{
1327 u32 rxhash;
1328 int i, count = 0;
1329
1330 rxhash = skb_get_hash(skb);
1331 for (i = 0; i < ROLLOVER_HLEN; i++)
1332 if (po->rollover->history[i] == rxhash)
1333 count++;
1334
1335 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1336 return count > (ROLLOVER_HLEN >> 1);
1337}
1338
77f65ebd
WB		 1339static unsigned int fanout_demux_hash(struct packet_fanout *f,
1340 struct sk_buff *skb,
1341 unsigned int num)
dc99f600 1342{
424848bd 1343 return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
dc99f600
DM		 1344}
1345
77f65ebd
WB		 1346static unsigned int fanout_demux_lb(struct packet_fanout *f,
1347 struct sk_buff *skb,
1348 unsigned int num)
dc99f600 1349{
468479e6 1350 unsigned int val = atomic_inc_return(&f->rr_cur);
dc99f600 1351
468479e6 1352 return val % num;
77f65ebd
WB		 1353}
1354
1355static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1356 struct sk_buff *skb,
1357 unsigned int num)
1358{
1359 return smp_processor_id() % num;
dc99f600
DM		 1360}
1361
5df0ddfb
DB		 1362static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1363 struct sk_buff *skb,
1364 unsigned int num)
1365{
f337db64 1366 return prandom_u32_max(num);
5df0ddfb
DB		 1367}
1368
77f65ebd
WB		 1369static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1370 struct sk_buff *skb,
ad377cab 1371 unsigned int idx, bool try_self,
77f65ebd 1372 unsigned int num)
95ec3eb4 1373{
4633c9e0 1374 struct packet_sock *po, *po_next, *po_skip = NULL;
a9b63918 1375 unsigned int i, j, room = ROOM_NONE;
95ec3eb4 1376
0648ab70 1377 po = pkt_sk(f->arr[idx]);
3b3a5b0a
WB		 1378
1379 if (try_self) {
1380 room = packet_rcv_has_room(po, skb);
1381 if (room == ROOM_NORMAL ||
1382 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1383 return idx;
4633c9e0 1384 po_skip = po;
3b3a5b0a 1385 }
ad377cab 1386
0648ab70 1387 i = j = min_t(int, po->rollover->sock, num - 1);
77f65ebd 1388 do {
2ccdbaa6 1389 po_next = pkt_sk(f->arr[i]);
4633c9e0 1390 if (po_next != po_skip && !po_next->pressure &&
2ccdbaa6 1391 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
77f65ebd 1392 if (i != j)
0648ab70 1393 po->rollover->sock = i;
a9b63918
WB		 1394 atomic_long_inc(&po->rollover->num);
1395 if (room == ROOM_LOW)
1396 atomic_long_inc(&po->rollover->num_huge);
77f65ebd
WB		 1397 return i;
1398 }
ad377cab 1399
77f65ebd
WB		 1400 if (++i == num)
1401 i = 0;
1402 } while (i != j);
1403
a9b63918 1404 atomic_long_inc(&po->rollover->num_failed);
77f65ebd
WB		 1405 return idx;
1406}
1407
2d36097d
NH		 1408static unsigned int fanout_demux_qm(struct packet_fanout *f,
1409 struct sk_buff *skb,
1410 unsigned int num)
1411{
1412 return skb_get_queue_mapping(skb) % num;
1413}
1414
47dceb8e
WB		 1415static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1416 struct sk_buff *skb,
1417 unsigned int num)
1418{
1419 struct bpf_prog *prog;
1420 unsigned int ret = 0;
1421
1422 rcu_read_lock();
1423 prog = rcu_dereference(f->bpf_prog);
1424 if (prog)
ff936a04 1425 ret = bpf_prog_run_clear_cb(prog, skb) % num;
47dceb8e
WB		 1426 rcu_read_unlock();
1427
1428 return ret;
1429}
1430
77f65ebd
WB		 1431static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1432{
1433 return f->flags & (flag >> 8);
95ec3eb4
DM		 1434}
1435
95ec3eb4
DM		 1436static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1437 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1438{
1439 struct packet_fanout *f = pt->af_packet_priv;
f98f4514 1440 unsigned int num = READ_ONCE(f->num_members);
19bcf9f2 1441 struct net *net = read_pnet(&f->net);
dc99f600 1442 struct packet_sock *po;
77f65ebd 1443 unsigned int idx;
dc99f600 1444
19bcf9f2 1445 if (!net_eq(dev_net(dev), net) || !num) {
dc99f600
DM		 1446 kfree_skb(skb);
1447 return 0;
1448 }
1449
3f34b24a 1450 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
19bcf9f2 1451 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
3f34b24a
AD		 1452 if (!skb)
1453 return 0;
1454 }
95ec3eb4
DM		 1455 switch (f->type) {
1456 case PACKET_FANOUT_HASH:
1457 default:
77f65ebd 1458 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1459 break;
1460 case PACKET_FANOUT_LB:
77f65ebd 1461 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1462 break;
1463 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1464 idx = fanout_demux_cpu(f, skb, num);
1465 break;
5df0ddfb
DB		 1466 case PACKET_FANOUT_RND:
1467 idx = fanout_demux_rnd(f, skb, num);
1468 break;
2d36097d
NH		 1469 case PACKET_FANOUT_QM:
1470 idx = fanout_demux_qm(f, skb, num);
1471 break;
77f65ebd 1472 case PACKET_FANOUT_ROLLOVER:
ad377cab 1473 idx = fanout_demux_rollover(f, skb, 0, false, num);
95ec3eb4 1474 break;
47dceb8e 1475 case PACKET_FANOUT_CBPF:
f2e52095 1476 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1477 idx = fanout_demux_bpf(f, skb, num);
1478 break;
dc99f600
DM		 1479 }
1480
ad377cab
WB		 1481 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1482 idx = fanout_demux_rollover(f, skb, idx, true, num);
dc99f600 1483
ad377cab 1484 po = pkt_sk(f->arr[idx]);
dc99f600
DM		 1485 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1486}
1487
fff3321d
PE		 1488DEFINE_MUTEX(fanout_mutex);
1489EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1490static LIST_HEAD(fanout_list);
1491
1492static void __fanout_link(struct sock *sk, struct packet_sock *po)
1493{
1494 struct packet_fanout *f = po->fanout;
1495
1496 spin_lock(&f->lock);
1497 f->arr[f->num_members] = sk;
1498 smp_wmb();
1499 f->num_members++;
fe41cfb4
AS		 1500 if (f->num_members == 1)
1501 dev_add_pack(&f->prot_hook);
dc99f600
DM		 1502 spin_unlock(&f->lock);
1503}
1504
1505static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1506{
1507 struct packet_fanout *f = po->fanout;
1508 int i;
1509
1510 spin_lock(&f->lock);
1511 for (i = 0; i < f->num_members; i++) {
1512 if (f->arr[i] == sk)
1513 break;
1514 }
1515 BUG_ON(i >= f->num_members);
1516 f->arr[i] = f->arr[f->num_members - 1];
1517 f->num_members--;
fe41cfb4
AS		 1518 if (f->num_members == 0)
1519 __dev_remove_pack(&f->prot_hook);
dc99f600
DM		 1520 spin_unlock(&f->lock);
1521}
1522
d4dd8aee 1523static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1524{
161642e2
ED		 1525 if (sk->sk_family != PF_PACKET)
1526 return false;
c0de08d0 1527
161642e2 1528 return ptype->af_packet_priv == pkt_sk(sk)->fanout;
c0de08d0
EL		 1529}
1530
47dceb8e
WB		 1531static void fanout_init_data(struct packet_fanout *f)
1532{
1533 switch (f->type) {
1534 case PACKET_FANOUT_LB:
1535 atomic_set(&f->rr_cur, 0);
1536 break;
1537 case PACKET_FANOUT_CBPF:
f2e52095 1538 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1539 RCU_INIT_POINTER(f->bpf_prog, NULL);
1540 break;
1541 }
1542}
1543
1544static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1545{
1546 struct bpf_prog *old;
1547
1548 spin_lock(&f->lock);
1549 old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1550 rcu_assign_pointer(f->bpf_prog, new);
1551 spin_unlock(&f->lock);
1552
1553 if (old) {
1554 synchronize_net();
1555 bpf_prog_destroy(old);
1556 }
1557}
1558
1559static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data,
1560 unsigned int len)
1561{
1562 struct bpf_prog *new;
1563 struct sock_fprog fprog;
1564 int ret;
1565
1566 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1567 return -EPERM;
1568 if (len != sizeof(fprog))
1569 return -EINVAL;
1570 if (copy_from_user(&fprog, data, len))
1571 return -EFAULT;
1572
bab18991 1573 ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
47dceb8e
WB		 1574 if (ret)
1575 return ret;
1576
1577 __fanout_set_data_bpf(po->fanout, new);
1578 return 0;
1579}
1580
f2e52095
WB		 1581static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data,
1582 unsigned int len)
1583{
1584 struct bpf_prog *new;
1585 u32 fd;
1586
1587 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1588 return -EPERM;
1589 if (len != sizeof(fd))
1590 return -EINVAL;
1591 if (copy_from_user(&fd, data, len))
1592 return -EFAULT;
1593
1594 new = bpf_prog_get(fd);
1595 if (IS_ERR(new))
1596 return PTR_ERR(new);
1597 if (new->type != BPF_PROG_TYPE_SOCKET_FILTER) {
1598 bpf_prog_put(new);
1599 return -EINVAL;
1600 }
1601
1602 __fanout_set_data_bpf(po->fanout, new);
1603 return 0;
1604}
1605
47dceb8e
WB		 1606static int fanout_set_data(struct packet_sock *po, char __user *data,
1607 unsigned int len)
1608{
1609 switch (po->fanout->type) {
1610 case PACKET_FANOUT_CBPF:
1611 return fanout_set_data_cbpf(po, data, len);
f2e52095
WB		 1612 case PACKET_FANOUT_EBPF:
1613 return fanout_set_data_ebpf(po, data, len);
47dceb8e
WB		 1614 default:
1615 return -EINVAL;
1616 };
1617}
1618
1619static void fanout_release_data(struct packet_fanout *f)
1620{
1621 switch (f->type) {
1622 case PACKET_FANOUT_CBPF:
f2e52095 1623 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1624 __fanout_set_data_bpf(f, NULL);
1625 };
1626}
1627
7736d33f 1628static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600 1629{
abd672de 1630 struct packet_rollover *rollover = NULL;
dc99f600
DM		 1631 struct packet_sock *po = pkt_sk(sk);
1632 struct packet_fanout *f, *match;
7736d33f 1633 u8 type = type_flags & 0xff;
77f65ebd 1634 u8 flags = type_flags >> 8;
dc99f600
DM		 1635 int err;
1636
1637 switch (type) {
77f65ebd
WB		 1638 case PACKET_FANOUT_ROLLOVER:
1639 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1640 return -EINVAL;
dc99f600
DM		 1641 case PACKET_FANOUT_HASH:
1642 case PACKET_FANOUT_LB:
95ec3eb4 1643 case PACKET_FANOUT_CPU:
5df0ddfb 1644 case PACKET_FANOUT_RND:
2d36097d 1645 case PACKET_FANOUT_QM:
47dceb8e 1646 case PACKET_FANOUT_CBPF:
f2e52095 1647 case PACKET_FANOUT_EBPF:
dc99f600
DM		 1648 break;
1649 default:
1650 return -EINVAL;
1651 }
1652
abd672de
ED		 1653 mutex_lock(&fanout_mutex);
1654
abd672de 1655 err = -EALREADY;
dc99f600 1656 if (po->fanout)
abd672de 1657 goto out;
dc99f600 1658
4633c9e0
WB		 1659 if (type == PACKET_FANOUT_ROLLOVER ||
1660 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
abd672de
ED		 1661 err = -ENOMEM;
1662 rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1663 if (!rollover)
1664 goto out;
1665 atomic_long_set(&rollover->num, 0);
1666 atomic_long_set(&rollover->num_huge, 0);
1667 atomic_long_set(&rollover->num_failed, 0);
1cac41cb 1668 po->rollover = rollover;
0648ab70
WB		 1669 }
1670
dc99f600
DM		 1671 match = NULL;
1672 list_for_each_entry(f, &fanout_list, list) {
1673 if (f->id == id &&
1674 read_pnet(&f->net) == sock_net(sk)) {
1675 match = f;
1676 break;
1677 }
1678 }
afe62c68 1679 err = -EINVAL;
77f65ebd 1680 if (match && match->flags != flags)
afe62c68 1681 goto out;
dc99f600 1682 if (!match) {
afe62c68 1683 err = -ENOMEM;
dc99f600 1684 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1685 if (!match)
1686 goto out;
1687 write_pnet(&match->net, sock_net(sk));
1688 match->id = id;
1689 match->type = type;
77f65ebd 1690 match->flags = flags;
afe62c68
ED		 1691 INIT_LIST_HEAD(&match->list);
1692 spin_lock_init(&match->lock);
1693 atomic_set(&match->sk_ref, 0);
47dceb8e 1694 fanout_init_data(match);
afe62c68
ED		 1695 match->prot_hook.type = po->prot_hook.type;
1696 match->prot_hook.dev = po->prot_hook.dev;
1697 match->prot_hook.func = packet_rcv_fanout;
1698 match->prot_hook.af_packet_priv = match;
c0de08d0 1699 match->prot_hook.id_match = match_fanout_group;
afe62c68 1700 list_add(&match->list, &fanout_list);
dc99f600 1701 }
afe62c68 1702 err = -EINVAL;
5be6824b
WB		 1703
1704 spin_lock(&po->bind_lock);
1705 if (po->running &&
1706 match->type == type &&
afe62c68
ED		 1707 match->prot_hook.type == po->prot_hook.type &&
1708 match->prot_hook.dev == po->prot_hook.dev) {
1709 err = -ENOSPC;
1710 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1711 __dev_remove_pack(&po->prot_hook);
1712 po->fanout = match;
f50e9c87
MM		 1713 po->rollover = rollover;
1714 rollover = NULL;
afe62c68
ED		 1715 atomic_inc(&match->sk_ref);
1716 __fanout_link(sk, po);
1717 err = 0;
dc99f600
DM		 1718 }
1719 }
5be6824b
WB		 1720 spin_unlock(&po->bind_lock);
1721
1299f7e1 1722 if (err && !atomic_read(&match->sk_ref)) {
5be6824b
WB		 1723 list_del(&match->list);
1724 kfree(match);
1725 }
1726
afe62c68 1727out:
f50e9c87 1728 kfree(rollover);
abd672de 1729 mutex_unlock(&fanout_mutex);
dc99f600
DM		 1730 return err;
1731}
1732
fe41cfb4
AS		 1733/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1734 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1735 * It is the responsibility of the caller to call fanout_release_data() and
1736 * free the returned packet_fanout (after synchronize_net())
1737 */
1738static struct packet_fanout *fanout_release(struct sock *sk)
dc99f600
DM		 1739{
1740 struct packet_sock *po = pkt_sk(sk);
1741 struct packet_fanout *f;
1742
fff3321d 1743 mutex_lock(&fanout_mutex);
abd672de
ED		 1744 f = po->fanout;
1745 if (f) {
1746 po->fanout = NULL;
1747
fe41cfb4 1748 if (atomic_dec_and_test(&f->sk_ref))
abd672de 1749 list_del(&f->list);
fe41cfb4
AS		 1750 else
1751 f = NULL;
dc99f600
DM		 1752 }
1753 mutex_unlock(&fanout_mutex);
fe41cfb4
AS		 1754
1755 return f;
dc99f600 1756}
1da177e4 1757
3c70c132
DB		 1758static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1759 struct sk_buff *skb)
1760{
1761 /* Earlier code assumed this would be a VLAN pkt, double-check
1762 * this now that we have the actual packet in hand. We can only
1763 * do this check on Ethernet devices.
1764 */
1765 if (unlikely(dev->type != ARPHRD_ETHER))
1766 return false;
1767
1768 skb_reset_mac_header(skb);
1769 return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1770}
1771
90ddc4f0 1772static const struct proto_ops packet_ops;
1da177e4 1773
90ddc4f0 1774static const struct proto_ops packet_ops_spkt;
1da177e4 1775
40d4e3df
ED		 1776static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1777 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1778{
1779 struct sock *sk;
1780 struct sockaddr_pkt *spkt;
1781
1782 /*
1783 * When we registered the protocol we saved the socket in the data
1784 * field for just this event.
1785 */
1786
1787 sk = pt->af_packet_priv;
1ce4f28b 1788
1da177e4
LT		 1789 /*
1790 * Yank back the headers [hope the device set this
1791 * right or kerboom...]
1792 *
1793 * Incoming packets have ll header pulled,
1794 * push it back.
1795 *
98e399f8 1796 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1797 * so that this procedure is noop.
1798 */
1799
1800 if (skb->pkt_type == PACKET_LOOPBACK)
1801 goto out;
1802
09ad9bc7 1803 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1804 goto out;
1805
40d4e3df
ED		 1806 skb = skb_share_check(skb, GFP_ATOMIC);
1807 if (skb == NULL)
1da177e4
LT		 1808 goto oom;
1809
1810 /* drop any routing info */
adf30907 1811 skb_dst_drop(skb);
1da177e4 1812
84531c24
PO		 1813 /* drop conntrack reference */
1814 nf_reset(skb);
1815
ffbc6111 1816 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1817
98e399f8 1818 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1819
1820 /*
1821 * The SOCK_PACKET socket receives _all_ frames.
1822 */
1823
1824 spkt->spkt_family = dev->type;
1825 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1826 spkt->spkt_protocol = skb->protocol;
1827
1828 /*
1829 * Charge the memory to the socket. This is done specifically
1830 * to prevent sockets using all the memory up.
1831 */
1832
40d4e3df 1833 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1834 return 0;
1835
1836out:
1837 kfree_skb(skb);
1838oom:
1839 return 0;
1840}
1841
1842
1843/*
1844 * Output a raw packet to a device layer. This bypasses all the other
1845 * protocol layers and you must therefore supply it with a complete frame
1846 */
1ce4f28b 1847
1b784140
YX		 1848static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1849 size_t len)
1da177e4
LT		 1850{
1851 struct sock *sk = sock->sk;
342dfc30 1852 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1853 struct sk_buff *skb = NULL;
1da177e4 1854 struct net_device *dev;
40d4e3df 1855 __be16 proto = 0;
1da177e4 1856 int err;
3bdc0eba 1857 int extra_len = 0;
1ce4f28b 1858
1da177e4 1859 /*
1ce4f28b 1860 * Get and verify the address.
1da177e4
LT		 1861 */
1862
40d4e3df 1863 if (saddr) {
1da177e4 1864 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1865 return -EINVAL;
1866 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1867 proto = saddr->spkt_protocol;
1868 } else
1869 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1870
1871 /*
1ce4f28b 1872 * Find the device first to size check it
1da177e4
LT		 1873 */
1874
de74e92a 1875 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1876retry:
654d1f8a
ED		 1877 rcu_read_lock();
1878 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1879 err = -ENODEV;
1880 if (dev == NULL)
1881 goto out_unlock;
1ce4f28b 1882
d5e76b0a
DM		 1883 err = -ENETDOWN;
1884 if (!(dev->flags & IFF_UP))
1885 goto out_unlock;
1886
1da177e4 1887 /*
40d4e3df
ED		 1888 * You may not queue a frame bigger than the mtu. This is the lowest level
1889 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1890 */
1ce4f28b 1891
3bdc0eba
BG		 1892 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1893 if (!netif_supports_nofcs(dev)) {
1894 err = -EPROTONOSUPPORT;
1895 goto out_unlock;
1896 }
1897 extra_len = 4; /* We're doing our own CRC */
1898 }
1899
1da177e4 1900 err = -EMSGSIZE;
3bdc0eba 1901 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1902 goto out_unlock;
1903
1a35ca80
ED		 1904 if (!skb) {
1905 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1906 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1907 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1908
1909 rcu_read_unlock();
4ce40912 1910 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1911 if (skb == NULL)
1912 return -ENOBUFS;
1913 /* FIXME: Save some space for broken drivers that write a hard
1914 * header at transmission time by themselves. PPP is the notable
1915 * one here. This should really be fixed at the driver level.
1916 */
1917 skb_reserve(skb, reserved);
1918 skb_reset_network_header(skb);
1919
1920 /* Try to align data part correctly */
1921 if (hhlen) {
1922 skb->data -= hhlen;
1923 skb->tail -= hhlen;
1924 if (len < hhlen)
1925 skb_reset_network_header(skb);
1926 }
6ce8e9ce 1927 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1a35ca80
ED		 1928 if (err)
1929 goto out_free;
1930 goto retry;
1da177e4
LT		 1931 }
1932
edb60bc7
WB		 1933 if (!dev_validate_header(dev, skb->data, len)) {
1934 err = -EINVAL;
1935 goto out_unlock;
1936 }
3c70c132
DB		 1937 if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
1938 !packet_extra_vlan_len_allowed(dev, skb)) {
1939 err = -EMSGSIZE;
1940 goto out_unlock;
57f89bfa 1941 }
1a35ca80 1942
1da177e4
LT		 1943 skb->protocol = proto;
1944 skb->dev = dev;
1945 skb->priority = sk->sk_priority;
2d37a186 1946 skb->mark = sk->sk_mark;
bf84a010
DB		 1947
1948 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1949
3bdc0eba
BG		 1950 if (unlikely(extra_len == 4))
1951 skb->no_fcs = 1;
1952
40893fd0 1953 skb_probe_transport_header(skb, 0);
c1aad275 1954
1da177e4 1955 dev_queue_xmit(skb);
654d1f8a 1956 rcu_read_unlock();
40d4e3df 1957 return len;
1da177e4 1958
1da177e4 1959out_unlock:
654d1f8a 1960 rcu_read_unlock();
1a35ca80
ED		 1961out_free:
1962 kfree_skb(skb);
1da177e4
LT		 1963 return err;
1964}
1da177e4 1965
ff936a04
AS		 1966static unsigned int run_filter(struct sk_buff *skb,
1967 const struct sock *sk,
1968 unsigned int res)
1da177e4
LT		 1969{
1970 struct sk_filter *filter;
fda9ef5d 1971
80f8f102
ED		 1972 rcu_read_lock();
1973 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1974 if (filter != NULL)
ff936a04 1975 res = bpf_prog_run_clear_cb(filter->prog, skb);
80f8f102 1976 rcu_read_unlock();
1da177e4 1977
dbcb5855 1978 return res;
1da177e4
LT		 1979}
1980
1981/*
62ab0812
ED		 1982 * This function makes lazy skb cloning in hope that most of packets
1983 * are discarded by BPF.
1984 *
1985 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1986 * and skb->cb are mangled. It works because (and until) packets
1987 * falling here are owned by current CPU. Output packets are cloned
1988 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1989 * sequencially, so that if we return skb to original state on exit,
1990 * we will not harm anyone.
1da177e4
LT		 1991 */
1992
40d4e3df
ED		 1993static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1994 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1995{
1996 struct sock *sk;
1997 struct sockaddr_ll *sll;
1998 struct packet_sock *po;
40d4e3df 1999 u8 *skb_head = skb->data;
1da177e4 2000 int skb_len = skb->len;
dbcb5855 2001 unsigned int snaplen, res;
1da177e4
LT		 2002
2003 if (skb->pkt_type == PACKET_LOOPBACK)
2004 goto drop;
2005
2006 sk = pt->af_packet_priv;
2007 po = pkt_sk(sk);
2008
09ad9bc7 2009 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 2010 goto drop;
2011
1da177e4
LT		 2012 skb->dev = dev;
2013
3b04ddde 2014 if (dev->header_ops) {
1da177e4 2015 /* The device has an explicit notion of ll header,
62ab0812
ED		 2016 * exported to higher levels.
2017 *
2018 * Otherwise, the device hides details of its frame
2019 * structure, so that corresponding packet head is
2020 * never delivered to user.
1da177e4
LT		 2021 */
2022 if (sk->sk_type != SOCK_DGRAM)
98e399f8 2023 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 2024 else if (skb->pkt_type == PACKET_OUTGOING) {
2025 /* Special case: outgoing packets have ll header at head */
bbe735e4 2026 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 2027 }
2028 }
2029
2030 snaplen = skb->len;
2031
dbcb5855
DM		 2032 res = run_filter(skb, sk, snaplen);
2033 if (!res)
fda9ef5d 2034 goto drop_n_restore;
dbcb5855
DM		 2035 if (snaplen > res)
2036 snaplen = res;
1da177e4 2037
0fd7bac6 2038 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 2039 goto drop_n_acct;
2040
2041 if (skb_shared(skb)) {
2042 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2043 if (nskb == NULL)
2044 goto drop_n_acct;
2045
2046 if (skb_head != skb->data) {
2047 skb->data = skb_head;
2048 skb->len = skb_len;
2049 }
abc4e4fa 2050 consume_skb(skb);
1da177e4
LT		 2051 skb = nskb;
2052 }
2053
b4772ef8 2054 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
ffbc6111
HX		 2055
2056 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4 2057 sll->sll_hatype = dev->type;
1da177e4 2058 sll->sll_pkttype = skb->pkt_type;
8032b464 2059 if (unlikely(po->origdev))
80feaacb
PWJ		 2060 sll->sll_ifindex = orig_dev->ifindex;
2061 else
2062 sll->sll_ifindex = dev->ifindex;
1da177e4 2063
b95cce35 2064 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 2065
2472d761
EB		 2066 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2067 * Use their space for storing the original skb length.
2068 */
2069 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
8dc41944 2070
1da177e4
LT		 2071 if (pskb_trim(skb, snaplen))
2072 goto drop_n_acct;
2073
2074 skb_set_owner_r(skb, sk);
2075 skb->dev = NULL;
adf30907 2076 skb_dst_drop(skb);
1da177e4 2077
84531c24
PO		 2078 /* drop conntrack reference */
2079 nf_reset(skb);
2080
1da177e4 2081 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 2082 po->stats.stats1.tp_packets++;
3bc3b96f 2083 sock_skb_set_dropcount(sk, skb);
1da177e4
LT		 2084 __skb_queue_tail(&sk->sk_receive_queue, skb);
2085 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 2086 sk->sk_data_ready(sk);
1da177e4
LT		 2087 return 0;
2088
2089drop_n_acct:
7091fbd8 2090 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 2091 po->stats.stats1.tp_drops++;
7091fbd8
WB		 2092 atomic_inc(&sk->sk_drops);
2093 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 2094
2095drop_n_restore:
2096 if (skb_head != skb->data && skb_shared(skb)) {
2097 skb->data = skb_head;
2098 skb->len = skb_len;
2099 }
2100drop:
ead2ceb0 2101 consume_skb(skb);
1da177e4
LT		 2102 return 0;
2103}
2104
40d4e3df
ED		 2105static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2106 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 2107{
2108 struct sock *sk;
2109 struct packet_sock *po;
2110 struct sockaddr_ll *sll;
184f489e 2111 union tpacket_uhdr h;
40d4e3df 2112 u8 *skb_head = skb->data;
1da177e4 2113 int skb_len = skb->len;
dbcb5855 2114 unsigned int snaplen, res;
f6fb8f10 2115 unsigned long status = TP_STATUS_USER;
bbd6ef87 2116 unsigned short macoff, netoff, hdrlen;
1da177e4 2117 struct sk_buff *copy_skb = NULL;
bbd6ef87 2118 struct timespec ts;
b9c32fb2 2119 __u32 ts_status;
1da177e4 2120
51846355
AW		 2121 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2122 * We may add members to them until current aligned size without forcing
2123 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2124 */
2125 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2126 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2127
1da177e4
LT		 2128 if (skb->pkt_type == PACKET_LOOPBACK)
2129 goto drop;
2130
2131 sk = pt->af_packet_priv;
2132 po = pkt_sk(sk);
2133
09ad9bc7 2134 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 2135 goto drop;
2136
3b04ddde 2137 if (dev->header_ops) {
1da177e4 2138 if (sk->sk_type != SOCK_DGRAM)
98e399f8 2139 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 2140 else if (skb->pkt_type == PACKET_OUTGOING) {
2141 /* Special case: outgoing packets have ll header at head */
bbe735e4 2142 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 2143 }
2144 }
2145
2146 snaplen = skb->len;
2147
dbcb5855
DM		 2148 res = run_filter(skb, sk, snaplen);
2149 if (!res)
fda9ef5d 2150 goto drop_n_restore;
68c2e5de
AD		 2151
2152 if (skb->ip_summed == CHECKSUM_PARTIAL)
2153 status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 2154 else if (skb->pkt_type != PACKET_OUTGOING &&
2155 (skb->ip_summed == CHECKSUM_COMPLETE ||
2156 skb_csum_unnecessary(skb)))
2157 status |= TP_STATUS_CSUM_VALID;
68c2e5de 2158
dbcb5855
DM		 2159 if (snaplen > res)
2160 snaplen = res;
1da177e4
LT		 2161
2162 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 2163 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2164 po->tp_reserve;
1da177e4 2165 } else {
95c96174 2166 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 2167 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 2168 (maclen < 16 ? 16 : maclen)) +
2169 po->tp_reserve;
1da177e4
LT		 2170 macoff = netoff - maclen;
2171 }
f6fb8f10 2172 if (po->tp_version <= TPACKET_V2) {
2173 if (macoff + snaplen > po->rx_ring.frame_size) {
2174 if (po->copy_thresh &&
0fd7bac6 2175 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 2176 if (skb_shared(skb)) {
2177 copy_skb = skb_clone(skb, GFP_ATOMIC);
2178 } else {
2179 copy_skb = skb_get(skb);
2180 skb_head = skb->data;
2181 }
2182 if (copy_skb)
2183 skb_set_owner_r(copy_skb, sk);
1da177e4 2184 }
f6fb8f10 2185 snaplen = po->rx_ring.frame_size - macoff;
2186 if ((int)snaplen < 0)
2187 snaplen = 0;
1da177e4 2188 }
dc808110
ED		 2189 } else if (unlikely(macoff + snaplen >
2190 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2191 u32 nval;
2192
2193 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2194 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2195 snaplen, nval, macoff);
2196 snaplen = nval;
2197 if (unlikely((int)snaplen < 0)) {
2198 snaplen = 0;
2199 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2200 }
1da177e4 2201 }
1da177e4 2202 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 2203 h.raw = packet_current_rx_frame(po, skb,
2204 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 2205 if (!h.raw)
1da177e4 2206 goto ring_is_full;
f6fb8f10 2207 if (po->tp_version <= TPACKET_V2) {
2208 packet_increment_rx_head(po, &po->rx_ring);
2209 /*
2210 * LOSING will be reported till you read the stats,
2211 * because it's COR - Clear On Read.
2212 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2213 * at packet level.
2214 */
ee80fbf3 2215 if (po->stats.stats1.tp_drops)
f6fb8f10 2216 status |= TP_STATUS_LOSING;
2217 }
ee80fbf3 2218 po->stats.stats1.tp_packets++;
1da177e4
LT		 2219 if (copy_skb) {
2220 status |= TP_STATUS_COPY;
2221 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2222 }
1da177e4
LT		 2223 spin_unlock(&sk->sk_receive_queue.lock);
2224
bbd6ef87 2225 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 2226
2227 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 2228 getnstimeofday(&ts);
1da177e4 2229
b9c32fb2
DB		 2230 status |= ts_status;
2231
bbd6ef87
PM		 2232 switch (po->tp_version) {
2233 case TPACKET_V1:
2234 h.h1->tp_len = skb->len;
2235 h.h1->tp_snaplen = snaplen;
2236 h.h1->tp_mac = macoff;
2237 h.h1->tp_net = netoff;
4b457bdf
DB		 2238 h.h1->tp_sec = ts.tv_sec;
2239 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 2240 hdrlen = sizeof(*h.h1);
2241 break;
2242 case TPACKET_V2:
2243 h.h2->tp_len = skb->len;
2244 h.h2->tp_snaplen = snaplen;
2245 h.h2->tp_mac = macoff;
2246 h.h2->tp_net = netoff;
bbd6ef87
PM		 2247 h.h2->tp_sec = ts.tv_sec;
2248 h.h2->tp_nsec = ts.tv_nsec;
df8a39de
JP		 2249 if (skb_vlan_tag_present(skb)) {
2250 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 2251 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2252 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2253 } else {
2254 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2255 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2256 }
e4d26f4b 2257 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2258 hdrlen = sizeof(*h.h2);
2259 break;
f6fb8f10 2260 case TPACKET_V3:
2261 /* tp_nxt_offset,vlan are already populated above.
2262 * So DONT clear those fields here
2263 */
2264 h.h3->tp_status |= status;
2265 h.h3->tp_len = skb->len;
2266 h.h3->tp_snaplen = snaplen;
2267 h.h3->tp_mac = macoff;
2268 h.h3->tp_net = netoff;
f6fb8f10 2269 h.h3->tp_sec = ts.tv_sec;
2270 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2271 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2272 hdrlen = sizeof(*h.h3);
2273 break;
bbd6ef87
PM		 2274 default:
2275 BUG();
2276 }
1da177e4 2277
bbd6ef87 2278 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2279 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2280 sll->sll_family = AF_PACKET;
2281 sll->sll_hatype = dev->type;
2282 sll->sll_protocol = skb->protocol;
2283 sll->sll_pkttype = skb->pkt_type;
8032b464 2284 if (unlikely(po->origdev))
80feaacb
PWJ		 2285 sll->sll_ifindex = orig_dev->ifindex;
2286 else
2287 sll->sll_ifindex = dev->ifindex;
1da177e4 2288
e16aa207 2289 smp_mb();
f0d4eb29 2290
f6dafa95 2291#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2292 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2293 u8 *start, *end;
2294
f0d4eb29
DB		 2295 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2296 macoff + snaplen);
2297
2298 for (start = h.raw; start < end; start += PAGE_SIZE)
2299 flush_dcache_page(pgv_to_page(start));
1da177e4 2300 }
f0d4eb29 2301 smp_wmb();
f6dafa95 2302#endif
f0d4eb29 2303
da413eec 2304 if (po->tp_version <= TPACKET_V2) {
f6fb8f10 2305 __packet_set_status(po, h.raw, status);
da413eec
DC		 2306 sk->sk_data_ready(sk);
2307 } else {
f6fb8f10 2308 prb_clear_blk_fill_status(&po->rx_ring);
da413eec 2309 }
1da177e4
LT		 2310
2311drop_n_restore:
2312 if (skb_head != skb->data && skb_shared(skb)) {
2313 skb->data = skb_head;
2314 skb->len = skb_len;
2315 }
2316drop:
1ce4f28b 2317 kfree_skb(skb);
1da177e4
LT		 2318 return 0;
2319
2320ring_is_full:
ee80fbf3 2321 po->stats.stats1.tp_drops++;
1da177e4
LT		 2322 spin_unlock(&sk->sk_receive_queue.lock);
2323
676d2369 2324 sk->sk_data_ready(sk);
acb5d75b 2325 kfree_skb(copy_skb);
1da177e4
LT		 2326 goto drop_n_restore;
2327}
2328
69e3c75f
JB		 2329static void tpacket_destruct_skb(struct sk_buff *skb)
2330{
2331 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2332
69e3c75f 2333 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2334 void *ph;
b9c32fb2
DB		 2335 __u32 ts;
2336
69e3c75f 2337 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2338 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2339
2340 ts = __packet_set_timestamp(po, ph, skb);
2341 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2342 }
2343
2344 sock_wfree(skb);
2345}
2346
c72219b7
DB		 2347static void tpacket_set_protocol(const struct net_device *dev,
2348 struct sk_buff *skb)
2349{
2350 if (dev->type == ARPHRD_ETHER) {
2351 skb_reset_mac_header(skb);
2352 skb->protocol = eth_hdr(skb)->h_proto;
2353 }
2354}
2355
40d4e3df
ED		 2356static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2357 void *frame, struct net_device *dev, int size_max,
ae641949 2358 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2359{
184f489e 2360 union tpacket_uhdr ph;
09effa67 2361 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2362 struct socket *sock = po->sk.sk_socket;
2363 struct page *page;
2364 void *data;
2365 int err;
2366
2367 ph.raw = frame;
2368
2369 skb->protocol = proto;
2370 skb->dev = dev;
2371 skb->priority = po->sk.sk_priority;
2d37a186 2372 skb->mark = po->sk.sk_mark;
2e31396f 2373 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2374 skb_shinfo(skb)->destructor_arg = ph.raw;
2375
2376 switch (po->tp_version) {
2377 case TPACKET_V2:
2378 tp_len = ph.h2->tp_len;
2379 break;
2380 default:
2381 tp_len = ph.h1->tp_len;
2382 break;
2383 }
09effa67
DM		 2384 if (unlikely(tp_len > size_max)) {
2385 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2386 return -EMSGSIZE;
2387 }
69e3c75f 2388
ae641949 2389 skb_reserve(skb, hlen);
69e3c75f 2390 skb_reset_network_header(skb);
c1aad275 2391
d346a3fa 2392 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2393 int off_min, off_max, off;
2394 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2395 off_max = po->tx_ring.frame_size - tp_len;
2396 if (sock->type == SOCK_DGRAM) {
2397 switch (po->tp_version) {
2398 case TPACKET_V2:
2399 off = ph.h2->tp_net;
2400 break;
2401 default:
2402 off = ph.h1->tp_net;
2403 break;
2404 }
2405 } else {
2406 switch (po->tp_version) {
2407 case TPACKET_V2:
2408 off = ph.h2->tp_mac;
2409 break;
2410 default:
2411 off = ph.h1->tp_mac;
2412 break;
2413 }
2414 }
2415 if (unlikely((off < off_min) || (off_max < off)))
2416 return -EINVAL;
2417 data = ph.raw + off;
2418 } else {
2419 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2420 }
69e3c75f
JB		 2421 to_write = tp_len;
2422
2423 if (sock->type == SOCK_DGRAM) {
2424 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2425 NULL, tp_len);
2426 if (unlikely(err < 0))
2427 return -EINVAL;
40d4e3df 2428 } else if (dev->hard_header_len) {
edb60bc7 2429 int hdrlen = min_t(int, dev->hard_header_len, tp_len);
69e3c75f
JB		 2430
2431 skb_push(skb, dev->hard_header_len);
edb60bc7 2432 err = skb_store_bits(skb, 0, data, hdrlen);
69e3c75f
JB		 2433 if (unlikely(err))
2434 return err;
edb60bc7
WB		 2435 if (!dev_validate_header(dev, skb->data, hdrlen))
2436 return -EINVAL;
c72219b7
DB		 2437 if (!skb->protocol)
2438 tpacket_set_protocol(dev, skb);
69e3c75f 2439
edb60bc7
WB		 2440 data += hdrlen;
2441 to_write -= hdrlen;
69e3c75f
JB		 2442 }
2443
69e3c75f
JB		 2444 offset = offset_in_page(data);
2445 len_max = PAGE_SIZE - offset;
2446 len = ((to_write > len_max) ? len_max : to_write);
2447
2448 skb->data_len = to_write;
2449 skb->len += to_write;
2450 skb->truesize += to_write;
2451 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2452
2453 while (likely(to_write)) {
2454 nr_frags = skb_shinfo(skb)->nr_frags;
2455
2456 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2457 pr_err("Packet exceed the number of skb frags(%lu)\n",
2458 MAX_SKB_FRAGS);
69e3c75f
JB		 2459 return -EFAULT;
2460 }
2461
0af55bb5
CG		 2462 page = pgv_to_page(data);
2463 data += len;
69e3c75f
JB		 2464 flush_dcache_page(page);
2465 get_page(page);
0af55bb5 2466 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2467 to_write -= len;
2468 offset = 0;
2469 len_max = PAGE_SIZE;
2470 len = ((to_write > len_max) ? len_max : to_write);
2471 }
2472
8fd6c80d 2473 skb_probe_transport_header(skb, 0);
efdfa2f7 2474
69e3c75f
JB		 2475 return tp_len;
2476}
2477
2478static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2479{
69e3c75f
JB		 2480 struct sk_buff *skb;
2481 struct net_device *dev;
2482 __be16 proto;
09effa67 2483 int err, reserve = 0;
40d4e3df 2484 void *ph;
342dfc30 2485 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2486 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2487 int tp_len, size_max;
2488 unsigned char *addr;
2489 int len_sum = 0;
9e67030a 2490 int status = TP_STATUS_AVAILABLE;
ae641949 2491 int hlen, tlen;
69e3c75f 2492
69e3c75f
JB		 2493 mutex_lock(&po->pg_vec_lock);
2494
66e56cd4 2495 if (likely(saddr == NULL)) {
e40526cb 2496 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2497 proto = po->num;
2498 addr = NULL;
2499 } else {
2500 err = -EINVAL;
2501 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2502 goto out;
2503 if (msg->msg_namelen < (saddr->sll_halen
2504 + offsetof(struct sockaddr_ll,
2505 sll_addr)))
2506 goto out;
69e3c75f
JB		 2507 proto = saddr->sll_protocol;
2508 addr = saddr->sll_addr;
827d9780 2509 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2510 }
2511
69e3c75f
JB		 2512 err = -ENXIO;
2513 if (unlikely(dev == NULL))
2514 goto out;
69e3c75f
JB		 2515 err = -ENETDOWN;
2516 if (unlikely(!(dev->flags & IFF_UP)))
2517 goto out_put;
2518
5cfb4c8d
DB		 2519 if (po->sk.sk_socket->type == SOCK_RAW)
2520 reserve = dev->hard_header_len;
69e3c75f 2521 size_max = po->tx_ring.frame_size
b5dd884e 2522 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2523
5cfb4c8d
DB		 2524 if (size_max > dev->mtu + reserve + VLAN_HLEN)
2525 size_max = dev->mtu + reserve + VLAN_HLEN;
09effa67 2526
69e3c75f
JB		 2527 do {
2528 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2529 TP_STATUS_SEND_REQUEST);
69e3c75f 2530 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2531 if (need_wait && need_resched())
2532 schedule();
69e3c75f
JB		 2533 continue;
2534 }
2535
2536 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2537 hlen = LL_RESERVED_SPACE(dev);
2538 tlen = dev->needed_tailroom;
69e3c75f 2539 skb = sock_alloc_send_skb(&po->sk,
ae641949 2540 hlen + tlen + sizeof(struct sockaddr_ll),
fbf33a28 2541 !need_wait, &err);
69e3c75f 2542
fbf33a28
KM		 2543 if (unlikely(skb == NULL)) {
2544 /* we assume the socket was initially writeable ... */
2545 if (likely(len_sum > 0))
2546 err = len_sum;
69e3c75f 2547 goto out_status;
fbf33a28 2548 }
69e3c75f 2549 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f 2550 addr, hlen);
dbd46ab4 2551 if (likely(tp_len >= 0) &&
5cfb4c8d 2552 tp_len > dev->mtu + reserve &&
3c70c132
DB		 2553 !packet_extra_vlan_len_allowed(dev, skb))
2554 tp_len = -EMSGSIZE;
69e3c75f
JB		 2555
2556 if (unlikely(tp_len < 0)) {
2557 if (po->tp_loss) {
2558 __packet_set_status(po, ph,
2559 TP_STATUS_AVAILABLE);
2560 packet_increment_head(&po->tx_ring);
2561 kfree_skb(skb);
2562 continue;
2563 } else {
2564 status = TP_STATUS_WRONG_FORMAT;
2565 err = tp_len;
2566 goto out_status;
2567 }
2568 }
2569
0fd5d57b
DB		 2570 packet_pick_tx_queue(dev, skb);
2571
69e3c75f
JB		 2572 skb->destructor = tpacket_destruct_skb;
2573 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2574 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2575
2576 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2577 err = po->xmit(skb);
eb70df13
JP		 2578 if (unlikely(err > 0)) {
2579 err = net_xmit_errno(err);
2580 if (err && __packet_get_status(po, ph) ==
2581 TP_STATUS_AVAILABLE) {
2582 /* skb was destructed already */
2583 skb = NULL;
2584 goto out_status;
2585 }
2586 /*
2587 * skb was dropped but not destructed yet;
2588 * let's treat it like congestion or err < 0
2589 */
2590 err = 0;
2591 }
69e3c75f
JB		 2592 packet_increment_head(&po->tx_ring);
2593 len_sum += tp_len;
b0138408
DB		 2594 } while (likely((ph != NULL) ||
2595 /* Note: packet_read_pending() might be slow if we have
2596 * to call it as it's per_cpu variable, but in fast-path
2597 * we already short-circuit the loop with the first
2598 * condition, and luckily don't have to go that path
2599 * anyway.
2600 */
2601 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2602
2603 err = len_sum;
2604 goto out_put;
2605
69e3c75f
JB		 2606out_status:
2607 __packet_set_status(po, ph, status);
2608 kfree_skb(skb);
2609out_put:
e40526cb 2610 dev_put(dev);
69e3c75f
JB		 2611out:
2612 mutex_unlock(&po->pg_vec_lock);
2613 return err;
2614}
69e3c75f 2615
eea49cc9
OJ		 2616static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2617 size_t reserve, size_t len,
2618 size_t linear, int noblock,
2619 int *err)
bfd5f4a3
SS		 2620{
2621 struct sk_buff *skb;
2622
2623 /* Under a page? Don't bother with paged skb. */
2624 if (prepad + len < PAGE_SIZE || !linear)
2625 linear = len;
2626
2627 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2628 err, 0);
bfd5f4a3
SS		 2629 if (!skb)
2630 return NULL;
2631
2632 skb_reserve(skb, reserve);
2633 skb_put(skb, linear);
2634 skb->data_len = len - linear;
2635 skb->len += len - linear;
2636
2637 return skb;
2638}
2639
d346a3fa 2640static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2641{
2642 struct sock *sk = sock->sk;
342dfc30 2643 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2644 struct sk_buff *skb;
2645 struct net_device *dev;
0e11c91e 2646 __be16 proto;
1da177e4 2647 unsigned char *addr;
827d9780 2648 int err, reserve = 0;
c7d39e32 2649 struct sockcm_cookie sockc;
bfd5f4a3
SS		 2650 struct virtio_net_hdr vnet_hdr = { 0 };
2651 int offset = 0;
2652 int vnet_hdr_len;
2653 struct packet_sock *po = pkt_sk(sk);
2654 unsigned short gso_type = 0;
1299f7e1 2655 bool has_vnet_hdr = false;
9117c897 2656 int hlen, tlen, linear;
3bdc0eba 2657 int extra_len = 0;
8feb2fb2 2658 ssize_t n;
1da177e4
LT		 2659
2660 /*
1ce4f28b 2661 * Get and verify the address.
1da177e4 2662 */
1ce4f28b 2663
66e56cd4 2664 if (likely(saddr == NULL)) {
e40526cb 2665 dev = packet_cached_dev_get(po);
1da177e4
LT		 2666 proto = po->num;
2667 addr = NULL;
2668 } else {
2669 err = -EINVAL;
2670 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2671 goto out;
0fb375fb
EB		 2672 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2673 goto out;
1da177e4
LT		 2674 proto = saddr->sll_protocol;
2675 addr = saddr->sll_addr;
827d9780 2676 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2677 }
2678
1da177e4 2679 err = -ENXIO;
e40526cb 2680 if (unlikely(dev == NULL))
1da177e4 2681 goto out_unlock;
d5e76b0a 2682 err = -ENETDOWN;
e40526cb 2683 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2684 goto out_unlock;
2685
c7d39e32
EJ		 2686 sockc.mark = sk->sk_mark;
2687 if (msg->msg_controllen) {
2688 err = sock_cmsg_send(sk, msg, &sockc);
2689 if (unlikely(err))
2690 goto out_unlock;
2691 }
2692
e40526cb
DB		 2693 if (sock->type == SOCK_RAW)
2694 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2695 if (po->has_vnet_hdr) {
2696 vnet_hdr_len = sizeof(vnet_hdr);
2697
2698 err = -EINVAL;
2699 if (len < vnet_hdr_len)
2700 goto out_unlock;
2701
2702 len -= vnet_hdr_len;
2703
8feb2fb2 2704 err = -EFAULT;
c0371da6 2705 n = copy_from_iter(&vnet_hdr, vnet_hdr_len, &msg->msg_iter);
8feb2fb2 2706 if (n != vnet_hdr_len)
bfd5f4a3
SS		 2707 goto out_unlock;
2708
2709 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
d3869efe
DW		 2710 (__virtio16_to_cpu(vio_le(), vnet_hdr.csum_start) +
2711 __virtio16_to_cpu(vio_le(), vnet_hdr.csum_offset) + 2 >
2712 __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len)))
2713 vnet_hdr.hdr_len = __cpu_to_virtio16(vio_le(),
2714 __virtio16_to_cpu(vio_le(), vnet_hdr.csum_start) +
2715 __virtio16_to_cpu(vio_le(), vnet_hdr.csum_offset) + 2);
bfd5f4a3
SS		 2716
2717 err = -EINVAL;
d3869efe 2718 if (__virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len) > len)
bfd5f4a3
SS		 2719 goto out_unlock;
2720
2721 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2722 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2723 case VIRTIO_NET_HDR_GSO_TCPV4:
2724 gso_type = SKB_GSO_TCPV4;
2725 break;
2726 case VIRTIO_NET_HDR_GSO_TCPV6:
2727 gso_type = SKB_GSO_TCPV6;
2728 break;
2729 case VIRTIO_NET_HDR_GSO_UDP:
2730 gso_type = SKB_GSO_UDP;
2731 break;
2732 default:
2733 goto out_unlock;
2734 }
2735
2736 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2737 gso_type |= SKB_GSO_TCP_ECN;
2738
2739 if (vnet_hdr.gso_size == 0)
2740 goto out_unlock;
2741
2742 }
1299f7e1 2743 has_vnet_hdr = true;
bfd5f4a3
SS		 2744 }
2745
3bdc0eba
BG		 2746 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2747 if (!netif_supports_nofcs(dev)) {
2748 err = -EPROTONOSUPPORT;
2749 goto out_unlock;
2750 }
2751 extra_len = 4; /* We're doing our own CRC */
2752 }
2753
1da177e4 2754 err = -EMSGSIZE;
3bdc0eba 2755 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2756 goto out_unlock;
2757
bfd5f4a3 2758 err = -ENOBUFS;
ae641949
HX		 2759 hlen = LL_RESERVED_SPACE(dev);
2760 tlen = dev->needed_tailroom;
9117c897
WB		 2761 linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
2762 linear = max(linear, min_t(int, len, dev->hard_header_len));
2763 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
bfd5f4a3 2764 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2765 if (skb == NULL)
1da177e4
LT		 2766 goto out_unlock;
2767
bfd5f4a3 2768 skb_set_network_header(skb, reserve);
1da177e4 2769
0c4e8581 2770 err = -EINVAL;
9c707762
WB		 2771 if (sock->type == SOCK_DGRAM) {
2772 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
46d2cfb1 2773 if (unlikely(offset < 0))
9c707762 2774 goto out_free;
9c707762 2775 }
1da177e4
LT		 2776
2777 /* Returns -EFAULT on error */
c0371da6 2778 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
1da177e4
LT		 2779 if (err)
2780 goto out_free;
bf84a010 2781
edb60bc7
WB		 2782 if (sock->type == SOCK_RAW &&
2783 !dev_validate_header(dev, skb->data, len)) {
2784 err = -EINVAL;
2785 goto out_free;
2786 }
2787
bf84a010 2788 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2789
3c70c132
DB		 2790 if (!gso_type && (len > dev->mtu + reserve + extra_len) &&
2791 !packet_extra_vlan_len_allowed(dev, skb)) {
2792 err = -EMSGSIZE;
2793 goto out_free;
57f89bfa
BG		 2794 }
2795
09effa67
DM		 2796 skb->protocol = proto;
2797 skb->dev = dev;
1da177e4 2798 skb->priority = sk->sk_priority;
c7d39e32 2799 skb->mark = sockc.mark;
0fd5d57b
DB		 2800
2801 packet_pick_tx_queue(dev, skb);
1da177e4 2802
1299f7e1 2803 if (has_vnet_hdr) {
bfd5f4a3 2804 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
d3869efe
DW		 2805 u16 s = __virtio16_to_cpu(vio_le(), vnet_hdr.csum_start);
2806 u16 o = __virtio16_to_cpu(vio_le(), vnet_hdr.csum_offset);
dc9e5153 2807 if (!skb_partial_csum_set(skb, s, o)) {
bfd5f4a3
SS		 2808 err = -EINVAL;
2809 goto out_free;
2810 }
2811 }
2812
dc9e5153 2813 skb_shinfo(skb)->gso_size =
d3869efe 2814 __virtio16_to_cpu(vio_le(), vnet_hdr.gso_size);
bfd5f4a3
SS		 2815 skb_shinfo(skb)->gso_type = gso_type;
2816
2817 /* Header must be checked, and gso_segs computed. */
2818 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2819 skb_shinfo(skb)->gso_segs = 0;
2820
2821 len += vnet_hdr_len;
2822 }
2823
8fd6c80d
DB		 2824 skb_probe_transport_header(skb, reserve);
2825
3bdc0eba
BG		 2826 if (unlikely(extra_len == 4))
2827 skb->no_fcs = 1;
2828
d346a3fa 2829 err = po->xmit(skb);
1da177e4
LT		 2830 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2831 goto out_unlock;
2832
e40526cb 2833 dev_put(dev);
1da177e4 2834
40d4e3df 2835 return len;
1da177e4
LT		 2836
2837out_free:
2838 kfree_skb(skb);
2839out_unlock:
e40526cb 2840 if (dev)
1da177e4
LT		 2841 dev_put(dev);
2842out:
2843 return err;
2844}
2845
1b784140 2846static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
69e3c75f 2847{
69e3c75f
JB		 2848 struct sock *sk = sock->sk;
2849 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2850
69e3c75f
JB		 2851 if (po->tx_ring.pg_vec)
2852 return tpacket_snd(po, msg);
2853 else
69e3c75f
JB		 2854 return packet_snd(sock, msg, len);
2855}
2856
1da177e4
LT		 2857/*
2858 * Close a PACKET socket. This is fairly simple. We immediately go
2859 * to 'closed' state and remove our protocol entry in the device list.
2860 */
2861
2862static int packet_release(struct socket *sock)
2863{
2864 struct sock *sk = sock->sk;
2865 struct packet_sock *po;
fe41cfb4 2866 struct packet_fanout *f;
d12d01d6 2867 struct net *net;
f6fb8f10 2868 union tpacket_req_u req_u;
1da177e4
LT		 2869
2870 if (!sk)
2871 return 0;
2872
3b1e0a65 2873 net = sock_net(sk);
1da177e4
LT		 2874 po = pkt_sk(sk);
2875
0fa7fa98 2876 mutex_lock(&net->packet.sklist_lock);
808f5114 2877 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2878 mutex_unlock(&net->packet.sklist_lock);
2879
2880 preempt_disable();
920de804 2881 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2882 preempt_enable();
1da177e4 2883
808f5114 2884 spin_lock(&po->bind_lock);
ce06b03e 2885 unregister_prot_hook(sk, false);
66e56cd4
DB		 2886 packet_cached_dev_reset(po);
2887
160ff18a
BG		 2888 if (po->prot_hook.dev) {
2889 dev_put(po->prot_hook.dev);
2890 po->prot_hook.dev = NULL;
2891 }
808f5114 2892 spin_unlock(&po->bind_lock);
1da177e4 2893
1da177e4 2894 packet_flush_mclist(sk);
1da177e4 2895
9665d5d6
PS		 2896 if (po->rx_ring.pg_vec) {
2897 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2898 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2899 }
69e3c75f 2900
9665d5d6
PS		 2901 if (po->tx_ring.pg_vec) {
2902 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2903 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2904 }
1da177e4 2905
fe41cfb4 2906 f = fanout_release(sk);
dc99f600 2907
808f5114 2908 synchronize_net();
fe41cfb4
AS		 2909
2910 if (f) {
f50e9c87 2911 kfree(po->rollover);
fe41cfb4
AS		 2912 fanout_release_data(f);
2913 kfree(f);
2914 }
1da177e4
LT		 2915 /*
2916 * Now the socket is dead. No more input will appear.
2917 */
1da177e4
LT		 2918 sock_orphan(sk);
2919 sock->sk = NULL;
2920
2921 /* Purge queues */
2922
2923 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2924 packet_free_pending(po);
17ab56a2 2925 sk_refcnt_debug_release(sk);
1da177e4
LT		 2926
2927 sock_put(sk);
2928 return 0;
2929}
2930
2931/*
2932 * Attach a packet hook.
2933 */
2934
30f7ea1c
FR		 2935static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
2936 __be16 proto)
1da177e4
LT		 2937{
2938 struct packet_sock *po = pkt_sk(sk);
158cd4af 2939 struct net_device *dev_curr;
902fefb8
DB		 2940 __be16 proto_curr;
2941 bool need_rehook;
30f7ea1c
FR		 2942 struct net_device *dev = NULL;
2943 int ret = 0;
2944 bool unlisted = false;
dc99f600 2945
1da177e4 2946 lock_sock(sk);
1da177e4 2947 spin_lock(&po->bind_lock);
30f7ea1c
FR		 2948 rcu_read_lock();
2949
1b6c80e7
WB		 2950 if (po->fanout) {
2951 ret = -EINVAL;
2952 goto out_unlock;
2953 }
2954
30f7ea1c
FR		 2955 if (name) {
2956 dev = dev_get_by_name_rcu(sock_net(sk), name);
2957 if (!dev) {
2958 ret = -ENODEV;
2959 goto out_unlock;
2960 }
2961 } else if (ifindex) {
2962 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
2963 if (!dev) {
2964 ret = -ENODEV;
2965 goto out_unlock;
2966 }
2967 }
2968
2969 if (dev)
2970 dev_hold(dev);
66e56cd4 2971
902fefb8
DB		 2972 proto_curr = po->prot_hook.type;
2973 dev_curr = po->prot_hook.dev;
2974
2975 need_rehook = proto_curr != proto || dev_curr != dev;
2976
2977 if (need_rehook) {
30f7ea1c
FR		 2978 if (po->running) {
2979 rcu_read_unlock();
b90f87c6
ED		 2980 /* prevents packet_notifier() from calling
2981 * register_prot_hook()
2982 */
2983 po->num = 0;
30f7ea1c
FR		 2984 __unregister_prot_hook(sk, true);
2985 rcu_read_lock();
2986 dev_curr = po->prot_hook.dev;
2987 if (dev)
2988 unlisted = !dev_get_by_index_rcu(sock_net(sk),
2989 dev->ifindex);
2990 }
1da177e4 2991
b90f87c6 2992 BUG_ON(po->running);
902fefb8
DB		 2993 po->num = proto;
2994 po->prot_hook.type = proto;
902fefb8 2995
30f7ea1c
FR		 2996 if (unlikely(unlisted)) {
2997 dev_put(dev);
2998 po->prot_hook.dev = NULL;
2999 po->ifindex = -1;
3000 packet_cached_dev_reset(po);
3001 } else {
3002 po->prot_hook.dev = dev;
3003 po->ifindex = dev ? dev->ifindex : 0;
3004 packet_cached_dev_assign(po, dev);
3005 }
902fefb8 3006 }
158cd4af
LW		 3007 if (dev_curr)
3008 dev_put(dev_curr);
66e56cd4 3009
902fefb8 3010 if (proto == 0 || !need_rehook)
1da177e4
LT		 3011 goto out_unlock;
3012
30f7ea1c 3013 if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
ce06b03e 3014 register_prot_hook(sk);
be85d4ad
UT		 3015 } else {
3016 sk->sk_err = ENETDOWN;
3017 if (!sock_flag(sk, SOCK_DEAD))
3018 sk->sk_error_report(sk);
1da177e4
LT		 3019 }
3020
3021out_unlock:
30f7ea1c 3022 rcu_read_unlock();
1da177e4
LT		 3023 spin_unlock(&po->bind_lock);
3024 release_sock(sk);
30f7ea1c 3025 return ret;
1da177e4
LT		 3026}
3027
3028/*
3029 * Bind a packet socket to a device
3030 */
3031
40d4e3df
ED		 3032static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3033 int addr_len)
1da177e4 3034{
40d4e3df 3035 struct sock *sk = sock->sk;
f331d644 3036 char name[sizeof(uaddr->sa_data) + 1];
1ce4f28b 3037
1da177e4
LT		 3038 /*
3039 * Check legality
3040 */
1ce4f28b 3041
8ae55f04 3042 if (addr_len != sizeof(struct sockaddr))
1da177e4 3043 return -EINVAL;
f331d644
AP		 3044 /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3045 * zero-terminated.
3046 */
3047 memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
3048 name[sizeof(uaddr->sa_data)] = 0;
1da177e4 3049
30f7ea1c 3050 return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
1da177e4 3051}
1da177e4
LT		 3052
3053static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3054{
40d4e3df
ED		 3055 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3056 struct sock *sk = sock->sk;
1da177e4
LT		 3057
3058 /*
3059 * Check legality
3060 */
1ce4f28b 3061
1da177e4
LT		 3062 if (addr_len < sizeof(struct sockaddr_ll))
3063 return -EINVAL;
3064 if (sll->sll_family != AF_PACKET)
3065 return -EINVAL;
3066
30f7ea1c
FR		 3067 return packet_do_bind(sk, NULL, sll->sll_ifindex,
3068 sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 3069}
3070
3071static struct proto packet_proto = {
3072 .name = "PACKET",
3073 .owner = THIS_MODULE,
3074 .obj_size = sizeof(struct packet_sock),
3075};
3076
3077/*
1ce4f28b 3078 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 3079 */
3080
3f378b68
EP		 3081static int packet_create(struct net *net, struct socket *sock, int protocol,
3082 int kern)
1da177e4
LT		 3083{
3084 struct sock *sk;
3085 struct packet_sock *po;
0e11c91e 3086 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 3087 int err;
3088
df008c91 3089 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 3090 return -EPERM;
be02097c
DM		 3091 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3092 sock->type != SOCK_PACKET)
1da177e4
LT		 3093 return -ESOCKTNOSUPPORT;
3094
3095 sock->state = SS_UNCONNECTED;
3096
3097 err = -ENOBUFS;
11aa9c28 3098 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
1da177e4
LT		 3099 if (sk == NULL)
3100 goto out;
3101
3102 sock->ops = &packet_ops;
1da177e4
LT		 3103 if (sock->type == SOCK_PACKET)
3104 sock->ops = &packet_ops_spkt;
be02097c 3105
1da177e4
LT		 3106 sock_init_data(sock, sk);
3107
3108 po = pkt_sk(sk);
3109 sk->sk_family = PF_PACKET;
0e11c91e 3110 po->num = proto;
d346a3fa 3111 po->xmit = dev_queue_xmit;
66e56cd4 3112
b0138408
DB		 3113 err = packet_alloc_pending(po);
3114 if (err)
3115 goto out2;
3116
66e56cd4 3117 packet_cached_dev_reset(po);
1da177e4
LT		 3118
3119 sk->sk_destruct = packet_sock_destruct;
17ab56a2 3120 sk_refcnt_debug_inc(sk);
1da177e4
LT		 3121
3122 /*
3123 * Attach a protocol block
3124 */
3125
3126 spin_lock_init(&po->bind_lock);
905db440 3127 mutex_init(&po->pg_vec_lock);
0648ab70 3128 po->rollover = NULL;
1da177e4 3129 po->prot_hook.func = packet_rcv;
be02097c 3130
1da177e4
LT		 3131 if (sock->type == SOCK_PACKET)
3132 po->prot_hook.func = packet_rcv_spkt;
be02097c 3133
1da177e4
LT		 3134 po->prot_hook.af_packet_priv = sk;
3135
0e11c91e
AV		 3136 if (proto) {
3137 po->prot_hook.type = proto;
ce06b03e 3138 register_prot_hook(sk);
1da177e4
LT		 3139 }
3140
0fa7fa98 3141 mutex_lock(&net->packet.sklist_lock);
808f5114 3142 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 3143 mutex_unlock(&net->packet.sklist_lock);
3144
3145 preempt_disable();
3680453c 3146 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 3147 preempt_enable();
808f5114 3148
40d4e3df 3149 return 0;
b0138408
DB		 3150out2:
3151 sk_free(sk);
1da177e4
LT		 3152out:
3153 return err;
3154}
3155
3156/*
3157 * Pull a packet from our receive queue and hand it to the user.
3158 * If necessary we block.
3159 */
3160
1b784140
YX		 3161static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3162 int flags)
1da177e4
LT		 3163{
3164 struct sock *sk = sock->sk;
3165 struct sk_buff *skb;
3166 int copied, err;
bfd5f4a3 3167 int vnet_hdr_len = 0;
2472d761 3168 unsigned int origlen = 0;
1da177e4
LT		 3169
3170 err = -EINVAL;
ed85b565 3171 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 3172 goto out;
3173
3174#if 0
3175 /* What error should we return now? EUNATTACH? */
3176 if (pkt_sk(sk)->ifindex < 0)
3177 return -ENODEV;
3178#endif
3179
ed85b565 3180 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 3181 err = sock_recv_errqueue(sk, msg, len,
3182 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 3183 goto out;
3184 }
3185
1da177e4
LT		 3186 /*
3187 * Call the generic datagram receiver. This handles all sorts
3188 * of horrible races and re-entrancy so we can forget about it
3189 * in the protocol layers.
3190 *
3191 * Now it will return ENETDOWN, if device have just gone down,
3192 * but then it will block.
3193 */
3194
40d4e3df 3195 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 3196
3197 /*
1ce4f28b 3198 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 3199 * handles the blocking we don't see and worry about blocking
3200 * retries.
3201 */
3202
8ae55f04 3203 if (skb == NULL)
1da177e4
LT		 3204 goto out;
3205
2ccdbaa6
WB		 3206 if (pkt_sk(sk)->pressure)
3207 packet_rcv_has_room(pkt_sk(sk), NULL);
3208
bfd5f4a3
SS		 3209 if (pkt_sk(sk)->has_vnet_hdr) {
3210 struct virtio_net_hdr vnet_hdr = { 0 };
3211
3212 err = -EINVAL;
3213 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 3214 if (len < vnet_hdr_len)
bfd5f4a3
SS		 3215 goto out_free;
3216
1f18b717
MK		 3217 len -= vnet_hdr_len;
3218
bfd5f4a3
SS		 3219 if (skb_is_gso(skb)) {
3220 struct skb_shared_info *sinfo = skb_shinfo(skb);
3221
3222 /* This is a hint as to how much should be linear. */
dc9e5153 3223 vnet_hdr.hdr_len =
d3869efe 3224 __cpu_to_virtio16(vio_le(), skb_headlen(skb));
dc9e5153 3225 vnet_hdr.gso_size =
d3869efe 3226 __cpu_to_virtio16(vio_le(), sinfo->gso_size);
bfd5f4a3
SS		 3227 if (sinfo->gso_type & SKB_GSO_TCPV4)
3228 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
3229 else if (sinfo->gso_type & SKB_GSO_TCPV6)
3230 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
3231 else if (sinfo->gso_type & SKB_GSO_UDP)
3232 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
3233 else if (sinfo->gso_type & SKB_GSO_FCOE)
3234 goto out_free;
3235 else
3236 BUG();
3237 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
3238 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
3239 } else
3240 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
3241
3242 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3243 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
d3869efe 3244 vnet_hdr.csum_start = __cpu_to_virtio16(vio_le(),
dc9e5153 3245 skb_checksum_start_offset(skb));
d3869efe 3246 vnet_hdr.csum_offset = __cpu_to_virtio16(vio_le(),
dc9e5153 3247 skb->csum_offset);
10a8d94a
JW		 3248 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3249 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 3250 } /* else everything is zero */
3251
7eab8d9e 3252 err = memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_len);
bfd5f4a3
SS		 3253 if (err < 0)
3254 goto out_free;
3255 }
3256
f3d33426
HFS		 3257 /* You lose any data beyond the buffer you gave. If it worries
3258 * a user program they can ask the device for its MTU
3259 * anyway.
1da177e4 3260 */
1da177e4 3261 copied = skb->len;
40d4e3df
ED		 3262 if (copied > len) {
3263 copied = len;
3264 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 3265 }
3266
51f3d02b 3267 err = skb_copy_datagram_msg(skb, 0, msg, copied);
1da177e4
LT		 3268 if (err)
3269 goto out_free;
3270
2472d761
EB		 3271 if (sock->type != SOCK_PACKET) {
3272 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3273
3274 /* Original length was stored in sockaddr_ll fields */
3275 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3276 sll->sll_family = AF_PACKET;
3277 sll->sll_protocol = skb->protocol;
3278 }
3279
3b885787 3280 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 3281
f3d33426
HFS		 3282 if (msg->msg_name) {
3283 /* If the address length field is there to be filled
3284 * in, we fill it in now.
3285 */
3286 if (sock->type == SOCK_PACKET) {
342dfc30 3287 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 3288 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3289 } else {
3290 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2472d761 3291
f3d33426
HFS		 3292 msg->msg_namelen = sll->sll_halen +
3293 offsetof(struct sockaddr_ll, sll_addr);
3294 }
ffbc6111
HX		 3295 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3296 msg->msg_namelen);
f3d33426 3297 }
1da177e4 3298
8dc41944 3299 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 3300 struct tpacket_auxdata aux;
3301
3302 aux.tp_status = TP_STATUS_USER;
3303 if (skb->ip_summed == CHECKSUM_PARTIAL)
3304 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 3305 else if (skb->pkt_type != PACKET_OUTGOING &&
3306 (skb->ip_summed == CHECKSUM_COMPLETE ||
3307 skb_csum_unnecessary(skb)))
3308 aux.tp_status |= TP_STATUS_CSUM_VALID;
3309
2472d761 3310 aux.tp_len = origlen;
ffbc6111
HX		 3311 aux.tp_snaplen = skb->len;
3312 aux.tp_mac = 0;
bbe735e4 3313 aux.tp_net = skb_network_offset(skb);
df8a39de
JP		 3314 if (skb_vlan_tag_present(skb)) {
3315 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 3316 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3317 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 3318 } else {
3319 aux.tp_vlan_tci = 0;
a0cdfcf3 3320 aux.tp_vlan_tpid = 0;
a3bcc23e 3321 }
ffbc6111 3322 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3323 }
3324
1da177e4
LT		 3325 /*
3326 * Free or return the buffer as appropriate. Again this
3327 * hides all the races and re-entrancy issues from us.
3328 */
bfd5f4a3 3329 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3330
3331out_free:
3332 skb_free_datagram(sk, skb);
3333out:
3334 return err;
3335}
3336
1da177e4
LT		 3337static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3338 int *uaddr_len, int peer)
3339{
3340 struct net_device *dev;
3341 struct sock *sk = sock->sk;
3342
3343 if (peer)
3344 return -EOPNOTSUPP;
3345
3346 uaddr->sa_family = AF_PACKET;
2dc85bf3 3347 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3348 rcu_read_lock();
3349 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3350 if (dev)
2dc85bf3 3351 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3352 rcu_read_unlock();
1da177e4
LT		 3353 *uaddr_len = sizeof(*uaddr);
3354
3355 return 0;
3356}
1da177e4
LT		 3357
3358static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3359 int *uaddr_len, int peer)
3360{
3361 struct net_device *dev;
3362 struct sock *sk = sock->sk;
3363 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3364 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3365
3366 if (peer)
3367 return -EOPNOTSUPP;
3368
3369 sll->sll_family = AF_PACKET;
3370 sll->sll_ifindex = po->ifindex;
3371 sll->sll_protocol = po->num;
67286640 3372 sll->sll_pkttype = 0;
654d1f8a
ED		 3373 rcu_read_lock();
3374 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3375 if (dev) {
3376 sll->sll_hatype = dev->type;
3377 sll->sll_halen = dev->addr_len;
3378 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3379 } else {
3380 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3381 sll->sll_halen = 0;
3382 }
654d1f8a 3383 rcu_read_unlock();
0fb375fb 3384 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3385
3386 return 0;
3387}
3388
2aeb0b88
WC		 3389static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3390 int what)
1da177e4
LT		 3391{
3392 switch (i->type) {
3393 case PACKET_MR_MULTICAST:
1162563f
JP		 3394 if (i->alen != dev->addr_len)
3395 return -EINVAL;
1da177e4 3396 if (what > 0)
22bedad3 3397 return dev_mc_add(dev, i->addr);
1da177e4 3398 else
22bedad3 3399 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3400 break;
3401 case PACKET_MR_PROMISC:
2aeb0b88 3402 return dev_set_promiscuity(dev, what);
1da177e4 3403 case PACKET_MR_ALLMULTI:
2aeb0b88 3404 return dev_set_allmulti(dev, what);
d95ed927 3405 case PACKET_MR_UNICAST:
1162563f
JP		 3406 if (i->alen != dev->addr_len)
3407 return -EINVAL;
d95ed927 3408 if (what > 0)
a748ee24 3409 return dev_uc_add(dev, i->addr);
d95ed927 3410 else
a748ee24 3411 return dev_uc_del(dev, i->addr);
d95ed927 3412 break;
40d4e3df
ED		 3413 default:
3414 break;
1da177e4 3415 }
2aeb0b88 3416 return 0;
1da177e4
LT		 3417}
3418
82f17091
FR		 3419static void packet_dev_mclist_delete(struct net_device *dev,
3420 struct packet_mclist **mlp)
1da177e4 3421{
82f17091
FR		 3422 struct packet_mclist *ml;
3423
3424 while ((ml = *mlp) != NULL) {
3425 if (ml->ifindex == dev->ifindex) {
3426 packet_dev_mc(dev, ml, -1);
3427 *mlp = ml->next;
3428 kfree(ml);
3429 } else
3430 mlp = &ml->next;
1da177e4
LT		 3431 }
3432}
3433
0fb375fb 3434static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3435{
3436 struct packet_sock *po = pkt_sk(sk);
3437 struct packet_mclist *ml, *i;
3438 struct net_device *dev;
3439 int err;
3440
3441 rtnl_lock();
3442
3443 err = -ENODEV;
3b1e0a65 3444 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3445 if (!dev)
3446 goto done;
3447
3448 err = -EINVAL;
1162563f 3449 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3450 goto done;
3451
3452 err = -ENOBUFS;
8b3a7005 3453 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3454 if (i == NULL)
3455 goto done;
3456
3457 err = 0;
3458 for (ml = po->mclist; ml; ml = ml->next) {
3459 if (ml->ifindex == mreq->mr_ifindex &&
3460 ml->type == mreq->mr_type &&
3461 ml->alen == mreq->mr_alen &&
3462 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3463 ml->count++;
3464 /* Free the new element ... */
3465 kfree(i);
3466 goto done;
3467 }
3468 }
3469
3470 i->type = mreq->mr_type;
3471 i->ifindex = mreq->mr_ifindex;
3472 i->alen = mreq->mr_alen;
3473 memcpy(i->addr, mreq->mr_address, i->alen);
a416c948 3474 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
1da177e4
LT		 3475 i->count = 1;
3476 i->next = po->mclist;
3477 po->mclist = i;
2aeb0b88
WC		 3478 err = packet_dev_mc(dev, i, 1);
3479 if (err) {
3480 po->mclist = i->next;
3481 kfree(i);
3482 }
1da177e4
LT		 3483
3484done:
3485 rtnl_unlock();
3486 return err;
3487}
3488
0fb375fb 3489static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3490{
3491 struct packet_mclist *ml, **mlp;
3492
3493 rtnl_lock();
3494
3495 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3496 if (ml->ifindex == mreq->mr_ifindex &&
3497 ml->type == mreq->mr_type &&
3498 ml->alen == mreq->mr_alen &&
3499 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3500 if (--ml->count == 0) {
3501 struct net_device *dev;
3502 *mlp = ml->next;
ad959e76
ED		 3503 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3504 if (dev)
1da177e4 3505 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3506 kfree(ml);
3507 }
82f17091 3508 break;
1da177e4
LT		 3509 }
3510 }
3511 rtnl_unlock();
82f17091 3512 return 0;
1da177e4
LT		 3513}
3514
3515static void packet_flush_mclist(struct sock *sk)
3516{
3517 struct packet_sock *po = pkt_sk(sk);
3518 struct packet_mclist *ml;
3519
3520 if (!po->mclist)
3521 return;
3522
3523 rtnl_lock();
3524 while ((ml = po->mclist) != NULL) {
3525 struct net_device *dev;
3526
3527 po->mclist = ml->next;
ad959e76
ED		 3528 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3529 if (dev != NULL)
1da177e4 3530 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3531 kfree(ml);
3532 }
3533 rtnl_unlock();
3534}
1da177e4
LT		 3535
3536static int
b7058842 3537packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3538{
3539 struct sock *sk = sock->sk;
8dc41944 3540 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3541 int ret;
3542
3543 if (level != SOL_PACKET)
3544 return -ENOPROTOOPT;
3545
69e3c75f 3546 switch (optname) {
1ce4f28b 3547 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3548 case PACKET_DROP_MEMBERSHIP:
3549 {
0fb375fb
EB		 3550 struct packet_mreq_max mreq;
3551 int len = optlen;
3552 memset(&mreq, 0, sizeof(mreq));
3553 if (len < sizeof(struct packet_mreq))
1da177e4 3554 return -EINVAL;
0fb375fb
EB		 3555 if (len > sizeof(mreq))
3556 len = sizeof(mreq);
40d4e3df 3557 if (copy_from_user(&mreq, optval, len))
1da177e4 3558 return -EFAULT;
0fb375fb
EB		 3559 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3560 return -EINVAL;
1da177e4
LT		 3561 if (optname == PACKET_ADD_MEMBERSHIP)
3562 ret = packet_mc_add(sk, &mreq);
3563 else
3564 ret = packet_mc_drop(sk, &mreq);
3565 return ret;
3566 }
a2efcfa0 3567
1da177e4 3568 case PACKET_RX_RING:
69e3c75f 3569 case PACKET_TX_RING:
1da177e4 3570 {
f6fb8f10 3571 union tpacket_req_u req_u;
3572 int len;
1da177e4 3573
f6fb8f10 3574 switch (po->tp_version) {
3575 case TPACKET_V1:
3576 case TPACKET_V2:
3577 len = sizeof(req_u.req);
3578 break;
3579 case TPACKET_V3:
3580 default:
3581 len = sizeof(req_u.req3);
3582 break;
3583 }
3584 if (optlen < len)
1da177e4 3585 return -EINVAL;
bfd5f4a3
SS		 3586 if (pkt_sk(sk)->has_vnet_hdr)
3587 return -EINVAL;
f6fb8f10 3588 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3589 return -EFAULT;
f6fb8f10 3590 return packet_set_ring(sk, &req_u, 0,
3591 optname == PACKET_TX_RING);
1da177e4
LT		 3592 }
3593 case PACKET_COPY_THRESH:
3594 {
3595 int val;
3596
40d4e3df 3597 if (optlen != sizeof(val))
1da177e4 3598 return -EINVAL;
40d4e3df 3599 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3600 return -EFAULT;
3601
3602 pkt_sk(sk)->copy_thresh = val;
3603 return 0;
3604 }
bbd6ef87
PM		 3605 case PACKET_VERSION:
3606 {
3607 int val;
3608
3609 if (optlen != sizeof(val))
3610 return -EINVAL;
bbd6ef87
PM		 3611 if (copy_from_user(&val, optval, sizeof(val)))
3612 return -EFAULT;
3613 switch (val) {
3614 case TPACKET_V1:
3615 case TPACKET_V2:
f6fb8f10 3616 case TPACKET_V3:
5a01eaf1 3617 break;
bbd6ef87
PM		 3618 default:
3619 return -EINVAL;
3620 }
5a01eaf1
PP		 3621 lock_sock(sk);
3622 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3623 ret = -EBUSY;
3624 } else {
3625 po->tp_version = val;
3626 ret = 0;
3627 }
3628 release_sock(sk);
3629 return ret;
bbd6ef87 3630 }
8913336a
PM		 3631 case PACKET_RESERVE:
3632 {
3633 unsigned int val;
3634
3635 if (optlen != sizeof(val))
3636 return -EINVAL;
8913336a
PM		 3637 if (copy_from_user(&val, optval, sizeof(val)))
3638 return -EFAULT;
25adf4e3
AK		 3639 if (val > INT_MAX)
3640 return -EINVAL;
63364a50
WB		 3641 lock_sock(sk);
3642 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3643 ret = -EBUSY;
3644 } else {
3645 po->tp_reserve = val;
3646 ret = 0;
3647 }
3648 release_sock(sk);
3649 return ret;
8913336a 3650 }
69e3c75f
JB		 3651 case PACKET_LOSS:
3652 {
3653 unsigned int val;
3654
3655 if (optlen != sizeof(val))
3656 return -EINVAL;
3657 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3658 return -EBUSY;
3659 if (copy_from_user(&val, optval, sizeof(val)))
3660 return -EFAULT;
3661 po->tp_loss = !!val;
3662 return 0;
3663 }
8dc41944
HX		 3664 case PACKET_AUXDATA:
3665 {
3666 int val;
3667
3668 if (optlen < sizeof(val))
3669 return -EINVAL;
3670 if (copy_from_user(&val, optval, sizeof(val)))
3671 return -EFAULT;
3672
3673 po->auxdata = !!val;
3674 return 0;
3675 }
80feaacb
PWJ		 3676 case PACKET_ORIGDEV:
3677 {
3678 int val;
3679
3680 if (optlen < sizeof(val))
3681 return -EINVAL;
3682 if (copy_from_user(&val, optval, sizeof(val)))
3683 return -EFAULT;
3684
3685 po->origdev = !!val;
3686 return 0;
3687 }
bfd5f4a3
SS		 3688 case PACKET_VNET_HDR:
3689 {
3690 int val;
3691
3692 if (sock->type != SOCK_RAW)
3693 return -EINVAL;
3694 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3695 return -EBUSY;
3696 if (optlen < sizeof(val))
3697 return -EINVAL;
3698 if (copy_from_user(&val, optval, sizeof(val)))
3699 return -EFAULT;
3700
3701 po->has_vnet_hdr = !!val;
3702 return 0;
3703 }
614f60fa
SM		 3704 case PACKET_TIMESTAMP:
3705 {
3706 int val;
3707
3708 if (optlen != sizeof(val))
3709 return -EINVAL;
3710 if (copy_from_user(&val, optval, sizeof(val)))
3711 return -EFAULT;
3712
3713 po->tp_tstamp = val;
3714 return 0;
3715 }
dc99f600
DM		 3716 case PACKET_FANOUT:
3717 {
3718 int val;
3719
3720 if (optlen != sizeof(val))
3721 return -EINVAL;
3722 if (copy_from_user(&val, optval, sizeof(val)))
3723 return -EFAULT;
3724
3725 return fanout_add(sk, val & 0xffff, val >> 16);
3726 }
47dceb8e
WB		 3727 case PACKET_FANOUT_DATA:
3728 {
3729 if (!po->fanout)
3730 return -EINVAL;
3731
3732 return fanout_set_data(po, optval, optlen);
3733 }
5920cd3a
PC		 3734 case PACKET_TX_HAS_OFF:
3735 {
3736 unsigned int val;
3737
3738 if (optlen != sizeof(val))
3739 return -EINVAL;
3740 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3741 return -EBUSY;
3742 if (copy_from_user(&val, optval, sizeof(val)))
3743 return -EFAULT;
3744 po->tp_tx_has_off = !!val;
3745 return 0;
3746 }
d346a3fa
DB		 3747 case PACKET_QDISC_BYPASS:
3748 {
3749 int val;
3750
3751 if (optlen != sizeof(val))
3752 return -EINVAL;
3753 if (copy_from_user(&val, optval, sizeof(val)))
3754 return -EFAULT;
3755
3756 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3757 return 0;
3758 }
1da177e4
LT		 3759 default:
3760 return -ENOPROTOOPT;
3761 }
3762}
3763
3764static int packet_getsockopt(struct socket *sock, int level, int optname,
3765 char __user *optval, int __user *optlen)
3766{
3767 int len;
c06fff6e 3768 int val, lv = sizeof(val);
1da177e4
LT		 3769 struct sock *sk = sock->sk;
3770 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3771 void *data = &val;
ee80fbf3 3772 union tpacket_stats_u st;
a9b63918 3773 struct tpacket_rollover_stats rstats;
1da177e4
LT		 3774
3775 if (level != SOL_PACKET)
3776 return -ENOPROTOOPT;
3777
8ae55f04
KK		 3778 if (get_user(len, optlen))
3779 return -EFAULT;
1da177e4
LT		 3780
3781 if (len < 0)
3782 return -EINVAL;
1ce4f28b 3783
69e3c75f 3784 switch (optname) {
1da177e4 3785 case PACKET_STATISTICS:
1da177e4 3786 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3787 memcpy(&st, &po->stats, sizeof(st));
3788 memset(&po->stats, 0, sizeof(po->stats));
3789 spin_unlock_bh(&sk->sk_receive_queue.lock);
3790
f6fb8f10 3791 if (po->tp_version == TPACKET_V3) {
c06fff6e 3792 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3793 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3794 data = &st.stats3;
f6fb8f10 3795 } else {
c06fff6e 3796 lv = sizeof(struct tpacket_stats);
8bcdeaff 3797 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3798 data = &st.stats1;
f6fb8f10 3799 }
ee80fbf3 3800
8dc41944
HX		 3801 break;
3802 case PACKET_AUXDATA:
8dc41944 3803 val = po->auxdata;
80feaacb
PWJ		 3804 break;
3805 case PACKET_ORIGDEV:
80feaacb 3806 val = po->origdev;
bfd5f4a3
SS		 3807 break;
3808 case PACKET_VNET_HDR:
bfd5f4a3 3809 val = po->has_vnet_hdr;
1da177e4 3810 break;
bbd6ef87 3811 case PACKET_VERSION:
bbd6ef87 3812 val = po->tp_version;
bbd6ef87
PM		 3813 break;
3814 case PACKET_HDRLEN:
3815 if (len > sizeof(int))
3816 len = sizeof(int);
fa63895f
AP		 3817 if (len < sizeof(int))
3818 return -EINVAL;
bbd6ef87
PM		 3819 if (copy_from_user(&val, optval, len))
3820 return -EFAULT;
3821 switch (val) {
3822 case TPACKET_V1:
3823 val = sizeof(struct tpacket_hdr);
3824 break;
3825 case TPACKET_V2:
3826 val = sizeof(struct tpacket2_hdr);
3827 break;
f6fb8f10 3828 case TPACKET_V3:
3829 val = sizeof(struct tpacket3_hdr);
3830 break;
bbd6ef87
PM		 3831 default:
3832 return -EINVAL;
3833 }
bbd6ef87 3834 break;
8913336a 3835 case PACKET_RESERVE:
8913336a 3836 val = po->tp_reserve;
8913336a 3837 break;
69e3c75f 3838 case PACKET_LOSS:
69e3c75f 3839 val = po->tp_loss;
69e3c75f 3840 break;
614f60fa 3841 case PACKET_TIMESTAMP:
614f60fa 3842 val = po->tp_tstamp;
614f60fa 3843 break;
dc99f600 3844 case PACKET_FANOUT:
dc99f600
DM		 3845 val = (po->fanout ?
3846 ((u32)po->fanout->id |
77f65ebd
WB		 3847 ((u32)po->fanout->type << 16) |
3848 ((u32)po->fanout->flags << 24)) :
dc99f600 3849 0);
dc99f600 3850 break;
a9b63918 3851 case PACKET_ROLLOVER_STATS:
f50e9c87 3852 if (!po->rollover)
a9b63918 3853 return -EINVAL;
f50e9c87
MM		 3854 rstats.tp_all = atomic_long_read(&po->rollover->num);
3855 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3856 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3857 data = &rstats;
3858 lv = sizeof(rstats);
a9b63918 3859 break;
5920cd3a
PC		 3860 case PACKET_TX_HAS_OFF:
3861 val = po->tp_tx_has_off;
3862 break;
d346a3fa
DB		 3863 case PACKET_QDISC_BYPASS:
3864 val = packet_use_direct_xmit(po);
3865 break;
1da177e4
LT		 3866 default:
3867 return -ENOPROTOOPT;
3868 }
3869
c06fff6e
ED		 3870 if (len > lv)
3871 len = lv;
8ae55f04
KK		 3872 if (put_user(len, optlen))
3873 return -EFAULT;
8dc41944
HX		 3874 if (copy_to_user(optval, data, len))
3875 return -EFAULT;
8ae55f04 3876 return 0;
1da177e4
LT		 3877}
3878
3879
351638e7
JP		 3880static int packet_notifier(struct notifier_block *this,
3881 unsigned long msg, void *ptr)
1da177e4
LT		 3882{
3883 struct sock *sk;
351638e7 3884 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3885 struct net *net = dev_net(dev);
1da177e4 3886
808f5114 3887 rcu_read_lock();
b67bfe0d 3888 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3889 struct packet_sock *po = pkt_sk(sk);
3890
3891 switch (msg) {
3892 case NETDEV_UNREGISTER:
1da177e4 3893 if (po->mclist)
82f17091 3894 packet_dev_mclist_delete(dev, &po->mclist);
a2efcfa0
DM		 3895 /* fallthrough */
3896
1da177e4
LT		 3897 case NETDEV_DOWN:
3898 if (dev->ifindex == po->ifindex) {
3899 spin_lock(&po->bind_lock);
3900 if (po->running) {
ce06b03e 3901 __unregister_prot_hook(sk, false);
1da177e4
LT		 3902 sk->sk_err = ENETDOWN;
3903 if (!sock_flag(sk, SOCK_DEAD))
3904 sk->sk_error_report(sk);
3905 }
3906 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3907 packet_cached_dev_reset(po);
1da177e4 3908 po->ifindex = -1;
160ff18a
BG		 3909 if (po->prot_hook.dev)
3910 dev_put(po->prot_hook.dev);
1da177e4
LT		 3911 po->prot_hook.dev = NULL;
3912 }
3913 spin_unlock(&po->bind_lock);
3914 }
3915 break;
3916 case NETDEV_UP:
808f5114 3917 if (dev->ifindex == po->ifindex) {
3918 spin_lock(&po->bind_lock);
ce06b03e
DM		 3919 if (po->num)
3920 register_prot_hook(sk);
808f5114 3921 spin_unlock(&po->bind_lock);
1da177e4 3922 }
1da177e4
LT		 3923 break;
3924 }
3925 }
808f5114 3926 rcu_read_unlock();
1da177e4
LT		 3927 return NOTIFY_DONE;
3928}
3929
3930
3931static int packet_ioctl(struct socket *sock, unsigned int cmd,
3932 unsigned long arg)
3933{
3934 struct sock *sk = sock->sk;
3935
69e3c75f 3936 switch (cmd) {
40d4e3df
ED		 3937 case SIOCOUTQ:
3938 {
3939 int amount = sk_wmem_alloc_get(sk);
31e6d363 3940
40d4e3df
ED		 3941 return put_user(amount, (int __user *)arg);
3942 }
3943 case SIOCINQ:
3944 {
3945 struct sk_buff *skb;
3946 int amount = 0;
3947
3948 spin_lock_bh(&sk->sk_receive_queue.lock);
3949 skb = skb_peek(&sk->sk_receive_queue);
3950 if (skb)
3951 amount = skb->len;
3952 spin_unlock_bh(&sk->sk_receive_queue.lock);
3953 return put_user(amount, (int __user *)arg);
3954 }
3955 case SIOCGSTAMP:
3956 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3957 case SIOCGSTAMPNS:
3958 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3959
1da177e4 3960#ifdef CONFIG_INET
40d4e3df
ED		 3961 case SIOCADDRT:
3962 case SIOCDELRT:
3963 case SIOCDARP:
3964 case SIOCGARP:
3965 case SIOCSARP:
3966 case SIOCGIFADDR:
3967 case SIOCSIFADDR:
3968 case SIOCGIFBRDADDR:
3969 case SIOCSIFBRDADDR:
3970 case SIOCGIFNETMASK:
3971 case SIOCSIFNETMASK:
3972 case SIOCGIFDSTADDR:
3973 case SIOCSIFDSTADDR:
3974 case SIOCSIFFLAGS:
40d4e3df 3975 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3976#endif
3977
40d4e3df
ED		 3978 default:
3979 return -ENOIOCTLCMD;
1da177e4
LT		 3980 }
3981 return 0;
3982}
3983
40d4e3df 3984static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3985 poll_table *wait)
3986{
3987 struct sock *sk = sock->sk;
3988 struct packet_sock *po = pkt_sk(sk);
3989 unsigned int mask = datagram_poll(file, sock, wait);
3990
3991 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3992 if (po->rx_ring.pg_vec) {
f6fb8f10 3993 if (!packet_previous_rx_frame(po, &po->rx_ring,
3994 TP_STATUS_KERNEL))
1da177e4
LT		 3995 mask |= POLLIN | POLLRDNORM;
3996 }
2ccdbaa6 3997 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
54d7c01d 3998 po->pressure = 0;
1da177e4 3999 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 4000 spin_lock_bh(&sk->sk_write_queue.lock);
4001 if (po->tx_ring.pg_vec) {
4002 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
4003 mask |= POLLOUT | POLLWRNORM;
4004 }
4005 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 4006 return mask;
4007}
4008
4009
4010/* Dirty? Well, I still did not learn better way to account
4011 * for user mmaps.
4012 */
4013
4014static void packet_mm_open(struct vm_area_struct *vma)
4015{
4016 struct file *file = vma->vm_file;
40d4e3df 4017 struct socket *sock = file->private_data;
1da177e4 4018 struct sock *sk = sock->sk;
1ce4f28b 4019
1da177e4
LT		 4020 if (sk)
4021 atomic_inc(&pkt_sk(sk)->mapped);
4022}
4023
4024static void packet_mm_close(struct vm_area_struct *vma)
4025{
4026 struct file *file = vma->vm_file;
40d4e3df 4027 struct socket *sock = file->private_data;
1da177e4 4028 struct sock *sk = sock->sk;
1ce4f28b 4029
1da177e4
LT		 4030 if (sk)
4031 atomic_dec(&pkt_sk(sk)->mapped);
4032}
4033
f0f37e2f 4034static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 4035 .open = packet_mm_open,
4036 .close = packet_mm_close,
1da177e4
LT		 4037};
4038
0e3125c7
NH		 4039static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4040 unsigned int len)
1da177e4
LT		 4041{
4042 int i;
4043
4ebf0ae2 4044 for (i = 0; i < len; i++) {
0e3125c7 4045 if (likely(pg_vec[i].buffer)) {
c56b4d90 4046 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 4047 vfree(pg_vec[i].buffer);
4048 else
4049 free_pages((unsigned long)pg_vec[i].buffer,
4050 order);
4051 pg_vec[i].buffer = NULL;
4052 }
1da177e4
LT		 4053 }
4054 kfree(pg_vec);
4055}
4056
eea49cc9 4057static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 4058{
f0d4eb29 4059 char *buffer;
0e3125c7
NH		 4060 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4061 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
4062
4063 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 4064 if (buffer)
4065 return buffer;
4066
f0d4eb29 4067 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 4068 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 4069 if (buffer)
4070 return buffer;
4071
f0d4eb29 4072 /* vmalloc failed, lets dig into swap here */
0e3125c7 4073 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 4074 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 4075 if (buffer)
4076 return buffer;
4077
f0d4eb29 4078 /* complete and utter failure */
0e3125c7 4079 return NULL;
4ebf0ae2
DM		 4080}
4081
0e3125c7 4082static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 4083{
4084 unsigned int block_nr = req->tp_block_nr;
0e3125c7 4085 struct pgv *pg_vec;
4ebf0ae2
DM		 4086 int i;
4087
0e3125c7 4088 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 4089 if (unlikely(!pg_vec))
4090 goto out;
4091
4092 for (i = 0; i < block_nr; i++) {
c56b4d90 4093 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 4094 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 4095 goto out_free_pgvec;
4096 }
4097
4098out:
4099 return pg_vec;
4100
4101out_free_pgvec:
4102 free_pg_vec(pg_vec, order, block_nr);
4103 pg_vec = NULL;
4104 goto out;
4105}
1da177e4 4106
f6fb8f10 4107static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 4108 int closing, int tx_ring)
1da177e4 4109{
0e3125c7 4110 struct pgv *pg_vec = NULL;
1da177e4 4111 struct packet_sock *po = pkt_sk(sk);
0e11c91e 4112 int was_running, order = 0;
69e3c75f
JB		 4113 struct packet_ring_buffer *rb;
4114 struct sk_buff_head *rb_queue;
0e11c91e 4115 __be16 num;
f6fb8f10 4116 int err = -EINVAL;
4117 /* Added to avoid minimal code churn */
4118 struct tpacket_req *req = &req_u->req;
4119
5a01eaf1 4120 lock_sock(sk);
f6fb8f10 4121 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
4122 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
4123 WARN(1, "Tx-ring is not supported.\n");
4124 goto out;
4125 }
1ce4f28b 4126
69e3c75f
JB		 4127 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4128 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 4129
69e3c75f
JB		 4130 err = -EBUSY;
4131 if (!closing) {
4132 if (atomic_read(&po->mapped))
4133 goto out;
b0138408 4134 if (packet_read_pending(rb))
69e3c75f
JB		 4135 goto out;
4136 }
1da177e4 4137
69e3c75f
JB		 4138 if (req->tp_block_nr) {
4139 /* Sanity tests and some calculations */
4140 err = -EBUSY;
4141 if (unlikely(rb->pg_vec))
4142 goto out;
1da177e4 4143
bbd6ef87
PM		 4144 switch (po->tp_version) {
4145 case TPACKET_V1:
4146 po->tp_hdrlen = TPACKET_HDRLEN;
4147 break;
4148 case TPACKET_V2:
4149 po->tp_hdrlen = TPACKET2_HDRLEN;
4150 break;
f6fb8f10 4151 case TPACKET_V3:
4152 po->tp_hdrlen = TPACKET3_HDRLEN;
4153 break;
bbd6ef87
PM		 4154 }
4155
69e3c75f 4156 err = -EINVAL;
4ebf0ae2 4157 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 4158 goto out;
90836b67 4159 if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
69e3c75f 4160 goto out;
dc808110 4161 if (po->tp_version >= TPACKET_V3 &&
d35f8fa0
AK		 4162 req->tp_block_size <=
4163 BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv))
dc808110 4164 goto out;
8913336a 4165 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 4166 po->tp_reserve))
4167 goto out;
4ebf0ae2 4168 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 4169 goto out;
1da177e4 4170
4194b491
TK		 4171 rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4172 if (unlikely(rb->frames_per_block == 0))
69e3c75f 4173 goto out;
cf71bd41
AK		 4174 if (unlikely(req->tp_block_size > UINT_MAX / req->tp_block_nr))
4175 goto out;
69e3c75f
JB		 4176 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4177 req->tp_frame_nr))
4178 goto out;
1da177e4
LT		 4179
4180 err = -ENOMEM;
4ebf0ae2
DM		 4181 order = get_order(req->tp_block_size);
4182 pg_vec = alloc_pg_vec(req, order);
4183 if (unlikely(!pg_vec))
1da177e4 4184 goto out;
f6fb8f10 4185 switch (po->tp_version) {
4186 case TPACKET_V3:
4187 /* Transmit path is not supported. We checked
4188 * it above but just being paranoid
4189 */
4190 if (!tx_ring)
e8e85cc5 4191 init_prb_bdqc(po, rb, pg_vec, req_u);
d7cf0c34 4192 break;
f6fb8f10 4193 default:
4194 break;
4195 }
69e3c75f
JB		 4196 }
4197 /* Done */
4198 else {
4199 err = -EINVAL;
4ebf0ae2 4200 if (unlikely(req->tp_frame_nr))
69e3c75f 4201 goto out;
1da177e4
LT		 4202 }
4203
1da177e4
LT		 4204
4205 /* Detach socket from network */
4206 spin_lock(&po->bind_lock);
4207 was_running = po->running;
4208 num = po->num;
4209 if (was_running) {
1da177e4 4210 po->num = 0;
ce06b03e 4211 __unregister_prot_hook(sk, false);
1da177e4
LT		 4212 }
4213 spin_unlock(&po->bind_lock);
1ce4f28b 4214
1da177e4
LT		 4215 synchronize_net();
4216
4217 err = -EBUSY;
905db440 4218 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 4219 if (closing || atomic_read(&po->mapped) == 0) {
4220 err = 0;
69e3c75f 4221 spin_lock_bh(&rb_queue->lock);
c053fd96 4222 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 4223 rb->frame_max = (req->tp_frame_nr - 1);
4224 rb->head = 0;
4225 rb->frame_size = req->tp_frame_size;
4226 spin_unlock_bh(&rb_queue->lock);
4227
c053fd96
CG		 4228 swap(rb->pg_vec_order, order);
4229 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 4230
4231 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4232 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4233 tpacket_rcv : packet_rcv;
4234 skb_queue_purge(rb_queue);
1da177e4 4235 if (atomic_read(&po->mapped))
40d4e3df
ED		 4236 pr_err("packet_mmap: vma is busy: %d\n",
4237 atomic_read(&po->mapped));
1da177e4 4238 }
905db440 4239 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4240
4241 spin_lock(&po->bind_lock);
ce06b03e 4242 if (was_running) {
1da177e4 4243 po->num = num;
ce06b03e 4244 register_prot_hook(sk);
1da177e4
LT		 4245 }
4246 spin_unlock(&po->bind_lock);
49933896 4247 if (pg_vec && (po->tp_version > TPACKET_V2)) {
f6fb8f10 4248 /* Because we don't support block-based V3 on tx-ring */
4249 if (!tx_ring)
73d0fcf2 4250 prb_shutdown_retire_blk_timer(po, rb_queue);
f6fb8f10 4251 }
1da177e4 4252
1da177e4
LT		 4253 if (pg_vec)
4254 free_pg_vec(pg_vec, order, req->tp_block_nr);
4255out:
5a01eaf1 4256 release_sock(sk);
1da177e4
LT		 4257 return err;
4258}
4259
69e3c75f
JB		 4260static int packet_mmap(struct file *file, struct socket *sock,
4261 struct vm_area_struct *vma)
1da177e4
LT		 4262{
4263 struct sock *sk = sock->sk;
4264 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 4265 unsigned long size, expected_size;
4266 struct packet_ring_buffer *rb;
1da177e4
LT		 4267 unsigned long start;
4268 int err = -EINVAL;
4269 int i;
4270
4271 if (vma->vm_pgoff)
4272 return -EINVAL;
4273
905db440 4274 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 4275
4276 expected_size = 0;
4277 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4278 if (rb->pg_vec) {
4279 expected_size += rb->pg_vec_len
4280 * rb->pg_vec_pages
4281 * PAGE_SIZE;
4282 }
4283 }
4284
4285 if (expected_size == 0)
1da177e4 4286 goto out;
69e3c75f
JB		 4287
4288 size = vma->vm_end - vma->vm_start;
4289 if (size != expected_size)
1da177e4
LT		 4290 goto out;
4291
1da177e4 4292 start = vma->vm_start;
69e3c75f
JB		 4293 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4294 if (rb->pg_vec == NULL)
4295 continue;
4296
4297 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 4298 struct page *page;
4299 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 4300 int pg_num;
4301
c56b4d90
CG		 4302 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4303 page = pgv_to_page(kaddr);
69e3c75f
JB		 4304 err = vm_insert_page(vma, start, page);
4305 if (unlikely(err))
4306 goto out;
4307 start += PAGE_SIZE;
0e3125c7 4308 kaddr += PAGE_SIZE;
69e3c75f 4309 }
4ebf0ae2 4310 }
1da177e4 4311 }
69e3c75f 4312
4ebf0ae2 4313 atomic_inc(&po->mapped);
1da177e4
LT		 4314 vma->vm_ops = &packet_mmap_ops;
4315 err = 0;
4316
4317out:
905db440 4318 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4319 return err;
4320}
1da177e4 4321
90ddc4f0 4322static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 4323 .family = PF_PACKET,
4324 .owner = THIS_MODULE,
4325 .release = packet_release,
4326 .bind = packet_bind_spkt,
4327 .connect = sock_no_connect,
4328 .socketpair = sock_no_socketpair,
4329 .accept = sock_no_accept,
4330 .getname = packet_getname_spkt,
4331 .poll = datagram_poll,
4332 .ioctl = packet_ioctl,
4333 .listen = sock_no_listen,
4334 .shutdown = sock_no_shutdown,
4335 .setsockopt = sock_no_setsockopt,
4336 .getsockopt = sock_no_getsockopt,
4337 .sendmsg = packet_sendmsg_spkt,
4338 .recvmsg = packet_recvmsg,
4339 .mmap = sock_no_mmap,
4340 .sendpage = sock_no_sendpage,
4341};
1da177e4 4342
90ddc4f0 4343static const struct proto_ops packet_ops = {
1da177e4
LT		 4344 .family = PF_PACKET,
4345 .owner = THIS_MODULE,
4346 .release = packet_release,
4347 .bind = packet_bind,
4348 .connect = sock_no_connect,
4349 .socketpair = sock_no_socketpair,
4350 .accept = sock_no_accept,
1ce4f28b 4351 .getname = packet_getname,
1da177e4
LT		 4352 .poll = packet_poll,
4353 .ioctl = packet_ioctl,
4354 .listen = sock_no_listen,
4355 .shutdown = sock_no_shutdown,
4356 .setsockopt = packet_setsockopt,
4357 .getsockopt = packet_getsockopt,
4358 .sendmsg = packet_sendmsg,
4359 .recvmsg = packet_recvmsg,
4360 .mmap = packet_mmap,
4361 .sendpage = sock_no_sendpage,
4362};
4363
ec1b4cf7 4364static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4365 .family = PF_PACKET,
4366 .create = packet_create,
4367 .owner = THIS_MODULE,
4368};
4369
4370static struct notifier_block packet_netdev_notifier = {
40d4e3df 4371 .notifier_call = packet_notifier,
1da177e4
LT		 4372};
4373
4374#ifdef CONFIG_PROC_FS
1da177e4
LT		 4375
4376static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4377 __acquires(RCU)
1da177e4 4378{
e372c414 4379 struct net *net = seq_file_net(seq);
808f5114 4380
4381 rcu_read_lock();
4382 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4383}
4384
4385static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4386{
1bf40954 4387 struct net *net = seq_file_net(seq);
808f5114 4388 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4389}
4390
4391static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4392 __releases(RCU)
1da177e4 4393{
808f5114 4394 rcu_read_unlock();
1da177e4
LT		 4395}
4396
1ce4f28b 4397static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4398{
4399 if (v == SEQ_START_TOKEN)
4400 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4401 else {
b7ceabd9 4402 struct sock *s = sk_entry(v);
1da177e4
LT		 4403 const struct packet_sock *po = pkt_sk(s);
4404
4405 seq_printf(seq,
71338aa7 4406 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4407 s,
4408 atomic_read(&s->sk_refcnt),
4409 s->sk_type,
4410 ntohs(po->num),
4411 po->ifindex,
4412 po->running,
4413 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4414 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4415 sock_i_ino(s));
1da177e4
LT		 4416 }
4417
4418 return 0;
4419}
4420
56b3d975 4421static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4422 .start = packet_seq_start,
4423 .next = packet_seq_next,
4424 .stop = packet_seq_stop,
4425 .show = packet_seq_show,
4426};
4427
4428static int packet_seq_open(struct inode *inode, struct file *file)
4429{
e372c414
DL		 4430 return seq_open_net(inode, file, &packet_seq_ops,
4431 sizeof(struct seq_net_private));
1da177e4
LT		 4432}
4433
da7071d7 4434static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4435 .owner = THIS_MODULE,
4436 .open = packet_seq_open,
4437 .read = seq_read,
4438 .llseek = seq_lseek,
e372c414 4439 .release = seq_release_net,
1da177e4
LT		 4440};
4441
4442#endif
4443
2c8c1e72 4444static int __net_init packet_net_init(struct net *net)
d12d01d6 4445{
0fa7fa98 4446 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4447 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4448
d4beaa66 4449 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4450 return -ENOMEM;
4451
4452 return 0;
4453}
4454
2c8c1e72 4455static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4456{
ece31ffd 4457 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4458}
4459
4460static struct pernet_operations packet_net_ops = {
4461 .init = packet_net_init,
4462 .exit = packet_net_exit,
4463};
4464
4465
1da177e4
LT		 4466static void __exit packet_exit(void)
4467{
1da177e4 4468 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4469 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4470 sock_unregister(PF_PACKET);
4471 proto_unregister(&packet_proto);
4472}
4473
4474static int __init packet_init(void)
4475{
4476 int rc = proto_register(&packet_proto, 0);
4477
4478 if (rc != 0)
4479 goto out;
4480
4481 sock_register(&packet_family_ops);
d12d01d6 4482 register_pernet_subsys(&packet_net_ops);
1da177e4 4483 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4484out:
4485 return rc;
4486}
4487
4488module_init(packet_init);
4489module_exit(packet_exit);
4490MODULE_LICENSE("GPL");
4491MODULE_ALIAS_NETPROTO(PF_PACKET);