projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'v3.10.58' into update
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / net / packet / af_packet.c
1 /*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
8 * Authors: Ross Biro
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
12 * Fixes:
13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
35 * Ulises Alonso : Frame number limit removal and
36 * packet_set_ring memory leak.
37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
40 * byte arrays at the end of sockaddr_ll
41 * and packet_mreq.
42 * Johann Baudy : Added TX RING.
43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
54
55 #include <linux/types.h>
56 #include <linux/mm.h>
57 #include <linux/capability.h>
58 #include <linux/fcntl.h>
59 #include <linux/socket.h>
60 #include <linux/in.h>
61 #include <linux/inet.h>
62 #include <linux/netdevice.h>
63 #include <linux/if_packet.h>
64 #include <linux/wireless.h>
65 #include <linux/kernel.h>
66 #include <linux/kmod.h>
67 #include <linux/slab.h>
68 #include <linux/vmalloc.h>
69 #include <net/net_namespace.h>
70 #include <net/ip.h>
71 #include <net/protocol.h>
72 #include <linux/skbuff.h>
73 #include <net/sock.h>
74 #include <linux/errno.h>
75 #include <linux/timer.h>
76 #include <asm/uaccess.h>
77 #include <asm/ioctls.h>
78 #include <asm/page.h>
79 #include <asm/cacheflush.h>
80 #include <asm/io.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83 #include <linux/poll.h>
84 #include <linux/module.h>
85 #include <linux/init.h>
86 #include <linux/mutex.h>
87 #include <linux/if_vlan.h>
88 #include <linux/virtio_net.h>
89 #include <linux/errqueue.h>
90 #include <linux/net_tstamp.h>
91
92 #ifdef CONFIG_INET
93 #include <net/inet_common.h>
94 #endif
95
96 #include "internal.h"
97
98 /*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109 On receive:
110 -----------
111
112 Incoming, dev->hard_header!=NULL
113 mac_header -> ll header
114 data -> data
115
116 Outgoing, dev->hard_header!=NULL
117 mac_header -> ll header
118 data -> ll header
119
120 Incoming, dev->hard_header==NULL
121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
123 assymetry between rx and tx paths.
124 data -> data
125
126 Outgoing, dev->hard_header==NULL
127 mac_header -> data. ll header is still not built!
128 data -> data
129
130 Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134 On transmit:
135 ------------
136
137 dev->hard_header != NULL
138 mac_header -> ll header
139 data -> ll header
140
141 dev->hard_header == NULL (ll header is added by device, we cannot control it)
142 mac_header -> data
143 data -> data
144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
149 /* Private packet socket structures. */
150
151 /* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
154 struct packet_mreq_max {
155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
159 };
160
161 union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166 };
167
168 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
169 int closing, int tx_ring);
170
171 #define V3_ALIGNMENT (8)
172
173 #define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
174
175 #define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
178 #define PGV_FROM_VMALLOC 1
179
180 #define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181 #define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182 #define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183 #define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184 #define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185 #define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186 #define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
188 struct packet_sock;
189 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
190 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
192
193 static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196 static void packet_increment_head(struct packet_ring_buffer *buff);
197 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200 struct packet_sock *);
201 static void prb_retire_current_block(struct tpacket_kbdq_core *,
202 struct packet_sock *, unsigned int status);
203 static int prb_queue_frozen(struct tpacket_kbdq_core *);
204 static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
206 static void prb_retire_rx_blk_timer_expired(unsigned long);
207 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208 static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
216 static void packet_flush_mclist(struct sock *sk);
217
218 struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224 };
225
226 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
227
228 #define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
229 #define GET_PBLOCK_DESC(x, bid) \
230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
231 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
233 #define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
237 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238 static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
240 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
241 {
242 struct net_device *dev;
243
244 rcu_read_lock();
245 dev = rcu_dereference(po->cached_dev);
246 if (likely(dev))
247 dev_hold(dev);
248 rcu_read_unlock();
249
250 return dev;
251 }
252
253 static void packet_cached_dev_assign(struct packet_sock *po,
254 struct net_device *dev)
255 {
256 rcu_assign_pointer(po->cached_dev, dev);
257 }
258
259 static void packet_cached_dev_reset(struct packet_sock *po)
260 {
261 RCU_INIT_POINTER(po->cached_dev, NULL);
262 }
263
264 /* register_prot_hook must be invoked with the po->bind_lock held,
265 * or from a context in which asynchronous accesses to the packet
266 * socket is not possible (packet_create()).
267 */
268 static void register_prot_hook(struct sock *sk)
269 {
270 struct packet_sock *po = pkt_sk(sk);
271
272 if (!po->running) {
273 if (po->fanout)
274 __fanout_link(sk, po);
275 else
276 dev_add_pack(&po->prot_hook);
277
278 sock_hold(sk);
279 po->running = 1;
280 }
281 }
282
283 /* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
284 * held. If the sync parameter is true, we will temporarily drop
285 * the po->bind_lock and do a synchronize_net to make sure no
286 * asynchronous packet processing paths still refer to the elements
287 * of po->prot_hook. If the sync parameter is false, it is the
288 * callers responsibility to take care of this.
289 */
290 static void __unregister_prot_hook(struct sock *sk, bool sync)
291 {
292 struct packet_sock *po = pkt_sk(sk);
293
294 po->running = 0;
295
296 if (po->fanout)
297 __fanout_unlink(sk, po);
298 else
299 __dev_remove_pack(&po->prot_hook);
300
301 __sock_put(sk);
302
303 if (sync) {
304 spin_unlock(&po->bind_lock);
305 synchronize_net();
306 spin_lock(&po->bind_lock);
307 }
308 }
309
310 static void unregister_prot_hook(struct sock *sk, bool sync)
311 {
312 struct packet_sock *po = pkt_sk(sk);
313
314 if (po->running)
315 __unregister_prot_hook(sk, sync);
316 }
317
318 static inline __pure struct page *pgv_to_page(void *addr)
319 {
320 if (is_vmalloc_addr(addr))
321 return vmalloc_to_page(addr);
322 return virt_to_page(addr);
323 }
324
325 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
326 {
327 union tpacket_uhdr h;
328
329 h.raw = frame;
330 switch (po->tp_version) {
331 case TPACKET_V1:
332 h.h1->tp_status = status;
333 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
334 break;
335 case TPACKET_V2:
336 h.h2->tp_status = status;
337 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
338 break;
339 case TPACKET_V3:
340 default:
341 WARN(1, "TPACKET version not supported.\n");
342 BUG();
343 }
344
345 smp_wmb();
346 }
347
348 static int __packet_get_status(struct packet_sock *po, void *frame)
349 {
350 union tpacket_uhdr h;
351
352 smp_rmb();
353
354 h.raw = frame;
355 switch (po->tp_version) {
356 case TPACKET_V1:
357 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
358 return h.h1->tp_status;
359 case TPACKET_V2:
360 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
361 return h.h2->tp_status;
362 case TPACKET_V3:
363 default:
364 WARN(1, "TPACKET version not supported.\n");
365 BUG();
366 return 0;
367 }
368 }
369
370 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
371 unsigned int flags)
372 {
373 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
374
375 if (shhwtstamps) {
376 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
377 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
378 return TP_STATUS_TS_SYS_HARDWARE;
379 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
380 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
381 return TP_STATUS_TS_RAW_HARDWARE;
382 }
383
384 if (ktime_to_timespec_cond(skb->tstamp, ts))
385 return TP_STATUS_TS_SOFTWARE;
386
387 return 0;
388 }
389
390 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
391 struct sk_buff *skb)
392 {
393 union tpacket_uhdr h;
394 struct timespec ts;
395 __u32 ts_status;
396
397 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
398 return 0;
399
400 h.raw = frame;
401 switch (po->tp_version) {
402 case TPACKET_V1:
403 h.h1->tp_sec = ts.tv_sec;
404 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
405 break;
406 case TPACKET_V2:
407 h.h2->tp_sec = ts.tv_sec;
408 h.h2->tp_nsec = ts.tv_nsec;
409 break;
410 case TPACKET_V3:
411 default:
412 WARN(1, "TPACKET version not supported.\n");
413 BUG();
414 }
415
416 /* one flush is safe, as both fields always lie on the same cacheline */
417 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
418 smp_wmb();
419
420 return ts_status;
421 }
422
423 static void *packet_lookup_frame(struct packet_sock *po,
424 struct packet_ring_buffer *rb,
425 unsigned int position,
426 int status)
427 {
428 unsigned int pg_vec_pos, frame_offset;
429 union tpacket_uhdr h;
430
431 pg_vec_pos = position / rb->frames_per_block;
432 frame_offset = position % rb->frames_per_block;
433
434 h.raw = rb->pg_vec[pg_vec_pos].buffer +
435 (frame_offset * rb->frame_size);
436
437 if (status != __packet_get_status(po, h.raw))
438 return NULL;
439
440 return h.raw;
441 }
442
443 static void *packet_current_frame(struct packet_sock *po,
444 struct packet_ring_buffer *rb,
445 int status)
446 {
447 return packet_lookup_frame(po, rb, rb->head, status);
448 }
449
450 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
451 {
452 del_timer_sync(&pkc->retire_blk_timer);
453 }
454
455 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
456 int tx_ring,
457 struct sk_buff_head *rb_queue)
458 {
459 struct tpacket_kbdq_core *pkc;
460
461 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
462
463 spin_lock_bh(&rb_queue->lock);
464 pkc->delete_blk_timer = 1;
465 spin_unlock_bh(&rb_queue->lock);
466
467 prb_del_retire_blk_timer(pkc);
468 }
469
470 static void prb_init_blk_timer(struct packet_sock *po,
471 struct tpacket_kbdq_core *pkc,
472 void (*func) (unsigned long))
473 {
474 init_timer(&pkc->retire_blk_timer);
475 pkc->retire_blk_timer.data = (long)po;
476 pkc->retire_blk_timer.function = func;
477 pkc->retire_blk_timer.expires = jiffies;
478 }
479
480 static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
481 {
482 struct tpacket_kbdq_core *pkc;
483
484 if (tx_ring)
485 BUG();
486
487 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
488 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
489 }
490
491 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
492 int blk_size_in_bytes)
493 {
494 struct net_device *dev;
495 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
496 struct ethtool_cmd ecmd;
497 int err;
498 u32 speed;
499
500 rtnl_lock();
501 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
502 if (unlikely(!dev)) {
503 rtnl_unlock();
504 return DEFAULT_PRB_RETIRE_TOV;
505 }
506 err = __ethtool_get_settings(dev, &ecmd);
507 speed = ethtool_cmd_speed(&ecmd);
508 rtnl_unlock();
509 if (!err) {
510 /*
511 * If the link speed is so slow you don't really
512 * need to worry about perf anyways
513 */
514 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
515 return DEFAULT_PRB_RETIRE_TOV;
516 } else {
517 msec = 1;
518 div = speed / 1000;
519 }
520 }
521
522 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
523
524 if (div)
525 mbits /= div;
526
527 tmo = mbits * msec;
528
529 if (div)
530 return tmo+1;
531 return tmo;
532 }
533
534 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
535 union tpacket_req_u *req_u)
536 {
537 p1->feature_req_word = req_u->req3.tp_feature_req_word;
538 }
539
540 static void init_prb_bdqc(struct packet_sock *po,
541 struct packet_ring_buffer *rb,
542 struct pgv *pg_vec,
543 union tpacket_req_u *req_u, int tx_ring)
544 {
545 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
546 struct tpacket_block_desc *pbd;
547
548 memset(p1, 0x0, sizeof(*p1));
549
550 p1->knxt_seq_num = 1;
551 p1->pkbdq = pg_vec;
552 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
553 p1->pkblk_start = pg_vec[0].buffer;
554 p1->kblk_size = req_u->req3.tp_block_size;
555 p1->knum_blocks = req_u->req3.tp_block_nr;
556 p1->hdrlen = po->tp_hdrlen;
557 p1->version = po->tp_version;
558 p1->last_kactive_blk_num = 0;
559 po->stats.stats3.tp_freeze_q_cnt = 0;
560 if (req_u->req3.tp_retire_blk_tov)
561 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
562 else
563 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
564 req_u->req3.tp_block_size);
565 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
566 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
567
568 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
569 prb_init_ft_ops(p1, req_u);
570 prb_setup_retire_blk_timer(po, tx_ring);
571 prb_open_block(p1, pbd);
572 }
573
574 /* Do NOT update the last_blk_num first.
575 * Assumes sk_buff_head lock is held.
576 */
577 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
578 {
579 mod_timer(&pkc->retire_blk_timer,
580 jiffies + pkc->tov_in_jiffies);
581 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
582 }
583
584 /*
585 * Timer logic:
586 * 1) We refresh the timer only when we open a block.
587 * By doing this we don't waste cycles refreshing the timer
588 * on packet-by-packet basis.
589 *
590 * With a 1MB block-size, on a 1Gbps line, it will take
591 * i) ~8 ms to fill a block + ii) memcpy etc.
592 * In this cut we are not accounting for the memcpy time.
593 *
594 * So, if the user sets the 'tmo' to 10ms then the timer
595 * will never fire while the block is still getting filled
596 * (which is what we want). However, the user could choose
597 * to close a block early and that's fine.
598 *
599 * But when the timer does fire, we check whether or not to refresh it.
600 * Since the tmo granularity is in msecs, it is not too expensive
601 * to refresh the timer, lets say every '8' msecs.
602 * Either the user can set the 'tmo' or we can derive it based on
603 * a) line-speed and b) block-size.
604 * prb_calc_retire_blk_tmo() calculates the tmo.
605 *
606 */
607 static void prb_retire_rx_blk_timer_expired(unsigned long data)
608 {
609 struct packet_sock *po = (struct packet_sock *)data;
610 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
611 unsigned int frozen;
612 struct tpacket_block_desc *pbd;
613
614 spin_lock(&po->sk.sk_receive_queue.lock);
615
616 frozen = prb_queue_frozen(pkc);
617 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
618
619 if (unlikely(pkc->delete_blk_timer))
620 goto out;
621
622 /* We only need to plug the race when the block is partially filled.
623 * tpacket_rcv:
624 * lock(); increment BLOCK_NUM_PKTS; unlock()
625 * copy_bits() is in progress ...
626 * timer fires on other cpu:
627 * we can't retire the current block because copy_bits
628 * is in progress.
629 *
630 */
631 if (BLOCK_NUM_PKTS(pbd)) {
632 while (atomic_read(&pkc->blk_fill_in_prog)) {
633 /* Waiting for skb_copy_bits to finish... */
634 cpu_relax();
635 }
636 }
637
638 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
639 if (!frozen) {
640 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
641 if (!prb_dispatch_next_block(pkc, po))
642 goto refresh_timer;
643 else
644 goto out;
645 } else {
646 /* Case 1. Queue was frozen because user-space was
647 * lagging behind.
648 */
649 if (prb_curr_blk_in_use(pkc, pbd)) {
650 /*
651 * Ok, user-space is still behind.
652 * So just refresh the timer.
653 */
654 goto refresh_timer;
655 } else {
656 /* Case 2. queue was frozen,user-space caught up,
657 * now the link went idle && the timer fired.
658 * We don't have a block to close.So we open this
659 * block and restart the timer.
660 * opening a block thaws the queue,restarts timer
661 * Thawing/timer-refresh is a side effect.
662 */
663 prb_open_block(pkc, pbd);
664 goto out;
665 }
666 }
667 }
668
669 refresh_timer:
670 _prb_refresh_rx_retire_blk_timer(pkc);
671
672 out:
673 spin_unlock(&po->sk.sk_receive_queue.lock);
674 }
675
676 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
677 struct tpacket_block_desc *pbd1, __u32 status)
678 {
679 /* Flush everything minus the block header */
680
681 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
682 u8 *start, *end;
683
684 start = (u8 *)pbd1;
685
686 /* Skip the block header(we know header WILL fit in 4K) */
687 start += PAGE_SIZE;
688
689 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
690 for (; start < end; start += PAGE_SIZE)
691 flush_dcache_page(pgv_to_page(start));
692
693 smp_wmb();
694 #endif
695
696 /* Now update the block status. */
697
698 BLOCK_STATUS(pbd1) = status;
699
700 /* Flush the block header */
701
702 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
703 start = (u8 *)pbd1;
704 flush_dcache_page(pgv_to_page(start));
705
706 smp_wmb();
707 #endif
708 }
709
710 /*
711 * Side effect:
712 *
713 * 1) flush the block
714 * 2) Increment active_blk_num
715 *
716 * Note:We DONT refresh the timer on purpose.
717 * Because almost always the next block will be opened.
718 */
719 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
720 struct tpacket_block_desc *pbd1,
721 struct packet_sock *po, unsigned int stat)
722 {
723 __u32 status = TP_STATUS_USER | stat;
724
725 struct tpacket3_hdr *last_pkt;
726 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
727
728 if (po->stats.stats3.tp_drops)
729 status |= TP_STATUS_LOSING;
730
731 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
732 last_pkt->tp_next_offset = 0;
733
734 /* Get the ts of the last pkt */
735 if (BLOCK_NUM_PKTS(pbd1)) {
736 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
737 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
738 } else {
739 /* Ok, we tmo'd - so get the current time */
740 struct timespec ts;
741 getnstimeofday(&ts);
742 h1->ts_last_pkt.ts_sec = ts.tv_sec;
743 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
744 }
745
746 smp_wmb();
747
748 /* Flush the block */
749 prb_flush_block(pkc1, pbd1, status);
750
751 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
752 }
753
754 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
755 {
756 pkc->reset_pending_on_curr_blk = 0;
757 }
758
759 /*
760 * Side effect of opening a block:
761 *
762 * 1) prb_queue is thawed.
763 * 2) retire_blk_timer is refreshed.
764 *
765 */
766 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
767 struct tpacket_block_desc *pbd1)
768 {
769 struct timespec ts;
770 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
771
772 smp_rmb();
773
774 /* We could have just memset this but we will lose the
775 * flexibility of making the priv area sticky
776 */
777
778 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
779 BLOCK_NUM_PKTS(pbd1) = 0;
780 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
781
782 getnstimeofday(&ts);
783
784 h1->ts_first_pkt.ts_sec = ts.tv_sec;
785 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
786
787 pkc1->pkblk_start = (char *)pbd1;
788 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
789
790 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
791 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
792
793 pbd1->version = pkc1->version;
794 pkc1->prev = pkc1->nxt_offset;
795 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
796
797 prb_thaw_queue(pkc1);
798 _prb_refresh_rx_retire_blk_timer(pkc1);
799
800 smp_wmb();
801 }
802
803 /*
804 * Queue freeze logic:
805 * 1) Assume tp_block_nr = 8 blocks.
806 * 2) At time 't0', user opens Rx ring.
807 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
808 * 4) user-space is either sleeping or processing block '0'.
809 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
810 * it will close block-7,loop around and try to fill block '0'.
811 * call-flow:
812 * __packet_lookup_frame_in_block
813 * prb_retire_current_block()
814 * prb_dispatch_next_block()
815 * |->(BLOCK_STATUS == USER) evaluates to true
816 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
817 * 6) Now there are two cases:
818 * 6.1) Link goes idle right after the queue is frozen.
819 * But remember, the last open_block() refreshed the timer.
820 * When this timer expires,it will refresh itself so that we can
821 * re-open block-0 in near future.
822 * 6.2) Link is busy and keeps on receiving packets. This is a simple
823 * case and __packet_lookup_frame_in_block will check if block-0
824 * is free and can now be re-used.
825 */
826 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
827 struct packet_sock *po)
828 {
829 pkc->reset_pending_on_curr_blk = 1;
830 po->stats.stats3.tp_freeze_q_cnt++;
831 }
832
833 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
834
835 /*
836 * If the next block is free then we will dispatch it
837 * and return a good offset.
838 * Else, we will freeze the queue.
839 * So, caller must check the return value.
840 */
841 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
842 struct packet_sock *po)
843 {
844 struct tpacket_block_desc *pbd;
845
846 smp_rmb();
847
848 /* 1. Get current block num */
849 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
850
851 /* 2. If this block is currently in_use then freeze the queue */
852 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
853 prb_freeze_queue(pkc, po);
854 return NULL;
855 }
856
857 /*
858 * 3.
859 * open this block and return the offset where the first packet
860 * needs to get stored.
861 */
862 prb_open_block(pkc, pbd);
863 return (void *)pkc->nxt_offset;
864 }
865
866 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
867 struct packet_sock *po, unsigned int status)
868 {
869 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
870
871 /* retire/close the current block */
872 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
873 /*
874 * Plug the case where copy_bits() is in progress on
875 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
876 * have space to copy the pkt in the current block and
877 * called prb_retire_current_block()
878 *
879 * We don't need to worry about the TMO case because
880 * the timer-handler already handled this case.
881 */
882 if (!(status & TP_STATUS_BLK_TMO)) {
883 while (atomic_read(&pkc->blk_fill_in_prog)) {
884 /* Waiting for skb_copy_bits to finish... */
885 cpu_relax();
886 }
887 }
888 prb_close_block(pkc, pbd, po, status);
889 return;
890 }
891 }
892
893 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
894 struct tpacket_block_desc *pbd)
895 {
896 return TP_STATUS_USER & BLOCK_STATUS(pbd);
897 }
898
899 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
900 {
901 return pkc->reset_pending_on_curr_blk;
902 }
903
904 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
905 {
906 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
907 atomic_dec(&pkc->blk_fill_in_prog);
908 }
909
910 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
911 struct tpacket3_hdr *ppd)
912 {
913 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
914 }
915
916 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
917 struct tpacket3_hdr *ppd)
918 {
919 ppd->hv1.tp_rxhash = 0;
920 }
921
922 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
923 struct tpacket3_hdr *ppd)
924 {
925 if (vlan_tx_tag_present(pkc->skb)) {
926 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
927 ppd->tp_status = TP_STATUS_VLAN_VALID;
928 } else {
929 ppd->hv1.tp_vlan_tci = 0;
930 ppd->tp_status = TP_STATUS_AVAILABLE;
931 }
932 }
933
934 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
935 struct tpacket3_hdr *ppd)
936 {
937 prb_fill_vlan_info(pkc, ppd);
938
939 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
940 prb_fill_rxhash(pkc, ppd);
941 else
942 prb_clear_rxhash(pkc, ppd);
943 }
944
945 static void prb_fill_curr_block(char *curr,
946 struct tpacket_kbdq_core *pkc,
947 struct tpacket_block_desc *pbd,
948 unsigned int len)
949 {
950 struct tpacket3_hdr *ppd;
951
952 ppd = (struct tpacket3_hdr *)curr;
953 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
954 pkc->prev = curr;
955 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
956 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
957 BLOCK_NUM_PKTS(pbd) += 1;
958 atomic_inc(&pkc->blk_fill_in_prog);
959 prb_run_all_ft_ops(pkc, ppd);
960 }
961
962 /* Assumes caller has the sk->rx_queue.lock */
963 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
964 struct sk_buff *skb,
965 int status,
966 unsigned int len
967 )
968 {
969 struct tpacket_kbdq_core *pkc;
970 struct tpacket_block_desc *pbd;
971 char *curr, *end;
972
973 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
974 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
975
976 /* Queue is frozen when user space is lagging behind */
977 if (prb_queue_frozen(pkc)) {
978 /*
979 * Check if that last block which caused the queue to freeze,
980 * is still in_use by user-space.
981 */
982 if (prb_curr_blk_in_use(pkc, pbd)) {
983 /* Can't record this packet */
984 return NULL;
985 } else {
986 /*
987 * Ok, the block was released by user-space.
988 * Now let's open that block.
989 * opening a block also thaws the queue.
990 * Thawing is a side effect.
991 */
992 prb_open_block(pkc, pbd);
993 }
994 }
995
996 smp_mb();
997 curr = pkc->nxt_offset;
998 pkc->skb = skb;
999 end = (char *)pbd + pkc->kblk_size;
1000
1001 /* first try the current block */
1002 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1003 prb_fill_curr_block(curr, pkc, pbd, len);
1004 return (void *)curr;
1005 }
1006
1007 /* Ok, close the current block */
1008 prb_retire_current_block(pkc, po, 0);
1009
1010 /* Now, try to dispatch the next block */
1011 curr = (char *)prb_dispatch_next_block(pkc, po);
1012 if (curr) {
1013 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1014 prb_fill_curr_block(curr, pkc, pbd, len);
1015 return (void *)curr;
1016 }
1017
1018 /*
1019 * No free blocks are available.user_space hasn't caught up yet.
1020 * Queue was just frozen and now this packet will get dropped.
1021 */
1022 return NULL;
1023 }
1024
1025 static void *packet_current_rx_frame(struct packet_sock *po,
1026 struct sk_buff *skb,
1027 int status, unsigned int len)
1028 {
1029 char *curr = NULL;
1030 switch (po->tp_version) {
1031 case TPACKET_V1:
1032 case TPACKET_V2:
1033 curr = packet_lookup_frame(po, &po->rx_ring,
1034 po->rx_ring.head, status);
1035 return curr;
1036 case TPACKET_V3:
1037 return __packet_lookup_frame_in_block(po, skb, status, len);
1038 default:
1039 WARN(1, "TPACKET version not supported\n");
1040 BUG();
1041 return NULL;
1042 }
1043 }
1044
1045 static void *prb_lookup_block(struct packet_sock *po,
1046 struct packet_ring_buffer *rb,
1047 unsigned int idx,
1048 int status)
1049 {
1050 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1051 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1052
1053 if (status != BLOCK_STATUS(pbd))
1054 return NULL;
1055 return pbd;
1056 }
1057
1058 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1059 {
1060 unsigned int prev;
1061 if (rb->prb_bdqc.kactive_blk_num)
1062 prev = rb->prb_bdqc.kactive_blk_num-1;
1063 else
1064 prev = rb->prb_bdqc.knum_blocks-1;
1065 return prev;
1066 }
1067
1068 /* Assumes caller has held the rx_queue.lock */
1069 static void *__prb_previous_block(struct packet_sock *po,
1070 struct packet_ring_buffer *rb,
1071 int status)
1072 {
1073 unsigned int previous = prb_previous_blk_num(rb);
1074 return prb_lookup_block(po, rb, previous, status);
1075 }
1076
1077 static void *packet_previous_rx_frame(struct packet_sock *po,
1078 struct packet_ring_buffer *rb,
1079 int status)
1080 {
1081 if (po->tp_version <= TPACKET_V2)
1082 return packet_previous_frame(po, rb, status);
1083
1084 return __prb_previous_block(po, rb, status);
1085 }
1086
1087 static void packet_increment_rx_head(struct packet_sock *po,
1088 struct packet_ring_buffer *rb)
1089 {
1090 switch (po->tp_version) {
1091 case TPACKET_V1:
1092 case TPACKET_V2:
1093 return packet_increment_head(rb);
1094 case TPACKET_V3:
1095 default:
1096 WARN(1, "TPACKET version not supported.\n");
1097 BUG();
1098 return;
1099 }
1100 }
1101
1102 static void *packet_previous_frame(struct packet_sock *po,
1103 struct packet_ring_buffer *rb,
1104 int status)
1105 {
1106 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1107 return packet_lookup_frame(po, rb, previous, status);
1108 }
1109
1110 static void packet_increment_head(struct packet_ring_buffer *buff)
1111 {
1112 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1113 }
1114
1115 static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1116 {
1117 struct sock *sk = &po->sk;
1118 bool has_room;
1119
1120 if (po->prot_hook.func != tpacket_rcv)
1121 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1122 <= sk->sk_rcvbuf;
1123
1124 spin_lock(&sk->sk_receive_queue.lock);
1125 if (po->tp_version == TPACKET_V3)
1126 has_room = prb_lookup_block(po, &po->rx_ring,
1127 po->rx_ring.prb_bdqc.kactive_blk_num,
1128 TP_STATUS_KERNEL);
1129 else
1130 has_room = packet_lookup_frame(po, &po->rx_ring,
1131 po->rx_ring.head,
1132 TP_STATUS_KERNEL);
1133 spin_unlock(&sk->sk_receive_queue.lock);
1134
1135 return has_room;
1136 }
1137
1138 static void packet_sock_destruct(struct sock *sk)
1139 {
1140 skb_queue_purge(&sk->sk_error_queue);
1141
1142 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1143 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1144
1145 if (!sock_flag(sk, SOCK_DEAD)) {
1146 pr_err("Attempt to release alive packet socket: %p\n", sk);
1147 return;
1148 }
1149
1150 sk_refcnt_debug_dec(sk);
1151 }
1152
1153 static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1154 {
1155 int x = atomic_read(&f->rr_cur) + 1;
1156
1157 if (x >= num)
1158 x = 0;
1159
1160 return x;
1161 }
1162
1163 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1164 struct sk_buff *skb,
1165 unsigned int num)
1166 {
1167 return (((u64)skb->rxhash) * num) >> 32;
1168 }
1169
1170 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1171 struct sk_buff *skb,
1172 unsigned int num)
1173 {
1174 int cur, old;
1175
1176 cur = atomic_read(&f->rr_cur);
1177 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1178 fanout_rr_next(f, num))) != cur)
1179 cur = old;
1180 return cur;
1181 }
1182
1183 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1184 struct sk_buff *skb,
1185 unsigned int num)
1186 {
1187 return smp_processor_id() % num;
1188 }
1189
1190 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1191 struct sk_buff *skb,
1192 unsigned int idx, unsigned int skip,
1193 unsigned int num)
1194 {
1195 unsigned int i, j;
1196
1197 i = j = min_t(int, f->next[idx], num - 1);
1198 do {
1199 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1200 if (i != j)
1201 f->next[idx] = i;
1202 return i;
1203 }
1204 if (++i == num)
1205 i = 0;
1206 } while (i != j);
1207
1208 return idx;
1209 }
1210
1211 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1212 {
1213 return f->flags & (flag >> 8);
1214 }
1215
1216 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1217 struct packet_type *pt, struct net_device *orig_dev)
1218 {
1219 struct packet_fanout *f = pt->af_packet_priv;
1220 unsigned int num = f->num_members;
1221 struct packet_sock *po;
1222 unsigned int idx;
1223
1224 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1225 !num) {
1226 kfree_skb(skb);
1227 return 0;
1228 }
1229
1230 switch (f->type) {
1231 case PACKET_FANOUT_HASH:
1232 default:
1233 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1234 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1235 if (!skb)
1236 return 0;
1237 }
1238 skb_get_rxhash(skb);
1239 idx = fanout_demux_hash(f, skb, num);
1240 break;
1241 case PACKET_FANOUT_LB:
1242 idx = fanout_demux_lb(f, skb, num);
1243 break;
1244 case PACKET_FANOUT_CPU:
1245 idx = fanout_demux_cpu(f, skb, num);
1246 break;
1247 case PACKET_FANOUT_ROLLOVER:
1248 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
1249 break;
1250 }
1251
1252 po = pkt_sk(f->arr[idx]);
1253 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1254 unlikely(!packet_rcv_has_room(po, skb))) {
1255 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1256 po = pkt_sk(f->arr[idx]);
1257 }
1258
1259 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1260 }
1261
1262 DEFINE_MUTEX(fanout_mutex);
1263 EXPORT_SYMBOL_GPL(fanout_mutex);
1264 static LIST_HEAD(fanout_list);
1265
1266 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1267 {
1268 struct packet_fanout *f = po->fanout;
1269
1270 spin_lock(&f->lock);
1271 f->arr[f->num_members] = sk;
1272 smp_wmb();
1273 f->num_members++;
1274 spin_unlock(&f->lock);
1275 }
1276
1277 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1278 {
1279 struct packet_fanout *f = po->fanout;
1280 int i;
1281
1282 spin_lock(&f->lock);
1283 for (i = 0; i < f->num_members; i++) {
1284 if (f->arr[i] == sk)
1285 break;
1286 }
1287 BUG_ON(i >= f->num_members);
1288 f->arr[i] = f->arr[f->num_members - 1];
1289 f->num_members--;
1290 spin_unlock(&f->lock);
1291 }
1292
1293 static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
1294 {
1295 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1296 return true;
1297
1298 return false;
1299 }
1300
1301 static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1302 {
1303 struct packet_sock *po = pkt_sk(sk);
1304 struct packet_fanout *f, *match;
1305 u8 type = type_flags & 0xff;
1306 u8 flags = type_flags >> 8;
1307 int err;
1308
1309 switch (type) {
1310 case PACKET_FANOUT_ROLLOVER:
1311 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1312 return -EINVAL;
1313 case PACKET_FANOUT_HASH:
1314 case PACKET_FANOUT_LB:
1315 case PACKET_FANOUT_CPU:
1316 break;
1317 default:
1318 return -EINVAL;
1319 }
1320
1321 if (!po->running)
1322 return -EINVAL;
1323
1324 if (po->fanout)
1325 return -EALREADY;
1326
1327 mutex_lock(&fanout_mutex);
1328 match = NULL;
1329 list_for_each_entry(f, &fanout_list, list) {
1330 if (f->id == id &&
1331 read_pnet(&f->net) == sock_net(sk)) {
1332 match = f;
1333 break;
1334 }
1335 }
1336 err = -EINVAL;
1337 if (match && match->flags != flags)
1338 goto out;
1339 if (!match) {
1340 err = -ENOMEM;
1341 match = kzalloc(sizeof(*match), GFP_KERNEL);
1342 if (!match)
1343 goto out;
1344 write_pnet(&match->net, sock_net(sk));
1345 match->id = id;
1346 match->type = type;
1347 match->flags = flags;
1348 atomic_set(&match->rr_cur, 0);
1349 INIT_LIST_HEAD(&match->list);
1350 spin_lock_init(&match->lock);
1351 atomic_set(&match->sk_ref, 0);
1352 match->prot_hook.type = po->prot_hook.type;
1353 match->prot_hook.dev = po->prot_hook.dev;
1354 match->prot_hook.func = packet_rcv_fanout;
1355 match->prot_hook.af_packet_priv = match;
1356 match->prot_hook.id_match = match_fanout_group;
1357 dev_add_pack(&match->prot_hook);
1358 list_add(&match->list, &fanout_list);
1359 }
1360 err = -EINVAL;
1361 if (match->type == type &&
1362 match->prot_hook.type == po->prot_hook.type &&
1363 match->prot_hook.dev == po->prot_hook.dev) {
1364 err = -ENOSPC;
1365 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1366 __dev_remove_pack(&po->prot_hook);
1367 po->fanout = match;
1368 atomic_inc(&match->sk_ref);
1369 __fanout_link(sk, po);
1370 err = 0;
1371 }
1372 }
1373 out:
1374 mutex_unlock(&fanout_mutex);
1375 return err;
1376 }
1377
1378 static void fanout_release(struct sock *sk)
1379 {
1380 struct packet_sock *po = pkt_sk(sk);
1381 struct packet_fanout *f;
1382
1383 f = po->fanout;
1384 if (!f)
1385 return;
1386
1387 mutex_lock(&fanout_mutex);
1388 po->fanout = NULL;
1389
1390 if (atomic_dec_and_test(&f->sk_ref)) {
1391 list_del(&f->list);
1392 dev_remove_pack(&f->prot_hook);
1393 kfree(f);
1394 }
1395 mutex_unlock(&fanout_mutex);
1396 }
1397
1398 static const struct proto_ops packet_ops;
1399
1400 static const struct proto_ops packet_ops_spkt;
1401
1402 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1403 struct packet_type *pt, struct net_device *orig_dev)
1404 {
1405 struct sock *sk;
1406 struct sockaddr_pkt *spkt;
1407
1408 /*
1409 * When we registered the protocol we saved the socket in the data
1410 * field for just this event.
1411 */
1412
1413 sk = pt->af_packet_priv;
1414
1415 /*
1416 * Yank back the headers [hope the device set this
1417 * right or kerboom...]
1418 *
1419 * Incoming packets have ll header pulled,
1420 * push it back.
1421 *
1422 * For outgoing ones skb->data == skb_mac_header(skb)
1423 * so that this procedure is noop.
1424 */
1425
1426 if (skb->pkt_type == PACKET_LOOPBACK)
1427 goto out;
1428
1429 if (!net_eq(dev_net(dev), sock_net(sk)))
1430 goto out;
1431
1432 skb = skb_share_check(skb, GFP_ATOMIC);
1433 if (skb == NULL)
1434 goto oom;
1435
1436 /* drop any routing info */
1437 skb_dst_drop(skb);
1438
1439 /* drop conntrack reference */
1440 nf_reset(skb);
1441
1442 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1443
1444 skb_push(skb, skb->data - skb_mac_header(skb));
1445
1446 /*
1447 * The SOCK_PACKET socket receives _all_ frames.
1448 */
1449
1450 spkt->spkt_family = dev->type;
1451 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1452 spkt->spkt_protocol = skb->protocol;
1453
1454 /*
1455 * Charge the memory to the socket. This is done specifically
1456 * to prevent sockets using all the memory up.
1457 */
1458
1459 if (sock_queue_rcv_skb(sk, skb) == 0)
1460 return 0;
1461
1462 out:
1463 kfree_skb(skb);
1464 oom:
1465 return 0;
1466 }
1467
1468
1469 /*
1470 * Output a raw packet to a device layer. This bypasses all the other
1471 * protocol layers and you must therefore supply it with a complete frame
1472 */
1473
1474 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1475 struct msghdr *msg, size_t len)
1476 {
1477 struct sock *sk = sock->sk;
1478 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1479 struct sk_buff *skb = NULL;
1480 struct net_device *dev;
1481 __be16 proto = 0;
1482 int err;
1483 int extra_len = 0;
1484
1485 /*
1486 * Get and verify the address.
1487 */
1488
1489 if (saddr) {
1490 if (msg->msg_namelen < sizeof(struct sockaddr))
1491 return -EINVAL;
1492 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1493 proto = saddr->spkt_protocol;
1494 } else
1495 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1496
1497 /*
1498 * Find the device first to size check it
1499 */
1500
1501 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1502 retry:
1503 rcu_read_lock();
1504 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1505 err = -ENODEV;
1506 if (dev == NULL)
1507 goto out_unlock;
1508
1509 err = -ENETDOWN;
1510 if (!(dev->flags & IFF_UP))
1511 goto out_unlock;
1512
1513 /*
1514 * You may not queue a frame bigger than the mtu. This is the lowest level
1515 * raw protocol and you must do your own fragmentation at this level.
1516 */
1517
1518 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1519 if (!netif_supports_nofcs(dev)) {
1520 err = -EPROTONOSUPPORT;
1521 goto out_unlock;
1522 }
1523 extra_len = 4; /* We're doing our own CRC */
1524 }
1525
1526 err = -EMSGSIZE;
1527 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1528 goto out_unlock;
1529
1530 if (!skb) {
1531 size_t reserved = LL_RESERVED_SPACE(dev);
1532 int tlen = dev->needed_tailroom;
1533 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1534
1535 rcu_read_unlock();
1536 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1537 if (skb == NULL)
1538 return -ENOBUFS;
1539 /* FIXME: Save some space for broken drivers that write a hard
1540 * header at transmission time by themselves. PPP is the notable
1541 * one here. This should really be fixed at the driver level.
1542 */
1543 skb_reserve(skb, reserved);
1544 skb_reset_network_header(skb);
1545
1546 /* Try to align data part correctly */
1547 if (hhlen) {
1548 skb->data -= hhlen;
1549 skb->tail -= hhlen;
1550 if (len < hhlen)
1551 skb_reset_network_header(skb);
1552 }
1553 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1554 if (err)
1555 goto out_free;
1556 goto retry;
1557 }
1558
1559 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
1560 /* Earlier code assumed this would be a VLAN pkt,
1561 * double-check this now that we have the actual
1562 * packet in hand.
1563 */
1564 struct ethhdr *ehdr;
1565 skb_reset_mac_header(skb);
1566 ehdr = eth_hdr(skb);
1567 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1568 err = -EMSGSIZE;
1569 goto out_unlock;
1570 }
1571 }
1572
1573 skb->protocol = proto;
1574 skb->dev = dev;
1575 skb->priority = sk->sk_priority;
1576 skb->mark = sk->sk_mark;
1577
1578 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1579
1580 if (unlikely(extra_len == 4))
1581 skb->no_fcs = 1;
1582
1583 skb_probe_transport_header(skb, 0);
1584
1585 dev_queue_xmit(skb);
1586 rcu_read_unlock();
1587 return len;
1588
1589 out_unlock:
1590 rcu_read_unlock();
1591 out_free:
1592 kfree_skb(skb);
1593 return err;
1594 }
1595
1596 static unsigned int run_filter(const struct sk_buff *skb,
1597 const struct sock *sk,
1598 unsigned int res)
1599 {
1600 struct sk_filter *filter;
1601
1602 rcu_read_lock();
1603 filter = rcu_dereference(sk->sk_filter);
1604 if (filter != NULL)
1605 res = SK_RUN_FILTER(filter, skb);
1606 rcu_read_unlock();
1607
1608 return res;
1609 }
1610
1611 /*
1612 * This function makes lazy skb cloning in hope that most of packets
1613 * are discarded by BPF.
1614 *
1615 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1616 * and skb->cb are mangled. It works because (and until) packets
1617 * falling here are owned by current CPU. Output packets are cloned
1618 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1619 * sequencially, so that if we return skb to original state on exit,
1620 * we will not harm anyone.
1621 */
1622
1623 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1624 struct packet_type *pt, struct net_device *orig_dev)
1625 {
1626 struct sock *sk;
1627 struct sockaddr_ll *sll;
1628 struct packet_sock *po;
1629 u8 *skb_head = skb->data;
1630 int skb_len = skb->len;
1631 unsigned int snaplen, res;
1632
1633 if (skb->pkt_type == PACKET_LOOPBACK)
1634 goto drop;
1635
1636 sk = pt->af_packet_priv;
1637 po = pkt_sk(sk);
1638
1639 if (!net_eq(dev_net(dev), sock_net(sk)))
1640 goto drop;
1641
1642 skb->dev = dev;
1643
1644 if (dev->header_ops) {
1645 /* The device has an explicit notion of ll header,
1646 * exported to higher levels.
1647 *
1648 * Otherwise, the device hides details of its frame
1649 * structure, so that corresponding packet head is
1650 * never delivered to user.
1651 */
1652 if (sk->sk_type != SOCK_DGRAM)
1653 skb_push(skb, skb->data - skb_mac_header(skb));
1654 else if (skb->pkt_type == PACKET_OUTGOING) {
1655 /* Special case: outgoing packets have ll header at head */
1656 skb_pull(skb, skb_network_offset(skb));
1657 }
1658 }
1659
1660 snaplen = skb->len;
1661
1662 res = run_filter(skb, sk, snaplen);
1663 if (!res)
1664 goto drop_n_restore;
1665 if (snaplen > res)
1666 snaplen = res;
1667
1668 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1669 goto drop_n_acct;
1670
1671 if (skb_shared(skb)) {
1672 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1673 if (nskb == NULL)
1674 goto drop_n_acct;
1675
1676 if (skb_head != skb->data) {
1677 skb->data = skb_head;
1678 skb->len = skb_len;
1679 }
1680 consume_skb(skb);
1681 skb = nskb;
1682 }
1683
1684 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1685 sizeof(skb->cb));
1686
1687 sll = &PACKET_SKB_CB(skb)->sa.ll;
1688 sll->sll_family = AF_PACKET;
1689 sll->sll_hatype = dev->type;
1690 sll->sll_protocol = skb->protocol;
1691 sll->sll_pkttype = skb->pkt_type;
1692 if (unlikely(po->origdev))
1693 sll->sll_ifindex = orig_dev->ifindex;
1694 else
1695 sll->sll_ifindex = dev->ifindex;
1696
1697 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1698
1699 PACKET_SKB_CB(skb)->origlen = skb->len;
1700
1701 if (pskb_trim(skb, snaplen))
1702 goto drop_n_acct;
1703
1704 skb_set_owner_r(skb, sk);
1705 skb->dev = NULL;
1706 skb_dst_drop(skb);
1707
1708 /* drop conntrack reference */
1709 nf_reset(skb);
1710
1711 spin_lock(&sk->sk_receive_queue.lock);
1712 po->stats.stats1.tp_packets++;
1713 skb->dropcount = atomic_read(&sk->sk_drops);
1714 __skb_queue_tail(&sk->sk_receive_queue, skb);
1715 spin_unlock(&sk->sk_receive_queue.lock);
1716 sk->sk_data_ready(sk, skb->len);
1717 return 0;
1718
1719 drop_n_acct:
1720 spin_lock(&sk->sk_receive_queue.lock);
1721 po->stats.stats1.tp_drops++;
1722 atomic_inc(&sk->sk_drops);
1723 spin_unlock(&sk->sk_receive_queue.lock);
1724
1725 drop_n_restore:
1726 if (skb_head != skb->data && skb_shared(skb)) {
1727 skb->data = skb_head;
1728 skb->len = skb_len;
1729 }
1730 drop:
1731 consume_skb(skb);
1732 return 0;
1733 }
1734
1735 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1736 struct packet_type *pt, struct net_device *orig_dev)
1737 {
1738 struct sock *sk;
1739 struct packet_sock *po;
1740 struct sockaddr_ll *sll;
1741 union tpacket_uhdr h;
1742 u8 *skb_head = skb->data;
1743 int skb_len = skb->len;
1744 unsigned int snaplen, res;
1745 unsigned long status = TP_STATUS_USER;
1746 unsigned short macoff, netoff, hdrlen;
1747 struct sk_buff *copy_skb = NULL;
1748 struct timespec ts;
1749 __u32 ts_status;
1750
1751 if (skb->pkt_type == PACKET_LOOPBACK)
1752 goto drop;
1753
1754 sk = pt->af_packet_priv;
1755 po = pkt_sk(sk);
1756
1757 if (!net_eq(dev_net(dev), sock_net(sk)))
1758 goto drop;
1759
1760 if (dev->header_ops) {
1761 if (sk->sk_type != SOCK_DGRAM)
1762 skb_push(skb, skb->data - skb_mac_header(skb));
1763 else if (skb->pkt_type == PACKET_OUTGOING) {
1764 /* Special case: outgoing packets have ll header at head */
1765 skb_pull(skb, skb_network_offset(skb));
1766 }
1767 }
1768
1769 if (skb->ip_summed == CHECKSUM_PARTIAL)
1770 status |= TP_STATUS_CSUMNOTREADY;
1771
1772 snaplen = skb->len;
1773
1774 res = run_filter(skb, sk, snaplen);
1775 if (!res)
1776 goto drop_n_restore;
1777 if (snaplen > res)
1778 snaplen = res;
1779
1780 if (sk->sk_type == SOCK_DGRAM) {
1781 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1782 po->tp_reserve;
1783 } else {
1784 unsigned int maclen = skb_network_offset(skb);
1785 netoff = TPACKET_ALIGN(po->tp_hdrlen +
1786 (maclen < 16 ? 16 : maclen)) +
1787 po->tp_reserve;
1788 macoff = netoff - maclen;
1789 }
1790 if (po->tp_version <= TPACKET_V2) {
1791 if (macoff + snaplen > po->rx_ring.frame_size) {
1792 if (po->copy_thresh &&
1793 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
1794 if (skb_shared(skb)) {
1795 copy_skb = skb_clone(skb, GFP_ATOMIC);
1796 } else {
1797 copy_skb = skb_get(skb);
1798 skb_head = skb->data;
1799 }
1800 if (copy_skb)
1801 skb_set_owner_r(copy_skb, sk);
1802 }
1803 snaplen = po->rx_ring.frame_size - macoff;
1804 if ((int)snaplen < 0)
1805 snaplen = 0;
1806 }
1807 } else if (unlikely(macoff + snaplen >
1808 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1809 u32 nval;
1810
1811 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1812 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1813 snaplen, nval, macoff);
1814 snaplen = nval;
1815 if (unlikely((int)snaplen < 0)) {
1816 snaplen = 0;
1817 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1818 }
1819 }
1820 spin_lock(&sk->sk_receive_queue.lock);
1821 h.raw = packet_current_rx_frame(po, skb,
1822 TP_STATUS_KERNEL, (macoff+snaplen));
1823 if (!h.raw)
1824 goto ring_is_full;
1825 if (po->tp_version <= TPACKET_V2) {
1826 packet_increment_rx_head(po, &po->rx_ring);
1827 /*
1828 * LOSING will be reported till you read the stats,
1829 * because it's COR - Clear On Read.
1830 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1831 * at packet level.
1832 */
1833 if (po->stats.stats1.tp_drops)
1834 status |= TP_STATUS_LOSING;
1835 }
1836 po->stats.stats1.tp_packets++;
1837 if (copy_skb) {
1838 status |= TP_STATUS_COPY;
1839 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1840 }
1841 spin_unlock(&sk->sk_receive_queue.lock);
1842
1843 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1844
1845 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
1846 getnstimeofday(&ts);
1847
1848 status |= ts_status;
1849
1850 switch (po->tp_version) {
1851 case TPACKET_V1:
1852 h.h1->tp_len = skb->len;
1853 h.h1->tp_snaplen = snaplen;
1854 h.h1->tp_mac = macoff;
1855 h.h1->tp_net = netoff;
1856 h.h1->tp_sec = ts.tv_sec;
1857 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
1858 hdrlen = sizeof(*h.h1);
1859 break;
1860 case TPACKET_V2:
1861 h.h2->tp_len = skb->len;
1862 h.h2->tp_snaplen = snaplen;
1863 h.h2->tp_mac = macoff;
1864 h.h2->tp_net = netoff;
1865 h.h2->tp_sec = ts.tv_sec;
1866 h.h2->tp_nsec = ts.tv_nsec;
1867 if (vlan_tx_tag_present(skb)) {
1868 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1869 status |= TP_STATUS_VLAN_VALID;
1870 } else {
1871 h.h2->tp_vlan_tci = 0;
1872 }
1873 h.h2->tp_padding = 0;
1874 hdrlen = sizeof(*h.h2);
1875 break;
1876 case TPACKET_V3:
1877 /* tp_nxt_offset,vlan are already populated above.
1878 * So DONT clear those fields here
1879 */
1880 h.h3->tp_status |= status;
1881 h.h3->tp_len = skb->len;
1882 h.h3->tp_snaplen = snaplen;
1883 h.h3->tp_mac = macoff;
1884 h.h3->tp_net = netoff;
1885 h.h3->tp_sec = ts.tv_sec;
1886 h.h3->tp_nsec = ts.tv_nsec;
1887 hdrlen = sizeof(*h.h3);
1888 break;
1889 default:
1890 BUG();
1891 }
1892
1893 sll = h.raw + TPACKET_ALIGN(hdrlen);
1894 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1895 sll->sll_family = AF_PACKET;
1896 sll->sll_hatype = dev->type;
1897 sll->sll_protocol = skb->protocol;
1898 sll->sll_pkttype = skb->pkt_type;
1899 if (unlikely(po->origdev))
1900 sll->sll_ifindex = orig_dev->ifindex;
1901 else
1902 sll->sll_ifindex = dev->ifindex;
1903
1904 smp_mb();
1905 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1906 {
1907 u8 *start, *end;
1908
1909 if (po->tp_version <= TPACKET_V2) {
1910 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1911 + macoff + snaplen);
1912 for (start = h.raw; start < end; start += PAGE_SIZE)
1913 flush_dcache_page(pgv_to_page(start));
1914 }
1915 smp_wmb();
1916 }
1917 #endif
1918 if (po->tp_version <= TPACKET_V2)
1919 __packet_set_status(po, h.raw, status);
1920 else
1921 prb_clear_blk_fill_status(&po->rx_ring);
1922
1923 sk->sk_data_ready(sk, 0);
1924
1925 drop_n_restore:
1926 if (skb_head != skb->data && skb_shared(skb)) {
1927 skb->data = skb_head;
1928 skb->len = skb_len;
1929 }
1930 drop:
1931 kfree_skb(skb);
1932 return 0;
1933
1934 ring_is_full:
1935 po->stats.stats1.tp_drops++;
1936 spin_unlock(&sk->sk_receive_queue.lock);
1937
1938 sk->sk_data_ready(sk, 0);
1939 kfree_skb(copy_skb);
1940 goto drop_n_restore;
1941 }
1942
1943 static void tpacket_destruct_skb(struct sk_buff *skb)
1944 {
1945 struct packet_sock *po = pkt_sk(skb->sk);
1946 void *ph;
1947
1948 if (likely(po->tx_ring.pg_vec)) {
1949 __u32 ts;
1950
1951 ph = skb_shinfo(skb)->destructor_arg;
1952 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1953 atomic_dec(&po->tx_ring.pending);
1954
1955 ts = __packet_set_timestamp(po, ph, skb);
1956 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
1957 }
1958
1959 sock_wfree(skb);
1960 }
1961
1962 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1963 void *frame, struct net_device *dev, int size_max,
1964 __be16 proto, unsigned char *addr, int hlen)
1965 {
1966 union tpacket_uhdr ph;
1967 int to_write, offset, len, tp_len, nr_frags, len_max;
1968 struct socket *sock = po->sk.sk_socket;
1969 struct page *page;
1970 void *data;
1971 int err;
1972
1973 ph.raw = frame;
1974
1975 skb->protocol = proto;
1976 skb->dev = dev;
1977 skb->priority = po->sk.sk_priority;
1978 skb->mark = po->sk.sk_mark;
1979 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
1980 skb_shinfo(skb)->destructor_arg = ph.raw;
1981
1982 switch (po->tp_version) {
1983 case TPACKET_V2:
1984 tp_len = ph.h2->tp_len;
1985 break;
1986 default:
1987 tp_len = ph.h1->tp_len;
1988 break;
1989 }
1990 if (unlikely(tp_len > size_max)) {
1991 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
1992 return -EMSGSIZE;
1993 }
1994
1995 skb_reserve(skb, hlen);
1996 skb_reset_network_header(skb);
1997 skb_probe_transport_header(skb, 0);
1998
1999 if (po->tp_tx_has_off) {
2000 int off_min, off_max, off;
2001 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2002 off_max = po->tx_ring.frame_size - tp_len;
2003 if (sock->type == SOCK_DGRAM) {
2004 switch (po->tp_version) {
2005 case TPACKET_V2:
2006 off = ph.h2->tp_net;
2007 break;
2008 default:
2009 off = ph.h1->tp_net;
2010 break;
2011 }
2012 } else {
2013 switch (po->tp_version) {
2014 case TPACKET_V2:
2015 off = ph.h2->tp_mac;
2016 break;
2017 default:
2018 off = ph.h1->tp_mac;
2019 break;
2020 }
2021 }
2022 if (unlikely((off < off_min) || (off_max < off)))
2023 return -EINVAL;
2024 data = ph.raw + off;
2025 } else {
2026 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2027 }
2028 to_write = tp_len;
2029
2030 if (sock->type == SOCK_DGRAM) {
2031 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2032 NULL, tp_len);
2033 if (unlikely(err < 0))
2034 return -EINVAL;
2035 } else if (dev->hard_header_len) {
2036 /* net device doesn't like empty head */
2037 if (unlikely(tp_len <= dev->hard_header_len)) {
2038 pr_err("packet size is too short (%d < %d)\n",
2039 tp_len, dev->hard_header_len);
2040 return -EINVAL;
2041 }
2042
2043 skb_push(skb, dev->hard_header_len);
2044 err = skb_store_bits(skb, 0, data,
2045 dev->hard_header_len);
2046 if (unlikely(err))
2047 return err;
2048
2049 data += dev->hard_header_len;
2050 to_write -= dev->hard_header_len;
2051 }
2052
2053 offset = offset_in_page(data);
2054 len_max = PAGE_SIZE - offset;
2055 len = ((to_write > len_max) ? len_max : to_write);
2056
2057 skb->data_len = to_write;
2058 skb->len += to_write;
2059 skb->truesize += to_write;
2060 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2061
2062 while (likely(to_write)) {
2063 nr_frags = skb_shinfo(skb)->nr_frags;
2064
2065 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2066 pr_err("Packet exceed the number of skb frags(%lu)\n",
2067 MAX_SKB_FRAGS);
2068 return -EFAULT;
2069 }
2070
2071 page = pgv_to_page(data);
2072 data += len;
2073 flush_dcache_page(page);
2074 get_page(page);
2075 skb_fill_page_desc(skb, nr_frags, page, offset, len);
2076 to_write -= len;
2077 offset = 0;
2078 len_max = PAGE_SIZE;
2079 len = ((to_write > len_max) ? len_max : to_write);
2080 }
2081
2082 return tp_len;
2083 }
2084
2085 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2086 {
2087 struct sk_buff *skb;
2088 struct net_device *dev;
2089 __be16 proto;
2090 int err, reserve = 0;
2091 void *ph;
2092 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2093 int tp_len, size_max;
2094 unsigned char *addr;
2095 int len_sum = 0;
2096 int status = TP_STATUS_AVAILABLE;
2097 int hlen, tlen;
2098
2099 mutex_lock(&po->pg_vec_lock);
2100
2101 if (likely(saddr == NULL)) {
2102 dev = packet_cached_dev_get(po);
2103 proto = po->num;
2104 addr = NULL;
2105 } else {
2106 err = -EINVAL;
2107 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2108 goto out;
2109 if (msg->msg_namelen < (saddr->sll_halen
2110 + offsetof(struct sockaddr_ll,
2111 sll_addr)))
2112 goto out;
2113 proto = saddr->sll_protocol;
2114 addr = saddr->sll_addr;
2115 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2116 }
2117
2118 err = -ENXIO;
2119 if (unlikely(dev == NULL))
2120 goto out;
2121 err = -ENETDOWN;
2122 if (unlikely(!(dev->flags & IFF_UP)))
2123 goto out_put;
2124
2125 reserve = dev->hard_header_len;
2126
2127 size_max = po->tx_ring.frame_size
2128 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2129
2130 if (size_max > dev->mtu + reserve)
2131 size_max = dev->mtu + reserve;
2132
2133 do {
2134 ph = packet_current_frame(po, &po->tx_ring,
2135 TP_STATUS_SEND_REQUEST);
2136
2137 if (unlikely(ph == NULL)) {
2138 schedule();
2139 continue;
2140 }
2141
2142 status = TP_STATUS_SEND_REQUEST;
2143 hlen = LL_RESERVED_SPACE(dev);
2144 tlen = dev->needed_tailroom;
2145 skb = sock_alloc_send_skb(&po->sk,
2146 hlen + tlen + sizeof(struct sockaddr_ll),
2147 0, &err);
2148
2149 if (unlikely(skb == NULL))
2150 goto out_status;
2151
2152 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2153 addr, hlen);
2154
2155 if (unlikely(tp_len < 0)) {
2156 if (po->tp_loss) {
2157 __packet_set_status(po, ph,
2158 TP_STATUS_AVAILABLE);
2159 packet_increment_head(&po->tx_ring);
2160 kfree_skb(skb);
2161 continue;
2162 } else {
2163 status = TP_STATUS_WRONG_FORMAT;
2164 err = tp_len;
2165 goto out_status;
2166 }
2167 }
2168
2169 skb->destructor = tpacket_destruct_skb;
2170 __packet_set_status(po, ph, TP_STATUS_SENDING);
2171 atomic_inc(&po->tx_ring.pending);
2172
2173 status = TP_STATUS_SEND_REQUEST;
2174 err = dev_queue_xmit(skb);
2175 if (unlikely(err > 0)) {
2176 err = net_xmit_errno(err);
2177 if (err && __packet_get_status(po, ph) ==
2178 TP_STATUS_AVAILABLE) {
2179 /* skb was destructed already */
2180 skb = NULL;
2181 goto out_status;
2182 }
2183 /*
2184 * skb was dropped but not destructed yet;
2185 * let's treat it like congestion or err < 0
2186 */
2187 err = 0;
2188 }
2189 packet_increment_head(&po->tx_ring);
2190 len_sum += tp_len;
2191 } while (likely((ph != NULL) ||
2192 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2193 (atomic_read(&po->tx_ring.pending))))
2194 );
2195
2196 err = len_sum;
2197 goto out_put;
2198
2199 out_status:
2200 __packet_set_status(po, ph, status);
2201 kfree_skb(skb);
2202 out_put:
2203 dev_put(dev);
2204 out:
2205 mutex_unlock(&po->pg_vec_lock);
2206 return err;
2207 }
2208
2209 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2210 size_t reserve, size_t len,
2211 size_t linear, int noblock,
2212 int *err)
2213 {
2214 struct sk_buff *skb;
2215
2216 /* Under a page? Don't bother with paged skb. */
2217 if (prepad + len < PAGE_SIZE || !linear)
2218 linear = len;
2219
2220 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2221 err);
2222 if (!skb)
2223 return NULL;
2224
2225 skb_reserve(skb, reserve);
2226 skb_put(skb, linear);
2227 skb->data_len = len - linear;
2228 skb->len += len - linear;
2229
2230 return skb;
2231 }
2232
2233 static int packet_snd(struct socket *sock,
2234 struct msghdr *msg, size_t len)
2235 {
2236 struct sock *sk = sock->sk;
2237 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2238 struct sk_buff *skb;
2239 struct net_device *dev;
2240 __be16 proto;
2241 unsigned char *addr;
2242 int err, reserve = 0;
2243 struct virtio_net_hdr vnet_hdr = { 0 };
2244 int offset = 0;
2245 int vnet_hdr_len;
2246 struct packet_sock *po = pkt_sk(sk);
2247 unsigned short gso_type = 0;
2248 int hlen, tlen;
2249 int extra_len = 0;
2250
2251 /*
2252 * Get and verify the address.
2253 */
2254
2255 if (likely(saddr == NULL)) {
2256 dev = packet_cached_dev_get(po);
2257 proto = po->num;
2258 addr = NULL;
2259 } else {
2260 err = -EINVAL;
2261 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2262 goto out;
2263 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2264 goto out;
2265 proto = saddr->sll_protocol;
2266 addr = saddr->sll_addr;
2267 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2268 }
2269
2270 err = -ENXIO;
2271 if (unlikely(dev == NULL))
2272 goto out_unlock;
2273 err = -ENETDOWN;
2274 if (unlikely(!(dev->flags & IFF_UP)))
2275 goto out_unlock;
2276
2277 if (sock->type == SOCK_RAW)
2278 reserve = dev->hard_header_len;
2279 if (po->has_vnet_hdr) {
2280 vnet_hdr_len = sizeof(vnet_hdr);
2281
2282 err = -EINVAL;
2283 if (len < vnet_hdr_len)
2284 goto out_unlock;
2285
2286 len -= vnet_hdr_len;
2287
2288 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2289 vnet_hdr_len);
2290 if (err < 0)
2291 goto out_unlock;
2292
2293 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2294 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2295 vnet_hdr.hdr_len))
2296 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2297 vnet_hdr.csum_offset + 2;
2298
2299 err = -EINVAL;
2300 if (vnet_hdr.hdr_len > len)
2301 goto out_unlock;
2302
2303 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2304 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2305 case VIRTIO_NET_HDR_GSO_TCPV4:
2306 gso_type = SKB_GSO_TCPV4;
2307 break;
2308 case VIRTIO_NET_HDR_GSO_TCPV6:
2309 gso_type = SKB_GSO_TCPV6;
2310 break;
2311 case VIRTIO_NET_HDR_GSO_UDP:
2312 gso_type = SKB_GSO_UDP;
2313 break;
2314 default:
2315 goto out_unlock;
2316 }
2317
2318 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2319 gso_type |= SKB_GSO_TCP_ECN;
2320
2321 if (vnet_hdr.gso_size == 0)
2322 goto out_unlock;
2323
2324 }
2325 }
2326
2327 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2328 if (!netif_supports_nofcs(dev)) {
2329 err = -EPROTONOSUPPORT;
2330 goto out_unlock;
2331 }
2332 extra_len = 4; /* We're doing our own CRC */
2333 }
2334
2335 err = -EMSGSIZE;
2336 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2337 goto out_unlock;
2338
2339 err = -ENOBUFS;
2340 hlen = LL_RESERVED_SPACE(dev);
2341 tlen = dev->needed_tailroom;
2342 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
2343 msg->msg_flags & MSG_DONTWAIT, &err);
2344 if (skb == NULL)
2345 goto out_unlock;
2346
2347 skb_set_network_header(skb, reserve);
2348
2349 err = -EINVAL;
2350 if (sock->type == SOCK_DGRAM &&
2351 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
2352 goto out_free;
2353
2354 /* Returns -EFAULT on error */
2355 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
2356 if (err)
2357 goto out_free;
2358
2359 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
2360
2361 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
2362 /* Earlier code assumed this would be a VLAN pkt,
2363 * double-check this now that we have the actual
2364 * packet in hand.
2365 */
2366 struct ethhdr *ehdr;
2367 skb_reset_mac_header(skb);
2368 ehdr = eth_hdr(skb);
2369 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2370 err = -EMSGSIZE;
2371 goto out_free;
2372 }
2373 }
2374
2375 skb->protocol = proto;
2376 skb->dev = dev;
2377 skb->priority = sk->sk_priority;
2378 skb->mark = sk->sk_mark;
2379
2380 if (po->has_vnet_hdr) {
2381 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2382 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2383 vnet_hdr.csum_offset)) {
2384 err = -EINVAL;
2385 goto out_free;
2386 }
2387 }
2388
2389 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2390 skb_shinfo(skb)->gso_type = gso_type;
2391
2392 /* Header must be checked, and gso_segs computed. */
2393 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2394 skb_shinfo(skb)->gso_segs = 0;
2395
2396 len += vnet_hdr_len;
2397 }
2398
2399 skb_probe_transport_header(skb, reserve);
2400
2401 if (unlikely(extra_len == 4))
2402 skb->no_fcs = 1;
2403
2404 /*
2405 * Now send it
2406 */
2407
2408 err = dev_queue_xmit(skb);
2409 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2410 goto out_unlock;
2411
2412 dev_put(dev);
2413
2414 return len;
2415
2416 out_free:
2417 kfree_skb(skb);
2418 out_unlock:
2419 if (dev)
2420 dev_put(dev);
2421 out:
2422 return err;
2423 }
2424
2425 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2426 struct msghdr *msg, size_t len)
2427 {
2428 struct sock *sk = sock->sk;
2429 struct packet_sock *po = pkt_sk(sk);
2430 if (po->tx_ring.pg_vec)
2431 return tpacket_snd(po, msg);
2432 else
2433 return packet_snd(sock, msg, len);
2434 }
2435
2436 /*
2437 * Close a PACKET socket. This is fairly simple. We immediately go
2438 * to 'closed' state and remove our protocol entry in the device list.
2439 */
2440
2441 static int packet_release(struct socket *sock)
2442 {
2443 struct sock *sk = sock->sk;
2444 struct packet_sock *po;
2445 struct net *net;
2446 union tpacket_req_u req_u;
2447
2448 if (!sk)
2449 return 0;
2450
2451 net = sock_net(sk);
2452 po = pkt_sk(sk);
2453
2454 mutex_lock(&net->packet.sklist_lock);
2455 sk_del_node_init_rcu(sk);
2456 mutex_unlock(&net->packet.sklist_lock);
2457
2458 preempt_disable();
2459 sock_prot_inuse_add(net, sk->sk_prot, -1);
2460 preempt_enable();
2461
2462 spin_lock(&po->bind_lock);
2463 unregister_prot_hook(sk, false);
2464 packet_cached_dev_reset(po);
2465
2466 if (po->prot_hook.dev) {
2467 dev_put(po->prot_hook.dev);
2468 po->prot_hook.dev = NULL;
2469 }
2470 spin_unlock(&po->bind_lock);
2471
2472 packet_flush_mclist(sk);
2473
2474 if (po->rx_ring.pg_vec) {
2475 memset(&req_u, 0, sizeof(req_u));
2476 packet_set_ring(sk, &req_u, 1, 0);
2477 }
2478
2479 if (po->tx_ring.pg_vec) {
2480 memset(&req_u, 0, sizeof(req_u));
2481 packet_set_ring(sk, &req_u, 1, 1);
2482 }
2483
2484 fanout_release(sk);
2485
2486 synchronize_net();
2487 /*
2488 * Now the socket is dead. No more input will appear.
2489 */
2490 sock_orphan(sk);
2491 sock->sk = NULL;
2492
2493 /* Purge queues */
2494
2495 skb_queue_purge(&sk->sk_receive_queue);
2496 sk_refcnt_debug_release(sk);
2497
2498 sock_put(sk);
2499 return 0;
2500 }
2501
2502 /*
2503 * Attach a packet hook.
2504 */
2505
2506 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
2507 {
2508 struct packet_sock *po = pkt_sk(sk);
2509
2510 if (po->fanout) {
2511 if (dev)
2512 dev_put(dev);
2513
2514 return -EINVAL;
2515 }
2516
2517 lock_sock(sk);
2518
2519 spin_lock(&po->bind_lock);
2520 unregister_prot_hook(sk, true);
2521
2522 po->num = protocol;
2523 po->prot_hook.type = protocol;
2524 if (po->prot_hook.dev)
2525 dev_put(po->prot_hook.dev);
2526
2527 po->prot_hook.dev = dev;
2528 po->ifindex = dev ? dev->ifindex : 0;
2529
2530 packet_cached_dev_assign(po, dev);
2531
2532 if (protocol == 0)
2533 goto out_unlock;
2534
2535 if (!dev || (dev->flags & IFF_UP)) {
2536 register_prot_hook(sk);
2537 } else {
2538 sk->sk_err = ENETDOWN;
2539 if (!sock_flag(sk, SOCK_DEAD))
2540 sk->sk_error_report(sk);
2541 }
2542
2543 out_unlock:
2544 spin_unlock(&po->bind_lock);
2545 release_sock(sk);
2546 return 0;
2547 }
2548
2549 /*
2550 * Bind a packet socket to a device
2551 */
2552
2553 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2554 int addr_len)
2555 {
2556 struct sock *sk = sock->sk;
2557 char name[15];
2558 struct net_device *dev;
2559 int err = -ENODEV;
2560
2561 /*
2562 * Check legality
2563 */
2564
2565 if (addr_len != sizeof(struct sockaddr))
2566 return -EINVAL;
2567 strlcpy(name, uaddr->sa_data, sizeof(name));
2568
2569 dev = dev_get_by_name(sock_net(sk), name);
2570 if (dev)
2571 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
2572 return err;
2573 }
2574
2575 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2576 {
2577 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2578 struct sock *sk = sock->sk;
2579 struct net_device *dev = NULL;
2580 int err;
2581
2582
2583 /*
2584 * Check legality
2585 */
2586
2587 if (addr_len < sizeof(struct sockaddr_ll))
2588 return -EINVAL;
2589 if (sll->sll_family != AF_PACKET)
2590 return -EINVAL;
2591
2592 if (sll->sll_ifindex) {
2593 err = -ENODEV;
2594 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
2595 if (dev == NULL)
2596 goto out;
2597 }
2598 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
2599
2600 out:
2601 return err;
2602 }
2603
2604 static struct proto packet_proto = {
2605 .name = "PACKET",
2606 .owner = THIS_MODULE,
2607 .obj_size = sizeof(struct packet_sock),
2608 };
2609
2610 /*
2611 * Create a packet of type SOCK_PACKET.
2612 */
2613
2614 static int packet_create(struct net *net, struct socket *sock, int protocol,
2615 int kern)
2616 {
2617 struct sock *sk;
2618 struct packet_sock *po;
2619 __be16 proto = (__force __be16)protocol; /* weird, but documented */
2620 int err;
2621
2622 if (!ns_capable(net->user_ns, CAP_NET_RAW))
2623 return -EPERM;
2624 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2625 sock->type != SOCK_PACKET)
2626 return -ESOCKTNOSUPPORT;
2627
2628 sock->state = SS_UNCONNECTED;
2629
2630 err = -ENOBUFS;
2631 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
2632 if (sk == NULL)
2633 goto out;
2634
2635 sock->ops = &packet_ops;
2636 if (sock->type == SOCK_PACKET)
2637 sock->ops = &packet_ops_spkt;
2638
2639 sock_init_data(sock, sk);
2640
2641 po = pkt_sk(sk);
2642 sk->sk_family = PF_PACKET;
2643 po->num = proto;
2644
2645 packet_cached_dev_reset(po);
2646
2647 sk->sk_destruct = packet_sock_destruct;
2648 sk_refcnt_debug_inc(sk);
2649
2650 /*
2651 * Attach a protocol block
2652 */
2653
2654 spin_lock_init(&po->bind_lock);
2655 mutex_init(&po->pg_vec_lock);
2656 po->prot_hook.func = packet_rcv;
2657
2658 if (sock->type == SOCK_PACKET)
2659 po->prot_hook.func = packet_rcv_spkt;
2660
2661 po->prot_hook.af_packet_priv = sk;
2662
2663 if (proto) {
2664 po->prot_hook.type = proto;
2665 register_prot_hook(sk);
2666 }
2667
2668 mutex_lock(&net->packet.sklist_lock);
2669 sk_add_node_rcu(sk, &net->packet.sklist);
2670 mutex_unlock(&net->packet.sklist_lock);
2671
2672 preempt_disable();
2673 sock_prot_inuse_add(net, &packet_proto, 1);
2674 preempt_enable();
2675
2676 return 0;
2677 out:
2678 return err;
2679 }
2680
2681 static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2682 {
2683 struct sock_exterr_skb *serr;
2684 struct sk_buff *skb, *skb2;
2685 int copied, err;
2686
2687 err = -EAGAIN;
2688 skb = skb_dequeue(&sk->sk_error_queue);
2689 if (skb == NULL)
2690 goto out;
2691
2692 copied = skb->len;
2693 if (copied > len) {
2694 msg->msg_flags |= MSG_TRUNC;
2695 copied = len;
2696 }
2697 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2698 if (err)
2699 goto out_free_skb;
2700
2701 sock_recv_timestamp(msg, sk, skb);
2702
2703 serr = SKB_EXT_ERR(skb);
2704 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2705 sizeof(serr->ee), &serr->ee);
2706
2707 msg->msg_flags |= MSG_ERRQUEUE;
2708 err = copied;
2709
2710 /* Reset and regenerate socket error */
2711 spin_lock_bh(&sk->sk_error_queue.lock);
2712 sk->sk_err = 0;
2713 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2714 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2715 spin_unlock_bh(&sk->sk_error_queue.lock);
2716 sk->sk_error_report(sk);
2717 } else
2718 spin_unlock_bh(&sk->sk_error_queue.lock);
2719
2720 out_free_skb:
2721 kfree_skb(skb);
2722 out:
2723 return err;
2724 }
2725
2726 /*
2727 * Pull a packet from our receive queue and hand it to the user.
2728 * If necessary we block.
2729 */
2730
2731 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2732 struct msghdr *msg, size_t len, int flags)
2733 {
2734 struct sock *sk = sock->sk;
2735 struct sk_buff *skb;
2736 int copied, err;
2737 int vnet_hdr_len = 0;
2738
2739 err = -EINVAL;
2740 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
2741 goto out;
2742
2743 #if 0
2744 /* What error should we return now? EUNATTACH? */
2745 if (pkt_sk(sk)->ifindex < 0)
2746 return -ENODEV;
2747 #endif
2748
2749 if (flags & MSG_ERRQUEUE) {
2750 err = packet_recv_error(sk, msg, len);
2751 goto out;
2752 }
2753
2754 /*
2755 * Call the generic datagram receiver. This handles all sorts
2756 * of horrible races and re-entrancy so we can forget about it
2757 * in the protocol layers.
2758 *
2759 * Now it will return ENETDOWN, if device have just gone down,
2760 * but then it will block.
2761 */
2762
2763 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
2764
2765 /*
2766 * An error occurred so return it. Because skb_recv_datagram()
2767 * handles the blocking we don't see and worry about blocking
2768 * retries.
2769 */
2770
2771 if (skb == NULL)
2772 goto out;
2773
2774 if (pkt_sk(sk)->has_vnet_hdr) {
2775 struct virtio_net_hdr vnet_hdr = { 0 };
2776
2777 err = -EINVAL;
2778 vnet_hdr_len = sizeof(vnet_hdr);
2779 if (len < vnet_hdr_len)
2780 goto out_free;
2781
2782 len -= vnet_hdr_len;
2783
2784 if (skb_is_gso(skb)) {
2785 struct skb_shared_info *sinfo = skb_shinfo(skb);
2786
2787 /* This is a hint as to how much should be linear. */
2788 vnet_hdr.hdr_len = skb_headlen(skb);
2789 vnet_hdr.gso_size = sinfo->gso_size;
2790 if (sinfo->gso_type & SKB_GSO_TCPV4)
2791 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2792 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2793 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2794 else if (sinfo->gso_type & SKB_GSO_UDP)
2795 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2796 else if (sinfo->gso_type & SKB_GSO_FCOE)
2797 goto out_free;
2798 else
2799 BUG();
2800 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2801 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2802 } else
2803 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2804
2805 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2806 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
2807 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
2808 vnet_hdr.csum_offset = skb->csum_offset;
2809 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2810 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
2811 } /* else everything is zero */
2812
2813 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2814 vnet_hdr_len);
2815 if (err < 0)
2816 goto out_free;
2817 }
2818
2819 /* You lose any data beyond the buffer you gave. If it worries
2820 * a user program they can ask the device for its MTU
2821 * anyway.
2822 */
2823 copied = skb->len;
2824 if (copied > len) {
2825 copied = len;
2826 msg->msg_flags |= MSG_TRUNC;
2827 }
2828
2829 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2830 if (err)
2831 goto out_free;
2832
2833 sock_recv_ts_and_drops(msg, sk, skb);
2834
2835 if (msg->msg_name) {
2836 /* If the address length field is there to be filled
2837 * in, we fill it in now.
2838 */
2839 if (sock->type == SOCK_PACKET) {
2840 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2841 } else {
2842 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2843 msg->msg_namelen = sll->sll_halen +
2844 offsetof(struct sockaddr_ll, sll_addr);
2845 }
2846 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2847 msg->msg_namelen);
2848 }
2849
2850 if (pkt_sk(sk)->auxdata) {
2851 struct tpacket_auxdata aux;
2852
2853 aux.tp_status = TP_STATUS_USER;
2854 if (skb->ip_summed == CHECKSUM_PARTIAL)
2855 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2856 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2857 aux.tp_snaplen = skb->len;
2858 aux.tp_mac = 0;
2859 aux.tp_net = skb_network_offset(skb);
2860 if (vlan_tx_tag_present(skb)) {
2861 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2862 aux.tp_status |= TP_STATUS_VLAN_VALID;
2863 } else {
2864 aux.tp_vlan_tci = 0;
2865 }
2866 aux.tp_padding = 0;
2867 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
2868 }
2869
2870 /*
2871 * Free or return the buffer as appropriate. Again this
2872 * hides all the races and re-entrancy issues from us.
2873 */
2874 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
2875
2876 out_free:
2877 skb_free_datagram(sk, skb);
2878 out:
2879 return err;
2880 }
2881
2882 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2883 int *uaddr_len, int peer)
2884 {
2885 struct net_device *dev;
2886 struct sock *sk = sock->sk;
2887
2888 if (peer)
2889 return -EOPNOTSUPP;
2890
2891 uaddr->sa_family = AF_PACKET;
2892 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
2893 rcu_read_lock();
2894 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2895 if (dev)
2896 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
2897 rcu_read_unlock();
2898 *uaddr_len = sizeof(*uaddr);
2899
2900 return 0;
2901 }
2902
2903 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2904 int *uaddr_len, int peer)
2905 {
2906 struct net_device *dev;
2907 struct sock *sk = sock->sk;
2908 struct packet_sock *po = pkt_sk(sk);
2909 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
2910
2911 if (peer)
2912 return -EOPNOTSUPP;
2913
2914 sll->sll_family = AF_PACKET;
2915 sll->sll_ifindex = po->ifindex;
2916 sll->sll_protocol = po->num;
2917 sll->sll_pkttype = 0;
2918 rcu_read_lock();
2919 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
2920 if (dev) {
2921 sll->sll_hatype = dev->type;
2922 sll->sll_halen = dev->addr_len;
2923 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
2924 } else {
2925 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2926 sll->sll_halen = 0;
2927 }
2928 rcu_read_unlock();
2929 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
2930
2931 return 0;
2932 }
2933
2934 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2935 int what)
2936 {
2937 switch (i->type) {
2938 case PACKET_MR_MULTICAST:
2939 if (i->alen != dev->addr_len)
2940 return -EINVAL;
2941 if (what > 0)
2942 return dev_mc_add(dev, i->addr);
2943 else
2944 return dev_mc_del(dev, i->addr);
2945 break;
2946 case PACKET_MR_PROMISC:
2947 return dev_set_promiscuity(dev, what);
2948 break;
2949 case PACKET_MR_ALLMULTI:
2950 return dev_set_allmulti(dev, what);
2951 break;
2952 case PACKET_MR_UNICAST:
2953 if (i->alen != dev->addr_len)
2954 return -EINVAL;
2955 if (what > 0)
2956 return dev_uc_add(dev, i->addr);
2957 else
2958 return dev_uc_del(dev, i->addr);
2959 break;
2960 default:
2961 break;
2962 }
2963 return 0;
2964 }
2965
2966 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2967 {
2968 for ( ; i; i = i->next) {
2969 if (i->ifindex == dev->ifindex)
2970 packet_dev_mc(dev, i, what);
2971 }
2972 }
2973
2974 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
2975 {
2976 struct packet_sock *po = pkt_sk(sk);
2977 struct packet_mclist *ml, *i;
2978 struct net_device *dev;
2979 int err;
2980
2981 rtnl_lock();
2982
2983 err = -ENODEV;
2984 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
2985 if (!dev)
2986 goto done;
2987
2988 err = -EINVAL;
2989 if (mreq->mr_alen > dev->addr_len)
2990 goto done;
2991
2992 err = -ENOBUFS;
2993 i = kmalloc(sizeof(*i), GFP_KERNEL);
2994 if (i == NULL)
2995 goto done;
2996
2997 err = 0;
2998 for (ml = po->mclist; ml; ml = ml->next) {
2999 if (ml->ifindex == mreq->mr_ifindex &&
3000 ml->type == mreq->mr_type &&
3001 ml->alen == mreq->mr_alen &&
3002 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3003 ml->count++;
3004 /* Free the new element ... */
3005 kfree(i);
3006 goto done;
3007 }
3008 }
3009
3010 i->type = mreq->mr_type;
3011 i->ifindex = mreq->mr_ifindex;
3012 i->alen = mreq->mr_alen;
3013 memcpy(i->addr, mreq->mr_address, i->alen);
3014 i->count = 1;
3015 i->next = po->mclist;
3016 po->mclist = i;
3017 err = packet_dev_mc(dev, i, 1);
3018 if (err) {
3019 po->mclist = i->next;
3020 kfree(i);
3021 }
3022
3023 done:
3024 rtnl_unlock();
3025 return err;
3026 }
3027
3028 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3029 {
3030 struct packet_mclist *ml, **mlp;
3031
3032 rtnl_lock();
3033
3034 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3035 if (ml->ifindex == mreq->mr_ifindex &&
3036 ml->type == mreq->mr_type &&
3037 ml->alen == mreq->mr_alen &&
3038 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3039 if (--ml->count == 0) {
3040 struct net_device *dev;
3041 *mlp = ml->next;
3042 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3043 if (dev)
3044 packet_dev_mc(dev, ml, -1);
3045 kfree(ml);
3046 }
3047 rtnl_unlock();
3048 return 0;
3049 }
3050 }
3051 rtnl_unlock();
3052 return -EADDRNOTAVAIL;
3053 }
3054
3055 static void packet_flush_mclist(struct sock *sk)
3056 {
3057 struct packet_sock *po = pkt_sk(sk);
3058 struct packet_mclist *ml;
3059
3060 if (!po->mclist)
3061 return;
3062
3063 rtnl_lock();
3064 while ((ml = po->mclist) != NULL) {
3065 struct net_device *dev;
3066
3067 po->mclist = ml->next;
3068 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3069 if (dev != NULL)
3070 packet_dev_mc(dev, ml, -1);
3071 kfree(ml);
3072 }
3073 rtnl_unlock();
3074 }
3075
3076 static int
3077 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3078 {
3079 struct sock *sk = sock->sk;
3080 struct packet_sock *po = pkt_sk(sk);
3081 int ret;
3082
3083 if (level != SOL_PACKET)
3084 return -ENOPROTOOPT;
3085
3086 switch (optname) {
3087 case PACKET_ADD_MEMBERSHIP:
3088 case PACKET_DROP_MEMBERSHIP:
3089 {
3090 struct packet_mreq_max mreq;
3091 int len = optlen;
3092 memset(&mreq, 0, sizeof(mreq));
3093 if (len < sizeof(struct packet_mreq))
3094 return -EINVAL;
3095 if (len > sizeof(mreq))
3096 len = sizeof(mreq);
3097 if (copy_from_user(&mreq, optval, len))
3098 return -EFAULT;
3099 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3100 return -EINVAL;
3101 if (optname == PACKET_ADD_MEMBERSHIP)
3102 ret = packet_mc_add(sk, &mreq);
3103 else
3104 ret = packet_mc_drop(sk, &mreq);
3105 return ret;
3106 }
3107
3108 case PACKET_RX_RING:
3109 case PACKET_TX_RING:
3110 {
3111 union tpacket_req_u req_u;
3112 int len;
3113
3114 switch (po->tp_version) {
3115 case TPACKET_V1:
3116 case TPACKET_V2:
3117 len = sizeof(req_u.req);
3118 break;
3119 case TPACKET_V3:
3120 default:
3121 len = sizeof(req_u.req3);
3122 break;
3123 }
3124 if (optlen < len)
3125 return -EINVAL;
3126 if (pkt_sk(sk)->has_vnet_hdr)
3127 return -EINVAL;
3128 if (copy_from_user(&req_u.req, optval, len))
3129 return -EFAULT;
3130 return packet_set_ring(sk, &req_u, 0,
3131 optname == PACKET_TX_RING);
3132 }
3133 case PACKET_COPY_THRESH:
3134 {
3135 int val;
3136
3137 if (optlen != sizeof(val))
3138 return -EINVAL;
3139 if (copy_from_user(&val, optval, sizeof(val)))
3140 return -EFAULT;
3141
3142 pkt_sk(sk)->copy_thresh = val;
3143 return 0;
3144 }
3145 case PACKET_VERSION:
3146 {
3147 int val;
3148
3149 if (optlen != sizeof(val))
3150 return -EINVAL;
3151 if (copy_from_user(&val, optval, sizeof(val)))
3152 return -EFAULT;
3153 switch (val) {
3154 case TPACKET_V1:
3155 case TPACKET_V2:
3156 case TPACKET_V3:
3157 break;
3158 default:
3159 return -EINVAL;
3160 }
3161 lock_sock(sk);
3162 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3163 ret = -EBUSY;
3164 } else {
3165 po->tp_version = val;
3166 ret = 0;
3167 }
3168 release_sock(sk);
3169 return ret;
3170 }
3171 case PACKET_RESERVE:
3172 {
3173 unsigned int val;
3174
3175 if (optlen != sizeof(val))
3176 return -EINVAL;
3177 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3178 return -EBUSY;
3179 if (copy_from_user(&val, optval, sizeof(val)))
3180 return -EFAULT;
3181 if (val > INT_MAX)
3182 return -EINVAL;
3183 po->tp_reserve = val;
3184 return 0;
3185 }
3186 case PACKET_LOSS:
3187 {
3188 unsigned int val;
3189
3190 if (optlen != sizeof(val))
3191 return -EINVAL;
3192 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3193 return -EBUSY;
3194 if (copy_from_user(&val, optval, sizeof(val)))
3195 return -EFAULT;
3196 po->tp_loss = !!val;
3197 return 0;
3198 }
3199 case PACKET_AUXDATA:
3200 {
3201 int val;
3202
3203 if (optlen < sizeof(val))
3204 return -EINVAL;
3205 if (copy_from_user(&val, optval, sizeof(val)))
3206 return -EFAULT;
3207
3208 po->auxdata = !!val;
3209 return 0;
3210 }
3211 case PACKET_ORIGDEV:
3212 {
3213 int val;
3214
3215 if (optlen < sizeof(val))
3216 return -EINVAL;
3217 if (copy_from_user(&val, optval, sizeof(val)))
3218 return -EFAULT;
3219
3220 po->origdev = !!val;
3221 return 0;
3222 }
3223 case PACKET_VNET_HDR:
3224 {
3225 int val;
3226
3227 if (sock->type != SOCK_RAW)
3228 return -EINVAL;
3229 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3230 return -EBUSY;
3231 if (optlen < sizeof(val))
3232 return -EINVAL;
3233 if (copy_from_user(&val, optval, sizeof(val)))
3234 return -EFAULT;
3235
3236 po->has_vnet_hdr = !!val;
3237 return 0;
3238 }
3239 case PACKET_TIMESTAMP:
3240 {
3241 int val;
3242
3243 if (optlen != sizeof(val))
3244 return -EINVAL;
3245 if (copy_from_user(&val, optval, sizeof(val)))
3246 return -EFAULT;
3247
3248 po->tp_tstamp = val;
3249 return 0;
3250 }
3251 case PACKET_FANOUT:
3252 {
3253 int val;
3254
3255 if (optlen != sizeof(val))
3256 return -EINVAL;
3257 if (copy_from_user(&val, optval, sizeof(val)))
3258 return -EFAULT;
3259
3260 return fanout_add(sk, val & 0xffff, val >> 16);
3261 }
3262 case PACKET_TX_HAS_OFF:
3263 {
3264 unsigned int val;
3265
3266 if (optlen != sizeof(val))
3267 return -EINVAL;
3268 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3269 return -EBUSY;
3270 if (copy_from_user(&val, optval, sizeof(val)))
3271 return -EFAULT;
3272 po->tp_tx_has_off = !!val;
3273 return 0;
3274 }
3275 default:
3276 return -ENOPROTOOPT;
3277 }
3278 }
3279
3280 static int packet_getsockopt(struct socket *sock, int level, int optname,
3281 char __user *optval, int __user *optlen)
3282 {
3283 int len;
3284 int val, lv = sizeof(val);
3285 struct sock *sk = sock->sk;
3286 struct packet_sock *po = pkt_sk(sk);
3287 void *data = &val;
3288 union tpacket_stats_u st;
3289
3290 if (level != SOL_PACKET)
3291 return -ENOPROTOOPT;
3292
3293 if (get_user(len, optlen))
3294 return -EFAULT;
3295
3296 if (len < 0)
3297 return -EINVAL;
3298
3299 switch (optname) {
3300 case PACKET_STATISTICS:
3301 spin_lock_bh(&sk->sk_receive_queue.lock);
3302 memcpy(&st, &po->stats, sizeof(st));
3303 memset(&po->stats, 0, sizeof(po->stats));
3304 spin_unlock_bh(&sk->sk_receive_queue.lock);
3305
3306 if (po->tp_version == TPACKET_V3) {
3307 lv = sizeof(struct tpacket_stats_v3);
3308 st.stats3.tp_packets += st.stats3.tp_drops;
3309 data = &st.stats3;
3310 } else {
3311 lv = sizeof(struct tpacket_stats);
3312 st.stats1.tp_packets += st.stats1.tp_drops;
3313 data = &st.stats1;
3314 }
3315
3316 break;
3317 case PACKET_AUXDATA:
3318 val = po->auxdata;
3319 break;
3320 case PACKET_ORIGDEV:
3321 val = po->origdev;
3322 break;
3323 case PACKET_VNET_HDR:
3324 val = po->has_vnet_hdr;
3325 break;
3326 case PACKET_VERSION:
3327 val = po->tp_version;
3328 break;
3329 case PACKET_HDRLEN:
3330 if (len > sizeof(int))
3331 len = sizeof(int);
3332 if (copy_from_user(&val, optval, len))
3333 return -EFAULT;
3334 switch (val) {
3335 case TPACKET_V1:
3336 val = sizeof(struct tpacket_hdr);
3337 break;
3338 case TPACKET_V2:
3339 val = sizeof(struct tpacket2_hdr);
3340 break;
3341 case TPACKET_V3:
3342 val = sizeof(struct tpacket3_hdr);
3343 break;
3344 default:
3345 return -EINVAL;
3346 }
3347 break;
3348 case PACKET_RESERVE:
3349 val = po->tp_reserve;
3350 break;
3351 case PACKET_LOSS:
3352 val = po->tp_loss;
3353 break;
3354 case PACKET_TIMESTAMP:
3355 val = po->tp_tstamp;
3356 break;
3357 case PACKET_FANOUT:
3358 val = (po->fanout ?
3359 ((u32)po->fanout->id |
3360 ((u32)po->fanout->type << 16) |
3361 ((u32)po->fanout->flags << 24)) :
3362 0);
3363 break;
3364 case PACKET_TX_HAS_OFF:
3365 val = po->tp_tx_has_off;
3366 break;
3367 default:
3368 return -ENOPROTOOPT;
3369 }
3370
3371 if (len > lv)
3372 len = lv;
3373 if (put_user(len, optlen))
3374 return -EFAULT;
3375 if (copy_to_user(optval, data, len))
3376 return -EFAULT;
3377 return 0;
3378 }
3379
3380
3381 static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3382 {
3383 struct sock *sk;
3384 struct net_device *dev = data;
3385 struct net *net = dev_net(dev);
3386
3387 rcu_read_lock();
3388 sk_for_each_rcu(sk, &net->packet.sklist) {
3389 struct packet_sock *po = pkt_sk(sk);
3390
3391 switch (msg) {
3392 case NETDEV_UNREGISTER:
3393 if (po->mclist)
3394 packet_dev_mclist(dev, po->mclist, -1);
3395 /* fallthrough */
3396
3397 case NETDEV_DOWN:
3398 if (dev->ifindex == po->ifindex) {
3399 spin_lock(&po->bind_lock);
3400 if (po->running) {
3401 __unregister_prot_hook(sk, false);
3402 sk->sk_err = ENETDOWN;
3403 if (!sock_flag(sk, SOCK_DEAD))
3404 sk->sk_error_report(sk);
3405 }
3406 if (msg == NETDEV_UNREGISTER) {
3407 packet_cached_dev_reset(po);
3408 po->ifindex = -1;
3409 if (po->prot_hook.dev)
3410 dev_put(po->prot_hook.dev);
3411 po->prot_hook.dev = NULL;
3412 }
3413 spin_unlock(&po->bind_lock);
3414 }
3415 break;
3416 case NETDEV_UP:
3417 if (dev->ifindex == po->ifindex) {
3418 spin_lock(&po->bind_lock);
3419 if (po->num)
3420 register_prot_hook(sk);
3421 spin_unlock(&po->bind_lock);
3422 }
3423 break;
3424 }
3425 }
3426 rcu_read_unlock();
3427 return NOTIFY_DONE;
3428 }
3429
3430
3431 static int packet_ioctl(struct socket *sock, unsigned int cmd,
3432 unsigned long arg)
3433 {
3434 struct sock *sk = sock->sk;
3435
3436 switch (cmd) {
3437 case SIOCOUTQ:
3438 {
3439 int amount = sk_wmem_alloc_get(sk);
3440
3441 return put_user(amount, (int __user *)arg);
3442 }
3443 case SIOCINQ:
3444 {
3445 struct sk_buff *skb;
3446 int amount = 0;
3447
3448 spin_lock_bh(&sk->sk_receive_queue.lock);
3449 skb = skb_peek(&sk->sk_receive_queue);
3450 if (skb)
3451 amount = skb->len;
3452 spin_unlock_bh(&sk->sk_receive_queue.lock);
3453 return put_user(amount, (int __user *)arg);
3454 }
3455 case SIOCGSTAMP:
3456 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3457 case SIOCGSTAMPNS:
3458 return sock_get_timestampns(sk, (struct timespec __user *)arg);
3459
3460 #ifdef CONFIG_INET
3461 case SIOCADDRT:
3462 case SIOCDELRT:
3463 case SIOCDARP:
3464 case SIOCGARP:
3465 case SIOCSARP:
3466 case SIOCGIFADDR:
3467 case SIOCSIFADDR:
3468 case SIOCGIFBRDADDR:
3469 case SIOCSIFBRDADDR:
3470 case SIOCGIFNETMASK:
3471 case SIOCSIFNETMASK:
3472 case SIOCGIFDSTADDR:
3473 case SIOCSIFDSTADDR:
3474 case SIOCSIFFLAGS:
3475 return inet_dgram_ops.ioctl(sock, cmd, arg);
3476 #endif
3477
3478 default:
3479 return -ENOIOCTLCMD;
3480 }
3481 return 0;
3482 }
3483
3484 static unsigned int packet_poll(struct file *file, struct socket *sock,
3485 poll_table *wait)
3486 {
3487 struct sock *sk = sock->sk;
3488 struct packet_sock *po = pkt_sk(sk);
3489 unsigned int mask = datagram_poll(file, sock, wait);
3490
3491 spin_lock_bh(&sk->sk_receive_queue.lock);
3492 if (po->rx_ring.pg_vec) {
3493 if (!packet_previous_rx_frame(po, &po->rx_ring,
3494 TP_STATUS_KERNEL))
3495 mask |= POLLIN | POLLRDNORM;
3496 }
3497 spin_unlock_bh(&sk->sk_receive_queue.lock);
3498 spin_lock_bh(&sk->sk_write_queue.lock);
3499 if (po->tx_ring.pg_vec) {
3500 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3501 mask |= POLLOUT | POLLWRNORM;
3502 }
3503 spin_unlock_bh(&sk->sk_write_queue.lock);
3504 return mask;
3505 }
3506
3507
3508 /* Dirty? Well, I still did not learn better way to account
3509 * for user mmaps.
3510 */
3511
3512 static void packet_mm_open(struct vm_area_struct *vma)
3513 {
3514 struct file *file = vma->vm_file;
3515 struct socket *sock = file->private_data;
3516 struct sock *sk = sock->sk;
3517
3518 if (sk)
3519 atomic_inc(&pkt_sk(sk)->mapped);
3520 }
3521
3522 static void packet_mm_close(struct vm_area_struct *vma)
3523 {
3524 struct file *file = vma->vm_file;
3525 struct socket *sock = file->private_data;
3526 struct sock *sk = sock->sk;
3527
3528 if (sk)
3529 atomic_dec(&pkt_sk(sk)->mapped);
3530 }
3531
3532 static const struct vm_operations_struct packet_mmap_ops = {
3533 .open = packet_mm_open,
3534 .close = packet_mm_close,
3535 };
3536
3537 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3538 unsigned int len)
3539 {
3540 int i;
3541
3542 for (i = 0; i < len; i++) {
3543 if (likely(pg_vec[i].buffer)) {
3544 if (is_vmalloc_addr(pg_vec[i].buffer))
3545 vfree(pg_vec[i].buffer);
3546 else
3547 free_pages((unsigned long)pg_vec[i].buffer,
3548 order);
3549 pg_vec[i].buffer = NULL;
3550 }
3551 }
3552 kfree(pg_vec);
3553 }
3554
3555 static char *alloc_one_pg_vec_page(unsigned long order)
3556 {
3557 char *buffer = NULL;
3558 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3559 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3560
3561 buffer = (char *) __get_free_pages(gfp_flags, order);
3562
3563 if (buffer)
3564 return buffer;
3565
3566 /*
3567 * __get_free_pages failed, fall back to vmalloc
3568 */
3569 buffer = vzalloc((1 << order) * PAGE_SIZE);
3570
3571 if (buffer)
3572 return buffer;
3573
3574 /*
3575 * vmalloc failed, lets dig into swap here
3576 */
3577 gfp_flags &= ~__GFP_NORETRY;
3578 buffer = (char *)__get_free_pages(gfp_flags, order);
3579 if (buffer)
3580 return buffer;
3581
3582 /*
3583 * complete and utter failure
3584 */
3585 return NULL;
3586 }
3587
3588 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
3589 {
3590 unsigned int block_nr = req->tp_block_nr;
3591 struct pgv *pg_vec;
3592 int i;
3593
3594 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
3595 if (unlikely(!pg_vec))
3596 goto out;
3597
3598 for (i = 0; i < block_nr; i++) {
3599 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
3600 if (unlikely(!pg_vec[i].buffer))
3601 goto out_free_pgvec;
3602 }
3603
3604 out:
3605 return pg_vec;
3606
3607 out_free_pgvec:
3608 free_pg_vec(pg_vec, order, block_nr);
3609 pg_vec = NULL;
3610 goto out;
3611 }
3612
3613 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
3614 int closing, int tx_ring)
3615 {
3616 struct pgv *pg_vec = NULL;
3617 struct packet_sock *po = pkt_sk(sk);
3618 int was_running, order = 0;
3619 struct packet_ring_buffer *rb;
3620 struct sk_buff_head *rb_queue;
3621 __be16 num;
3622 int err = -EINVAL;
3623 /* Added to avoid minimal code churn */
3624 struct tpacket_req *req = &req_u->req;
3625
3626 lock_sock(sk);
3627 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3628 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3629 WARN(1, "Tx-ring is not supported.\n");
3630 goto out;
3631 }
3632
3633 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3634 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3635
3636 err = -EBUSY;
3637 if (!closing) {
3638 if (atomic_read(&po->mapped))
3639 goto out;
3640 if (atomic_read(&rb->pending))
3641 goto out;
3642 }
3643
3644 if (req->tp_block_nr) {
3645 /* Sanity tests and some calculations */
3646 err = -EBUSY;
3647 if (unlikely(rb->pg_vec))
3648 goto out;
3649
3650 switch (po->tp_version) {
3651 case TPACKET_V1:
3652 po->tp_hdrlen = TPACKET_HDRLEN;
3653 break;
3654 case TPACKET_V2:
3655 po->tp_hdrlen = TPACKET2_HDRLEN;
3656 break;
3657 case TPACKET_V3:
3658 po->tp_hdrlen = TPACKET3_HDRLEN;
3659 break;
3660 }
3661
3662 err = -EINVAL;
3663 if (unlikely((int)req->tp_block_size <= 0))
3664 goto out;
3665 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
3666 goto out;
3667 if (po->tp_version >= TPACKET_V3 &&
3668 (int)(req->tp_block_size -
3669 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3670 goto out;
3671 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
3672 po->tp_reserve))
3673 goto out;
3674 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
3675 goto out;
3676
3677 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3678 if (unlikely(rb->frames_per_block <= 0))
3679 goto out;
3680 if (unlikely(req->tp_block_size > UINT_MAX / req->tp_block_nr))
3681 goto out;
3682 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3683 req->tp_frame_nr))
3684 goto out;
3685
3686 err = -ENOMEM;
3687 order = get_order(req->tp_block_size);
3688 pg_vec = alloc_pg_vec(req, order);
3689 if (unlikely(!pg_vec))
3690 goto out;
3691 switch (po->tp_version) {
3692 case TPACKET_V3:
3693 /* Transmit path is not supported. We checked
3694 * it above but just being paranoid
3695 */
3696 if (!tx_ring)
3697 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3698 break;
3699 default:
3700 break;
3701 }
3702 }
3703 /* Done */
3704 else {
3705 err = -EINVAL;
3706 if (unlikely(req->tp_frame_nr))
3707 goto out;
3708 }
3709
3710
3711 /* Detach socket from network */
3712 spin_lock(&po->bind_lock);
3713 was_running = po->running;
3714 num = po->num;
3715 if (was_running) {
3716 po->num = 0;
3717 __unregister_prot_hook(sk, false);
3718 }
3719 spin_unlock(&po->bind_lock);
3720
3721 synchronize_net();
3722
3723 err = -EBUSY;
3724 mutex_lock(&po->pg_vec_lock);
3725 if (closing || atomic_read(&po->mapped) == 0) {
3726 err = 0;
3727 spin_lock_bh(&rb_queue->lock);
3728 swap(rb->pg_vec, pg_vec);
3729 rb->frame_max = (req->tp_frame_nr - 1);
3730 rb->head = 0;
3731 rb->frame_size = req->tp_frame_size;
3732 spin_unlock_bh(&rb_queue->lock);
3733
3734 swap(rb->pg_vec_order, order);
3735 swap(rb->pg_vec_len, req->tp_block_nr);
3736
3737 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3738 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3739 tpacket_rcv : packet_rcv;
3740 skb_queue_purge(rb_queue);
3741 if (atomic_read(&po->mapped))
3742 pr_err("packet_mmap: vma is busy: %d\n",
3743 atomic_read(&po->mapped));
3744 }
3745 mutex_unlock(&po->pg_vec_lock);
3746
3747 spin_lock(&po->bind_lock);
3748 if (was_running) {
3749 po->num = num;
3750 register_prot_hook(sk);
3751 }
3752 spin_unlock(&po->bind_lock);
3753 if (closing && (po->tp_version > TPACKET_V2)) {
3754 /* Because we don't support block-based V3 on tx-ring */
3755 if (!tx_ring)
3756 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3757 }
3758
3759 if (pg_vec)
3760 free_pg_vec(pg_vec, order, req->tp_block_nr);
3761 out:
3762 release_sock(sk);
3763 return err;
3764 }
3765
3766 static int packet_mmap(struct file *file, struct socket *sock,
3767 struct vm_area_struct *vma)
3768 {
3769 struct sock *sk = sock->sk;
3770 struct packet_sock *po = pkt_sk(sk);
3771 unsigned long size, expected_size;
3772 struct packet_ring_buffer *rb;
3773 unsigned long start;
3774 int err = -EINVAL;
3775 int i;
3776
3777 if (vma->vm_pgoff)
3778 return -EINVAL;
3779
3780 mutex_lock(&po->pg_vec_lock);
3781
3782 expected_size = 0;
3783 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3784 if (rb->pg_vec) {
3785 expected_size += rb->pg_vec_len
3786 * rb->pg_vec_pages
3787 * PAGE_SIZE;
3788 }
3789 }
3790
3791 if (expected_size == 0)
3792 goto out;
3793
3794 size = vma->vm_end - vma->vm_start;
3795 if (size != expected_size)
3796 goto out;
3797
3798 start = vma->vm_start;
3799 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3800 if (rb->pg_vec == NULL)
3801 continue;
3802
3803 for (i = 0; i < rb->pg_vec_len; i++) {
3804 struct page *page;
3805 void *kaddr = rb->pg_vec[i].buffer;
3806 int pg_num;
3807
3808 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3809 page = pgv_to_page(kaddr);
3810 err = vm_insert_page(vma, start, page);
3811 if (unlikely(err))
3812 goto out;
3813 start += PAGE_SIZE;
3814 kaddr += PAGE_SIZE;
3815 }
3816 }
3817 }
3818
3819 atomic_inc(&po->mapped);
3820 vma->vm_ops = &packet_mmap_ops;
3821 err = 0;
3822
3823 out:
3824 mutex_unlock(&po->pg_vec_lock);
3825 return err;
3826 }
3827
3828 static const struct proto_ops packet_ops_spkt = {
3829 .family = PF_PACKET,
3830 .owner = THIS_MODULE,
3831 .release = packet_release,
3832 .bind = packet_bind_spkt,
3833 .connect = sock_no_connect,
3834 .socketpair = sock_no_socketpair,
3835 .accept = sock_no_accept,
3836 .getname = packet_getname_spkt,
3837 .poll = datagram_poll,
3838 .ioctl = packet_ioctl,
3839 .listen = sock_no_listen,
3840 .shutdown = sock_no_shutdown,
3841 .setsockopt = sock_no_setsockopt,
3842 .getsockopt = sock_no_getsockopt,
3843 .sendmsg = packet_sendmsg_spkt,
3844 .recvmsg = packet_recvmsg,
3845 .mmap = sock_no_mmap,
3846 .sendpage = sock_no_sendpage,
3847 };
3848
3849 static const struct proto_ops packet_ops = {
3850 .family = PF_PACKET,
3851 .owner = THIS_MODULE,
3852 .release = packet_release,
3853 .bind = packet_bind,
3854 .connect = sock_no_connect,
3855 .socketpair = sock_no_socketpair,
3856 .accept = sock_no_accept,
3857 .getname = packet_getname,
3858 .poll = packet_poll,
3859 .ioctl = packet_ioctl,
3860 .listen = sock_no_listen,
3861 .shutdown = sock_no_shutdown,
3862 .setsockopt = packet_setsockopt,
3863 .getsockopt = packet_getsockopt,
3864 .sendmsg = packet_sendmsg,
3865 .recvmsg = packet_recvmsg,
3866 .mmap = packet_mmap,
3867 .sendpage = sock_no_sendpage,
3868 };
3869
3870 static const struct net_proto_family packet_family_ops = {
3871 .family = PF_PACKET,
3872 .create = packet_create,
3873 .owner = THIS_MODULE,
3874 };
3875
3876 static struct notifier_block packet_netdev_notifier = {
3877 .notifier_call = packet_notifier,
3878 };
3879
3880 #ifdef CONFIG_PROC_FS
3881
3882 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
3883 __acquires(RCU)
3884 {
3885 struct net *net = seq_file_net(seq);
3886
3887 rcu_read_lock();
3888 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
3889 }
3890
3891 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3892 {
3893 struct net *net = seq_file_net(seq);
3894 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
3895 }
3896
3897 static void packet_seq_stop(struct seq_file *seq, void *v)
3898 __releases(RCU)
3899 {
3900 rcu_read_unlock();
3901 }
3902
3903 static int packet_seq_show(struct seq_file *seq, void *v)
3904 {
3905 if (v == SEQ_START_TOKEN)
3906 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3907 else {
3908 struct sock *s = sk_entry(v);
3909 const struct packet_sock *po = pkt_sk(s);
3910
3911 seq_printf(seq,
3912 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
3913 s,
3914 atomic_read(&s->sk_refcnt),
3915 s->sk_type,
3916 ntohs(po->num),
3917 po->ifindex,
3918 po->running,
3919 atomic_read(&s->sk_rmem_alloc),
3920 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
3921 sock_i_ino(s));
3922 }
3923
3924 return 0;
3925 }
3926
3927 static const struct seq_operations packet_seq_ops = {
3928 .start = packet_seq_start,
3929 .next = packet_seq_next,
3930 .stop = packet_seq_stop,
3931 .show = packet_seq_show,
3932 };
3933
3934 static int packet_seq_open(struct inode *inode, struct file *file)
3935 {
3936 return seq_open_net(inode, file, &packet_seq_ops,
3937 sizeof(struct seq_net_private));
3938 }
3939
3940 static const struct file_operations packet_seq_fops = {
3941 .owner = THIS_MODULE,
3942 .open = packet_seq_open,
3943 .read = seq_read,
3944 .llseek = seq_lseek,
3945 .release = seq_release_net,
3946 };
3947
3948 #endif
3949
3950 static int __net_init packet_net_init(struct net *net)
3951 {
3952 mutex_init(&net->packet.sklist_lock);
3953 INIT_HLIST_HEAD(&net->packet.sklist);
3954
3955 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
3956 return -ENOMEM;
3957
3958 return 0;
3959 }
3960
3961 static void __net_exit packet_net_exit(struct net *net)
3962 {
3963 remove_proc_entry("packet", net->proc_net);
3964 }
3965
3966 static struct pernet_operations packet_net_ops = {
3967 .init = packet_net_init,
3968 .exit = packet_net_exit,
3969 };
3970
3971
3972 static void __exit packet_exit(void)
3973 {
3974 unregister_netdevice_notifier(&packet_netdev_notifier);
3975 unregister_pernet_subsys(&packet_net_ops);
3976 sock_unregister(PF_PACKET);
3977 proto_unregister(&packet_proto);
3978 }
3979
3980 static int __init packet_init(void)
3981 {
3982 int rc = proto_register(&packet_proto, 0);
3983
3984 if (rc != 0)
3985 goto out;
3986
3987 sock_register(&packet_family_ops);
3988 register_pernet_subsys(&packet_net_ops);
3989 register_netdevice_notifier(&packet_netdev_notifier);
3990 out:
3991 return rc;
3992 }
3993
3994 module_init(packet_init);
3995 module_exit(packet_exit);
3996 MODULE_LICENSE("GPL");
3997 MODULE_ALIAS_NETPROTO(PF_PACKET);
kernel source for telekom puls (alcatel/mediatek)