projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ax25: Fix missing break
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
CommitLineData
92915f71
GR		 1/*******************************************************************************
2
3 Intel 82599 Virtual Function driver
5c47a2b6 4 Copyright(c) 1999 - 2012 Intel Corporation.
92915f71
GR		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28
29/******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31******************************************************************************/
dbd9636e
JK		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
92915f71 35#include <linux/types.h>
dadcd65f 36#include <linux/bitops.h>
92915f71
GR		 37#include <linux/module.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/vmalloc.h>
41#include <linux/string.h>
42#include <linux/in.h>
43#include <linux/ip.h>
44#include <linux/tcp.h>
45#include <linux/ipv6.h>
5a0e3ad6 46#include <linux/slab.h>
92915f71
GR		 47#include <net/checksum.h>
48#include <net/ip6_checksum.h>
49#include <linux/ethtool.h>
01789349 50#include <linux/if.h>
92915f71 51#include <linux/if_vlan.h>
70c71606 52#include <linux/prefetch.h>
92915f71
GR		 53
54#include "ixgbevf.h"
55
3d8fe98f 56const char ixgbevf_driver_name[] = "ixgbevf";
92915f71 57static const char ixgbevf_driver_string[] =
422e05d1 58 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
92915f71 59
9cd9130d 60#define DRV_VERSION "2.6.0-k"
92915f71 61const char ixgbevf_driver_version[] = DRV_VERSION;
66c87bd5 62static char ixgbevf_copyright[] =
5c47a2b6 63 "Copyright (c) 2009 - 2012 Intel Corporation.";
92915f71
GR		 64
65static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
2316aa2a
GR		 66 [board_82599_vf] = &ixgbevf_82599_vf_info,
67 [board_X540_vf] = &ixgbevf_X540_vf_info,
92915f71
GR		 68};
69
70/* ixgbevf_pci_tbl - PCI Device ID Table
71 *
72 * Wildcard entries (PCI_ANY_ID) should come last
73 * Last entry must be all 0s
74 *
75 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
76 * Class, Class Mask, private data (not used) }
77 */
78static struct pci_device_id ixgbevf_pci_tbl[] = {
79 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF),
80 board_82599_vf},
2316aa2a
GR		 81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF),
82 board_X540_vf},
92915f71
GR		 83
84 /* required last entry */
85 {0, }
86};
87MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
88
89MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
90MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
91MODULE_LICENSE("GPL");
92MODULE_VERSION(DRV_VERSION);
93
b3f4d599 94#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
95static int debug = -1;
96module_param(debug, int, 0);
97MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
92915f71
GR		 98
99/* forward decls */
100static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector);
101static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx,
102 u32 itr_reg);
103
104static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
105 struct ixgbevf_ring *rx_ring,
106 u32 val)
107{
108 /*
109 * Force memory writes to complete before letting h/w
110 * know there are new descriptors to fetch. (Only
111 * applicable for weak-ordered memory model archs,
112 * such as IA-64).
113 */
114 wmb();
115 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
116}
117
118/*
65d676c8 119 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
92915f71
GR		 120 * @adapter: pointer to adapter struct
121 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
122 * @queue: queue to map the corresponding interrupt to
123 * @msix_vector: the vector to map to the corresponding queue
124 *
125 */
126static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
127 u8 queue, u8 msix_vector)
128{
129 u32 ivar, index;
130 struct ixgbe_hw *hw = &adapter->hw;
131 if (direction == -1) {
132 /* other causes */
133 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
134 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
135 ivar &= ~0xFF;
136 ivar |= msix_vector;
137 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
138 } else {
139 /* tx or rx causes */
140 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
141 index = ((16 * (queue & 1)) + (8 * direction));
142 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
143 ivar &= ~(0xFF << index);
144 ivar |= (msix_vector << index);
145 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
146 }
147}
148
149static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_adapter *adapter,
150 struct ixgbevf_tx_buffer
151 *tx_buffer_info)
152{
153 if (tx_buffer_info->dma) {
154 if (tx_buffer_info->mapped_as_page)
2a1f8794 155 dma_unmap_page(&adapter->pdev->dev,
92915f71
GR		 156 tx_buffer_info->dma,
157 tx_buffer_info->length,
2a1f8794 158 DMA_TO_DEVICE);
92915f71 159 else
2a1f8794 160 dma_unmap_single(&adapter->pdev->dev,
92915f71
GR		 161 tx_buffer_info->dma,
162 tx_buffer_info->length,
2a1f8794 163 DMA_TO_DEVICE);
92915f71
GR		 164 tx_buffer_info->dma = 0;
165 }
166 if (tx_buffer_info->skb) {
167 dev_kfree_skb_any(tx_buffer_info->skb);
168 tx_buffer_info->skb = NULL;
169 }
170 tx_buffer_info->time_stamp = 0;
171 /* tx_buffer_info must be completely set up in the transmit path */
172}
173
92915f71
GR		 174#define IXGBE_MAX_TXD_PWR 14
175#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
176
177/* Tx Descriptors needed, worst case */
178#define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \
179 (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
180#ifdef MAX_SKB_FRAGS
181#define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \
182 MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1) /* for context */
183#else
184#define DESC_NEEDED TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD)
185#endif
186
187static void ixgbevf_tx_timeout(struct net_device *netdev);
188
189/**
190 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
191 * @adapter: board private structure
192 * @tx_ring: tx ring to clean
193 **/
194static bool ixgbevf_clean_tx_irq(struct ixgbevf_adapter *adapter,
195 struct ixgbevf_ring *tx_ring)
196{
197 struct net_device *netdev = adapter->netdev;
198 struct ixgbe_hw *hw = &adapter->hw;
199 union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
200 struct ixgbevf_tx_buffer *tx_buffer_info;
201 unsigned int i, eop, count = 0;
202 unsigned int total_bytes = 0, total_packets = 0;
203
204 i = tx_ring->next_to_clean;
205 eop = tx_ring->tx_buffer_info[i].next_to_watch;
206 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop);
207
208 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
209 (count < tx_ring->work_limit)) {
210 bool cleaned = false;
2d0bb1c1 211 rmb(); /* read buffer_info after eop_desc */
98b9e48f
GR		 212 /* eop could change between read and DD-check */
213 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch))
214 goto cont_loop;
92915f71
GR		 215 for ( ; !cleaned; count++) {
216 struct sk_buff *skb;
217 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i);
218 tx_buffer_info = &tx_ring->tx_buffer_info[i];
219 cleaned = (i == eop);
220 skb = tx_buffer_info->skb;
221
222 if (cleaned && skb) {
223 unsigned int segs, bytecount;
224
225 /* gso_segs is currently only valid for tcp */
226 segs = skb_shinfo(skb)->gso_segs ?: 1;
227 /* multiply data chunks by size of headers */
228 bytecount = ((segs - 1) * skb_headlen(skb)) +
229 skb->len;
230 total_packets += segs;
231 total_bytes += bytecount;
232 }
233
234 ixgbevf_unmap_and_free_tx_resource(adapter,
235 tx_buffer_info);
236
237 tx_desc->wb.status = 0;
238
239 i++;
240 if (i == tx_ring->count)
241 i = 0;
242 }
243
98b9e48f 244cont_loop:
92915f71
GR		 245 eop = tx_ring->tx_buffer_info[i].next_to_watch;
246 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop);
247 }
248
249 tx_ring->next_to_clean = i;
250
251#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
252 if (unlikely(count && netif_carrier_ok(netdev) &&
253 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
254 /* Make sure that anybody stopping the queue after this
255 * sees the new next_to_clean.
256 */
257 smp_mb();
258#ifdef HAVE_TX_MQ
259 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
260 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
261 netif_wake_subqueue(netdev, tx_ring->queue_index);
262 ++adapter->restart_queue;
263 }
264#else
265 if (netif_queue_stopped(netdev) &&
266 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
267 netif_wake_queue(netdev);
268 ++adapter->restart_queue;
269 }
270#endif
271 }
272
92915f71
GR		 273 /* re-arm the interrupt */
274 if ((count >= tx_ring->work_limit) &&
275 (!test_bit(__IXGBEVF_DOWN, &adapter->state))) {
276 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, tx_ring->v_idx);
277 }
278
4197aa7b 279 u64_stats_update_begin(&tx_ring->syncp);
92915f71
GR		 280 tx_ring->total_bytes += total_bytes;
281 tx_ring->total_packets += total_packets;
4197aa7b 282 u64_stats_update_end(&tx_ring->syncp);
92915f71 283
807540ba 284 return count < tx_ring->work_limit;
92915f71
GR		 285}
286
287/**
288 * ixgbevf_receive_skb - Send a completed packet up the stack
289 * @q_vector: structure containing interrupt and ring information
290 * @skb: packet to send up
291 * @status: hardware indication of status of receive
292 * @rx_ring: rx descriptor ring (for a specific queue) to setup
293 * @rx_desc: rx descriptor
294 **/
295static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
296 struct sk_buff *skb, u8 status,
297 struct ixgbevf_ring *ring,
298 union ixgbe_adv_rx_desc *rx_desc)
299{
300 struct ixgbevf_adapter *adapter = q_vector->adapter;
301 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
dd1ed3b7 302 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
92915f71 303
dd1ed3b7 304 if (is_vlan && test_bit(tag, adapter->active_vlans))
dadcd65f 305 __vlan_hwaccel_put_tag(skb, tag);
dadcd65f
JP		 306
307 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
92915f71 308 napi_gro_receive(&q_vector->napi, skb);
dadcd65f 309 else
c82a538e 310 netif_rx(skb);
92915f71
GR		 311}
312
313/**
314 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
315 * @adapter: address of board private structure
316 * @status_err: hardware indication of status of receive
317 * @skb: skb currently being received and modified
318 **/
319static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter,
320 u32 status_err, struct sk_buff *skb)
321{
bc8acf2c 322 skb_checksum_none_assert(skb);
92915f71
GR		 323
324 /* Rx csum disabled */
325 if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
326 return;
327
328 /* if IP and error */
329 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
330 (status_err & IXGBE_RXDADV_ERR_IPE)) {
331 adapter->hw_csum_rx_error++;
332 return;
333 }
334
335 if (!(status_err & IXGBE_RXD_STAT_L4CS))
336 return;
337
338 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
339 adapter->hw_csum_rx_error++;
340 return;
341 }
342
343 /* It must be a TCP or UDP packet with a valid checksum */
344 skb->ip_summed = CHECKSUM_UNNECESSARY;
345 adapter->hw_csum_rx_good++;
346}
347
348/**
349 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
350 * @adapter: address of board private structure
351 **/
352static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
353 struct ixgbevf_ring *rx_ring,
354 int cleaned_count)
355{
356 struct pci_dev *pdev = adapter->pdev;
357 union ixgbe_adv_rx_desc *rx_desc;
358 struct ixgbevf_rx_buffer *bi;
359 struct sk_buff *skb;
360 unsigned int i;
361 unsigned int bufsz = rx_ring->rx_buf_len + NET_IP_ALIGN;
362
363 i = rx_ring->next_to_use;
364 bi = &rx_ring->rx_buffer_info[i];
365
366 while (cleaned_count--) {
367 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i);
368
369 if (!bi->page_dma &&
370 (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)) {
371 if (!bi->page) {
1f2149c1 372 bi->page = alloc_page(GFP_ATOMIC | __GFP_COLD);
92915f71
GR		 373 if (!bi->page) {
374 adapter->alloc_rx_page_failed++;
375 goto no_buffers;
376 }
377 bi->page_offset = 0;
378 } else {
379 /* use a half page if we're re-using */
380 bi->page_offset ^= (PAGE_SIZE / 2);
381 }
382
2a1f8794 383 bi->page_dma = dma_map_page(&pdev->dev, bi->page,
92915f71
GR		 384 bi->page_offset,
385 (PAGE_SIZE / 2),
2a1f8794 386 DMA_FROM_DEVICE);
92915f71
GR		 387 }
388
389 skb = bi->skb;
390 if (!skb) {
391 skb = netdev_alloc_skb(adapter->netdev,
392 bufsz);
393
394 if (!skb) {
395 adapter->alloc_rx_buff_failed++;
396 goto no_buffers;
397 }
398
399 /*
400 * Make buffer alignment 2 beyond a 16 byte boundary
401 * this will result in a 16 byte aligned IP header after
402 * the 14 byte MAC header is removed
403 */
404 skb_reserve(skb, NET_IP_ALIGN);
405
406 bi->skb = skb;
407 }
408 if (!bi->dma) {
2a1f8794 409 bi->dma = dma_map_single(&pdev->dev, skb->data,
92915f71 410 rx_ring->rx_buf_len,
2a1f8794 411 DMA_FROM_DEVICE);
92915f71
GR		 412 }
413 /* Refresh the desc even if buffer_addrs didn't change because
414 * each write-back erases this info. */
415 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
416 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
417 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
418 } else {
419 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
420 }
421
422 i++;
423 if (i == rx_ring->count)
424 i = 0;
425 bi = &rx_ring->rx_buffer_info[i];
426 }
427
428no_buffers:
429 if (rx_ring->next_to_use != i) {
430 rx_ring->next_to_use = i;
431 if (i-- == 0)
432 i = (rx_ring->count - 1);
433
434 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
435 }
436}
437
438static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
439 u64 qmask)
440{
441 u32 mask;
442 struct ixgbe_hw *hw = &adapter->hw;
443
444 mask = (qmask & 0xFFFFFFFF);
445 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
446}
447
448static inline u16 ixgbevf_get_hdr_info(union ixgbe_adv_rx_desc *rx_desc)
449{
450 return rx_desc->wb.lower.lo_dword.hs_rss.hdr_info;
451}
452
453static inline u16 ixgbevf_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc)
454{
455 return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
456}
457
458static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
459 struct ixgbevf_ring *rx_ring,
460 int *work_done, int work_to_do)
461{
462 struct ixgbevf_adapter *adapter = q_vector->adapter;
463 struct pci_dev *pdev = adapter->pdev;
464 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
465 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
466 struct sk_buff *skb;
467 unsigned int i;
468 u32 len, staterr;
469 u16 hdr_info;
470 bool cleaned = false;
471 int cleaned_count = 0;
472 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
473
474 i = rx_ring->next_to_clean;
475 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i);
476 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
477 rx_buffer_info = &rx_ring->rx_buffer_info[i];
478
479 while (staterr & IXGBE_RXD_STAT_DD) {
480 u32 upper_len = 0;
481 if (*work_done >= work_to_do)
482 break;
483 (*work_done)++;
484
2d0bb1c1 485 rmb(); /* read descriptor and rx_buffer_info after status DD */
92915f71
GR		 486 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
487 hdr_info = le16_to_cpu(ixgbevf_get_hdr_info(rx_desc));
488 len = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
489 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
490 if (hdr_info & IXGBE_RXDADV_SPH)
491 adapter->rx_hdr_split++;
492 if (len > IXGBEVF_RX_HDR_SIZE)
493 len = IXGBEVF_RX_HDR_SIZE;
494 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
495 } else {
496 len = le16_to_cpu(rx_desc->wb.upper.length);
497 }
498 cleaned = true;
499 skb = rx_buffer_info->skb;
500 prefetch(skb->data - NET_IP_ALIGN);
501 rx_buffer_info->skb = NULL;
502
503 if (rx_buffer_info->dma) {
2a1f8794 504 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
92915f71 505 rx_ring->rx_buf_len,
2a1f8794 506 DMA_FROM_DEVICE);
92915f71
GR		 507 rx_buffer_info->dma = 0;
508 skb_put(skb, len);
509 }
510
511 if (upper_len) {
2a1f8794
NN		 512 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma,
513 PAGE_SIZE / 2, DMA_FROM_DEVICE);
92915f71
GR		 514 rx_buffer_info->page_dma = 0;
515 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
516 rx_buffer_info->page,
517 rx_buffer_info->page_offset,
518 upper_len);
519
520 if ((rx_ring->rx_buf_len > (PAGE_SIZE / 2)) ||
521 (page_count(rx_buffer_info->page) != 1))
522 rx_buffer_info->page = NULL;
523 else
524 get_page(rx_buffer_info->page);
525
526 skb->len += upper_len;
527 skb->data_len += upper_len;
528 skb->truesize += upper_len;
529 }
530
531 i++;
532 if (i == rx_ring->count)
533 i = 0;
534
535 next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i);
536 prefetch(next_rxd);
537 cleaned_count++;
538
539 next_buffer = &rx_ring->rx_buffer_info[i];
540
541 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
542 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
543 rx_buffer_info->skb = next_buffer->skb;
544 rx_buffer_info->dma = next_buffer->dma;
545 next_buffer->skb = skb;
546 next_buffer->dma = 0;
547 } else {
548 skb->next = next_buffer->skb;
549 skb->next->prev = skb;
550 }
551 adapter->non_eop_descs++;
552 goto next_desc;
553 }
554
555 /* ERR_MASK will only have valid bits if EOP set */
556 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
557 dev_kfree_skb_irq(skb);
558 goto next_desc;
559 }
560
561 ixgbevf_rx_checksum(adapter, staterr, skb);
562
563 /* probably a little skewed due to removing CRC */
564 total_rx_bytes += skb->len;
565 total_rx_packets++;
566
567 /*
568 * Work around issue of some types of VM to VM loop back
569 * packets not getting split correctly
570 */
571 if (staterr & IXGBE_RXD_STAT_LB) {
e743d313 572 u32 header_fixup_len = skb_headlen(skb);
92915f71
GR		 573 if (header_fixup_len < 14)
574 skb_push(skb, header_fixup_len);
575 }
576 skb->protocol = eth_type_trans(skb, adapter->netdev);
577
578 ixgbevf_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
92915f71
GR		 579
580next_desc:
581 rx_desc->wb.upper.status_error = 0;
582
583 /* return some buffers to hardware, one at a time is too slow */
584 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
585 ixgbevf_alloc_rx_buffers(adapter, rx_ring,
586 cleaned_count);
587 cleaned_count = 0;
588 }
589
590 /* use prefetched values */
591 rx_desc = next_rxd;
592 rx_buffer_info = &rx_ring->rx_buffer_info[i];
593
594 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
595 }
596
597 rx_ring->next_to_clean = i;
598 cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
599
600 if (cleaned_count)
601 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
602
4197aa7b 603 u64_stats_update_begin(&rx_ring->syncp);
92915f71
GR		 604 rx_ring->total_packets += total_rx_packets;
605 rx_ring->total_bytes += total_rx_bytes;
4197aa7b 606 u64_stats_update_end(&rx_ring->syncp);
92915f71
GR		 607
608 return cleaned;
609}
610
611/**
612 * ixgbevf_clean_rxonly - msix (aka one shot) rx clean routine
613 * @napi: napi struct with our devices info in it
614 * @budget: amount of work driver is allowed to do this pass, in packets
615 *
616 * This function is optimized for cleaning one queue only on a single
617 * q_vector!!!
618 **/
619static int ixgbevf_clean_rxonly(struct napi_struct *napi, int budget)
620{
621 struct ixgbevf_q_vector *q_vector =
622 container_of(napi, struct ixgbevf_q_vector, napi);
623 struct ixgbevf_adapter *adapter = q_vector->adapter;
624 struct ixgbevf_ring *rx_ring = NULL;
625 int work_done = 0;
626 long r_idx;
627
628 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
629 rx_ring = &(adapter->rx_ring[r_idx]);
630
631 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget);
632
633 /* If all Rx work done, exit the polling mode */
634 if (work_done < budget) {
635 napi_complete(napi);
636 if (adapter->itr_setting & 1)
637 ixgbevf_set_itr_msix(q_vector);
638 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
639 ixgbevf_irq_enable_queues(adapter, rx_ring->v_idx);
640 }
641
642 return work_done;
643}
644
645/**
646 * ixgbevf_clean_rxonly_many - msix (aka one shot) rx clean routine
647 * @napi: napi struct with our devices info in it
648 * @budget: amount of work driver is allowed to do this pass, in packets
649 *
650 * This function will clean more than one rx queue associated with a
651 * q_vector.
652 **/
653static int ixgbevf_clean_rxonly_many(struct napi_struct *napi, int budget)
654{
655 struct ixgbevf_q_vector *q_vector =
656 container_of(napi, struct ixgbevf_q_vector, napi);
657 struct ixgbevf_adapter *adapter = q_vector->adapter;
658 struct ixgbevf_ring *rx_ring = NULL;
659 int work_done = 0, i;
660 long r_idx;
661 u64 enable_mask = 0;
662
663 /* attempt to distribute budget to each queue fairly, but don't allow
664 * the budget to go below 1 because we'll exit polling */
665 budget /= (q_vector->rxr_count ?: 1);
666 budget = max(budget, 1);
667 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
668 for (i = 0; i < q_vector->rxr_count; i++) {
669 rx_ring = &(adapter->rx_ring[r_idx]);
670 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget);
671 enable_mask |= rx_ring->v_idx;
672 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
673 r_idx + 1);
674 }
675
676#ifndef HAVE_NETDEV_NAPI_LIST
677 if (!netif_running(adapter->netdev))
678 work_done = 0;
679
680#endif
681 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
682 rx_ring = &(adapter->rx_ring[r_idx]);
683
684 /* If all Rx work done, exit the polling mode */
685 if (work_done < budget) {
686 napi_complete(napi);
687 if (adapter->itr_setting & 1)
688 ixgbevf_set_itr_msix(q_vector);
689 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
690 ixgbevf_irq_enable_queues(adapter, enable_mask);
691 }
692
693 return work_done;
694}
695
696
697/**
698 * ixgbevf_configure_msix - Configure MSI-X hardware
699 * @adapter: board private structure
700 *
701 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
702 * interrupts.
703 **/
704static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
705{
706 struct ixgbevf_q_vector *q_vector;
707 struct ixgbe_hw *hw = &adapter->hw;
708 int i, j, q_vectors, v_idx, r_idx;
709 u32 mask;
710
711 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
712
713 /*
714 * Populate the IVAR table and set the ITR values to the
715 * corresponding register.
716 */
717 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
718 q_vector = adapter->q_vector[v_idx];
984b3f57 719 /* XXX for_each_set_bit(...) */
92915f71
GR		 720 r_idx = find_first_bit(q_vector->rxr_idx,
721 adapter->num_rx_queues);
722
723 for (i = 0; i < q_vector->rxr_count; i++) {
724 j = adapter->rx_ring[r_idx].reg_idx;
725 ixgbevf_set_ivar(adapter, 0, j, v_idx);
726 r_idx = find_next_bit(q_vector->rxr_idx,
727 adapter->num_rx_queues,
728 r_idx + 1);
729 }
730 r_idx = find_first_bit(q_vector->txr_idx,
731 adapter->num_tx_queues);
732
733 for (i = 0; i < q_vector->txr_count; i++) {
734 j = adapter->tx_ring[r_idx].reg_idx;
735 ixgbevf_set_ivar(adapter, 1, j, v_idx);
736 r_idx = find_next_bit(q_vector->txr_idx,
737 adapter->num_tx_queues,
738 r_idx + 1);
739 }
740
741 /* if this is a tx only vector halve the interrupt rate */
742 if (q_vector->txr_count && !q_vector->rxr_count)
743 q_vector->eitr = (adapter->eitr_param >> 1);
744 else if (q_vector->rxr_count)
745 /* rx only */
746 q_vector->eitr = adapter->eitr_param;
747
748 ixgbevf_write_eitr(adapter, v_idx, q_vector->eitr);
749 }
750
751 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
752
753 /* set up to autoclear timer, and the vectors */
754 mask = IXGBE_EIMS_ENABLE_MASK;
755 mask &= ~IXGBE_EIMS_OTHER;
756 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
757}
758
759enum latency_range {
760 lowest_latency = 0,
761 low_latency = 1,
762 bulk_latency = 2,
763 latency_invalid = 255
764};
765
766/**
767 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
768 * @adapter: pointer to adapter
769 * @eitr: eitr setting (ints per sec) to give last timeslice
770 * @itr_setting: current throttle rate in ints/second
771 * @packets: the number of packets during this measurement interval
772 * @bytes: the number of bytes during this measurement interval
773 *
774 * Stores a new ITR value based on packets and byte
775 * counts during the last interrupt. The advantage of per interrupt
776 * computation is faster updates and more accurate ITR for the current
777 * traffic pattern. Constants in this function were computed
778 * based on theoretical maximum wire speed and thresholds were set based
779 * on testing data as well as attempting to minimize response time
780 * while increasing bulk throughput.
781 **/
782static u8 ixgbevf_update_itr(struct ixgbevf_adapter *adapter,
783 u32 eitr, u8 itr_setting,
784 int packets, int bytes)
785{
786 unsigned int retval = itr_setting;
787 u32 timepassed_us;
788 u64 bytes_perint;
789
790 if (packets == 0)
791 goto update_itr_done;
792
793
794 /* simple throttlerate management
795 * 0-20MB/s lowest (100000 ints/s)
796 * 20-100MB/s low (20000 ints/s)
797 * 100-1249MB/s bulk (8000 ints/s)
798 */
799 /* what was last interrupt timeslice? */
800 timepassed_us = 1000000/eitr;
801 bytes_perint = bytes / timepassed_us; /* bytes/usec */
802
803 switch (itr_setting) {
804 case lowest_latency:
805 if (bytes_perint > adapter->eitr_low)
806 retval = low_latency;
807 break;
808 case low_latency:
809 if (bytes_perint > adapter->eitr_high)
810 retval = bulk_latency;
811 else if (bytes_perint <= adapter->eitr_low)
812 retval = lowest_latency;
813 break;
814 case bulk_latency:
815 if (bytes_perint <= adapter->eitr_high)
816 retval = low_latency;
817 break;
818 }
819
820update_itr_done:
821 return retval;
822}
823
824/**
825 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
826 * @adapter: pointer to adapter struct
827 * @v_idx: vector index into q_vector array
828 * @itr_reg: new value to be written in *register* format, not ints/s
829 *
830 * This function is made to be called by ethtool and by the driver
831 * when it needs to update VTEITR registers at runtime. Hardware
832 * specific quirks/differences are taken care of here.
833 */
834static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx,
835 u32 itr_reg)
836{
837 struct ixgbe_hw *hw = &adapter->hw;
838
839 itr_reg = EITR_INTS_PER_SEC_TO_REG(itr_reg);
840
841 /*
842 * set the WDIS bit to not clear the timer bits and cause an
843 * immediate assertion of the interrupt
844 */
845 itr_reg |= IXGBE_EITR_CNT_WDIS;
846
847 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
848}
849
850static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector)
851{
852 struct ixgbevf_adapter *adapter = q_vector->adapter;
853 u32 new_itr;
854 u8 current_itr, ret_itr;
855 int i, r_idx, v_idx = q_vector->v_idx;
856 struct ixgbevf_ring *rx_ring, *tx_ring;
857
858 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
859 for (i = 0; i < q_vector->txr_count; i++) {
860 tx_ring = &(adapter->tx_ring[r_idx]);
861 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr,
862 q_vector->tx_itr,
863 tx_ring->total_packets,
864 tx_ring->total_bytes);
865 /* if the result for this queue would decrease interrupt
866 * rate for this vector then use that result */
867 q_vector->tx_itr = ((q_vector->tx_itr > ret_itr) ?
868 q_vector->tx_itr - 1 : ret_itr);
869 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
870 r_idx + 1);
871 }
872
873 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
874 for (i = 0; i < q_vector->rxr_count; i++) {
875 rx_ring = &(adapter->rx_ring[r_idx]);
876 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr,
877 q_vector->rx_itr,
878 rx_ring->total_packets,
879 rx_ring->total_bytes);
880 /* if the result for this queue would decrease interrupt
881 * rate for this vector then use that result */
882 q_vector->rx_itr = ((q_vector->rx_itr > ret_itr) ?
883 q_vector->rx_itr - 1 : ret_itr);
884 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
885 r_idx + 1);
886 }
887
888 current_itr = max(q_vector->rx_itr, q_vector->tx_itr);
889
890 switch (current_itr) {
891 /* counts and packets in update_itr are dependent on these numbers */
892 case lowest_latency:
893 new_itr = 100000;
894 break;
895 case low_latency:
896 new_itr = 20000; /* aka hwitr = ~200 */
897 break;
898 case bulk_latency:
899 default:
900 new_itr = 8000;
901 break;
902 }
903
904 if (new_itr != q_vector->eitr) {
905 u32 itr_reg;
906
907 /* save the algorithm value here, not the smoothed one */
908 q_vector->eitr = new_itr;
909 /* do an exponential smoothing */
910 new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100);
911 itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr);
912 ixgbevf_write_eitr(adapter, v_idx, itr_reg);
913 }
92915f71
GR		 914}
915
916static irqreturn_t ixgbevf_msix_mbx(int irq, void *data)
917{
918 struct net_device *netdev = data;
919 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
920 struct ixgbe_hw *hw = &adapter->hw;
921 u32 eicr;
a9ee25a2 922 u32 msg;
375b27cf 923 bool got_ack = false;
92915f71
GR		 924
925 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICS);
926 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, eicr);
927
375b27cf
GR		 928 if (!hw->mbx.ops.check_for_ack(hw))
929 got_ack = true;
08259594 930
375b27cf
GR		 931 if (!hw->mbx.ops.check_for_msg(hw)) {
932 hw->mbx.ops.read(hw, &msg, 1);
a9ee25a2 933
375b27cf
GR		 934 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG)
935 mod_timer(&adapter->watchdog_timer,
936 round_jiffies(jiffies + 1));
a9ee25a2 937
375b27cf
GR		 938 if (msg & IXGBE_VT_MSGTYPE_NACK)
939 pr_warn("Last Request of type %2.2x to PF Nacked\n",
940 msg & 0xFF);
3a2c4033
GR		 941 /*
942 * Restore the PFSTS bit in case someone is polling for a
943 * return message from the PF
944 */
945 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS;
375b27cf
GR		 946 }
947
948 /*
949 * checking for the ack clears the PFACK bit. Place
950 * it back in the v2p_mailbox cache so that anyone
951 * polling for an ack will not miss it
952 */
953 if (got_ack)
954 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK;
3a2c4033 955
92915f71
GR		 956 return IRQ_HANDLED;
957}
958
959static irqreturn_t ixgbevf_msix_clean_tx(int irq, void *data)
960{
961 struct ixgbevf_q_vector *q_vector = data;
962 struct ixgbevf_adapter *adapter = q_vector->adapter;
963 struct ixgbevf_ring *tx_ring;
964 int i, r_idx;
965
966 if (!q_vector->txr_count)
967 return IRQ_HANDLED;
968
969 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
970 for (i = 0; i < q_vector->txr_count; i++) {
971 tx_ring = &(adapter->tx_ring[r_idx]);
92915f71
GR		 972 ixgbevf_clean_tx_irq(adapter, tx_ring);
973 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
974 r_idx + 1);
975 }
976
977 if (adapter->itr_setting & 1)
978 ixgbevf_set_itr_msix(q_vector);
979
980 return IRQ_HANDLED;
981}
982
983/**
65d676c8 984 * ixgbevf_msix_clean_rx - single unshared vector rx clean (all queues)
92915f71
GR		 985 * @irq: unused
986 * @data: pointer to our q_vector struct for this interrupt vector
987 **/
988static irqreturn_t ixgbevf_msix_clean_rx(int irq, void *data)
989{
990 struct ixgbevf_q_vector *q_vector = data;
991 struct ixgbevf_adapter *adapter = q_vector->adapter;
992 struct ixgbe_hw *hw = &adapter->hw;
993 struct ixgbevf_ring *rx_ring;
994 int r_idx;
92915f71
GR		 995
996 if (!q_vector->rxr_count)
997 return IRQ_HANDLED;
998
999 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
1000 rx_ring = &(adapter->rx_ring[r_idx]);
1001 /* disable interrupts on this vector only */
1002 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, rx_ring->v_idx);
1003 napi_schedule(&q_vector->napi);
1004
1005
1006 return IRQ_HANDLED;
1007}
1008
1009static irqreturn_t ixgbevf_msix_clean_many(int irq, void *data)
1010{
1011 ixgbevf_msix_clean_rx(irq, data);
1012 ixgbevf_msix_clean_tx(irq, data);
1013
1014 return IRQ_HANDLED;
1015}
1016
1017static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
1018 int r_idx)
1019{
1020 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1021
1022 set_bit(r_idx, q_vector->rxr_idx);
1023 q_vector->rxr_count++;
1024 a->rx_ring[r_idx].v_idx = 1 << v_idx;
1025}
1026
1027static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
1028 int t_idx)
1029{
1030 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1031
1032 set_bit(t_idx, q_vector->txr_idx);
1033 q_vector->txr_count++;
1034 a->tx_ring[t_idx].v_idx = 1 << v_idx;
1035}
1036
1037/**
1038 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
1039 * @adapter: board private structure to initialize
1040 *
1041 * This function maps descriptor rings to the queue-specific vectors
1042 * we were allotted through the MSI-X enabling code. Ideally, we'd have
1043 * one vector per ring/queue, but on a constrained vector budget, we
1044 * group the rings as "efficiently" as possible. You would add new
1045 * mapping configurations in here.
1046 **/
1047static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
1048{
1049 int q_vectors;
1050 int v_start = 0;
1051 int rxr_idx = 0, txr_idx = 0;
1052 int rxr_remaining = adapter->num_rx_queues;
1053 int txr_remaining = adapter->num_tx_queues;
1054 int i, j;
1055 int rqpv, tqpv;
1056 int err = 0;
1057
1058 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1059
1060 /*
1061 * The ideal configuration...
1062 * We have enough vectors to map one per queue.
1063 */
1064 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
1065 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
1066 map_vector_to_rxq(adapter, v_start, rxr_idx);
1067
1068 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
1069 map_vector_to_txq(adapter, v_start, txr_idx);
1070 goto out;
1071 }
1072
1073 /*
1074 * If we don't have enough vectors for a 1-to-1
1075 * mapping, we'll have to group them so there are
1076 * multiple queues per vector.
1077 */
1078 /* Re-adjusting *qpv takes care of the remainder. */
1079 for (i = v_start; i < q_vectors; i++) {
1080 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
1081 for (j = 0; j < rqpv; j++) {
1082 map_vector_to_rxq(adapter, i, rxr_idx);
1083 rxr_idx++;
1084 rxr_remaining--;
1085 }
1086 }
1087 for (i = v_start; i < q_vectors; i++) {
1088 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
1089 for (j = 0; j < tqpv; j++) {
1090 map_vector_to_txq(adapter, i, txr_idx);
1091 txr_idx++;
1092 txr_remaining--;
1093 }
1094 }
1095
1096out:
1097 return err;
1098}
1099
1100/**
1101 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1102 * @adapter: board private structure
1103 *
1104 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1105 * interrupts from the kernel.
1106 **/
1107static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1108{
1109 struct net_device *netdev = adapter->netdev;
1110 irqreturn_t (*handler)(int, void *);
1111 int i, vector, q_vectors, err;
1112 int ri = 0, ti = 0;
1113
1114 /* Decrement for Other and TCP Timer vectors */
1115 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1116
1117#define SET_HANDLER(_v) (((_v)->rxr_count && (_v)->txr_count) \
1118 ? &ixgbevf_msix_clean_many : \
1119 (_v)->rxr_count ? &ixgbevf_msix_clean_rx : \
1120 (_v)->txr_count ? &ixgbevf_msix_clean_tx : \
1121 NULL)
1122 for (vector = 0; vector < q_vectors; vector++) {
1123 handler = SET_HANDLER(adapter->q_vector[vector]);
1124
1125 if (handler == &ixgbevf_msix_clean_rx) {
1126 sprintf(adapter->name[vector], "%s-%s-%d",
1127 netdev->name, "rx", ri++);
1128 } else if (handler == &ixgbevf_msix_clean_tx) {
1129 sprintf(adapter->name[vector], "%s-%s-%d",
1130 netdev->name, "tx", ti++);
1131 } else if (handler == &ixgbevf_msix_clean_many) {
1132 sprintf(adapter->name[vector], "%s-%s-%d",
1133 netdev->name, "TxRx", vector);
1134 } else {
1135 /* skip this unused q_vector */
1136 continue;
1137 }
1138 err = request_irq(adapter->msix_entries[vector].vector,
1139 handler, 0, adapter->name[vector],
1140 adapter->q_vector[vector]);
1141 if (err) {
1142 hw_dbg(&adapter->hw,
1143 "request_irq failed for MSIX interrupt "
1144 "Error: %d\n", err);
1145 goto free_queue_irqs;
1146 }
1147 }
1148
1149 sprintf(adapter->name[vector], "%s:mbx", netdev->name);
1150 err = request_irq(adapter->msix_entries[vector].vector,
1151 &ixgbevf_msix_mbx, 0, adapter->name[vector], netdev);
1152 if (err) {
1153 hw_dbg(&adapter->hw,
1154 "request_irq for msix_mbx failed: %d\n", err);
1155 goto free_queue_irqs;
1156 }
1157
1158 return 0;
1159
1160free_queue_irqs:
1161 for (i = vector - 1; i >= 0; i--)
1162 free_irq(adapter->msix_entries[--vector].vector,
1163 &(adapter->q_vector[i]));
1164 pci_disable_msix(adapter->pdev);
1165 kfree(adapter->msix_entries);
1166 adapter->msix_entries = NULL;
1167 return err;
1168}
1169
1170static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1171{
1172 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1173
1174 for (i = 0; i < q_vectors; i++) {
1175 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
1176 bitmap_zero(q_vector->rxr_idx, MAX_RX_QUEUES);
1177 bitmap_zero(q_vector->txr_idx, MAX_TX_QUEUES);
1178 q_vector->rxr_count = 0;
1179 q_vector->txr_count = 0;
1180 q_vector->eitr = adapter->eitr_param;
1181 }
1182}
1183
1184/**
1185 * ixgbevf_request_irq - initialize interrupts
1186 * @adapter: board private structure
1187 *
1188 * Attempts to configure interrupts using the best available
1189 * capabilities of the hardware and kernel.
1190 **/
1191static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1192{
1193 int err = 0;
1194
1195 err = ixgbevf_request_msix_irqs(adapter);
1196
1197 if (err)
1198 hw_dbg(&adapter->hw,
1199 "request_irq failed, Error %d\n", err);
1200
1201 return err;
1202}
1203
1204static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1205{
1206 struct net_device *netdev = adapter->netdev;
1207 int i, q_vectors;
1208
1209 q_vectors = adapter->num_msix_vectors;
1210
1211 i = q_vectors - 1;
1212
1213 free_irq(adapter->msix_entries[i].vector, netdev);
1214 i--;
1215
1216 for (; i >= 0; i--) {
1217 free_irq(adapter->msix_entries[i].vector,
1218 adapter->q_vector[i]);
1219 }
1220
1221 ixgbevf_reset_q_vectors(adapter);
1222}
1223
1224/**
1225 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1226 * @adapter: board private structure
1227 **/
1228static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1229{
1230 int i;
1231 struct ixgbe_hw *hw = &adapter->hw;
1232
1233 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1234
1235 IXGBE_WRITE_FLUSH(hw);
1236
1237 for (i = 0; i < adapter->num_msix_vectors; i++)
1238 synchronize_irq(adapter->msix_entries[i].vector);
1239}
1240
1241/**
1242 * ixgbevf_irq_enable - Enable default interrupt generation settings
1243 * @adapter: board private structure
1244 **/
1245static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter,
1246 bool queues, bool flush)
1247{
1248 struct ixgbe_hw *hw = &adapter->hw;
1249 u32 mask;
1250 u64 qmask;
1251
1252 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1253 qmask = ~0;
1254
1255 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
1256
1257 if (queues)
1258 ixgbevf_irq_enable_queues(adapter, qmask);
1259
1260 if (flush)
1261 IXGBE_WRITE_FLUSH(hw);
1262}
1263
1264/**
1265 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1266 * @adapter: board private structure
1267 *
1268 * Configure the Tx unit of the MAC after a reset.
1269 **/
1270static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1271{
1272 u64 tdba;
1273 struct ixgbe_hw *hw = &adapter->hw;
1274 u32 i, j, tdlen, txctrl;
1275
1276 /* Setup the HW Tx Head and Tail descriptor pointers */
1277 for (i = 0; i < adapter->num_tx_queues; i++) {
1278 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1279 j = ring->reg_idx;
1280 tdba = ring->dma;
1281 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1282 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1283 (tdba & DMA_BIT_MASK(32)));
1284 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1285 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1286 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1287 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1288 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1289 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1290 /* Disable Tx Head Writeback RO bit, since this hoses
1291 * bookkeeping if things aren't delivered in order.
1292 */
1293 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1294 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1295 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1296 }
1297}
1298
1299#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1300
1301static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1302{
1303 struct ixgbevf_ring *rx_ring;
1304 struct ixgbe_hw *hw = &adapter->hw;
1305 u32 srrctl;
1306
1307 rx_ring = &adapter->rx_ring[index];
1308
1309 srrctl = IXGBE_SRRCTL_DROP_EN;
1310
1311 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
1312 u16 bufsz = IXGBEVF_RXBUFFER_2048;
1313 /* grow the amount we can receive on large page machines */
1314 if (bufsz < (PAGE_SIZE / 2))
1315 bufsz = (PAGE_SIZE / 2);
1316 /* cap the bufsz at our largest descriptor size */
1317 bufsz = min((u16)IXGBEVF_MAX_RXBUFFER, bufsz);
1318
1319 srrctl |= bufsz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1320 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1321 srrctl |= ((IXGBEVF_RX_HDR_SIZE <<
1322 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
1323 IXGBE_SRRCTL_BSIZEHDR_MASK);
1324 } else {
1325 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1326
1327 if (rx_ring->rx_buf_len == MAXIMUM_ETHERNET_VLAN_SIZE)
1328 srrctl |= IXGBEVF_RXBUFFER_2048 >>
1329 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1330 else
1331 srrctl |= rx_ring->rx_buf_len >>
1332 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1333 }
1334 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1335}
1336
1337/**
1338 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1339 * @adapter: board private structure
1340 *
1341 * Configure the Rx unit of the MAC after a reset.
1342 **/
1343static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1344{
1345 u64 rdba;
1346 struct ixgbe_hw *hw = &adapter->hw;
1347 struct net_device *netdev = adapter->netdev;
1348 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1349 int i, j;
1350 u32 rdlen;
1351 int rx_buf_len;
1352
1353 /* Decide whether to use packet split mode or not */
1354 if (netdev->mtu > ETH_DATA_LEN) {
1355 if (adapter->flags & IXGBE_FLAG_RX_PS_CAPABLE)
1356 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
1357 else
1358 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
1359 } else {
1360 if (adapter->flags & IXGBE_FLAG_RX_1BUF_CAPABLE)
1361 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
1362 else
1363 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
1364 }
1365
1366 /* Set the RX buffer length according to the mode */
1367 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
1368 /* PSRTYPE must be initialized in 82599 */
1369 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
1370 IXGBE_PSRTYPE_UDPHDR |
1371 IXGBE_PSRTYPE_IPV4HDR |
1372 IXGBE_PSRTYPE_IPV6HDR |
1373 IXGBE_PSRTYPE_L2HDR;
1374 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1375 rx_buf_len = IXGBEVF_RX_HDR_SIZE;
1376 } else {
1377 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
1378 if (netdev->mtu <= ETH_DATA_LEN)
1379 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1380 else
1381 rx_buf_len = ALIGN(max_frame, 1024);
1382 }
1383
1384 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1385 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1386 * the Base and Length of the Rx Descriptor Ring */
1387 for (i = 0; i < adapter->num_rx_queues; i++) {
1388 rdba = adapter->rx_ring[i].dma;
1389 j = adapter->rx_ring[i].reg_idx;
1390 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1391 (rdba & DMA_BIT_MASK(32)));
1392 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1393 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1394 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1395 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1396 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1397 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
1398 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1399
1400 ixgbevf_configure_srrctl(adapter, j);
1401 }
1402}
1403
8e586137 1404static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
92915f71
GR		 1405{
1406 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1407 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1408
1409 /* add VID to filter table */
1410 if (hw->mac.ops.set_vfta)
1411 hw->mac.ops.set_vfta(hw, vid, 0, true);
dadcd65f 1412 set_bit(vid, adapter->active_vlans);
8e586137
JP		 1413
1414 return 0;
92915f71
GR		 1415}
1416
8e586137 1417static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
92915f71
GR		 1418{
1419 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1420 struct ixgbe_hw *hw = &adapter->hw;
1421
92915f71
GR		 1422 /* remove VID from filter table */
1423 if (hw->mac.ops.set_vfta)
1424 hw->mac.ops.set_vfta(hw, vid, 0, false);
dadcd65f 1425 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 1426
1427 return 0;
92915f71
GR		 1428}
1429
1430static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1431{
dadcd65f 1432 u16 vid;
92915f71 1433
dadcd65f
JP		 1434 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1435 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
92915f71
GR		 1436}
1437
46ec20ff
GR		 1438static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1439{
1440 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1441 struct ixgbe_hw *hw = &adapter->hw;
1442 int count = 0;
1443
1444 if ((netdev_uc_count(netdev)) > 10) {
dbd9636e 1445 pr_err("Too many unicast filters - No Space\n");
46ec20ff
GR		 1446 return -ENOSPC;
1447 }
1448
1449 if (!netdev_uc_empty(netdev)) {
1450 struct netdev_hw_addr *ha;
1451 netdev_for_each_uc_addr(ha, netdev) {
1452 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1453 udelay(200);
1454 }
1455 } else {
1456 /*
1457 * If the list is empty then send message to PF driver to
1458 * clear all macvlans on this VF.
1459 */
1460 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1461 }
1462
1463 return count;
1464}
1465
92915f71
GR		 1466/**
1467 * ixgbevf_set_rx_mode - Multicast set
1468 * @netdev: network interface device structure
1469 *
1470 * The set_rx_method entry point is called whenever the multicast address
1471 * list or the network interface flags are updated. This routine is
1472 * responsible for configuring the hardware for proper multicast mode.
1473 **/
1474static void ixgbevf_set_rx_mode(struct net_device *netdev)
1475{
1476 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1477 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1478
1479 /* reprogram multicast list */
92915f71 1480 if (hw->mac.ops.update_mc_addr_list)
5c58c47a 1481 hw->mac.ops.update_mc_addr_list(hw, netdev);
46ec20ff
GR		 1482
1483 ixgbevf_write_uc_addr_list(netdev);
92915f71
GR		 1484}
1485
1486static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1487{
1488 int q_idx;
1489 struct ixgbevf_q_vector *q_vector;
1490 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1491
1492 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1493 struct napi_struct *napi;
1494 q_vector = adapter->q_vector[q_idx];
1495 if (!q_vector->rxr_count)
1496 continue;
1497 napi = &q_vector->napi;
1498 if (q_vector->rxr_count > 1)
1499 napi->poll = &ixgbevf_clean_rxonly_many;
1500
1501 napi_enable(napi);
1502 }
1503}
1504
1505static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1506{
1507 int q_idx;
1508 struct ixgbevf_q_vector *q_vector;
1509 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1510
1511 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1512 q_vector = adapter->q_vector[q_idx];
1513 if (!q_vector->rxr_count)
1514 continue;
1515 napi_disable(&q_vector->napi);
1516 }
1517}
1518
1519static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1520{
1521 struct net_device *netdev = adapter->netdev;
1522 int i;
1523
1524 ixgbevf_set_rx_mode(netdev);
1525
1526 ixgbevf_restore_vlan(adapter);
1527
1528 ixgbevf_configure_tx(adapter);
1529 ixgbevf_configure_rx(adapter);
1530 for (i = 0; i < adapter->num_rx_queues; i++) {
1531 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
1532 ixgbevf_alloc_rx_buffers(adapter, ring, ring->count);
1533 ring->next_to_use = ring->count - 1;
1534 writel(ring->next_to_use, adapter->hw.hw_addr + ring->tail);
1535 }
1536}
1537
1538#define IXGBE_MAX_RX_DESC_POLL 10
1539static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1540 int rxr)
1541{
1542 struct ixgbe_hw *hw = &adapter->hw;
1543 int j = adapter->rx_ring[rxr].reg_idx;
1544 int k;
1545
1546 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1547 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1548 break;
1549 else
1550 msleep(1);
1551 }
1552 if (k >= IXGBE_MAX_RX_DESC_POLL) {
1553 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1554 "not set within the polling period\n", rxr);
1555 }
1556
1557 ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr],
1558 (adapter->rx_ring[rxr].count - 1));
1559}
1560
33bd9f60
GR		 1561static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1562{
1563 /* Only save pre-reset stats if there are some */
1564 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1565 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1566 adapter->stats.base_vfgprc;
1567 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1568 adapter->stats.base_vfgptc;
1569 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1570 adapter->stats.base_vfgorc;
1571 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1572 adapter->stats.base_vfgotc;
1573 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1574 adapter->stats.base_vfmprc;
1575 }
1576}
1577
1578static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1579{
1580 struct ixgbe_hw *hw = &adapter->hw;
1581
1582 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1583 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1584 adapter->stats.last_vfgorc |=
1585 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1586 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1587 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1588 adapter->stats.last_vfgotc |=
1589 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1590 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1591
1592 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1593 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1594 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1595 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1596 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1597}
1598
795180d8 1599static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
92915f71
GR		 1600{
1601 struct net_device *netdev = adapter->netdev;
1602 struct ixgbe_hw *hw = &adapter->hw;
1603 int i, j = 0;
1604 int num_rx_rings = adapter->num_rx_queues;
1605 u32 txdctl, rxdctl;
795180d8 1606 u32 msg[2];
92915f71
GR		 1607
1608 for (i = 0; i < adapter->num_tx_queues; i++) {
1609 j = adapter->tx_ring[i].reg_idx;
1610 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1611 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1612 txdctl |= (8 << 16);
1613 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1614 }
1615
1616 for (i = 0; i < adapter->num_tx_queues; i++) {
1617 j = adapter->tx_ring[i].reg_idx;
1618 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1619 txdctl |= IXGBE_TXDCTL_ENABLE;
1620 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1621 }
1622
1623 for (i = 0; i < num_rx_rings; i++) {
1624 j = adapter->rx_ring[i].reg_idx;
1625 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
dadcd65f 1626 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
69bfbec4
GR		 1627 if (hw->mac.type == ixgbe_mac_X540_vf) {
1628 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1629 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1630 IXGBE_RXDCTL_RLPML_EN);
1631 }
92915f71
GR		 1632 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1633 ixgbevf_rx_desc_queue_enable(adapter, i);
1634 }
1635
1636 ixgbevf_configure_msix(adapter);
1637
1638 if (hw->mac.ops.set_rar) {
1639 if (is_valid_ether_addr(hw->mac.addr))
1640 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1641 else
1642 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1643 }
1644
795180d8
GR		 1645 msg[0] = IXGBE_VF_SET_LPE;
1646 msg[1] = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1647 hw->mbx.ops.write_posted(hw, msg, 2);
1648
92915f71
GR		 1649 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1650 ixgbevf_napi_enable_all(adapter);
1651
1652 /* enable transmits */
1653 netif_tx_start_all_queues(netdev);
1654
33bd9f60
GR		 1655 ixgbevf_save_reset_stats(adapter);
1656 ixgbevf_init_last_counter_stats(adapter);
1657
92915f71
GR		 1658 /* bring the link up in the watchdog, this could race with our first
1659 * link up interrupt but shouldn't be a problem */
1660 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1661 adapter->link_check_timeout = jiffies;
1662 mod_timer(&adapter->watchdog_timer, jiffies);
92915f71
GR		 1663}
1664
795180d8 1665void ixgbevf_up(struct ixgbevf_adapter *adapter)
92915f71 1666{
92915f71
GR		 1667 struct ixgbe_hw *hw = &adapter->hw;
1668
1669 ixgbevf_configure(adapter);
1670
795180d8 1671 ixgbevf_up_complete(adapter);
92915f71
GR		 1672
1673 /* clear any pending interrupts, may auto mask */
1674 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1675
1676 ixgbevf_irq_enable(adapter, true, true);
92915f71
GR		 1677}
1678
1679/**
1680 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1681 * @adapter: board private structure
1682 * @rx_ring: ring to free buffers from
1683 **/
1684static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1685 struct ixgbevf_ring *rx_ring)
1686{
1687 struct pci_dev *pdev = adapter->pdev;
1688 unsigned long size;
1689 unsigned int i;
1690
c0456c23
GR		 1691 if (!rx_ring->rx_buffer_info)
1692 return;
92915f71 1693
c0456c23 1694 /* Free all the Rx ring sk_buffs */
92915f71
GR		 1695 for (i = 0; i < rx_ring->count; i++) {
1696 struct ixgbevf_rx_buffer *rx_buffer_info;
1697
1698 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1699 if (rx_buffer_info->dma) {
2a1f8794 1700 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
92915f71 1701 rx_ring->rx_buf_len,
2a1f8794 1702 DMA_FROM_DEVICE);
92915f71
GR		 1703 rx_buffer_info->dma = 0;
1704 }
1705 if (rx_buffer_info->skb) {
1706 struct sk_buff *skb = rx_buffer_info->skb;
1707 rx_buffer_info->skb = NULL;
1708 do {
1709 struct sk_buff *this = skb;
1710 skb = skb->prev;
1711 dev_kfree_skb(this);
1712 } while (skb);
1713 }
1714 if (!rx_buffer_info->page)
1715 continue;
2a1f8794
NN		 1716 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma,
1717 PAGE_SIZE / 2, DMA_FROM_DEVICE);
92915f71
GR		 1718 rx_buffer_info->page_dma = 0;
1719 put_page(rx_buffer_info->page);
1720 rx_buffer_info->page = NULL;
1721 rx_buffer_info->page_offset = 0;
1722 }
1723
1724 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1725 memset(rx_ring->rx_buffer_info, 0, size);
1726
1727 /* Zero out the descriptor ring */
1728 memset(rx_ring->desc, 0, rx_ring->size);
1729
1730 rx_ring->next_to_clean = 0;
1731 rx_ring->next_to_use = 0;
1732
1733 if (rx_ring->head)
1734 writel(0, adapter->hw.hw_addr + rx_ring->head);
1735 if (rx_ring->tail)
1736 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1737}
1738
1739/**
1740 * ixgbevf_clean_tx_ring - Free Tx Buffers
1741 * @adapter: board private structure
1742 * @tx_ring: ring to be cleaned
1743 **/
1744static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1745 struct ixgbevf_ring *tx_ring)
1746{
1747 struct ixgbevf_tx_buffer *tx_buffer_info;
1748 unsigned long size;
1749 unsigned int i;
1750
c0456c23
GR		 1751 if (!tx_ring->tx_buffer_info)
1752 return;
1753
92915f71
GR		 1754 /* Free all the Tx ring sk_buffs */
1755
1756 for (i = 0; i < tx_ring->count; i++) {
1757 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1758 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info);
1759 }
1760
1761 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1762 memset(tx_ring->tx_buffer_info, 0, size);
1763
1764 memset(tx_ring->desc, 0, tx_ring->size);
1765
1766 tx_ring->next_to_use = 0;
1767 tx_ring->next_to_clean = 0;
1768
1769 if (tx_ring->head)
1770 writel(0, adapter->hw.hw_addr + tx_ring->head);
1771 if (tx_ring->tail)
1772 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1773}
1774
1775/**
1776 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1777 * @adapter: board private structure
1778 **/
1779static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1780{
1781 int i;
1782
1783 for (i = 0; i < adapter->num_rx_queues; i++)
1784 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1785}
1786
1787/**
1788 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1789 * @adapter: board private structure
1790 **/
1791static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1792{
1793 int i;
1794
1795 for (i = 0; i < adapter->num_tx_queues; i++)
1796 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1797}
1798
1799void ixgbevf_down(struct ixgbevf_adapter *adapter)
1800{
1801 struct net_device *netdev = adapter->netdev;
1802 struct ixgbe_hw *hw = &adapter->hw;
1803 u32 txdctl;
1804 int i, j;
1805
1806 /* signal that we are down to the interrupt handler */
1807 set_bit(__IXGBEVF_DOWN, &adapter->state);
1808 /* disable receives */
1809
1810 netif_tx_disable(netdev);
1811
1812 msleep(10);
1813
1814 netif_tx_stop_all_queues(netdev);
1815
1816 ixgbevf_irq_disable(adapter);
1817
1818 ixgbevf_napi_disable_all(adapter);
1819
1820 del_timer_sync(&adapter->watchdog_timer);
1821 /* can't call flush scheduled work here because it can deadlock
1822 * if linkwatch_event tries to acquire the rtnl_lock which we are
1823 * holding */
1824 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1825 msleep(1);
1826
1827 /* disable transmits in the hardware now that interrupts are off */
1828 for (i = 0; i < adapter->num_tx_queues; i++) {
1829 j = adapter->tx_ring[i].reg_idx;
1830 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1831 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1832 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1833 }
1834
1835 netif_carrier_off(netdev);
1836
1837 if (!pci_channel_offline(adapter->pdev))
1838 ixgbevf_reset(adapter);
1839
1840 ixgbevf_clean_all_tx_rings(adapter);
1841 ixgbevf_clean_all_rx_rings(adapter);
1842}
1843
1844void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1845{
c0456c23
GR		 1846 struct ixgbe_hw *hw = &adapter->hw;
1847
92915f71 1848 WARN_ON(in_interrupt());
c0456c23 1849
92915f71
GR		 1850 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1851 msleep(1);
1852
c0456c23
GR		 1853 /*
1854 * Check if PF is up before re-init. If not then skip until
1855 * later when the PF is up and ready to service requests from
1856 * the VF via mailbox. If the VF is up and running then the
1857 * watchdog task will continue to schedule reset tasks until
1858 * the PF is up and running.
1859 */
1860 if (!hw->mac.ops.reset_hw(hw)) {
1861 ixgbevf_down(adapter);
1862 ixgbevf_up(adapter);
1863 }
92915f71
GR		 1864
1865 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1866}
1867
1868void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1869{
1870 struct ixgbe_hw *hw = &adapter->hw;
1871 struct net_device *netdev = adapter->netdev;
1872
1873 if (hw->mac.ops.reset_hw(hw))
1874 hw_dbg(hw, "PF still resetting\n");
1875 else
1876 hw->mac.ops.init_hw(hw);
1877
1878 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1879 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1880 netdev->addr_len);
1881 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1882 netdev->addr_len);
1883 }
1884}
1885
1886static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1887 int vectors)
1888{
1889 int err, vector_threshold;
1890
1891 /* We'll want at least 3 (vector_threshold):
1892 * 1) TxQ[0] Cleanup
1893 * 2) RxQ[0] Cleanup
1894 * 3) Other (Link Status Change, etc.)
1895 */
1896 vector_threshold = MIN_MSIX_COUNT;
1897
1898 /* The more we get, the more we will assign to Tx/Rx Cleanup
1899 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1900 * Right now, we simply care about how many we'll get; we'll
1901 * set them up later while requesting irq's.
1902 */
1903 while (vectors >= vector_threshold) {
1904 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1905 vectors);
1906 if (!err) /* Success in acquiring all requested vectors. */
1907 break;
1908 else if (err < 0)
1909 vectors = 0; /* Nasty failure, quit now */
1910 else /* err == number of vectors we should try again with */
1911 vectors = err;
1912 }
1913
1914 if (vectors < vector_threshold) {
1915 /* Can't allocate enough MSI-X interrupts? Oh well.
1916 * This just means we'll go with either a single MSI
1917 * vector or fall back to legacy interrupts.
1918 */
1919 hw_dbg(&adapter->hw,
1920 "Unable to allocate MSI-X interrupts\n");
1921 kfree(adapter->msix_entries);
1922 adapter->msix_entries = NULL;
1923 } else {
1924 /*
1925 * Adjust for only the vectors we'll use, which is minimum
1926 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1927 * vectors we were allocated.
1928 */
1929 adapter->num_msix_vectors = vectors;
1930 }
1931}
1932
1933/*
25985edc 1934 * ixgbevf_set_num_queues: Allocate queues for device, feature dependent
92915f71
GR		 1935 * @adapter: board private structure to initialize
1936 *
1937 * This is the top level queue allocation routine. The order here is very
1938 * important, starting with the "most" number of features turned on at once,
1939 * and ending with the smallest set of features. This way large combinations
1940 * can be allocated if they're turned on, and smaller combinations are the
1941 * fallthrough conditions.
1942 *
1943 **/
1944static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1945{
1946 /* Start with base case */
1947 adapter->num_rx_queues = 1;
1948 adapter->num_tx_queues = 1;
1949 adapter->num_rx_pools = adapter->num_rx_queues;
1950 adapter->num_rx_queues_per_pool = 1;
1951}
1952
1953/**
1954 * ixgbevf_alloc_queues - Allocate memory for all rings
1955 * @adapter: board private structure to initialize
1956 *
1957 * We allocate one ring per queue at run-time since we don't know the
1958 * number of queues at compile-time. The polling_netdev array is
1959 * intended for Multiqueue, but should work fine with a single queue.
1960 **/
1961static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1962{
1963 int i;
1964
1965 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1966 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1967 if (!adapter->tx_ring)
1968 goto err_tx_ring_allocation;
1969
1970 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1971 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1972 if (!adapter->rx_ring)
1973 goto err_rx_ring_allocation;
1974
1975 for (i = 0; i < adapter->num_tx_queues; i++) {
1976 adapter->tx_ring[i].count = adapter->tx_ring_count;
1977 adapter->tx_ring[i].queue_index = i;
1978 adapter->tx_ring[i].reg_idx = i;
1979 }
1980
1981 for (i = 0; i < adapter->num_rx_queues; i++) {
1982 adapter->rx_ring[i].count = adapter->rx_ring_count;
1983 adapter->rx_ring[i].queue_index = i;
1984 adapter->rx_ring[i].reg_idx = i;
1985 }
1986
1987 return 0;
1988
1989err_rx_ring_allocation:
1990 kfree(adapter->tx_ring);
1991err_tx_ring_allocation:
1992 return -ENOMEM;
1993}
1994
1995/**
1996 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1997 * @adapter: board private structure to initialize
1998 *
1999 * Attempt to configure the interrupts using the best available
2000 * capabilities of the hardware and the kernel.
2001 **/
2002static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2003{
2004 int err = 0;
2005 int vector, v_budget;
2006
2007 /*
2008 * It's easy to be greedy for MSI-X vectors, but it really
2009 * doesn't do us much good if we have a lot more vectors
2010 * than CPU's. So let's be conservative and only ask for
2011 * (roughly) twice the number of vectors as there are CPU's.
2012 */
2013 v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues,
2014 (int)(num_online_cpus() * 2)) + NON_Q_VECTORS;
2015
2016 /* A failure in MSI-X entry allocation isn't fatal, but it does
2017 * mean we disable MSI-X capabilities of the adapter. */
2018 adapter->msix_entries = kcalloc(v_budget,
2019 sizeof(struct msix_entry), GFP_KERNEL);
2020 if (!adapter->msix_entries) {
2021 err = -ENOMEM;
2022 goto out;
2023 }
2024
2025 for (vector = 0; vector < v_budget; vector++)
2026 adapter->msix_entries[vector].entry = vector;
2027
2028 ixgbevf_acquire_msix_vectors(adapter, v_budget);
2029
2030out:
2031 return err;
2032}
2033
2034/**
2035 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2036 * @adapter: board private structure to initialize
2037 *
2038 * We allocate one q_vector per queue interrupt. If allocation fails we
2039 * return -ENOMEM.
2040 **/
2041static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2042{
2043 int q_idx, num_q_vectors;
2044 struct ixgbevf_q_vector *q_vector;
2045 int napi_vectors;
2046 int (*poll)(struct napi_struct *, int);
2047
2048 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2049 napi_vectors = adapter->num_rx_queues;
2050 poll = &ixgbevf_clean_rxonly;
2051
2052 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2053 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2054 if (!q_vector)
2055 goto err_out;
2056 q_vector->adapter = adapter;
2057 q_vector->v_idx = q_idx;
2058 q_vector->eitr = adapter->eitr_param;
2059 if (q_idx < napi_vectors)
2060 netif_napi_add(adapter->netdev, &q_vector->napi,
2061 (*poll), 64);
2062 adapter->q_vector[q_idx] = q_vector;
2063 }
2064
2065 return 0;
2066
2067err_out:
2068 while (q_idx) {
2069 q_idx--;
2070 q_vector = adapter->q_vector[q_idx];
2071 netif_napi_del(&q_vector->napi);
2072 kfree(q_vector);
2073 adapter->q_vector[q_idx] = NULL;
2074 }
2075 return -ENOMEM;
2076}
2077
2078/**
2079 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2080 * @adapter: board private structure to initialize
2081 *
2082 * This function frees the memory allocated to the q_vectors. In addition if
2083 * NAPI is enabled it will delete any references to the NAPI struct prior
2084 * to freeing the q_vector.
2085 **/
2086static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2087{
2088 int q_idx, num_q_vectors;
2089 int napi_vectors;
2090
2091 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2092 napi_vectors = adapter->num_rx_queues;
2093
2094 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2095 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2096
2097 adapter->q_vector[q_idx] = NULL;
2098 if (q_idx < napi_vectors)
2099 netif_napi_del(&q_vector->napi);
2100 kfree(q_vector);
2101 }
2102}
2103
2104/**
2105 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2106 * @adapter: board private structure
2107 *
2108 **/
2109static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2110{
2111 pci_disable_msix(adapter->pdev);
2112 kfree(adapter->msix_entries);
2113 adapter->msix_entries = NULL;
92915f71
GR		 2114}
2115
2116/**
2117 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2118 * @adapter: board private structure to initialize
2119 *
2120 **/
2121static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2122{
2123 int err;
2124
2125 /* Number of supported queues */
2126 ixgbevf_set_num_queues(adapter);
2127
2128 err = ixgbevf_set_interrupt_capability(adapter);
2129 if (err) {
2130 hw_dbg(&adapter->hw,
2131 "Unable to setup interrupt capabilities\n");
2132 goto err_set_interrupt;
2133 }
2134
2135 err = ixgbevf_alloc_q_vectors(adapter);
2136 if (err) {
2137 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2138 "vectors\n");
2139 goto err_alloc_q_vectors;
2140 }
2141
2142 err = ixgbevf_alloc_queues(adapter);
2143 if (err) {
dbd9636e 2144 pr_err("Unable to allocate memory for queues\n");
92915f71
GR		 2145 goto err_alloc_queues;
2146 }
2147
2148 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2149 "Tx Queue count = %u\n",
2150 (adapter->num_rx_queues > 1) ? "Enabled" :
2151 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2152
2153 set_bit(__IXGBEVF_DOWN, &adapter->state);
2154
2155 return 0;
2156err_alloc_queues:
2157 ixgbevf_free_q_vectors(adapter);
2158err_alloc_q_vectors:
2159 ixgbevf_reset_interrupt_capability(adapter);
2160err_set_interrupt:
2161 return err;
2162}
2163
2164/**
2165 * ixgbevf_sw_init - Initialize general software structures
2166 * (struct ixgbevf_adapter)
2167 * @adapter: board private structure to initialize
2168 *
2169 * ixgbevf_sw_init initializes the Adapter private data structure.
2170 * Fields are initialized based on PCI device information and
2171 * OS network device settings (MTU size).
2172 **/
2173static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
2174{
2175 struct ixgbe_hw *hw = &adapter->hw;
2176 struct pci_dev *pdev = adapter->pdev;
2177 int err;
2178
2179 /* PCI config space info */
2180
2181 hw->vendor_id = pdev->vendor;
2182 hw->device_id = pdev->device;
ff938e43 2183 hw->revision_id = pdev->revision;
92915f71
GR		 2184 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2185 hw->subsystem_device_id = pdev->subsystem_device;
2186
2187 hw->mbx.ops.init_params(hw);
2188 hw->mac.max_tx_queues = MAX_TX_QUEUES;
2189 hw->mac.max_rx_queues = MAX_RX_QUEUES;
2190 err = hw->mac.ops.reset_hw(hw);
2191 if (err) {
2192 dev_info(&pdev->dev,
2193 "PF still in reset state, assigning new address\n");
1a0d6ae5
DK		 2194 eth_hw_addr_random(adapter->netdev);
2195 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr,
2196 adapter->netdev->addr_len);
92915f71
GR		 2197 } else {
2198 err = hw->mac.ops.init_hw(hw);
2199 if (err) {
dbd9636e 2200 pr_err("init_shared_code failed: %d\n", err);
92915f71
GR		 2201 goto out;
2202 }
1a0d6ae5
DK		 2203 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr,
2204 adapter->netdev->addr_len);
92915f71
GR		 2205 }
2206
2207 /* Enable dynamic interrupt throttling rates */
2208 adapter->eitr_param = 20000;
2209 adapter->itr_setting = 1;
2210
2211 /* set defaults for eitr in MegaBytes */
2212 adapter->eitr_low = 10;
2213 adapter->eitr_high = 20;
2214
2215 /* set default ring sizes */
2216 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2217 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2218
2219 /* enable rx csum by default */
2220 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
2221
2222 set_bit(__IXGBEVF_DOWN, &adapter->state);
1a0d6ae5 2223 return 0;
92915f71
GR		 2224
2225out:
2226 return err;
2227}
2228
92915f71
GR		 2229#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
2230 { \
2231 u32 current_counter = IXGBE_READ_REG(hw, reg); \
2232 if (current_counter < last_counter) \
2233 counter += 0x100000000LL; \
2234 last_counter = current_counter; \
2235 counter &= 0xFFFFFFFF00000000LL; \
2236 counter |= current_counter; \
2237 }
2238
2239#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2240 { \
2241 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2242 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2243 u64 current_counter = (current_counter_msb << 32) | \
2244 current_counter_lsb; \
2245 if (current_counter < last_counter) \
2246 counter += 0x1000000000LL; \
2247 last_counter = current_counter; \
2248 counter &= 0xFFFFFFF000000000LL; \
2249 counter |= current_counter; \
2250 }
2251/**
2252 * ixgbevf_update_stats - Update the board statistics counters.
2253 * @adapter: board private structure
2254 **/
2255void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2256{
2257 struct ixgbe_hw *hw = &adapter->hw;
2258
2259 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2260 adapter->stats.vfgprc);
2261 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2262 adapter->stats.vfgptc);
2263 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2264 adapter->stats.last_vfgorc,
2265 adapter->stats.vfgorc);
2266 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2267 adapter->stats.last_vfgotc,
2268 adapter->stats.vfgotc);
2269 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2270 adapter->stats.vfmprc);
92915f71
GR		 2271}
2272
2273/**
2274 * ixgbevf_watchdog - Timer Call-back
2275 * @data: pointer to adapter cast into an unsigned long
2276 **/
2277static void ixgbevf_watchdog(unsigned long data)
2278{
2279 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2280 struct ixgbe_hw *hw = &adapter->hw;
2281 u64 eics = 0;
2282 int i;
2283
2284 /*
2285 * Do the watchdog outside of interrupt context due to the lovely
2286 * delays that some of the newer hardware requires
2287 */
2288
2289 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2290 goto watchdog_short_circuit;
2291
2292 /* get one bit for every active tx/rx interrupt vector */
2293 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2294 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2295 if (qv->rxr_count || qv->txr_count)
2296 eics |= (1 << i);
2297 }
2298
2299 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, (u32)eics);
2300
2301watchdog_short_circuit:
2302 schedule_work(&adapter->watchdog_task);
2303}
2304
2305/**
2306 * ixgbevf_tx_timeout - Respond to a Tx Hang
2307 * @netdev: network interface device structure
2308 **/
2309static void ixgbevf_tx_timeout(struct net_device *netdev)
2310{
2311 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2312
2313 /* Do the reset outside of interrupt context */
2314 schedule_work(&adapter->reset_task);
2315}
2316
2317static void ixgbevf_reset_task(struct work_struct *work)
2318{
2319 struct ixgbevf_adapter *adapter;
2320 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2321
2322 /* If we're already down or resetting, just bail */
2323 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2324 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2325 return;
2326
2327 adapter->tx_timeout_count++;
2328
2329 ixgbevf_reinit_locked(adapter);
2330}
2331
2332/**
2333 * ixgbevf_watchdog_task - worker thread to bring link up
2334 * @work: pointer to work_struct containing our data
2335 **/
2336static void ixgbevf_watchdog_task(struct work_struct *work)
2337{
2338 struct ixgbevf_adapter *adapter = container_of(work,
2339 struct ixgbevf_adapter,
2340 watchdog_task);
2341 struct net_device *netdev = adapter->netdev;
2342 struct ixgbe_hw *hw = &adapter->hw;
2343 u32 link_speed = adapter->link_speed;
2344 bool link_up = adapter->link_up;
2345
2346 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2347
2348 /*
2349 * Always check the link on the watchdog because we have
2350 * no LSC interrupt
2351 */
2352 if (hw->mac.ops.check_link) {
2353 if ((hw->mac.ops.check_link(hw, &link_speed,
2354 &link_up, false)) != 0) {
2355 adapter->link_up = link_up;
2356 adapter->link_speed = link_speed;
da6b3330
GR		 2357 netif_carrier_off(netdev);
2358 netif_tx_stop_all_queues(netdev);
92915f71
GR		 2359 schedule_work(&adapter->reset_task);
2360 goto pf_has_reset;
2361 }
2362 } else {
2363 /* always assume link is up, if no check link
2364 * function */
2365 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
2366 link_up = true;
2367 }
2368 adapter->link_up = link_up;
2369 adapter->link_speed = link_speed;
2370
2371 if (link_up) {
2372 if (!netif_carrier_ok(netdev)) {
300bc060
JP		 2373 hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n",
2374 (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
2375 10 : 1);
92915f71
GR		 2376 netif_carrier_on(netdev);
2377 netif_tx_wake_all_queues(netdev);
92915f71
GR		 2378 }
2379 } else {
2380 adapter->link_up = false;
2381 adapter->link_speed = 0;
2382 if (netif_carrier_ok(netdev)) {
2383 hw_dbg(&adapter->hw, "NIC Link is Down\n");
2384 netif_carrier_off(netdev);
2385 netif_tx_stop_all_queues(netdev);
2386 }
2387 }
2388
92915f71
GR		 2389 ixgbevf_update_stats(adapter);
2390
33bd9f60 2391pf_has_reset:
92915f71
GR		 2392 /* Reset the timer */
2393 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2394 mod_timer(&adapter->watchdog_timer,
2395 round_jiffies(jiffies + (2 * HZ)));
2396
2397 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2398}
2399
2400/**
2401 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2402 * @adapter: board private structure
2403 * @tx_ring: Tx descriptor ring for a specific queue
2404 *
2405 * Free all transmit software resources
2406 **/
2407void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2408 struct ixgbevf_ring *tx_ring)
2409{
2410 struct pci_dev *pdev = adapter->pdev;
2411
92915f71
GR		 2412 ixgbevf_clean_tx_ring(adapter, tx_ring);
2413
2414 vfree(tx_ring->tx_buffer_info);
2415 tx_ring->tx_buffer_info = NULL;
2416
2a1f8794
NN		 2417 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2418 tx_ring->dma);
92915f71
GR		 2419
2420 tx_ring->desc = NULL;
2421}
2422
2423/**
2424 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2425 * @adapter: board private structure
2426 *
2427 * Free all transmit software resources
2428 **/
2429static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2430{
2431 int i;
2432
2433 for (i = 0; i < adapter->num_tx_queues; i++)
2434 if (adapter->tx_ring[i].desc)
2435 ixgbevf_free_tx_resources(adapter,
2436 &adapter->tx_ring[i]);
2437
2438}
2439
2440/**
2441 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2442 * @adapter: board private structure
2443 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2444 *
2445 * Return 0 on success, negative on failure
2446 **/
2447int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2448 struct ixgbevf_ring *tx_ring)
2449{
2450 struct pci_dev *pdev = adapter->pdev;
2451 int size;
2452
2453 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
89bf67f1 2454 tx_ring->tx_buffer_info = vzalloc(size);
92915f71
GR		 2455 if (!tx_ring->tx_buffer_info)
2456 goto err;
92915f71
GR		 2457
2458 /* round up to nearest 4K */
2459 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2460 tx_ring->size = ALIGN(tx_ring->size, 4096);
2461
2a1f8794
NN		 2462 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2463 &tx_ring->dma, GFP_KERNEL);
92915f71
GR		 2464 if (!tx_ring->desc)
2465 goto err;
2466
2467 tx_ring->next_to_use = 0;
2468 tx_ring->next_to_clean = 0;
2469 tx_ring->work_limit = tx_ring->count;
2470 return 0;
2471
2472err:
2473 vfree(tx_ring->tx_buffer_info);
2474 tx_ring->tx_buffer_info = NULL;
2475 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2476 "descriptor ring\n");
2477 return -ENOMEM;
2478}
2479
2480/**
2481 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2482 * @adapter: board private structure
2483 *
2484 * If this function returns with an error, then it's possible one or
2485 * more of the rings is populated (while the rest are not). It is the
2486 * callers duty to clean those orphaned rings.
2487 *
2488 * Return 0 on success, negative on failure
2489 **/
2490static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2491{
2492 int i, err = 0;
2493
2494 for (i = 0; i < adapter->num_tx_queues; i++) {
2495 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2496 if (!err)
2497 continue;
2498 hw_dbg(&adapter->hw,
2499 "Allocation for Tx Queue %u failed\n", i);
2500 break;
2501 }
2502
2503 return err;
2504}
2505
2506/**
2507 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2508 * @adapter: board private structure
2509 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2510 *
2511 * Returns 0 on success, negative on failure
2512 **/
2513int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2514 struct ixgbevf_ring *rx_ring)
2515{
2516 struct pci_dev *pdev = adapter->pdev;
2517 int size;
2518
2519 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
89bf67f1 2520 rx_ring->rx_buffer_info = vzalloc(size);
e404decb 2521 if (!rx_ring->rx_buffer_info)
92915f71 2522 goto alloc_failed;
92915f71
GR		 2523
2524 /* Round up to nearest 4K */
2525 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2526 rx_ring->size = ALIGN(rx_ring->size, 4096);
2527
2a1f8794
NN		 2528 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2529 &rx_ring->dma, GFP_KERNEL);
92915f71
GR		 2530
2531 if (!rx_ring->desc) {
2532 hw_dbg(&adapter->hw,
2533 "Unable to allocate memory for "
2534 "the receive descriptor ring\n");
2535 vfree(rx_ring->rx_buffer_info);
2536 rx_ring->rx_buffer_info = NULL;
2537 goto alloc_failed;
2538 }
2539
2540 rx_ring->next_to_clean = 0;
2541 rx_ring->next_to_use = 0;
2542
2543 return 0;
2544alloc_failed:
2545 return -ENOMEM;
2546}
2547
2548/**
2549 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2550 * @adapter: board private structure
2551 *
2552 * If this function returns with an error, then it's possible one or
2553 * more of the rings is populated (while the rest are not). It is the
2554 * callers duty to clean those orphaned rings.
2555 *
2556 * Return 0 on success, negative on failure
2557 **/
2558static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2559{
2560 int i, err = 0;
2561
2562 for (i = 0; i < adapter->num_rx_queues; i++) {
2563 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2564 if (!err)
2565 continue;
2566 hw_dbg(&adapter->hw,
2567 "Allocation for Rx Queue %u failed\n", i);
2568 break;
2569 }
2570 return err;
2571}
2572
2573/**
2574 * ixgbevf_free_rx_resources - Free Rx Resources
2575 * @adapter: board private structure
2576 * @rx_ring: ring to clean the resources from
2577 *
2578 * Free all receive software resources
2579 **/
2580void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2581 struct ixgbevf_ring *rx_ring)
2582{
2583 struct pci_dev *pdev = adapter->pdev;
2584
2585 ixgbevf_clean_rx_ring(adapter, rx_ring);
2586
2587 vfree(rx_ring->rx_buffer_info);
2588 rx_ring->rx_buffer_info = NULL;
2589
2a1f8794
NN		 2590 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2591 rx_ring->dma);
92915f71
GR		 2592
2593 rx_ring->desc = NULL;
2594}
2595
2596/**
2597 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2598 * @adapter: board private structure
2599 *
2600 * Free all receive software resources
2601 **/
2602static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2603{
2604 int i;
2605
2606 for (i = 0; i < adapter->num_rx_queues; i++)
2607 if (adapter->rx_ring[i].desc)
2608 ixgbevf_free_rx_resources(adapter,
2609 &adapter->rx_ring[i]);
2610}
2611
2612/**
2613 * ixgbevf_open - Called when a network interface is made active
2614 * @netdev: network interface device structure
2615 *
2616 * Returns 0 on success, negative value on failure
2617 *
2618 * The open entry point is called when a network interface is made
2619 * active by the system (IFF_UP). At this point all resources needed
2620 * for transmit and receive operations are allocated, the interrupt
2621 * handler is registered with the OS, the watchdog timer is started,
2622 * and the stack is notified that the interface is ready.
2623 **/
2624static int ixgbevf_open(struct net_device *netdev)
2625{
2626 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2627 struct ixgbe_hw *hw = &adapter->hw;
2628 int err;
2629
2630 /* disallow open during test */
2631 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2632 return -EBUSY;
2633
2634 if (hw->adapter_stopped) {
2635 ixgbevf_reset(adapter);
2636 /* if adapter is still stopped then PF isn't up and
2637 * the vf can't start. */
2638 if (hw->adapter_stopped) {
2639 err = IXGBE_ERR_MBX;
dbd9636e
JK		 2640 pr_err("Unable to start - perhaps the PF Driver isn't "
2641 "up yet\n");
92915f71
GR		 2642 goto err_setup_reset;
2643 }
2644 }
2645
2646 /* allocate transmit descriptors */
2647 err = ixgbevf_setup_all_tx_resources(adapter);
2648 if (err)
2649 goto err_setup_tx;
2650
2651 /* allocate receive descriptors */
2652 err = ixgbevf_setup_all_rx_resources(adapter);
2653 if (err)
2654 goto err_setup_rx;
2655
2656 ixgbevf_configure(adapter);
2657
2658 /*
2659 * Map the Tx/Rx rings to the vectors we were allotted.
2660 * if request_irq will be called in this function map_rings
2661 * must be called *before* up_complete
2662 */
2663 ixgbevf_map_rings_to_vectors(adapter);
2664
795180d8 2665 ixgbevf_up_complete(adapter);
92915f71
GR		 2666
2667 /* clear any pending interrupts, may auto mask */
2668 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2669 err = ixgbevf_request_irq(adapter);
2670 if (err)
2671 goto err_req_irq;
2672
2673 ixgbevf_irq_enable(adapter, true, true);
2674
2675 return 0;
2676
2677err_req_irq:
2678 ixgbevf_down(adapter);
92915f71
GR		 2679 ixgbevf_free_irq(adapter);
2680err_setup_rx:
2681 ixgbevf_free_all_rx_resources(adapter);
2682err_setup_tx:
2683 ixgbevf_free_all_tx_resources(adapter);
2684 ixgbevf_reset(adapter);
2685
2686err_setup_reset:
2687
2688 return err;
2689}
2690
2691/**
2692 * ixgbevf_close - Disables a network interface
2693 * @netdev: network interface device structure
2694 *
2695 * Returns 0, this is not allowed to fail
2696 *
2697 * The close entry point is called when an interface is de-activated
2698 * by the OS. The hardware is still under the drivers control, but
2699 * needs to be disabled. A global MAC reset is issued to stop the
2700 * hardware, and all transmit and receive resources are freed.
2701 **/
2702static int ixgbevf_close(struct net_device *netdev)
2703{
2704 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2705
2706 ixgbevf_down(adapter);
2707 ixgbevf_free_irq(adapter);
2708
2709 ixgbevf_free_all_tx_resources(adapter);
2710 ixgbevf_free_all_rx_resources(adapter);
2711
2712 return 0;
2713}
2714
2715static int ixgbevf_tso(struct ixgbevf_adapter *adapter,
2716 struct ixgbevf_ring *tx_ring,
2717 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2718{
2719 struct ixgbe_adv_tx_context_desc *context_desc;
2720 unsigned int i;
2721 int err;
2722 struct ixgbevf_tx_buffer *tx_buffer_info;
2723 u32 vlan_macip_lens = 0, type_tucmd_mlhl;
2724 u32 mss_l4len_idx, l4len;
2725
2726 if (skb_is_gso(skb)) {
2727 if (skb_header_cloned(skb)) {
2728 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2729 if (err)
2730 return err;
2731 }
2732 l4len = tcp_hdrlen(skb);
2733 *hdr_len += l4len;
2734
2735 if (skb->protocol == htons(ETH_P_IP)) {
2736 struct iphdr *iph = ip_hdr(skb);
2737 iph->tot_len = 0;
2738 iph->check = 0;
2739 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2740 iph->daddr, 0,
2741 IPPROTO_TCP,
2742 0);
2743 adapter->hw_tso_ctxt++;
9010bc33 2744 } else if (skb_is_gso_v6(skb)) {
92915f71
GR		 2745 ipv6_hdr(skb)->payload_len = 0;
2746 tcp_hdr(skb)->check =
2747 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2748 &ipv6_hdr(skb)->daddr,
2749 0, IPPROTO_TCP, 0);
2750 adapter->hw_tso6_ctxt++;
2751 }
2752
2753 i = tx_ring->next_to_use;
2754
2755 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2756 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i);
2757
2758 /* VLAN MACLEN IPLEN */
2759 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2760 vlan_macip_lens |=
2761 (tx_flags & IXGBE_TX_FLAGS_VLAN_MASK);
2762 vlan_macip_lens |= ((skb_network_offset(skb)) <<
2763 IXGBE_ADVTXD_MACLEN_SHIFT);
2764 *hdr_len += skb_network_offset(skb);
2765 vlan_macip_lens |=
2766 (skb_transport_header(skb) - skb_network_header(skb));
2767 *hdr_len +=
2768 (skb_transport_header(skb) - skb_network_header(skb));
2769 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2770 context_desc->seqnum_seed = 0;
2771
2772 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2773 type_tucmd_mlhl = (IXGBE_TXD_CMD_DEXT |
2774 IXGBE_ADVTXD_DTYP_CTXT);
2775
2776 if (skb->protocol == htons(ETH_P_IP))
2777 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2778 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2779 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl);
2780
2781 /* MSS L4LEN IDX */
2782 mss_l4len_idx =
2783 (skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT);
2784 mss_l4len_idx |= (l4len << IXGBE_ADVTXD_L4LEN_SHIFT);
2785 /* use index 1 for TSO */
2786 mss_l4len_idx |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2787 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2788
2789 tx_buffer_info->time_stamp = jiffies;
2790 tx_buffer_info->next_to_watch = i;
2791
2792 i++;
2793 if (i == tx_ring->count)
2794 i = 0;
2795 tx_ring->next_to_use = i;
2796
2797 return true;
2798 }
2799
2800 return false;
2801}
2802
2803static bool ixgbevf_tx_csum(struct ixgbevf_adapter *adapter,
2804 struct ixgbevf_ring *tx_ring,
2805 struct sk_buff *skb, u32 tx_flags)
2806{
2807 struct ixgbe_adv_tx_context_desc *context_desc;
2808 unsigned int i;
2809 struct ixgbevf_tx_buffer *tx_buffer_info;
2810 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2811
2812 if (skb->ip_summed == CHECKSUM_PARTIAL ||
2813 (tx_flags & IXGBE_TX_FLAGS_VLAN)) {
2814 i = tx_ring->next_to_use;
2815 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2816 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i);
2817
2818 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2819 vlan_macip_lens |= (tx_flags &
2820 IXGBE_TX_FLAGS_VLAN_MASK);
2821 vlan_macip_lens |= (skb_network_offset(skb) <<
2822 IXGBE_ADVTXD_MACLEN_SHIFT);
2823 if (skb->ip_summed == CHECKSUM_PARTIAL)
2824 vlan_macip_lens |= (skb_transport_header(skb) -
2825 skb_network_header(skb));
2826
2827 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2828 context_desc->seqnum_seed = 0;
2829
2830 type_tucmd_mlhl |= (IXGBE_TXD_CMD_DEXT |
2831 IXGBE_ADVTXD_DTYP_CTXT);
2832
2833 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2834 switch (skb->protocol) {
2835 case __constant_htons(ETH_P_IP):
2836 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2837 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2838 type_tucmd_mlhl |=
2839 IXGBE_ADVTXD_TUCMD_L4T_TCP;
2840 break;
2841 case __constant_htons(ETH_P_IPV6):
2842 /* XXX what about other V6 headers?? */
2843 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2844 type_tucmd_mlhl |=
2845 IXGBE_ADVTXD_TUCMD_L4T_TCP;
2846 break;
2847 default:
2848 if (unlikely(net_ratelimit())) {
dbd9636e
JK		 2849 pr_warn("partial checksum but "
2850 "proto=%x!\n", skb->protocol);
92915f71
GR		 2851 }
2852 break;
2853 }
2854 }
2855
2856 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl);
2857 /* use index zero for tx checksum offload */
2858 context_desc->mss_l4len_idx = 0;
2859
2860 tx_buffer_info->time_stamp = jiffies;
2861 tx_buffer_info->next_to_watch = i;
2862
2863 adapter->hw_csum_tx_good++;
2864 i++;
2865 if (i == tx_ring->count)
2866 i = 0;
2867 tx_ring->next_to_use = i;
2868
2869 return true;
2870 }
2871
2872 return false;
2873}
2874
2875static int ixgbevf_tx_map(struct ixgbevf_adapter *adapter,
2876 struct ixgbevf_ring *tx_ring,
2877 struct sk_buff *skb, u32 tx_flags,
2878 unsigned int first)
2879{
2880 struct pci_dev *pdev = adapter->pdev;
2881 struct ixgbevf_tx_buffer *tx_buffer_info;
2882 unsigned int len;
2883 unsigned int total = skb->len;
2540ddb5
KV		 2884 unsigned int offset = 0, size;
2885 int count = 0;
92915f71
GR		 2886 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2887 unsigned int f;
65deeed7 2888 int i;
92915f71
GR		 2889
2890 i = tx_ring->next_to_use;
2891
2892 len = min(skb_headlen(skb), total);
2893 while (len) {
2894 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2895 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2896
2897 tx_buffer_info->length = size;
2898 tx_buffer_info->mapped_as_page = false;
2a1f8794 2899 tx_buffer_info->dma = dma_map_single(&adapter->pdev->dev,
92915f71 2900 skb->data + offset,
2a1f8794
NN		 2901 size, DMA_TO_DEVICE);
2902 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma))
92915f71
GR		 2903 goto dma_error;
2904 tx_buffer_info->time_stamp = jiffies;
2905 tx_buffer_info->next_to_watch = i;
2906
2907 len -= size;
2908 total -= size;
2909 offset += size;
2910 count++;
2911 i++;
2912 if (i == tx_ring->count)
2913 i = 0;
2914 }
2915
2916 for (f = 0; f < nr_frags; f++) {
9e903e08 2917 const struct skb_frag_struct *frag;
92915f71
GR		 2918
2919 frag = &skb_shinfo(skb)->frags[f];
9e903e08 2920 len = min((unsigned int)skb_frag_size(frag), total);
877749bf 2921 offset = 0;
92915f71
GR		 2922
2923 while (len) {
2924 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2925 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2926
2927 tx_buffer_info->length = size;
877749bf
IC		 2928 tx_buffer_info->dma =
2929 skb_frag_dma_map(&adapter->pdev->dev, frag,
2930 offset, size, DMA_TO_DEVICE);
92915f71 2931 tx_buffer_info->mapped_as_page = true;
2a1f8794 2932 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma))
92915f71
GR		 2933 goto dma_error;
2934 tx_buffer_info->time_stamp = jiffies;
2935 tx_buffer_info->next_to_watch = i;
2936
2937 len -= size;
2938 total -= size;
2939 offset += size;
2940 count++;
2941 i++;
2942 if (i == tx_ring->count)
2943 i = 0;
2944 }
2945 if (total == 0)
2946 break;
2947 }
2948
2949 if (i == 0)
2950 i = tx_ring->count - 1;
2951 else
2952 i = i - 1;
2953 tx_ring->tx_buffer_info[i].skb = skb;
2954 tx_ring->tx_buffer_info[first].next_to_watch = i;
2955
2956 return count;
2957
2958dma_error:
2959 dev_err(&pdev->dev, "TX DMA map failed\n");
2960
2961 /* clear timestamp and dma mappings for failed tx_buffer_info map */
2962 tx_buffer_info->dma = 0;
2963 tx_buffer_info->time_stamp = 0;
2964 tx_buffer_info->next_to_watch = 0;
2965 count--;
2966
2967 /* clear timestamp and dma mappings for remaining portion of packet */
2968 while (count >= 0) {
2969 count--;
2970 i--;
2971 if (i < 0)
2972 i += tx_ring->count;
2973 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2974 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info);
2975 }
2976
2977 return count;
2978}
2979
2980static void ixgbevf_tx_queue(struct ixgbevf_adapter *adapter,
2981 struct ixgbevf_ring *tx_ring, int tx_flags,
2982 int count, u32 paylen, u8 hdr_len)
2983{
2984 union ixgbe_adv_tx_desc *tx_desc = NULL;
2985 struct ixgbevf_tx_buffer *tx_buffer_info;
2986 u32 olinfo_status = 0, cmd_type_len = 0;
2987 unsigned int i;
2988
2989 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2990
2991 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2992
2993 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2994
2995 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2996 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2997
2998 if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2999 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3000
3001 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
3002 IXGBE_ADVTXD_POPTS_SHIFT;
3003
3004 /* use index 1 context for tso */
3005 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
3006 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3007 olinfo_status |= IXGBE_TXD_POPTS_IXSM <<
3008 IXGBE_ADVTXD_POPTS_SHIFT;
3009
3010 } else if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3011 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
3012 IXGBE_ADVTXD_POPTS_SHIFT;
3013
3014 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
3015
3016 i = tx_ring->next_to_use;
3017 while (count--) {
3018 tx_buffer_info = &tx_ring->tx_buffer_info[i];
3019 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i);
3020 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
3021 tx_desc->read.cmd_type_len =
3022 cpu_to_le32(cmd_type_len | tx_buffer_info->length);
3023 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3024 i++;
3025 if (i == tx_ring->count)
3026 i = 0;
3027 }
3028
3029 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
3030
3031 /*
3032 * Force memory writes to complete before letting h/w
3033 * know there are new descriptors to fetch. (Only
3034 * applicable for weak-ordered memory model archs,
3035 * such as IA-64).
3036 */
3037 wmb();
3038
3039 tx_ring->next_to_use = i;
3040 writel(i, adapter->hw.hw_addr + tx_ring->tail);
3041}
3042
3043static int __ixgbevf_maybe_stop_tx(struct net_device *netdev,
3044 struct ixgbevf_ring *tx_ring, int size)
3045{
3046 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3047
3048 netif_stop_subqueue(netdev, tx_ring->queue_index);
3049 /* Herbert's original patch had:
3050 * smp_mb__after_netif_stop_queue();
3051 * but since that doesn't exist yet, just open code it. */
3052 smp_mb();
3053
3054 /* We need to check again in a case another CPU has just
3055 * made room available. */
3056 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
3057 return -EBUSY;
3058
3059 /* A reprieve! - use start_queue because it doesn't call schedule */
3060 netif_start_subqueue(netdev, tx_ring->queue_index);
3061 ++adapter->restart_queue;
3062 return 0;
3063}
3064
3065static int ixgbevf_maybe_stop_tx(struct net_device *netdev,
3066 struct ixgbevf_ring *tx_ring, int size)
3067{
3068 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
3069 return 0;
3070 return __ixgbevf_maybe_stop_tx(netdev, tx_ring, size);
3071}
3072
3073static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3074{
3075 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3076 struct ixgbevf_ring *tx_ring;
3077 unsigned int first;
3078 unsigned int tx_flags = 0;
3079 u8 hdr_len = 0;
3080 int r_idx = 0, tso;
3081 int count = 0;
3082
3083 unsigned int f;
3084
3085 tx_ring = &adapter->tx_ring[r_idx];
3086
eab6d18d 3087 if (vlan_tx_tag_present(skb)) {
92915f71
GR		 3088 tx_flags |= vlan_tx_tag_get(skb);
3089 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3090 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3091 }
3092
3093 /* four things can cause us to need a context descriptor */
3094 if (skb_is_gso(skb) ||
3095 (skb->ip_summed == CHECKSUM_PARTIAL) ||
3096 (tx_flags & IXGBE_TX_FLAGS_VLAN))
3097 count++;
3098
3099 count += TXD_USE_COUNT(skb_headlen(skb));
3100 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
9e903e08 3101 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]));
92915f71
GR		 3102
3103 if (ixgbevf_maybe_stop_tx(netdev, tx_ring, count)) {
3104 adapter->tx_busy++;
3105 return NETDEV_TX_BUSY;
3106 }
3107
3108 first = tx_ring->next_to_use;
3109
3110 if (skb->protocol == htons(ETH_P_IP))
3111 tx_flags |= IXGBE_TX_FLAGS_IPV4;
3112 tso = ixgbevf_tso(adapter, tx_ring, skb, tx_flags, &hdr_len);
3113 if (tso < 0) {
3114 dev_kfree_skb_any(skb);
3115 return NETDEV_TX_OK;
3116 }
3117
3118 if (tso)
3119 tx_flags |= IXGBE_TX_FLAGS_TSO;
3120 else if (ixgbevf_tx_csum(adapter, tx_ring, skb, tx_flags) &&
3121 (skb->ip_summed == CHECKSUM_PARTIAL))
3122 tx_flags |= IXGBE_TX_FLAGS_CSUM;
3123
3124 ixgbevf_tx_queue(adapter, tx_ring, tx_flags,
3125 ixgbevf_tx_map(adapter, tx_ring, skb, tx_flags, first),
3126 skb->len, hdr_len);
3127
92915f71
GR		 3128 ixgbevf_maybe_stop_tx(netdev, tx_ring, DESC_NEEDED);
3129
3130 return NETDEV_TX_OK;
3131}
3132
92915f71
GR		 3133/**
3134 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3135 * @netdev: network interface device structure
3136 * @p: pointer to an address structure
3137 *
3138 * Returns 0 on success, negative on failure
3139 **/
3140static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3141{
3142 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3143 struct ixgbe_hw *hw = &adapter->hw;
3144 struct sockaddr *addr = p;
3145
3146 if (!is_valid_ether_addr(addr->sa_data))
3147 return -EADDRNOTAVAIL;
3148
3149 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3150 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3151
3152 if (hw->mac.ops.set_rar)
3153 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
3154
3155 return 0;
3156}
3157
3158/**
3159 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3160 * @netdev: network interface device structure
3161 * @new_mtu: new value for maximum frame size
3162 *
3163 * Returns 0 on success, negative on failure
3164 **/
3165static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3166{
3167 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
69bfbec4 3168 struct ixgbe_hw *hw = &adapter->hw;
92915f71 3169 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
69bfbec4
GR		 3170 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
3171 u32 msg[2];
3172
3173 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3174 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
92915f71
GR		 3175
3176 /* MTU < 68 is an error and causes problems on some kernels */
69bfbec4 3177 if ((new_mtu < 68) || (max_frame > max_possible_frame))
92915f71
GR		 3178 return -EINVAL;
3179
3180 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3181 netdev->mtu, new_mtu);
3182 /* must set new MTU before calling down or up */
3183 netdev->mtu = new_mtu;
3184
795180d8
GR		 3185 if (!netif_running(netdev)) {
3186 msg[0] = IXGBE_VF_SET_LPE;
3187 msg[1] = max_frame;
3188 hw->mbx.ops.write_posted(hw, msg, 2);
3189 }
69bfbec4 3190
92915f71
GR		 3191 if (netif_running(netdev))
3192 ixgbevf_reinit_locked(adapter);
3193
3194 return 0;
3195}
3196
3197static void ixgbevf_shutdown(struct pci_dev *pdev)
3198{
3199 struct net_device *netdev = pci_get_drvdata(pdev);
3200 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3201
3202 netif_device_detach(netdev);
3203
3204 if (netif_running(netdev)) {
3205 ixgbevf_down(adapter);
3206 ixgbevf_free_irq(adapter);
3207 ixgbevf_free_all_tx_resources(adapter);
3208 ixgbevf_free_all_rx_resources(adapter);
3209 }
3210
3211#ifdef CONFIG_PM
3212 pci_save_state(pdev);
3213#endif
3214
3215 pci_disable_device(pdev);
3216}
3217
4197aa7b
ED		 3218static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3219 struct rtnl_link_stats64 *stats)
3220{
3221 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3222 unsigned int start;
3223 u64 bytes, packets;
3224 const struct ixgbevf_ring *ring;
3225 int i;
3226
3227 ixgbevf_update_stats(adapter);
3228
3229 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3230
3231 for (i = 0; i < adapter->num_rx_queues; i++) {
3232 ring = &adapter->rx_ring[i];
3233 do {
3234 start = u64_stats_fetch_begin_bh(&ring->syncp);
3235 bytes = ring->total_bytes;
3236 packets = ring->total_packets;
3237 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3238 stats->rx_bytes += bytes;
3239 stats->rx_packets += packets;
3240 }
3241
3242 for (i = 0; i < adapter->num_tx_queues; i++) {
3243 ring = &adapter->tx_ring[i];
3244 do {
3245 start = u64_stats_fetch_begin_bh(&ring->syncp);
3246 bytes = ring->total_bytes;
3247 packets = ring->total_packets;
3248 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3249 stats->tx_bytes += bytes;
3250 stats->tx_packets += packets;
3251 }
3252
3253 return stats;
3254}
3255
c8f44aff
MM		 3256static int ixgbevf_set_features(struct net_device *netdev,
3257 netdev_features_t features)
471a76de
MM		 3258{
3259 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3260
3261 if (features & NETIF_F_RXCSUM)
3262 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
3263 else
3264 adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED;
3265
3266 return 0;
3267}
3268
92915f71 3269static const struct net_device_ops ixgbe_netdev_ops = {
c12db769
SH		 3270 .ndo_open = ixgbevf_open,
3271 .ndo_stop = ixgbevf_close,
3272 .ndo_start_xmit = ixgbevf_xmit_frame,
3273 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
4197aa7b 3274 .ndo_get_stats64 = ixgbevf_get_stats,
92915f71 3275 .ndo_validate_addr = eth_validate_addr,
c12db769
SH		 3276 .ndo_set_mac_address = ixgbevf_set_mac,
3277 .ndo_change_mtu = ixgbevf_change_mtu,
3278 .ndo_tx_timeout = ixgbevf_tx_timeout,
c12db769
SH		 3279 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3280 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
471a76de 3281 .ndo_set_features = ixgbevf_set_features,
92915f71 3282};
92915f71
GR		 3283
3284static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3285{
92915f71 3286 dev->netdev_ops = &ixgbe_netdev_ops;
92915f71
GR		 3287 ixgbevf_set_ethtool_ops(dev);
3288 dev->watchdog_timeo = 5 * HZ;
3289}
3290
3291/**
3292 * ixgbevf_probe - Device Initialization Routine
3293 * @pdev: PCI device information struct
3294 * @ent: entry in ixgbevf_pci_tbl
3295 *
3296 * Returns 0 on success, negative on failure
3297 *
3298 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3299 * The OS initialization, configuring of the adapter private structure,
3300 * and a hardware reset occur.
3301 **/
3302static int __devinit ixgbevf_probe(struct pci_dev *pdev,
3303 const struct pci_device_id *ent)
3304{
3305 struct net_device *netdev;
3306 struct ixgbevf_adapter *adapter = NULL;
3307 struct ixgbe_hw *hw = NULL;
3308 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3309 static int cards_found;
3310 int err, pci_using_dac;
3311
3312 err = pci_enable_device(pdev);
3313 if (err)
3314 return err;
3315
2a1f8794
NN		 3316 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
3317 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
92915f71
GR		 3318 pci_using_dac = 1;
3319 } else {
2a1f8794 3320 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
92915f71 3321 if (err) {
2a1f8794
NN		 3322 err = dma_set_coherent_mask(&pdev->dev,
3323 DMA_BIT_MASK(32));
92915f71
GR		 3324 if (err) {
3325 dev_err(&pdev->dev, "No usable DMA "
3326 "configuration, aborting\n");
3327 goto err_dma;
3328 }
3329 }
3330 pci_using_dac = 0;
3331 }
3332
3333 err = pci_request_regions(pdev, ixgbevf_driver_name);
3334 if (err) {
3335 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3336 goto err_pci_reg;
3337 }
3338
3339 pci_set_master(pdev);
3340
3341#ifdef HAVE_TX_MQ
3342 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3343 MAX_TX_QUEUES);
3344#else
3345 netdev = alloc_etherdev(sizeof(struct ixgbevf_adapter));
3346#endif
3347 if (!netdev) {
3348 err = -ENOMEM;
3349 goto err_alloc_etherdev;
3350 }
3351
3352 SET_NETDEV_DEV(netdev, &pdev->dev);
3353
3354 pci_set_drvdata(pdev, netdev);
3355 adapter = netdev_priv(netdev);
3356
3357 adapter->netdev = netdev;
3358 adapter->pdev = pdev;
3359 hw = &adapter->hw;
3360 hw->back = adapter;
b3f4d599 3361 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
92915f71
GR		 3362
3363 /*
3364 * call save state here in standalone driver because it relies on
3365 * adapter struct to exist, and needs to call netdev_priv
3366 */
3367 pci_save_state(pdev);
3368
3369 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3370 pci_resource_len(pdev, 0));
3371 if (!hw->hw_addr) {
3372 err = -EIO;
3373 goto err_ioremap;
3374 }
3375
3376 ixgbevf_assign_netdev_ops(netdev);
3377
3378 adapter->bd_number = cards_found;
3379
3380 /* Setup hw api */
3381 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3382 hw->mac.type = ii->mac;
3383
3384 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
f416dfc0 3385 sizeof(struct ixgbe_mbx_operations));
92915f71
GR		 3386
3387 adapter->flags &= ~IXGBE_FLAG_RX_PS_CAPABLE;
3388 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3389 adapter->flags |= IXGBE_FLAG_RX_1BUF_CAPABLE;
3390
3391 /* setup the private structure */
3392 err = ixgbevf_sw_init(adapter);
1a0d6ae5
DK		 3393 if (err)
3394 goto err_sw_init;
3395
3396 /* The HW MAC address was set and/or determined in sw_init */
3397 memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
3398
3399 if (!is_valid_ether_addr(netdev->dev_addr)) {
3400 pr_err("invalid MAC address\n");
3401 err = -EIO;
3402 goto err_sw_init;
3403 }
92915f71 3404
471a76de 3405 netdev->hw_features = NETIF_F_SG |
92915f71 3406 NETIF_F_IP_CSUM |
471a76de
MM		 3407 NETIF_F_IPV6_CSUM |
3408 NETIF_F_TSO |
3409 NETIF_F_TSO6 |
3410 NETIF_F_RXCSUM;
3411
3412 netdev->features = netdev->hw_features |
92915f71
GR		 3413 NETIF_F_HW_VLAN_TX |
3414 NETIF_F_HW_VLAN_RX |
3415 NETIF_F_HW_VLAN_FILTER;
3416
92915f71
GR		 3417 netdev->vlan_features |= NETIF_F_TSO;
3418 netdev->vlan_features |= NETIF_F_TSO6;
3419 netdev->vlan_features |= NETIF_F_IP_CSUM;
3bfacf96 3420 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
92915f71
GR		 3421 netdev->vlan_features |= NETIF_F_SG;
3422
3423 if (pci_using_dac)
3424 netdev->features |= NETIF_F_HIGHDMA;
3425
01789349
JP		 3426 netdev->priv_flags |= IFF_UNICAST_FLT;
3427
92915f71 3428 init_timer(&adapter->watchdog_timer);
c061b18d 3429 adapter->watchdog_timer.function = ixgbevf_watchdog;
92915f71
GR		 3430 adapter->watchdog_timer.data = (unsigned long)adapter;
3431
3432 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3433 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3434
3435 err = ixgbevf_init_interrupt_scheme(adapter);
3436 if (err)
3437 goto err_sw_init;
3438
3439 /* pick up the PCI bus settings for reporting later */
3440 if (hw->mac.ops.get_bus_info)
3441 hw->mac.ops.get_bus_info(hw);
3442
92915f71
GR		 3443 strcpy(netdev->name, "eth%d");
3444
3445 err = register_netdev(netdev);
3446 if (err)
3447 goto err_register;
3448
3449 adapter->netdev_registered = true;
3450
5d426ad1
GR		 3451 netif_carrier_off(netdev);
3452
33bd9f60
GR		 3453 ixgbevf_init_last_counter_stats(adapter);
3454
92915f71 3455 /* print the MAC address */
f794e7ef 3456 hw_dbg(hw, "%pM\n", netdev->dev_addr);
92915f71
GR		 3457
3458 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3459
d6dbee86 3460 hw_dbg(hw, "LRO is disabled\n");
92915f71
GR		 3461
3462 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3463 cards_found++;
3464 return 0;
3465
3466err_register:
3467err_sw_init:
3468 ixgbevf_reset_interrupt_capability(adapter);
3469 iounmap(hw->hw_addr);
3470err_ioremap:
3471 free_netdev(netdev);
3472err_alloc_etherdev:
3473 pci_release_regions(pdev);
3474err_pci_reg:
3475err_dma:
3476 pci_disable_device(pdev);
3477 return err;
3478}
3479
3480/**
3481 * ixgbevf_remove - Device Removal Routine
3482 * @pdev: PCI device information struct
3483 *
3484 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3485 * that it should release a PCI device. The could be caused by a
3486 * Hot-Plug event, or because the driver is going to be removed from
3487 * memory.
3488 **/
3489static void __devexit ixgbevf_remove(struct pci_dev *pdev)
3490{
3491 struct net_device *netdev = pci_get_drvdata(pdev);
3492 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3493
3494 set_bit(__IXGBEVF_DOWN, &adapter->state);
3495
3496 del_timer_sync(&adapter->watchdog_timer);
3497
23f333a2 3498 cancel_work_sync(&adapter->reset_task);
92915f71
GR		 3499 cancel_work_sync(&adapter->watchdog_task);
3500
92915f71
GR		 3501 if (adapter->netdev_registered) {
3502 unregister_netdev(netdev);
3503 adapter->netdev_registered = false;
3504 }
3505
3506 ixgbevf_reset_interrupt_capability(adapter);
3507
3508 iounmap(adapter->hw.hw_addr);
3509 pci_release_regions(pdev);
3510
3511 hw_dbg(&adapter->hw, "Remove complete\n");
3512
3513 kfree(adapter->tx_ring);
3514 kfree(adapter->rx_ring);
3515
3516 free_netdev(netdev);
3517
3518 pci_disable_device(pdev);
3519}
3520
3521static struct pci_driver ixgbevf_driver = {
3522 .name = ixgbevf_driver_name,
3523 .id_table = ixgbevf_pci_tbl,
3524 .probe = ixgbevf_probe,
3525 .remove = __devexit_p(ixgbevf_remove),
3526 .shutdown = ixgbevf_shutdown,
3527};
3528
3529/**
65d676c8 3530 * ixgbevf_init_module - Driver Registration Routine
92915f71 3531 *
65d676c8 3532 * ixgbevf_init_module is the first routine called when the driver is
92915f71
GR		 3533 * loaded. All it does is register with the PCI subsystem.
3534 **/
3535static int __init ixgbevf_init_module(void)
3536{
3537 int ret;
dbd9636e
JK		 3538 pr_info("%s - version %s\n", ixgbevf_driver_string,
3539 ixgbevf_driver_version);
92915f71 3540
dbd9636e 3541 pr_info("%s\n", ixgbevf_copyright);
92915f71
GR		 3542
3543 ret = pci_register_driver(&ixgbevf_driver);
3544 return ret;
3545}
3546
3547module_init(ixgbevf_init_module);
3548
3549/**
65d676c8 3550 * ixgbevf_exit_module - Driver Exit Cleanup Routine
92915f71 3551 *
65d676c8 3552 * ixgbevf_exit_module is called just before the driver is removed
92915f71
GR		 3553 * from memory.
3554 **/
3555static void __exit ixgbevf_exit_module(void)
3556{
3557 pci_unregister_driver(&ixgbevf_driver);
3558}
3559
3560#ifdef DEBUG
3561/**
65d676c8 3562 * ixgbevf_get_hw_dev_name - return device name string
92915f71
GR		 3563 * used by hardware layer to print debugging information
3564 **/
3565char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3566{
3567 struct ixgbevf_adapter *adapter = hw->back;
3568 return adapter->netdev->name;
3569}
3570
3571#endif
3572module_exit(ixgbevf_exit_module);
3573
3574/* ixgbevf_main.c */
kernel source for telekom puls (alcatel/mediatek)