projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: Update max_frame_size to account for an optional VLAN tag if present
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4297f99b 4 Copyright(c) 2007-2011 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
b2cb09b1 31#include <linux/bitops.h>
9d5c8243
AK		 32#include <linux/vmalloc.h>
33#include <linux/pagemap.h>
34#include <linux/netdevice.h>
9d5c8243 35#include <linux/ipv6.h>
5a0e3ad6 36#include <linux/slab.h>
9d5c8243
AK		 37#include <net/checksum.h>
38#include <net/ip6_checksum.h>
c6cb090b 39#include <linux/net_tstamp.h>
9d5c8243
AK		 40#include <linux/mii.h>
41#include <linux/ethtool.h>
01789349 42#include <linux/if.h>
9d5c8243
AK		 43#include <linux/if_vlan.h>
44#include <linux/pci.h>
c54106bb 45#include <linux/pci-aspm.h>
9d5c8243
AK		 46#include <linux/delay.h>
47#include <linux/interrupt.h>
48#include <linux/if_ether.h>
40a914fa 49#include <linux/aer.h>
70c71606 50#include <linux/prefetch.h>
421e02f0 51#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 52#include <linux/dca.h>
53#endif
9d5c8243
AK		 54#include "igb.h"
55
0d1fe82d
CW		 56#define MAJ 3
57#define MIN 0
58#define BUILD 6
0d1fe82d 59#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 60__stringify(BUILD) "-k"
9d5c8243
AK		 61char igb_driver_name[] = "igb";
62char igb_driver_version[] = DRV_VERSION;
63static const char igb_driver_string[] =
64 "Intel(R) Gigabit Ethernet Network Driver";
4c4b42cb 65static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
9d5c8243 66
9d5c8243
AK		 67static const struct e1000_info *igb_info_tbl[] = {
68 [board_82575] = &e1000_82575_info,
69};
70
a3aa1884 71static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
97 /* required last entry */
98 {0, }
99};
100
101MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
102
103void igb_reset(struct igb_adapter *);
104static int igb_setup_all_tx_resources(struct igb_adapter *);
105static int igb_setup_all_rx_resources(struct igb_adapter *);
106static void igb_free_all_tx_resources(struct igb_adapter *);
107static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 108static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 109static int igb_probe(struct pci_dev *, const struct pci_device_id *);
110static void __devexit igb_remove(struct pci_dev *pdev);
673b8b70 111static void igb_init_hw_timer(struct igb_adapter *adapter);
9d5c8243
AK		 112static int igb_sw_init(struct igb_adapter *);
113static int igb_open(struct net_device *);
114static int igb_close(struct net_device *);
115static void igb_configure_tx(struct igb_adapter *);
116static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 117static void igb_clean_all_tx_rings(struct igb_adapter *);
118static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 119static void igb_clean_tx_ring(struct igb_ring *);
120static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 121static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 122static void igb_update_phy_info(unsigned long);
123static void igb_watchdog(unsigned long);
124static void igb_watchdog_task(struct work_struct *);
b1a436c3 125static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 126static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
127 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 128static int igb_change_mtu(struct net_device *, int);
129static int igb_set_mac(struct net_device *, void *);
68d480c4 130static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 131static irqreturn_t igb_intr(int irq, void *);
132static irqreturn_t igb_intr_msi(int irq, void *);
133static irqreturn_t igb_msix_other(int irq, void *);
047e0030 134static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 135#ifdef CONFIG_IGB_DCA
047e0030 136static void igb_update_dca(struct igb_q_vector *);
fe4506b6 137static void igb_setup_dca(struct igb_adapter *);
421e02f0 138#endif /* CONFIG_IGB_DCA */
047e0030 139static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 140static int igb_poll(struct napi_struct *, int);
047e0030 141static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 142static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
143static void igb_tx_timeout(struct net_device *);
144static void igb_reset_task(struct work_struct *);
b2cb09b1 145static void igb_vlan_mode(struct net_device *netdev, u32 features);
9d5c8243
AK		 146static void igb_vlan_rx_add_vid(struct net_device *, u16);
147static void igb_vlan_rx_kill_vid(struct net_device *, u16);
148static void igb_restore_vlan(struct igb_adapter *);
26ad9178 149static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 150static void igb_ping_all_vfs(struct igb_adapter *);
151static void igb_msg_task(struct igb_adapter *);
4ae196df 152static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 153static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 154static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 155static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
156static int igb_ndo_set_vf_vlan(struct net_device *netdev,
157 int vf, u16 vlan, u8 qos);
158static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
159static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
160 struct ifla_vf_info *ivi);
17dc566c 161static void igb_check_vf_rate_limit(struct igb_adapter *);
9d5c8243 162
9d5c8243 163#ifdef CONFIG_PM
3fe7c4c9 164static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 165static int igb_resume(struct pci_dev *);
166#endif
167static void igb_shutdown(struct pci_dev *);
421e02f0 168#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 169static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
170static struct notifier_block dca_notifier = {
171 .notifier_call = igb_notify_dca,
172 .next = NULL,
173 .priority = 0
174};
175#endif
9d5c8243
AK		 176#ifdef CONFIG_NET_POLL_CONTROLLER
177/* for netdump / net console */
178static void igb_netpoll(struct net_device *);
179#endif
37680117 180#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 181static unsigned int max_vfs = 0;
182module_param(max_vfs, uint, 0);
183MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
184 "per physical function");
185#endif /* CONFIG_PCI_IOV */
186
9d5c8243
AK		 187static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
188 pci_channel_state_t);
189static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
190static void igb_io_resume(struct pci_dev *);
191
192static struct pci_error_handlers igb_err_handler = {
193 .error_detected = igb_io_error_detected,
194 .slot_reset = igb_io_slot_reset,
195 .resume = igb_io_resume,
196};
197
198
199static struct pci_driver igb_driver = {
200 .name = igb_driver_name,
201 .id_table = igb_pci_tbl,
202 .probe = igb_probe,
203 .remove = __devexit_p(igb_remove),
204#ifdef CONFIG_PM
25985edc 205 /* Power Management Hooks */
9d5c8243
AK		 206 .suspend = igb_suspend,
207 .resume = igb_resume,
208#endif
209 .shutdown = igb_shutdown,
210 .err_handler = &igb_err_handler
211};
212
213MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
214MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
215MODULE_LICENSE("GPL");
216MODULE_VERSION(DRV_VERSION);
217
c97ec42a
TI		 218struct igb_reg_info {
219 u32 ofs;
220 char *name;
221};
222
223static const struct igb_reg_info igb_reg_info_tbl[] = {
224
225 /* General Registers */
226 {E1000_CTRL, "CTRL"},
227 {E1000_STATUS, "STATUS"},
228 {E1000_CTRL_EXT, "CTRL_EXT"},
229
230 /* Interrupt Registers */
231 {E1000_ICR, "ICR"},
232
233 /* RX Registers */
234 {E1000_RCTL, "RCTL"},
235 {E1000_RDLEN(0), "RDLEN"},
236 {E1000_RDH(0), "RDH"},
237 {E1000_RDT(0), "RDT"},
238 {E1000_RXDCTL(0), "RXDCTL"},
239 {E1000_RDBAL(0), "RDBAL"},
240 {E1000_RDBAH(0), "RDBAH"},
241
242 /* TX Registers */
243 {E1000_TCTL, "TCTL"},
244 {E1000_TDBAL(0), "TDBAL"},
245 {E1000_TDBAH(0), "TDBAH"},
246 {E1000_TDLEN(0), "TDLEN"},
247 {E1000_TDH(0), "TDH"},
248 {E1000_TDT(0), "TDT"},
249 {E1000_TXDCTL(0), "TXDCTL"},
250 {E1000_TDFH, "TDFH"},
251 {E1000_TDFT, "TDFT"},
252 {E1000_TDFHS, "TDFHS"},
253 {E1000_TDFPC, "TDFPC"},
254
255 /* List Terminator */
256 {}
257};
258
259/*
260 * igb_regdump - register printout routine
261 */
262static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
263{
264 int n = 0;
265 char rname[16];
266 u32 regs[8];
267
268 switch (reginfo->ofs) {
269 case E1000_RDLEN(0):
270 for (n = 0; n < 4; n++)
271 regs[n] = rd32(E1000_RDLEN(n));
272 break;
273 case E1000_RDH(0):
274 for (n = 0; n < 4; n++)
275 regs[n] = rd32(E1000_RDH(n));
276 break;
277 case E1000_RDT(0):
278 for (n = 0; n < 4; n++)
279 regs[n] = rd32(E1000_RDT(n));
280 break;
281 case E1000_RXDCTL(0):
282 for (n = 0; n < 4; n++)
283 regs[n] = rd32(E1000_RXDCTL(n));
284 break;
285 case E1000_RDBAL(0):
286 for (n = 0; n < 4; n++)
287 regs[n] = rd32(E1000_RDBAL(n));
288 break;
289 case E1000_RDBAH(0):
290 for (n = 0; n < 4; n++)
291 regs[n] = rd32(E1000_RDBAH(n));
292 break;
293 case E1000_TDBAL(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_RDBAL(n));
296 break;
297 case E1000_TDBAH(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_TDBAH(n));
300 break;
301 case E1000_TDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_TDLEN(n));
304 break;
305 case E1000_TDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_TDH(n));
308 break;
309 case E1000_TDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_TDT(n));
312 break;
313 case E1000_TXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_TXDCTL(n));
316 break;
317 default:
318 printk(KERN_INFO "%-15s %08x\n",
319 reginfo->name, rd32(reginfo->ofs));
320 return;
321 }
322
323 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
324 printk(KERN_INFO "%-15s ", rname);
325 for (n = 0; n < 4; n++)
326 printk(KERN_CONT "%08x ", regs[n]);
327 printk(KERN_CONT "\n");
328}
329
330/*
331 * igb_dump - Print registers, tx-rings and rx-rings
332 */
333static void igb_dump(struct igb_adapter *adapter)
334{
335 struct net_device *netdev = adapter->netdev;
336 struct e1000_hw *hw = &adapter->hw;
337 struct igb_reg_info *reginfo;
338 int n = 0;
339 struct igb_ring *tx_ring;
340 union e1000_adv_tx_desc *tx_desc;
341 struct my_u0 { u64 a; u64 b; } *u0;
342 struct igb_buffer *buffer_info;
343 struct igb_ring *rx_ring;
344 union e1000_adv_rx_desc *rx_desc;
345 u32 staterr;
346 int i = 0;
347
348 if (!netif_msg_hw(adapter))
349 return;
350
351 /* Print netdevice Info */
352 if (netdev) {
353 dev_info(&adapter->pdev->dev, "Net device Info\n");
354 printk(KERN_INFO "Device Name state "
355 "trans_start last_rx\n");
356 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
357 netdev->name,
358 netdev->state,
359 netdev->trans_start,
360 netdev->last_rx);
361 }
362
363 /* Print Registers */
364 dev_info(&adapter->pdev->dev, "Register Dump\n");
365 printk(KERN_INFO " Register Name Value\n");
366 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
367 reginfo->name; reginfo++) {
368 igb_regdump(hw, reginfo);
369 }
370
371 /* Print TX Ring Summary */
372 if (!netdev || !netif_running(netdev))
373 goto exit;
374
375 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
376 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
377 " leng ntw timestamp\n");
378 for (n = 0; n < adapter->num_tx_queues; n++) {
379 tx_ring = adapter->tx_ring[n];
380 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
381 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
382 n, tx_ring->next_to_use, tx_ring->next_to_clean,
383 (u64)buffer_info->dma,
384 buffer_info->length,
385 buffer_info->next_to_watch,
386 (u64)buffer_info->time_stamp);
387 }
388
389 /* Print TX Rings */
390 if (!netif_msg_tx_done(adapter))
391 goto rx_ring_summary;
392
393 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
394
395 /* Transmit Descriptor Formats
396 *
397 * Advanced Transmit Descriptor
398 * +--------------------------------------------------------------+
399 * 0 | Buffer Address [63:0] |
400 * +--------------------------------------------------------------+
401 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
402 * +--------------------------------------------------------------+
403 * 63 46 45 40 39 38 36 35 32 31 24 15 0
404 */
405
406 for (n = 0; n < adapter->num_tx_queues; n++) {
407 tx_ring = adapter->tx_ring[n];
408 printk(KERN_INFO "------------------------------------\n");
409 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
410 printk(KERN_INFO "------------------------------------\n");
411 printk(KERN_INFO "T [desc] [address 63:0 ] "
412 "[PlPOCIStDDM Ln] [bi->dma ] "
413 "leng ntw timestamp bi->skb\n");
414
415 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
416 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
417 buffer_info = &tx_ring->buffer_info[i];
418 u0 = (struct my_u0 *)tx_desc;
419 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
420 " %04X %3X %016llX %p", i,
421 le64_to_cpu(u0->a),
422 le64_to_cpu(u0->b),
423 (u64)buffer_info->dma,
424 buffer_info->length,
425 buffer_info->next_to_watch,
426 (u64)buffer_info->time_stamp,
427 buffer_info->skb);
428 if (i == tx_ring->next_to_use &&
429 i == tx_ring->next_to_clean)
430 printk(KERN_CONT " NTC/U\n");
431 else if (i == tx_ring->next_to_use)
432 printk(KERN_CONT " NTU\n");
433 else if (i == tx_ring->next_to_clean)
434 printk(KERN_CONT " NTC\n");
435 else
436 printk(KERN_CONT "\n");
437
438 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
439 print_hex_dump(KERN_INFO, "",
440 DUMP_PREFIX_ADDRESS,
441 16, 1, phys_to_virt(buffer_info->dma),
442 buffer_info->length, true);
443 }
444 }
445
446 /* Print RX Rings Summary */
447rx_ring_summary:
448 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
449 printk(KERN_INFO "Queue [NTU] [NTC]\n");
450 for (n = 0; n < adapter->num_rx_queues; n++) {
451 rx_ring = adapter->rx_ring[n];
452 printk(KERN_INFO " %5d %5X %5X\n", n,
453 rx_ring->next_to_use, rx_ring->next_to_clean);
454 }
455
456 /* Print RX Rings */
457 if (!netif_msg_rx_status(adapter))
458 goto exit;
459
460 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
461
462 /* Advanced Receive Descriptor (Read) Format
463 * 63 1 0
464 * +-----------------------------------------------------+
465 * 0 | Packet Buffer Address [63:1] |A0/NSE|
466 * +----------------------------------------------+------+
467 * 8 | Header Buffer Address [63:1] | DD |
468 * +-----------------------------------------------------+
469 *
470 *
471 * Advanced Receive Descriptor (Write-Back) Format
472 *
473 * 63 48 47 32 31 30 21 20 17 16 4 3 0
474 * +------------------------------------------------------+
475 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
476 * | Checksum Ident | | | | Type | Type |
477 * +------------------------------------------------------+
478 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
479 * +------------------------------------------------------+
480 * 63 48 47 32 31 20 19 0
481 */
482
483 for (n = 0; n < adapter->num_rx_queues; n++) {
484 rx_ring = adapter->rx_ring[n];
485 printk(KERN_INFO "------------------------------------\n");
486 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
487 printk(KERN_INFO "------------------------------------\n");
488 printk(KERN_INFO "R [desc] [ PktBuf A0] "
489 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
490 "<-- Adv Rx Read format\n");
491 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
492 "[vl er S cks ln] ---------------- [bi->skb] "
493 "<-- Adv Rx Write-Back format\n");
494
495 for (i = 0; i < rx_ring->count; i++) {
496 buffer_info = &rx_ring->buffer_info[i];
497 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
498 u0 = (struct my_u0 *)rx_desc;
499 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
500 if (staterr & E1000_RXD_STAT_DD) {
501 /* Descriptor Done */
502 printk(KERN_INFO "RWB[0x%03X] %016llX "
503 "%016llX ---------------- %p", i,
504 le64_to_cpu(u0->a),
505 le64_to_cpu(u0->b),
506 buffer_info->skb);
507 } else {
508 printk(KERN_INFO "R [0x%03X] %016llX "
509 "%016llX %016llX %p", i,
510 le64_to_cpu(u0->a),
511 le64_to_cpu(u0->b),
512 (u64)buffer_info->dma,
513 buffer_info->skb);
514
515 if (netif_msg_pktdata(adapter)) {
516 print_hex_dump(KERN_INFO, "",
517 DUMP_PREFIX_ADDRESS,
518 16, 1,
519 phys_to_virt(buffer_info->dma),
520 rx_ring->rx_buffer_len, true);
521 if (rx_ring->rx_buffer_len
522 < IGB_RXBUFFER_1024)
523 print_hex_dump(KERN_INFO, "",
524 DUMP_PREFIX_ADDRESS,
525 16, 1,
526 phys_to_virt(
527 buffer_info->page_dma +
528 buffer_info->page_offset),
529 PAGE_SIZE/2, true);
530 }
531 }
532
533 if (i == rx_ring->next_to_use)
534 printk(KERN_CONT " NTU\n");
535 else if (i == rx_ring->next_to_clean)
536 printk(KERN_CONT " NTC\n");
537 else
538 printk(KERN_CONT "\n");
539
540 }
541 }
542
543exit:
544 return;
545}
546
547
38c845c7
PO		 548/**
549 * igb_read_clock - read raw cycle counter (to be used by time counter)
550 */
551static cycle_t igb_read_clock(const struct cyclecounter *tc)
552{
553 struct igb_adapter *adapter =
554 container_of(tc, struct igb_adapter, cycles);
555 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 556 u64 stamp = 0;
557 int shift = 0;
38c845c7 558
55cac248
AD		 559 /*
560 * The timestamp latches on lowest register read. For the 82580
561 * the lowest register is SYSTIMR instead of SYSTIML. However we never
562 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
563 */
564 if (hw->mac.type == e1000_82580) {
565 stamp = rd32(E1000_SYSTIMR) >> 8;
566 shift = IGB_82580_TSYNC_SHIFT;
567 }
568
c5b9bd5e
AD		 569 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
570 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 571 return stamp;
572}
573
9d5c8243 574/**
c041076a 575 * igb_get_hw_dev - return device
9d5c8243
AK		 576 * used by hardware layer to print debugging information
577 **/
c041076a 578struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 579{
580 struct igb_adapter *adapter = hw->back;
c041076a 581 return adapter->netdev;
9d5c8243 582}
38c845c7 583
9d5c8243
AK		 584/**
585 * igb_init_module - Driver Registration Routine
586 *
587 * igb_init_module is the first routine called when the driver is
588 * loaded. All it does is register with the PCI subsystem.
589 **/
590static int __init igb_init_module(void)
591{
592 int ret;
593 printk(KERN_INFO "%s - version %s\n",
594 igb_driver_string, igb_driver_version);
595
596 printk(KERN_INFO "%s\n", igb_copyright);
597
421e02f0 598#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 599 dca_register_notify(&dca_notifier);
600#endif
bbd98fe4 601 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 602 return ret;
603}
604
605module_init(igb_init_module);
606
607/**
608 * igb_exit_module - Driver Exit Cleanup Routine
609 *
610 * igb_exit_module is called just before the driver is removed
611 * from memory.
612 **/
613static void __exit igb_exit_module(void)
614{
421e02f0 615#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 616 dca_unregister_notify(&dca_notifier);
617#endif
9d5c8243
AK		 618 pci_unregister_driver(&igb_driver);
619}
620
621module_exit(igb_exit_module);
622
26bc19ec
AD		 623#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
624/**
625 * igb_cache_ring_register - Descriptor ring to register mapping
626 * @adapter: board private structure to initialize
627 *
628 * Once we know the feature-set enabled for the device, we'll cache
629 * the register offset the descriptor ring is assigned to.
630 **/
631static void igb_cache_ring_register(struct igb_adapter *adapter)
632{
ee1b9f06 633 int i = 0, j = 0;
047e0030 634 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 635
636 switch (adapter->hw.mac.type) {
637 case e1000_82576:
638 /* The queues are allocated for virtualization such that VF 0
639 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
640 * In order to avoid collision we start at the first free queue
641 * and continue consuming queues in the same sequence
642 */
ee1b9f06 643 if (adapter->vfs_allocated_count) {
a99955fc 644 for (; i < adapter->rss_queues; i++)
3025a446
AD		 645 adapter->rx_ring[i]->reg_idx = rbase_offset +
646 Q_IDX_82576(i);
ee1b9f06 647 }
26bc19ec 648 case e1000_82575:
55cac248 649 case e1000_82580:
d2ba2ed8 650 case e1000_i350:
26bc19ec 651 default:
ee1b9f06 652 for (; i < adapter->num_rx_queues; i++)
3025a446 653 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 654 for (; j < adapter->num_tx_queues; j++)
3025a446 655 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 656 break;
657 }
658}
659
047e0030
AD		 660static void igb_free_queues(struct igb_adapter *adapter)
661{
3025a446 662 int i;
047e0030 663
3025a446
AD		 664 for (i = 0; i < adapter->num_tx_queues; i++) {
665 kfree(adapter->tx_ring[i]);
666 adapter->tx_ring[i] = NULL;
667 }
668 for (i = 0; i < adapter->num_rx_queues; i++) {
669 kfree(adapter->rx_ring[i]);
670 adapter->rx_ring[i] = NULL;
671 }
047e0030
AD		 672 adapter->num_rx_queues = 0;
673 adapter->num_tx_queues = 0;
674}
675
9d5c8243
AK		 676/**
677 * igb_alloc_queues - Allocate memory for all rings
678 * @adapter: board private structure to initialize
679 *
680 * We allocate one ring per queue at run-time since we don't know the
681 * number of queues at compile-time.
682 **/
683static int igb_alloc_queues(struct igb_adapter *adapter)
684{
3025a446 685 struct igb_ring *ring;
9d5c8243
AK		 686 int i;
687
661086df 688 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 689 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
690 if (!ring)
691 goto err;
68fd9910 692 ring->count = adapter->tx_ring_count;
661086df 693 ring->queue_index = i;
59d71989 694 ring->dev = &adapter->pdev->dev;
e694e964 695 ring->netdev = adapter->netdev;
85ad76b2
AD		 696 /* For 82575, context index must be unique per ring. */
697 if (adapter->hw.mac.type == e1000_82575)
698 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 699 adapter->tx_ring[i] = ring;
661086df 700 }
85ad76b2 701
9d5c8243 702 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 703 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
704 if (!ring)
705 goto err;
68fd9910 706 ring->count = adapter->rx_ring_count;
844290e5 707 ring->queue_index = i;
59d71989 708 ring->dev = &adapter->pdev->dev;
e694e964 709 ring->netdev = adapter->netdev;
4c844851 710 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 711 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
712 /* set flag indicating ring supports SCTP checksum offload */
713 if (adapter->hw.mac.type >= e1000_82576)
714 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 715 adapter->rx_ring[i] = ring;
9d5c8243 716 }
26bc19ec
AD		 717
718 igb_cache_ring_register(adapter);
9d5c8243 719
047e0030 720 return 0;
a88f10ec 721
047e0030
AD		 722err:
723 igb_free_queues(adapter);
d1a8c9e1 724
047e0030 725 return -ENOMEM;
a88f10ec
AD		 726}
727
9d5c8243 728#define IGB_N0_QUEUE -1
047e0030 729static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 730{
731 u32 msixbm = 0;
047e0030 732 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 733 struct e1000_hw *hw = &adapter->hw;
2d064c06 734 u32 ivar, index;
047e0030
AD		 735 int rx_queue = IGB_N0_QUEUE;
736 int tx_queue = IGB_N0_QUEUE;
737
738 if (q_vector->rx_ring)
739 rx_queue = q_vector->rx_ring->reg_idx;
740 if (q_vector->tx_ring)
741 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 742
743 switch (hw->mac.type) {
744 case e1000_82575:
9d5c8243
AK		 745 /* The 82575 assigns vectors using a bitmask, which matches the
746 bitmask for the EICR/EIMS/EIMC registers. To assign one
747 or more queues to a vector, we write the appropriate bits
748 into the MSIXBM register for that vector. */
047e0030 749 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 750 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 751 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 752 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 753 if (!adapter->msix_entries && msix_vector == 0)
754 msixbm |= E1000_EIMS_OTHER;
9d5c8243 755 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 756 q_vector->eims_value = msixbm;
2d064c06
AD		 757 break;
758 case e1000_82576:
26bc19ec 759 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 760 Each queue has a single entry in the table to which we write
761 a vector number along with a "valid" bit. Sadly, the layout
762 of the table is somewhat counterintuitive. */
763 if (rx_queue > IGB_N0_QUEUE) {
047e0030 764 index = (rx_queue & 0x7);
2d064c06 765 ivar = array_rd32(E1000_IVAR0, index);
047e0030 766 if (rx_queue < 8) {
26bc19ec
AD		 767 /* vector goes into low byte of register */
768 ivar = ivar & 0xFFFFFF00;
769 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 770 } else {
771 /* vector goes into third byte of register */
772 ivar = ivar & 0xFF00FFFF;
773 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 774 }
2d064c06
AD		 775 array_wr32(E1000_IVAR0, index, ivar);
776 }
777 if (tx_queue > IGB_N0_QUEUE) {
047e0030 778 index = (tx_queue & 0x7);
2d064c06 779 ivar = array_rd32(E1000_IVAR0, index);
047e0030 780 if (tx_queue < 8) {
26bc19ec
AD		 781 /* vector goes into second byte of register */
782 ivar = ivar & 0xFFFF00FF;
783 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 784 } else {
785 /* vector goes into high byte of register */
786 ivar = ivar & 0x00FFFFFF;
787 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 788 }
2d064c06
AD		 789 array_wr32(E1000_IVAR0, index, ivar);
790 }
047e0030 791 q_vector->eims_value = 1 << msix_vector;
2d064c06 792 break;
55cac248 793 case e1000_82580:
d2ba2ed8 794 case e1000_i350:
55cac248
AD		 795 /* 82580 uses the same table-based approach as 82576 but has fewer
796 entries as a result we carry over for queues greater than 4. */
797 if (rx_queue > IGB_N0_QUEUE) {
798 index = (rx_queue >> 1);
799 ivar = array_rd32(E1000_IVAR0, index);
800 if (rx_queue & 0x1) {
801 /* vector goes into third byte of register */
802 ivar = ivar & 0xFF00FFFF;
803 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
804 } else {
805 /* vector goes into low byte of register */
806 ivar = ivar & 0xFFFFFF00;
807 ivar |= msix_vector | E1000_IVAR_VALID;
808 }
809 array_wr32(E1000_IVAR0, index, ivar);
810 }
811 if (tx_queue > IGB_N0_QUEUE) {
812 index = (tx_queue >> 1);
813 ivar = array_rd32(E1000_IVAR0, index);
814 if (tx_queue & 0x1) {
815 /* vector goes into high byte of register */
816 ivar = ivar & 0x00FFFFFF;
817 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
818 } else {
819 /* vector goes into second byte of register */
820 ivar = ivar & 0xFFFF00FF;
821 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
822 }
823 array_wr32(E1000_IVAR0, index, ivar);
824 }
825 q_vector->eims_value = 1 << msix_vector;
826 break;
2d064c06
AD		 827 default:
828 BUG();
829 break;
830 }
26b39276
AD		 831
832 /* add q_vector eims value to global eims_enable_mask */
833 adapter->eims_enable_mask |= q_vector->eims_value;
834
835 /* configure q_vector to set itr on first interrupt */
836 q_vector->set_itr = 1;
9d5c8243
AK		 837}
838
839/**
840 * igb_configure_msix - Configure MSI-X hardware
841 *
842 * igb_configure_msix sets up the hardware to properly
843 * generate MSI-X interrupts.
844 **/
845static void igb_configure_msix(struct igb_adapter *adapter)
846{
847 u32 tmp;
848 int i, vector = 0;
849 struct e1000_hw *hw = &adapter->hw;
850
851 adapter->eims_enable_mask = 0;
9d5c8243
AK		 852
853 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 854 switch (hw->mac.type) {
855 case e1000_82575:
9d5c8243
AK		 856 tmp = rd32(E1000_CTRL_EXT);
857 /* enable MSI-X PBA support*/
858 tmp |= E1000_CTRL_EXT_PBA_CLR;
859
860 /* Auto-Mask interrupts upon ICR read. */
861 tmp |= E1000_CTRL_EXT_EIAME;
862 tmp |= E1000_CTRL_EXT_IRCA;
863
864 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 865
866 /* enable msix_other interrupt */
867 array_wr32(E1000_MSIXBM(0), vector++,
868 E1000_EIMS_OTHER);
844290e5 869 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 870
2d064c06
AD		 871 break;
872
873 case e1000_82576:
55cac248 874 case e1000_82580:
d2ba2ed8 875 case e1000_i350:
047e0030
AD		 876 /* Turn on MSI-X capability first, or our settings
877 * won't stick. And it will take days to debug. */
878 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
879 E1000_GPIE_PBA | E1000_GPIE_EIAME |
880 E1000_GPIE_NSICR);
881
882 /* enable msix_other interrupt */
883 adapter->eims_other = 1 << vector;
2d064c06 884 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 885
047e0030 886 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 887 break;
888 default:
889 /* do nothing, since nothing else supports MSI-X */
890 break;
891 } /* switch (hw->mac.type) */
047e0030
AD		 892
893 adapter->eims_enable_mask |= adapter->eims_other;
894
26b39276
AD		 895 for (i = 0; i < adapter->num_q_vectors; i++)
896 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 897
9d5c8243
AK		 898 wrfl();
899}
900
901/**
902 * igb_request_msix - Initialize MSI-X interrupts
903 *
904 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
905 * kernel.
906 **/
907static int igb_request_msix(struct igb_adapter *adapter)
908{
909 struct net_device *netdev = adapter->netdev;
047e0030 910 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 911 int i, err = 0, vector = 0;
912
047e0030 913 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 914 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 915 if (err)
916 goto out;
917 vector++;
918
919 for (i = 0; i < adapter->num_q_vectors; i++) {
920 struct igb_q_vector *q_vector = adapter->q_vector[i];
921
922 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
923
924 if (q_vector->rx_ring && q_vector->tx_ring)
925 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
926 q_vector->rx_ring->queue_index);
927 else if (q_vector->tx_ring)
928 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
929 q_vector->tx_ring->queue_index);
930 else if (q_vector->rx_ring)
931 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
932 q_vector->rx_ring->queue_index);
9d5c8243 933 else
047e0030
AD		 934 sprintf(q_vector->name, "%s-unused", netdev->name);
935
9d5c8243 936 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 937 igb_msix_ring, 0, q_vector->name,
047e0030 938 q_vector);
9d5c8243
AK		 939 if (err)
940 goto out;
9d5c8243
AK		 941 vector++;
942 }
943
9d5c8243
AK		 944 igb_configure_msix(adapter);
945 return 0;
946out:
947 return err;
948}
949
950static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
951{
952 if (adapter->msix_entries) {
953 pci_disable_msix(adapter->pdev);
954 kfree(adapter->msix_entries);
955 adapter->msix_entries = NULL;
047e0030 956 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 957 pci_disable_msi(adapter->pdev);
047e0030 958 }
9d5c8243
AK		 959}
960
047e0030
AD		 961/**
962 * igb_free_q_vectors - Free memory allocated for interrupt vectors
963 * @adapter: board private structure to initialize
964 *
965 * This function frees the memory allocated to the q_vectors. In addition if
966 * NAPI is enabled it will delete any references to the NAPI struct prior
967 * to freeing the q_vector.
968 **/
969static void igb_free_q_vectors(struct igb_adapter *adapter)
970{
971 int v_idx;
972
973 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
974 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
975 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 976 if (!q_vector)
977 continue;
047e0030
AD		 978 netif_napi_del(&q_vector->napi);
979 kfree(q_vector);
980 }
981 adapter->num_q_vectors = 0;
982}
983
984/**
985 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
986 *
987 * This function resets the device so that it has 0 rx queues, tx queues, and
988 * MSI-X interrupts allocated.
989 */
990static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
991{
992 igb_free_queues(adapter);
993 igb_free_q_vectors(adapter);
994 igb_reset_interrupt_capability(adapter);
995}
9d5c8243
AK		 996
997/**
998 * igb_set_interrupt_capability - set MSI or MSI-X if supported
999 *
1000 * Attempt to configure interrupts using the best available
1001 * capabilities of the hardware and kernel.
1002 **/
21adef3e 1003static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1004{
1005 int err;
1006 int numvecs, i;
1007
83b7180d 1008 /* Number of supported queues. */
a99955fc 1009 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1010 if (adapter->vfs_allocated_count)
1011 adapter->num_tx_queues = 1;
1012 else
1013 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1014
047e0030
AD		 1015 /* start with one vector for every rx queue */
1016 numvecs = adapter->num_rx_queues;
1017
3ad2f3fb 1018 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1019 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1020 numvecs += adapter->num_tx_queues;
047e0030
AD		 1021
1022 /* store the number of vectors reserved for queues */
1023 adapter->num_q_vectors = numvecs;
1024
1025 /* add 1 vector for link status interrupts */
1026 numvecs++;
9d5c8243
AK		 1027 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1028 GFP_KERNEL);
1029 if (!adapter->msix_entries)
1030 goto msi_only;
1031
1032 for (i = 0; i < numvecs; i++)
1033 adapter->msix_entries[i].entry = i;
1034
1035 err = pci_enable_msix(adapter->pdev,
1036 adapter->msix_entries,
1037 numvecs);
1038 if (err == 0)
34a20e89 1039 goto out;
9d5c8243
AK		 1040
1041 igb_reset_interrupt_capability(adapter);
1042
1043 /* If we can't do MSI-X, try MSI */
1044msi_only:
2a3abf6d
AD		 1045#ifdef CONFIG_PCI_IOV
1046 /* disable SR-IOV for non MSI-X configurations */
1047 if (adapter->vf_data) {
1048 struct e1000_hw *hw = &adapter->hw;
1049 /* disable iov and allow time for transactions to clear */
1050 pci_disable_sriov(adapter->pdev);
1051 msleep(500);
1052
1053 kfree(adapter->vf_data);
1054 adapter->vf_data = NULL;
1055 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1056 wrfl();
2a3abf6d
AD		 1057 msleep(100);
1058 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1059 }
1060#endif
4fc82adf 1061 adapter->vfs_allocated_count = 0;
a99955fc 1062 adapter->rss_queues = 1;
4fc82adf 1063 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1064 adapter->num_rx_queues = 1;
661086df 1065 adapter->num_tx_queues = 1;
047e0030 1066 adapter->num_q_vectors = 1;
9d5c8243 1067 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1068 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1069out:
21adef3e
BH		 1070 /* Notify the stack of the (possibly) reduced queue counts. */
1071 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1072 return netif_set_real_num_rx_queues(adapter->netdev,
1073 adapter->num_rx_queues);
9d5c8243
AK		 1074}
1075
047e0030
AD		 1076/**
1077 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1078 * @adapter: board private structure to initialize
1079 *
1080 * We allocate one q_vector per queue interrupt. If allocation fails we
1081 * return -ENOMEM.
1082 **/
1083static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1084{
1085 struct igb_q_vector *q_vector;
1086 struct e1000_hw *hw = &adapter->hw;
1087 int v_idx;
1088
1089 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1090 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1091 if (!q_vector)
1092 goto err_out;
1093 q_vector->adapter = adapter;
047e0030
AD		 1094 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1095 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1096 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1097 adapter->q_vector[v_idx] = q_vector;
1098 }
1099 return 0;
1100
1101err_out:
fe0592b4 1102 igb_free_q_vectors(adapter);
047e0030
AD		 1103 return -ENOMEM;
1104}
1105
1106static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1107 int ring_idx, int v_idx)
1108{
3025a446 1109 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1110
3025a446 1111 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1112 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1113 q_vector->itr_val = adapter->rx_itr_setting;
1114 if (q_vector->itr_val && q_vector->itr_val <= 3)
1115 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1116}
1117
1118static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1119 int ring_idx, int v_idx)
1120{
3025a446 1121 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1122
3025a446 1123 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1124 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1125 q_vector->itr_val = adapter->tx_itr_setting;
1126 if (q_vector->itr_val && q_vector->itr_val <= 3)
1127 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1128}
1129
1130/**
1131 * igb_map_ring_to_vector - maps allocated queues to vectors
1132 *
1133 * This function maps the recently allocated queues to vectors.
1134 **/
1135static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1136{
1137 int i;
1138 int v_idx = 0;
1139
1140 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1141 (adapter->num_q_vectors < adapter->num_tx_queues))
1142 return -ENOMEM;
1143
1144 if (adapter->num_q_vectors >=
1145 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1146 for (i = 0; i < adapter->num_rx_queues; i++)
1147 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1148 for (i = 0; i < adapter->num_tx_queues; i++)
1149 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1150 } else {
1151 for (i = 0; i < adapter->num_rx_queues; i++) {
1152 if (i < adapter->num_tx_queues)
1153 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1154 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1155 }
1156 for (; i < adapter->num_tx_queues; i++)
1157 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1158 }
1159 return 0;
1160}
1161
1162/**
1163 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1164 *
1165 * This function initializes the interrupts and allocates all of the queues.
1166 **/
1167static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1168{
1169 struct pci_dev *pdev = adapter->pdev;
1170 int err;
1171
21adef3e
BH		 1172 err = igb_set_interrupt_capability(adapter);
1173 if (err)
1174 return err;
047e0030
AD		 1175
1176 err = igb_alloc_q_vectors(adapter);
1177 if (err) {
1178 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1179 goto err_alloc_q_vectors;
1180 }
1181
1182 err = igb_alloc_queues(adapter);
1183 if (err) {
1184 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1185 goto err_alloc_queues;
1186 }
1187
1188 err = igb_map_ring_to_vector(adapter);
1189 if (err) {
1190 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1191 goto err_map_queues;
1192 }
1193
1194
1195 return 0;
1196err_map_queues:
1197 igb_free_queues(adapter);
1198err_alloc_queues:
1199 igb_free_q_vectors(adapter);
1200err_alloc_q_vectors:
1201 igb_reset_interrupt_capability(adapter);
1202 return err;
1203}
1204
9d5c8243
AK		 1205/**
1206 * igb_request_irq - initialize interrupts
1207 *
1208 * Attempts to configure interrupts using the best available
1209 * capabilities of the hardware and kernel.
1210 **/
1211static int igb_request_irq(struct igb_adapter *adapter)
1212{
1213 struct net_device *netdev = adapter->netdev;
047e0030 1214 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1215 int err = 0;
1216
1217 if (adapter->msix_entries) {
1218 err = igb_request_msix(adapter);
844290e5 1219 if (!err)
9d5c8243 1220 goto request_done;
9d5c8243 1221 /* fall back to MSI */
047e0030 1222 igb_clear_interrupt_scheme(adapter);
9d5c8243 1223 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1224 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1225 igb_free_all_tx_resources(adapter);
1226 igb_free_all_rx_resources(adapter);
047e0030 1227 adapter->num_tx_queues = 1;
9d5c8243 1228 adapter->num_rx_queues = 1;
047e0030
AD		 1229 adapter->num_q_vectors = 1;
1230 err = igb_alloc_q_vectors(adapter);
1231 if (err) {
1232 dev_err(&pdev->dev,
1233 "Unable to allocate memory for vectors\n");
1234 goto request_done;
1235 }
1236 err = igb_alloc_queues(adapter);
1237 if (err) {
1238 dev_err(&pdev->dev,
1239 "Unable to allocate memory for queues\n");
1240 igb_free_q_vectors(adapter);
1241 goto request_done;
1242 }
1243 igb_setup_all_tx_resources(adapter);
1244 igb_setup_all_rx_resources(adapter);
844290e5 1245 } else {
feeb2721 1246 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1247 }
844290e5 1248
7dfc16fa 1249 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1250 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1251 netdev->name, adapter);
9d5c8243
AK		 1252 if (!err)
1253 goto request_done;
047e0030 1254
9d5c8243
AK		 1255 /* fall back to legacy interrupts */
1256 igb_reset_interrupt_capability(adapter);
7dfc16fa 1257 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1258 }
1259
a0607fd3 1260 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1261 netdev->name, adapter);
9d5c8243 1262
6cb5e577 1263 if (err)
9d5c8243
AK		 1264 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1265 err);
9d5c8243
AK		 1266
1267request_done:
1268 return err;
1269}
1270
1271static void igb_free_irq(struct igb_adapter *adapter)
1272{
9d5c8243
AK		 1273 if (adapter->msix_entries) {
1274 int vector = 0, i;
1275
047e0030 1276 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1277
047e0030
AD		 1278 for (i = 0; i < adapter->num_q_vectors; i++) {
1279 struct igb_q_vector *q_vector = adapter->q_vector[i];
1280 free_irq(adapter->msix_entries[vector++].vector,
1281 q_vector);
1282 }
1283 } else {
1284 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1285 }
9d5c8243
AK		 1286}
1287
1288/**
1289 * igb_irq_disable - Mask off interrupt generation on the NIC
1290 * @adapter: board private structure
1291 **/
1292static void igb_irq_disable(struct igb_adapter *adapter)
1293{
1294 struct e1000_hw *hw = &adapter->hw;
1295
25568a53
AD		 1296 /*
1297 * we need to be careful when disabling interrupts. The VFs are also
1298 * mapped into these registers and so clearing the bits can cause
1299 * issues on the VF drivers so we only need to clear what we set
1300 */
9d5c8243 1301 if (adapter->msix_entries) {
2dfd1212
AD		 1302 u32 regval = rd32(E1000_EIAM);
1303 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1304 wr32(E1000_EIMC, adapter->eims_enable_mask);
1305 regval = rd32(E1000_EIAC);
1306 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1307 }
844290e5
PW		 1308
1309 wr32(E1000_IAM, 0);
9d5c8243
AK		 1310 wr32(E1000_IMC, ~0);
1311 wrfl();
81a61859
ET		 1312 if (adapter->msix_entries) {
1313 int i;
1314 for (i = 0; i < adapter->num_q_vectors; i++)
1315 synchronize_irq(adapter->msix_entries[i].vector);
1316 } else {
1317 synchronize_irq(adapter->pdev->irq);
1318 }
9d5c8243
AK		 1319}
1320
1321/**
1322 * igb_irq_enable - Enable default interrupt generation settings
1323 * @adapter: board private structure
1324 **/
1325static void igb_irq_enable(struct igb_adapter *adapter)
1326{
1327 struct e1000_hw *hw = &adapter->hw;
1328
1329 if (adapter->msix_entries) {
25568a53 1330 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1331 u32 regval = rd32(E1000_EIAC);
1332 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1333 regval = rd32(E1000_EIAM);
1334 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1335 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1336 if (adapter->vfs_allocated_count) {
4ae196df 1337 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1338 ims |= E1000_IMS_VMMB;
1339 }
55cac248
AD		 1340 if (adapter->hw.mac.type == e1000_82580)
1341 ims |= E1000_IMS_DRSTA;
1342
25568a53 1343 wr32(E1000_IMS, ims);
844290e5 1344 } else {
55cac248
AD		 1345 wr32(E1000_IMS, IMS_ENABLE_MASK |
1346 E1000_IMS_DRSTA);
1347 wr32(E1000_IAM, IMS_ENABLE_MASK |
1348 E1000_IMS_DRSTA);
844290e5 1349 }
9d5c8243
AK		 1350}
1351
1352static void igb_update_mng_vlan(struct igb_adapter *adapter)
1353{
51466239 1354 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1355 u16 vid = adapter->hw.mng_cookie.vlan_id;
1356 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1357
1358 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1359 /* add VID to filter table */
1360 igb_vfta_set(hw, vid, true);
1361 adapter->mng_vlan_id = vid;
1362 } else {
1363 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1364 }
1365
1366 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1367 (vid != old_vid) &&
b2cb09b1 1368 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1369 /* remove VID from filter table */
1370 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1371 }
1372}
1373
1374/**
1375 * igb_release_hw_control - release control of the h/w to f/w
1376 * @adapter: address of board private structure
1377 *
1378 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1379 * For ASF and Pass Through versions of f/w this means that the
1380 * driver is no longer loaded.
1381 *
1382 **/
1383static void igb_release_hw_control(struct igb_adapter *adapter)
1384{
1385 struct e1000_hw *hw = &adapter->hw;
1386 u32 ctrl_ext;
1387
1388 /* Let firmware take over control of h/w */
1389 ctrl_ext = rd32(E1000_CTRL_EXT);
1390 wr32(E1000_CTRL_EXT,
1391 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1392}
1393
9d5c8243
AK		 1394/**
1395 * igb_get_hw_control - get control of the h/w from f/w
1396 * @adapter: address of board private structure
1397 *
1398 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1399 * For ASF and Pass Through versions of f/w this means that
1400 * the driver is loaded.
1401 *
1402 **/
1403static void igb_get_hw_control(struct igb_adapter *adapter)
1404{
1405 struct e1000_hw *hw = &adapter->hw;
1406 u32 ctrl_ext;
1407
1408 /* Let firmware know the driver has taken over */
1409 ctrl_ext = rd32(E1000_CTRL_EXT);
1410 wr32(E1000_CTRL_EXT,
1411 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1412}
1413
9d5c8243
AK		 1414/**
1415 * igb_configure - configure the hardware for RX and TX
1416 * @adapter: private board structure
1417 **/
1418static void igb_configure(struct igb_adapter *adapter)
1419{
1420 struct net_device *netdev = adapter->netdev;
1421 int i;
1422
1423 igb_get_hw_control(adapter);
ff41f8dc 1424 igb_set_rx_mode(netdev);
9d5c8243
AK		 1425
1426 igb_restore_vlan(adapter);
9d5c8243 1427
85b430b4 1428 igb_setup_tctl(adapter);
06cf2666 1429 igb_setup_mrqc(adapter);
9d5c8243 1430 igb_setup_rctl(adapter);
85b430b4
AD		 1431
1432 igb_configure_tx(adapter);
9d5c8243 1433 igb_configure_rx(adapter);
662d7205
AD		 1434
1435 igb_rx_fifo_flush_82575(&adapter->hw);
1436
c493ea45 1437 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1438 * at least 1 descriptor unused to make sure
1439 * next_to_use != next_to_clean */
1440 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1441 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1442 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1443 }
9d5c8243
AK		 1444}
1445
88a268c1
NN		 1446/**
1447 * igb_power_up_link - Power up the phy/serdes link
1448 * @adapter: address of board private structure
1449 **/
1450void igb_power_up_link(struct igb_adapter *adapter)
1451{
1452 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1453 igb_power_up_phy_copper(&adapter->hw);
1454 else
1455 igb_power_up_serdes_link_82575(&adapter->hw);
1456}
1457
1458/**
1459 * igb_power_down_link - Power down the phy/serdes link
1460 * @adapter: address of board private structure
1461 */
1462static void igb_power_down_link(struct igb_adapter *adapter)
1463{
1464 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1465 igb_power_down_phy_copper_82575(&adapter->hw);
1466 else
1467 igb_shutdown_serdes_link_82575(&adapter->hw);
1468}
9d5c8243
AK		 1469
1470/**
1471 * igb_up - Open the interface and prepare it to handle traffic
1472 * @adapter: board private structure
1473 **/
9d5c8243
AK		 1474int igb_up(struct igb_adapter *adapter)
1475{
1476 struct e1000_hw *hw = &adapter->hw;
1477 int i;
1478
1479 /* hardware has been reset, we need to reload some things */
1480 igb_configure(adapter);
1481
1482 clear_bit(__IGB_DOWN, &adapter->state);
1483
047e0030
AD		 1484 for (i = 0; i < adapter->num_q_vectors; i++) {
1485 struct igb_q_vector *q_vector = adapter->q_vector[i];
1486 napi_enable(&q_vector->napi);
1487 }
844290e5 1488 if (adapter->msix_entries)
9d5c8243 1489 igb_configure_msix(adapter);
feeb2721
AD		 1490 else
1491 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1492
1493 /* Clear any pending interrupts. */
1494 rd32(E1000_ICR);
1495 igb_irq_enable(adapter);
1496
d4960307
AD		 1497 /* notify VFs that reset has been completed */
1498 if (adapter->vfs_allocated_count) {
1499 u32 reg_data = rd32(E1000_CTRL_EXT);
1500 reg_data |= E1000_CTRL_EXT_PFRSTD;
1501 wr32(E1000_CTRL_EXT, reg_data);
1502 }
1503
4cb9be7a
JB		 1504 netif_tx_start_all_queues(adapter->netdev);
1505
25568a53
AD		 1506 /* start the watchdog. */
1507 hw->mac.get_link_status = 1;
1508 schedule_work(&adapter->watchdog_task);
1509
9d5c8243
AK		 1510 return 0;
1511}
1512
1513void igb_down(struct igb_adapter *adapter)
1514{
9d5c8243 1515 struct net_device *netdev = adapter->netdev;
330a6d6a 1516 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1517 u32 tctl, rctl;
1518 int i;
1519
1520 /* signal that we're down so the interrupt handler does not
1521 * reschedule our watchdog timer */
1522 set_bit(__IGB_DOWN, &adapter->state);
1523
1524 /* disable receives in the hardware */
1525 rctl = rd32(E1000_RCTL);
1526 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1527 /* flush and sleep below */
1528
fd2ea0a7 1529 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1530
1531 /* disable transmits in the hardware */
1532 tctl = rd32(E1000_TCTL);
1533 tctl &= ~E1000_TCTL_EN;
1534 wr32(E1000_TCTL, tctl);
1535 /* flush both disables and wait for them to finish */
1536 wrfl();
1537 msleep(10);
1538
047e0030
AD		 1539 for (i = 0; i < adapter->num_q_vectors; i++) {
1540 struct igb_q_vector *q_vector = adapter->q_vector[i];
1541 napi_disable(&q_vector->napi);
1542 }
9d5c8243 1543
9d5c8243
AK		 1544 igb_irq_disable(adapter);
1545
1546 del_timer_sync(&adapter->watchdog_timer);
1547 del_timer_sync(&adapter->phy_info_timer);
1548
9d5c8243 1549 netif_carrier_off(netdev);
04fe6358
AD		 1550
1551 /* record the stats before reset*/
12dcd86b
ED		 1552 spin_lock(&adapter->stats64_lock);
1553 igb_update_stats(adapter, &adapter->stats64);
1554 spin_unlock(&adapter->stats64_lock);
04fe6358 1555
9d5c8243
AK		 1556 adapter->link_speed = 0;
1557 adapter->link_duplex = 0;
1558
3023682e
JK		 1559 if (!pci_channel_offline(adapter->pdev))
1560 igb_reset(adapter);
9d5c8243
AK		 1561 igb_clean_all_tx_rings(adapter);
1562 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1563#ifdef CONFIG_IGB_DCA
1564
1565 /* since we reset the hardware DCA settings were cleared */
1566 igb_setup_dca(adapter);
1567#endif
9d5c8243
AK		 1568}
1569
1570void igb_reinit_locked(struct igb_adapter *adapter)
1571{
1572 WARN_ON(in_interrupt());
1573 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1574 msleep(1);
1575 igb_down(adapter);
1576 igb_up(adapter);
1577 clear_bit(__IGB_RESETTING, &adapter->state);
1578}
1579
1580void igb_reset(struct igb_adapter *adapter)
1581{
090b1795 1582 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1583 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1584 struct e1000_mac_info *mac = &hw->mac;
1585 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1586 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1587 u16 hwm;
1588
1589 /* Repartition Pba for greater than 9k mtu
1590 * To take effect CTRL.RST is required.
1591 */
fa4dfae0 1592 switch (mac->type) {
d2ba2ed8 1593 case e1000_i350:
55cac248
AD		 1594 case e1000_82580:
1595 pba = rd32(E1000_RXPBS);
1596 pba = igb_rxpbs_adjust_82580(pba);
1597 break;
fa4dfae0 1598 case e1000_82576:
d249be54
AD		 1599 pba = rd32(E1000_RXPBS);
1600 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1601 break;
1602 case e1000_82575:
1603 default:
1604 pba = E1000_PBA_34K;
1605 break;
2d064c06 1606 }
9d5c8243 1607
2d064c06
AD		 1608 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1609 (mac->type < e1000_82576)) {
9d5c8243
AK		 1610 /* adjust PBA for jumbo frames */
1611 wr32(E1000_PBA, pba);
1612
1613 /* To maintain wire speed transmits, the Tx FIFO should be
1614 * large enough to accommodate two full transmit packets,
1615 * rounded up to the next 1KB and expressed in KB. Likewise,
1616 * the Rx FIFO should be large enough to accommodate at least
1617 * one full receive packet and is similarly rounded up and
1618 * expressed in KB. */
1619 pba = rd32(E1000_PBA);
1620 /* upper 16 bits has Tx packet buffer allocation size in KB */
1621 tx_space = pba >> 16;
1622 /* lower 16 bits has Rx packet buffer allocation size in KB */
1623 pba &= 0xffff;
1624 /* the tx fifo also stores 16 bytes of information about the tx
1625 * but don't include ethernet FCS because hardware appends it */
1626 min_tx_space = (adapter->max_frame_size +
85e8d004 1627 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1628 ETH_FCS_LEN) * 2;
1629 min_tx_space = ALIGN(min_tx_space, 1024);
1630 min_tx_space >>= 10;
1631 /* software strips receive CRC, so leave room for it */
1632 min_rx_space = adapter->max_frame_size;
1633 min_rx_space = ALIGN(min_rx_space, 1024);
1634 min_rx_space >>= 10;
1635
1636 /* If current Tx allocation is less than the min Tx FIFO size,
1637 * and the min Tx FIFO size is less than the current Rx FIFO
1638 * allocation, take space away from current Rx allocation */
1639 if (tx_space < min_tx_space &&
1640 ((min_tx_space - tx_space) < pba)) {
1641 pba = pba - (min_tx_space - tx_space);
1642
1643 /* if short on rx space, rx wins and must trump tx
1644 * adjustment */
1645 if (pba < min_rx_space)
1646 pba = min_rx_space;
1647 }
2d064c06 1648 wr32(E1000_PBA, pba);
9d5c8243 1649 }
9d5c8243
AK		 1650
1651 /* flow control settings */
1652 /* The high water mark must be low enough to fit one full frame
1653 * (or the size used for early receive) above it in the Rx FIFO.
1654 * Set it to the lower of:
1655 * - 90% of the Rx FIFO size, or
1656 * - the full Rx FIFO size minus one full frame */
1657 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1658 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1659
d405ea3e
AD		 1660 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1661 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1662 fc->pause_time = 0xFFFF;
1663 fc->send_xon = 1;
0cce119a 1664 fc->current_mode = fc->requested_mode;
9d5c8243 1665
4ae196df
AD		 1666 /* disable receive for all VFs and wait one second */
1667 if (adapter->vfs_allocated_count) {
1668 int i;
1669 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1670 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1671
1672 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1673 igb_ping_all_vfs(adapter);
4ae196df
AD		 1674
1675 /* disable transmits and receives */
1676 wr32(E1000_VFRE, 0);
1677 wr32(E1000_VFTE, 0);
1678 }
1679
9d5c8243 1680 /* Allow time for pending master requests to run */
330a6d6a 1681 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1682 wr32(E1000_WUC, 0);
1683
330a6d6a 1684 if (hw->mac.ops.init_hw(hw))
090b1795 1685 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4
CW		 1686 if (hw->mac.type > e1000_82580) {
1687 if (adapter->flags & IGB_FLAG_DMAC) {
1688 u32 reg;
1689
1690 /*
1691 * DMA Coalescing high water mark needs to be higher
1692 * than * the * Rx threshold. The Rx threshold is
1693 * currently * pba - 6, so we * should use a high water
1694 * mark of pba * - 4. */
1695 hwm = (pba - 4) << 10;
1696
1697 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
1698 & E1000_DMACR_DMACTHR_MASK);
1699
1700 /* transition to L0x or L1 if available..*/
1701 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
1702
1703 /* watchdog timer= +-1000 usec in 32usec intervals */
1704 reg |= (1000 >> 5);
1705 wr32(E1000_DMACR, reg);
1706
1707 /* no lower threshold to disable coalescing(smart fifb)
1708 * -UTRESH=0*/
1709 wr32(E1000_DMCRTRH, 0);
1710
1711 /* set hwm to PBA - 2 * max frame size */
1712 wr32(E1000_FCRTC, hwm);
1713
1714 /*
1715 * This sets the time to wait before requesting tran-
1716 * sition to * low power state to number of usecs needed
1717 * to receive 1 512 * byte frame at gigabit line rate
1718 */
1719 reg = rd32(E1000_DMCTLX);
1720 reg |= IGB_DMCTLX_DCFLUSH_DIS;
1721
1722 /* Delay 255 usec before entering Lx state. */
1723 reg |= 0xFF;
1724 wr32(E1000_DMCTLX, reg);
1725
1726 /* free space in Tx packet buffer to wake from DMAC */
1727 wr32(E1000_DMCTXTH,
1728 (IGB_MIN_TXPBSIZE -
1729 (IGB_TX_BUF_4096 + adapter->max_frame_size))
1730 >> 6);
1731
1732 /* make low power state decision controlled by DMAC */
1733 reg = rd32(E1000_PCIEMISC);
1734 reg |= E1000_PCIEMISC_LX_DECISION;
1735 wr32(E1000_PCIEMISC, reg);
1736 } /* end if IGB_FLAG_DMAC set */
1737 }
55cac248
AD		 1738 if (hw->mac.type == e1000_82580) {
1739 u32 reg = rd32(E1000_PCIEMISC);
1740 wr32(E1000_PCIEMISC,
1741 reg & ~E1000_PCIEMISC_LX_DECISION);
1742 }
88a268c1
NN		 1743 if (!netif_running(adapter->netdev))
1744 igb_power_down_link(adapter);
1745
9d5c8243
AK		 1746 igb_update_mng_vlan(adapter);
1747
1748 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1749 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1750
330a6d6a 1751 igb_get_phy_info(hw);
9d5c8243
AK		 1752}
1753
b2cb09b1
JP		 1754static u32 igb_fix_features(struct net_device *netdev, u32 features)
1755{
1756 /*
1757 * Since there is no support for separate rx/tx vlan accel
1758 * enable/disable make sure tx flag is always in same state as rx.
1759 */
1760 if (features & NETIF_F_HW_VLAN_RX)
1761 features |= NETIF_F_HW_VLAN_TX;
1762 else
1763 features &= ~NETIF_F_HW_VLAN_TX;
1764
1765 return features;
1766}
1767
ac52caa3
MM		 1768static int igb_set_features(struct net_device *netdev, u32 features)
1769{
1770 struct igb_adapter *adapter = netdev_priv(netdev);
1771 int i;
b2cb09b1 1772 u32 changed = netdev->features ^ features;
ac52caa3
MM		 1773
1774 for (i = 0; i < adapter->num_rx_queues; i++) {
1775 if (features & NETIF_F_RXCSUM)
1776 adapter->rx_ring[i]->flags |= IGB_RING_FLAG_RX_CSUM;
1777 else
1778 adapter->rx_ring[i]->flags &= ~IGB_RING_FLAG_RX_CSUM;
1779 }
1780
b2cb09b1
JP		 1781 if (changed & NETIF_F_HW_VLAN_RX)
1782 igb_vlan_mode(netdev, features);
1783
ac52caa3
MM		 1784 return 0;
1785}
1786
2e5c6922 1787static const struct net_device_ops igb_netdev_ops = {
559e9c49 1788 .ndo_open = igb_open,
2e5c6922 1789 .ndo_stop = igb_close,
00829823 1790 .ndo_start_xmit = igb_xmit_frame_adv,
12dcd86b 1791 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1792 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1793 .ndo_set_mac_address = igb_set_mac,
1794 .ndo_change_mtu = igb_change_mtu,
1795 .ndo_do_ioctl = igb_ioctl,
1796 .ndo_tx_timeout = igb_tx_timeout,
1797 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1798 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1799 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1800 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1801 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1802 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1803 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1804#ifdef CONFIG_NET_POLL_CONTROLLER
1805 .ndo_poll_controller = igb_netpoll,
1806#endif
b2cb09b1
JP		 1807 .ndo_fix_features = igb_fix_features,
1808 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1809};
1810
9d5c8243
AK		 1811/**
1812 * igb_probe - Device Initialization Routine
1813 * @pdev: PCI device information struct
1814 * @ent: entry in igb_pci_tbl
1815 *
1816 * Returns 0 on success, negative on failure
1817 *
1818 * igb_probe initializes an adapter identified by a pci_dev structure.
1819 * The OS initialization, configuring of the adapter private structure,
1820 * and a hardware reset occur.
1821 **/
1822static int __devinit igb_probe(struct pci_dev *pdev,
1823 const struct pci_device_id *ent)
1824{
1825 struct net_device *netdev;
1826 struct igb_adapter *adapter;
1827 struct e1000_hw *hw;
4337e993 1828 u16 eeprom_data = 0;
9835fd73 1829 s32 ret_val;
4337e993 1830 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1831 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1832 unsigned long mmio_start, mmio_len;
2d6a5e95 1833 int err, pci_using_dac;
9d5c8243 1834 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1835 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1836
bded64a7
AG		 1837 /* Catch broken hardware that put the wrong VF device ID in
1838 * the PCIe SR-IOV capability.
1839 */
1840 if (pdev->is_virtfn) {
1841 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1842 pci_name(pdev), pdev->vendor, pdev->device);
1843 return -EINVAL;
1844 }
1845
aed5dec3 1846 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1847 if (err)
1848 return err;
1849
1850 pci_using_dac = 0;
59d71989 1851 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1852 if (!err) {
59d71989 1853 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1854 if (!err)
1855 pci_using_dac = 1;
1856 } else {
59d71989 1857 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1858 if (err) {
59d71989 1859 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1860 if (err) {
1861 dev_err(&pdev->dev, "No usable DMA "
1862 "configuration, aborting\n");
1863 goto err_dma;
1864 }
1865 }
1866 }
1867
aed5dec3
AD		 1868 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1869 IORESOURCE_MEM),
1870 igb_driver_name);
9d5c8243
AK		 1871 if (err)
1872 goto err_pci_reg;
1873
19d5afd4 1874 pci_enable_pcie_error_reporting(pdev);
40a914fa 1875
9d5c8243 1876 pci_set_master(pdev);
c682fc23 1877 pci_save_state(pdev);
9d5c8243
AK		 1878
1879 err = -ENOMEM;
1bfaf07b
AD		 1880 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1881 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1882 if (!netdev)
1883 goto err_alloc_etherdev;
1884
1885 SET_NETDEV_DEV(netdev, &pdev->dev);
1886
1887 pci_set_drvdata(pdev, netdev);
1888 adapter = netdev_priv(netdev);
1889 adapter->netdev = netdev;
1890 adapter->pdev = pdev;
1891 hw = &adapter->hw;
1892 hw->back = adapter;
1893 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1894
1895 mmio_start = pci_resource_start(pdev, 0);
1896 mmio_len = pci_resource_len(pdev, 0);
1897
1898 err = -EIO;
28b0759c
AD		 1899 hw->hw_addr = ioremap(mmio_start, mmio_len);
1900 if (!hw->hw_addr)
9d5c8243
AK		 1901 goto err_ioremap;
1902
2e5c6922 1903 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1904 igb_set_ethtool_ops(netdev);
9d5c8243 1905 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1906
1907 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1908
1909 netdev->mem_start = mmio_start;
1910 netdev->mem_end = mmio_start + mmio_len;
1911
9d5c8243
AK		 1912 /* PCI config space info */
1913 hw->vendor_id = pdev->vendor;
1914 hw->device_id = pdev->device;
1915 hw->revision_id = pdev->revision;
1916 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1917 hw->subsystem_device_id = pdev->subsystem_device;
1918
9d5c8243
AK		 1919 /* Copy the default MAC, PHY and NVM function pointers */
1920 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1921 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1922 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1923 /* Initialize skew-specific constants */
1924 err = ei->get_invariants(hw);
1925 if (err)
450c87c8 1926 goto err_sw_init;
9d5c8243 1927
450c87c8 1928 /* setup the private structure */
9d5c8243
AK		 1929 err = igb_sw_init(adapter);
1930 if (err)
1931 goto err_sw_init;
1932
1933 igb_get_bus_info_pcie(hw);
1934
1935 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1936
1937 /* Copper options */
1938 if (hw->phy.media_type == e1000_media_type_copper) {
1939 hw->phy.mdix = AUTO_ALL_MODES;
1940 hw->phy.disable_polarity_correction = false;
1941 hw->phy.ms_type = e1000_ms_hw_default;
1942 }
1943
1944 if (igb_check_reset_block(hw))
1945 dev_info(&pdev->dev,
1946 "PHY reset is blocked due to SOL/IDER session.\n");
1947
ac52caa3 1948 netdev->hw_features = NETIF_F_SG |
7d8eb29e 1949 NETIF_F_IP_CSUM |
ac52caa3
MM		 1950 NETIF_F_IPV6_CSUM |
1951 NETIF_F_TSO |
1952 NETIF_F_TSO6 |
b2cb09b1
JP		 1953 NETIF_F_RXCSUM |
1954 NETIF_F_HW_VLAN_RX;
ac52caa3
MM		 1955
1956 netdev->features = netdev->hw_features |
9d5c8243 1957 NETIF_F_HW_VLAN_TX |
9d5c8243
AK		 1958 NETIF_F_HW_VLAN_FILTER;
1959
48f29ffc
JK		 1960 netdev->vlan_features |= NETIF_F_TSO;
1961 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1962 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1963 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1964 netdev->vlan_features |= NETIF_F_SG;
1965
7b872a55 1966 if (pci_using_dac) {
9d5c8243 1967 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1968 netdev->vlan_features |= NETIF_F_HIGHDMA;
1969 }
9d5c8243 1970
ac52caa3
MM		 1971 if (hw->mac.type >= e1000_82576) {
1972 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 1973 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 1974 }
b9473560 1975
01789349
JP		 1976 netdev->priv_flags |= IFF_UNICAST_FLT;
1977
330a6d6a 1978 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1979
1980 /* before reading the NVM, reset the controller to put the device in a
1981 * known good starting state */
1982 hw->mac.ops.reset_hw(hw);
1983
1984 /* make sure the NVM is good */
4322e561 1985 if (hw->nvm.ops.validate(hw) < 0) {
9d5c8243
AK		 1986 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1987 err = -EIO;
1988 goto err_eeprom;
1989 }
1990
1991 /* copy the MAC address out of the NVM */
1992 if (hw->mac.ops.read_mac_addr(hw))
1993 dev_err(&pdev->dev, "NVM Read Error\n");
1994
1995 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1996 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1997
1998 if (!is_valid_ether_addr(netdev->perm_addr)) {
1999 dev_err(&pdev->dev, "Invalid MAC Address\n");
2000 err = -EIO;
2001 goto err_eeprom;
2002 }
2003
c061b18d 2004 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2005 (unsigned long) adapter);
c061b18d 2006 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2007 (unsigned long) adapter);
9d5c8243
AK		 2008
2009 INIT_WORK(&adapter->reset_task, igb_reset_task);
2010 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2011
450c87c8 2012 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2013 adapter->fc_autoneg = true;
2014 hw->mac.autoneg = true;
2015 hw->phy.autoneg_advertised = 0x2f;
2016
0cce119a
AD		 2017 hw->fc.requested_mode = e1000_fc_default;
2018 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2019
9d5c8243
AK		 2020 igb_validate_mdi_setting(hw);
2021
9d5c8243
AK		 2022 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2023 * enable the ACPI Magic Packet filter
2024 */
2025
a2cf8b6c 2026 if (hw->bus.func == 0)
312c75ae 2027 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2028 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2029 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2030 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2031 &eeprom_data);
a2cf8b6c
AD		 2032 else if (hw->bus.func == 1)
2033 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2034
2035 if (eeprom_data & eeprom_apme_mask)
2036 adapter->eeprom_wol |= E1000_WUFC_MAG;
2037
2038 /* now that we have the eeprom settings, apply the special cases where
2039 * the eeprom may be wrong or the board simply won't support wake on
2040 * lan on a particular port */
2041 switch (pdev->device) {
2042 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2043 adapter->eeprom_wol = 0;
2044 break;
2045 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2046 case E1000_DEV_ID_82576_FIBER:
2047 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2048 /* Wake events only supported on port A for dual fiber
2049 * regardless of eeprom setting */
2050 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2051 adapter->eeprom_wol = 0;
2052 break;
c8ea5ea9 2053 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2054 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2055 /* if quad port adapter, disable WoL on all but port A */
2056 if (global_quad_port_a != 0)
2057 adapter->eeprom_wol = 0;
2058 else
2059 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2060 /* Reset for multiple quad port adapters */
2061 if (++global_quad_port_a == 4)
2062 global_quad_port_a = 0;
2063 break;
9d5c8243
AK		 2064 }
2065
2066 /* initialize the wol settings based on the eeprom settings */
2067 adapter->wol = adapter->eeprom_wol;
e1b86d84 2068 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2069
2070 /* reset the hardware with the new settings */
2071 igb_reset(adapter);
2072
2073 /* let the f/w know that the h/w is now under the control of the
2074 * driver. */
2075 igb_get_hw_control(adapter);
2076
9d5c8243
AK		 2077 strcpy(netdev->name, "eth%d");
2078 err = register_netdev(netdev);
2079 if (err)
2080 goto err_register;
2081
b2cb09b1
JP		 2082 igb_vlan_mode(netdev, netdev->features);
2083
b168dfc5
JB		 2084 /* carrier off reporting is important to ethtool even BEFORE open */
2085 netif_carrier_off(netdev);
2086
421e02f0 2087#ifdef CONFIG_IGB_DCA
bbd98fe4 2088 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2089 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2090 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2091 igb_setup_dca(adapter);
2092 }
fe4506b6 2093
38c845c7 2094#endif
673b8b70
AB		 2095 /* do hw tstamp init after resetting */
2096 igb_init_hw_timer(adapter);
2097
9d5c8243
AK		 2098 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2099 /* print bus type/speed/width info */
7c510e4b 2100 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2101 netdev->name,
559e9c49 2102 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2103 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2104 "unknown"),
59c3de89
AD		 2105 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2106 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2107 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2108 "unknown"),
7c510e4b 2109 netdev->dev_addr);
9d5c8243 2110
9835fd73
CW		 2111 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2112 if (ret_val)
2113 strcpy(part_str, "Unknown");
2114 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2115 dev_info(&pdev->dev,
2116 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2117 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2118 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2119 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2120 switch (hw->mac.type) {
2121 case e1000_i350:
2122 igb_set_eee_i350(hw);
2123 break;
2124 default:
2125 break;
2126 }
9d5c8243
AK		 2127 return 0;
2128
2129err_register:
2130 igb_release_hw_control(adapter);
2131err_eeprom:
2132 if (!igb_check_reset_block(hw))
f5f4cf08 2133 igb_reset_phy(hw);
9d5c8243
AK		 2134
2135 if (hw->flash_address)
2136 iounmap(hw->flash_address);
9d5c8243 2137err_sw_init:
047e0030 2138 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2139 iounmap(hw->hw_addr);
2140err_ioremap:
2141 free_netdev(netdev);
2142err_alloc_etherdev:
559e9c49
AD		 2143 pci_release_selected_regions(pdev,
2144 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2145err_pci_reg:
2146err_dma:
2147 pci_disable_device(pdev);
2148 return err;
2149}
2150
2151/**
2152 * igb_remove - Device Removal Routine
2153 * @pdev: PCI device information struct
2154 *
2155 * igb_remove is called by the PCI subsystem to alert the driver
2156 * that it should release a PCI device. The could be caused by a
2157 * Hot-Plug event, or because the driver is going to be removed from
2158 * memory.
2159 **/
2160static void __devexit igb_remove(struct pci_dev *pdev)
2161{
2162 struct net_device *netdev = pci_get_drvdata(pdev);
2163 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2164 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2165
760141a5
TH		 2166 /*
2167 * The watchdog timer may be rescheduled, so explicitly
2168 * disable watchdog from being rescheduled.
2169 */
9d5c8243
AK		 2170 set_bit(__IGB_DOWN, &adapter->state);
2171 del_timer_sync(&adapter->watchdog_timer);
2172 del_timer_sync(&adapter->phy_info_timer);
2173
760141a5
TH		 2174 cancel_work_sync(&adapter->reset_task);
2175 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2176
421e02f0 2177#ifdef CONFIG_IGB_DCA
7dfc16fa 2178 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2179 dev_info(&pdev->dev, "DCA disabled\n");
2180 dca_remove_requester(&pdev->dev);
7dfc16fa 2181 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2182 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2183 }
2184#endif
2185
9d5c8243
AK		 2186 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2187 * would have already happened in close and is redundant. */
2188 igb_release_hw_control(adapter);
2189
2190 unregister_netdev(netdev);
2191
047e0030 2192 igb_clear_interrupt_scheme(adapter);
9d5c8243 2193
37680117
AD		 2194#ifdef CONFIG_PCI_IOV
2195 /* reclaim resources allocated to VFs */
2196 if (adapter->vf_data) {
2197 /* disable iov and allow time for transactions to clear */
2198 pci_disable_sriov(pdev);
2199 msleep(500);
2200
2201 kfree(adapter->vf_data);
2202 adapter->vf_data = NULL;
2203 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2204 wrfl();
37680117
AD		 2205 msleep(100);
2206 dev_info(&pdev->dev, "IOV Disabled\n");
2207 }
2208#endif
559e9c49 2209
28b0759c
AD		 2210 iounmap(hw->hw_addr);
2211 if (hw->flash_address)
2212 iounmap(hw->flash_address);
559e9c49
AD		 2213 pci_release_selected_regions(pdev,
2214 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2215
2216 free_netdev(netdev);
2217
19d5afd4 2218 pci_disable_pcie_error_reporting(pdev);
40a914fa 2219
9d5c8243
AK		 2220 pci_disable_device(pdev);
2221}
2222
a6b623e0
AD		 2223/**
2224 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2225 * @adapter: board private structure to initialize
2226 *
2227 * This function initializes the vf specific data storage and then attempts to
2228 * allocate the VFs. The reason for ordering it this way is because it is much
2229 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2230 * the memory for the VFs.
2231 **/
2232static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2233{
2234#ifdef CONFIG_PCI_IOV
2235 struct pci_dev *pdev = adapter->pdev;
2236
a6b623e0
AD		 2237 if (adapter->vfs_allocated_count) {
2238 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2239 sizeof(struct vf_data_storage),
2240 GFP_KERNEL);
2241 /* if allocation failed then we do not support SR-IOV */
2242 if (!adapter->vf_data) {
2243 adapter->vfs_allocated_count = 0;
2244 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2245 "Data Storage\n");
2246 }
2247 }
2248
2249 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2250 kfree(adapter->vf_data);
2251 adapter->vf_data = NULL;
2252#endif /* CONFIG_PCI_IOV */
2253 adapter->vfs_allocated_count = 0;
2254#ifdef CONFIG_PCI_IOV
2255 } else {
2256 unsigned char mac_addr[ETH_ALEN];
2257 int i;
2258 dev_info(&pdev->dev, "%d vfs allocated\n",
2259 adapter->vfs_allocated_count);
2260 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2261 random_ether_addr(mac_addr);
2262 igb_set_vf_mac(adapter, i, mac_addr);
2263 }
831ec0b4
CW		 2264 /* DMA Coalescing is not supported in IOV mode. */
2265 if (adapter->flags & IGB_FLAG_DMAC)
2266 adapter->flags &= ~IGB_FLAG_DMAC;
a6b623e0
AD		 2267 }
2268#endif /* CONFIG_PCI_IOV */
2269}
2270
115f459a
AD		 2271
2272/**
2273 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2274 * @adapter: board private structure to initialize
2275 *
2276 * igb_init_hw_timer initializes the function pointer and values for the hw
2277 * timer found in hardware.
2278 **/
2279static void igb_init_hw_timer(struct igb_adapter *adapter)
2280{
2281 struct e1000_hw *hw = &adapter->hw;
2282
2283 switch (hw->mac.type) {
d2ba2ed8 2284 case e1000_i350:
55cac248
AD		 2285 case e1000_82580:
2286 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2287 adapter->cycles.read = igb_read_clock;
2288 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2289 adapter->cycles.mult = 1;
2290 /*
2291 * The 82580 timesync updates the system timer every 8ns by 8ns
2292 * and the value cannot be shifted. Instead we need to shift
2293 * the registers to generate a 64bit timer value. As a result
2294 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2295 * 24 in order to generate a larger value for synchronization.
2296 */
2297 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2298 /* disable system timer temporarily by setting bit 31 */
2299 wr32(E1000_TSAUXC, 0x80000000);
2300 wrfl();
2301
2302 /* Set registers so that rollover occurs soon to test this. */
2303 wr32(E1000_SYSTIMR, 0x00000000);
2304 wr32(E1000_SYSTIML, 0x80000000);
2305 wr32(E1000_SYSTIMH, 0x000000FF);
2306 wrfl();
2307
2308 /* enable system timer by clearing bit 31 */
2309 wr32(E1000_TSAUXC, 0x0);
2310 wrfl();
2311
2312 timecounter_init(&adapter->clock,
2313 &adapter->cycles,
2314 ktime_to_ns(ktime_get_real()));
2315 /*
2316 * Synchronize our NIC clock against system wall clock. NIC
2317 * time stamp reading requires ~3us per sample, each sample
2318 * was pretty stable even under load => only require 10
2319 * samples for each offset comparison.
2320 */
2321 memset(&adapter->compare, 0, sizeof(adapter->compare));
2322 adapter->compare.source = &adapter->clock;
2323 adapter->compare.target = ktime_get_real;
2324 adapter->compare.num_samples = 10;
2325 timecompare_update(&adapter->compare, 0);
2326 break;
115f459a
AD		 2327 case e1000_82576:
2328 /*
2329 * Initialize hardware timer: we keep it running just in case
2330 * that some program needs it later on.
2331 */
2332 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2333 adapter->cycles.read = igb_read_clock;
2334 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2335 adapter->cycles.mult = 1;
2336 /**
2337 * Scale the NIC clock cycle by a large factor so that
2338 * relatively small clock corrections can be added or
25985edc 2339 * subtracted at each clock tick. The drawbacks of a large
115f459a
AD		 2340 * factor are a) that the clock register overflows more quickly
2341 * (not such a big deal) and b) that the increment per tick has
2342 * to fit into 24 bits. As a result we need to use a shift of
2343 * 19 so we can fit a value of 16 into the TIMINCA register.
2344 */
2345 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2346 wr32(E1000_TIMINCA,
2347 (1 << E1000_TIMINCA_16NS_SHIFT) |
2348 (16 << IGB_82576_TSYNC_SHIFT));
2349
2350 /* Set registers so that rollover occurs soon to test this. */
2351 wr32(E1000_SYSTIML, 0x00000000);
2352 wr32(E1000_SYSTIMH, 0xFF800000);
2353 wrfl();
2354
2355 timecounter_init(&adapter->clock,
2356 &adapter->cycles,
2357 ktime_to_ns(ktime_get_real()));
2358 /*
2359 * Synchronize our NIC clock against system wall clock. NIC
2360 * time stamp reading requires ~3us per sample, each sample
2361 * was pretty stable even under load => only require 10
2362 * samples for each offset comparison.
2363 */
2364 memset(&adapter->compare, 0, sizeof(adapter->compare));
2365 adapter->compare.source = &adapter->clock;
2366 adapter->compare.target = ktime_get_real;
2367 adapter->compare.num_samples = 10;
2368 timecompare_update(&adapter->compare, 0);
2369 break;
2370 case e1000_82575:
2371 /* 82575 does not support timesync */
2372 default:
2373 break;
2374 }
2375
2376}
2377
9d5c8243
AK		 2378/**
2379 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2380 * @adapter: board private structure to initialize
2381 *
2382 * igb_sw_init initializes the Adapter private data structure.
2383 * Fields are initialized based on PCI device information and
2384 * OS network device settings (MTU size).
2385 **/
2386static int __devinit igb_sw_init(struct igb_adapter *adapter)
2387{
2388 struct e1000_hw *hw = &adapter->hw;
2389 struct net_device *netdev = adapter->netdev;
2390 struct pci_dev *pdev = adapter->pdev;
2391
2392 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2393
68fd9910
AD		 2394 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2395 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2396 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2397 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2398
153285f9
AD		 2399 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2400 VLAN_HLEN;
9d5c8243
AK		 2401 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2402
12dcd86b 2403 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2404#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2405 switch (hw->mac.type) {
2406 case e1000_82576:
2407 case e1000_i350:
9b082d73
SA		 2408 if (max_vfs > 7) {
2409 dev_warn(&pdev->dev,
2410 "Maximum of 7 VFs per PF, using max\n");
2411 adapter->vfs_allocated_count = 7;
2412 } else
2413 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2414 break;
2415 default:
2416 break;
2417 }
a6b623e0 2418#endif /* CONFIG_PCI_IOV */
a99955fc 2419 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
665c8c8e
WM		 2420 /* i350 cannot do RSS and SR-IOV at the same time */
2421 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2422 adapter->rss_queues = 1;
a99955fc
AD		 2423
2424 /*
2425 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2426 * then we should combine the queues into a queue pair in order to
2427 * conserve interrupts due to limited supply
2428 */
2429 if ((adapter->rss_queues > 4) ||
2430 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2431 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2432
a6b623e0 2433 /* This call may decrease the number of queues */
047e0030 2434 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2435 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2436 return -ENOMEM;
2437 }
2438
a6b623e0
AD		 2439 igb_probe_vfs(adapter);
2440
9d5c8243
AK		 2441 /* Explicitly disable IRQ since the NIC can be in any state. */
2442 igb_irq_disable(adapter);
2443
831ec0b4
CW		 2444 if (hw->mac.type == e1000_i350)
2445 adapter->flags &= ~IGB_FLAG_DMAC;
2446
9d5c8243
AK		 2447 set_bit(__IGB_DOWN, &adapter->state);
2448 return 0;
2449}
2450
2451/**
2452 * igb_open - Called when a network interface is made active
2453 * @netdev: network interface device structure
2454 *
2455 * Returns 0 on success, negative value on failure
2456 *
2457 * The open entry point is called when a network interface is made
2458 * active by the system (IFF_UP). At this point all resources needed
2459 * for transmit and receive operations are allocated, the interrupt
2460 * handler is registered with the OS, the watchdog timer is started,
2461 * and the stack is notified that the interface is ready.
2462 **/
2463static int igb_open(struct net_device *netdev)
2464{
2465 struct igb_adapter *adapter = netdev_priv(netdev);
2466 struct e1000_hw *hw = &adapter->hw;
2467 int err;
2468 int i;
2469
2470 /* disallow open during test */
2471 if (test_bit(__IGB_TESTING, &adapter->state))
2472 return -EBUSY;
2473
b168dfc5
JB		 2474 netif_carrier_off(netdev);
2475
9d5c8243
AK		 2476 /* allocate transmit descriptors */
2477 err = igb_setup_all_tx_resources(adapter);
2478 if (err)
2479 goto err_setup_tx;
2480
2481 /* allocate receive descriptors */
2482 err = igb_setup_all_rx_resources(adapter);
2483 if (err)
2484 goto err_setup_rx;
2485
88a268c1 2486 igb_power_up_link(adapter);
9d5c8243 2487
9d5c8243
AK		 2488 /* before we allocate an interrupt, we must be ready to handle it.
2489 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2490 * as soon as we call pci_request_irq, so we have to setup our
2491 * clean_rx handler before we do so. */
2492 igb_configure(adapter);
2493
2494 err = igb_request_irq(adapter);
2495 if (err)
2496 goto err_req_irq;
2497
2498 /* From here on the code is the same as igb_up() */
2499 clear_bit(__IGB_DOWN, &adapter->state);
2500
047e0030
AD		 2501 for (i = 0; i < adapter->num_q_vectors; i++) {
2502 struct igb_q_vector *q_vector = adapter->q_vector[i];
2503 napi_enable(&q_vector->napi);
2504 }
9d5c8243
AK		 2505
2506 /* Clear any pending interrupts. */
2507 rd32(E1000_ICR);
844290e5
PW		 2508
2509 igb_irq_enable(adapter);
2510
d4960307
AD		 2511 /* notify VFs that reset has been completed */
2512 if (adapter->vfs_allocated_count) {
2513 u32 reg_data = rd32(E1000_CTRL_EXT);
2514 reg_data |= E1000_CTRL_EXT_PFRSTD;
2515 wr32(E1000_CTRL_EXT, reg_data);
2516 }
2517
d55b53ff
JK		 2518 netif_tx_start_all_queues(netdev);
2519
25568a53
AD		 2520 /* start the watchdog. */
2521 hw->mac.get_link_status = 1;
2522 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2523
2524 return 0;
2525
2526err_req_irq:
2527 igb_release_hw_control(adapter);
88a268c1 2528 igb_power_down_link(adapter);
9d5c8243
AK		 2529 igb_free_all_rx_resources(adapter);
2530err_setup_rx:
2531 igb_free_all_tx_resources(adapter);
2532err_setup_tx:
2533 igb_reset(adapter);
2534
2535 return err;
2536}
2537
2538/**
2539 * igb_close - Disables a network interface
2540 * @netdev: network interface device structure
2541 *
2542 * Returns 0, this is not allowed to fail
2543 *
2544 * The close entry point is called when an interface is de-activated
2545 * by the OS. The hardware is still under the driver's control, but
2546 * needs to be disabled. A global MAC reset is issued to stop the
2547 * hardware, and all transmit and receive resources are freed.
2548 **/
2549static int igb_close(struct net_device *netdev)
2550{
2551 struct igb_adapter *adapter = netdev_priv(netdev);
2552
2553 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2554 igb_down(adapter);
2555
2556 igb_free_irq(adapter);
2557
2558 igb_free_all_tx_resources(adapter);
2559 igb_free_all_rx_resources(adapter);
2560
9d5c8243
AK		 2561 return 0;
2562}
2563
2564/**
2565 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2566 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2567 *
2568 * Return 0 on success, negative on failure
2569 **/
80785298 2570int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2571{
59d71989 2572 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2573 int size;
2574
2575 size = sizeof(struct igb_buffer) * tx_ring->count;
89bf67f1 2576 tx_ring->buffer_info = vzalloc(size);
9d5c8243
AK		 2577 if (!tx_ring->buffer_info)
2578 goto err;
9d5c8243
AK		 2579
2580 /* round up to nearest 4K */
85e8d004 2581 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2582 tx_ring->size = ALIGN(tx_ring->size, 4096);
2583
59d71989
AD		 2584 tx_ring->desc = dma_alloc_coherent(dev,
2585 tx_ring->size,
2586 &tx_ring->dma,
2587 GFP_KERNEL);
9d5c8243
AK		 2588
2589 if (!tx_ring->desc)
2590 goto err;
2591
9d5c8243
AK		 2592 tx_ring->next_to_use = 0;
2593 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2594 return 0;
2595
2596err:
2597 vfree(tx_ring->buffer_info);
59d71989 2598 dev_err(dev,
9d5c8243
AK		 2599 "Unable to allocate memory for the transmit descriptor ring\n");
2600 return -ENOMEM;
2601}
2602
2603/**
2604 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2605 * (Descriptors) for all queues
2606 * @adapter: board private structure
2607 *
2608 * Return 0 on success, negative on failure
2609 **/
2610static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2611{
439705e1 2612 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2613 int i, err = 0;
2614
2615 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2616 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2617 if (err) {
439705e1 2618 dev_err(&pdev->dev,
9d5c8243
AK		 2619 "Allocation for Tx Queue %u failed\n", i);
2620 for (i--; i >= 0; i--)
3025a446 2621 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2622 break;
2623 }
2624 }
2625
a99955fc 2626 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2627 int r_idx = i % adapter->num_tx_queues;
3025a446 2628 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2629 }
9d5c8243
AK		 2630 return err;
2631}
2632
2633/**
85b430b4
AD		 2634 * igb_setup_tctl - configure the transmit control registers
2635 * @adapter: Board private structure
9d5c8243 2636 **/
d7ee5b3a 2637void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2638{
9d5c8243
AK		 2639 struct e1000_hw *hw = &adapter->hw;
2640 u32 tctl;
9d5c8243 2641
85b430b4
AD		 2642 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2643 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2644
2645 /* Program the Transmit Control Register */
9d5c8243
AK		 2646 tctl = rd32(E1000_TCTL);
2647 tctl &= ~E1000_TCTL_CT;
2648 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2649 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2650
2651 igb_config_collision_dist(hw);
2652
9d5c8243
AK		 2653 /* Enable transmits */
2654 tctl |= E1000_TCTL_EN;
2655
2656 wr32(E1000_TCTL, tctl);
2657}
2658
85b430b4
AD		 2659/**
2660 * igb_configure_tx_ring - Configure transmit ring after Reset
2661 * @adapter: board private structure
2662 * @ring: tx ring to configure
2663 *
2664 * Configure a transmit ring after a reset.
2665 **/
d7ee5b3a
AD		 2666void igb_configure_tx_ring(struct igb_adapter *adapter,
2667 struct igb_ring *ring)
85b430b4
AD		 2668{
2669 struct e1000_hw *hw = &adapter->hw;
a74420e0 2670 u32 txdctl = 0;
85b430b4
AD		 2671 u64 tdba = ring->dma;
2672 int reg_idx = ring->reg_idx;
2673
2674 /* disable the queue */
a74420e0 2675 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2676 wrfl();
2677 mdelay(10);
2678
2679 wr32(E1000_TDLEN(reg_idx),
2680 ring->count * sizeof(union e1000_adv_tx_desc));
2681 wr32(E1000_TDBAL(reg_idx),
2682 tdba & 0x00000000ffffffffULL);
2683 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2684
fce99e34
AD		 2685 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2686 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2687 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2688 writel(0, ring->tail);
85b430b4
AD		 2689
2690 txdctl |= IGB_TX_PTHRESH;
2691 txdctl |= IGB_TX_HTHRESH << 8;
2692 txdctl |= IGB_TX_WTHRESH << 16;
2693
2694 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2695 wr32(E1000_TXDCTL(reg_idx), txdctl);
2696}
2697
2698/**
2699 * igb_configure_tx - Configure transmit Unit after Reset
2700 * @adapter: board private structure
2701 *
2702 * Configure the Tx unit of the MAC after a reset.
2703 **/
2704static void igb_configure_tx(struct igb_adapter *adapter)
2705{
2706 int i;
2707
2708 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2709 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2710}
2711
9d5c8243
AK		 2712/**
2713 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2714 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2715 *
2716 * Returns 0 on success, negative on failure
2717 **/
80785298 2718int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2719{
59d71989 2720 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2721 int size, desc_len;
2722
2723 size = sizeof(struct igb_buffer) * rx_ring->count;
89bf67f1 2724 rx_ring->buffer_info = vzalloc(size);
9d5c8243
AK		 2725 if (!rx_ring->buffer_info)
2726 goto err;
9d5c8243
AK		 2727
2728 desc_len = sizeof(union e1000_adv_rx_desc);
2729
2730 /* Round up to nearest 4K */
2731 rx_ring->size = rx_ring->count * desc_len;
2732 rx_ring->size = ALIGN(rx_ring->size, 4096);
2733
59d71989
AD		 2734 rx_ring->desc = dma_alloc_coherent(dev,
2735 rx_ring->size,
2736 &rx_ring->dma,
2737 GFP_KERNEL);
9d5c8243
AK		 2738
2739 if (!rx_ring->desc)
2740 goto err;
2741
2742 rx_ring->next_to_clean = 0;
2743 rx_ring->next_to_use = 0;
9d5c8243 2744
9d5c8243
AK		 2745 return 0;
2746
2747err:
2748 vfree(rx_ring->buffer_info);
439705e1 2749 rx_ring->buffer_info = NULL;
59d71989
AD		 2750 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2751 " ring\n");
9d5c8243
AK		 2752 return -ENOMEM;
2753}
2754
2755/**
2756 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2757 * (Descriptors) for all queues
2758 * @adapter: board private structure
2759 *
2760 * Return 0 on success, negative on failure
2761 **/
2762static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2763{
439705e1 2764 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2765 int i, err = 0;
2766
2767 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2768 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2769 if (err) {
439705e1 2770 dev_err(&pdev->dev,
9d5c8243
AK		 2771 "Allocation for Rx Queue %u failed\n", i);
2772 for (i--; i >= 0; i--)
3025a446 2773 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2774 break;
2775 }
2776 }
2777
2778 return err;
2779}
2780
06cf2666
AD		 2781/**
2782 * igb_setup_mrqc - configure the multiple receive queue control registers
2783 * @adapter: Board private structure
2784 **/
2785static void igb_setup_mrqc(struct igb_adapter *adapter)
2786{
2787 struct e1000_hw *hw = &adapter->hw;
2788 u32 mrqc, rxcsum;
2789 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2790 union e1000_reta {
2791 u32 dword;
2792 u8 bytes[4];
2793 } reta;
2794 static const u8 rsshash[40] = {
2795 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2796 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2797 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2798 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2799
2800 /* Fill out hash function seeds */
2801 for (j = 0; j < 10; j++) {
2802 u32 rsskey = rsshash[(j * 4)];
2803 rsskey |= rsshash[(j * 4) + 1] << 8;
2804 rsskey |= rsshash[(j * 4) + 2] << 16;
2805 rsskey |= rsshash[(j * 4) + 3] << 24;
2806 array_wr32(E1000_RSSRK(0), j, rsskey);
2807 }
2808
a99955fc 2809 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2810
2811 if (adapter->vfs_allocated_count) {
2812 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2813 switch (hw->mac.type) {
d2ba2ed8 2814 case e1000_i350:
55cac248
AD		 2815 case e1000_82580:
2816 num_rx_queues = 1;
2817 shift = 0;
2818 break;
06cf2666
AD		 2819 case e1000_82576:
2820 shift = 3;
2821 num_rx_queues = 2;
2822 break;
2823 case e1000_82575:
2824 shift = 2;
2825 shift2 = 6;
2826 default:
2827 break;
2828 }
2829 } else {
2830 if (hw->mac.type == e1000_82575)
2831 shift = 6;
2832 }
2833
2834 for (j = 0; j < (32 * 4); j++) {
2835 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2836 if (shift2)
2837 reta.bytes[j & 3] |= num_rx_queues << shift2;
2838 if ((j & 3) == 3)
2839 wr32(E1000_RETA(j >> 2), reta.dword);
2840 }
2841
2842 /*
2843 * Disable raw packet checksumming so that RSS hash is placed in
2844 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2845 * offloads as they are enabled by default
2846 */
2847 rxcsum = rd32(E1000_RXCSUM);
2848 rxcsum |= E1000_RXCSUM_PCSD;
2849
2850 if (adapter->hw.mac.type >= e1000_82576)
2851 /* Enable Receive Checksum Offload for SCTP */
2852 rxcsum |= E1000_RXCSUM_CRCOFL;
2853
2854 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2855 wr32(E1000_RXCSUM, rxcsum);
2856
2857 /* If VMDq is enabled then we set the appropriate mode for that, else
2858 * we default to RSS so that an RSS hash is calculated per packet even
2859 * if we are only using one queue */
2860 if (adapter->vfs_allocated_count) {
2861 if (hw->mac.type > e1000_82575) {
2862 /* Set the default pool for the PF's first queue */
2863 u32 vtctl = rd32(E1000_VT_CTL);
2864 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2865 E1000_VT_CTL_DISABLE_DEF_POOL);
2866 vtctl |= adapter->vfs_allocated_count <<
2867 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2868 wr32(E1000_VT_CTL, vtctl);
2869 }
a99955fc 2870 if (adapter->rss_queues > 1)
06cf2666
AD		 2871 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2872 else
2873 mrqc = E1000_MRQC_ENABLE_VMDQ;
2874 } else {
2875 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2876 }
2877 igb_vmm_control(adapter);
2878
4478a9cd
AD		 2879 /*
2880 * Generate RSS hash based on TCP port numbers and/or
2881 * IPv4/v6 src and dst addresses since UDP cannot be
2882 * hashed reliably due to IP fragmentation
2883 */
2884 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2885 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2886 E1000_MRQC_RSS_FIELD_IPV6 |
2887 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2888 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2889
2890 wr32(E1000_MRQC, mrqc);
2891}
2892
9d5c8243
AK		 2893/**
2894 * igb_setup_rctl - configure the receive control registers
2895 * @adapter: Board private structure
2896 **/
d7ee5b3a 2897void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2898{
2899 struct e1000_hw *hw = &adapter->hw;
2900 u32 rctl;
9d5c8243
AK		 2901
2902 rctl = rd32(E1000_RCTL);
2903
2904 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2905 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2906
69d728ba 2907 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2908 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2909
87cb7e8c
AK		 2910 /*
2911 * enable stripping of CRC. It's unlikely this will break BMC
2912 * redirection as it did with e1000. Newer features require
2913 * that the HW strips the CRC.
73cd78f1 2914 */
87cb7e8c 2915 rctl |= E1000_RCTL_SECRC;
9d5c8243 2916
559e9c49 2917 /* disable store bad packets and clear size bits. */
ec54d7d6 2918 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2919
6ec43fe6
AD		 2920 /* enable LPE to prevent packets larger than max_frame_size */
2921 rctl |= E1000_RCTL_LPE;
9d5c8243 2922
952f72a8
AD		 2923 /* disable queue 0 to prevent tail write w/o re-config */
2924 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2925
e1739522
AD		 2926 /* Attention!!! For SR-IOV PF driver operations you must enable
2927 * queue drop for all VF and PF queues to prevent head of line blocking
2928 * if an un-trusted VF does not provide descriptors to hardware.
2929 */
2930 if (adapter->vfs_allocated_count) {
e1739522
AD		 2931 /* set all queue drop enable bits */
2932 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2933 }
2934
9d5c8243
AK		 2935 wr32(E1000_RCTL, rctl);
2936}
2937
7d5753f0
AD		 2938static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2939 int vfn)
2940{
2941 struct e1000_hw *hw = &adapter->hw;
2942 u32 vmolr;
2943
2944 /* if it isn't the PF check to see if VFs are enabled and
2945 * increase the size to support vlan tags */
2946 if (vfn < adapter->vfs_allocated_count &&
2947 adapter->vf_data[vfn].vlans_enabled)
2948 size += VLAN_TAG_SIZE;
2949
2950 vmolr = rd32(E1000_VMOLR(vfn));
2951 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2952 vmolr |= size | E1000_VMOLR_LPE;
2953 wr32(E1000_VMOLR(vfn), vmolr);
2954
2955 return 0;
2956}
2957
e1739522
AD		 2958/**
2959 * igb_rlpml_set - set maximum receive packet size
2960 * @adapter: board private structure
2961 *
2962 * Configure maximum receivable packet size.
2963 **/
2964static void igb_rlpml_set(struct igb_adapter *adapter)
2965{
153285f9 2966 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 2967 struct e1000_hw *hw = &adapter->hw;
2968 u16 pf_id = adapter->vfs_allocated_count;
2969
e1739522
AD		 2970 if (pf_id) {
2971 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 2972 /*
2973 * If we're in VMDQ or SR-IOV mode, then set global RLPML
2974 * to our max jumbo frame size, in case we need to enable
2975 * jumbo frames on one of the rings later.
2976 * This will not pass over-length frames into the default
2977 * queue because it's gated by the VMOLR.RLPML.
2978 */
7d5753f0 2979 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2980 }
2981
2982 wr32(E1000_RLPML, max_frame_size);
2983}
2984
8151d294
WM		 2985static inline void igb_set_vmolr(struct igb_adapter *adapter,
2986 int vfn, bool aupe)
7d5753f0
AD		 2987{
2988 struct e1000_hw *hw = &adapter->hw;
2989 u32 vmolr;
2990
2991 /*
2992 * This register exists only on 82576 and newer so if we are older then
2993 * we should exit and do nothing
2994 */
2995 if (hw->mac.type < e1000_82576)
2996 return;
2997
2998 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2999 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3000 if (aupe)
3001 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3002 else
3003 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3004
3005 /* clear all bits that might not be set */
3006 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3007
a99955fc 3008 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3009 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3010 /*
3011 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3012 * multicast packets
3013 */
3014 if (vfn <= adapter->vfs_allocated_count)
3015 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3016
3017 wr32(E1000_VMOLR(vfn), vmolr);
3018}
3019
85b430b4
AD		 3020/**
3021 * igb_configure_rx_ring - Configure a receive ring after Reset
3022 * @adapter: board private structure
3023 * @ring: receive ring to be configured
3024 *
3025 * Configure the Rx unit of the MAC after a reset.
3026 **/
d7ee5b3a
AD		 3027void igb_configure_rx_ring(struct igb_adapter *adapter,
3028 struct igb_ring *ring)
85b430b4
AD		 3029{
3030 struct e1000_hw *hw = &adapter->hw;
3031 u64 rdba = ring->dma;
3032 int reg_idx = ring->reg_idx;
a74420e0 3033 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3034
3035 /* disable the queue */
a74420e0 3036 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3037
3038 /* Set DMA base address registers */
3039 wr32(E1000_RDBAL(reg_idx),
3040 rdba & 0x00000000ffffffffULL);
3041 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3042 wr32(E1000_RDLEN(reg_idx),
3043 ring->count * sizeof(union e1000_adv_rx_desc));
3044
3045 /* initialize head and tail */
fce99e34
AD		 3046 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
3047 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3048 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3049 writel(0, ring->tail);
85b430b4 3050
952f72a8 3051 /* set descriptor configuration */
4c844851
AD		 3052 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
3053 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 3054 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3055#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3056 srrctl |= IGB_RXBUFFER_16384 >>
3057 E1000_SRRCTL_BSIZEPKT_SHIFT;
3058#else
3059 srrctl |= (PAGE_SIZE / 2) >>
3060 E1000_SRRCTL_BSIZEPKT_SHIFT;
3061#endif
3062 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3063 } else {
4c844851 3064 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 3065 E1000_SRRCTL_BSIZEPKT_SHIFT;
3066 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3067 }
757b77e2
NN		 3068 if (hw->mac.type == e1000_82580)
3069 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3070 /* Only set Drop Enable if we are supporting multiple queues */
3071 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3072 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3073
3074 wr32(E1000_SRRCTL(reg_idx), srrctl);
3075
7d5753f0 3076 /* set filtering for VMDQ pools */
8151d294 3077 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3078
85b430b4
AD		 3079 rxdctl |= IGB_RX_PTHRESH;
3080 rxdctl |= IGB_RX_HTHRESH << 8;
3081 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3082
3083 /* enable receive descriptor fetching */
3084 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3085 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3086}
3087
9d5c8243
AK		 3088/**
3089 * igb_configure_rx - Configure receive Unit after Reset
3090 * @adapter: board private structure
3091 *
3092 * Configure the Rx unit of the MAC after a reset.
3093 **/
3094static void igb_configure_rx(struct igb_adapter *adapter)
3095{
9107584e 3096 int i;
9d5c8243 3097
68d480c4
AD		 3098 /* set UTA to appropriate mode */
3099 igb_set_uta(adapter);
3100
26ad9178
AD		 3101 /* set the correct pool for the PF default MAC address in entry 0 */
3102 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3103 adapter->vfs_allocated_count);
3104
06cf2666
AD		 3105 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3106 * the Base and Length of the Rx Descriptor Ring */
3107 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3108 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3109}
3110
3111/**
3112 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3113 * @tx_ring: Tx descriptor ring for a specific queue
3114 *
3115 * Free all transmit software resources
3116 **/
68fd9910 3117void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3118{
3b644cf6 3119 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 3120
3121 vfree(tx_ring->buffer_info);
3122 tx_ring->buffer_info = NULL;
3123
439705e1
AD		 3124 /* if not set, then don't free */
3125 if (!tx_ring->desc)
3126 return;
3127
59d71989
AD		 3128 dma_free_coherent(tx_ring->dev, tx_ring->size,
3129 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3130
3131 tx_ring->desc = NULL;
3132}
3133
3134/**
3135 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3136 * @adapter: board private structure
3137 *
3138 * Free all transmit software resources
3139 **/
3140static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3141{
3142 int i;
3143
3144 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3145 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3146}
3147
b1a436c3
AD		 3148void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3149 struct igb_buffer *buffer_info)
9d5c8243 3150{
6366ad33
AD		 3151 if (buffer_info->dma) {
3152 if (buffer_info->mapped_as_page)
59d71989 3153 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3154 buffer_info->dma,
3155 buffer_info->length,
59d71989 3156 DMA_TO_DEVICE);
6366ad33 3157 else
59d71989 3158 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3159 buffer_info->dma,
3160 buffer_info->length,
59d71989 3161 DMA_TO_DEVICE);
6366ad33
AD		 3162 buffer_info->dma = 0;
3163 }
9d5c8243
AK		 3164 if (buffer_info->skb) {
3165 dev_kfree_skb_any(buffer_info->skb);
3166 buffer_info->skb = NULL;
3167 }
3168 buffer_info->time_stamp = 0;
6366ad33
AD		 3169 buffer_info->length = 0;
3170 buffer_info->next_to_watch = 0;
3171 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3172}
3173
3174/**
3175 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3176 * @tx_ring: ring to be cleaned
3177 **/
3b644cf6 3178static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3179{
3180 struct igb_buffer *buffer_info;
3181 unsigned long size;
3182 unsigned int i;
3183
3184 if (!tx_ring->buffer_info)
3185 return;
3186 /* Free all the Tx ring sk_buffs */
3187
3188 for (i = 0; i < tx_ring->count; i++) {
3189 buffer_info = &tx_ring->buffer_info[i];
80785298 3190 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3191 }
3192
3193 size = sizeof(struct igb_buffer) * tx_ring->count;
3194 memset(tx_ring->buffer_info, 0, size);
3195
3196 /* Zero out the descriptor ring */
9d5c8243
AK		 3197 memset(tx_ring->desc, 0, tx_ring->size);
3198
3199 tx_ring->next_to_use = 0;
3200 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3201}
3202
3203/**
3204 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3205 * @adapter: board private structure
3206 **/
3207static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3208{
3209 int i;
3210
3211 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3212 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3213}
3214
3215/**
3216 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3217 * @rx_ring: ring to clean the resources from
3218 *
3219 * Free all receive software resources
3220 **/
68fd9910 3221void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3222{
3b644cf6 3223 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3224
3225 vfree(rx_ring->buffer_info);
3226 rx_ring->buffer_info = NULL;
3227
439705e1
AD		 3228 /* if not set, then don't free */
3229 if (!rx_ring->desc)
3230 return;
3231
59d71989
AD		 3232 dma_free_coherent(rx_ring->dev, rx_ring->size,
3233 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3234
3235 rx_ring->desc = NULL;
3236}
3237
3238/**
3239 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3240 * @adapter: board private structure
3241 *
3242 * Free all receive software resources
3243 **/
3244static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3245{
3246 int i;
3247
3248 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3249 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3250}
3251
3252/**
3253 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3254 * @rx_ring: ring to free buffers from
3255 **/
3b644cf6 3256static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3257{
3258 struct igb_buffer *buffer_info;
9d5c8243
AK		 3259 unsigned long size;
3260 unsigned int i;
3261
3262 if (!rx_ring->buffer_info)
3263 return;
439705e1 3264
9d5c8243
AK		 3265 /* Free all the Rx ring sk_buffs */
3266 for (i = 0; i < rx_ring->count; i++) {
3267 buffer_info = &rx_ring->buffer_info[i];
3268 if (buffer_info->dma) {
59d71989 3269 dma_unmap_single(rx_ring->dev,
80785298 3270 buffer_info->dma,
4c844851 3271 rx_ring->rx_buffer_len,
59d71989 3272 DMA_FROM_DEVICE);
9d5c8243
AK		 3273 buffer_info->dma = 0;
3274 }
3275
3276 if (buffer_info->skb) {
3277 dev_kfree_skb(buffer_info->skb);
3278 buffer_info->skb = NULL;
3279 }
6ec43fe6 3280 if (buffer_info->page_dma) {
59d71989 3281 dma_unmap_page(rx_ring->dev,
80785298 3282 buffer_info->page_dma,
6ec43fe6 3283 PAGE_SIZE / 2,
59d71989 3284 DMA_FROM_DEVICE);
6ec43fe6
AD		 3285 buffer_info->page_dma = 0;
3286 }
9d5c8243 3287 if (buffer_info->page) {
9d5c8243
AK		 3288 put_page(buffer_info->page);
3289 buffer_info->page = NULL;
bf36c1a0 3290 buffer_info->page_offset = 0;
9d5c8243
AK		 3291 }
3292 }
3293
9d5c8243
AK		 3294 size = sizeof(struct igb_buffer) * rx_ring->count;
3295 memset(rx_ring->buffer_info, 0, size);
3296
3297 /* Zero out the descriptor ring */
3298 memset(rx_ring->desc, 0, rx_ring->size);
3299
3300 rx_ring->next_to_clean = 0;
3301 rx_ring->next_to_use = 0;
9d5c8243
AK		 3302}
3303
3304/**
3305 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3306 * @adapter: board private structure
3307 **/
3308static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3309{
3310 int i;
3311
3312 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3313 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3314}
3315
3316/**
3317 * igb_set_mac - Change the Ethernet Address of the NIC
3318 * @netdev: network interface device structure
3319 * @p: pointer to an address structure
3320 *
3321 * Returns 0 on success, negative on failure
3322 **/
3323static int igb_set_mac(struct net_device *netdev, void *p)
3324{
3325 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3326 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3327 struct sockaddr *addr = p;
3328
3329 if (!is_valid_ether_addr(addr->sa_data))
3330 return -EADDRNOTAVAIL;
3331
3332 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3333 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3334
26ad9178
AD		 3335 /* set the correct pool for the new PF MAC address in entry 0 */
3336 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3337 adapter->vfs_allocated_count);
e1739522 3338
9d5c8243
AK		 3339 return 0;
3340}
3341
3342/**
68d480c4 3343 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3344 * @netdev: network interface device structure
3345 *
68d480c4
AD		 3346 * Writes multicast address list to the MTA hash table.
3347 * Returns: -ENOMEM on failure
3348 * 0 on no addresses written
3349 * X on writing X addresses to MTA
9d5c8243 3350 **/
68d480c4 3351static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3352{
3353 struct igb_adapter *adapter = netdev_priv(netdev);
3354 struct e1000_hw *hw = &adapter->hw;
22bedad3 3355 struct netdev_hw_addr *ha;
68d480c4 3356 u8 *mta_list;
9d5c8243
AK		 3357 int i;
3358
4cd24eaf 3359 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3360 /* nothing to program, so clear mc list */
3361 igb_update_mc_addr_list(hw, NULL, 0);
3362 igb_restore_vf_multicasts(adapter);
3363 return 0;
3364 }
9d5c8243 3365
4cd24eaf 3366 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3367 if (!mta_list)
3368 return -ENOMEM;
ff41f8dc 3369
68d480c4 3370 /* The shared function expects a packed array of only addresses. */
48e2f183 3371 i = 0;
22bedad3
JP		 3372 netdev_for_each_mc_addr(ha, netdev)
3373 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3374
68d480c4
AD		 3375 igb_update_mc_addr_list(hw, mta_list, i);
3376 kfree(mta_list);
3377
4cd24eaf 3378 return netdev_mc_count(netdev);
68d480c4
AD		 3379}
3380
3381/**
3382 * igb_write_uc_addr_list - write unicast addresses to RAR table
3383 * @netdev: network interface device structure
3384 *
3385 * Writes unicast address list to the RAR table.
3386 * Returns: -ENOMEM on failure/insufficient address space
3387 * 0 on no addresses written
3388 * X on writing X addresses to the RAR table
3389 **/
3390static int igb_write_uc_addr_list(struct net_device *netdev)
3391{
3392 struct igb_adapter *adapter = netdev_priv(netdev);
3393 struct e1000_hw *hw = &adapter->hw;
3394 unsigned int vfn = adapter->vfs_allocated_count;
3395 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3396 int count = 0;
3397
3398 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3399 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3400 return -ENOMEM;
9d5c8243 3401
32e7bfc4 3402 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3403 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3404
3405 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3406 if (!rar_entries)
3407 break;
26ad9178
AD		 3408 igb_rar_set_qsel(adapter, ha->addr,
3409 rar_entries--,
68d480c4
AD		 3410 vfn);
3411 count++;
ff41f8dc
AD		 3412 }
3413 }
3414 /* write the addresses in reverse order to avoid write combining */
3415 for (; rar_entries > 0 ; rar_entries--) {
3416 wr32(E1000_RAH(rar_entries), 0);
3417 wr32(E1000_RAL(rar_entries), 0);
3418 }
3419 wrfl();
3420
68d480c4
AD		 3421 return count;
3422}
3423
3424/**
3425 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3426 * @netdev: network interface device structure
3427 *
3428 * The set_rx_mode entry point is called whenever the unicast or multicast
3429 * address lists or the network interface flags are updated. This routine is
3430 * responsible for configuring the hardware for proper unicast, multicast,
3431 * promiscuous mode, and all-multi behavior.
3432 **/
3433static void igb_set_rx_mode(struct net_device *netdev)
3434{
3435 struct igb_adapter *adapter = netdev_priv(netdev);
3436 struct e1000_hw *hw = &adapter->hw;
3437 unsigned int vfn = adapter->vfs_allocated_count;
3438 u32 rctl, vmolr = 0;
3439 int count;
3440
3441 /* Check for Promiscuous and All Multicast modes */
3442 rctl = rd32(E1000_RCTL);
3443
3444 /* clear the effected bits */
3445 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3446
3447 if (netdev->flags & IFF_PROMISC) {
3448 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3449 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3450 } else {
3451 if (netdev->flags & IFF_ALLMULTI) {
3452 rctl |= E1000_RCTL_MPE;
3453 vmolr |= E1000_VMOLR_MPME;
3454 } else {
3455 /*
3456 * Write addresses to the MTA, if the attempt fails
25985edc 3457 * then we should just turn on promiscuous mode so
68d480c4
AD		 3458 * that we can at least receive multicast traffic
3459 */
3460 count = igb_write_mc_addr_list(netdev);
3461 if (count < 0) {
3462 rctl |= E1000_RCTL_MPE;
3463 vmolr |= E1000_VMOLR_MPME;
3464 } else if (count) {
3465 vmolr |= E1000_VMOLR_ROMPE;
3466 }
3467 }
3468 /*
3469 * Write addresses to available RAR registers, if there is not
3470 * sufficient space to store all the addresses then enable
25985edc 3471 * unicast promiscuous mode
68d480c4
AD		 3472 */
3473 count = igb_write_uc_addr_list(netdev);
3474 if (count < 0) {
3475 rctl |= E1000_RCTL_UPE;
3476 vmolr |= E1000_VMOLR_ROPE;
3477 }
3478 rctl |= E1000_RCTL_VFE;
28fc06f5 3479 }
68d480c4 3480 wr32(E1000_RCTL, rctl);
28fc06f5 3481
68d480c4
AD		 3482 /*
3483 * In order to support SR-IOV and eventually VMDq it is necessary to set
3484 * the VMOLR to enable the appropriate modes. Without this workaround
3485 * we will have issues with VLAN tag stripping not being done for frames
3486 * that are only arriving because we are the default pool
3487 */
3488 if (hw->mac.type < e1000_82576)
28fc06f5 3489 return;
9d5c8243 3490
68d480c4
AD		 3491 vmolr |= rd32(E1000_VMOLR(vfn)) &
3492 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3493 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3494 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3495}
3496
13800469
GR		 3497static void igb_check_wvbr(struct igb_adapter *adapter)
3498{
3499 struct e1000_hw *hw = &adapter->hw;
3500 u32 wvbr = 0;
3501
3502 switch (hw->mac.type) {
3503 case e1000_82576:
3504 case e1000_i350:
3505 if (!(wvbr = rd32(E1000_WVBR)))
3506 return;
3507 break;
3508 default:
3509 break;
3510 }
3511
3512 adapter->wvbr |= wvbr;
3513}
3514
3515#define IGB_STAGGERED_QUEUE_OFFSET 8
3516
3517static void igb_spoof_check(struct igb_adapter *adapter)
3518{
3519 int j;
3520
3521 if (!adapter->wvbr)
3522 return;
3523
3524 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3525 if (adapter->wvbr & (1 << j) ||
3526 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3527 dev_warn(&adapter->pdev->dev,
3528 "Spoof event(s) detected on VF %d\n", j);
3529 adapter->wvbr &=
3530 ~((1 << j) |
3531 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3532 }
3533 }
3534}
3535
9d5c8243
AK		 3536/* Need to wait a few seconds after link up to get diagnostic information from
3537 * the phy */
3538static void igb_update_phy_info(unsigned long data)
3539{
3540 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3541 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3542}
3543
4d6b725e
AD		 3544/**
3545 * igb_has_link - check shared code for link and determine up/down
3546 * @adapter: pointer to driver private info
3547 **/
3145535a 3548bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3549{
3550 struct e1000_hw *hw = &adapter->hw;
3551 bool link_active = false;
3552 s32 ret_val = 0;
3553
3554 /* get_link_status is set on LSC (link status) interrupt or
3555 * rx sequence error interrupt. get_link_status will stay
3556 * false until the e1000_check_for_link establishes link
3557 * for copper adapters ONLY
3558 */
3559 switch (hw->phy.media_type) {
3560 case e1000_media_type_copper:
3561 if (hw->mac.get_link_status) {
3562 ret_val = hw->mac.ops.check_for_link(hw);
3563 link_active = !hw->mac.get_link_status;
3564 } else {
3565 link_active = true;
3566 }
3567 break;
4d6b725e
AD		 3568 case e1000_media_type_internal_serdes:
3569 ret_val = hw->mac.ops.check_for_link(hw);
3570 link_active = hw->mac.serdes_has_link;
3571 break;
3572 default:
3573 case e1000_media_type_unknown:
3574 break;
3575 }
3576
3577 return link_active;
3578}
3579
563988dc
SA		 3580static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3581{
3582 bool ret = false;
3583 u32 ctrl_ext, thstat;
3584
3585 /* check for thermal sensor event on i350, copper only */
3586 if (hw->mac.type == e1000_i350) {
3587 thstat = rd32(E1000_THSTAT);
3588 ctrl_ext = rd32(E1000_CTRL_EXT);
3589
3590 if ((hw->phy.media_type == e1000_media_type_copper) &&
3591 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3592 ret = !!(thstat & event);
3593 }
3594 }
3595
3596 return ret;
3597}
3598
9d5c8243
AK		 3599/**
3600 * igb_watchdog - Timer Call-back
3601 * @data: pointer to adapter cast into an unsigned long
3602 **/
3603static void igb_watchdog(unsigned long data)
3604{
3605 struct igb_adapter *adapter = (struct igb_adapter *)data;
3606 /* Do the rest outside of interrupt context */
3607 schedule_work(&adapter->watchdog_task);
3608}
3609
3610static void igb_watchdog_task(struct work_struct *work)
3611{
3612 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3613 struct igb_adapter,
3614 watchdog_task);
9d5c8243 3615 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3616 struct net_device *netdev = adapter->netdev;
563988dc 3617 u32 link;
7a6ea550 3618 int i;
9d5c8243 3619
4d6b725e 3620 link = igb_has_link(adapter);
9d5c8243
AK		 3621 if (link) {
3622 if (!netif_carrier_ok(netdev)) {
3623 u32 ctrl;
330a6d6a
AD		 3624 hw->mac.ops.get_speed_and_duplex(hw,
3625 &adapter->link_speed,
3626 &adapter->link_duplex);
9d5c8243
AK		 3627
3628 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3629 /* Links status message must follow this format */
3630 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3631 "Flow Control: %s\n",
559e9c49
AD		 3632 netdev->name,
3633 adapter->link_speed,
3634 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3635 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3636 ((ctrl & E1000_CTRL_TFCE) &&
3637 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3638 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3639 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3640
563988dc
SA		 3641 /* check for thermal sensor event */
3642 if (igb_thermal_sensor_event(hw, E1000_THSTAT_LINK_THROTTLE)) {
3643 printk(KERN_INFO "igb: %s The network adapter "
3644 "link speed was downshifted "
3645 "because it overheated.\n",
3646 netdev->name);
7ef5ed1c 3647 }
563988dc 3648
d07f3e37 3649 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3650 adapter->tx_timeout_factor = 1;
3651 switch (adapter->link_speed) {
3652 case SPEED_10:
9d5c8243
AK		 3653 adapter->tx_timeout_factor = 14;
3654 break;
3655 case SPEED_100:
9d5c8243
AK		 3656 /* maybe add some timeout factor ? */
3657 break;
3658 }
3659
3660 netif_carrier_on(netdev);
9d5c8243 3661
4ae196df 3662 igb_ping_all_vfs(adapter);
17dc566c 3663 igb_check_vf_rate_limit(adapter);
4ae196df 3664
4b1a9877 3665 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3666 if (!test_bit(__IGB_DOWN, &adapter->state))
3667 mod_timer(&adapter->phy_info_timer,
3668 round_jiffies(jiffies + 2 * HZ));
3669 }
3670 } else {
3671 if (netif_carrier_ok(netdev)) {
3672 adapter->link_speed = 0;
3673 adapter->link_duplex = 0;
563988dc
SA		 3674
3675 /* check for thermal sensor event */
3676 if (igb_thermal_sensor_event(hw, E1000_THSTAT_PWR_DOWN)) {
3677 printk(KERN_ERR "igb: %s The network adapter "
3678 "was stopped because it "
3679 "overheated.\n",
7ef5ed1c 3680 netdev->name);
7ef5ed1c 3681 }
563988dc 3682
527d47c1
AD		 3683 /* Links status message must follow this format */
3684 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3685 netdev->name);
9d5c8243 3686 netif_carrier_off(netdev);
4b1a9877 3687
4ae196df
AD		 3688 igb_ping_all_vfs(adapter);
3689
4b1a9877 3690 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3691 if (!test_bit(__IGB_DOWN, &adapter->state))
3692 mod_timer(&adapter->phy_info_timer,
3693 round_jiffies(jiffies + 2 * HZ));
3694 }
3695 }
3696
12dcd86b
ED		 3697 spin_lock(&adapter->stats64_lock);
3698 igb_update_stats(adapter, &adapter->stats64);
3699 spin_unlock(&adapter->stats64_lock);
9d5c8243 3700
dbabb065 3701 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3702 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3703 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3704 /* We've lost link, so the controller stops DMA,
3705 * but we've got queued Tx work that's never going
3706 * to get done, so reset controller to flush Tx.
3707 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3708 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3709 adapter->tx_timeout_count++;
3710 schedule_work(&adapter->reset_task);
3711 /* return immediately since reset is imminent */
3712 return;
3713 }
9d5c8243 3714 }
9d5c8243 3715
dbabb065
AD		 3716 /* Force detection of hung controller every watchdog period */
3717 tx_ring->detect_tx_hung = true;
3718 }
f7ba205e 3719
9d5c8243 3720 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3721 if (adapter->msix_entries) {
047e0030
AD		 3722 u32 eics = 0;
3723 for (i = 0; i < adapter->num_q_vectors; i++) {
3724 struct igb_q_vector *q_vector = adapter->q_vector[i];
3725 eics |= q_vector->eims_value;
3726 }
7a6ea550
AD		 3727 wr32(E1000_EICS, eics);
3728 } else {
3729 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3730 }
9d5c8243 3731
13800469
GR		 3732 igb_spoof_check(adapter);
3733
9d5c8243
AK		 3734 /* Reset the timer */
3735 if (!test_bit(__IGB_DOWN, &adapter->state))
3736 mod_timer(&adapter->watchdog_timer,
3737 round_jiffies(jiffies + 2 * HZ));
3738}
3739
3740enum latency_range {
3741 lowest_latency = 0,
3742 low_latency = 1,
3743 bulk_latency = 2,
3744 latency_invalid = 255
3745};
3746
6eb5a7f1
AD		 3747/**
3748 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3749 *
3750 * Stores a new ITR value based on strictly on packet size. This
3751 * algorithm is less sophisticated than that used in igb_update_itr,
3752 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3753 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3754 * were determined based on theoretical maximum wire speed and testing
3755 * data, in order to minimize response time while increasing bulk
3756 * throughput.
3757 * This functionality is controlled by the InterruptThrottleRate module
3758 * parameter (see igb_param.c)
3759 * NOTE: This function is called only when operating in a multiqueue
3760 * receive environment.
047e0030 3761 * @q_vector: pointer to q_vector
6eb5a7f1 3762 **/
047e0030 3763static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3764{
047e0030 3765 int new_val = q_vector->itr_val;
6eb5a7f1 3766 int avg_wire_size = 0;
047e0030 3767 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b
ED		 3768 struct igb_ring *ring;
3769 unsigned int packets;
9d5c8243 3770
6eb5a7f1
AD		 3771 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3772 * ints/sec - ITR timer value of 120 ticks.
3773 */
3774 if (adapter->link_speed != SPEED_1000) {
047e0030 3775 new_val = 976;
6eb5a7f1 3776 goto set_itr_val;
9d5c8243 3777 }
047e0030 3778
12dcd86b
ED		 3779 ring = q_vector->rx_ring;
3780 if (ring) {
3781 packets = ACCESS_ONCE(ring->total_packets);
3782
3783 if (packets)
3784 avg_wire_size = ring->total_bytes / packets;
047e0030
AD		 3785 }
3786
12dcd86b
ED		 3787 ring = q_vector->tx_ring;
3788 if (ring) {
3789 packets = ACCESS_ONCE(ring->total_packets);
3790
3791 if (packets)
3792 avg_wire_size = max_t(u32, avg_wire_size,
3793 ring->total_bytes / packets);
047e0030
AD		 3794 }
3795
3796 /* if avg_wire_size isn't set no work was done */
3797 if (!avg_wire_size)
3798 goto clear_counts;
9d5c8243 3799
6eb5a7f1
AD		 3800 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3801 avg_wire_size += 24;
3802
3803 /* Don't starve jumbo frames */
3804 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3805
6eb5a7f1
AD		 3806 /* Give a little boost to mid-size frames */
3807 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3808 new_val = avg_wire_size / 3;
3809 else
3810 new_val = avg_wire_size / 2;
9d5c8243 3811
abe1c363
NN		 3812 /* when in itr mode 3 do not exceed 20K ints/sec */
3813 if (adapter->rx_itr_setting == 3 && new_val < 196)
3814 new_val = 196;
3815
6eb5a7f1 3816set_itr_val:
047e0030
AD		 3817 if (new_val != q_vector->itr_val) {
3818 q_vector->itr_val = new_val;
3819 q_vector->set_itr = 1;
9d5c8243 3820 }
6eb5a7f1 3821clear_counts:
047e0030
AD		 3822 if (q_vector->rx_ring) {
3823 q_vector->rx_ring->total_bytes = 0;
3824 q_vector->rx_ring->total_packets = 0;
3825 }
3826 if (q_vector->tx_ring) {
3827 q_vector->tx_ring->total_bytes = 0;
3828 q_vector->tx_ring->total_packets = 0;
3829 }
9d5c8243
AK		 3830}
3831
3832/**
3833 * igb_update_itr - update the dynamic ITR value based on statistics
3834 * Stores a new ITR value based on packets and byte
3835 * counts during the last interrupt. The advantage of per interrupt
3836 * computation is faster updates and more accurate ITR for the current
3837 * traffic pattern. Constants in this function were computed
3838 * based on theoretical maximum wire speed and thresholds were set based
3839 * on testing data as well as attempting to minimize response time
3840 * while increasing bulk throughput.
3841 * this functionality is controlled by the InterruptThrottleRate module
3842 * parameter (see igb_param.c)
3843 * NOTE: These calculations are only valid when operating in a single-
3844 * queue environment.
3845 * @adapter: pointer to adapter
047e0030 3846 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3847 * @packets: the number of packets during this measurement interval
3848 * @bytes: the number of bytes during this measurement interval
3849 **/
3850static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3851 int packets, int bytes)
3852{
3853 unsigned int retval = itr_setting;
3854
3855 if (packets == 0)
3856 goto update_itr_done;
3857
3858 switch (itr_setting) {
3859 case lowest_latency:
3860 /* handle TSO and jumbo frames */
3861 if (bytes/packets > 8000)
3862 retval = bulk_latency;
3863 else if ((packets < 5) && (bytes > 512))
3864 retval = low_latency;
3865 break;
3866 case low_latency: /* 50 usec aka 20000 ints/s */
3867 if (bytes > 10000) {
3868 /* this if handles the TSO accounting */
3869 if (bytes/packets > 8000) {
3870 retval = bulk_latency;
3871 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3872 retval = bulk_latency;
3873 } else if ((packets > 35)) {
3874 retval = lowest_latency;
3875 }
3876 } else if (bytes/packets > 2000) {
3877 retval = bulk_latency;
3878 } else if (packets <= 2 && bytes < 512) {
3879 retval = lowest_latency;
3880 }
3881 break;
3882 case bulk_latency: /* 250 usec aka 4000 ints/s */
3883 if (bytes > 25000) {
3884 if (packets > 35)
3885 retval = low_latency;
1e5c3d21 3886 } else if (bytes < 1500) {
9d5c8243
AK		 3887 retval = low_latency;
3888 }
3889 break;
3890 }
3891
3892update_itr_done:
3893 return retval;
3894}
3895
6eb5a7f1 3896static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3897{
047e0030 3898 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3899 u16 current_itr;
047e0030 3900 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3901
3902 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3903 if (adapter->link_speed != SPEED_1000) {
3904 current_itr = 0;
3905 new_itr = 4000;
3906 goto set_itr_now;
3907 }
3908
3909 adapter->rx_itr = igb_update_itr(adapter,
3910 adapter->rx_itr,
3025a446
AD		 3911 q_vector->rx_ring->total_packets,
3912 q_vector->rx_ring->total_bytes);
9d5c8243 3913
047e0030
AD		 3914 adapter->tx_itr = igb_update_itr(adapter,
3915 adapter->tx_itr,
3025a446
AD		 3916 q_vector->tx_ring->total_packets,
3917 q_vector->tx_ring->total_bytes);
047e0030 3918 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3919
6eb5a7f1 3920 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3921 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3922 current_itr = low_latency;
3923
9d5c8243
AK		 3924 switch (current_itr) {
3925 /* counts and packets in update_itr are dependent on these numbers */
3926 case lowest_latency:
78b1f607 3927 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3928 break;
3929 case low_latency:
78b1f607 3930 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3931 break;
3932 case bulk_latency:
78b1f607 3933 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3934 break;
3935 default:
3936 break;
3937 }
3938
3939set_itr_now:
3025a446
AD		 3940 q_vector->rx_ring->total_bytes = 0;
3941 q_vector->rx_ring->total_packets = 0;
3942 q_vector->tx_ring->total_bytes = 0;
3943 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3944
047e0030 3945 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3946 /* this attempts to bias the interrupt rate towards Bulk
3947 * by adding intermediate steps when interrupt rate is
3948 * increasing */
047e0030
AD		 3949 new_itr = new_itr > q_vector->itr_val ?
3950 max((new_itr * q_vector->itr_val) /
3951 (new_itr + (q_vector->itr_val >> 2)),
3952 new_itr) :
9d5c8243
AK		 3953 new_itr;
3954 /* Don't write the value here; it resets the adapter's
3955 * internal timer, and causes us to delay far longer than
3956 * we should between interrupts. Instead, we write the ITR
3957 * value at the beginning of the next interrupt so the timing
3958 * ends up being correct.
3959 */
047e0030
AD		 3960 q_vector->itr_val = new_itr;
3961 q_vector->set_itr = 1;
9d5c8243 3962 }
9d5c8243
AK		 3963}
3964
9d5c8243
AK		 3965#define IGB_TX_FLAGS_CSUM 0x00000001
3966#define IGB_TX_FLAGS_VLAN 0x00000002
3967#define IGB_TX_FLAGS_TSO 0x00000004
3968#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3969#define IGB_TX_FLAGS_TSTAMP 0x00000010
3970#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3971#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3972
85ad76b2 3973static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3974 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3975{
3976 struct e1000_adv_tx_context_desc *context_desc;
3977 unsigned int i;
3978 int err;
3979 struct igb_buffer *buffer_info;
3980 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3981 u32 mss_l4len_idx;
3982 u8 l4len;
9d5c8243
AK		 3983
3984 if (skb_header_cloned(skb)) {
3985 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3986 if (err)
3987 return err;
3988 }
3989
3990 l4len = tcp_hdrlen(skb);
3991 *hdr_len += l4len;
3992
3993 if (skb->protocol == htons(ETH_P_IP)) {
3994 struct iphdr *iph = ip_hdr(skb);
3995 iph->tot_len = 0;
3996 iph->check = 0;
3997 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3998 iph->daddr, 0,
3999 IPPROTO_TCP,
4000 0);
8e1e8a47 4001 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4002 ipv6_hdr(skb)->payload_len = 0;
4003 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4004 &ipv6_hdr(skb)->daddr,
4005 0, IPPROTO_TCP, 0);
4006 }
4007
4008 i = tx_ring->next_to_use;
4009
4010 buffer_info = &tx_ring->buffer_info[i];
4011 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
4012 /* VLAN MACLEN IPLEN */
4013 if (tx_flags & IGB_TX_FLAGS_VLAN)
4014 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
4015 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4016 *hdr_len += skb_network_offset(skb);
4017 info |= skb_network_header_len(skb);
4018 *hdr_len += skb_network_header_len(skb);
4019 context_desc->vlan_macip_lens = cpu_to_le32(info);
4020
4021 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4022 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4023
4024 if (skb->protocol == htons(ETH_P_IP))
4025 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
4026 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4027
4028 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4029
4030 /* MSS L4LEN IDX */
4031 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
4032 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
4033
73cd78f1 4034 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 4035 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
4036 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4037
4038 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4039 context_desc->seqnum_seed = 0;
4040
4041 buffer_info->time_stamp = jiffies;
0e014cb1 4042 buffer_info->next_to_watch = i;
9d5c8243
AK		 4043 buffer_info->dma = 0;
4044 i++;
4045 if (i == tx_ring->count)
4046 i = 0;
4047
4048 tx_ring->next_to_use = i;
4049
4050 return true;
4051}
4052
85ad76b2
AD		 4053static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
4054 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 4055{
4056 struct e1000_adv_tx_context_desc *context_desc;
59d71989 4057 struct device *dev = tx_ring->dev;
9d5c8243
AK		 4058 struct igb_buffer *buffer_info;
4059 u32 info = 0, tu_cmd = 0;
80785298 4060 unsigned int i;
9d5c8243
AK		 4061
4062 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
4063 (tx_flags & IGB_TX_FLAGS_VLAN)) {
4064 i = tx_ring->next_to_use;
4065 buffer_info = &tx_ring->buffer_info[i];
4066 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
4067
4068 if (tx_flags & IGB_TX_FLAGS_VLAN)
4069 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 4070
9d5c8243
AK		 4071 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4072 if (skb->ip_summed == CHECKSUM_PARTIAL)
4073 info |= skb_network_header_len(skb);
4074
4075 context_desc->vlan_macip_lens = cpu_to_le32(info);
4076
4077 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4078
4079 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 4080 __be16 protocol;
4081
4082 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
4083 const struct vlan_ethhdr *vhdr =
4084 (const struct vlan_ethhdr*)skb->data;
4085
4086 protocol = vhdr->h_vlan_encapsulated_proto;
4087 } else {
4088 protocol = skb->protocol;
4089 }
4090
4091 switch (protocol) {
09640e63 4092 case cpu_to_be16(ETH_P_IP):
9d5c8243 4093 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 4094 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
4095 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 4096 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
4097 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 4098 break;
09640e63 4099 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 4100 /* XXX what about other V6 headers?? */
4101 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
4102 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 4103 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
4104 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 4105 break;
4106 default:
4107 if (unlikely(net_ratelimit()))
59d71989 4108 dev_warn(dev,
44b0cda3
MW		 4109 "partial checksum but proto=%x!\n",
4110 skb->protocol);
4111 break;
4112 }
9d5c8243
AK		 4113 }
4114
4115 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4116 context_desc->seqnum_seed = 0;
85ad76b2 4117 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 4118 context_desc->mss_l4len_idx =
85ad76b2 4119 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 4120
4121 buffer_info->time_stamp = jiffies;
0e014cb1 4122 buffer_info->next_to_watch = i;
9d5c8243
AK		 4123 buffer_info->dma = 0;
4124
4125 i++;
4126 if (i == tx_ring->count)
4127 i = 0;
4128 tx_ring->next_to_use = i;
4129
4130 return true;
4131 }
9d5c8243
AK		 4132 return false;
4133}
4134
4135#define IGB_MAX_TXD_PWR 16
4136#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4137
80785298 4138static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 4139 unsigned int first)
9d5c8243
AK		 4140{
4141 struct igb_buffer *buffer_info;
59d71989 4142 struct device *dev = tx_ring->dev;
2873957d 4143 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 4144 unsigned int count = 0, i;
4145 unsigned int f;
2873957d 4146 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 4147
4148 i = tx_ring->next_to_use;
4149
4150 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 4151 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
4152 buffer_info->length = hlen;
9d5c8243
AK		 4153 /* set time_stamp *before* dma to help avoid a possible race */
4154 buffer_info->time_stamp = jiffies;
0e014cb1 4155 buffer_info->next_to_watch = i;
2873957d 4156 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 4157 DMA_TO_DEVICE);
4158 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 4159 goto dma_error;
9d5c8243
AK		 4160
4161 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 4162 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
4163 unsigned int len = frag->size;
9d5c8243 4164
8581145f 4165 count++;
65689fef
AD		 4166 i++;
4167 if (i == tx_ring->count)
4168 i = 0;
4169
9d5c8243
AK		 4170 buffer_info = &tx_ring->buffer_info[i];
4171 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
4172 buffer_info->length = len;
4173 buffer_info->time_stamp = jiffies;
0e014cb1 4174 buffer_info->next_to_watch = i;
6366ad33 4175 buffer_info->mapped_as_page = true;
877749bf 4176 buffer_info->dma = skb_frag_dma_map(dev, frag, 0, len,
59d71989
AD		 4177 DMA_TO_DEVICE);
4178 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 4179 goto dma_error;
4180
9d5c8243
AK		 4181 }
4182
9d5c8243 4183 tx_ring->buffer_info[i].skb = skb;
2244d07b 4184 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
2873957d
NN		 4185 /* multiply data chunks by size of headers */
4186 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
4187 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 4188 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 4189
cdfd01fc 4190 return ++count;
6366ad33
AD		 4191
4192dma_error:
59d71989 4193 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 4194
4195 /* clear timestamp and dma mappings for failed buffer_info mapping */
4196 buffer_info->dma = 0;
4197 buffer_info->time_stamp = 0;
4198 buffer_info->length = 0;
4199 buffer_info->next_to_watch = 0;
4200 buffer_info->mapped_as_page = false;
6366ad33
AD		 4201
4202 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 4203 while (count--) {
4204 if (i == 0)
4205 i = tx_ring->count;
6366ad33 4206 i--;
6366ad33
AD		 4207 buffer_info = &tx_ring->buffer_info[i];
4208 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4209 }
4210
4211 return 0;
9d5c8243
AK		 4212}
4213
85ad76b2 4214static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 4215 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 4216 u8 hdr_len)
4217{
cdfd01fc 4218 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 4219 struct igb_buffer *buffer_info;
4220 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 4221 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 4222
4223 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4224 E1000_ADVTXD_DCMD_DEXT);
4225
4226 if (tx_flags & IGB_TX_FLAGS_VLAN)
4227 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4228
33af6bcc
PO		 4229 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4230 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4231
9d5c8243
AK		 4232 if (tx_flags & IGB_TX_FLAGS_TSO) {
4233 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4234
4235 /* insert tcp checksum */
4236 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4237
4238 /* insert ip checksum */
4239 if (tx_flags & IGB_TX_FLAGS_IPV4)
4240 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4241
4242 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4243 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4244 }
4245
85ad76b2
AD		 4246 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4247 (tx_flags & (IGB_TX_FLAGS_CSUM |
4248 IGB_TX_FLAGS_TSO |
7dfc16fa 4249 IGB_TX_FLAGS_VLAN)))
85ad76b2 4250 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4251
4252 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4253
cdfd01fc 4254 do {
9d5c8243
AK		 4255 buffer_info = &tx_ring->buffer_info[i];
4256 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4257 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4258 tx_desc->read.cmd_type_len =
4259 cpu_to_le32(cmd_type_len | buffer_info->length);
4260 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4261 count--;
9d5c8243
AK		 4262 i++;
4263 if (i == tx_ring->count)
4264 i = 0;
cdfd01fc 4265 } while (count > 0);
9d5c8243 4266
85ad76b2 4267 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4268 /* Force memory writes to complete before letting h/w
4269 * know there are new descriptors to fetch. (Only
4270 * applicable for weak-ordered memory model archs,
4271 * such as IA-64). */
4272 wmb();
4273
4274 tx_ring->next_to_use = i;
fce99e34 4275 writel(i, tx_ring->tail);
9d5c8243
AK		 4276 /* we need this if more than one processor can write to our tail
4277 * at a time, it syncronizes IO on IA64/Altix systems */
4278 mmiowb();
4279}
4280
e694e964 4281static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4282{
e694e964
AD		 4283 struct net_device *netdev = tx_ring->netdev;
4284
661086df 4285 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4286
9d5c8243
AK		 4287 /* Herbert's original patch had:
4288 * smp_mb__after_netif_stop_queue();
4289 * but since that doesn't exist yet, just open code it. */
4290 smp_mb();
4291
4292 /* We need to check again in a case another CPU has just
4293 * made room available. */
c493ea45 4294 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4295 return -EBUSY;
4296
4297 /* A reprieve! */
661086df 4298 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4299
4300 u64_stats_update_begin(&tx_ring->tx_syncp2);
4301 tx_ring->tx_stats.restart_queue2++;
4302 u64_stats_update_end(&tx_ring->tx_syncp2);
4303
9d5c8243
AK		 4304 return 0;
4305}
4306
717ba089 4307static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4308{
c493ea45 4309 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4310 return 0;
e694e964 4311 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4312}
4313
b1a436c3
AD		 4314netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4315 struct igb_ring *tx_ring)
9d5c8243 4316{
cdfd01fc 4317 int tso = 0, count;
91d4ee33
NN		 4318 u32 tx_flags = 0;
4319 u16 first;
4320 u8 hdr_len = 0;
9d5c8243 4321
9d5c8243
AK		 4322 /* need: 1 descriptor per page,
4323 * + 2 desc gap to keep tail from touching head,
4324 * + 1 desc for skb->data,
4325 * + 1 desc for context descriptor,
4326 * otherwise try next time */
e694e964 4327 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4328 /* this is a hard error */
9d5c8243
AK		 4329 return NETDEV_TX_BUSY;
4330 }
33af6bcc 4331
2244d07b
OH		 4332 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4333 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4334 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4335 }
9d5c8243 4336
eab6d18d 4337 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4338 tx_flags |= IGB_TX_FLAGS_VLAN;
4339 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4340 }
4341
661086df
PWJ		 4342 if (skb->protocol == htons(ETH_P_IP))
4343 tx_flags |= IGB_TX_FLAGS_IPV4;
4344
0e014cb1 4345 first = tx_ring->next_to_use;
85ad76b2
AD		 4346 if (skb_is_gso(skb)) {
4347 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4348
85ad76b2
AD		 4349 if (tso < 0) {
4350 dev_kfree_skb_any(skb);
4351 return NETDEV_TX_OK;
4352 }
9d5c8243
AK		 4353 }
4354
4355 if (tso)
4356 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4357 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4358 (skb->ip_summed == CHECKSUM_PARTIAL))
4359 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4360
65689fef 4361 /*
cdfd01fc 4362 * count reflects descriptors mapped, if 0 or less then mapping error
25985edc 4363 * has occurred and we need to rewind the descriptor queue
65689fef 4364 */
80785298 4365 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4366 if (!count) {
65689fef
AD		 4367 dev_kfree_skb_any(skb);
4368 tx_ring->buffer_info[first].time_stamp = 0;
4369 tx_ring->next_to_use = first;
85ad76b2 4370 return NETDEV_TX_OK;
65689fef 4371 }
9d5c8243 4372
85ad76b2
AD		 4373 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4374
4375 /* Make sure there is space in the ring for the next send. */
e694e964 4376 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4377
9d5c8243
AK		 4378 return NETDEV_TX_OK;
4379}
4380
3b29a56d
SH		 4381static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4382 struct net_device *netdev)
9d5c8243
AK		 4383{
4384 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4385 struct igb_ring *tx_ring;
661086df 4386 int r_idx = 0;
b1a436c3
AD		 4387
4388 if (test_bit(__IGB_DOWN, &adapter->state)) {
4389 dev_kfree_skb_any(skb);
4390 return NETDEV_TX_OK;
4391 }
4392
4393 if (skb->len <= 0) {
4394 dev_kfree_skb_any(skb);
4395 return NETDEV_TX_OK;
4396 }
4397
1bfaf07b 4398 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4399 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4400
4401 /* This goes back to the question of how to logically map a tx queue
4402 * to a flow. Right now, performance is impacted slightly negatively
4403 * if using multiple tx queues. If the stack breaks away from a
4404 * single qdisc implementation, we can look at this again. */
e694e964 4405 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4406}
4407
4408/**
4409 * igb_tx_timeout - Respond to a Tx Hang
4410 * @netdev: network interface device structure
4411 **/
4412static void igb_tx_timeout(struct net_device *netdev)
4413{
4414 struct igb_adapter *adapter = netdev_priv(netdev);
4415 struct e1000_hw *hw = &adapter->hw;
4416
4417 /* Do the reset outside of interrupt context */
4418 adapter->tx_timeout_count++;
f7ba205e 4419
55cac248
AD		 4420 if (hw->mac.type == e1000_82580)
4421 hw->dev_spec._82575.global_device_reset = true;
4422
9d5c8243 4423 schedule_work(&adapter->reset_task);
265de409
AD		 4424 wr32(E1000_EICS,
4425 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4426}
4427
4428static void igb_reset_task(struct work_struct *work)
4429{
4430 struct igb_adapter *adapter;
4431 adapter = container_of(work, struct igb_adapter, reset_task);
4432
c97ec42a
TI		 4433 igb_dump(adapter);
4434 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4435 igb_reinit_locked(adapter);
4436}
4437
4438/**
12dcd86b 4439 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4440 * @netdev: network interface device structure
12dcd86b 4441 * @stats: rtnl_link_stats64 pointer
9d5c8243 4442 *
9d5c8243 4443 **/
12dcd86b
ED		 4444static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4445 struct rtnl_link_stats64 *stats)
9d5c8243 4446{
12dcd86b
ED		 4447 struct igb_adapter *adapter = netdev_priv(netdev);
4448
4449 spin_lock(&adapter->stats64_lock);
4450 igb_update_stats(adapter, &adapter->stats64);
4451 memcpy(stats, &adapter->stats64, sizeof(*stats));
4452 spin_unlock(&adapter->stats64_lock);
4453
4454 return stats;
9d5c8243
AK		 4455}
4456
4457/**
4458 * igb_change_mtu - Change the Maximum Transfer Unit
4459 * @netdev: network interface device structure
4460 * @new_mtu: new value for maximum frame size
4461 *
4462 * Returns 0 on success, negative on failure
4463 **/
4464static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4465{
4466 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4467 struct pci_dev *pdev = adapter->pdev;
153285f9 4468 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4c844851 4469 u32 rx_buffer_len, i;
9d5c8243 4470
c809d227 4471 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4472 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4473 return -EINVAL;
4474 }
4475
153285f9 4476#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4477 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4478 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4479 return -EINVAL;
4480 }
4481
4482 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4483 msleep(1);
73cd78f1 4484
9d5c8243
AK		 4485 /* igb_down has a dependency on max_frame_size */
4486 adapter->max_frame_size = max_frame;
559e9c49 4487
9d5c8243
AK		 4488 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4489 * means we reserve 2 more, this pushes us to allocate from the next
4490 * larger slab size.
4491 * i.e. RXBUFFER_2048 --> size-4096 slab
4492 */
4493
757b77e2
NN		 4494 if (adapter->hw.mac.type == e1000_82580)
4495 max_frame += IGB_TS_HDR_LEN;
4496
7d95b717 4497 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4498 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4499 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4500 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4501 else
4c844851
AD		 4502 rx_buffer_len = IGB_RXBUFFER_128;
4503
757b77e2
NN		 4504 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4505 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4506 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4507
4508 if ((adapter->hw.mac.type == e1000_82580) &&
4509 (rx_buffer_len == IGB_RXBUFFER_128))
4510 rx_buffer_len += IGB_RXBUFFER_64;
4511
4c844851
AD		 4512 if (netif_running(netdev))
4513 igb_down(adapter);
9d5c8243 4514
090b1795 4515 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4516 netdev->mtu, new_mtu);
4517 netdev->mtu = new_mtu;
4518
4c844851 4519 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4520 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4521
9d5c8243
AK		 4522 if (netif_running(netdev))
4523 igb_up(adapter);
4524 else
4525 igb_reset(adapter);
4526
4527 clear_bit(__IGB_RESETTING, &adapter->state);
4528
4529 return 0;
4530}
4531
4532/**
4533 * igb_update_stats - Update the board statistics counters
4534 * @adapter: board private structure
4535 **/
4536
12dcd86b
ED		 4537void igb_update_stats(struct igb_adapter *adapter,
4538 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4539{
4540 struct e1000_hw *hw = &adapter->hw;
4541 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4542 u32 reg, mpc;
9d5c8243 4543 u16 phy_tmp;
3f9c0164
AD		 4544 int i;
4545 u64 bytes, packets;
12dcd86b
ED		 4546 unsigned int start;
4547 u64 _bytes, _packets;
9d5c8243
AK		 4548
4549#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4550
4551 /*
4552 * Prevent stats update while adapter is being reset, or if the pci
4553 * connection is down.
4554 */
4555 if (adapter->link_speed == 0)
4556 return;
4557 if (pci_channel_offline(pdev))
4558 return;
4559
3f9c0164
AD		 4560 bytes = 0;
4561 packets = 0;
4562 for (i = 0; i < adapter->num_rx_queues; i++) {
4563 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4564 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4565
3025a446 4566 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4567 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4568
4569 do {
4570 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4571 _bytes = ring->rx_stats.bytes;
4572 _packets = ring->rx_stats.packets;
4573 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4574 bytes += _bytes;
4575 packets += _packets;
3f9c0164
AD		 4576 }
4577
128e45eb
AD		 4578 net_stats->rx_bytes = bytes;
4579 net_stats->rx_packets = packets;
3f9c0164
AD		 4580
4581 bytes = 0;
4582 packets = 0;
4583 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4584 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4585 do {
4586 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4587 _bytes = ring->tx_stats.bytes;
4588 _packets = ring->tx_stats.packets;
4589 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4590 bytes += _bytes;
4591 packets += _packets;
3f9c0164 4592 }
128e45eb
AD		 4593 net_stats->tx_bytes = bytes;
4594 net_stats->tx_packets = packets;
3f9c0164
AD		 4595
4596 /* read stats registers */
9d5c8243
AK		 4597 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4598 adapter->stats.gprc += rd32(E1000_GPRC);
4599 adapter->stats.gorc += rd32(E1000_GORCL);
4600 rd32(E1000_GORCH); /* clear GORCL */
4601 adapter->stats.bprc += rd32(E1000_BPRC);
4602 adapter->stats.mprc += rd32(E1000_MPRC);
4603 adapter->stats.roc += rd32(E1000_ROC);
4604
4605 adapter->stats.prc64 += rd32(E1000_PRC64);
4606 adapter->stats.prc127 += rd32(E1000_PRC127);
4607 adapter->stats.prc255 += rd32(E1000_PRC255);
4608 adapter->stats.prc511 += rd32(E1000_PRC511);
4609 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4610 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4611 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4612 adapter->stats.sec += rd32(E1000_SEC);
4613
fa3d9a6d
MW		 4614 mpc = rd32(E1000_MPC);
4615 adapter->stats.mpc += mpc;
4616 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4617 adapter->stats.scc += rd32(E1000_SCC);
4618 adapter->stats.ecol += rd32(E1000_ECOL);
4619 adapter->stats.mcc += rd32(E1000_MCC);
4620 adapter->stats.latecol += rd32(E1000_LATECOL);
4621 adapter->stats.dc += rd32(E1000_DC);
4622 adapter->stats.rlec += rd32(E1000_RLEC);
4623 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4624 adapter->stats.xontxc += rd32(E1000_XONTXC);
4625 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4626 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4627 adapter->stats.fcruc += rd32(E1000_FCRUC);
4628 adapter->stats.gptc += rd32(E1000_GPTC);
4629 adapter->stats.gotc += rd32(E1000_GOTCL);
4630 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4631 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4632 adapter->stats.ruc += rd32(E1000_RUC);
4633 adapter->stats.rfc += rd32(E1000_RFC);
4634 adapter->stats.rjc += rd32(E1000_RJC);
4635 adapter->stats.tor += rd32(E1000_TORH);
4636 adapter->stats.tot += rd32(E1000_TOTH);
4637 adapter->stats.tpr += rd32(E1000_TPR);
4638
4639 adapter->stats.ptc64 += rd32(E1000_PTC64);
4640 adapter->stats.ptc127 += rd32(E1000_PTC127);
4641 adapter->stats.ptc255 += rd32(E1000_PTC255);
4642 adapter->stats.ptc511 += rd32(E1000_PTC511);
4643 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4644 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4645
4646 adapter->stats.mptc += rd32(E1000_MPTC);
4647 adapter->stats.bptc += rd32(E1000_BPTC);
4648
2d0b0f69
NN		 4649 adapter->stats.tpt += rd32(E1000_TPT);
4650 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4651
4652 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4653 /* read internal phy specific stats */
4654 reg = rd32(E1000_CTRL_EXT);
4655 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4656 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4657 adapter->stats.tncrs += rd32(E1000_TNCRS);
4658 }
4659
9d5c8243
AK		 4660 adapter->stats.tsctc += rd32(E1000_TSCTC);
4661 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4662
4663 adapter->stats.iac += rd32(E1000_IAC);
4664 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4665 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4666 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4667 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4668 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4669 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4670 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4671 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4672
4673 /* Fill out the OS statistics structure */
128e45eb
AD		 4674 net_stats->multicast = adapter->stats.mprc;
4675 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4676
4677 /* Rx Errors */
4678
4679 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4680 * our own version based on RUC and ROC */
128e45eb 4681 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4682 adapter->stats.crcerrs + adapter->stats.algnerrc +
4683 adapter->stats.ruc + adapter->stats.roc +
4684 adapter->stats.cexterr;
128e45eb
AD		 4685 net_stats->rx_length_errors = adapter->stats.ruc +
4686 adapter->stats.roc;
4687 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4688 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4689 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4690
4691 /* Tx Errors */
128e45eb
AD		 4692 net_stats->tx_errors = adapter->stats.ecol +
4693 adapter->stats.latecol;
4694 net_stats->tx_aborted_errors = adapter->stats.ecol;
4695 net_stats->tx_window_errors = adapter->stats.latecol;
4696 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4697
4698 /* Tx Dropped needs to be maintained elsewhere */
4699
4700 /* Phy Stats */
4701 if (hw->phy.media_type == e1000_media_type_copper) {
4702 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4703 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4704 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4705 adapter->phy_stats.idle_errors += phy_tmp;
4706 }
4707 }
4708
4709 /* Management Stats */
4710 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4711 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4712 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4713
4714 /* OS2BMC Stats */
4715 reg = rd32(E1000_MANC);
4716 if (reg & E1000_MANC_EN_BMC2OS) {
4717 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4718 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4719 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4720 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4721 }
9d5c8243
AK		 4722}
4723
9d5c8243
AK		 4724static irqreturn_t igb_msix_other(int irq, void *data)
4725{
047e0030 4726 struct igb_adapter *adapter = data;
9d5c8243 4727 struct e1000_hw *hw = &adapter->hw;
844290e5 4728 u32 icr = rd32(E1000_ICR);
844290e5 4729 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4730
7f081d40
AD		 4731 if (icr & E1000_ICR_DRSTA)
4732 schedule_work(&adapter->reset_task);
4733
047e0030 4734 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4735 /* HW is reporting DMA is out of sync */
4736 adapter->stats.doosync++;
13800469
GR		 4737 /* The DMA Out of Sync is also indication of a spoof event
4738 * in IOV mode. Check the Wrong VM Behavior register to
4739 * see if it is really a spoof event. */
4740 igb_check_wvbr(adapter);
dda0e083 4741 }
eebbbdba 4742
4ae196df
AD		 4743 /* Check for a mailbox event */
4744 if (icr & E1000_ICR_VMMB)
4745 igb_msg_task(adapter);
4746
4747 if (icr & E1000_ICR_LSC) {
4748 hw->mac.get_link_status = 1;
4749 /* guard against interrupt when we're going down */
4750 if (!test_bit(__IGB_DOWN, &adapter->state))
4751 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4752 }
4753
25568a53
AD		 4754 if (adapter->vfs_allocated_count)
4755 wr32(E1000_IMS, E1000_IMS_LSC |
4756 E1000_IMS_VMMB |
4757 E1000_IMS_DOUTSYNC);
4758 else
4759 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4760 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4761
4762 return IRQ_HANDLED;
4763}
4764
047e0030 4765static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4766{
26b39276 4767 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4768 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4769
047e0030
AD		 4770 if (!q_vector->set_itr)
4771 return;
73cd78f1 4772
047e0030
AD		 4773 if (!itr_val)
4774 itr_val = 0x4;
661086df 4775
26b39276
AD		 4776 if (adapter->hw.mac.type == e1000_82575)
4777 itr_val |= itr_val << 16;
661086df 4778 else
047e0030 4779 itr_val |= 0x8000000;
661086df 4780
047e0030
AD		 4781 writel(itr_val, q_vector->itr_register);
4782 q_vector->set_itr = 0;
6eb5a7f1
AD		 4783}
4784
047e0030 4785static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4786{
047e0030 4787 struct igb_q_vector *q_vector = data;
9d5c8243 4788
047e0030
AD		 4789 /* Write the ITR value calculated from the previous interrupt. */
4790 igb_write_itr(q_vector);
9d5c8243 4791
047e0030 4792 napi_schedule(&q_vector->napi);
844290e5 4793
047e0030 4794 return IRQ_HANDLED;
fe4506b6
JC		 4795}
4796
421e02f0 4797#ifdef CONFIG_IGB_DCA
047e0030 4798static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4799{
047e0030 4800 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4801 struct e1000_hw *hw = &adapter->hw;
4802 int cpu = get_cpu();
fe4506b6 4803
047e0030
AD		 4804 if (q_vector->cpu == cpu)
4805 goto out_no_update;
4806
4807 if (q_vector->tx_ring) {
4808 int q = q_vector->tx_ring->reg_idx;
4809 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4810 if (hw->mac.type == e1000_82575) {
4811 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4812 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4813 } else {
047e0030
AD		 4814 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4815 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4816 E1000_DCA_TXCTRL_CPUID_SHIFT;
4817 }
4818 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4819 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4820 }
4821 if (q_vector->rx_ring) {
4822 int q = q_vector->rx_ring->reg_idx;
4823 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4824 if (hw->mac.type == e1000_82575) {
2d064c06 4825 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4826 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4827 } else {
4828 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4829 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4830 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4831 }
fe4506b6
JC		 4832 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4833 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4834 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4835 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4836 }
047e0030
AD		 4837 q_vector->cpu = cpu;
4838out_no_update:
fe4506b6
JC		 4839 put_cpu();
4840}
4841
4842static void igb_setup_dca(struct igb_adapter *adapter)
4843{
7e0e99ef 4844 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4845 int i;
4846
7dfc16fa 4847 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4848 return;
4849
7e0e99ef
AD		 4850 /* Always use CB2 mode, difference is masked in the CB driver. */
4851 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4852
047e0030 4853 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4854 adapter->q_vector[i]->cpu = -1;
4855 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4856 }
4857}
4858
4859static int __igb_notify_dca(struct device *dev, void *data)
4860{
4861 struct net_device *netdev = dev_get_drvdata(dev);
4862 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4863 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4864 struct e1000_hw *hw = &adapter->hw;
4865 unsigned long event = *(unsigned long *)data;
4866
4867 switch (event) {
4868 case DCA_PROVIDER_ADD:
4869 /* if already enabled, don't do it again */
7dfc16fa 4870 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4871 break;
fe4506b6 4872 if (dca_add_requester(dev) == 0) {
bbd98fe4 4873 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4874 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4875 igb_setup_dca(adapter);
4876 break;
4877 }
4878 /* Fall Through since DCA is disabled. */
4879 case DCA_PROVIDER_REMOVE:
7dfc16fa 4880 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4881 /* without this a class_device is left
047e0030 4882 * hanging around in the sysfs model */
fe4506b6 4883 dca_remove_requester(dev);
090b1795 4884 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4885 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4886 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4887 }
4888 break;
4889 }
bbd98fe4 4890
fe4506b6 4891 return 0;
9d5c8243
AK		 4892}
4893
fe4506b6
JC		 4894static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4895 void *p)
4896{
4897 int ret_val;
4898
4899 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4900 __igb_notify_dca);
4901
4902 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4903}
421e02f0 4904#endif /* CONFIG_IGB_DCA */
9d5c8243 4905
4ae196df
AD		 4906static void igb_ping_all_vfs(struct igb_adapter *adapter)
4907{
4908 struct e1000_hw *hw = &adapter->hw;
4909 u32 ping;
4910 int i;
4911
4912 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4913 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4914 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4915 ping |= E1000_VT_MSGTYPE_CTS;
4916 igb_write_mbx(hw, &ping, 1, i);
4917 }
4918}
4919
7d5753f0
AD		 4920static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4921{
4922 struct e1000_hw *hw = &adapter->hw;
4923 u32 vmolr = rd32(E1000_VMOLR(vf));
4924 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4925
d85b9004 4926 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4927 IGB_VF_FLAG_MULTI_PROMISC);
4928 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4929
4930 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4931 vmolr |= E1000_VMOLR_MPME;
d85b9004 4932 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4933 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4934 } else {
4935 /*
4936 * if we have hashes and we are clearing a multicast promisc
4937 * flag we need to write the hashes to the MTA as this step
4938 * was previously skipped
4939 */
4940 if (vf_data->num_vf_mc_hashes > 30) {
4941 vmolr |= E1000_VMOLR_MPME;
4942 } else if (vf_data->num_vf_mc_hashes) {
4943 int j;
4944 vmolr |= E1000_VMOLR_ROMPE;
4945 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4946 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4947 }
4948 }
4949
4950 wr32(E1000_VMOLR(vf), vmolr);
4951
4952 /* there are flags left unprocessed, likely not supported */
4953 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4954 return -EINVAL;
4955
4956 return 0;
4957
4958}
4959
4ae196df
AD		 4960static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4961 u32 *msgbuf, u32 vf)
4962{
4963 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4964 u16 *hash_list = (u16 *)&msgbuf[1];
4965 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4966 int i;
4967
7d5753f0 4968 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4969 * to this VF for later use to restore when the PF multi cast
4970 * list changes
4971 */
4972 vf_data->num_vf_mc_hashes = n;
4973
7d5753f0
AD		 4974 /* only up to 30 hash values supported */
4975 if (n > 30)
4976 n = 30;
4977
4978 /* store the hashes for later use */
4ae196df 4979 for (i = 0; i < n; i++)
a419aef8 4980 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4981
4982 /* Flush and reset the mta with the new values */
ff41f8dc 4983 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4984
4985 return 0;
4986}
4987
4988static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4989{
4990 struct e1000_hw *hw = &adapter->hw;
4991 struct vf_data_storage *vf_data;
4992 int i, j;
4993
4994 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4995 u32 vmolr = rd32(E1000_VMOLR(i));
4996 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4997
4ae196df 4998 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4999
5000 if ((vf_data->num_vf_mc_hashes > 30) ||
5001 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5002 vmolr |= E1000_VMOLR_MPME;
5003 } else if (vf_data->num_vf_mc_hashes) {
5004 vmolr |= E1000_VMOLR_ROMPE;
5005 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5006 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5007 }
5008 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5009 }
5010}
5011
5012static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5013{
5014 struct e1000_hw *hw = &adapter->hw;
5015 u32 pool_mask, reg, vid;
5016 int i;
5017
5018 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5019
5020 /* Find the vlan filter for this id */
5021 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5022 reg = rd32(E1000_VLVF(i));
5023
5024 /* remove the vf from the pool */
5025 reg &= ~pool_mask;
5026
5027 /* if pool is empty then remove entry from vfta */
5028 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5029 (reg & E1000_VLVF_VLANID_ENABLE)) {
5030 reg = 0;
5031 vid = reg & E1000_VLVF_VLANID_MASK;
5032 igb_vfta_set(hw, vid, false);
5033 }
5034
5035 wr32(E1000_VLVF(i), reg);
5036 }
ae641bdc
AD		 5037
5038 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5039}
5040
5041static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5042{
5043 struct e1000_hw *hw = &adapter->hw;
5044 u32 reg, i;
5045
51466239
AD		 5046 /* The vlvf table only exists on 82576 hardware and newer */
5047 if (hw->mac.type < e1000_82576)
5048 return -1;
5049
5050 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5051 if (!adapter->vfs_allocated_count)
5052 return -1;
5053
5054 /* Find the vlan filter for this id */
5055 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5056 reg = rd32(E1000_VLVF(i));
5057 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5058 vid == (reg & E1000_VLVF_VLANID_MASK))
5059 break;
5060 }
5061
5062 if (add) {
5063 if (i == E1000_VLVF_ARRAY_SIZE) {
5064 /* Did not find a matching VLAN ID entry that was
5065 * enabled. Search for a free filter entry, i.e.
5066 * one without the enable bit set
5067 */
5068 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5069 reg = rd32(E1000_VLVF(i));
5070 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5071 break;
5072 }
5073 }
5074 if (i < E1000_VLVF_ARRAY_SIZE) {
5075 /* Found an enabled/available entry */
5076 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5077
5078 /* if !enabled we need to set this up in vfta */
5079 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5080 /* add VID to filter table */
5081 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5082 reg |= E1000_VLVF_VLANID_ENABLE;
5083 }
cad6d05f
AD		 5084 reg &= ~E1000_VLVF_VLANID_MASK;
5085 reg |= vid;
4ae196df 5086 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5087
5088 /* do not modify RLPML for PF devices */
5089 if (vf >= adapter->vfs_allocated_count)
5090 return 0;
5091
5092 if (!adapter->vf_data[vf].vlans_enabled) {
5093 u32 size;
5094 reg = rd32(E1000_VMOLR(vf));
5095 size = reg & E1000_VMOLR_RLPML_MASK;
5096 size += 4;
5097 reg &= ~E1000_VMOLR_RLPML_MASK;
5098 reg |= size;
5099 wr32(E1000_VMOLR(vf), reg);
5100 }
ae641bdc 5101
51466239 5102 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5103 return 0;
5104 }
5105 } else {
5106 if (i < E1000_VLVF_ARRAY_SIZE) {
5107 /* remove vf from the pool */
5108 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5109 /* if pool is empty then remove entry from vfta */
5110 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5111 reg = 0;
5112 igb_vfta_set(hw, vid, false);
5113 }
5114 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5115
5116 /* do not modify RLPML for PF devices */
5117 if (vf >= adapter->vfs_allocated_count)
5118 return 0;
5119
5120 adapter->vf_data[vf].vlans_enabled--;
5121 if (!adapter->vf_data[vf].vlans_enabled) {
5122 u32 size;
5123 reg = rd32(E1000_VMOLR(vf));
5124 size = reg & E1000_VMOLR_RLPML_MASK;
5125 size -= 4;
5126 reg &= ~E1000_VMOLR_RLPML_MASK;
5127 reg |= size;
5128 wr32(E1000_VMOLR(vf), reg);
5129 }
4ae196df
AD		 5130 }
5131 }
8151d294
WM		 5132 return 0;
5133}
5134
5135static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5136{
5137 struct e1000_hw *hw = &adapter->hw;
5138
5139 if (vid)
5140 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5141 else
5142 wr32(E1000_VMVIR(vf), 0);
5143}
5144
5145static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5146 int vf, u16 vlan, u8 qos)
5147{
5148 int err = 0;
5149 struct igb_adapter *adapter = netdev_priv(netdev);
5150
5151 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5152 return -EINVAL;
5153 if (vlan || qos) {
5154 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5155 if (err)
5156 goto out;
5157 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5158 igb_set_vmolr(adapter, vf, !vlan);
5159 adapter->vf_data[vf].pf_vlan = vlan;
5160 adapter->vf_data[vf].pf_qos = qos;
5161 dev_info(&adapter->pdev->dev,
5162 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5163 if (test_bit(__IGB_DOWN, &adapter->state)) {
5164 dev_warn(&adapter->pdev->dev,
5165 "The VF VLAN has been set,"
5166 " but the PF device is not up.\n");
5167 dev_warn(&adapter->pdev->dev,
5168 "Bring the PF device up before"
5169 " attempting to use the VF device.\n");
5170 }
5171 } else {
5172 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5173 false, vf);
5174 igb_set_vmvir(adapter, vlan, vf);
5175 igb_set_vmolr(adapter, vf, true);
5176 adapter->vf_data[vf].pf_vlan = 0;
5177 adapter->vf_data[vf].pf_qos = 0;
5178 }
5179out:
5180 return err;
4ae196df
AD		 5181}
5182
5183static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5184{
5185 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5186 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5187
5188 return igb_vlvf_set(adapter, vid, add, vf);
5189}
5190
f2ca0dbe 5191static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5192{
8fa7e0f7
GR		 5193 /* clear flags - except flag that indicates PF has set the MAC */
5194 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5195 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5196
5197 /* reset offloads to defaults */
8151d294 5198 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5199
5200 /* reset vlans for device */
5201 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5202 if (adapter->vf_data[vf].pf_vlan)
5203 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5204 adapter->vf_data[vf].pf_vlan,
5205 adapter->vf_data[vf].pf_qos);
5206 else
5207 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5208
5209 /* reset multicast table array for vf */
5210 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5211
5212 /* Flush and reset the mta with the new values */
ff41f8dc 5213 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5214}
5215
f2ca0dbe
AD		 5216static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5217{
5218 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5219
5220 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5221 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5222 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5223
5224 /* process remaining reset events */
5225 igb_vf_reset(adapter, vf);
5226}
5227
5228static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5229{
5230 struct e1000_hw *hw = &adapter->hw;
5231 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5232 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5233 u32 reg, msgbuf[3];
5234 u8 *addr = (u8 *)(&msgbuf[1]);
5235
5236 /* process all the same items cleared in a function level reset */
f2ca0dbe 5237 igb_vf_reset(adapter, vf);
4ae196df
AD		 5238
5239 /* set vf mac address */
26ad9178 5240 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5241
5242 /* enable transmit and receive for vf */
5243 reg = rd32(E1000_VFTE);
5244 wr32(E1000_VFTE, reg | (1 << vf));
5245 reg = rd32(E1000_VFRE);
5246 wr32(E1000_VFRE, reg | (1 << vf));
5247
8fa7e0f7 5248 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5249
5250 /* reply to reset with ack and vf mac address */
5251 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5252 memcpy(addr, vf_mac, 6);
5253 igb_write_mbx(hw, msgbuf, 3, vf);
5254}
5255
5256static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5257{
de42edde
GR		 5258 /*
5259 * The VF MAC Address is stored in a packed array of bytes
5260 * starting at the second 32 bit word of the msg array
5261 */
f2ca0dbe
AD		 5262 unsigned char *addr = (char *)&msg[1];
5263 int err = -1;
4ae196df 5264
f2ca0dbe
AD		 5265 if (is_valid_ether_addr(addr))
5266 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5267
f2ca0dbe 5268 return err;
4ae196df
AD		 5269}
5270
5271static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5272{
5273 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5274 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5275 u32 msg = E1000_VT_MSGTYPE_NACK;
5276
5277 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5278 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5279 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5280 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5281 vf_data->last_nack = jiffies;
4ae196df
AD		 5282 }
5283}
5284
f2ca0dbe 5285static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5286{
f2ca0dbe
AD		 5287 struct pci_dev *pdev = adapter->pdev;
5288 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5289 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5290 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5291 s32 retval;
5292
f2ca0dbe 5293 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5294
fef45f4c
AD		 5295 if (retval) {
5296 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5297 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5298 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5299 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5300 return;
5301 goto out;
5302 }
4ae196df
AD		 5303
5304 /* this is a message we already processed, do nothing */
5305 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5306 return;
4ae196df
AD		 5307
5308 /*
5309 * until the vf completes a reset it should not be
5310 * allowed to start any configuration.
5311 */
5312
5313 if (msgbuf[0] == E1000_VF_RESET) {
5314 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5315 return;
4ae196df
AD		 5316 }
5317
f2ca0dbe 5318 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5319 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5320 return;
5321 retval = -1;
5322 goto out;
4ae196df
AD		 5323 }
5324
5325 switch ((msgbuf[0] & 0xFFFF)) {
5326 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5327 retval = -EINVAL;
5328 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5329 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5330 else
5331 dev_warn(&pdev->dev,
5332 "VF %d attempted to override administratively "
5333 "set MAC address\nReload the VF driver to "
5334 "resume operations\n", vf);
4ae196df 5335 break;
7d5753f0
AD		 5336 case E1000_VF_SET_PROMISC:
5337 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5338 break;
4ae196df
AD		 5339 case E1000_VF_SET_MULTICAST:
5340 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5341 break;
5342 case E1000_VF_SET_LPE:
5343 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5344 break;
5345 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5346 retval = -1;
5347 if (vf_data->pf_vlan)
5348 dev_warn(&pdev->dev,
5349 "VF %d attempted to override administratively "
5350 "set VLAN tag\nReload the VF driver to "
5351 "resume operations\n", vf);
8151d294
WM		 5352 else
5353 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5354 break;
5355 default:
090b1795 5356 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5357 retval = -1;
5358 break;
5359 }
5360
fef45f4c
AD		 5361 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5362out:
4ae196df
AD		 5363 /* notify the VF of the results of what it sent us */
5364 if (retval)
5365 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5366 else
5367 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5368
4ae196df 5369 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5370}
4ae196df 5371
f2ca0dbe
AD		 5372static void igb_msg_task(struct igb_adapter *adapter)
5373{
5374 struct e1000_hw *hw = &adapter->hw;
5375 u32 vf;
5376
5377 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5378 /* process any reset requests */
5379 if (!igb_check_for_rst(hw, vf))
5380 igb_vf_reset_event(adapter, vf);
5381
5382 /* process any messages pending */
5383 if (!igb_check_for_msg(hw, vf))
5384 igb_rcv_msg_from_vf(adapter, vf);
5385
5386 /* process any acks */
5387 if (!igb_check_for_ack(hw, vf))
5388 igb_rcv_ack_from_vf(adapter, vf);
5389 }
4ae196df
AD		 5390}
5391
68d480c4
AD		 5392/**
5393 * igb_set_uta - Set unicast filter table address
5394 * @adapter: board private structure
5395 *
5396 * The unicast table address is a register array of 32-bit registers.
5397 * The table is meant to be used in a way similar to how the MTA is used
5398 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5399 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5400 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5401 **/
5402static void igb_set_uta(struct igb_adapter *adapter)
5403{
5404 struct e1000_hw *hw = &adapter->hw;
5405 int i;
5406
5407 /* The UTA table only exists on 82576 hardware and newer */
5408 if (hw->mac.type < e1000_82576)
5409 return;
5410
5411 /* we only need to do this if VMDq is enabled */
5412 if (!adapter->vfs_allocated_count)
5413 return;
5414
5415 for (i = 0; i < hw->mac.uta_reg_count; i++)
5416 array_wr32(E1000_UTA, i, ~0);
5417}
5418
9d5c8243
AK		 5419/**
5420 * igb_intr_msi - Interrupt Handler
5421 * @irq: interrupt number
5422 * @data: pointer to a network interface device structure
5423 **/
5424static irqreturn_t igb_intr_msi(int irq, void *data)
5425{
047e0030
AD		 5426 struct igb_adapter *adapter = data;
5427 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5428 struct e1000_hw *hw = &adapter->hw;
5429 /* read ICR disables interrupts using IAM */
5430 u32 icr = rd32(E1000_ICR);
5431
047e0030 5432 igb_write_itr(q_vector);
9d5c8243 5433
7f081d40
AD		 5434 if (icr & E1000_ICR_DRSTA)
5435 schedule_work(&adapter->reset_task);
5436
047e0030 5437 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5438 /* HW is reporting DMA is out of sync */
5439 adapter->stats.doosync++;
5440 }
5441
9d5c8243
AK		 5442 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5443 hw->mac.get_link_status = 1;
5444 if (!test_bit(__IGB_DOWN, &adapter->state))
5445 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5446 }
5447
047e0030 5448 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5449
5450 return IRQ_HANDLED;
5451}
5452
5453/**
4a3c6433 5454 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5455 * @irq: interrupt number
5456 * @data: pointer to a network interface device structure
5457 **/
5458static irqreturn_t igb_intr(int irq, void *data)
5459{
047e0030
AD		 5460 struct igb_adapter *adapter = data;
5461 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5462 struct e1000_hw *hw = &adapter->hw;
5463 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5464 * need for the IMC write */
5465 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5466 if (!icr)
5467 return IRQ_NONE; /* Not our interrupt */
5468
047e0030 5469 igb_write_itr(q_vector);
9d5c8243
AK		 5470
5471 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5472 * not set, then the adapter didn't send an interrupt */
5473 if (!(icr & E1000_ICR_INT_ASSERTED))
5474 return IRQ_NONE;
5475
7f081d40
AD		 5476 if (icr & E1000_ICR_DRSTA)
5477 schedule_work(&adapter->reset_task);
5478
047e0030 5479 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5480 /* HW is reporting DMA is out of sync */
5481 adapter->stats.doosync++;
5482 }
5483
9d5c8243
AK		 5484 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5485 hw->mac.get_link_status = 1;
5486 /* guard against interrupt when we're going down */
5487 if (!test_bit(__IGB_DOWN, &adapter->state))
5488 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5489 }
5490
047e0030 5491 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5492
5493 return IRQ_HANDLED;
5494}
5495
047e0030 5496static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5497{
047e0030 5498 struct igb_adapter *adapter = q_vector->adapter;
46544258 5499 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5500
4fc82adf
AD		 5501 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5502 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5503 if (!adapter->msix_entries)
6eb5a7f1 5504 igb_set_itr(adapter);
46544258 5505 else
047e0030 5506 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5507 }
5508
46544258
AD		 5509 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5510 if (adapter->msix_entries)
047e0030 5511 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5512 else
5513 igb_irq_enable(adapter);
5514 }
9d5c8243
AK		 5515}
5516
46544258
AD		 5517/**
5518 * igb_poll - NAPI Rx polling callback
5519 * @napi: napi polling structure
5520 * @budget: count of how many packets we should handle
5521 **/
5522static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5523{
047e0030
AD		 5524 struct igb_q_vector *q_vector = container_of(napi,
5525 struct igb_q_vector,
5526 napi);
5527 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5528
421e02f0 5529#ifdef CONFIG_IGB_DCA
047e0030
AD		 5530 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5531 igb_update_dca(q_vector);
fe4506b6 5532#endif
047e0030
AD		 5533 if (q_vector->tx_ring)
5534 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5535
047e0030
AD		 5536 if (q_vector->rx_ring)
5537 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5538
5539 if (!tx_clean_complete)
5540 work_done = budget;
46544258 5541
9d5c8243 5542 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5543 if (work_done < budget) {
288379f0 5544 napi_complete(napi);
047e0030 5545 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5546 }
5547
46544258 5548 return work_done;
9d5c8243 5549}
6d8126f9 5550
33af6bcc 5551/**
c5b9bd5e 5552 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5553 * @adapter: board private structure
c5b9bd5e
AD		 5554 * @shhwtstamps: timestamp structure to update
5555 * @regval: unsigned 64bit system time value.
5556 *
5557 * We need to convert the system time value stored in the RX/TXSTMP registers
5558 * into a hwtstamp which can be used by the upper level timestamping functions
5559 */
5560static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5561 struct skb_shared_hwtstamps *shhwtstamps,
5562 u64 regval)
5563{
5564 u64 ns;
5565
55cac248
AD		 5566 /*
5567 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5568 * 24 to match clock shift we setup earlier.
5569 */
5570 if (adapter->hw.mac.type == e1000_82580)
5571 regval <<= IGB_82580_TSYNC_SHIFT;
5572
c5b9bd5e
AD		 5573 ns = timecounter_cyc2time(&adapter->clock, regval);
5574 timecompare_update(&adapter->compare, ns);
5575 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5576 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5577 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5578}
5579
5580/**
5581 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5582 * @q_vector: pointer to q_vector containing needed info
2873957d 5583 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5584 *
5585 * If we were asked to do hardware stamping and such a time stamp is
5586 * available, then it must have been for this skb here because we only
5587 * allow only one such packet into the queue.
5588 */
2873957d 5589static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5590{
c5b9bd5e 5591 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5592 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5593 struct skb_shared_hwtstamps shhwtstamps;
5594 u64 regval;
33af6bcc 5595
c5b9bd5e 5596 /* if skb does not support hw timestamp or TX stamp not valid exit */
2244d07b 5597 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
c5b9bd5e
AD		 5598 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5599 return;
5600
5601 regval = rd32(E1000_TXSTMPL);
5602 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5603
5604 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5605 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5606}
5607
9d5c8243
AK		 5608/**
5609 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5610 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5611 * returns true if ring is completely cleaned
5612 **/
047e0030 5613static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5614{
047e0030
AD		 5615 struct igb_adapter *adapter = q_vector->adapter;
5616 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5617 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5618 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5619 struct igb_buffer *buffer_info;
0e014cb1 5620 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5621 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5622 unsigned int i, eop, count = 0;
5623 bool cleaned = false;
9d5c8243 5624
9d5c8243 5625 i = tx_ring->next_to_clean;
0e014cb1
AD		 5626 eop = tx_ring->buffer_info[i].next_to_watch;
5627 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5628
5629 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5630 (count < tx_ring->count)) {
2d0bb1c1 5631 rmb(); /* read buffer_info after eop_desc status */
0e014cb1
AD		 5632 for (cleaned = false; !cleaned; count++) {
5633 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5634 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5635 cleaned = (i == eop);
9d5c8243 5636
2873957d
NN		 5637 if (buffer_info->skb) {
5638 total_bytes += buffer_info->bytecount;
9d5c8243 5639 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5640 total_packets += buffer_info->gso_segs;
5641 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5642 }
5643
80785298 5644 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5645 tx_desc->wb.status = 0;
9d5c8243
AK		 5646
5647 i++;
5648 if (i == tx_ring->count)
5649 i = 0;
9d5c8243 5650 }
0e014cb1
AD		 5651 eop = tx_ring->buffer_info[i].next_to_watch;
5652 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5653 }
5654
9d5c8243
AK		 5655 tx_ring->next_to_clean = i;
5656
fc7d345d 5657 if (unlikely(count &&
9d5c8243 5658 netif_carrier_ok(netdev) &&
c493ea45 5659 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5660 /* Make sure that anybody stopping the queue after this
5661 * sees the new next_to_clean.
5662 */
5663 smp_mb();
661086df
PWJ		 5664 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5665 !(test_bit(__IGB_DOWN, &adapter->state))) {
5666 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 5667
5668 u64_stats_update_begin(&tx_ring->tx_syncp);
04a5fcaa 5669 tx_ring->tx_stats.restart_queue++;
12dcd86b 5670 u64_stats_update_end(&tx_ring->tx_syncp);
661086df 5671 }
9d5c8243
AK		 5672 }
5673
5674 if (tx_ring->detect_tx_hung) {
5675 /* Detect a transmit hang in hardware, this serializes the
5676 * check with the clearing of time_stamp and movement of i */
5677 tx_ring->detect_tx_hung = false;
5678 if (tx_ring->buffer_info[i].time_stamp &&
5679 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5680 (adapter->tx_timeout_factor * HZ)) &&
5681 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5682
9d5c8243 5683 /* detected Tx unit hang */
59d71989 5684 dev_err(tx_ring->dev,
9d5c8243 5685 "Detected Tx Unit Hang\n"
2d064c06 5686 " Tx Queue <%d>\n"
9d5c8243
AK		 5687 " TDH <%x>\n"
5688 " TDT <%x>\n"
5689 " next_to_use <%x>\n"
5690 " next_to_clean <%x>\n"
9d5c8243
AK		 5691 "buffer_info[next_to_clean]\n"
5692 " time_stamp <%lx>\n"
0e014cb1 5693 " next_to_watch <%x>\n"
9d5c8243
AK		 5694 " jiffies <%lx>\n"
5695 " desc.status <%x>\n",
2d064c06 5696 tx_ring->queue_index,
fce99e34
AD		 5697 readl(tx_ring->head),
5698 readl(tx_ring->tail),
9d5c8243
AK		 5699 tx_ring->next_to_use,
5700 tx_ring->next_to_clean,
f7ba205e 5701 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5702 eop,
9d5c8243 5703 jiffies,
0e014cb1 5704 eop_desc->wb.status);
661086df 5705 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5706 }
5707 }
5708 tx_ring->total_bytes += total_bytes;
5709 tx_ring->total_packets += total_packets;
12dcd86b 5710 u64_stats_update_begin(&tx_ring->tx_syncp);
e21ed353
AD		 5711 tx_ring->tx_stats.bytes += total_bytes;
5712 tx_ring->tx_stats.packets += total_packets;
12dcd86b 5713 u64_stats_update_end(&tx_ring->tx_syncp);
807540ba 5714 return count < tx_ring->count;
9d5c8243
AK		 5715}
5716
04a5fcaa 5717static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5718 u32 status_err, struct sk_buff *skb)
5719{
bc8acf2c 5720 skb_checksum_none_assert(skb);
9d5c8243
AK		 5721
5722 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5723 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5724 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5725 return;
85ad76b2 5726
9d5c8243
AK		 5727 /* TCP/UDP checksum error bit is set */
5728 if (status_err &
5729 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5730 /*
5731 * work around errata with sctp packets where the TCPE aka
5732 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5733 * packets, (aka let the stack check the crc32c)
5734 */
85ad76b2 5735 if ((skb->len == 60) &&
12dcd86b
ED		 5736 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5737 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5738 ring->rx_stats.csum_err++;
12dcd86b
ED		 5739 u64_stats_update_end(&ring->rx_syncp);
5740 }
9d5c8243 5741 /* let the stack verify checksum errors */
9d5c8243
AK		 5742 return;
5743 }
5744 /* It must be a TCP or UDP packet with a valid checksum */
5745 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5746 skb->ip_summed = CHECKSUM_UNNECESSARY;
5747
59d71989 5748 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5749}
5750
757b77e2 5751static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5752 struct sk_buff *skb)
5753{
5754 struct igb_adapter *adapter = q_vector->adapter;
5755 struct e1000_hw *hw = &adapter->hw;
5756 u64 regval;
5757
5758 /*
5759 * If this bit is set, then the RX registers contain the time stamp. No
5760 * other packet will be time stamped until we read these registers, so
5761 * read the registers to make them available again. Because only one
5762 * packet can be time stamped at a time, we know that the register
5763 * values must belong to this one here and therefore we don't need to
5764 * compare any of the additional attributes stored for it.
5765 *
2244d07b 5766 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5767 * can turn into a skb_shared_hwtstamps.
5768 */
757b77e2
NN		 5769 if (staterr & E1000_RXDADV_STAT_TSIP) {
5770 u32 *stamp = (u32 *)skb->data;
5771 regval = le32_to_cpu(*(stamp + 2));
5772 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5773 skb_pull(skb, IGB_TS_HDR_LEN);
5774 } else {
5775 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5776 return;
c5b9bd5e 5777
757b77e2
NN		 5778 regval = rd32(E1000_RXSTMPL);
5779 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5780 }
c5b9bd5e
AD		 5781
5782 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5783}
4c844851 5784static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5785 union e1000_adv_rx_desc *rx_desc)
5786{
5787 /* HW will not DMA in data larger than the given buffer, even if it
5788 * parses the (NFS, of course) header to be larger. In that case, it
5789 * fills the header buffer and spills the rest into the page.
5790 */
5791 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5792 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5793 if (hlen > rx_ring->rx_buffer_len)
5794 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5795 return hlen;
5796}
5797
047e0030
AD		 5798static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5799 int *work_done, int budget)
9d5c8243 5800{
047e0030 5801 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5802 struct net_device *netdev = rx_ring->netdev;
59d71989 5803 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5804 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5805 struct igb_buffer *buffer_info , *next_buffer;
5806 struct sk_buff *skb;
9d5c8243
AK		 5807 bool cleaned = false;
5808 int cleaned_count = 0;
d1eff350 5809 int current_node = numa_node_id();
9d5c8243 5810 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5811 unsigned int i;
2d94d8ab
AD		 5812 u32 staterr;
5813 u16 length;
9d5c8243
AK		 5814
5815 i = rx_ring->next_to_clean;
69d3ca53 5816 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5817 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5818 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5819
5820 while (staterr & E1000_RXD_STAT_DD) {
5821 if (*work_done >= budget)
5822 break;
5823 (*work_done)++;
2d0bb1c1 5824 rmb(); /* read descriptor and rx_buffer_info after status DD */
9d5c8243 5825
69d3ca53
AD		 5826 skb = buffer_info->skb;
5827 prefetch(skb->data - NET_IP_ALIGN);
5828 buffer_info->skb = NULL;
5829
5830 i++;
5831 if (i == rx_ring->count)
5832 i = 0;
42d0781a 5833
69d3ca53
AD		 5834 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5835 prefetch(next_rxd);
5836 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5837
5838 length = le16_to_cpu(rx_desc->wb.upper.length);
5839 cleaned = true;
5840 cleaned_count++;
5841
2d94d8ab 5842 if (buffer_info->dma) {
59d71989 5843 dma_unmap_single(dev, buffer_info->dma,
4c844851 5844 rx_ring->rx_buffer_len,
59d71989 5845 DMA_FROM_DEVICE);
91615f76 5846 buffer_info->dma = 0;
4c844851 5847 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5848 skb_put(skb, length);
5849 goto send_up;
5850 }
4c844851 5851 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5852 }
5853
5854 if (length) {
59d71989
AD		 5855 dma_unmap_page(dev, buffer_info->page_dma,
5856 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5857 buffer_info->page_dma = 0;
bf36c1a0 5858
aa913403 5859 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5860 buffer_info->page,
5861 buffer_info->page_offset,
5862 length);
5863
d1eff350
AD		 5864 if ((page_count(buffer_info->page) != 1) ||
5865 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5866 buffer_info->page = NULL;
5867 else
5868 get_page(buffer_info->page);
9d5c8243
AK		 5869
5870 skb->len += length;
5871 skb->data_len += length;
bf36c1a0 5872 skb->truesize += length;
9d5c8243 5873 }
9d5c8243 5874
bf36c1a0 5875 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5876 buffer_info->skb = next_buffer->skb;
5877 buffer_info->dma = next_buffer->dma;
5878 next_buffer->skb = skb;
5879 next_buffer->dma = 0;
bf36c1a0
AD		 5880 goto next_desc;
5881 }
69d3ca53 5882send_up:
9d5c8243
AK		 5883 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5884 dev_kfree_skb_irq(skb);
5885 goto next_desc;
5886 }
9d5c8243 5887
757b77e2
NN		 5888 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5889 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5890 total_bytes += skb->len;
5891 total_packets++;
5892
85ad76b2 5893 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5894
5895 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5896 skb_record_rx_queue(skb, rx_ring->queue_index);
5897
b2cb09b1
JP		 5898 if (staterr & E1000_RXD_STAT_VP) {
5899 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
9d5c8243 5900
b2cb09b1
JP		 5901 __vlan_hwaccel_put_tag(skb, vid);
5902 }
5903 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 5904
9d5c8243
AK		 5905next_desc:
5906 rx_desc->wb.upper.status_error = 0;
5907
5908 /* return some buffers to hardware, one at a time is too slow */
5909 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5910 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5911 cleaned_count = 0;
5912 }
5913
5914 /* use prefetched values */
5915 rx_desc = next_rxd;
5916 buffer_info = next_buffer;
9d5c8243
AK		 5917 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5918 }
bf36c1a0 5919
9d5c8243 5920 rx_ring->next_to_clean = i;
c493ea45 5921 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5922
5923 if (cleaned_count)
3b644cf6 5924 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5925
5926 rx_ring->total_packets += total_packets;
5927 rx_ring->total_bytes += total_bytes;
12dcd86b 5928 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 5929 rx_ring->rx_stats.packets += total_packets;
5930 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 5931 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5932 return cleaned;
5933}
5934
9d5c8243
AK		 5935/**
5936 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5937 * @adapter: address of board private structure
5938 **/
d7ee5b3a 5939void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5940{
e694e964 5941 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5942 union e1000_adv_rx_desc *rx_desc;
5943 struct igb_buffer *buffer_info;
5944 struct sk_buff *skb;
5945 unsigned int i;
db761762 5946 int bufsz;
9d5c8243
AK		 5947
5948 i = rx_ring->next_to_use;
5949 buffer_info = &rx_ring->buffer_info[i];
5950
4c844851 5951 bufsz = rx_ring->rx_buffer_len;
db761762 5952
9d5c8243
AK		 5953 while (cleaned_count--) {
5954 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5955
6ec43fe6 5956 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5957 if (!buffer_info->page) {
42d0781a 5958 buffer_info->page = netdev_alloc_page(netdev);
12dcd86b
ED		 5959 if (unlikely(!buffer_info->page)) {
5960 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5961 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5962 u64_stats_update_end(&rx_ring->rx_syncp);
bf36c1a0
AD		 5963 goto no_buffers;
5964 }
5965 buffer_info->page_offset = 0;
5966 } else {
5967 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5968 }
5969 buffer_info->page_dma =
59d71989 5970 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5971 buffer_info->page_offset,
5972 PAGE_SIZE / 2,
59d71989
AD		 5973 DMA_FROM_DEVICE);
5974 if (dma_mapping_error(rx_ring->dev,
5975 buffer_info->page_dma)) {
42d0781a 5976 buffer_info->page_dma = 0;
12dcd86b 5977 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 5978 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5979 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 5980 goto no_buffers;
5981 }
9d5c8243
AK		 5982 }
5983
42d0781a
AD		 5984 skb = buffer_info->skb;
5985 if (!skb) {
89d71a66 5986 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
12dcd86b
ED		 5987 if (unlikely(!skb)) {
5988 u64_stats_update_begin(&rx_ring->rx_syncp);
04a5fcaa 5989 rx_ring->rx_stats.alloc_failed++;
12dcd86b 5990 u64_stats_update_end(&rx_ring->rx_syncp);
9d5c8243
AK		 5991 goto no_buffers;
5992 }
5993
9d5c8243 5994 buffer_info->skb = skb;
42d0781a
AD		 5995 }
5996 if (!buffer_info->dma) {
59d71989 5997 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5998 skb->data,
9d5c8243 5999 bufsz,
59d71989
AD		 6000 DMA_FROM_DEVICE);
6001 if (dma_mapping_error(rx_ring->dev,
6002 buffer_info->dma)) {
42d0781a 6003 buffer_info->dma = 0;
12dcd86b 6004 u64_stats_update_begin(&rx_ring->rx_syncp);
42d0781a 6005 rx_ring->rx_stats.alloc_failed++;
12dcd86b 6006 u64_stats_update_end(&rx_ring->rx_syncp);
42d0781a
AD		 6007 goto no_buffers;
6008 }
9d5c8243
AK		 6009 }
6010 /* Refresh the desc even if buffer_addrs didn't change because
6011 * each write-back erases this info. */
6ec43fe6 6012 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 6013 rx_desc->read.pkt_addr =
6014 cpu_to_le64(buffer_info->page_dma);
6015 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
6016 } else {
42d0781a 6017 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 6018 rx_desc->read.hdr_addr = 0;
6019 }
6020
6021 i++;
6022 if (i == rx_ring->count)
6023 i = 0;
6024 buffer_info = &rx_ring->buffer_info[i];
6025 }
6026
6027no_buffers:
6028 if (rx_ring->next_to_use != i) {
6029 rx_ring->next_to_use = i;
6030 if (i == 0)
6031 i = (rx_ring->count - 1);
6032 else
6033 i--;
6034
6035 /* Force memory writes to complete before letting h/w
6036 * know there are new descriptors to fetch. (Only
6037 * applicable for weak-ordered memory model archs,
6038 * such as IA-64). */
6039 wmb();
fce99e34 6040 writel(i, rx_ring->tail);
9d5c8243
AK		 6041 }
6042}
6043
6044/**
6045 * igb_mii_ioctl -
6046 * @netdev:
6047 * @ifreq:
6048 * @cmd:
6049 **/
6050static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6051{
6052 struct igb_adapter *adapter = netdev_priv(netdev);
6053 struct mii_ioctl_data *data = if_mii(ifr);
6054
6055 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6056 return -EOPNOTSUPP;
6057
6058 switch (cmd) {
6059 case SIOCGMIIPHY:
6060 data->phy_id = adapter->hw.phy.addr;
6061 break;
6062 case SIOCGMIIREG:
f5f4cf08
AD		 6063 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6064 &data->val_out))
9d5c8243
AK		 6065 return -EIO;
6066 break;
6067 case SIOCSMIIREG:
6068 default:
6069 return -EOPNOTSUPP;
6070 }
6071 return 0;
6072}
6073
c6cb090b
PO		 6074/**
6075 * igb_hwtstamp_ioctl - control hardware time stamping
6076 * @netdev:
6077 * @ifreq:
6078 * @cmd:
6079 *
33af6bcc
PO		 6080 * Outgoing time stamping can be enabled and disabled. Play nice and
6081 * disable it when requested, although it shouldn't case any overhead
6082 * when no packet needs it. At most one packet in the queue may be
6083 * marked for time stamping, otherwise it would be impossible to tell
6084 * for sure to which packet the hardware time stamp belongs.
6085 *
6086 * Incoming time stamping has to be configured via the hardware
6087 * filters. Not all combinations are supported, in particular event
6088 * type has to be specified. Matching the kind of event packet is
6089 * not supported, with the exception of "all V2 events regardless of
6090 * level 2 or 4".
6091 *
c6cb090b
PO		 6092 **/
6093static int igb_hwtstamp_ioctl(struct net_device *netdev,
6094 struct ifreq *ifr, int cmd)
6095{
33af6bcc
PO		 6096 struct igb_adapter *adapter = netdev_priv(netdev);
6097 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6098 struct hwtstamp_config config;
c5b9bd5e
AD		 6099 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6100 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6101 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6102 bool is_l4 = false;
6103 bool is_l2 = false;
33af6bcc 6104 u32 regval;
c6cb090b
PO		 6105
6106 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6107 return -EFAULT;
6108
6109 /* reserved for future extensions */
6110 if (config.flags)
6111 return -EINVAL;
6112
33af6bcc
PO		 6113 switch (config.tx_type) {
6114 case HWTSTAMP_TX_OFF:
c5b9bd5e 6115 tsync_tx_ctl = 0;
33af6bcc 6116 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6117 break;
6118 default:
6119 return -ERANGE;
6120 }
6121
6122 switch (config.rx_filter) {
6123 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6124 tsync_rx_ctl = 0;
33af6bcc
PO		 6125 break;
6126 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6127 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6128 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6129 case HWTSTAMP_FILTER_ALL:
6130 /*
6131 * register TSYNCRXCFG must be set, therefore it is not
6132 * possible to time stamp both Sync and Delay_Req messages
6133 * => fall back to time stamping all packets
6134 */
c5b9bd5e 6135 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6136 config.rx_filter = HWTSTAMP_FILTER_ALL;
6137 break;
6138 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6139 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6140 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6141 is_l4 = true;
33af6bcc
PO		 6142 break;
6143 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6144 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6145 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6146 is_l4 = true;
33af6bcc
PO		 6147 break;
6148 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6149 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6150 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6151 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6152 is_l2 = true;
6153 is_l4 = true;
33af6bcc
PO		 6154 config.rx_filter = HWTSTAMP_FILTER_SOME;
6155 break;
6156 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6157 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6158 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6159 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6160 is_l2 = true;
6161 is_l4 = true;
33af6bcc
PO		 6162 config.rx_filter = HWTSTAMP_FILTER_SOME;
6163 break;
6164 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6165 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6166 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6167 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6168 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6169 is_l2 = true;
33af6bcc
PO		 6170 break;
6171 default:
6172 return -ERANGE;
6173 }
6174
c5b9bd5e
AD		 6175 if (hw->mac.type == e1000_82575) {
6176 if (tsync_rx_ctl | tsync_tx_ctl)
6177 return -EINVAL;
6178 return 0;
6179 }
6180
757b77e2
NN		 6181 /*
6182 * Per-packet timestamping only works if all packets are
6183 * timestamped, so enable timestamping in all packets as
6184 * long as one rx filter was configured.
6185 */
6186 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
6187 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6188 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6189 }
6190
33af6bcc
PO		 6191 /* enable/disable TX */
6192 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6193 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6194 regval |= tsync_tx_ctl;
33af6bcc
PO		 6195 wr32(E1000_TSYNCTXCTL, regval);
6196
c5b9bd5e 6197 /* enable/disable RX */
33af6bcc 6198 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6199 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6200 regval |= tsync_rx_ctl;
33af6bcc 6201 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6202
c5b9bd5e
AD		 6203 /* define which PTP packets are time stamped */
6204 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6205
c5b9bd5e
AD		 6206 /* define ethertype filter for timestamped packets */
6207 if (is_l2)
6208 wr32(E1000_ETQF(3),
6209 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6210 E1000_ETQF_1588 | /* enable timestamping */
6211 ETH_P_1588)); /* 1588 eth protocol type */
6212 else
6213 wr32(E1000_ETQF(3), 0);
6214
6215#define PTP_PORT 319
6216 /* L4 Queue Filter[3]: filter by destination port and protocol */
6217 if (is_l4) {
6218 u32 ftqf = (IPPROTO_UDP /* UDP */
6219 | E1000_FTQF_VF_BP /* VF not compared */
6220 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6221 | E1000_FTQF_MASK); /* mask all inputs */
6222 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6223
6224 wr32(E1000_IMIR(3), htons(PTP_PORT));
6225 wr32(E1000_IMIREXT(3),
6226 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6227 if (hw->mac.type == e1000_82576) {
6228 /* enable source port check */
6229 wr32(E1000_SPQF(3), htons(PTP_PORT));
6230 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6231 }
6232 wr32(E1000_FTQF(3), ftqf);
6233 } else {
6234 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6235 }
33af6bcc
PO		 6236 wrfl();
6237
6238 adapter->hwtstamp_config = config;
6239
6240 /* clear TX/RX time stamp registers, just to be sure */
6241 regval = rd32(E1000_TXSTMPH);
6242 regval = rd32(E1000_RXSTMPH);
c6cb090b 6243
33af6bcc
PO		 6244 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6245 -EFAULT : 0;
c6cb090b
PO		 6246}
6247
9d5c8243
AK		 6248/**
6249 * igb_ioctl -
6250 * @netdev:
6251 * @ifreq:
6252 * @cmd:
6253 **/
6254static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6255{
6256 switch (cmd) {
6257 case SIOCGMIIPHY:
6258 case SIOCGMIIREG:
6259 case SIOCSMIIREG:
6260 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6261 case SIOCSHWTSTAMP:
6262 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6263 default:
6264 return -EOPNOTSUPP;
6265 }
6266}
6267
009bc06e
AD		 6268s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6269{
6270 struct igb_adapter *adapter = hw->back;
6271 u16 cap_offset;
6272
bdaae04c 6273 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6274 if (!cap_offset)
6275 return -E1000_ERR_CONFIG;
6276
6277 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6278
6279 return 0;
6280}
6281
6282s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6283{
6284 struct igb_adapter *adapter = hw->back;
6285 u16 cap_offset;
6286
bdaae04c 6287 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6288 if (!cap_offset)
6289 return -E1000_ERR_CONFIG;
6290
6291 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6292
6293 return 0;
6294}
6295
b2cb09b1 6296static void igb_vlan_mode(struct net_device *netdev, u32 features)
9d5c8243
AK		 6297{
6298 struct igb_adapter *adapter = netdev_priv(netdev);
6299 struct e1000_hw *hw = &adapter->hw;
6300 u32 ctrl, rctl;
6301
6302 igb_irq_disable(adapter);
9d5c8243 6303
b2cb09b1 6304 if (features & NETIF_F_HW_VLAN_RX) {
9d5c8243
AK		 6305 /* enable VLAN tag insert/strip */
6306 ctrl = rd32(E1000_CTRL);
6307 ctrl |= E1000_CTRL_VME;
6308 wr32(E1000_CTRL, ctrl);
6309
51466239 6310 /* Disable CFI check */
9d5c8243 6311 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6312 rctl &= ~E1000_RCTL_CFIEN;
6313 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6314 } else {
6315 /* disable VLAN tag insert/strip */
6316 ctrl = rd32(E1000_CTRL);
6317 ctrl &= ~E1000_CTRL_VME;
6318 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6319 }
6320
e1739522
AD		 6321 igb_rlpml_set(adapter);
6322
9d5c8243
AK		 6323 if (!test_bit(__IGB_DOWN, &adapter->state))
6324 igb_irq_enable(adapter);
6325}
6326
6327static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6328{
6329 struct igb_adapter *adapter = netdev_priv(netdev);
6330 struct e1000_hw *hw = &adapter->hw;
4ae196df 6331 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6332
51466239
AD		 6333 /* attempt to add filter to vlvf array */
6334 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6335
51466239
AD		 6336 /* add the filter since PF can receive vlans w/o entry in vlvf */
6337 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6338
6339 set_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6340}
6341
6342static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6343{
6344 struct igb_adapter *adapter = netdev_priv(netdev);
6345 struct e1000_hw *hw = &adapter->hw;
4ae196df 6346 int pf_id = adapter->vfs_allocated_count;
51466239 6347 s32 err;
9d5c8243
AK		 6348
6349 igb_irq_disable(adapter);
9d5c8243
AK		 6350
6351 if (!test_bit(__IGB_DOWN, &adapter->state))
6352 igb_irq_enable(adapter);
6353
51466239
AD		 6354 /* remove vlan from VLVF table array */
6355 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6356
51466239
AD		 6357 /* if vid was not present in VLVF just remove it from table */
6358 if (err)
4ae196df 6359 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6360
6361 clear_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6362}
6363
6364static void igb_restore_vlan(struct igb_adapter *adapter)
6365{
b2cb09b1 6366 u16 vid;
9d5c8243 6367
b2cb09b1
JP		 6368 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6369 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6370}
6371
14ad2513 6372int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6373{
090b1795 6374 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6375 struct e1000_mac_info *mac = &adapter->hw.mac;
6376
6377 mac->autoneg = 0;
6378
14ad2513
DD		 6379 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6380 * for the switch() below to work */
6381 if ((spd & 1) || (dplx & ~1))
6382 goto err_inval;
6383
cd2638a8
CW		 6384 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6385 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6386 spd != SPEED_1000 &&
6387 dplx != DUPLEX_FULL)
6388 goto err_inval;
cd2638a8 6389
14ad2513 6390 switch (spd + dplx) {
9d5c8243
AK		 6391 case SPEED_10 + DUPLEX_HALF:
6392 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6393 break;
6394 case SPEED_10 + DUPLEX_FULL:
6395 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6396 break;
6397 case SPEED_100 + DUPLEX_HALF:
6398 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6399 break;
6400 case SPEED_100 + DUPLEX_FULL:
6401 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6402 break;
6403 case SPEED_1000 + DUPLEX_FULL:
6404 mac->autoneg = 1;
6405 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6406 break;
6407 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6408 default:
14ad2513 6409 goto err_inval;
9d5c8243
AK		 6410 }
6411 return 0;
14ad2513
DD		 6412
6413err_inval:
6414 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6415 return -EINVAL;
9d5c8243
AK		 6416}
6417
3fe7c4c9 6418static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6419{
6420 struct net_device *netdev = pci_get_drvdata(pdev);
6421 struct igb_adapter *adapter = netdev_priv(netdev);
6422 struct e1000_hw *hw = &adapter->hw;
2d064c06 6423 u32 ctrl, rctl, status;
9d5c8243
AK		 6424 u32 wufc = adapter->wol;
6425#ifdef CONFIG_PM
6426 int retval = 0;
6427#endif
6428
6429 netif_device_detach(netdev);
6430
a88f10ec
AD		 6431 if (netif_running(netdev))
6432 igb_close(netdev);
6433
047e0030 6434 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6435
6436#ifdef CONFIG_PM
6437 retval = pci_save_state(pdev);
6438 if (retval)
6439 return retval;
6440#endif
6441
6442 status = rd32(E1000_STATUS);
6443 if (status & E1000_STATUS_LU)
6444 wufc &= ~E1000_WUFC_LNKC;
6445
6446 if (wufc) {
6447 igb_setup_rctl(adapter);
ff41f8dc 6448 igb_set_rx_mode(netdev);
9d5c8243
AK		 6449
6450 /* turn on all-multi mode if wake on multicast is enabled */
6451 if (wufc & E1000_WUFC_MC) {
6452 rctl = rd32(E1000_RCTL);
6453 rctl |= E1000_RCTL_MPE;
6454 wr32(E1000_RCTL, rctl);
6455 }
6456
6457 ctrl = rd32(E1000_CTRL);
6458 /* advertise wake from D3Cold */
6459 #define E1000_CTRL_ADVD3WUC 0x00100000
6460 /* phy power management enable */
6461 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6462 ctrl |= E1000_CTRL_ADVD3WUC;
6463 wr32(E1000_CTRL, ctrl);
6464
9d5c8243 6465 /* Allow time for pending master requests to run */
330a6d6a 6466 igb_disable_pcie_master(hw);
9d5c8243
AK		 6467
6468 wr32(E1000_WUC, E1000_WUC_PME_EN);
6469 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6470 } else {
6471 wr32(E1000_WUC, 0);
6472 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6473 }
6474
3fe7c4c9
RW		 6475 *enable_wake = wufc || adapter->en_mng_pt;
6476 if (!*enable_wake)
88a268c1
NN		 6477 igb_power_down_link(adapter);
6478 else
6479 igb_power_up_link(adapter);
9d5c8243
AK		 6480
6481 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6482 * would have already happened in close and is redundant. */
6483 igb_release_hw_control(adapter);
6484
6485 pci_disable_device(pdev);
6486
9d5c8243
AK		 6487 return 0;
6488}
6489
6490#ifdef CONFIG_PM
3fe7c4c9
RW		 6491static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6492{
6493 int retval;
6494 bool wake;
6495
6496 retval = __igb_shutdown(pdev, &wake);
6497 if (retval)
6498 return retval;
6499
6500 if (wake) {
6501 pci_prepare_to_sleep(pdev);
6502 } else {
6503 pci_wake_from_d3(pdev, false);
6504 pci_set_power_state(pdev, PCI_D3hot);
6505 }
6506
6507 return 0;
6508}
6509
9d5c8243
AK		 6510static int igb_resume(struct pci_dev *pdev)
6511{
6512 struct net_device *netdev = pci_get_drvdata(pdev);
6513 struct igb_adapter *adapter = netdev_priv(netdev);
6514 struct e1000_hw *hw = &adapter->hw;
6515 u32 err;
6516
6517 pci_set_power_state(pdev, PCI_D0);
6518 pci_restore_state(pdev);
b94f2d77 6519 pci_save_state(pdev);
42bfd33a 6520
aed5dec3 6521 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6522 if (err) {
6523 dev_err(&pdev->dev,
6524 "igb: Cannot enable PCI device from suspend\n");
6525 return err;
6526 }
6527 pci_set_master(pdev);
6528
6529 pci_enable_wake(pdev, PCI_D3hot, 0);
6530 pci_enable_wake(pdev, PCI_D3cold, 0);
6531
047e0030 6532 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6533 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6534 return -ENOMEM;
9d5c8243
AK		 6535 }
6536
9d5c8243 6537 igb_reset(adapter);
a8564f03
AD		 6538
6539 /* let the f/w know that the h/w is now under the control of the
6540 * driver. */
6541 igb_get_hw_control(adapter);
6542
9d5c8243
AK		 6543 wr32(E1000_WUS, ~0);
6544
a88f10ec
AD		 6545 if (netif_running(netdev)) {
6546 err = igb_open(netdev);
6547 if (err)
6548 return err;
6549 }
9d5c8243
AK		 6550
6551 netif_device_attach(netdev);
6552
9d5c8243
AK		 6553 return 0;
6554}
6555#endif
6556
6557static void igb_shutdown(struct pci_dev *pdev)
6558{
3fe7c4c9
RW		 6559 bool wake;
6560
6561 __igb_shutdown(pdev, &wake);
6562
6563 if (system_state == SYSTEM_POWER_OFF) {
6564 pci_wake_from_d3(pdev, wake);
6565 pci_set_power_state(pdev, PCI_D3hot);
6566 }
9d5c8243
AK		 6567}
6568
6569#ifdef CONFIG_NET_POLL_CONTROLLER
6570/*
6571 * Polling 'interrupt' - used by things like netconsole to send skbs
6572 * without having to re-enable interrupts. It's not called while
6573 * the interrupt routine is executing.
6574 */
6575static void igb_netpoll(struct net_device *netdev)
6576{
6577 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6578 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6579 int i;
9d5c8243 6580
eebbbdba 6581 if (!adapter->msix_entries) {
047e0030 6582 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6583 igb_irq_disable(adapter);
047e0030 6584 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6585 return;
6586 }
9d5c8243 6587
047e0030
AD		 6588 for (i = 0; i < adapter->num_q_vectors; i++) {
6589 struct igb_q_vector *q_vector = adapter->q_vector[i];
6590 wr32(E1000_EIMC, q_vector->eims_value);
6591 napi_schedule(&q_vector->napi);
eebbbdba 6592 }
9d5c8243
AK		 6593}
6594#endif /* CONFIG_NET_POLL_CONTROLLER */
6595
6596/**
6597 * igb_io_error_detected - called when PCI error is detected
6598 * @pdev: Pointer to PCI device
6599 * @state: The current pci connection state
6600 *
6601 * This function is called after a PCI bus error affecting
6602 * this device has been detected.
6603 */
6604static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6605 pci_channel_state_t state)
6606{
6607 struct net_device *netdev = pci_get_drvdata(pdev);
6608 struct igb_adapter *adapter = netdev_priv(netdev);
6609
6610 netif_device_detach(netdev);
6611
59ed6eec
AD		 6612 if (state == pci_channel_io_perm_failure)
6613 return PCI_ERS_RESULT_DISCONNECT;
6614
9d5c8243
AK		 6615 if (netif_running(netdev))
6616 igb_down(adapter);
6617 pci_disable_device(pdev);
6618
6619 /* Request a slot slot reset. */
6620 return PCI_ERS_RESULT_NEED_RESET;
6621}
6622
6623/**
6624 * igb_io_slot_reset - called after the pci bus has been reset.
6625 * @pdev: Pointer to PCI device
6626 *
6627 * Restart the card from scratch, as if from a cold-boot. Implementation
6628 * resembles the first-half of the igb_resume routine.
6629 */
6630static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6631{
6632 struct net_device *netdev = pci_get_drvdata(pdev);
6633 struct igb_adapter *adapter = netdev_priv(netdev);
6634 struct e1000_hw *hw = &adapter->hw;
40a914fa 6635 pci_ers_result_t result;
42bfd33a 6636 int err;
9d5c8243 6637
aed5dec3 6638 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6639 dev_err(&pdev->dev,
6640 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6641 result = PCI_ERS_RESULT_DISCONNECT;
6642 } else {
6643 pci_set_master(pdev);
6644 pci_restore_state(pdev);
b94f2d77 6645 pci_save_state(pdev);
9d5c8243 6646
40a914fa
AD		 6647 pci_enable_wake(pdev, PCI_D3hot, 0);
6648 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6649
40a914fa
AD		 6650 igb_reset(adapter);
6651 wr32(E1000_WUS, ~0);
6652 result = PCI_ERS_RESULT_RECOVERED;
6653 }
9d5c8243 6654
ea943d41
JK		 6655 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6656 if (err) {
6657 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6658 "failed 0x%0x\n", err);
6659 /* non-fatal, continue */
6660 }
40a914fa
AD		 6661
6662 return result;
9d5c8243
AK		 6663}
6664
6665/**
6666 * igb_io_resume - called when traffic can start flowing again.
6667 * @pdev: Pointer to PCI device
6668 *
6669 * This callback is called when the error recovery driver tells us that
6670 * its OK to resume normal operation. Implementation resembles the
6671 * second-half of the igb_resume routine.
6672 */
6673static void igb_io_resume(struct pci_dev *pdev)
6674{
6675 struct net_device *netdev = pci_get_drvdata(pdev);
6676 struct igb_adapter *adapter = netdev_priv(netdev);
6677
9d5c8243
AK		 6678 if (netif_running(netdev)) {
6679 if (igb_up(adapter)) {
6680 dev_err(&pdev->dev, "igb_up failed after reset\n");
6681 return;
6682 }
6683 }
6684
6685 netif_device_attach(netdev);
6686
6687 /* let the f/w know that the h/w is now under the control of the
6688 * driver. */
6689 igb_get_hw_control(adapter);
9d5c8243
AK		 6690}
6691
26ad9178
AD		 6692static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6693 u8 qsel)
6694{
6695 u32 rar_low, rar_high;
6696 struct e1000_hw *hw = &adapter->hw;
6697
6698 /* HW expects these in little endian so we reverse the byte order
6699 * from network order (big endian) to little endian
6700 */
6701 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6702 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6703 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6704
6705 /* Indicate to hardware the Address is Valid. */
6706 rar_high |= E1000_RAH_AV;
6707
6708 if (hw->mac.type == e1000_82575)
6709 rar_high |= E1000_RAH_POOL_1 * qsel;
6710 else
6711 rar_high |= E1000_RAH_POOL_1 << qsel;
6712
6713 wr32(E1000_RAL(index), rar_low);
6714 wrfl();
6715 wr32(E1000_RAH(index), rar_high);
6716 wrfl();
6717}
6718
4ae196df
AD		 6719static int igb_set_vf_mac(struct igb_adapter *adapter,
6720 int vf, unsigned char *mac_addr)
6721{
6722 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6723 /* VF MAC addresses start at end of receive addresses and moves
6724 * torwards the first, as a result a collision should not be possible */
6725 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6726
37680117 6727 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6728
26ad9178 6729 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6730
6731 return 0;
6732}
6733
8151d294
WM		 6734static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6735{
6736 struct igb_adapter *adapter = netdev_priv(netdev);
6737 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6738 return -EINVAL;
6739 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6740 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6741 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6742 " change effective.");
6743 if (test_bit(__IGB_DOWN, &adapter->state)) {
6744 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6745 " but the PF device is not up.\n");
6746 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6747 " attempting to use the VF device.\n");
6748 }
6749 return igb_set_vf_mac(adapter, vf, mac);
6750}
6751
17dc566c
LL		 6752static int igb_link_mbps(int internal_link_speed)
6753{
6754 switch (internal_link_speed) {
6755 case SPEED_100:
6756 return 100;
6757 case SPEED_1000:
6758 return 1000;
6759 default:
6760 return 0;
6761 }
6762}
6763
6764static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6765 int link_speed)
6766{
6767 int rf_dec, rf_int;
6768 u32 bcnrc_val;
6769
6770 if (tx_rate != 0) {
6771 /* Calculate the rate factor values to set */
6772 rf_int = link_speed / tx_rate;
6773 rf_dec = (link_speed - (rf_int * tx_rate));
6774 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6775
6776 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6777 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6778 E1000_RTTBCNRC_RF_INT_MASK);
6779 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6780 } else {
6781 bcnrc_val = 0;
6782 }
6783
6784 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6785 wr32(E1000_RTTBCNRC, bcnrc_val);
6786}
6787
6788static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6789{
6790 int actual_link_speed, i;
6791 bool reset_rate = false;
6792
6793 /* VF TX rate limit was not set or not supported */
6794 if ((adapter->vf_rate_link_speed == 0) ||
6795 (adapter->hw.mac.type != e1000_82576))
6796 return;
6797
6798 actual_link_speed = igb_link_mbps(adapter->link_speed);
6799 if (actual_link_speed != adapter->vf_rate_link_speed) {
6800 reset_rate = true;
6801 adapter->vf_rate_link_speed = 0;
6802 dev_info(&adapter->pdev->dev,
6803 "Link speed has been changed. VF Transmit "
6804 "rate is disabled\n");
6805 }
6806
6807 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6808 if (reset_rate)
6809 adapter->vf_data[i].tx_rate = 0;
6810
6811 igb_set_vf_rate_limit(&adapter->hw, i,
6812 adapter->vf_data[i].tx_rate,
6813 actual_link_speed);
6814 }
6815}
6816
8151d294
WM		 6817static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6818{
17dc566c
LL		 6819 struct igb_adapter *adapter = netdev_priv(netdev);
6820 struct e1000_hw *hw = &adapter->hw;
6821 int actual_link_speed;
6822
6823 if (hw->mac.type != e1000_82576)
6824 return -EOPNOTSUPP;
6825
6826 actual_link_speed = igb_link_mbps(adapter->link_speed);
6827 if ((vf >= adapter->vfs_allocated_count) ||
6828 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6829 (tx_rate < 0) || (tx_rate > actual_link_speed))
6830 return -EINVAL;
6831
6832 adapter->vf_rate_link_speed = actual_link_speed;
6833 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6834 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6835
6836 return 0;
8151d294
WM		 6837}
6838
6839static int igb_ndo_get_vf_config(struct net_device *netdev,
6840 int vf, struct ifla_vf_info *ivi)
6841{
6842 struct igb_adapter *adapter = netdev_priv(netdev);
6843 if (vf >= adapter->vfs_allocated_count)
6844 return -EINVAL;
6845 ivi->vf = vf;
6846 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 6847 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 6848 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6849 ivi->qos = adapter->vf_data[vf].pf_qos;
6850 return 0;
6851}
6852
4ae196df
AD		 6853static void igb_vmm_control(struct igb_adapter *adapter)
6854{
6855 struct e1000_hw *hw = &adapter->hw;
10d8e907 6856 u32 reg;
4ae196df 6857
52a1dd4d
AD		 6858 switch (hw->mac.type) {
6859 case e1000_82575:
6860 default:
6861 /* replication is not supported for 82575 */
4ae196df 6862 return;
52a1dd4d
AD		 6863 case e1000_82576:
6864 /* notify HW that the MAC is adding vlan tags */
6865 reg = rd32(E1000_DTXCTL);
6866 reg |= E1000_DTXCTL_VLAN_ADDED;
6867 wr32(E1000_DTXCTL, reg);
6868 case e1000_82580:
6869 /* enable replication vlan tag stripping */
6870 reg = rd32(E1000_RPLOLR);
6871 reg |= E1000_RPLOLR_STRVLAN;
6872 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6873 case e1000_i350:
6874 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6875 break;
6876 }
10d8e907 6877
d4960307
AD		 6878 if (adapter->vfs_allocated_count) {
6879 igb_vmdq_set_loopback_pf(hw, true);
6880 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 6881 igb_vmdq_set_anti_spoofing_pf(hw, true,
6882 adapter->vfs_allocated_count);
d4960307
AD		 6883 } else {
6884 igb_vmdq_set_loopback_pf(hw, false);
6885 igb_vmdq_set_replication_pf(hw, false);
6886 }
4ae196df
AD		 6887}
6888
9d5c8243 6889/* igb_main.c */
kernel source for telekom puls (alcatel/mediatek)