projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
netxen: mask correctable error
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
2d064c06 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
87 /* required last entry */
88 {0, }
89};
90
91MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
92
93void igb_reset(struct igb_adapter *);
94static int igb_setup_all_tx_resources(struct igb_adapter *);
95static int igb_setup_all_rx_resources(struct igb_adapter *);
96static void igb_free_all_tx_resources(struct igb_adapter *);
97static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 98static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 99void igb_update_stats(struct igb_adapter *);
100static int igb_probe(struct pci_dev *, const struct pci_device_id *);
101static void __devexit igb_remove(struct pci_dev *pdev);
102static int igb_sw_init(struct igb_adapter *);
103static int igb_open(struct net_device *);
104static int igb_close(struct net_device *);
105static void igb_configure_tx(struct igb_adapter *);
106static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 107static void igb_clean_all_tx_rings(struct igb_adapter *);
108static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 109static void igb_clean_tx_ring(struct igb_ring *);
110static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 111static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 112static void igb_update_phy_info(unsigned long);
113static void igb_watchdog(unsigned long);
114static void igb_watchdog_task(struct work_struct *);
b1a436c3 115static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 116static struct net_device_stats *igb_get_stats(struct net_device *);
117static int igb_change_mtu(struct net_device *, int);
118static int igb_set_mac(struct net_device *, void *);
68d480c4 119static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 120static irqreturn_t igb_intr(int irq, void *);
121static irqreturn_t igb_intr_msi(int irq, void *);
122static irqreturn_t igb_msix_other(int irq, void *);
047e0030 123static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 124#ifdef CONFIG_IGB_DCA
047e0030 125static void igb_update_dca(struct igb_q_vector *);
fe4506b6 126static void igb_setup_dca(struct igb_adapter *);
421e02f0 127#endif /* CONFIG_IGB_DCA */
047e0030 128static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 129static int igb_poll(struct napi_struct *, int);
047e0030 130static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 131static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
132static void igb_tx_timeout(struct net_device *);
133static void igb_reset_task(struct work_struct *);
134static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
135static void igb_vlan_rx_add_vid(struct net_device *, u16);
136static void igb_vlan_rx_kill_vid(struct net_device *, u16);
137static void igb_restore_vlan(struct igb_adapter *);
26ad9178 138static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 139static void igb_ping_all_vfs(struct igb_adapter *);
140static void igb_msg_task(struct igb_adapter *);
4ae196df 141static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 142static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 143static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 144static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
145static int igb_ndo_set_vf_vlan(struct net_device *netdev,
146 int vf, u16 vlan, u8 qos);
147static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
148static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
149 struct ifla_vf_info *ivi);
9d5c8243 150
9d5c8243 151#ifdef CONFIG_PM
3fe7c4c9 152static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 153static int igb_resume(struct pci_dev *);
154#endif
155static void igb_shutdown(struct pci_dev *);
421e02f0 156#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 157static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
158static struct notifier_block dca_notifier = {
159 .notifier_call = igb_notify_dca,
160 .next = NULL,
161 .priority = 0
162};
163#endif
9d5c8243
AK		 164#ifdef CONFIG_NET_POLL_CONTROLLER
165/* for netdump / net console */
166static void igb_netpoll(struct net_device *);
167#endif
37680117 168#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 169static unsigned int max_vfs = 0;
170module_param(max_vfs, uint, 0);
171MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
172 "per physical function");
173#endif /* CONFIG_PCI_IOV */
174
9d5c8243
AK		 175static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
176 pci_channel_state_t);
177static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
178static void igb_io_resume(struct pci_dev *);
179
180static struct pci_error_handlers igb_err_handler = {
181 .error_detected = igb_io_error_detected,
182 .slot_reset = igb_io_slot_reset,
183 .resume = igb_io_resume,
184};
185
186
187static struct pci_driver igb_driver = {
188 .name = igb_driver_name,
189 .id_table = igb_pci_tbl,
190 .probe = igb_probe,
191 .remove = __devexit_p(igb_remove),
192#ifdef CONFIG_PM
193 /* Power Managment Hooks */
194 .suspend = igb_suspend,
195 .resume = igb_resume,
196#endif
197 .shutdown = igb_shutdown,
198 .err_handler = &igb_err_handler
199};
200
201MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
202MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
203MODULE_LICENSE("GPL");
204MODULE_VERSION(DRV_VERSION);
205
c97ec42a
TI		 206struct igb_reg_info {
207 u32 ofs;
208 char *name;
209};
210
211static const struct igb_reg_info igb_reg_info_tbl[] = {
212
213 /* General Registers */
214 {E1000_CTRL, "CTRL"},
215 {E1000_STATUS, "STATUS"},
216 {E1000_CTRL_EXT, "CTRL_EXT"},
217
218 /* Interrupt Registers */
219 {E1000_ICR, "ICR"},
220
221 /* RX Registers */
222 {E1000_RCTL, "RCTL"},
223 {E1000_RDLEN(0), "RDLEN"},
224 {E1000_RDH(0), "RDH"},
225 {E1000_RDT(0), "RDT"},
226 {E1000_RXDCTL(0), "RXDCTL"},
227 {E1000_RDBAL(0), "RDBAL"},
228 {E1000_RDBAH(0), "RDBAH"},
229
230 /* TX Registers */
231 {E1000_TCTL, "TCTL"},
232 {E1000_TDBAL(0), "TDBAL"},
233 {E1000_TDBAH(0), "TDBAH"},
234 {E1000_TDLEN(0), "TDLEN"},
235 {E1000_TDH(0), "TDH"},
236 {E1000_TDT(0), "TDT"},
237 {E1000_TXDCTL(0), "TXDCTL"},
238 {E1000_TDFH, "TDFH"},
239 {E1000_TDFT, "TDFT"},
240 {E1000_TDFHS, "TDFHS"},
241 {E1000_TDFPC, "TDFPC"},
242
243 /* List Terminator */
244 {}
245};
246
247/*
248 * igb_regdump - register printout routine
249 */
250static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
251{
252 int n = 0;
253 char rname[16];
254 u32 regs[8];
255
256 switch (reginfo->ofs) {
257 case E1000_RDLEN(0):
258 for (n = 0; n < 4; n++)
259 regs[n] = rd32(E1000_RDLEN(n));
260 break;
261 case E1000_RDH(0):
262 for (n = 0; n < 4; n++)
263 regs[n] = rd32(E1000_RDH(n));
264 break;
265 case E1000_RDT(0):
266 for (n = 0; n < 4; n++)
267 regs[n] = rd32(E1000_RDT(n));
268 break;
269 case E1000_RXDCTL(0):
270 for (n = 0; n < 4; n++)
271 regs[n] = rd32(E1000_RXDCTL(n));
272 break;
273 case E1000_RDBAL(0):
274 for (n = 0; n < 4; n++)
275 regs[n] = rd32(E1000_RDBAL(n));
276 break;
277 case E1000_RDBAH(0):
278 for (n = 0; n < 4; n++)
279 regs[n] = rd32(E1000_RDBAH(n));
280 break;
281 case E1000_TDBAL(0):
282 for (n = 0; n < 4; n++)
283 regs[n] = rd32(E1000_RDBAL(n));
284 break;
285 case E1000_TDBAH(0):
286 for (n = 0; n < 4; n++)
287 regs[n] = rd32(E1000_TDBAH(n));
288 break;
289 case E1000_TDLEN(0):
290 for (n = 0; n < 4; n++)
291 regs[n] = rd32(E1000_TDLEN(n));
292 break;
293 case E1000_TDH(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_TDH(n));
296 break;
297 case E1000_TDT(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_TDT(n));
300 break;
301 case E1000_TXDCTL(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_TXDCTL(n));
304 break;
305 default:
306 printk(KERN_INFO "%-15s %08x\n",
307 reginfo->name, rd32(reginfo->ofs));
308 return;
309 }
310
311 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
312 printk(KERN_INFO "%-15s ", rname);
313 for (n = 0; n < 4; n++)
314 printk(KERN_CONT "%08x ", regs[n]);
315 printk(KERN_CONT "\n");
316}
317
318/*
319 * igb_dump - Print registers, tx-rings and rx-rings
320 */
321static void igb_dump(struct igb_adapter *adapter)
322{
323 struct net_device *netdev = adapter->netdev;
324 struct e1000_hw *hw = &adapter->hw;
325 struct igb_reg_info *reginfo;
326 int n = 0;
327 struct igb_ring *tx_ring;
328 union e1000_adv_tx_desc *tx_desc;
329 struct my_u0 { u64 a; u64 b; } *u0;
330 struct igb_buffer *buffer_info;
331 struct igb_ring *rx_ring;
332 union e1000_adv_rx_desc *rx_desc;
333 u32 staterr;
334 int i = 0;
335
336 if (!netif_msg_hw(adapter))
337 return;
338
339 /* Print netdevice Info */
340 if (netdev) {
341 dev_info(&adapter->pdev->dev, "Net device Info\n");
342 printk(KERN_INFO "Device Name state "
343 "trans_start last_rx\n");
344 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
345 netdev->name,
346 netdev->state,
347 netdev->trans_start,
348 netdev->last_rx);
349 }
350
351 /* Print Registers */
352 dev_info(&adapter->pdev->dev, "Register Dump\n");
353 printk(KERN_INFO " Register Name Value\n");
354 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
355 reginfo->name; reginfo++) {
356 igb_regdump(hw, reginfo);
357 }
358
359 /* Print TX Ring Summary */
360 if (!netdev || !netif_running(netdev))
361 goto exit;
362
363 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
364 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
365 " leng ntw timestamp\n");
366 for (n = 0; n < adapter->num_tx_queues; n++) {
367 tx_ring = adapter->tx_ring[n];
368 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
369 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
370 n, tx_ring->next_to_use, tx_ring->next_to_clean,
371 (u64)buffer_info->dma,
372 buffer_info->length,
373 buffer_info->next_to_watch,
374 (u64)buffer_info->time_stamp);
375 }
376
377 /* Print TX Rings */
378 if (!netif_msg_tx_done(adapter))
379 goto rx_ring_summary;
380
381 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
382
383 /* Transmit Descriptor Formats
384 *
385 * Advanced Transmit Descriptor
386 * +--------------------------------------------------------------+
387 * 0 | Buffer Address [63:0] |
388 * +--------------------------------------------------------------+
389 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
390 * +--------------------------------------------------------------+
391 * 63 46 45 40 39 38 36 35 32 31 24 15 0
392 */
393
394 for (n = 0; n < adapter->num_tx_queues; n++) {
395 tx_ring = adapter->tx_ring[n];
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
398 printk(KERN_INFO "------------------------------------\n");
399 printk(KERN_INFO "T [desc] [address 63:0 ] "
400 "[PlPOCIStDDM Ln] [bi->dma ] "
401 "leng ntw timestamp bi->skb\n");
402
403 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
404 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
405 buffer_info = &tx_ring->buffer_info[i];
406 u0 = (struct my_u0 *)tx_desc;
407 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
408 " %04X %3X %016llX %p", i,
409 le64_to_cpu(u0->a),
410 le64_to_cpu(u0->b),
411 (u64)buffer_info->dma,
412 buffer_info->length,
413 buffer_info->next_to_watch,
414 (u64)buffer_info->time_stamp,
415 buffer_info->skb);
416 if (i == tx_ring->next_to_use &&
417 i == tx_ring->next_to_clean)
418 printk(KERN_CONT " NTC/U\n");
419 else if (i == tx_ring->next_to_use)
420 printk(KERN_CONT " NTU\n");
421 else if (i == tx_ring->next_to_clean)
422 printk(KERN_CONT " NTC\n");
423 else
424 printk(KERN_CONT "\n");
425
426 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
427 print_hex_dump(KERN_INFO, "",
428 DUMP_PREFIX_ADDRESS,
429 16, 1, phys_to_virt(buffer_info->dma),
430 buffer_info->length, true);
431 }
432 }
433
434 /* Print RX Rings Summary */
435rx_ring_summary:
436 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
437 printk(KERN_INFO "Queue [NTU] [NTC]\n");
438 for (n = 0; n < adapter->num_rx_queues; n++) {
439 rx_ring = adapter->rx_ring[n];
440 printk(KERN_INFO " %5d %5X %5X\n", n,
441 rx_ring->next_to_use, rx_ring->next_to_clean);
442 }
443
444 /* Print RX Rings */
445 if (!netif_msg_rx_status(adapter))
446 goto exit;
447
448 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
449
450 /* Advanced Receive Descriptor (Read) Format
451 * 63 1 0
452 * +-----------------------------------------------------+
453 * 0 | Packet Buffer Address [63:1] |A0/NSE|
454 * +----------------------------------------------+------+
455 * 8 | Header Buffer Address [63:1] | DD |
456 * +-----------------------------------------------------+
457 *
458 *
459 * Advanced Receive Descriptor (Write-Back) Format
460 *
461 * 63 48 47 32 31 30 21 20 17 16 4 3 0
462 * +------------------------------------------------------+
463 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
464 * | Checksum Ident | | | | Type | Type |
465 * +------------------------------------------------------+
466 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
467 * +------------------------------------------------------+
468 * 63 48 47 32 31 20 19 0
469 */
470
471 for (n = 0; n < adapter->num_rx_queues; n++) {
472 rx_ring = adapter->rx_ring[n];
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
475 printk(KERN_INFO "------------------------------------\n");
476 printk(KERN_INFO "R [desc] [ PktBuf A0] "
477 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
478 "<-- Adv Rx Read format\n");
479 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
480 "[vl er S cks ln] ---------------- [bi->skb] "
481 "<-- Adv Rx Write-Back format\n");
482
483 for (i = 0; i < rx_ring->count; i++) {
484 buffer_info = &rx_ring->buffer_info[i];
485 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
486 u0 = (struct my_u0 *)rx_desc;
487 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
488 if (staterr & E1000_RXD_STAT_DD) {
489 /* Descriptor Done */
490 printk(KERN_INFO "RWB[0x%03X] %016llX "
491 "%016llX ---------------- %p", i,
492 le64_to_cpu(u0->a),
493 le64_to_cpu(u0->b),
494 buffer_info->skb);
495 } else {
496 printk(KERN_INFO "R [0x%03X] %016llX "
497 "%016llX %016llX %p", i,
498 le64_to_cpu(u0->a),
499 le64_to_cpu(u0->b),
500 (u64)buffer_info->dma,
501 buffer_info->skb);
502
503 if (netif_msg_pktdata(adapter)) {
504 print_hex_dump(KERN_INFO, "",
505 DUMP_PREFIX_ADDRESS,
506 16, 1,
507 phys_to_virt(buffer_info->dma),
508 rx_ring->rx_buffer_len, true);
509 if (rx_ring->rx_buffer_len
510 < IGB_RXBUFFER_1024)
511 print_hex_dump(KERN_INFO, "",
512 DUMP_PREFIX_ADDRESS,
513 16, 1,
514 phys_to_virt(
515 buffer_info->page_dma +
516 buffer_info->page_offset),
517 PAGE_SIZE/2, true);
518 }
519 }
520
521 if (i == rx_ring->next_to_use)
522 printk(KERN_CONT " NTU\n");
523 else if (i == rx_ring->next_to_clean)
524 printk(KERN_CONT " NTC\n");
525 else
526 printk(KERN_CONT "\n");
527
528 }
529 }
530
531exit:
532 return;
533}
534
535
38c845c7
PO		 536/**
537 * igb_read_clock - read raw cycle counter (to be used by time counter)
538 */
539static cycle_t igb_read_clock(const struct cyclecounter *tc)
540{
541 struct igb_adapter *adapter =
542 container_of(tc, struct igb_adapter, cycles);
543 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 544 u64 stamp = 0;
545 int shift = 0;
38c845c7 546
55cac248
AD		 547 /*
548 * The timestamp latches on lowest register read. For the 82580
549 * the lowest register is SYSTIMR instead of SYSTIML. However we never
550 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
551 */
552 if (hw->mac.type == e1000_82580) {
553 stamp = rd32(E1000_SYSTIMR) >> 8;
554 shift = IGB_82580_TSYNC_SHIFT;
555 }
556
c5b9bd5e
AD		 557 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
558 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 559 return stamp;
560}
561
9d5c8243 562/**
c041076a 563 * igb_get_hw_dev - return device
9d5c8243
AK		 564 * used by hardware layer to print debugging information
565 **/
c041076a 566struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 567{
568 struct igb_adapter *adapter = hw->back;
c041076a 569 return adapter->netdev;
9d5c8243 570}
38c845c7 571
9d5c8243
AK		 572/**
573 * igb_init_module - Driver Registration Routine
574 *
575 * igb_init_module is the first routine called when the driver is
576 * loaded. All it does is register with the PCI subsystem.
577 **/
578static int __init igb_init_module(void)
579{
580 int ret;
581 printk(KERN_INFO "%s - version %s\n",
582 igb_driver_string, igb_driver_version);
583
584 printk(KERN_INFO "%s\n", igb_copyright);
585
421e02f0 586#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 587 dca_register_notify(&dca_notifier);
588#endif
bbd98fe4 589 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 590 return ret;
591}
592
593module_init(igb_init_module);
594
595/**
596 * igb_exit_module - Driver Exit Cleanup Routine
597 *
598 * igb_exit_module is called just before the driver is removed
599 * from memory.
600 **/
601static void __exit igb_exit_module(void)
602{
421e02f0 603#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 604 dca_unregister_notify(&dca_notifier);
605#endif
9d5c8243
AK		 606 pci_unregister_driver(&igb_driver);
607}
608
609module_exit(igb_exit_module);
610
26bc19ec
AD		 611#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
612/**
613 * igb_cache_ring_register - Descriptor ring to register mapping
614 * @adapter: board private structure to initialize
615 *
616 * Once we know the feature-set enabled for the device, we'll cache
617 * the register offset the descriptor ring is assigned to.
618 **/
619static void igb_cache_ring_register(struct igb_adapter *adapter)
620{
ee1b9f06 621 int i = 0, j = 0;
047e0030 622 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 623
624 switch (adapter->hw.mac.type) {
625 case e1000_82576:
626 /* The queues are allocated for virtualization such that VF 0
627 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
628 * In order to avoid collision we start at the first free queue
629 * and continue consuming queues in the same sequence
630 */
ee1b9f06 631 if (adapter->vfs_allocated_count) {
a99955fc 632 for (; i < adapter->rss_queues; i++)
3025a446
AD		 633 adapter->rx_ring[i]->reg_idx = rbase_offset +
634 Q_IDX_82576(i);
ee1b9f06 635 }
26bc19ec 636 case e1000_82575:
55cac248 637 case e1000_82580:
d2ba2ed8 638 case e1000_i350:
26bc19ec 639 default:
ee1b9f06 640 for (; i < adapter->num_rx_queues; i++)
3025a446 641 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 642 for (; j < adapter->num_tx_queues; j++)
3025a446 643 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 644 break;
645 }
646}
647
047e0030
AD		 648static void igb_free_queues(struct igb_adapter *adapter)
649{
3025a446 650 int i;
047e0030 651
3025a446
AD		 652 for (i = 0; i < adapter->num_tx_queues; i++) {
653 kfree(adapter->tx_ring[i]);
654 adapter->tx_ring[i] = NULL;
655 }
656 for (i = 0; i < adapter->num_rx_queues; i++) {
657 kfree(adapter->rx_ring[i]);
658 adapter->rx_ring[i] = NULL;
659 }
047e0030
AD		 660 adapter->num_rx_queues = 0;
661 adapter->num_tx_queues = 0;
662}
663
9d5c8243
AK		 664/**
665 * igb_alloc_queues - Allocate memory for all rings
666 * @adapter: board private structure to initialize
667 *
668 * We allocate one ring per queue at run-time since we don't know the
669 * number of queues at compile-time.
670 **/
671static int igb_alloc_queues(struct igb_adapter *adapter)
672{
3025a446 673 struct igb_ring *ring;
9d5c8243
AK		 674 int i;
675
661086df 676 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 677 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
678 if (!ring)
679 goto err;
68fd9910 680 ring->count = adapter->tx_ring_count;
661086df 681 ring->queue_index = i;
59d71989 682 ring->dev = &adapter->pdev->dev;
e694e964 683 ring->netdev = adapter->netdev;
85ad76b2
AD		 684 /* For 82575, context index must be unique per ring. */
685 if (adapter->hw.mac.type == e1000_82575)
686 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 687 adapter->tx_ring[i] = ring;
661086df 688 }
85ad76b2 689
9d5c8243 690 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 691 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
692 if (!ring)
693 goto err;
68fd9910 694 ring->count = adapter->rx_ring_count;
844290e5 695 ring->queue_index = i;
59d71989 696 ring->dev = &adapter->pdev->dev;
e694e964 697 ring->netdev = adapter->netdev;
4c844851 698 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 699 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
700 /* set flag indicating ring supports SCTP checksum offload */
701 if (adapter->hw.mac.type >= e1000_82576)
702 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 703 adapter->rx_ring[i] = ring;
9d5c8243 704 }
26bc19ec
AD		 705
706 igb_cache_ring_register(adapter);
9d5c8243 707
047e0030 708 return 0;
a88f10ec 709
047e0030
AD		 710err:
711 igb_free_queues(adapter);
d1a8c9e1 712
047e0030 713 return -ENOMEM;
a88f10ec
AD		 714}
715
9d5c8243 716#define IGB_N0_QUEUE -1
047e0030 717static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 718{
719 u32 msixbm = 0;
047e0030 720 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 721 struct e1000_hw *hw = &adapter->hw;
2d064c06 722 u32 ivar, index;
047e0030
AD		 723 int rx_queue = IGB_N0_QUEUE;
724 int tx_queue = IGB_N0_QUEUE;
725
726 if (q_vector->rx_ring)
727 rx_queue = q_vector->rx_ring->reg_idx;
728 if (q_vector->tx_ring)
729 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 730
731 switch (hw->mac.type) {
732 case e1000_82575:
9d5c8243
AK		 733 /* The 82575 assigns vectors using a bitmask, which matches the
734 bitmask for the EICR/EIMS/EIMC registers. To assign one
735 or more queues to a vector, we write the appropriate bits
736 into the MSIXBM register for that vector. */
047e0030 737 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 738 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 739 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 740 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 741 if (!adapter->msix_entries && msix_vector == 0)
742 msixbm |= E1000_EIMS_OTHER;
9d5c8243 743 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 744 q_vector->eims_value = msixbm;
2d064c06
AD		 745 break;
746 case e1000_82576:
26bc19ec 747 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 748 Each queue has a single entry in the table to which we write
749 a vector number along with a "valid" bit. Sadly, the layout
750 of the table is somewhat counterintuitive. */
751 if (rx_queue > IGB_N0_QUEUE) {
047e0030 752 index = (rx_queue & 0x7);
2d064c06 753 ivar = array_rd32(E1000_IVAR0, index);
047e0030 754 if (rx_queue < 8) {
26bc19ec
AD		 755 /* vector goes into low byte of register */
756 ivar = ivar & 0xFFFFFF00;
757 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 758 } else {
759 /* vector goes into third byte of register */
760 ivar = ivar & 0xFF00FFFF;
761 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 762 }
2d064c06
AD		 763 array_wr32(E1000_IVAR0, index, ivar);
764 }
765 if (tx_queue > IGB_N0_QUEUE) {
047e0030 766 index = (tx_queue & 0x7);
2d064c06 767 ivar = array_rd32(E1000_IVAR0, index);
047e0030 768 if (tx_queue < 8) {
26bc19ec
AD		 769 /* vector goes into second byte of register */
770 ivar = ivar & 0xFFFF00FF;
771 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 772 } else {
773 /* vector goes into high byte of register */
774 ivar = ivar & 0x00FFFFFF;
775 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 776 }
2d064c06
AD		 777 array_wr32(E1000_IVAR0, index, ivar);
778 }
047e0030 779 q_vector->eims_value = 1 << msix_vector;
2d064c06 780 break;
55cac248 781 case e1000_82580:
d2ba2ed8 782 case e1000_i350:
55cac248
AD		 783 /* 82580 uses the same table-based approach as 82576 but has fewer
784 entries as a result we carry over for queues greater than 4. */
785 if (rx_queue > IGB_N0_QUEUE) {
786 index = (rx_queue >> 1);
787 ivar = array_rd32(E1000_IVAR0, index);
788 if (rx_queue & 0x1) {
789 /* vector goes into third byte of register */
790 ivar = ivar & 0xFF00FFFF;
791 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
792 } else {
793 /* vector goes into low byte of register */
794 ivar = ivar & 0xFFFFFF00;
795 ivar |= msix_vector | E1000_IVAR_VALID;
796 }
797 array_wr32(E1000_IVAR0, index, ivar);
798 }
799 if (tx_queue > IGB_N0_QUEUE) {
800 index = (tx_queue >> 1);
801 ivar = array_rd32(E1000_IVAR0, index);
802 if (tx_queue & 0x1) {
803 /* vector goes into high byte of register */
804 ivar = ivar & 0x00FFFFFF;
805 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
806 } else {
807 /* vector goes into second byte of register */
808 ivar = ivar & 0xFFFF00FF;
809 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
810 }
811 array_wr32(E1000_IVAR0, index, ivar);
812 }
813 q_vector->eims_value = 1 << msix_vector;
814 break;
2d064c06
AD		 815 default:
816 BUG();
817 break;
818 }
26b39276
AD		 819
820 /* add q_vector eims value to global eims_enable_mask */
821 adapter->eims_enable_mask |= q_vector->eims_value;
822
823 /* configure q_vector to set itr on first interrupt */
824 q_vector->set_itr = 1;
9d5c8243
AK		 825}
826
827/**
828 * igb_configure_msix - Configure MSI-X hardware
829 *
830 * igb_configure_msix sets up the hardware to properly
831 * generate MSI-X interrupts.
832 **/
833static void igb_configure_msix(struct igb_adapter *adapter)
834{
835 u32 tmp;
836 int i, vector = 0;
837 struct e1000_hw *hw = &adapter->hw;
838
839 adapter->eims_enable_mask = 0;
9d5c8243
AK		 840
841 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 842 switch (hw->mac.type) {
843 case e1000_82575:
9d5c8243
AK		 844 tmp = rd32(E1000_CTRL_EXT);
845 /* enable MSI-X PBA support*/
846 tmp |= E1000_CTRL_EXT_PBA_CLR;
847
848 /* Auto-Mask interrupts upon ICR read. */
849 tmp |= E1000_CTRL_EXT_EIAME;
850 tmp |= E1000_CTRL_EXT_IRCA;
851
852 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 853
854 /* enable msix_other interrupt */
855 array_wr32(E1000_MSIXBM(0), vector++,
856 E1000_EIMS_OTHER);
844290e5 857 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 858
2d064c06
AD		 859 break;
860
861 case e1000_82576:
55cac248 862 case e1000_82580:
d2ba2ed8 863 case e1000_i350:
047e0030
AD		 864 /* Turn on MSI-X capability first, or our settings
865 * won't stick. And it will take days to debug. */
866 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
867 E1000_GPIE_PBA | E1000_GPIE_EIAME |
868 E1000_GPIE_NSICR);
869
870 /* enable msix_other interrupt */
871 adapter->eims_other = 1 << vector;
2d064c06 872 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 873
047e0030 874 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 875 break;
876 default:
877 /* do nothing, since nothing else supports MSI-X */
878 break;
879 } /* switch (hw->mac.type) */
047e0030
AD		 880
881 adapter->eims_enable_mask |= adapter->eims_other;
882
26b39276
AD		 883 for (i = 0; i < adapter->num_q_vectors; i++)
884 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 885
9d5c8243
AK		 886 wrfl();
887}
888
889/**
890 * igb_request_msix - Initialize MSI-X interrupts
891 *
892 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
893 * kernel.
894 **/
895static int igb_request_msix(struct igb_adapter *adapter)
896{
897 struct net_device *netdev = adapter->netdev;
047e0030 898 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 899 int i, err = 0, vector = 0;
900
047e0030 901 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 902 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 903 if (err)
904 goto out;
905 vector++;
906
907 for (i = 0; i < adapter->num_q_vectors; i++) {
908 struct igb_q_vector *q_vector = adapter->q_vector[i];
909
910 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
911
912 if (q_vector->rx_ring && q_vector->tx_ring)
913 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
914 q_vector->rx_ring->queue_index);
915 else if (q_vector->tx_ring)
916 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
917 q_vector->tx_ring->queue_index);
918 else if (q_vector->rx_ring)
919 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
920 q_vector->rx_ring->queue_index);
9d5c8243 921 else
047e0030
AD		 922 sprintf(q_vector->name, "%s-unused", netdev->name);
923
9d5c8243 924 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 925 igb_msix_ring, 0, q_vector->name,
047e0030 926 q_vector);
9d5c8243
AK		 927 if (err)
928 goto out;
9d5c8243
AK		 929 vector++;
930 }
931
9d5c8243
AK		 932 igb_configure_msix(adapter);
933 return 0;
934out:
935 return err;
936}
937
938static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
939{
940 if (adapter->msix_entries) {
941 pci_disable_msix(adapter->pdev);
942 kfree(adapter->msix_entries);
943 adapter->msix_entries = NULL;
047e0030 944 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 945 pci_disable_msi(adapter->pdev);
047e0030 946 }
9d5c8243
AK		 947}
948
047e0030
AD		 949/**
950 * igb_free_q_vectors - Free memory allocated for interrupt vectors
951 * @adapter: board private structure to initialize
952 *
953 * This function frees the memory allocated to the q_vectors. In addition if
954 * NAPI is enabled it will delete any references to the NAPI struct prior
955 * to freeing the q_vector.
956 **/
957static void igb_free_q_vectors(struct igb_adapter *adapter)
958{
959 int v_idx;
960
961 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
962 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
963 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 964 if (!q_vector)
965 continue;
047e0030
AD		 966 netif_napi_del(&q_vector->napi);
967 kfree(q_vector);
968 }
969 adapter->num_q_vectors = 0;
970}
971
972/**
973 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
974 *
975 * This function resets the device so that it has 0 rx queues, tx queues, and
976 * MSI-X interrupts allocated.
977 */
978static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
979{
980 igb_free_queues(adapter);
981 igb_free_q_vectors(adapter);
982 igb_reset_interrupt_capability(adapter);
983}
9d5c8243
AK		 984
985/**
986 * igb_set_interrupt_capability - set MSI or MSI-X if supported
987 *
988 * Attempt to configure interrupts using the best available
989 * capabilities of the hardware and kernel.
990 **/
21adef3e 991static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 992{
993 int err;
994 int numvecs, i;
995
83b7180d 996 /* Number of supported queues. */
a99955fc 997 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 998 if (adapter->vfs_allocated_count)
999 adapter->num_tx_queues = 1;
1000 else
1001 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1002
047e0030
AD		 1003 /* start with one vector for every rx queue */
1004 numvecs = adapter->num_rx_queues;
1005
3ad2f3fb 1006 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1007 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1008 numvecs += adapter->num_tx_queues;
047e0030
AD		 1009
1010 /* store the number of vectors reserved for queues */
1011 adapter->num_q_vectors = numvecs;
1012
1013 /* add 1 vector for link status interrupts */
1014 numvecs++;
9d5c8243
AK		 1015 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1016 GFP_KERNEL);
1017 if (!adapter->msix_entries)
1018 goto msi_only;
1019
1020 for (i = 0; i < numvecs; i++)
1021 adapter->msix_entries[i].entry = i;
1022
1023 err = pci_enable_msix(adapter->pdev,
1024 adapter->msix_entries,
1025 numvecs);
1026 if (err == 0)
34a20e89 1027 goto out;
9d5c8243
AK		 1028
1029 igb_reset_interrupt_capability(adapter);
1030
1031 /* If we can't do MSI-X, try MSI */
1032msi_only:
2a3abf6d
AD		 1033#ifdef CONFIG_PCI_IOV
1034 /* disable SR-IOV for non MSI-X configurations */
1035 if (adapter->vf_data) {
1036 struct e1000_hw *hw = &adapter->hw;
1037 /* disable iov and allow time for transactions to clear */
1038 pci_disable_sriov(adapter->pdev);
1039 msleep(500);
1040
1041 kfree(adapter->vf_data);
1042 adapter->vf_data = NULL;
1043 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1044 msleep(100);
1045 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1046 }
1047#endif
4fc82adf 1048 adapter->vfs_allocated_count = 0;
a99955fc 1049 adapter->rss_queues = 1;
4fc82adf 1050 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1051 adapter->num_rx_queues = 1;
661086df 1052 adapter->num_tx_queues = 1;
047e0030 1053 adapter->num_q_vectors = 1;
9d5c8243 1054 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1055 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1056out:
21adef3e
BH		 1057 /* Notify the stack of the (possibly) reduced queue counts. */
1058 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1059 return netif_set_real_num_rx_queues(adapter->netdev,
1060 adapter->num_rx_queues);
9d5c8243
AK		 1061}
1062
047e0030
AD		 1063/**
1064 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1065 * @adapter: board private structure to initialize
1066 *
1067 * We allocate one q_vector per queue interrupt. If allocation fails we
1068 * return -ENOMEM.
1069 **/
1070static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1071{
1072 struct igb_q_vector *q_vector;
1073 struct e1000_hw *hw = &adapter->hw;
1074 int v_idx;
1075
1076 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1077 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1078 if (!q_vector)
1079 goto err_out;
1080 q_vector->adapter = adapter;
047e0030
AD		 1081 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1082 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1083 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1084 adapter->q_vector[v_idx] = q_vector;
1085 }
1086 return 0;
1087
1088err_out:
fe0592b4 1089 igb_free_q_vectors(adapter);
047e0030
AD		 1090 return -ENOMEM;
1091}
1092
1093static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1094 int ring_idx, int v_idx)
1095{
3025a446 1096 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1097
3025a446 1098 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1099 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1100 q_vector->itr_val = adapter->rx_itr_setting;
1101 if (q_vector->itr_val && q_vector->itr_val <= 3)
1102 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1103}
1104
1105static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107{
3025a446 1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1109
3025a446 1110 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1111 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1112 q_vector->itr_val = adapter->tx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1115}
1116
1117/**
1118 * igb_map_ring_to_vector - maps allocated queues to vectors
1119 *
1120 * This function maps the recently allocated queues to vectors.
1121 **/
1122static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1123{
1124 int i;
1125 int v_idx = 0;
1126
1127 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1128 (adapter->num_q_vectors < adapter->num_tx_queues))
1129 return -ENOMEM;
1130
1131 if (adapter->num_q_vectors >=
1132 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1133 for (i = 0; i < adapter->num_rx_queues; i++)
1134 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1135 for (i = 0; i < adapter->num_tx_queues; i++)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1137 } else {
1138 for (i = 0; i < adapter->num_rx_queues; i++) {
1139 if (i < adapter->num_tx_queues)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1141 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1142 }
1143 for (; i < adapter->num_tx_queues; i++)
1144 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1145 }
1146 return 0;
1147}
1148
1149/**
1150 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1151 *
1152 * This function initializes the interrupts and allocates all of the queues.
1153 **/
1154static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1155{
1156 struct pci_dev *pdev = adapter->pdev;
1157 int err;
1158
21adef3e
BH		 1159 err = igb_set_interrupt_capability(adapter);
1160 if (err)
1161 return err;
047e0030
AD		 1162
1163 err = igb_alloc_q_vectors(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1166 goto err_alloc_q_vectors;
1167 }
1168
1169 err = igb_alloc_queues(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1172 goto err_alloc_queues;
1173 }
1174
1175 err = igb_map_ring_to_vector(adapter);
1176 if (err) {
1177 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1178 goto err_map_queues;
1179 }
1180
1181
1182 return 0;
1183err_map_queues:
1184 igb_free_queues(adapter);
1185err_alloc_queues:
1186 igb_free_q_vectors(adapter);
1187err_alloc_q_vectors:
1188 igb_reset_interrupt_capability(adapter);
1189 return err;
1190}
1191
9d5c8243
AK		 1192/**
1193 * igb_request_irq - initialize interrupts
1194 *
1195 * Attempts to configure interrupts using the best available
1196 * capabilities of the hardware and kernel.
1197 **/
1198static int igb_request_irq(struct igb_adapter *adapter)
1199{
1200 struct net_device *netdev = adapter->netdev;
047e0030 1201 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1202 int err = 0;
1203
1204 if (adapter->msix_entries) {
1205 err = igb_request_msix(adapter);
844290e5 1206 if (!err)
9d5c8243 1207 goto request_done;
9d5c8243 1208 /* fall back to MSI */
047e0030 1209 igb_clear_interrupt_scheme(adapter);
9d5c8243 1210 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1211 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1212 igb_free_all_tx_resources(adapter);
1213 igb_free_all_rx_resources(adapter);
047e0030 1214 adapter->num_tx_queues = 1;
9d5c8243 1215 adapter->num_rx_queues = 1;
047e0030
AD		 1216 adapter->num_q_vectors = 1;
1217 err = igb_alloc_q_vectors(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for vectors\n");
1221 goto request_done;
1222 }
1223 err = igb_alloc_queues(adapter);
1224 if (err) {
1225 dev_err(&pdev->dev,
1226 "Unable to allocate memory for queues\n");
1227 igb_free_q_vectors(adapter);
1228 goto request_done;
1229 }
1230 igb_setup_all_tx_resources(adapter);
1231 igb_setup_all_rx_resources(adapter);
844290e5 1232 } else {
feeb2721 1233 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1234 }
844290e5 1235
7dfc16fa 1236 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1237 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1238 netdev->name, adapter);
9d5c8243
AK		 1239 if (!err)
1240 goto request_done;
047e0030 1241
9d5c8243
AK		 1242 /* fall back to legacy interrupts */
1243 igb_reset_interrupt_capability(adapter);
7dfc16fa 1244 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1245 }
1246
a0607fd3 1247 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1248 netdev->name, adapter);
9d5c8243 1249
6cb5e577 1250 if (err)
9d5c8243
AK		 1251 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1252 err);
9d5c8243
AK		 1253
1254request_done:
1255 return err;
1256}
1257
1258static void igb_free_irq(struct igb_adapter *adapter)
1259{
9d5c8243
AK		 1260 if (adapter->msix_entries) {
1261 int vector = 0, i;
1262
047e0030 1263 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1264
047e0030
AD		 1265 for (i = 0; i < adapter->num_q_vectors; i++) {
1266 struct igb_q_vector *q_vector = adapter->q_vector[i];
1267 free_irq(adapter->msix_entries[vector++].vector,
1268 q_vector);
1269 }
1270 } else {
1271 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1272 }
9d5c8243
AK		 1273}
1274
1275/**
1276 * igb_irq_disable - Mask off interrupt generation on the NIC
1277 * @adapter: board private structure
1278 **/
1279static void igb_irq_disable(struct igb_adapter *adapter)
1280{
1281 struct e1000_hw *hw = &adapter->hw;
1282
25568a53
AD		 1283 /*
1284 * we need to be careful when disabling interrupts. The VFs are also
1285 * mapped into these registers and so clearing the bits can cause
1286 * issues on the VF drivers so we only need to clear what we set
1287 */
9d5c8243 1288 if (adapter->msix_entries) {
2dfd1212
AD		 1289 u32 regval = rd32(E1000_EIAM);
1290 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1291 wr32(E1000_EIMC, adapter->eims_enable_mask);
1292 regval = rd32(E1000_EIAC);
1293 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1294 }
844290e5
PW		 1295
1296 wr32(E1000_IAM, 0);
9d5c8243
AK		 1297 wr32(E1000_IMC, ~0);
1298 wrfl();
81a61859
ET		 1299 if (adapter->msix_entries) {
1300 int i;
1301 for (i = 0; i < adapter->num_q_vectors; i++)
1302 synchronize_irq(adapter->msix_entries[i].vector);
1303 } else {
1304 synchronize_irq(adapter->pdev->irq);
1305 }
9d5c8243
AK		 1306}
1307
1308/**
1309 * igb_irq_enable - Enable default interrupt generation settings
1310 * @adapter: board private structure
1311 **/
1312static void igb_irq_enable(struct igb_adapter *adapter)
1313{
1314 struct e1000_hw *hw = &adapter->hw;
1315
1316 if (adapter->msix_entries) {
25568a53 1317 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1318 u32 regval = rd32(E1000_EIAC);
1319 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1320 regval = rd32(E1000_EIAM);
1321 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1322 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1323 if (adapter->vfs_allocated_count) {
4ae196df 1324 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1325 ims |= E1000_IMS_VMMB;
1326 }
55cac248
AD		 1327 if (adapter->hw.mac.type == e1000_82580)
1328 ims |= E1000_IMS_DRSTA;
1329
25568a53 1330 wr32(E1000_IMS, ims);
844290e5 1331 } else {
55cac248
AD		 1332 wr32(E1000_IMS, IMS_ENABLE_MASK |
1333 E1000_IMS_DRSTA);
1334 wr32(E1000_IAM, IMS_ENABLE_MASK |
1335 E1000_IMS_DRSTA);
844290e5 1336 }
9d5c8243
AK		 1337}
1338
1339static void igb_update_mng_vlan(struct igb_adapter *adapter)
1340{
51466239 1341 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1342 u16 vid = adapter->hw.mng_cookie.vlan_id;
1343 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1344
1345 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1346 /* add VID to filter table */
1347 igb_vfta_set(hw, vid, true);
1348 adapter->mng_vlan_id = vid;
1349 } else {
1350 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1351 }
1352
1353 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1354 (vid != old_vid) &&
1355 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1356 /* remove VID from filter table */
1357 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1358 }
1359}
1360
1361/**
1362 * igb_release_hw_control - release control of the h/w to f/w
1363 * @adapter: address of board private structure
1364 *
1365 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1366 * For ASF and Pass Through versions of f/w this means that the
1367 * driver is no longer loaded.
1368 *
1369 **/
1370static void igb_release_hw_control(struct igb_adapter *adapter)
1371{
1372 struct e1000_hw *hw = &adapter->hw;
1373 u32 ctrl_ext;
1374
1375 /* Let firmware take over control of h/w */
1376 ctrl_ext = rd32(E1000_CTRL_EXT);
1377 wr32(E1000_CTRL_EXT,
1378 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1379}
1380
9d5c8243
AK		 1381/**
1382 * igb_get_hw_control - get control of the h/w from f/w
1383 * @adapter: address of board private structure
1384 *
1385 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1386 * For ASF and Pass Through versions of f/w this means that
1387 * the driver is loaded.
1388 *
1389 **/
1390static void igb_get_hw_control(struct igb_adapter *adapter)
1391{
1392 struct e1000_hw *hw = &adapter->hw;
1393 u32 ctrl_ext;
1394
1395 /* Let firmware know the driver has taken over */
1396 ctrl_ext = rd32(E1000_CTRL_EXT);
1397 wr32(E1000_CTRL_EXT,
1398 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1399}
1400
9d5c8243
AK		 1401/**
1402 * igb_configure - configure the hardware for RX and TX
1403 * @adapter: private board structure
1404 **/
1405static void igb_configure(struct igb_adapter *adapter)
1406{
1407 struct net_device *netdev = adapter->netdev;
1408 int i;
1409
1410 igb_get_hw_control(adapter);
ff41f8dc 1411 igb_set_rx_mode(netdev);
9d5c8243
AK		 1412
1413 igb_restore_vlan(adapter);
9d5c8243 1414
85b430b4 1415 igb_setup_tctl(adapter);
06cf2666 1416 igb_setup_mrqc(adapter);
9d5c8243 1417 igb_setup_rctl(adapter);
85b430b4
AD		 1418
1419 igb_configure_tx(adapter);
9d5c8243 1420 igb_configure_rx(adapter);
662d7205
AD		 1421
1422 igb_rx_fifo_flush_82575(&adapter->hw);
1423
c493ea45 1424 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1425 * at least 1 descriptor unused to make sure
1426 * next_to_use != next_to_clean */
1427 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1428 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1429 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1430 }
9d5c8243
AK		 1431}
1432
88a268c1
NN		 1433/**
1434 * igb_power_up_link - Power up the phy/serdes link
1435 * @adapter: address of board private structure
1436 **/
1437void igb_power_up_link(struct igb_adapter *adapter)
1438{
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_up_phy_copper(&adapter->hw);
1441 else
1442 igb_power_up_serdes_link_82575(&adapter->hw);
1443}
1444
1445/**
1446 * igb_power_down_link - Power down the phy/serdes link
1447 * @adapter: address of board private structure
1448 */
1449static void igb_power_down_link(struct igb_adapter *adapter)
1450{
1451 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1452 igb_power_down_phy_copper_82575(&adapter->hw);
1453 else
1454 igb_shutdown_serdes_link_82575(&adapter->hw);
1455}
9d5c8243
AK		 1456
1457/**
1458 * igb_up - Open the interface and prepare it to handle traffic
1459 * @adapter: board private structure
1460 **/
9d5c8243
AK		 1461int igb_up(struct igb_adapter *adapter)
1462{
1463 struct e1000_hw *hw = &adapter->hw;
1464 int i;
1465
1466 /* hardware has been reset, we need to reload some things */
1467 igb_configure(adapter);
1468
1469 clear_bit(__IGB_DOWN, &adapter->state);
1470
047e0030
AD		 1471 for (i = 0; i < adapter->num_q_vectors; i++) {
1472 struct igb_q_vector *q_vector = adapter->q_vector[i];
1473 napi_enable(&q_vector->napi);
1474 }
844290e5 1475 if (adapter->msix_entries)
9d5c8243 1476 igb_configure_msix(adapter);
feeb2721
AD		 1477 else
1478 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1479
1480 /* Clear any pending interrupts. */
1481 rd32(E1000_ICR);
1482 igb_irq_enable(adapter);
1483
d4960307
AD		 1484 /* notify VFs that reset has been completed */
1485 if (adapter->vfs_allocated_count) {
1486 u32 reg_data = rd32(E1000_CTRL_EXT);
1487 reg_data |= E1000_CTRL_EXT_PFRSTD;
1488 wr32(E1000_CTRL_EXT, reg_data);
1489 }
1490
4cb9be7a
JB		 1491 netif_tx_start_all_queues(adapter->netdev);
1492
25568a53
AD		 1493 /* start the watchdog. */
1494 hw->mac.get_link_status = 1;
1495 schedule_work(&adapter->watchdog_task);
1496
9d5c8243
AK		 1497 return 0;
1498}
1499
1500void igb_down(struct igb_adapter *adapter)
1501{
9d5c8243 1502 struct net_device *netdev = adapter->netdev;
330a6d6a 1503 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1504 u32 tctl, rctl;
1505 int i;
1506
1507 /* signal that we're down so the interrupt handler does not
1508 * reschedule our watchdog timer */
1509 set_bit(__IGB_DOWN, &adapter->state);
1510
1511 /* disable receives in the hardware */
1512 rctl = rd32(E1000_RCTL);
1513 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1514 /* flush and sleep below */
1515
fd2ea0a7 1516 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1517
1518 /* disable transmits in the hardware */
1519 tctl = rd32(E1000_TCTL);
1520 tctl &= ~E1000_TCTL_EN;
1521 wr32(E1000_TCTL, tctl);
1522 /* flush both disables and wait for them to finish */
1523 wrfl();
1524 msleep(10);
1525
047e0030
AD		 1526 for (i = 0; i < adapter->num_q_vectors; i++) {
1527 struct igb_q_vector *q_vector = adapter->q_vector[i];
1528 napi_disable(&q_vector->napi);
1529 }
9d5c8243 1530
9d5c8243
AK		 1531 igb_irq_disable(adapter);
1532
1533 del_timer_sync(&adapter->watchdog_timer);
1534 del_timer_sync(&adapter->phy_info_timer);
1535
9d5c8243 1536 netif_carrier_off(netdev);
04fe6358
AD		 1537
1538 /* record the stats before reset*/
1539 igb_update_stats(adapter);
1540
9d5c8243
AK		 1541 adapter->link_speed = 0;
1542 adapter->link_duplex = 0;
1543
3023682e
JK		 1544 if (!pci_channel_offline(adapter->pdev))
1545 igb_reset(adapter);
9d5c8243
AK		 1546 igb_clean_all_tx_rings(adapter);
1547 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1548#ifdef CONFIG_IGB_DCA
1549
1550 /* since we reset the hardware DCA settings were cleared */
1551 igb_setup_dca(adapter);
1552#endif
9d5c8243
AK		 1553}
1554
1555void igb_reinit_locked(struct igb_adapter *adapter)
1556{
1557 WARN_ON(in_interrupt());
1558 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1559 msleep(1);
1560 igb_down(adapter);
1561 igb_up(adapter);
1562 clear_bit(__IGB_RESETTING, &adapter->state);
1563}
1564
1565void igb_reset(struct igb_adapter *adapter)
1566{
090b1795 1567 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1568 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1569 struct e1000_mac_info *mac = &hw->mac;
1570 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1571 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1572 u16 hwm;
1573
1574 /* Repartition Pba for greater than 9k mtu
1575 * To take effect CTRL.RST is required.
1576 */
fa4dfae0 1577 switch (mac->type) {
d2ba2ed8 1578 case e1000_i350:
55cac248
AD		 1579 case e1000_82580:
1580 pba = rd32(E1000_RXPBS);
1581 pba = igb_rxpbs_adjust_82580(pba);
1582 break;
fa4dfae0 1583 case e1000_82576:
d249be54
AD		 1584 pba = rd32(E1000_RXPBS);
1585 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1586 break;
1587 case e1000_82575:
1588 default:
1589 pba = E1000_PBA_34K;
1590 break;
2d064c06 1591 }
9d5c8243 1592
2d064c06
AD		 1593 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1594 (mac->type < e1000_82576)) {
9d5c8243
AK		 1595 /* adjust PBA for jumbo frames */
1596 wr32(E1000_PBA, pba);
1597
1598 /* To maintain wire speed transmits, the Tx FIFO should be
1599 * large enough to accommodate two full transmit packets,
1600 * rounded up to the next 1KB and expressed in KB. Likewise,
1601 * the Rx FIFO should be large enough to accommodate at least
1602 * one full receive packet and is similarly rounded up and
1603 * expressed in KB. */
1604 pba = rd32(E1000_PBA);
1605 /* upper 16 bits has Tx packet buffer allocation size in KB */
1606 tx_space = pba >> 16;
1607 /* lower 16 bits has Rx packet buffer allocation size in KB */
1608 pba &= 0xffff;
1609 /* the tx fifo also stores 16 bytes of information about the tx
1610 * but don't include ethernet FCS because hardware appends it */
1611 min_tx_space = (adapter->max_frame_size +
85e8d004 1612 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1613 ETH_FCS_LEN) * 2;
1614 min_tx_space = ALIGN(min_tx_space, 1024);
1615 min_tx_space >>= 10;
1616 /* software strips receive CRC, so leave room for it */
1617 min_rx_space = adapter->max_frame_size;
1618 min_rx_space = ALIGN(min_rx_space, 1024);
1619 min_rx_space >>= 10;
1620
1621 /* If current Tx allocation is less than the min Tx FIFO size,
1622 * and the min Tx FIFO size is less than the current Rx FIFO
1623 * allocation, take space away from current Rx allocation */
1624 if (tx_space < min_tx_space &&
1625 ((min_tx_space - tx_space) < pba)) {
1626 pba = pba - (min_tx_space - tx_space);
1627
1628 /* if short on rx space, rx wins and must trump tx
1629 * adjustment */
1630 if (pba < min_rx_space)
1631 pba = min_rx_space;
1632 }
2d064c06 1633 wr32(E1000_PBA, pba);
9d5c8243 1634 }
9d5c8243
AK		 1635
1636 /* flow control settings */
1637 /* The high water mark must be low enough to fit one full frame
1638 * (or the size used for early receive) above it in the Rx FIFO.
1639 * Set it to the lower of:
1640 * - 90% of the Rx FIFO size, or
1641 * - the full Rx FIFO size minus one full frame */
1642 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1643 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1644
d405ea3e
AD		 1645 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1646 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1647 fc->pause_time = 0xFFFF;
1648 fc->send_xon = 1;
0cce119a 1649 fc->current_mode = fc->requested_mode;
9d5c8243 1650
4ae196df
AD		 1651 /* disable receive for all VFs and wait one second */
1652 if (adapter->vfs_allocated_count) {
1653 int i;
1654 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1655 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1656
1657 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1658 igb_ping_all_vfs(adapter);
4ae196df
AD		 1659
1660 /* disable transmits and receives */
1661 wr32(E1000_VFRE, 0);
1662 wr32(E1000_VFTE, 0);
1663 }
1664
9d5c8243 1665 /* Allow time for pending master requests to run */
330a6d6a 1666 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1667 wr32(E1000_WUC, 0);
1668
330a6d6a 1669 if (hw->mac.ops.init_hw(hw))
090b1795 1670 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1671
55cac248
AD		 1672 if (hw->mac.type == e1000_82580) {
1673 u32 reg = rd32(E1000_PCIEMISC);
1674 wr32(E1000_PCIEMISC,
1675 reg & ~E1000_PCIEMISC_LX_DECISION);
1676 }
88a268c1
NN		 1677 if (!netif_running(adapter->netdev))
1678 igb_power_down_link(adapter);
1679
9d5c8243
AK		 1680 igb_update_mng_vlan(adapter);
1681
1682 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1683 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1684
330a6d6a 1685 igb_get_phy_info(hw);
9d5c8243
AK		 1686}
1687
2e5c6922 1688static const struct net_device_ops igb_netdev_ops = {
559e9c49 1689 .ndo_open = igb_open,
2e5c6922 1690 .ndo_stop = igb_close,
00829823 1691 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1692 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1693 .ndo_set_rx_mode = igb_set_rx_mode,
1694 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1695 .ndo_set_mac_address = igb_set_mac,
1696 .ndo_change_mtu = igb_change_mtu,
1697 .ndo_do_ioctl = igb_ioctl,
1698 .ndo_tx_timeout = igb_tx_timeout,
1699 .ndo_validate_addr = eth_validate_addr,
1700 .ndo_vlan_rx_register = igb_vlan_rx_register,
1701 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1702 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1703 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1704 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1705 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1706 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1707#ifdef CONFIG_NET_POLL_CONTROLLER
1708 .ndo_poll_controller = igb_netpoll,
1709#endif
1710};
1711
9d5c8243
AK		 1712/**
1713 * igb_probe - Device Initialization Routine
1714 * @pdev: PCI device information struct
1715 * @ent: entry in igb_pci_tbl
1716 *
1717 * Returns 0 on success, negative on failure
1718 *
1719 * igb_probe initializes an adapter identified by a pci_dev structure.
1720 * The OS initialization, configuring of the adapter private structure,
1721 * and a hardware reset occur.
1722 **/
1723static int __devinit igb_probe(struct pci_dev *pdev,
1724 const struct pci_device_id *ent)
1725{
1726 struct net_device *netdev;
1727 struct igb_adapter *adapter;
1728 struct e1000_hw *hw;
4337e993
AD		 1729 u16 eeprom_data = 0;
1730 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1731 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1732 unsigned long mmio_start, mmio_len;
2d6a5e95 1733 int err, pci_using_dac;
9d5c8243
AK		 1734 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1735 u32 part_num;
1736
bded64a7
AG		 1737 /* Catch broken hardware that put the wrong VF device ID in
1738 * the PCIe SR-IOV capability.
1739 */
1740 if (pdev->is_virtfn) {
1741 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1742 pci_name(pdev), pdev->vendor, pdev->device);
1743 return -EINVAL;
1744 }
1745
aed5dec3 1746 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1747 if (err)
1748 return err;
1749
1750 pci_using_dac = 0;
59d71989 1751 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1752 if (!err) {
59d71989 1753 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1754 if (!err)
1755 pci_using_dac = 1;
1756 } else {
59d71989 1757 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1758 if (err) {
59d71989 1759 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1760 if (err) {
1761 dev_err(&pdev->dev, "No usable DMA "
1762 "configuration, aborting\n");
1763 goto err_dma;
1764 }
1765 }
1766 }
1767
aed5dec3
AD		 1768 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1769 IORESOURCE_MEM),
1770 igb_driver_name);
9d5c8243
AK		 1771 if (err)
1772 goto err_pci_reg;
1773
19d5afd4 1774 pci_enable_pcie_error_reporting(pdev);
40a914fa 1775
9d5c8243 1776 pci_set_master(pdev);
c682fc23 1777 pci_save_state(pdev);
9d5c8243
AK		 1778
1779 err = -ENOMEM;
1bfaf07b
AD		 1780 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1781 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1782 if (!netdev)
1783 goto err_alloc_etherdev;
1784
1785 SET_NETDEV_DEV(netdev, &pdev->dev);
1786
1787 pci_set_drvdata(pdev, netdev);
1788 adapter = netdev_priv(netdev);
1789 adapter->netdev = netdev;
1790 adapter->pdev = pdev;
1791 hw = &adapter->hw;
1792 hw->back = adapter;
1793 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1794
1795 mmio_start = pci_resource_start(pdev, 0);
1796 mmio_len = pci_resource_len(pdev, 0);
1797
1798 err = -EIO;
28b0759c
AD		 1799 hw->hw_addr = ioremap(mmio_start, mmio_len);
1800 if (!hw->hw_addr)
9d5c8243
AK		 1801 goto err_ioremap;
1802
2e5c6922 1803 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1804 igb_set_ethtool_ops(netdev);
9d5c8243 1805 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1806
1807 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1808
1809 netdev->mem_start = mmio_start;
1810 netdev->mem_end = mmio_start + mmio_len;
1811
9d5c8243
AK		 1812 /* PCI config space info */
1813 hw->vendor_id = pdev->vendor;
1814 hw->device_id = pdev->device;
1815 hw->revision_id = pdev->revision;
1816 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1817 hw->subsystem_device_id = pdev->subsystem_device;
1818
9d5c8243
AK		 1819 /* Copy the default MAC, PHY and NVM function pointers */
1820 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1821 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1822 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1823 /* Initialize skew-specific constants */
1824 err = ei->get_invariants(hw);
1825 if (err)
450c87c8 1826 goto err_sw_init;
9d5c8243 1827
450c87c8 1828 /* setup the private structure */
9d5c8243
AK		 1829 err = igb_sw_init(adapter);
1830 if (err)
1831 goto err_sw_init;
1832
1833 igb_get_bus_info_pcie(hw);
1834
1835 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1836
1837 /* Copper options */
1838 if (hw->phy.media_type == e1000_media_type_copper) {
1839 hw->phy.mdix = AUTO_ALL_MODES;
1840 hw->phy.disable_polarity_correction = false;
1841 hw->phy.ms_type = e1000_ms_hw_default;
1842 }
1843
1844 if (igb_check_reset_block(hw))
1845 dev_info(&pdev->dev,
1846 "PHY reset is blocked due to SOL/IDER session.\n");
1847
1848 netdev->features = NETIF_F_SG |
7d8eb29e 1849 NETIF_F_IP_CSUM |
9d5c8243
AK		 1850 NETIF_F_HW_VLAN_TX |
1851 NETIF_F_HW_VLAN_RX |
1852 NETIF_F_HW_VLAN_FILTER;
1853
7d8eb29e 1854 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1855 netdev->features |= NETIF_F_TSO;
9d5c8243 1856 netdev->features |= NETIF_F_TSO6;
5c0999b7 1857 netdev->features |= NETIF_F_GRO;
d3352520 1858
48f29ffc
JK		 1859 netdev->vlan_features |= NETIF_F_TSO;
1860 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1861 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1862 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1863 netdev->vlan_features |= NETIF_F_SG;
1864
7b872a55 1865 if (pci_using_dac) {
9d5c8243 1866 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1867 netdev->vlan_features |= NETIF_F_HIGHDMA;
1868 }
9d5c8243 1869
5b043fb0 1870 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1871 netdev->features |= NETIF_F_SCTP_CSUM;
1872
330a6d6a 1873 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1874
1875 /* before reading the NVM, reset the controller to put the device in a
1876 * known good starting state */
1877 hw->mac.ops.reset_hw(hw);
1878
1879 /* make sure the NVM is good */
1880 if (igb_validate_nvm_checksum(hw) < 0) {
1881 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1882 err = -EIO;
1883 goto err_eeprom;
1884 }
1885
1886 /* copy the MAC address out of the NVM */
1887 if (hw->mac.ops.read_mac_addr(hw))
1888 dev_err(&pdev->dev, "NVM Read Error\n");
1889
1890 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1891 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1892
1893 if (!is_valid_ether_addr(netdev->perm_addr)) {
1894 dev_err(&pdev->dev, "Invalid MAC Address\n");
1895 err = -EIO;
1896 goto err_eeprom;
1897 }
1898
c061b18d 1899 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 1900 (unsigned long) adapter);
c061b18d 1901 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 1902 (unsigned long) adapter);
9d5c8243
AK		 1903
1904 INIT_WORK(&adapter->reset_task, igb_reset_task);
1905 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1906
450c87c8 1907 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1908 adapter->fc_autoneg = true;
1909 hw->mac.autoneg = true;
1910 hw->phy.autoneg_advertised = 0x2f;
1911
0cce119a
AD		 1912 hw->fc.requested_mode = e1000_fc_default;
1913 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1914
9d5c8243
AK		 1915 igb_validate_mdi_setting(hw);
1916
9d5c8243
AK		 1917 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1918 * enable the ACPI Magic Packet filter
1919 */
1920
a2cf8b6c 1921 if (hw->bus.func == 0)
312c75ae 1922 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1923 else if (hw->mac.type == e1000_82580)
1924 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1925 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1926 &eeprom_data);
a2cf8b6c
AD		 1927 else if (hw->bus.func == 1)
1928 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1929
1930 if (eeprom_data & eeprom_apme_mask)
1931 adapter->eeprom_wol |= E1000_WUFC_MAG;
1932
1933 /* now that we have the eeprom settings, apply the special cases where
1934 * the eeprom may be wrong or the board simply won't support wake on
1935 * lan on a particular port */
1936 switch (pdev->device) {
1937 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1938 adapter->eeprom_wol = 0;
1939 break;
1940 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1941 case E1000_DEV_ID_82576_FIBER:
1942 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1943 /* Wake events only supported on port A for dual fiber
1944 * regardless of eeprom setting */
1945 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1946 adapter->eeprom_wol = 0;
1947 break;
c8ea5ea9 1948 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1949 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1950 /* if quad port adapter, disable WoL on all but port A */
1951 if (global_quad_port_a != 0)
1952 adapter->eeprom_wol = 0;
1953 else
1954 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1955 /* Reset for multiple quad port adapters */
1956 if (++global_quad_port_a == 4)
1957 global_quad_port_a = 0;
1958 break;
9d5c8243
AK		 1959 }
1960
1961 /* initialize the wol settings based on the eeprom settings */
1962 adapter->wol = adapter->eeprom_wol;
e1b86d84 1963 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1964
1965 /* reset the hardware with the new settings */
1966 igb_reset(adapter);
1967
1968 /* let the f/w know that the h/w is now under the control of the
1969 * driver. */
1970 igb_get_hw_control(adapter);
1971
9d5c8243
AK		 1972 strcpy(netdev->name, "eth%d");
1973 err = register_netdev(netdev);
1974 if (err)
1975 goto err_register;
1976
b168dfc5
JB		 1977 /* carrier off reporting is important to ethtool even BEFORE open */
1978 netif_carrier_off(netdev);
1979
421e02f0 1980#ifdef CONFIG_IGB_DCA
bbd98fe4 1981 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1982 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1983 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1984 igb_setup_dca(adapter);
1985 }
fe4506b6 1986
38c845c7 1987#endif
9d5c8243
AK		 1988 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1989 /* print bus type/speed/width info */
7c510e4b 1990 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1991 netdev->name,
559e9c49 1992 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 1993 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 1994 "unknown"),
59c3de89
AD		 1995 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1996 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1997 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1998 "unknown"),
7c510e4b 1999 netdev->dev_addr);
9d5c8243
AK		 2000
2001 igb_read_part_num(hw, &part_num);
2002 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
2003 (part_num >> 8), (part_num & 0xff));
2004
2005 dev_info(&pdev->dev,
2006 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2007 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2008 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 2009 adapter->num_rx_queues, adapter->num_tx_queues);
2010
9d5c8243
AK		 2011 return 0;
2012
2013err_register:
2014 igb_release_hw_control(adapter);
2015err_eeprom:
2016 if (!igb_check_reset_block(hw))
f5f4cf08 2017 igb_reset_phy(hw);
9d5c8243
AK		 2018
2019 if (hw->flash_address)
2020 iounmap(hw->flash_address);
9d5c8243 2021err_sw_init:
047e0030 2022 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2023 iounmap(hw->hw_addr);
2024err_ioremap:
2025 free_netdev(netdev);
2026err_alloc_etherdev:
559e9c49
AD		 2027 pci_release_selected_regions(pdev,
2028 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2029err_pci_reg:
2030err_dma:
2031 pci_disable_device(pdev);
2032 return err;
2033}
2034
2035/**
2036 * igb_remove - Device Removal Routine
2037 * @pdev: PCI device information struct
2038 *
2039 * igb_remove is called by the PCI subsystem to alert the driver
2040 * that it should release a PCI device. The could be caused by a
2041 * Hot-Plug event, or because the driver is going to be removed from
2042 * memory.
2043 **/
2044static void __devexit igb_remove(struct pci_dev *pdev)
2045{
2046 struct net_device *netdev = pci_get_drvdata(pdev);
2047 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2048 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2049
2050 /* flush_scheduled work may reschedule our watchdog task, so
2051 * explicitly disable watchdog tasks from being rescheduled */
2052 set_bit(__IGB_DOWN, &adapter->state);
2053 del_timer_sync(&adapter->watchdog_timer);
2054 del_timer_sync(&adapter->phy_info_timer);
2055
2056 flush_scheduled_work();
2057
421e02f0 2058#ifdef CONFIG_IGB_DCA
7dfc16fa 2059 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2060 dev_info(&pdev->dev, "DCA disabled\n");
2061 dca_remove_requester(&pdev->dev);
7dfc16fa 2062 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2063 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2064 }
2065#endif
2066
9d5c8243
AK		 2067 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2068 * would have already happened in close and is redundant. */
2069 igb_release_hw_control(adapter);
2070
2071 unregister_netdev(netdev);
2072
047e0030 2073 igb_clear_interrupt_scheme(adapter);
9d5c8243 2074
37680117
AD		 2075#ifdef CONFIG_PCI_IOV
2076 /* reclaim resources allocated to VFs */
2077 if (adapter->vf_data) {
2078 /* disable iov and allow time for transactions to clear */
2079 pci_disable_sriov(pdev);
2080 msleep(500);
2081
2082 kfree(adapter->vf_data);
2083 adapter->vf_data = NULL;
2084 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2085 msleep(100);
2086 dev_info(&pdev->dev, "IOV Disabled\n");
2087 }
2088#endif
559e9c49 2089
28b0759c
AD		 2090 iounmap(hw->hw_addr);
2091 if (hw->flash_address)
2092 iounmap(hw->flash_address);
559e9c49
AD		 2093 pci_release_selected_regions(pdev,
2094 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2095
2096 free_netdev(netdev);
2097
19d5afd4 2098 pci_disable_pcie_error_reporting(pdev);
40a914fa 2099
9d5c8243
AK		 2100 pci_disable_device(pdev);
2101}
2102
a6b623e0
AD		 2103/**
2104 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2105 * @adapter: board private structure to initialize
2106 *
2107 * This function initializes the vf specific data storage and then attempts to
2108 * allocate the VFs. The reason for ordering it this way is because it is much
2109 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2110 * the memory for the VFs.
2111 **/
2112static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2113{
2114#ifdef CONFIG_PCI_IOV
2115 struct pci_dev *pdev = adapter->pdev;
2116
a6b623e0
AD		 2117 if (adapter->vfs_allocated_count) {
2118 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2119 sizeof(struct vf_data_storage),
2120 GFP_KERNEL);
2121 /* if allocation failed then we do not support SR-IOV */
2122 if (!adapter->vf_data) {
2123 adapter->vfs_allocated_count = 0;
2124 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2125 "Data Storage\n");
2126 }
2127 }
2128
2129 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2130 kfree(adapter->vf_data);
2131 adapter->vf_data = NULL;
2132#endif /* CONFIG_PCI_IOV */
2133 adapter->vfs_allocated_count = 0;
2134#ifdef CONFIG_PCI_IOV
2135 } else {
2136 unsigned char mac_addr[ETH_ALEN];
2137 int i;
2138 dev_info(&pdev->dev, "%d vfs allocated\n",
2139 adapter->vfs_allocated_count);
2140 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2141 random_ether_addr(mac_addr);
2142 igb_set_vf_mac(adapter, i, mac_addr);
2143 }
2144 }
2145#endif /* CONFIG_PCI_IOV */
2146}
2147
115f459a
AD		 2148
2149/**
2150 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2151 * @adapter: board private structure to initialize
2152 *
2153 * igb_init_hw_timer initializes the function pointer and values for the hw
2154 * timer found in hardware.
2155 **/
2156static void igb_init_hw_timer(struct igb_adapter *adapter)
2157{
2158 struct e1000_hw *hw = &adapter->hw;
2159
2160 switch (hw->mac.type) {
d2ba2ed8 2161 case e1000_i350:
55cac248
AD		 2162 case e1000_82580:
2163 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2164 adapter->cycles.read = igb_read_clock;
2165 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2166 adapter->cycles.mult = 1;
2167 /*
2168 * The 82580 timesync updates the system timer every 8ns by 8ns
2169 * and the value cannot be shifted. Instead we need to shift
2170 * the registers to generate a 64bit timer value. As a result
2171 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2172 * 24 in order to generate a larger value for synchronization.
2173 */
2174 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2175 /* disable system timer temporarily by setting bit 31 */
2176 wr32(E1000_TSAUXC, 0x80000000);
2177 wrfl();
2178
2179 /* Set registers so that rollover occurs soon to test this. */
2180 wr32(E1000_SYSTIMR, 0x00000000);
2181 wr32(E1000_SYSTIML, 0x80000000);
2182 wr32(E1000_SYSTIMH, 0x000000FF);
2183 wrfl();
2184
2185 /* enable system timer by clearing bit 31 */
2186 wr32(E1000_TSAUXC, 0x0);
2187 wrfl();
2188
2189 timecounter_init(&adapter->clock,
2190 &adapter->cycles,
2191 ktime_to_ns(ktime_get_real()));
2192 /*
2193 * Synchronize our NIC clock against system wall clock. NIC
2194 * time stamp reading requires ~3us per sample, each sample
2195 * was pretty stable even under load => only require 10
2196 * samples for each offset comparison.
2197 */
2198 memset(&adapter->compare, 0, sizeof(adapter->compare));
2199 adapter->compare.source = &adapter->clock;
2200 adapter->compare.target = ktime_get_real;
2201 adapter->compare.num_samples = 10;
2202 timecompare_update(&adapter->compare, 0);
2203 break;
115f459a
AD		 2204 case e1000_82576:
2205 /*
2206 * Initialize hardware timer: we keep it running just in case
2207 * that some program needs it later on.
2208 */
2209 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2210 adapter->cycles.read = igb_read_clock;
2211 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2212 adapter->cycles.mult = 1;
2213 /**
2214 * Scale the NIC clock cycle by a large factor so that
2215 * relatively small clock corrections can be added or
2216 * substracted at each clock tick. The drawbacks of a large
2217 * factor are a) that the clock register overflows more quickly
2218 * (not such a big deal) and b) that the increment per tick has
2219 * to fit into 24 bits. As a result we need to use a shift of
2220 * 19 so we can fit a value of 16 into the TIMINCA register.
2221 */
2222 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2223 wr32(E1000_TIMINCA,
2224 (1 << E1000_TIMINCA_16NS_SHIFT) |
2225 (16 << IGB_82576_TSYNC_SHIFT));
2226
2227 /* Set registers so that rollover occurs soon to test this. */
2228 wr32(E1000_SYSTIML, 0x00000000);
2229 wr32(E1000_SYSTIMH, 0xFF800000);
2230 wrfl();
2231
2232 timecounter_init(&adapter->clock,
2233 &adapter->cycles,
2234 ktime_to_ns(ktime_get_real()));
2235 /*
2236 * Synchronize our NIC clock against system wall clock. NIC
2237 * time stamp reading requires ~3us per sample, each sample
2238 * was pretty stable even under load => only require 10
2239 * samples for each offset comparison.
2240 */
2241 memset(&adapter->compare, 0, sizeof(adapter->compare));
2242 adapter->compare.source = &adapter->clock;
2243 adapter->compare.target = ktime_get_real;
2244 adapter->compare.num_samples = 10;
2245 timecompare_update(&adapter->compare, 0);
2246 break;
2247 case e1000_82575:
2248 /* 82575 does not support timesync */
2249 default:
2250 break;
2251 }
2252
2253}
2254
9d5c8243
AK		 2255/**
2256 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2257 * @adapter: board private structure to initialize
2258 *
2259 * igb_sw_init initializes the Adapter private data structure.
2260 * Fields are initialized based on PCI device information and
2261 * OS network device settings (MTU size).
2262 **/
2263static int __devinit igb_sw_init(struct igb_adapter *adapter)
2264{
2265 struct e1000_hw *hw = &adapter->hw;
2266 struct net_device *netdev = adapter->netdev;
2267 struct pci_dev *pdev = adapter->pdev;
2268
2269 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2270
68fd9910
AD		 2271 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2272 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2273 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2274 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2275
9d5c8243
AK		 2276 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2277 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2278
a6b623e0
AD		 2279#ifdef CONFIG_PCI_IOV
2280 if (hw->mac.type == e1000_82576)
c0f2276f 2281 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
a6b623e0
AD		 2282
2283#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 2284 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2285
2286 /*
2287 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2288 * then we should combine the queues into a queue pair in order to
2289 * conserve interrupts due to limited supply
2290 */
2291 if ((adapter->rss_queues > 4) ||
2292 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2293 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2294
a6b623e0 2295 /* This call may decrease the number of queues */
047e0030 2296 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2297 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2298 return -ENOMEM;
2299 }
2300
115f459a 2301 igb_init_hw_timer(adapter);
a6b623e0
AD		 2302 igb_probe_vfs(adapter);
2303
9d5c8243
AK		 2304 /* Explicitly disable IRQ since the NIC can be in any state. */
2305 igb_irq_disable(adapter);
2306
2307 set_bit(__IGB_DOWN, &adapter->state);
2308 return 0;
2309}
2310
2311/**
2312 * igb_open - Called when a network interface is made active
2313 * @netdev: network interface device structure
2314 *
2315 * Returns 0 on success, negative value on failure
2316 *
2317 * The open entry point is called when a network interface is made
2318 * active by the system (IFF_UP). At this point all resources needed
2319 * for transmit and receive operations are allocated, the interrupt
2320 * handler is registered with the OS, the watchdog timer is started,
2321 * and the stack is notified that the interface is ready.
2322 **/
2323static int igb_open(struct net_device *netdev)
2324{
2325 struct igb_adapter *adapter = netdev_priv(netdev);
2326 struct e1000_hw *hw = &adapter->hw;
2327 int err;
2328 int i;
2329
2330 /* disallow open during test */
2331 if (test_bit(__IGB_TESTING, &adapter->state))
2332 return -EBUSY;
2333
b168dfc5
JB		 2334 netif_carrier_off(netdev);
2335
9d5c8243
AK		 2336 /* allocate transmit descriptors */
2337 err = igb_setup_all_tx_resources(adapter);
2338 if (err)
2339 goto err_setup_tx;
2340
2341 /* allocate receive descriptors */
2342 err = igb_setup_all_rx_resources(adapter);
2343 if (err)
2344 goto err_setup_rx;
2345
88a268c1 2346 igb_power_up_link(adapter);
9d5c8243 2347
9d5c8243
AK		 2348 /* before we allocate an interrupt, we must be ready to handle it.
2349 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2350 * as soon as we call pci_request_irq, so we have to setup our
2351 * clean_rx handler before we do so. */
2352 igb_configure(adapter);
2353
2354 err = igb_request_irq(adapter);
2355 if (err)
2356 goto err_req_irq;
2357
2358 /* From here on the code is the same as igb_up() */
2359 clear_bit(__IGB_DOWN, &adapter->state);
2360
047e0030
AD		 2361 for (i = 0; i < adapter->num_q_vectors; i++) {
2362 struct igb_q_vector *q_vector = adapter->q_vector[i];
2363 napi_enable(&q_vector->napi);
2364 }
9d5c8243
AK		 2365
2366 /* Clear any pending interrupts. */
2367 rd32(E1000_ICR);
844290e5
PW		 2368
2369 igb_irq_enable(adapter);
2370
d4960307
AD		 2371 /* notify VFs that reset has been completed */
2372 if (adapter->vfs_allocated_count) {
2373 u32 reg_data = rd32(E1000_CTRL_EXT);
2374 reg_data |= E1000_CTRL_EXT_PFRSTD;
2375 wr32(E1000_CTRL_EXT, reg_data);
2376 }
2377
d55b53ff
JK		 2378 netif_tx_start_all_queues(netdev);
2379
25568a53
AD		 2380 /* start the watchdog. */
2381 hw->mac.get_link_status = 1;
2382 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2383
2384 return 0;
2385
2386err_req_irq:
2387 igb_release_hw_control(adapter);
88a268c1 2388 igb_power_down_link(adapter);
9d5c8243
AK		 2389 igb_free_all_rx_resources(adapter);
2390err_setup_rx:
2391 igb_free_all_tx_resources(adapter);
2392err_setup_tx:
2393 igb_reset(adapter);
2394
2395 return err;
2396}
2397
2398/**
2399 * igb_close - Disables a network interface
2400 * @netdev: network interface device structure
2401 *
2402 * Returns 0, this is not allowed to fail
2403 *
2404 * The close entry point is called when an interface is de-activated
2405 * by the OS. The hardware is still under the driver's control, but
2406 * needs to be disabled. A global MAC reset is issued to stop the
2407 * hardware, and all transmit and receive resources are freed.
2408 **/
2409static int igb_close(struct net_device *netdev)
2410{
2411 struct igb_adapter *adapter = netdev_priv(netdev);
2412
2413 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2414 igb_down(adapter);
2415
2416 igb_free_irq(adapter);
2417
2418 igb_free_all_tx_resources(adapter);
2419 igb_free_all_rx_resources(adapter);
2420
9d5c8243
AK		 2421 return 0;
2422}
2423
2424/**
2425 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2426 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2427 *
2428 * Return 0 on success, negative on failure
2429 **/
80785298 2430int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2431{
59d71989 2432 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2433 int size;
2434
2435 size = sizeof(struct igb_buffer) * tx_ring->count;
2436 tx_ring->buffer_info = vmalloc(size);
2437 if (!tx_ring->buffer_info)
2438 goto err;
2439 memset(tx_ring->buffer_info, 0, size);
2440
2441 /* round up to nearest 4K */
85e8d004 2442 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2443 tx_ring->size = ALIGN(tx_ring->size, 4096);
2444
59d71989
AD		 2445 tx_ring->desc = dma_alloc_coherent(dev,
2446 tx_ring->size,
2447 &tx_ring->dma,
2448 GFP_KERNEL);
9d5c8243
AK		 2449
2450 if (!tx_ring->desc)
2451 goto err;
2452
9d5c8243
AK		 2453 tx_ring->next_to_use = 0;
2454 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2455 return 0;
2456
2457err:
2458 vfree(tx_ring->buffer_info);
59d71989 2459 dev_err(dev,
9d5c8243
AK		 2460 "Unable to allocate memory for the transmit descriptor ring\n");
2461 return -ENOMEM;
2462}
2463
2464/**
2465 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2466 * (Descriptors) for all queues
2467 * @adapter: board private structure
2468 *
2469 * Return 0 on success, negative on failure
2470 **/
2471static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2472{
439705e1 2473 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2474 int i, err = 0;
2475
2476 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2477 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2478 if (err) {
439705e1 2479 dev_err(&pdev->dev,
9d5c8243
AK		 2480 "Allocation for Tx Queue %u failed\n", i);
2481 for (i--; i >= 0; i--)
3025a446 2482 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2483 break;
2484 }
2485 }
2486
a99955fc 2487 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2488 int r_idx = i % adapter->num_tx_queues;
3025a446 2489 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2490 }
9d5c8243
AK		 2491 return err;
2492}
2493
2494/**
85b430b4
AD		 2495 * igb_setup_tctl - configure the transmit control registers
2496 * @adapter: Board private structure
9d5c8243 2497 **/
d7ee5b3a 2498void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2499{
9d5c8243
AK		 2500 struct e1000_hw *hw = &adapter->hw;
2501 u32 tctl;
9d5c8243 2502
85b430b4
AD		 2503 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2504 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2505
2506 /* Program the Transmit Control Register */
9d5c8243
AK		 2507 tctl = rd32(E1000_TCTL);
2508 tctl &= ~E1000_TCTL_CT;
2509 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2510 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2511
2512 igb_config_collision_dist(hw);
2513
9d5c8243
AK		 2514 /* Enable transmits */
2515 tctl |= E1000_TCTL_EN;
2516
2517 wr32(E1000_TCTL, tctl);
2518}
2519
85b430b4
AD		 2520/**
2521 * igb_configure_tx_ring - Configure transmit ring after Reset
2522 * @adapter: board private structure
2523 * @ring: tx ring to configure
2524 *
2525 * Configure a transmit ring after a reset.
2526 **/
d7ee5b3a
AD		 2527void igb_configure_tx_ring(struct igb_adapter *adapter,
2528 struct igb_ring *ring)
85b430b4
AD		 2529{
2530 struct e1000_hw *hw = &adapter->hw;
2531 u32 txdctl;
2532 u64 tdba = ring->dma;
2533 int reg_idx = ring->reg_idx;
2534
2535 /* disable the queue */
2536 txdctl = rd32(E1000_TXDCTL(reg_idx));
2537 wr32(E1000_TXDCTL(reg_idx),
2538 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2539 wrfl();
2540 mdelay(10);
2541
2542 wr32(E1000_TDLEN(reg_idx),
2543 ring->count * sizeof(union e1000_adv_tx_desc));
2544 wr32(E1000_TDBAL(reg_idx),
2545 tdba & 0x00000000ffffffffULL);
2546 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2547
fce99e34
AD		 2548 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2549 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2550 writel(0, ring->head);
2551 writel(0, ring->tail);
85b430b4
AD		 2552
2553 txdctl |= IGB_TX_PTHRESH;
2554 txdctl |= IGB_TX_HTHRESH << 8;
2555 txdctl |= IGB_TX_WTHRESH << 16;
2556
2557 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2558 wr32(E1000_TXDCTL(reg_idx), txdctl);
2559}
2560
2561/**
2562 * igb_configure_tx - Configure transmit Unit after Reset
2563 * @adapter: board private structure
2564 *
2565 * Configure the Tx unit of the MAC after a reset.
2566 **/
2567static void igb_configure_tx(struct igb_adapter *adapter)
2568{
2569 int i;
2570
2571 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2572 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2573}
2574
9d5c8243
AK		 2575/**
2576 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2577 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2578 *
2579 * Returns 0 on success, negative on failure
2580 **/
80785298 2581int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2582{
59d71989 2583 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2584 int size, desc_len;
2585
2586 size = sizeof(struct igb_buffer) * rx_ring->count;
2587 rx_ring->buffer_info = vmalloc(size);
2588 if (!rx_ring->buffer_info)
2589 goto err;
2590 memset(rx_ring->buffer_info, 0, size);
2591
2592 desc_len = sizeof(union e1000_adv_rx_desc);
2593
2594 /* Round up to nearest 4K */
2595 rx_ring->size = rx_ring->count * desc_len;
2596 rx_ring->size = ALIGN(rx_ring->size, 4096);
2597
59d71989
AD		 2598 rx_ring->desc = dma_alloc_coherent(dev,
2599 rx_ring->size,
2600 &rx_ring->dma,
2601 GFP_KERNEL);
9d5c8243
AK		 2602
2603 if (!rx_ring->desc)
2604 goto err;
2605
2606 rx_ring->next_to_clean = 0;
2607 rx_ring->next_to_use = 0;
9d5c8243 2608
9d5c8243
AK		 2609 return 0;
2610
2611err:
2612 vfree(rx_ring->buffer_info);
439705e1 2613 rx_ring->buffer_info = NULL;
59d71989
AD		 2614 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2615 " ring\n");
9d5c8243
AK		 2616 return -ENOMEM;
2617}
2618
2619/**
2620 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2621 * (Descriptors) for all queues
2622 * @adapter: board private structure
2623 *
2624 * Return 0 on success, negative on failure
2625 **/
2626static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2627{
439705e1 2628 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2629 int i, err = 0;
2630
2631 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2632 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2633 if (err) {
439705e1 2634 dev_err(&pdev->dev,
9d5c8243
AK		 2635 "Allocation for Rx Queue %u failed\n", i);
2636 for (i--; i >= 0; i--)
3025a446 2637 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2638 break;
2639 }
2640 }
2641
2642 return err;
2643}
2644
06cf2666
AD		 2645/**
2646 * igb_setup_mrqc - configure the multiple receive queue control registers
2647 * @adapter: Board private structure
2648 **/
2649static void igb_setup_mrqc(struct igb_adapter *adapter)
2650{
2651 struct e1000_hw *hw = &adapter->hw;
2652 u32 mrqc, rxcsum;
2653 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2654 union e1000_reta {
2655 u32 dword;
2656 u8 bytes[4];
2657 } reta;
2658 static const u8 rsshash[40] = {
2659 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2660 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2661 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2662 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2663
2664 /* Fill out hash function seeds */
2665 for (j = 0; j < 10; j++) {
2666 u32 rsskey = rsshash[(j * 4)];
2667 rsskey |= rsshash[(j * 4) + 1] << 8;
2668 rsskey |= rsshash[(j * 4) + 2] << 16;
2669 rsskey |= rsshash[(j * 4) + 3] << 24;
2670 array_wr32(E1000_RSSRK(0), j, rsskey);
2671 }
2672
a99955fc 2673 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2674
2675 if (adapter->vfs_allocated_count) {
2676 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2677 switch (hw->mac.type) {
d2ba2ed8 2678 case e1000_i350:
55cac248
AD		 2679 case e1000_82580:
2680 num_rx_queues = 1;
2681 shift = 0;
2682 break;
06cf2666
AD		 2683 case e1000_82576:
2684 shift = 3;
2685 num_rx_queues = 2;
2686 break;
2687 case e1000_82575:
2688 shift = 2;
2689 shift2 = 6;
2690 default:
2691 break;
2692 }
2693 } else {
2694 if (hw->mac.type == e1000_82575)
2695 shift = 6;
2696 }
2697
2698 for (j = 0; j < (32 * 4); j++) {
2699 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2700 if (shift2)
2701 reta.bytes[j & 3] |= num_rx_queues << shift2;
2702 if ((j & 3) == 3)
2703 wr32(E1000_RETA(j >> 2), reta.dword);
2704 }
2705
2706 /*
2707 * Disable raw packet checksumming so that RSS hash is placed in
2708 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2709 * offloads as they are enabled by default
2710 */
2711 rxcsum = rd32(E1000_RXCSUM);
2712 rxcsum |= E1000_RXCSUM_PCSD;
2713
2714 if (adapter->hw.mac.type >= e1000_82576)
2715 /* Enable Receive Checksum Offload for SCTP */
2716 rxcsum |= E1000_RXCSUM_CRCOFL;
2717
2718 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2719 wr32(E1000_RXCSUM, rxcsum);
2720
2721 /* If VMDq is enabled then we set the appropriate mode for that, else
2722 * we default to RSS so that an RSS hash is calculated per packet even
2723 * if we are only using one queue */
2724 if (adapter->vfs_allocated_count) {
2725 if (hw->mac.type > e1000_82575) {
2726 /* Set the default pool for the PF's first queue */
2727 u32 vtctl = rd32(E1000_VT_CTL);
2728 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2729 E1000_VT_CTL_DISABLE_DEF_POOL);
2730 vtctl |= adapter->vfs_allocated_count <<
2731 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2732 wr32(E1000_VT_CTL, vtctl);
2733 }
a99955fc 2734 if (adapter->rss_queues > 1)
06cf2666
AD		 2735 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2736 else
2737 mrqc = E1000_MRQC_ENABLE_VMDQ;
2738 } else {
2739 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2740 }
2741 igb_vmm_control(adapter);
2742
4478a9cd
AD		 2743 /*
2744 * Generate RSS hash based on TCP port numbers and/or
2745 * IPv4/v6 src and dst addresses since UDP cannot be
2746 * hashed reliably due to IP fragmentation
2747 */
2748 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2749 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2750 E1000_MRQC_RSS_FIELD_IPV6 |
2751 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2752 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2753
2754 wr32(E1000_MRQC, mrqc);
2755}
2756
9d5c8243
AK		 2757/**
2758 * igb_setup_rctl - configure the receive control registers
2759 * @adapter: Board private structure
2760 **/
d7ee5b3a 2761void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2762{
2763 struct e1000_hw *hw = &adapter->hw;
2764 u32 rctl;
9d5c8243
AK		 2765
2766 rctl = rd32(E1000_RCTL);
2767
2768 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2769 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2770
69d728ba 2771 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2772 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2773
87cb7e8c
AK		 2774 /*
2775 * enable stripping of CRC. It's unlikely this will break BMC
2776 * redirection as it did with e1000. Newer features require
2777 * that the HW strips the CRC.
73cd78f1 2778 */
87cb7e8c 2779 rctl |= E1000_RCTL_SECRC;
9d5c8243 2780
559e9c49 2781 /* disable store bad packets and clear size bits. */
ec54d7d6 2782 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2783
6ec43fe6
AD		 2784 /* enable LPE to prevent packets larger than max_frame_size */
2785 rctl |= E1000_RCTL_LPE;
9d5c8243 2786
952f72a8
AD		 2787 /* disable queue 0 to prevent tail write w/o re-config */
2788 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2789
e1739522
AD		 2790 /* Attention!!! For SR-IOV PF driver operations you must enable
2791 * queue drop for all VF and PF queues to prevent head of line blocking
2792 * if an un-trusted VF does not provide descriptors to hardware.
2793 */
2794 if (adapter->vfs_allocated_count) {
e1739522
AD		 2795 /* set all queue drop enable bits */
2796 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2797 }
2798
9d5c8243
AK		 2799 wr32(E1000_RCTL, rctl);
2800}
2801
7d5753f0
AD		 2802static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2803 int vfn)
2804{
2805 struct e1000_hw *hw = &adapter->hw;
2806 u32 vmolr;
2807
2808 /* if it isn't the PF check to see if VFs are enabled and
2809 * increase the size to support vlan tags */
2810 if (vfn < adapter->vfs_allocated_count &&
2811 adapter->vf_data[vfn].vlans_enabled)
2812 size += VLAN_TAG_SIZE;
2813
2814 vmolr = rd32(E1000_VMOLR(vfn));
2815 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2816 vmolr |= size | E1000_VMOLR_LPE;
2817 wr32(E1000_VMOLR(vfn), vmolr);
2818
2819 return 0;
2820}
2821
e1739522
AD		 2822/**
2823 * igb_rlpml_set - set maximum receive packet size
2824 * @adapter: board private structure
2825 *
2826 * Configure maximum receivable packet size.
2827 **/
2828static void igb_rlpml_set(struct igb_adapter *adapter)
2829{
2830 u32 max_frame_size = adapter->max_frame_size;
2831 struct e1000_hw *hw = &adapter->hw;
2832 u16 pf_id = adapter->vfs_allocated_count;
2833
2834 if (adapter->vlgrp)
2835 max_frame_size += VLAN_TAG_SIZE;
2836
2837 /* if vfs are enabled we set RLPML to the largest possible request
2838 * size and set the VMOLR RLPML to the size we need */
2839 if (pf_id) {
2840 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2841 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2842 }
2843
2844 wr32(E1000_RLPML, max_frame_size);
2845}
2846
8151d294
WM		 2847static inline void igb_set_vmolr(struct igb_adapter *adapter,
2848 int vfn, bool aupe)
7d5753f0
AD		 2849{
2850 struct e1000_hw *hw = &adapter->hw;
2851 u32 vmolr;
2852
2853 /*
2854 * This register exists only on 82576 and newer so if we are older then
2855 * we should exit and do nothing
2856 */
2857 if (hw->mac.type < e1000_82576)
2858 return;
2859
2860 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2861 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2862 if (aupe)
2863 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2864 else
2865 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2866
2867 /* clear all bits that might not be set */
2868 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2869
a99955fc 2870 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2871 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2872 /*
2873 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2874 * multicast packets
2875 */
2876 if (vfn <= adapter->vfs_allocated_count)
2877 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2878
2879 wr32(E1000_VMOLR(vfn), vmolr);
2880}
2881
85b430b4
AD		 2882/**
2883 * igb_configure_rx_ring - Configure a receive ring after Reset
2884 * @adapter: board private structure
2885 * @ring: receive ring to be configured
2886 *
2887 * Configure the Rx unit of the MAC after a reset.
2888 **/
d7ee5b3a
AD		 2889void igb_configure_rx_ring(struct igb_adapter *adapter,
2890 struct igb_ring *ring)
85b430b4
AD		 2891{
2892 struct e1000_hw *hw = &adapter->hw;
2893 u64 rdba = ring->dma;
2894 int reg_idx = ring->reg_idx;
952f72a8 2895 u32 srrctl, rxdctl;
85b430b4
AD		 2896
2897 /* disable the queue */
2898 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2899 wr32(E1000_RXDCTL(reg_idx),
2900 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2901
2902 /* Set DMA base address registers */
2903 wr32(E1000_RDBAL(reg_idx),
2904 rdba & 0x00000000ffffffffULL);
2905 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2906 wr32(E1000_RDLEN(reg_idx),
2907 ring->count * sizeof(union e1000_adv_rx_desc));
2908
2909 /* initialize head and tail */
fce99e34
AD		 2910 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2911 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2912 writel(0, ring->head);
2913 writel(0, ring->tail);
85b430b4 2914
952f72a8 2915 /* set descriptor configuration */
4c844851
AD		 2916 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2917 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2918 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2919#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2920 srrctl |= IGB_RXBUFFER_16384 >>
2921 E1000_SRRCTL_BSIZEPKT_SHIFT;
2922#else
2923 srrctl |= (PAGE_SIZE / 2) >>
2924 E1000_SRRCTL_BSIZEPKT_SHIFT;
2925#endif
2926 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2927 } else {
4c844851 2928 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2929 E1000_SRRCTL_BSIZEPKT_SHIFT;
2930 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2931 }
757b77e2
NN		 2932 if (hw->mac.type == e1000_82580)
2933 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2934 /* Only set Drop Enable if we are supporting multiple queues */
2935 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2936 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2937
2938 wr32(E1000_SRRCTL(reg_idx), srrctl);
2939
7d5753f0 2940 /* set filtering for VMDQ pools */
8151d294 2941 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2942
85b430b4
AD		 2943 /* enable receive descriptor fetching */
2944 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2945 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2946 rxdctl &= 0xFFF00000;
2947 rxdctl |= IGB_RX_PTHRESH;
2948 rxdctl |= IGB_RX_HTHRESH << 8;
2949 rxdctl |= IGB_RX_WTHRESH << 16;
2950 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2951}
2952
9d5c8243
AK		 2953/**
2954 * igb_configure_rx - Configure receive Unit after Reset
2955 * @adapter: board private structure
2956 *
2957 * Configure the Rx unit of the MAC after a reset.
2958 **/
2959static void igb_configure_rx(struct igb_adapter *adapter)
2960{
9107584e 2961 int i;
9d5c8243 2962
68d480c4
AD		 2963 /* set UTA to appropriate mode */
2964 igb_set_uta(adapter);
2965
26ad9178
AD		 2966 /* set the correct pool for the PF default MAC address in entry 0 */
2967 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2968 adapter->vfs_allocated_count);
2969
06cf2666
AD		 2970 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2971 * the Base and Length of the Rx Descriptor Ring */
2972 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2973 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2974}
2975
2976/**
2977 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2978 * @tx_ring: Tx descriptor ring for a specific queue
2979 *
2980 * Free all transmit software resources
2981 **/
68fd9910 2982void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2983{
3b644cf6 2984 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2985
2986 vfree(tx_ring->buffer_info);
2987 tx_ring->buffer_info = NULL;
2988
439705e1
AD		 2989 /* if not set, then don't free */
2990 if (!tx_ring->desc)
2991 return;
2992
59d71989
AD		 2993 dma_free_coherent(tx_ring->dev, tx_ring->size,
2994 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2995
2996 tx_ring->desc = NULL;
2997}
2998
2999/**
3000 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3001 * @adapter: board private structure
3002 *
3003 * Free all transmit software resources
3004 **/
3005static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3006{
3007 int i;
3008
3009 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3010 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3011}
3012
b1a436c3
AD		 3013void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3014 struct igb_buffer *buffer_info)
9d5c8243 3015{
6366ad33
AD		 3016 if (buffer_info->dma) {
3017 if (buffer_info->mapped_as_page)
59d71989 3018 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3019 buffer_info->dma,
3020 buffer_info->length,
59d71989 3021 DMA_TO_DEVICE);
6366ad33 3022 else
59d71989 3023 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3024 buffer_info->dma,
3025 buffer_info->length,
59d71989 3026 DMA_TO_DEVICE);
6366ad33
AD		 3027 buffer_info->dma = 0;
3028 }
9d5c8243
AK		 3029 if (buffer_info->skb) {
3030 dev_kfree_skb_any(buffer_info->skb);
3031 buffer_info->skb = NULL;
3032 }
3033 buffer_info->time_stamp = 0;
6366ad33
AD		 3034 buffer_info->length = 0;
3035 buffer_info->next_to_watch = 0;
3036 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3037}
3038
3039/**
3040 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3041 * @tx_ring: ring to be cleaned
3042 **/
3b644cf6 3043static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3044{
3045 struct igb_buffer *buffer_info;
3046 unsigned long size;
3047 unsigned int i;
3048
3049 if (!tx_ring->buffer_info)
3050 return;
3051 /* Free all the Tx ring sk_buffs */
3052
3053 for (i = 0; i < tx_ring->count; i++) {
3054 buffer_info = &tx_ring->buffer_info[i];
80785298 3055 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3056 }
3057
3058 size = sizeof(struct igb_buffer) * tx_ring->count;
3059 memset(tx_ring->buffer_info, 0, size);
3060
3061 /* Zero out the descriptor ring */
9d5c8243
AK		 3062 memset(tx_ring->desc, 0, tx_ring->size);
3063
3064 tx_ring->next_to_use = 0;
3065 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3066}
3067
3068/**
3069 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3070 * @adapter: board private structure
3071 **/
3072static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3073{
3074 int i;
3075
3076 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3077 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3078}
3079
3080/**
3081 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3082 * @rx_ring: ring to clean the resources from
3083 *
3084 * Free all receive software resources
3085 **/
68fd9910 3086void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3087{
3b644cf6 3088 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3089
3090 vfree(rx_ring->buffer_info);
3091 rx_ring->buffer_info = NULL;
3092
439705e1
AD		 3093 /* if not set, then don't free */
3094 if (!rx_ring->desc)
3095 return;
3096
59d71989
AD		 3097 dma_free_coherent(rx_ring->dev, rx_ring->size,
3098 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3099
3100 rx_ring->desc = NULL;
3101}
3102
3103/**
3104 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3105 * @adapter: board private structure
3106 *
3107 * Free all receive software resources
3108 **/
3109static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3110{
3111 int i;
3112
3113 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3114 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3115}
3116
3117/**
3118 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3119 * @rx_ring: ring to free buffers from
3120 **/
3b644cf6 3121static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3122{
3123 struct igb_buffer *buffer_info;
9d5c8243
AK		 3124 unsigned long size;
3125 unsigned int i;
3126
3127 if (!rx_ring->buffer_info)
3128 return;
439705e1 3129
9d5c8243
AK		 3130 /* Free all the Rx ring sk_buffs */
3131 for (i = 0; i < rx_ring->count; i++) {
3132 buffer_info = &rx_ring->buffer_info[i];
3133 if (buffer_info->dma) {
59d71989 3134 dma_unmap_single(rx_ring->dev,
80785298 3135 buffer_info->dma,
4c844851 3136 rx_ring->rx_buffer_len,
59d71989 3137 DMA_FROM_DEVICE);
9d5c8243
AK		 3138 buffer_info->dma = 0;
3139 }
3140
3141 if (buffer_info->skb) {
3142 dev_kfree_skb(buffer_info->skb);
3143 buffer_info->skb = NULL;
3144 }
6ec43fe6 3145 if (buffer_info->page_dma) {
59d71989 3146 dma_unmap_page(rx_ring->dev,
80785298 3147 buffer_info->page_dma,
6ec43fe6 3148 PAGE_SIZE / 2,
59d71989 3149 DMA_FROM_DEVICE);
6ec43fe6
AD		 3150 buffer_info->page_dma = 0;
3151 }
9d5c8243 3152 if (buffer_info->page) {
9d5c8243
AK		 3153 put_page(buffer_info->page);
3154 buffer_info->page = NULL;
bf36c1a0 3155 buffer_info->page_offset = 0;
9d5c8243
AK		 3156 }
3157 }
3158
9d5c8243
AK		 3159 size = sizeof(struct igb_buffer) * rx_ring->count;
3160 memset(rx_ring->buffer_info, 0, size);
3161
3162 /* Zero out the descriptor ring */
3163 memset(rx_ring->desc, 0, rx_ring->size);
3164
3165 rx_ring->next_to_clean = 0;
3166 rx_ring->next_to_use = 0;
9d5c8243
AK		 3167}
3168
3169/**
3170 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3171 * @adapter: board private structure
3172 **/
3173static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3174{
3175 int i;
3176
3177 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3178 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3179}
3180
3181/**
3182 * igb_set_mac - Change the Ethernet Address of the NIC
3183 * @netdev: network interface device structure
3184 * @p: pointer to an address structure
3185 *
3186 * Returns 0 on success, negative on failure
3187 **/
3188static int igb_set_mac(struct net_device *netdev, void *p)
3189{
3190 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3191 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3192 struct sockaddr *addr = p;
3193
3194 if (!is_valid_ether_addr(addr->sa_data))
3195 return -EADDRNOTAVAIL;
3196
3197 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3198 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3199
26ad9178
AD		 3200 /* set the correct pool for the new PF MAC address in entry 0 */
3201 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3202 adapter->vfs_allocated_count);
e1739522 3203
9d5c8243
AK		 3204 return 0;
3205}
3206
3207/**
68d480c4 3208 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3209 * @netdev: network interface device structure
3210 *
68d480c4
AD		 3211 * Writes multicast address list to the MTA hash table.
3212 * Returns: -ENOMEM on failure
3213 * 0 on no addresses written
3214 * X on writing X addresses to MTA
9d5c8243 3215 **/
68d480c4 3216static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3217{
3218 struct igb_adapter *adapter = netdev_priv(netdev);
3219 struct e1000_hw *hw = &adapter->hw;
22bedad3 3220 struct netdev_hw_addr *ha;
68d480c4 3221 u8 *mta_list;
9d5c8243
AK		 3222 int i;
3223
4cd24eaf 3224 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3225 /* nothing to program, so clear mc list */
3226 igb_update_mc_addr_list(hw, NULL, 0);
3227 igb_restore_vf_multicasts(adapter);
3228 return 0;
3229 }
9d5c8243 3230
4cd24eaf 3231 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3232 if (!mta_list)
3233 return -ENOMEM;
ff41f8dc 3234
68d480c4 3235 /* The shared function expects a packed array of only addresses. */
48e2f183 3236 i = 0;
22bedad3
JP		 3237 netdev_for_each_mc_addr(ha, netdev)
3238 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3239
68d480c4
AD		 3240 igb_update_mc_addr_list(hw, mta_list, i);
3241 kfree(mta_list);
3242
4cd24eaf 3243 return netdev_mc_count(netdev);
68d480c4
AD		 3244}
3245
3246/**
3247 * igb_write_uc_addr_list - write unicast addresses to RAR table
3248 * @netdev: network interface device structure
3249 *
3250 * Writes unicast address list to the RAR table.
3251 * Returns: -ENOMEM on failure/insufficient address space
3252 * 0 on no addresses written
3253 * X on writing X addresses to the RAR table
3254 **/
3255static int igb_write_uc_addr_list(struct net_device *netdev)
3256{
3257 struct igb_adapter *adapter = netdev_priv(netdev);
3258 struct e1000_hw *hw = &adapter->hw;
3259 unsigned int vfn = adapter->vfs_allocated_count;
3260 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3261 int count = 0;
3262
3263 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3264 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3265 return -ENOMEM;
9d5c8243 3266
32e7bfc4 3267 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3268 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3269
3270 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3271 if (!rar_entries)
3272 break;
26ad9178
AD		 3273 igb_rar_set_qsel(adapter, ha->addr,
3274 rar_entries--,
68d480c4
AD		 3275 vfn);
3276 count++;
ff41f8dc
AD		 3277 }
3278 }
3279 /* write the addresses in reverse order to avoid write combining */
3280 for (; rar_entries > 0 ; rar_entries--) {
3281 wr32(E1000_RAH(rar_entries), 0);
3282 wr32(E1000_RAL(rar_entries), 0);
3283 }
3284 wrfl();
3285
68d480c4
AD		 3286 return count;
3287}
3288
3289/**
3290 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3291 * @netdev: network interface device structure
3292 *
3293 * The set_rx_mode entry point is called whenever the unicast or multicast
3294 * address lists or the network interface flags are updated. This routine is
3295 * responsible for configuring the hardware for proper unicast, multicast,
3296 * promiscuous mode, and all-multi behavior.
3297 **/
3298static void igb_set_rx_mode(struct net_device *netdev)
3299{
3300 struct igb_adapter *adapter = netdev_priv(netdev);
3301 struct e1000_hw *hw = &adapter->hw;
3302 unsigned int vfn = adapter->vfs_allocated_count;
3303 u32 rctl, vmolr = 0;
3304 int count;
3305
3306 /* Check for Promiscuous and All Multicast modes */
3307 rctl = rd32(E1000_RCTL);
3308
3309 /* clear the effected bits */
3310 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3311
3312 if (netdev->flags & IFF_PROMISC) {
3313 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3314 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3315 } else {
3316 if (netdev->flags & IFF_ALLMULTI) {
3317 rctl |= E1000_RCTL_MPE;
3318 vmolr |= E1000_VMOLR_MPME;
3319 } else {
3320 /*
3321 * Write addresses to the MTA, if the attempt fails
3322 * then we should just turn on promiscous mode so
3323 * that we can at least receive multicast traffic
3324 */
3325 count = igb_write_mc_addr_list(netdev);
3326 if (count < 0) {
3327 rctl |= E1000_RCTL_MPE;
3328 vmolr |= E1000_VMOLR_MPME;
3329 } else if (count) {
3330 vmolr |= E1000_VMOLR_ROMPE;
3331 }
3332 }
3333 /*
3334 * Write addresses to available RAR registers, if there is not
3335 * sufficient space to store all the addresses then enable
3336 * unicast promiscous mode
3337 */
3338 count = igb_write_uc_addr_list(netdev);
3339 if (count < 0) {
3340 rctl |= E1000_RCTL_UPE;
3341 vmolr |= E1000_VMOLR_ROPE;
3342 }
3343 rctl |= E1000_RCTL_VFE;
28fc06f5 3344 }
68d480c4 3345 wr32(E1000_RCTL, rctl);
28fc06f5 3346
68d480c4
AD		 3347 /*
3348 * In order to support SR-IOV and eventually VMDq it is necessary to set
3349 * the VMOLR to enable the appropriate modes. Without this workaround
3350 * we will have issues with VLAN tag stripping not being done for frames
3351 * that are only arriving because we are the default pool
3352 */
3353 if (hw->mac.type < e1000_82576)
28fc06f5 3354 return;
9d5c8243 3355
68d480c4
AD		 3356 vmolr |= rd32(E1000_VMOLR(vfn)) &
3357 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3358 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3359 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3360}
3361
3362/* Need to wait a few seconds after link up to get diagnostic information from
3363 * the phy */
3364static void igb_update_phy_info(unsigned long data)
3365{
3366 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3367 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3368}
3369
4d6b725e
AD		 3370/**
3371 * igb_has_link - check shared code for link and determine up/down
3372 * @adapter: pointer to driver private info
3373 **/
3145535a 3374bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3375{
3376 struct e1000_hw *hw = &adapter->hw;
3377 bool link_active = false;
3378 s32 ret_val = 0;
3379
3380 /* get_link_status is set on LSC (link status) interrupt or
3381 * rx sequence error interrupt. get_link_status will stay
3382 * false until the e1000_check_for_link establishes link
3383 * for copper adapters ONLY
3384 */
3385 switch (hw->phy.media_type) {
3386 case e1000_media_type_copper:
3387 if (hw->mac.get_link_status) {
3388 ret_val = hw->mac.ops.check_for_link(hw);
3389 link_active = !hw->mac.get_link_status;
3390 } else {
3391 link_active = true;
3392 }
3393 break;
4d6b725e
AD		 3394 case e1000_media_type_internal_serdes:
3395 ret_val = hw->mac.ops.check_for_link(hw);
3396 link_active = hw->mac.serdes_has_link;
3397 break;
3398 default:
3399 case e1000_media_type_unknown:
3400 break;
3401 }
3402
3403 return link_active;
3404}
3405
9d5c8243
AK		 3406/**
3407 * igb_watchdog - Timer Call-back
3408 * @data: pointer to adapter cast into an unsigned long
3409 **/
3410static void igb_watchdog(unsigned long data)
3411{
3412 struct igb_adapter *adapter = (struct igb_adapter *)data;
3413 /* Do the rest outside of interrupt context */
3414 schedule_work(&adapter->watchdog_task);
3415}
3416
3417static void igb_watchdog_task(struct work_struct *work)
3418{
3419 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3420 struct igb_adapter,
3421 watchdog_task);
9d5c8243 3422 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3423 struct net_device *netdev = adapter->netdev;
9d5c8243 3424 u32 link;
7a6ea550 3425 int i;
9d5c8243 3426
4d6b725e 3427 link = igb_has_link(adapter);
9d5c8243
AK		 3428 if (link) {
3429 if (!netif_carrier_ok(netdev)) {
3430 u32 ctrl;
330a6d6a
AD		 3431 hw->mac.ops.get_speed_and_duplex(hw,
3432 &adapter->link_speed,
3433 &adapter->link_duplex);
9d5c8243
AK		 3434
3435 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3436 /* Links status message must follow this format */
3437 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3438 "Flow Control: %s\n",
559e9c49
AD		 3439 netdev->name,
3440 adapter->link_speed,
3441 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3442 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3443 ((ctrl & E1000_CTRL_TFCE) &&
3444 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3445 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3446 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3447
d07f3e37 3448 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3449 adapter->tx_timeout_factor = 1;
3450 switch (adapter->link_speed) {
3451 case SPEED_10:
9d5c8243
AK		 3452 adapter->tx_timeout_factor = 14;
3453 break;
3454 case SPEED_100:
9d5c8243
AK		 3455 /* maybe add some timeout factor ? */
3456 break;
3457 }
3458
3459 netif_carrier_on(netdev);
9d5c8243 3460
4ae196df
AD		 3461 igb_ping_all_vfs(adapter);
3462
4b1a9877 3463 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3464 if (!test_bit(__IGB_DOWN, &adapter->state))
3465 mod_timer(&adapter->phy_info_timer,
3466 round_jiffies(jiffies + 2 * HZ));
3467 }
3468 } else {
3469 if (netif_carrier_ok(netdev)) {
3470 adapter->link_speed = 0;
3471 adapter->link_duplex = 0;
527d47c1
AD		 3472 /* Links status message must follow this format */
3473 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3474 netdev->name);
9d5c8243 3475 netif_carrier_off(netdev);
4b1a9877 3476
4ae196df
AD		 3477 igb_ping_all_vfs(adapter);
3478
4b1a9877 3479 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3480 if (!test_bit(__IGB_DOWN, &adapter->state))
3481 mod_timer(&adapter->phy_info_timer,
3482 round_jiffies(jiffies + 2 * HZ));
3483 }
3484 }
3485
9d5c8243 3486 igb_update_stats(adapter);
9d5c8243 3487
dbabb065 3488 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3489 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3490 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3491 /* We've lost link, so the controller stops DMA,
3492 * but we've got queued Tx work that's never going
3493 * to get done, so reset controller to flush Tx.
3494 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3495 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3496 adapter->tx_timeout_count++;
3497 schedule_work(&adapter->reset_task);
3498 /* return immediately since reset is imminent */
3499 return;
3500 }
9d5c8243 3501 }
9d5c8243 3502
dbabb065
AD		 3503 /* Force detection of hung controller every watchdog period */
3504 tx_ring->detect_tx_hung = true;
3505 }
f7ba205e 3506
9d5c8243 3507 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3508 if (adapter->msix_entries) {
047e0030
AD		 3509 u32 eics = 0;
3510 for (i = 0; i < adapter->num_q_vectors; i++) {
3511 struct igb_q_vector *q_vector = adapter->q_vector[i];
3512 eics |= q_vector->eims_value;
3513 }
7a6ea550
AD		 3514 wr32(E1000_EICS, eics);
3515 } else {
3516 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3517 }
9d5c8243 3518
9d5c8243
AK		 3519 /* Reset the timer */
3520 if (!test_bit(__IGB_DOWN, &adapter->state))
3521 mod_timer(&adapter->watchdog_timer,
3522 round_jiffies(jiffies + 2 * HZ));
3523}
3524
3525enum latency_range {
3526 lowest_latency = 0,
3527 low_latency = 1,
3528 bulk_latency = 2,
3529 latency_invalid = 255
3530};
3531
6eb5a7f1
AD		 3532/**
3533 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3534 *
3535 * Stores a new ITR value based on strictly on packet size. This
3536 * algorithm is less sophisticated than that used in igb_update_itr,
3537 * due to the difficulty of synchronizing statistics across multiple
3538 * receive rings. The divisors and thresholds used by this fuction
3539 * were determined based on theoretical maximum wire speed and testing
3540 * data, in order to minimize response time while increasing bulk
3541 * throughput.
3542 * This functionality is controlled by the InterruptThrottleRate module
3543 * parameter (see igb_param.c)
3544 * NOTE: This function is called only when operating in a multiqueue
3545 * receive environment.
047e0030 3546 * @q_vector: pointer to q_vector
6eb5a7f1 3547 **/
047e0030 3548static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3549{
047e0030 3550 int new_val = q_vector->itr_val;
6eb5a7f1 3551 int avg_wire_size = 0;
047e0030 3552 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3553
6eb5a7f1
AD		 3554 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3555 * ints/sec - ITR timer value of 120 ticks.
3556 */
3557 if (adapter->link_speed != SPEED_1000) {
047e0030 3558 new_val = 976;
6eb5a7f1 3559 goto set_itr_val;
9d5c8243 3560 }
047e0030
AD		 3561
3562 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3563 struct igb_ring *ring = q_vector->rx_ring;
3564 avg_wire_size = ring->total_bytes / ring->total_packets;
3565 }
3566
3567 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3568 struct igb_ring *ring = q_vector->tx_ring;
3569 avg_wire_size = max_t(u32, avg_wire_size,
3570 (ring->total_bytes /
3571 ring->total_packets));
3572 }
3573
3574 /* if avg_wire_size isn't set no work was done */
3575 if (!avg_wire_size)
3576 goto clear_counts;
9d5c8243 3577
6eb5a7f1
AD		 3578 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3579 avg_wire_size += 24;
3580
3581 /* Don't starve jumbo frames */
3582 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3583
6eb5a7f1
AD		 3584 /* Give a little boost to mid-size frames */
3585 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3586 new_val = avg_wire_size / 3;
3587 else
3588 new_val = avg_wire_size / 2;
9d5c8243 3589
abe1c363
NN		 3590 /* when in itr mode 3 do not exceed 20K ints/sec */
3591 if (adapter->rx_itr_setting == 3 && new_val < 196)
3592 new_val = 196;
3593
6eb5a7f1 3594set_itr_val:
047e0030
AD		 3595 if (new_val != q_vector->itr_val) {
3596 q_vector->itr_val = new_val;
3597 q_vector->set_itr = 1;
9d5c8243 3598 }
6eb5a7f1 3599clear_counts:
047e0030
AD		 3600 if (q_vector->rx_ring) {
3601 q_vector->rx_ring->total_bytes = 0;
3602 q_vector->rx_ring->total_packets = 0;
3603 }
3604 if (q_vector->tx_ring) {
3605 q_vector->tx_ring->total_bytes = 0;
3606 q_vector->tx_ring->total_packets = 0;
3607 }
9d5c8243
AK		 3608}
3609
3610/**
3611 * igb_update_itr - update the dynamic ITR value based on statistics
3612 * Stores a new ITR value based on packets and byte
3613 * counts during the last interrupt. The advantage of per interrupt
3614 * computation is faster updates and more accurate ITR for the current
3615 * traffic pattern. Constants in this function were computed
3616 * based on theoretical maximum wire speed and thresholds were set based
3617 * on testing data as well as attempting to minimize response time
3618 * while increasing bulk throughput.
3619 * this functionality is controlled by the InterruptThrottleRate module
3620 * parameter (see igb_param.c)
3621 * NOTE: These calculations are only valid when operating in a single-
3622 * queue environment.
3623 * @adapter: pointer to adapter
047e0030 3624 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3625 * @packets: the number of packets during this measurement interval
3626 * @bytes: the number of bytes during this measurement interval
3627 **/
3628static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3629 int packets, int bytes)
3630{
3631 unsigned int retval = itr_setting;
3632
3633 if (packets == 0)
3634 goto update_itr_done;
3635
3636 switch (itr_setting) {
3637 case lowest_latency:
3638 /* handle TSO and jumbo frames */
3639 if (bytes/packets > 8000)
3640 retval = bulk_latency;
3641 else if ((packets < 5) && (bytes > 512))
3642 retval = low_latency;
3643 break;
3644 case low_latency: /* 50 usec aka 20000 ints/s */
3645 if (bytes > 10000) {
3646 /* this if handles the TSO accounting */
3647 if (bytes/packets > 8000) {
3648 retval = bulk_latency;
3649 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3650 retval = bulk_latency;
3651 } else if ((packets > 35)) {
3652 retval = lowest_latency;
3653 }
3654 } else if (bytes/packets > 2000) {
3655 retval = bulk_latency;
3656 } else if (packets <= 2 && bytes < 512) {
3657 retval = lowest_latency;
3658 }
3659 break;
3660 case bulk_latency: /* 250 usec aka 4000 ints/s */
3661 if (bytes > 25000) {
3662 if (packets > 35)
3663 retval = low_latency;
1e5c3d21 3664 } else if (bytes < 1500) {
9d5c8243
AK		 3665 retval = low_latency;
3666 }
3667 break;
3668 }
3669
3670update_itr_done:
3671 return retval;
3672}
3673
6eb5a7f1 3674static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3675{
047e0030 3676 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3677 u16 current_itr;
047e0030 3678 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3679
3680 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3681 if (adapter->link_speed != SPEED_1000) {
3682 current_itr = 0;
3683 new_itr = 4000;
3684 goto set_itr_now;
3685 }
3686
3687 adapter->rx_itr = igb_update_itr(adapter,
3688 adapter->rx_itr,
3025a446
AD		 3689 q_vector->rx_ring->total_packets,
3690 q_vector->rx_ring->total_bytes);
9d5c8243 3691
047e0030
AD		 3692 adapter->tx_itr = igb_update_itr(adapter,
3693 adapter->tx_itr,
3025a446
AD		 3694 q_vector->tx_ring->total_packets,
3695 q_vector->tx_ring->total_bytes);
047e0030 3696 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3697
6eb5a7f1 3698 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3699 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3700 current_itr = low_latency;
3701
9d5c8243
AK		 3702 switch (current_itr) {
3703 /* counts and packets in update_itr are dependent on these numbers */
3704 case lowest_latency:
78b1f607 3705 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3706 break;
3707 case low_latency:
78b1f607 3708 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3709 break;
3710 case bulk_latency:
78b1f607 3711 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3712 break;
3713 default:
3714 break;
3715 }
3716
3717set_itr_now:
3025a446
AD		 3718 q_vector->rx_ring->total_bytes = 0;
3719 q_vector->rx_ring->total_packets = 0;
3720 q_vector->tx_ring->total_bytes = 0;
3721 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3722
047e0030 3723 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3724 /* this attempts to bias the interrupt rate towards Bulk
3725 * by adding intermediate steps when interrupt rate is
3726 * increasing */
047e0030
AD		 3727 new_itr = new_itr > q_vector->itr_val ?
3728 max((new_itr * q_vector->itr_val) /
3729 (new_itr + (q_vector->itr_val >> 2)),
3730 new_itr) :
9d5c8243
AK		 3731 new_itr;
3732 /* Don't write the value here; it resets the adapter's
3733 * internal timer, and causes us to delay far longer than
3734 * we should between interrupts. Instead, we write the ITR
3735 * value at the beginning of the next interrupt so the timing
3736 * ends up being correct.
3737 */
047e0030
AD		 3738 q_vector->itr_val = new_itr;
3739 q_vector->set_itr = 1;
9d5c8243 3740 }
9d5c8243
AK		 3741}
3742
9d5c8243
AK		 3743#define IGB_TX_FLAGS_CSUM 0x00000001
3744#define IGB_TX_FLAGS_VLAN 0x00000002
3745#define IGB_TX_FLAGS_TSO 0x00000004
3746#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3747#define IGB_TX_FLAGS_TSTAMP 0x00000010
3748#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3749#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3750
85ad76b2 3751static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3752 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3753{
3754 struct e1000_adv_tx_context_desc *context_desc;
3755 unsigned int i;
3756 int err;
3757 struct igb_buffer *buffer_info;
3758 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3759 u32 mss_l4len_idx;
3760 u8 l4len;
9d5c8243
AK		 3761
3762 if (skb_header_cloned(skb)) {
3763 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3764 if (err)
3765 return err;
3766 }
3767
3768 l4len = tcp_hdrlen(skb);
3769 *hdr_len += l4len;
3770
3771 if (skb->protocol == htons(ETH_P_IP)) {
3772 struct iphdr *iph = ip_hdr(skb);
3773 iph->tot_len = 0;
3774 iph->check = 0;
3775 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3776 iph->daddr, 0,
3777 IPPROTO_TCP,
3778 0);
8e1e8a47 3779 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3780 ipv6_hdr(skb)->payload_len = 0;
3781 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3782 &ipv6_hdr(skb)->daddr,
3783 0, IPPROTO_TCP, 0);
3784 }
3785
3786 i = tx_ring->next_to_use;
3787
3788 buffer_info = &tx_ring->buffer_info[i];
3789 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3790 /* VLAN MACLEN IPLEN */
3791 if (tx_flags & IGB_TX_FLAGS_VLAN)
3792 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3793 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3794 *hdr_len += skb_network_offset(skb);
3795 info |= skb_network_header_len(skb);
3796 *hdr_len += skb_network_header_len(skb);
3797 context_desc->vlan_macip_lens = cpu_to_le32(info);
3798
3799 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3800 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3801
3802 if (skb->protocol == htons(ETH_P_IP))
3803 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3804 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3805
3806 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3807
3808 /* MSS L4LEN IDX */
3809 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3810 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3811
73cd78f1 3812 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3813 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3814 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3815
3816 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3817 context_desc->seqnum_seed = 0;
3818
3819 buffer_info->time_stamp = jiffies;
0e014cb1 3820 buffer_info->next_to_watch = i;
9d5c8243
AK		 3821 buffer_info->dma = 0;
3822 i++;
3823 if (i == tx_ring->count)
3824 i = 0;
3825
3826 tx_ring->next_to_use = i;
3827
3828 return true;
3829}
3830
85ad76b2
AD		 3831static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3832 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3833{
3834 struct e1000_adv_tx_context_desc *context_desc;
59d71989 3835 struct device *dev = tx_ring->dev;
9d5c8243
AK		 3836 struct igb_buffer *buffer_info;
3837 u32 info = 0, tu_cmd = 0;
80785298 3838 unsigned int i;
9d5c8243
AK		 3839
3840 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3841 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3842 i = tx_ring->next_to_use;
3843 buffer_info = &tx_ring->buffer_info[i];
3844 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3845
3846 if (tx_flags & IGB_TX_FLAGS_VLAN)
3847 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3848
9d5c8243
AK		 3849 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3850 if (skb->ip_summed == CHECKSUM_PARTIAL)
3851 info |= skb_network_header_len(skb);
3852
3853 context_desc->vlan_macip_lens = cpu_to_le32(info);
3854
3855 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3856
3857 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3858 __be16 protocol;
3859
3860 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3861 const struct vlan_ethhdr *vhdr =
3862 (const struct vlan_ethhdr*)skb->data;
3863
3864 protocol = vhdr->h_vlan_encapsulated_proto;
3865 } else {
3866 protocol = skb->protocol;
3867 }
3868
3869 switch (protocol) {
09640e63 3870 case cpu_to_be16(ETH_P_IP):
9d5c8243 3871 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3872 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3873 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3874 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3875 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3876 break;
09640e63 3877 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3878 /* XXX what about other V6 headers?? */
3879 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3880 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3881 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3882 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3883 break;
3884 default:
3885 if (unlikely(net_ratelimit()))
59d71989 3886 dev_warn(dev,
44b0cda3
MW		 3887 "partial checksum but proto=%x!\n",
3888 skb->protocol);
3889 break;
3890 }
9d5c8243
AK		 3891 }
3892
3893 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3894 context_desc->seqnum_seed = 0;
85ad76b2 3895 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3896 context_desc->mss_l4len_idx =
85ad76b2 3897 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3898
3899 buffer_info->time_stamp = jiffies;
0e014cb1 3900 buffer_info->next_to_watch = i;
9d5c8243
AK		 3901 buffer_info->dma = 0;
3902
3903 i++;
3904 if (i == tx_ring->count)
3905 i = 0;
3906 tx_ring->next_to_use = i;
3907
3908 return true;
3909 }
9d5c8243
AK		 3910 return false;
3911}
3912
3913#define IGB_MAX_TXD_PWR 16
3914#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3915
80785298 3916static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3917 unsigned int first)
9d5c8243
AK		 3918{
3919 struct igb_buffer *buffer_info;
59d71989 3920 struct device *dev = tx_ring->dev;
2873957d 3921 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 3922 unsigned int count = 0, i;
3923 unsigned int f;
2873957d 3924 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 3925
3926 i = tx_ring->next_to_use;
3927
3928 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 3929 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3930 buffer_info->length = hlen;
9d5c8243
AK		 3931 /* set time_stamp *before* dma to help avoid a possible race */
3932 buffer_info->time_stamp = jiffies;
0e014cb1 3933 buffer_info->next_to_watch = i;
2873957d 3934 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 3935 DMA_TO_DEVICE);
3936 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 3937 goto dma_error;
9d5c8243
AK		 3938
3939 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 3940 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3941 unsigned int len = frag->size;
9d5c8243 3942
8581145f 3943 count++;
65689fef
AD		 3944 i++;
3945 if (i == tx_ring->count)
3946 i = 0;
3947
9d5c8243
AK		 3948 buffer_info = &tx_ring->buffer_info[i];
3949 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3950 buffer_info->length = len;
3951 buffer_info->time_stamp = jiffies;
0e014cb1 3952 buffer_info->next_to_watch = i;
6366ad33 3953 buffer_info->mapped_as_page = true;
59d71989 3954 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 3955 frag->page,
3956 frag->page_offset,
3957 len,
59d71989
AD		 3958 DMA_TO_DEVICE);
3959 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 3960 goto dma_error;
3961
9d5c8243
AK		 3962 }
3963
9d5c8243 3964 tx_ring->buffer_info[i].skb = skb;
2244d07b 3965 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
2873957d
NN		 3966 /* multiply data chunks by size of headers */
3967 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3968 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 3969 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3970
cdfd01fc 3971 return ++count;
6366ad33
AD		 3972
3973dma_error:
59d71989 3974 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 3975
3976 /* clear timestamp and dma mappings for failed buffer_info mapping */
3977 buffer_info->dma = 0;
3978 buffer_info->time_stamp = 0;
3979 buffer_info->length = 0;
3980 buffer_info->next_to_watch = 0;
3981 buffer_info->mapped_as_page = false;
6366ad33
AD		 3982
3983 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3984 while (count--) {
3985 if (i == 0)
3986 i = tx_ring->count;
6366ad33 3987 i--;
6366ad33
AD		 3988 buffer_info = &tx_ring->buffer_info[i];
3989 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3990 }
3991
3992 return 0;
9d5c8243
AK		 3993}
3994
85ad76b2 3995static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3996 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3997 u8 hdr_len)
3998{
cdfd01fc 3999 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 4000 struct igb_buffer *buffer_info;
4001 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 4002 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 4003
4004 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4005 E1000_ADVTXD_DCMD_DEXT);
4006
4007 if (tx_flags & IGB_TX_FLAGS_VLAN)
4008 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4009
33af6bcc
PO		 4010 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4011 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4012
9d5c8243
AK		 4013 if (tx_flags & IGB_TX_FLAGS_TSO) {
4014 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4015
4016 /* insert tcp checksum */
4017 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4018
4019 /* insert ip checksum */
4020 if (tx_flags & IGB_TX_FLAGS_IPV4)
4021 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4022
4023 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4024 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4025 }
4026
85ad76b2
AD		 4027 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4028 (tx_flags & (IGB_TX_FLAGS_CSUM |
4029 IGB_TX_FLAGS_TSO |
7dfc16fa 4030 IGB_TX_FLAGS_VLAN)))
85ad76b2 4031 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4032
4033 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4034
cdfd01fc 4035 do {
9d5c8243
AK		 4036 buffer_info = &tx_ring->buffer_info[i];
4037 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4038 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4039 tx_desc->read.cmd_type_len =
4040 cpu_to_le32(cmd_type_len | buffer_info->length);
4041 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4042 count--;
9d5c8243
AK		 4043 i++;
4044 if (i == tx_ring->count)
4045 i = 0;
cdfd01fc 4046 } while (count > 0);
9d5c8243 4047
85ad76b2 4048 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4049 /* Force memory writes to complete before letting h/w
4050 * know there are new descriptors to fetch. (Only
4051 * applicable for weak-ordered memory model archs,
4052 * such as IA-64). */
4053 wmb();
4054
4055 tx_ring->next_to_use = i;
fce99e34 4056 writel(i, tx_ring->tail);
9d5c8243
AK		 4057 /* we need this if more than one processor can write to our tail
4058 * at a time, it syncronizes IO on IA64/Altix systems */
4059 mmiowb();
4060}
4061
e694e964 4062static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4063{
e694e964
AD		 4064 struct net_device *netdev = tx_ring->netdev;
4065
661086df 4066 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4067
9d5c8243
AK		 4068 /* Herbert's original patch had:
4069 * smp_mb__after_netif_stop_queue();
4070 * but since that doesn't exist yet, just open code it. */
4071 smp_mb();
4072
4073 /* We need to check again in a case another CPU has just
4074 * made room available. */
c493ea45 4075 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4076 return -EBUSY;
4077
4078 /* A reprieve! */
661086df 4079 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4080 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 4081 return 0;
4082}
4083
717ba089 4084static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4085{
c493ea45 4086 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4087 return 0;
e694e964 4088 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4089}
4090
b1a436c3
AD		 4091netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4092 struct igb_ring *tx_ring)
9d5c8243 4093{
e694e964 4094 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 4095 int tso = 0, count;
91d4ee33
NN		 4096 u32 tx_flags = 0;
4097 u16 first;
4098 u8 hdr_len = 0;
9d5c8243 4099
9d5c8243
AK		 4100 /* need: 1 descriptor per page,
4101 * + 2 desc gap to keep tail from touching head,
4102 * + 1 desc for skb->data,
4103 * + 1 desc for context descriptor,
4104 * otherwise try next time */
e694e964 4105 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4106 /* this is a hard error */
9d5c8243
AK		 4107 return NETDEV_TX_BUSY;
4108 }
33af6bcc 4109
2244d07b
OH		 4110 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4111 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4112 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4113 }
9d5c8243 4114
cdfd01fc 4115 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 4116 tx_flags |= IGB_TX_FLAGS_VLAN;
4117 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4118 }
4119
661086df
PWJ		 4120 if (skb->protocol == htons(ETH_P_IP))
4121 tx_flags |= IGB_TX_FLAGS_IPV4;
4122
0e014cb1 4123 first = tx_ring->next_to_use;
85ad76b2
AD		 4124 if (skb_is_gso(skb)) {
4125 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4126
85ad76b2
AD		 4127 if (tso < 0) {
4128 dev_kfree_skb_any(skb);
4129 return NETDEV_TX_OK;
4130 }
9d5c8243
AK		 4131 }
4132
4133 if (tso)
4134 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4135 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4136 (skb->ip_summed == CHECKSUM_PARTIAL))
4137 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4138
65689fef 4139 /*
cdfd01fc 4140 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 4141 * has occured and we need to rewind the descriptor queue
4142 */
80785298 4143 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4144 if (!count) {
65689fef
AD		 4145 dev_kfree_skb_any(skb);
4146 tx_ring->buffer_info[first].time_stamp = 0;
4147 tx_ring->next_to_use = first;
85ad76b2 4148 return NETDEV_TX_OK;
65689fef 4149 }
9d5c8243 4150
85ad76b2
AD		 4151 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4152
4153 /* Make sure there is space in the ring for the next send. */
e694e964 4154 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4155
9d5c8243
AK		 4156 return NETDEV_TX_OK;
4157}
4158
3b29a56d
SH		 4159static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4160 struct net_device *netdev)
9d5c8243
AK		 4161{
4162 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4163 struct igb_ring *tx_ring;
661086df 4164 int r_idx = 0;
b1a436c3
AD		 4165
4166 if (test_bit(__IGB_DOWN, &adapter->state)) {
4167 dev_kfree_skb_any(skb);
4168 return NETDEV_TX_OK;
4169 }
4170
4171 if (skb->len <= 0) {
4172 dev_kfree_skb_any(skb);
4173 return NETDEV_TX_OK;
4174 }
4175
1bfaf07b 4176 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4177 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4178
4179 /* This goes back to the question of how to logically map a tx queue
4180 * to a flow. Right now, performance is impacted slightly negatively
4181 * if using multiple tx queues. If the stack breaks away from a
4182 * single qdisc implementation, we can look at this again. */
e694e964 4183 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4184}
4185
4186/**
4187 * igb_tx_timeout - Respond to a Tx Hang
4188 * @netdev: network interface device structure
4189 **/
4190static void igb_tx_timeout(struct net_device *netdev)
4191{
4192 struct igb_adapter *adapter = netdev_priv(netdev);
4193 struct e1000_hw *hw = &adapter->hw;
4194
4195 /* Do the reset outside of interrupt context */
4196 adapter->tx_timeout_count++;
f7ba205e 4197
55cac248
AD		 4198 if (hw->mac.type == e1000_82580)
4199 hw->dev_spec._82575.global_device_reset = true;
4200
9d5c8243 4201 schedule_work(&adapter->reset_task);
265de409
AD		 4202 wr32(E1000_EICS,
4203 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4204}
4205
4206static void igb_reset_task(struct work_struct *work)
4207{
4208 struct igb_adapter *adapter;
4209 adapter = container_of(work, struct igb_adapter, reset_task);
4210
c97ec42a
TI		 4211 igb_dump(adapter);
4212 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4213 igb_reinit_locked(adapter);
4214}
4215
4216/**
4217 * igb_get_stats - Get System Network Statistics
4218 * @netdev: network interface device structure
4219 *
4220 * Returns the address of the device statistics structure.
4221 * The statistics are actually updated from the timer callback.
4222 **/
73cd78f1 4223static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 4224{
9d5c8243 4225 /* only return the current stats */
8d24e933 4226 return &netdev->stats;
9d5c8243
AK		 4227}
4228
4229/**
4230 * igb_change_mtu - Change the Maximum Transfer Unit
4231 * @netdev: network interface device structure
4232 * @new_mtu: new value for maximum frame size
4233 *
4234 * Returns 0 on success, negative on failure
4235 **/
4236static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4237{
4238 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4239 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4240 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4241 u32 rx_buffer_len, i;
9d5c8243 4242
c809d227 4243 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4244 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4245 return -EINVAL;
4246 }
4247
9d5c8243 4248 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4249 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4250 return -EINVAL;
4251 }
4252
4253 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4254 msleep(1);
73cd78f1 4255
9d5c8243
AK		 4256 /* igb_down has a dependency on max_frame_size */
4257 adapter->max_frame_size = max_frame;
559e9c49 4258
9d5c8243
AK		 4259 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4260 * means we reserve 2 more, this pushes us to allocate from the next
4261 * larger slab size.
4262 * i.e. RXBUFFER_2048 --> size-4096 slab
4263 */
4264
757b77e2
NN		 4265 if (adapter->hw.mac.type == e1000_82580)
4266 max_frame += IGB_TS_HDR_LEN;
4267
7d95b717 4268 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4269 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4270 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4271 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4272 else
4c844851
AD		 4273 rx_buffer_len = IGB_RXBUFFER_128;
4274
757b77e2
NN		 4275 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4276 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4277 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4278
4279 if ((adapter->hw.mac.type == e1000_82580) &&
4280 (rx_buffer_len == IGB_RXBUFFER_128))
4281 rx_buffer_len += IGB_RXBUFFER_64;
4282
4c844851
AD		 4283 if (netif_running(netdev))
4284 igb_down(adapter);
9d5c8243 4285
090b1795 4286 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4287 netdev->mtu, new_mtu);
4288 netdev->mtu = new_mtu;
4289
4c844851 4290 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4291 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4292
9d5c8243
AK		 4293 if (netif_running(netdev))
4294 igb_up(adapter);
4295 else
4296 igb_reset(adapter);
4297
4298 clear_bit(__IGB_RESETTING, &adapter->state);
4299
4300 return 0;
4301}
4302
4303/**
4304 * igb_update_stats - Update the board statistics counters
4305 * @adapter: board private structure
4306 **/
4307
4308void igb_update_stats(struct igb_adapter *adapter)
4309{
128e45eb 4310 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 4311 struct e1000_hw *hw = &adapter->hw;
4312 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4313 u32 reg, mpc;
9d5c8243 4314 u16 phy_tmp;
3f9c0164
AD		 4315 int i;
4316 u64 bytes, packets;
9d5c8243
AK		 4317
4318#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4319
4320 /*
4321 * Prevent stats update while adapter is being reset, or if the pci
4322 * connection is down.
4323 */
4324 if (adapter->link_speed == 0)
4325 return;
4326 if (pci_channel_offline(pdev))
4327 return;
4328
3f9c0164
AD		 4329 bytes = 0;
4330 packets = 0;
4331 for (i = 0; i < adapter->num_rx_queues; i++) {
4332 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 4333 struct igb_ring *ring = adapter->rx_ring[i];
4334 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4335 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 4336 bytes += ring->rx_stats.bytes;
4337 packets += ring->rx_stats.packets;
3f9c0164
AD		 4338 }
4339
128e45eb
AD		 4340 net_stats->rx_bytes = bytes;
4341 net_stats->rx_packets = packets;
3f9c0164
AD		 4342
4343 bytes = 0;
4344 packets = 0;
4345 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 4346 struct igb_ring *ring = adapter->tx_ring[i];
4347 bytes += ring->tx_stats.bytes;
4348 packets += ring->tx_stats.packets;
3f9c0164 4349 }
128e45eb
AD		 4350 net_stats->tx_bytes = bytes;
4351 net_stats->tx_packets = packets;
3f9c0164
AD		 4352
4353 /* read stats registers */
9d5c8243
AK		 4354 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4355 adapter->stats.gprc += rd32(E1000_GPRC);
4356 adapter->stats.gorc += rd32(E1000_GORCL);
4357 rd32(E1000_GORCH); /* clear GORCL */
4358 adapter->stats.bprc += rd32(E1000_BPRC);
4359 adapter->stats.mprc += rd32(E1000_MPRC);
4360 adapter->stats.roc += rd32(E1000_ROC);
4361
4362 adapter->stats.prc64 += rd32(E1000_PRC64);
4363 adapter->stats.prc127 += rd32(E1000_PRC127);
4364 adapter->stats.prc255 += rd32(E1000_PRC255);
4365 adapter->stats.prc511 += rd32(E1000_PRC511);
4366 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4367 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4368 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4369 adapter->stats.sec += rd32(E1000_SEC);
4370
fa3d9a6d
MW		 4371 mpc = rd32(E1000_MPC);
4372 adapter->stats.mpc += mpc;
4373 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4374 adapter->stats.scc += rd32(E1000_SCC);
4375 adapter->stats.ecol += rd32(E1000_ECOL);
4376 adapter->stats.mcc += rd32(E1000_MCC);
4377 adapter->stats.latecol += rd32(E1000_LATECOL);
4378 adapter->stats.dc += rd32(E1000_DC);
4379 adapter->stats.rlec += rd32(E1000_RLEC);
4380 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4381 adapter->stats.xontxc += rd32(E1000_XONTXC);
4382 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4383 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4384 adapter->stats.fcruc += rd32(E1000_FCRUC);
4385 adapter->stats.gptc += rd32(E1000_GPTC);
4386 adapter->stats.gotc += rd32(E1000_GOTCL);
4387 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4388 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4389 adapter->stats.ruc += rd32(E1000_RUC);
4390 adapter->stats.rfc += rd32(E1000_RFC);
4391 adapter->stats.rjc += rd32(E1000_RJC);
4392 adapter->stats.tor += rd32(E1000_TORH);
4393 adapter->stats.tot += rd32(E1000_TOTH);
4394 adapter->stats.tpr += rd32(E1000_TPR);
4395
4396 adapter->stats.ptc64 += rd32(E1000_PTC64);
4397 adapter->stats.ptc127 += rd32(E1000_PTC127);
4398 adapter->stats.ptc255 += rd32(E1000_PTC255);
4399 adapter->stats.ptc511 += rd32(E1000_PTC511);
4400 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4401 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4402
4403 adapter->stats.mptc += rd32(E1000_MPTC);
4404 adapter->stats.bptc += rd32(E1000_BPTC);
4405
2d0b0f69
NN		 4406 adapter->stats.tpt += rd32(E1000_TPT);
4407 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4408
4409 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4410 /* read internal phy specific stats */
4411 reg = rd32(E1000_CTRL_EXT);
4412 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4413 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4414 adapter->stats.tncrs += rd32(E1000_TNCRS);
4415 }
4416
9d5c8243
AK		 4417 adapter->stats.tsctc += rd32(E1000_TSCTC);
4418 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4419
4420 adapter->stats.iac += rd32(E1000_IAC);
4421 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4422 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4423 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4424 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4425 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4426 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4427 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4428 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4429
4430 /* Fill out the OS statistics structure */
128e45eb
AD		 4431 net_stats->multicast = adapter->stats.mprc;
4432 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4433
4434 /* Rx Errors */
4435
4436 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4437 * our own version based on RUC and ROC */
128e45eb 4438 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4439 adapter->stats.crcerrs + adapter->stats.algnerrc +
4440 adapter->stats.ruc + adapter->stats.roc +
4441 adapter->stats.cexterr;
128e45eb
AD		 4442 net_stats->rx_length_errors = adapter->stats.ruc +
4443 adapter->stats.roc;
4444 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4445 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4446 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4447
4448 /* Tx Errors */
128e45eb
AD		 4449 net_stats->tx_errors = adapter->stats.ecol +
4450 adapter->stats.latecol;
4451 net_stats->tx_aborted_errors = adapter->stats.ecol;
4452 net_stats->tx_window_errors = adapter->stats.latecol;
4453 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4454
4455 /* Tx Dropped needs to be maintained elsewhere */
4456
4457 /* Phy Stats */
4458 if (hw->phy.media_type == e1000_media_type_copper) {
4459 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4460 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4461 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4462 adapter->phy_stats.idle_errors += phy_tmp;
4463 }
4464 }
4465
4466 /* Management Stats */
4467 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4468 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4469 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4470}
4471
9d5c8243
AK		 4472static irqreturn_t igb_msix_other(int irq, void *data)
4473{
047e0030 4474 struct igb_adapter *adapter = data;
9d5c8243 4475 struct e1000_hw *hw = &adapter->hw;
844290e5 4476 u32 icr = rd32(E1000_ICR);
844290e5 4477 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4478
7f081d40
AD		 4479 if (icr & E1000_ICR_DRSTA)
4480 schedule_work(&adapter->reset_task);
4481
047e0030 4482 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4483 /* HW is reporting DMA is out of sync */
4484 adapter->stats.doosync++;
4485 }
eebbbdba 4486
4ae196df
AD		 4487 /* Check for a mailbox event */
4488 if (icr & E1000_ICR_VMMB)
4489 igb_msg_task(adapter);
4490
4491 if (icr & E1000_ICR_LSC) {
4492 hw->mac.get_link_status = 1;
4493 /* guard against interrupt when we're going down */
4494 if (!test_bit(__IGB_DOWN, &adapter->state))
4495 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4496 }
4497
25568a53
AD		 4498 if (adapter->vfs_allocated_count)
4499 wr32(E1000_IMS, E1000_IMS_LSC |
4500 E1000_IMS_VMMB |
4501 E1000_IMS_DOUTSYNC);
4502 else
4503 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4504 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4505
4506 return IRQ_HANDLED;
4507}
4508
047e0030 4509static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4510{
26b39276 4511 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4512 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4513
047e0030
AD		 4514 if (!q_vector->set_itr)
4515 return;
73cd78f1 4516
047e0030
AD		 4517 if (!itr_val)
4518 itr_val = 0x4;
661086df 4519
26b39276
AD		 4520 if (adapter->hw.mac.type == e1000_82575)
4521 itr_val |= itr_val << 16;
661086df 4522 else
047e0030 4523 itr_val |= 0x8000000;
661086df 4524
047e0030
AD		 4525 writel(itr_val, q_vector->itr_register);
4526 q_vector->set_itr = 0;
6eb5a7f1
AD		 4527}
4528
047e0030 4529static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4530{
047e0030 4531 struct igb_q_vector *q_vector = data;
9d5c8243 4532
047e0030
AD		 4533 /* Write the ITR value calculated from the previous interrupt. */
4534 igb_write_itr(q_vector);
9d5c8243 4535
047e0030 4536 napi_schedule(&q_vector->napi);
844290e5 4537
047e0030 4538 return IRQ_HANDLED;
fe4506b6
JC		 4539}
4540
421e02f0 4541#ifdef CONFIG_IGB_DCA
047e0030 4542static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4543{
047e0030 4544 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4545 struct e1000_hw *hw = &adapter->hw;
4546 int cpu = get_cpu();
fe4506b6 4547
047e0030
AD		 4548 if (q_vector->cpu == cpu)
4549 goto out_no_update;
4550
4551 if (q_vector->tx_ring) {
4552 int q = q_vector->tx_ring->reg_idx;
4553 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4554 if (hw->mac.type == e1000_82575) {
4555 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4556 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4557 } else {
047e0030
AD		 4558 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4559 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4560 E1000_DCA_TXCTRL_CPUID_SHIFT;
4561 }
4562 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4563 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4564 }
4565 if (q_vector->rx_ring) {
4566 int q = q_vector->rx_ring->reg_idx;
4567 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4568 if (hw->mac.type == e1000_82575) {
2d064c06 4569 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4570 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4571 } else {
4572 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4573 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4574 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4575 }
fe4506b6
JC		 4576 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4577 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4578 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4579 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4580 }
047e0030
AD		 4581 q_vector->cpu = cpu;
4582out_no_update:
fe4506b6
JC		 4583 put_cpu();
4584}
4585
4586static void igb_setup_dca(struct igb_adapter *adapter)
4587{
7e0e99ef 4588 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4589 int i;
4590
7dfc16fa 4591 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4592 return;
4593
7e0e99ef
AD		 4594 /* Always use CB2 mode, difference is masked in the CB driver. */
4595 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4596
047e0030 4597 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4598 adapter->q_vector[i]->cpu = -1;
4599 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4600 }
4601}
4602
4603static int __igb_notify_dca(struct device *dev, void *data)
4604{
4605 struct net_device *netdev = dev_get_drvdata(dev);
4606 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4607 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4608 struct e1000_hw *hw = &adapter->hw;
4609 unsigned long event = *(unsigned long *)data;
4610
4611 switch (event) {
4612 case DCA_PROVIDER_ADD:
4613 /* if already enabled, don't do it again */
7dfc16fa 4614 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4615 break;
fe4506b6 4616 if (dca_add_requester(dev) == 0) {
bbd98fe4 4617 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4618 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4619 igb_setup_dca(adapter);
4620 break;
4621 }
4622 /* Fall Through since DCA is disabled. */
4623 case DCA_PROVIDER_REMOVE:
7dfc16fa 4624 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4625 /* without this a class_device is left
047e0030 4626 * hanging around in the sysfs model */
fe4506b6 4627 dca_remove_requester(dev);
090b1795 4628 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4629 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4630 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4631 }
4632 break;
4633 }
bbd98fe4 4634
fe4506b6 4635 return 0;
9d5c8243
AK		 4636}
4637
fe4506b6
JC		 4638static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4639 void *p)
4640{
4641 int ret_val;
4642
4643 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4644 __igb_notify_dca);
4645
4646 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4647}
421e02f0 4648#endif /* CONFIG_IGB_DCA */
9d5c8243 4649
4ae196df
AD		 4650static void igb_ping_all_vfs(struct igb_adapter *adapter)
4651{
4652 struct e1000_hw *hw = &adapter->hw;
4653 u32 ping;
4654 int i;
4655
4656 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4657 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4658 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4659 ping |= E1000_VT_MSGTYPE_CTS;
4660 igb_write_mbx(hw, &ping, 1, i);
4661 }
4662}
4663
7d5753f0
AD		 4664static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4665{
4666 struct e1000_hw *hw = &adapter->hw;
4667 u32 vmolr = rd32(E1000_VMOLR(vf));
4668 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4669
d85b9004 4670 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4671 IGB_VF_FLAG_MULTI_PROMISC);
4672 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4673
4674 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4675 vmolr |= E1000_VMOLR_MPME;
d85b9004 4676 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4677 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4678 } else {
4679 /*
4680 * if we have hashes and we are clearing a multicast promisc
4681 * flag we need to write the hashes to the MTA as this step
4682 * was previously skipped
4683 */
4684 if (vf_data->num_vf_mc_hashes > 30) {
4685 vmolr |= E1000_VMOLR_MPME;
4686 } else if (vf_data->num_vf_mc_hashes) {
4687 int j;
4688 vmolr |= E1000_VMOLR_ROMPE;
4689 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4690 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4691 }
4692 }
4693
4694 wr32(E1000_VMOLR(vf), vmolr);
4695
4696 /* there are flags left unprocessed, likely not supported */
4697 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4698 return -EINVAL;
4699
4700 return 0;
4701
4702}
4703
4ae196df
AD		 4704static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4705 u32 *msgbuf, u32 vf)
4706{
4707 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4708 u16 *hash_list = (u16 *)&msgbuf[1];
4709 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4710 int i;
4711
7d5753f0 4712 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4713 * to this VF for later use to restore when the PF multi cast
4714 * list changes
4715 */
4716 vf_data->num_vf_mc_hashes = n;
4717
7d5753f0
AD		 4718 /* only up to 30 hash values supported */
4719 if (n > 30)
4720 n = 30;
4721
4722 /* store the hashes for later use */
4ae196df 4723 for (i = 0; i < n; i++)
a419aef8 4724 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4725
4726 /* Flush and reset the mta with the new values */
ff41f8dc 4727 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4728
4729 return 0;
4730}
4731
4732static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4733{
4734 struct e1000_hw *hw = &adapter->hw;
4735 struct vf_data_storage *vf_data;
4736 int i, j;
4737
4738 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4739 u32 vmolr = rd32(E1000_VMOLR(i));
4740 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4741
4ae196df 4742 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4743
4744 if ((vf_data->num_vf_mc_hashes > 30) ||
4745 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4746 vmolr |= E1000_VMOLR_MPME;
4747 } else if (vf_data->num_vf_mc_hashes) {
4748 vmolr |= E1000_VMOLR_ROMPE;
4749 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4750 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4751 }
4752 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4753 }
4754}
4755
4756static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4757{
4758 struct e1000_hw *hw = &adapter->hw;
4759 u32 pool_mask, reg, vid;
4760 int i;
4761
4762 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4763
4764 /* Find the vlan filter for this id */
4765 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4766 reg = rd32(E1000_VLVF(i));
4767
4768 /* remove the vf from the pool */
4769 reg &= ~pool_mask;
4770
4771 /* if pool is empty then remove entry from vfta */
4772 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4773 (reg & E1000_VLVF_VLANID_ENABLE)) {
4774 reg = 0;
4775 vid = reg & E1000_VLVF_VLANID_MASK;
4776 igb_vfta_set(hw, vid, false);
4777 }
4778
4779 wr32(E1000_VLVF(i), reg);
4780 }
ae641bdc
AD		 4781
4782 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4783}
4784
4785static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4786{
4787 struct e1000_hw *hw = &adapter->hw;
4788 u32 reg, i;
4789
51466239
AD		 4790 /* The vlvf table only exists on 82576 hardware and newer */
4791 if (hw->mac.type < e1000_82576)
4792 return -1;
4793
4794 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4795 if (!adapter->vfs_allocated_count)
4796 return -1;
4797
4798 /* Find the vlan filter for this id */
4799 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4800 reg = rd32(E1000_VLVF(i));
4801 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4802 vid == (reg & E1000_VLVF_VLANID_MASK))
4803 break;
4804 }
4805
4806 if (add) {
4807 if (i == E1000_VLVF_ARRAY_SIZE) {
4808 /* Did not find a matching VLAN ID entry that was
4809 * enabled. Search for a free filter entry, i.e.
4810 * one without the enable bit set
4811 */
4812 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4813 reg = rd32(E1000_VLVF(i));
4814 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4815 break;
4816 }
4817 }
4818 if (i < E1000_VLVF_ARRAY_SIZE) {
4819 /* Found an enabled/available entry */
4820 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4821
4822 /* if !enabled we need to set this up in vfta */
4823 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4824 /* add VID to filter table */
4825 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4826 reg |= E1000_VLVF_VLANID_ENABLE;
4827 }
cad6d05f
AD		 4828 reg &= ~E1000_VLVF_VLANID_MASK;
4829 reg |= vid;
4ae196df 4830 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4831
4832 /* do not modify RLPML for PF devices */
4833 if (vf >= adapter->vfs_allocated_count)
4834 return 0;
4835
4836 if (!adapter->vf_data[vf].vlans_enabled) {
4837 u32 size;
4838 reg = rd32(E1000_VMOLR(vf));
4839 size = reg & E1000_VMOLR_RLPML_MASK;
4840 size += 4;
4841 reg &= ~E1000_VMOLR_RLPML_MASK;
4842 reg |= size;
4843 wr32(E1000_VMOLR(vf), reg);
4844 }
ae641bdc 4845
51466239 4846 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4847 return 0;
4848 }
4849 } else {
4850 if (i < E1000_VLVF_ARRAY_SIZE) {
4851 /* remove vf from the pool */
4852 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4853 /* if pool is empty then remove entry from vfta */
4854 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4855 reg = 0;
4856 igb_vfta_set(hw, vid, false);
4857 }
4858 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4859
4860 /* do not modify RLPML for PF devices */
4861 if (vf >= adapter->vfs_allocated_count)
4862 return 0;
4863
4864 adapter->vf_data[vf].vlans_enabled--;
4865 if (!adapter->vf_data[vf].vlans_enabled) {
4866 u32 size;
4867 reg = rd32(E1000_VMOLR(vf));
4868 size = reg & E1000_VMOLR_RLPML_MASK;
4869 size -= 4;
4870 reg &= ~E1000_VMOLR_RLPML_MASK;
4871 reg |= size;
4872 wr32(E1000_VMOLR(vf), reg);
4873 }
4ae196df
AD		 4874 }
4875 }
8151d294
WM		 4876 return 0;
4877}
4878
4879static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4880{
4881 struct e1000_hw *hw = &adapter->hw;
4882
4883 if (vid)
4884 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4885 else
4886 wr32(E1000_VMVIR(vf), 0);
4887}
4888
4889static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4890 int vf, u16 vlan, u8 qos)
4891{
4892 int err = 0;
4893 struct igb_adapter *adapter = netdev_priv(netdev);
4894
4895 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4896 return -EINVAL;
4897 if (vlan || qos) {
4898 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4899 if (err)
4900 goto out;
4901 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4902 igb_set_vmolr(adapter, vf, !vlan);
4903 adapter->vf_data[vf].pf_vlan = vlan;
4904 adapter->vf_data[vf].pf_qos = qos;
4905 dev_info(&adapter->pdev->dev,
4906 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4907 if (test_bit(__IGB_DOWN, &adapter->state)) {
4908 dev_warn(&adapter->pdev->dev,
4909 "The VF VLAN has been set,"
4910 " but the PF device is not up.\n");
4911 dev_warn(&adapter->pdev->dev,
4912 "Bring the PF device up before"
4913 " attempting to use the VF device.\n");
4914 }
4915 } else {
4916 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4917 false, vf);
4918 igb_set_vmvir(adapter, vlan, vf);
4919 igb_set_vmolr(adapter, vf, true);
4920 adapter->vf_data[vf].pf_vlan = 0;
4921 adapter->vf_data[vf].pf_qos = 0;
4922 }
4923out:
4924 return err;
4ae196df
AD		 4925}
4926
4927static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4928{
4929 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4930 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4931
4932 return igb_vlvf_set(adapter, vid, add, vf);
4933}
4934
f2ca0dbe 4935static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4936{
8151d294
WM		 4937 /* clear flags */
4938 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4939 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4940
4941 /* reset offloads to defaults */
8151d294 4942 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4943
4944 /* reset vlans for device */
4945 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4946 if (adapter->vf_data[vf].pf_vlan)
4947 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4948 adapter->vf_data[vf].pf_vlan,
4949 adapter->vf_data[vf].pf_qos);
4950 else
4951 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4952
4953 /* reset multicast table array for vf */
4954 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4955
4956 /* Flush and reset the mta with the new values */
ff41f8dc 4957 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4958}
4959
f2ca0dbe
AD		 4960static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4961{
4962 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4963
4964 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4965 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4966 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4967
4968 /* process remaining reset events */
4969 igb_vf_reset(adapter, vf);
4970}
4971
4972static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4973{
4974 struct e1000_hw *hw = &adapter->hw;
4975 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4976 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4977 u32 reg, msgbuf[3];
4978 u8 *addr = (u8 *)(&msgbuf[1]);
4979
4980 /* process all the same items cleared in a function level reset */
f2ca0dbe 4981 igb_vf_reset(adapter, vf);
4ae196df
AD		 4982
4983 /* set vf mac address */
26ad9178 4984 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4985
4986 /* enable transmit and receive for vf */
4987 reg = rd32(E1000_VFTE);
4988 wr32(E1000_VFTE, reg | (1 << vf));
4989 reg = rd32(E1000_VFRE);
4990 wr32(E1000_VFRE, reg | (1 << vf));
4991
f2ca0dbe 4992 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4993
4994 /* reply to reset with ack and vf mac address */
4995 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4996 memcpy(addr, vf_mac, 6);
4997 igb_write_mbx(hw, msgbuf, 3, vf);
4998}
4999
5000static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5001{
de42edde
GR		 5002 /*
5003 * The VF MAC Address is stored in a packed array of bytes
5004 * starting at the second 32 bit word of the msg array
5005 */
f2ca0dbe
AD		 5006 unsigned char *addr = (char *)&msg[1];
5007 int err = -1;
4ae196df 5008
f2ca0dbe
AD		 5009 if (is_valid_ether_addr(addr))
5010 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5011
f2ca0dbe 5012 return err;
4ae196df
AD		 5013}
5014
5015static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5016{
5017 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5018 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5019 u32 msg = E1000_VT_MSGTYPE_NACK;
5020
5021 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5022 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5023 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5024 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5025 vf_data->last_nack = jiffies;
4ae196df
AD		 5026 }
5027}
5028
f2ca0dbe 5029static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5030{
f2ca0dbe
AD		 5031 struct pci_dev *pdev = adapter->pdev;
5032 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5033 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5034 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5035 s32 retval;
5036
f2ca0dbe 5037 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5038
fef45f4c
AD		 5039 if (retval) {
5040 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5041 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5042 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5043 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5044 return;
5045 goto out;
5046 }
4ae196df
AD		 5047
5048 /* this is a message we already processed, do nothing */
5049 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5050 return;
4ae196df
AD		 5051
5052 /*
5053 * until the vf completes a reset it should not be
5054 * allowed to start any configuration.
5055 */
5056
5057 if (msgbuf[0] == E1000_VF_RESET) {
5058 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5059 return;
4ae196df
AD		 5060 }
5061
f2ca0dbe 5062 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5063 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5064 return;
5065 retval = -1;
5066 goto out;
4ae196df
AD		 5067 }
5068
5069 switch ((msgbuf[0] & 0xFFFF)) {
5070 case E1000_VF_SET_MAC_ADDR:
5071 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5072 break;
7d5753f0
AD		 5073 case E1000_VF_SET_PROMISC:
5074 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5075 break;
4ae196df
AD		 5076 case E1000_VF_SET_MULTICAST:
5077 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5078 break;
5079 case E1000_VF_SET_LPE:
5080 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5081 break;
5082 case E1000_VF_SET_VLAN:
8151d294
WM		 5083 if (adapter->vf_data[vf].pf_vlan)
5084 retval = -1;
5085 else
5086 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5087 break;
5088 default:
090b1795 5089 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5090 retval = -1;
5091 break;
5092 }
5093
fef45f4c
AD		 5094 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5095out:
4ae196df
AD		 5096 /* notify the VF of the results of what it sent us */
5097 if (retval)
5098 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5099 else
5100 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5101
4ae196df 5102 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5103}
4ae196df 5104
f2ca0dbe
AD		 5105static void igb_msg_task(struct igb_adapter *adapter)
5106{
5107 struct e1000_hw *hw = &adapter->hw;
5108 u32 vf;
5109
5110 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5111 /* process any reset requests */
5112 if (!igb_check_for_rst(hw, vf))
5113 igb_vf_reset_event(adapter, vf);
5114
5115 /* process any messages pending */
5116 if (!igb_check_for_msg(hw, vf))
5117 igb_rcv_msg_from_vf(adapter, vf);
5118
5119 /* process any acks */
5120 if (!igb_check_for_ack(hw, vf))
5121 igb_rcv_ack_from_vf(adapter, vf);
5122 }
4ae196df
AD		 5123}
5124
68d480c4
AD		 5125/**
5126 * igb_set_uta - Set unicast filter table address
5127 * @adapter: board private structure
5128 *
5129 * The unicast table address is a register array of 32-bit registers.
5130 * The table is meant to be used in a way similar to how the MTA is used
5131 * however due to certain limitations in the hardware it is necessary to
5132 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5133 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5134 **/
5135static void igb_set_uta(struct igb_adapter *adapter)
5136{
5137 struct e1000_hw *hw = &adapter->hw;
5138 int i;
5139
5140 /* The UTA table only exists on 82576 hardware and newer */
5141 if (hw->mac.type < e1000_82576)
5142 return;
5143
5144 /* we only need to do this if VMDq is enabled */
5145 if (!adapter->vfs_allocated_count)
5146 return;
5147
5148 for (i = 0; i < hw->mac.uta_reg_count; i++)
5149 array_wr32(E1000_UTA, i, ~0);
5150}
5151
9d5c8243
AK		 5152/**
5153 * igb_intr_msi - Interrupt Handler
5154 * @irq: interrupt number
5155 * @data: pointer to a network interface device structure
5156 **/
5157static irqreturn_t igb_intr_msi(int irq, void *data)
5158{
047e0030
AD		 5159 struct igb_adapter *adapter = data;
5160 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5161 struct e1000_hw *hw = &adapter->hw;
5162 /* read ICR disables interrupts using IAM */
5163 u32 icr = rd32(E1000_ICR);
5164
047e0030 5165 igb_write_itr(q_vector);
9d5c8243 5166
7f081d40
AD		 5167 if (icr & E1000_ICR_DRSTA)
5168 schedule_work(&adapter->reset_task);
5169
047e0030 5170 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5171 /* HW is reporting DMA is out of sync */
5172 adapter->stats.doosync++;
5173 }
5174
9d5c8243
AK		 5175 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5176 hw->mac.get_link_status = 1;
5177 if (!test_bit(__IGB_DOWN, &adapter->state))
5178 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5179 }
5180
047e0030 5181 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5182
5183 return IRQ_HANDLED;
5184}
5185
5186/**
4a3c6433 5187 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5188 * @irq: interrupt number
5189 * @data: pointer to a network interface device structure
5190 **/
5191static irqreturn_t igb_intr(int irq, void *data)
5192{
047e0030
AD		 5193 struct igb_adapter *adapter = data;
5194 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5195 struct e1000_hw *hw = &adapter->hw;
5196 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5197 * need for the IMC write */
5198 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5199 if (!icr)
5200 return IRQ_NONE; /* Not our interrupt */
5201
047e0030 5202 igb_write_itr(q_vector);
9d5c8243
AK		 5203
5204 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5205 * not set, then the adapter didn't send an interrupt */
5206 if (!(icr & E1000_ICR_INT_ASSERTED))
5207 return IRQ_NONE;
5208
7f081d40
AD		 5209 if (icr & E1000_ICR_DRSTA)
5210 schedule_work(&adapter->reset_task);
5211
047e0030 5212 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5213 /* HW is reporting DMA is out of sync */
5214 adapter->stats.doosync++;
5215 }
5216
9d5c8243
AK		 5217 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5218 hw->mac.get_link_status = 1;
5219 /* guard against interrupt when we're going down */
5220 if (!test_bit(__IGB_DOWN, &adapter->state))
5221 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5222 }
5223
047e0030 5224 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5225
5226 return IRQ_HANDLED;
5227}
5228
047e0030 5229static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5230{
047e0030 5231 struct igb_adapter *adapter = q_vector->adapter;
46544258 5232 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5233
4fc82adf
AD		 5234 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5235 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5236 if (!adapter->msix_entries)
6eb5a7f1 5237 igb_set_itr(adapter);
46544258 5238 else
047e0030 5239 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5240 }
5241
46544258
AD		 5242 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5243 if (adapter->msix_entries)
047e0030 5244 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5245 else
5246 igb_irq_enable(adapter);
5247 }
9d5c8243
AK		 5248}
5249
46544258
AD		 5250/**
5251 * igb_poll - NAPI Rx polling callback
5252 * @napi: napi polling structure
5253 * @budget: count of how many packets we should handle
5254 **/
5255static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5256{
047e0030
AD		 5257 struct igb_q_vector *q_vector = container_of(napi,
5258 struct igb_q_vector,
5259 napi);
5260 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5261
421e02f0 5262#ifdef CONFIG_IGB_DCA
047e0030
AD		 5263 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5264 igb_update_dca(q_vector);
fe4506b6 5265#endif
047e0030
AD		 5266 if (q_vector->tx_ring)
5267 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5268
047e0030
AD		 5269 if (q_vector->rx_ring)
5270 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5271
5272 if (!tx_clean_complete)
5273 work_done = budget;
46544258 5274
9d5c8243 5275 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5276 if (work_done < budget) {
288379f0 5277 napi_complete(napi);
047e0030 5278 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5279 }
5280
46544258 5281 return work_done;
9d5c8243 5282}
6d8126f9 5283
33af6bcc 5284/**
c5b9bd5e 5285 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5286 * @adapter: board private structure
c5b9bd5e
AD		 5287 * @shhwtstamps: timestamp structure to update
5288 * @regval: unsigned 64bit system time value.
5289 *
5290 * We need to convert the system time value stored in the RX/TXSTMP registers
5291 * into a hwtstamp which can be used by the upper level timestamping functions
5292 */
5293static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5294 struct skb_shared_hwtstamps *shhwtstamps,
5295 u64 regval)
5296{
5297 u64 ns;
5298
55cac248
AD		 5299 /*
5300 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5301 * 24 to match clock shift we setup earlier.
5302 */
5303 if (adapter->hw.mac.type == e1000_82580)
5304 regval <<= IGB_82580_TSYNC_SHIFT;
5305
c5b9bd5e
AD		 5306 ns = timecounter_cyc2time(&adapter->clock, regval);
5307 timecompare_update(&adapter->compare, ns);
5308 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5309 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5310 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5311}
5312
5313/**
5314 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5315 * @q_vector: pointer to q_vector containing needed info
2873957d 5316 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5317 *
5318 * If we were asked to do hardware stamping and such a time stamp is
5319 * available, then it must have been for this skb here because we only
5320 * allow only one such packet into the queue.
5321 */
2873957d 5322static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5323{
c5b9bd5e 5324 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5325 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5326 struct skb_shared_hwtstamps shhwtstamps;
5327 u64 regval;
33af6bcc 5328
c5b9bd5e 5329 /* if skb does not support hw timestamp or TX stamp not valid exit */
2244d07b 5330 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
c5b9bd5e
AD		 5331 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5332 return;
5333
5334 regval = rd32(E1000_TXSTMPL);
5335 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5336
5337 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5338 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5339}
5340
9d5c8243
AK		 5341/**
5342 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5343 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5344 * returns true if ring is completely cleaned
5345 **/
047e0030 5346static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5347{
047e0030
AD		 5348 struct igb_adapter *adapter = q_vector->adapter;
5349 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5350 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5351 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5352 struct igb_buffer *buffer_info;
0e014cb1 5353 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5354 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5355 unsigned int i, eop, count = 0;
5356 bool cleaned = false;
9d5c8243 5357
9d5c8243 5358 i = tx_ring->next_to_clean;
0e014cb1
AD		 5359 eop = tx_ring->buffer_info[i].next_to_watch;
5360 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5361
5362 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5363 (count < tx_ring->count)) {
2d0bb1c1 5364 rmb(); /* read buffer_info after eop_desc status */
0e014cb1
AD		 5365 for (cleaned = false; !cleaned; count++) {
5366 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5367 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5368 cleaned = (i == eop);
9d5c8243 5369
2873957d
NN		 5370 if (buffer_info->skb) {
5371 total_bytes += buffer_info->bytecount;
9d5c8243 5372 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5373 total_packets += buffer_info->gso_segs;
5374 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5375 }
5376
80785298 5377 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5378 tx_desc->wb.status = 0;
9d5c8243
AK		 5379
5380 i++;
5381 if (i == tx_ring->count)
5382 i = 0;
9d5c8243 5383 }
0e014cb1
AD		 5384 eop = tx_ring->buffer_info[i].next_to_watch;
5385 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5386 }
5387
9d5c8243
AK		 5388 tx_ring->next_to_clean = i;
5389
fc7d345d 5390 if (unlikely(count &&
9d5c8243 5391 netif_carrier_ok(netdev) &&
c493ea45 5392 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5393 /* Make sure that anybody stopping the queue after this
5394 * sees the new next_to_clean.
5395 */
5396 smp_mb();
661086df
PWJ		 5397 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5398 !(test_bit(__IGB_DOWN, &adapter->state))) {
5399 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5400 tx_ring->tx_stats.restart_queue++;
661086df 5401 }
9d5c8243
AK		 5402 }
5403
5404 if (tx_ring->detect_tx_hung) {
5405 /* Detect a transmit hang in hardware, this serializes the
5406 * check with the clearing of time_stamp and movement of i */
5407 tx_ring->detect_tx_hung = false;
5408 if (tx_ring->buffer_info[i].time_stamp &&
5409 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5410 (adapter->tx_timeout_factor * HZ)) &&
5411 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5412
9d5c8243 5413 /* detected Tx unit hang */
59d71989 5414 dev_err(tx_ring->dev,
9d5c8243 5415 "Detected Tx Unit Hang\n"
2d064c06 5416 " Tx Queue <%d>\n"
9d5c8243
AK		 5417 " TDH <%x>\n"
5418 " TDT <%x>\n"
5419 " next_to_use <%x>\n"
5420 " next_to_clean <%x>\n"
9d5c8243
AK		 5421 "buffer_info[next_to_clean]\n"
5422 " time_stamp <%lx>\n"
0e014cb1 5423 " next_to_watch <%x>\n"
9d5c8243
AK		 5424 " jiffies <%lx>\n"
5425 " desc.status <%x>\n",
2d064c06 5426 tx_ring->queue_index,
fce99e34
AD		 5427 readl(tx_ring->head),
5428 readl(tx_ring->tail),
9d5c8243
AK		 5429 tx_ring->next_to_use,
5430 tx_ring->next_to_clean,
f7ba205e 5431 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5432 eop,
9d5c8243 5433 jiffies,
0e014cb1 5434 eop_desc->wb.status);
661086df 5435 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5436 }
5437 }
5438 tx_ring->total_bytes += total_bytes;
5439 tx_ring->total_packets += total_packets;
e21ed353
AD		 5440 tx_ring->tx_stats.bytes += total_bytes;
5441 tx_ring->tx_stats.packets += total_packets;
807540ba 5442 return count < tx_ring->count;
9d5c8243
AK		 5443}
5444
9d5c8243
AK		 5445/**
5446 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5447 * @q_vector: structure containing interrupt and ring information
5448 * @skb: packet to send up
5449 * @vlan_tag: vlan tag for packet
9d5c8243 5450 **/
047e0030
AD		 5451static void igb_receive_skb(struct igb_q_vector *q_vector,
5452 struct sk_buff *skb,
5453 u16 vlan_tag)
5454{
5455 struct igb_adapter *adapter = q_vector->adapter;
5456
31b24b95 5457 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5458 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5459 vlan_tag, skb);
182ff8df 5460 else
047e0030 5461 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5462}
5463
04a5fcaa 5464static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5465 u32 status_err, struct sk_buff *skb)
5466{
bc8acf2c 5467 skb_checksum_none_assert(skb);
9d5c8243
AK		 5468
5469 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5470 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5471 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5472 return;
85ad76b2 5473
9d5c8243
AK		 5474 /* TCP/UDP checksum error bit is set */
5475 if (status_err &
5476 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5477 /*
5478 * work around errata with sctp packets where the TCPE aka
5479 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5480 * packets, (aka let the stack check the crc32c)
5481 */
85ad76b2
AD		 5482 if ((skb->len == 60) &&
5483 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5484 ring->rx_stats.csum_err++;
85ad76b2 5485
9d5c8243 5486 /* let the stack verify checksum errors */
9d5c8243
AK		 5487 return;
5488 }
5489 /* It must be a TCP or UDP packet with a valid checksum */
5490 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5491 skb->ip_summed = CHECKSUM_UNNECESSARY;
5492
59d71989 5493 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5494}
5495
757b77e2 5496static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5497 struct sk_buff *skb)
5498{
5499 struct igb_adapter *adapter = q_vector->adapter;
5500 struct e1000_hw *hw = &adapter->hw;
5501 u64 regval;
5502
5503 /*
5504 * If this bit is set, then the RX registers contain the time stamp. No
5505 * other packet will be time stamped until we read these registers, so
5506 * read the registers to make them available again. Because only one
5507 * packet can be time stamped at a time, we know that the register
5508 * values must belong to this one here and therefore we don't need to
5509 * compare any of the additional attributes stored for it.
5510 *
2244d07b 5511 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5512 * can turn into a skb_shared_hwtstamps.
5513 */
757b77e2
NN		 5514 if (staterr & E1000_RXDADV_STAT_TSIP) {
5515 u32 *stamp = (u32 *)skb->data;
5516 regval = le32_to_cpu(*(stamp + 2));
5517 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5518 skb_pull(skb, IGB_TS_HDR_LEN);
5519 } else {
5520 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5521 return;
c5b9bd5e 5522
757b77e2
NN		 5523 regval = rd32(E1000_RXSTMPL);
5524 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5525 }
c5b9bd5e
AD		 5526
5527 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5528}
4c844851 5529static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5530 union e1000_adv_rx_desc *rx_desc)
5531{
5532 /* HW will not DMA in data larger than the given buffer, even if it
5533 * parses the (NFS, of course) header to be larger. In that case, it
5534 * fills the header buffer and spills the rest into the page.
5535 */
5536 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5537 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5538 if (hlen > rx_ring->rx_buffer_len)
5539 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5540 return hlen;
5541}
5542
047e0030
AD		 5543static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5544 int *work_done, int budget)
9d5c8243 5545{
047e0030 5546 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5547 struct net_device *netdev = rx_ring->netdev;
59d71989 5548 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5549 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5550 struct igb_buffer *buffer_info , *next_buffer;
5551 struct sk_buff *skb;
9d5c8243
AK		 5552 bool cleaned = false;
5553 int cleaned_count = 0;
d1eff350 5554 int current_node = numa_node_id();
9d5c8243 5555 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5556 unsigned int i;
2d94d8ab
AD		 5557 u32 staterr;
5558 u16 length;
047e0030 5559 u16 vlan_tag;
9d5c8243
AK		 5560
5561 i = rx_ring->next_to_clean;
69d3ca53 5562 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5563 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5564 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5565
5566 while (staterr & E1000_RXD_STAT_DD) {
5567 if (*work_done >= budget)
5568 break;
5569 (*work_done)++;
2d0bb1c1 5570 rmb(); /* read descriptor and rx_buffer_info after status DD */
9d5c8243 5571
69d3ca53
AD		 5572 skb = buffer_info->skb;
5573 prefetch(skb->data - NET_IP_ALIGN);
5574 buffer_info->skb = NULL;
5575
5576 i++;
5577 if (i == rx_ring->count)
5578 i = 0;
42d0781a 5579
69d3ca53
AD		 5580 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5581 prefetch(next_rxd);
5582 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5583
5584 length = le16_to_cpu(rx_desc->wb.upper.length);
5585 cleaned = true;
5586 cleaned_count++;
5587
2d94d8ab 5588 if (buffer_info->dma) {
59d71989 5589 dma_unmap_single(dev, buffer_info->dma,
4c844851 5590 rx_ring->rx_buffer_len,
59d71989 5591 DMA_FROM_DEVICE);
91615f76 5592 buffer_info->dma = 0;
4c844851 5593 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5594 skb_put(skb, length);
5595 goto send_up;
5596 }
4c844851 5597 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5598 }
5599
5600 if (length) {
59d71989
AD		 5601 dma_unmap_page(dev, buffer_info->page_dma,
5602 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5603 buffer_info->page_dma = 0;
bf36c1a0 5604
aa913403 5605 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5606 buffer_info->page,
5607 buffer_info->page_offset,
5608 length);
5609
d1eff350
AD		 5610 if ((page_count(buffer_info->page) != 1) ||
5611 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5612 buffer_info->page = NULL;
5613 else
5614 get_page(buffer_info->page);
9d5c8243
AK		 5615
5616 skb->len += length;
5617 skb->data_len += length;
bf36c1a0 5618 skb->truesize += length;
9d5c8243 5619 }
9d5c8243 5620
bf36c1a0 5621 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5622 buffer_info->skb = next_buffer->skb;
5623 buffer_info->dma = next_buffer->dma;
5624 next_buffer->skb = skb;
5625 next_buffer->dma = 0;
bf36c1a0
AD		 5626 goto next_desc;
5627 }
69d3ca53 5628send_up:
9d5c8243
AK		 5629 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5630 dev_kfree_skb_irq(skb);
5631 goto next_desc;
5632 }
9d5c8243 5633
757b77e2
NN		 5634 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5635 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5636 total_bytes += skb->len;
5637 total_packets++;
5638
85ad76b2 5639 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5640
5641 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5642 skb_record_rx_queue(skb, rx_ring->queue_index);
5643
5644 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5645 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5646
047e0030 5647 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5648
9d5c8243
AK		 5649next_desc:
5650 rx_desc->wb.upper.status_error = 0;
5651
5652 /* return some buffers to hardware, one at a time is too slow */
5653 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5654 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5655 cleaned_count = 0;
5656 }
5657
5658 /* use prefetched values */
5659 rx_desc = next_rxd;
5660 buffer_info = next_buffer;
9d5c8243
AK		 5661 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5662 }
bf36c1a0 5663
9d5c8243 5664 rx_ring->next_to_clean = i;
c493ea45 5665 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5666
5667 if (cleaned_count)
3b644cf6 5668 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5669
5670 rx_ring->total_packets += total_packets;
5671 rx_ring->total_bytes += total_bytes;
5672 rx_ring->rx_stats.packets += total_packets;
5673 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5674 return cleaned;
5675}
5676
9d5c8243
AK		 5677/**
5678 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5679 * @adapter: address of board private structure
5680 **/
d7ee5b3a 5681void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5682{
e694e964 5683 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5684 union e1000_adv_rx_desc *rx_desc;
5685 struct igb_buffer *buffer_info;
5686 struct sk_buff *skb;
5687 unsigned int i;
db761762 5688 int bufsz;
9d5c8243
AK		 5689
5690 i = rx_ring->next_to_use;
5691 buffer_info = &rx_ring->buffer_info[i];
5692
4c844851 5693 bufsz = rx_ring->rx_buffer_len;
db761762 5694
9d5c8243
AK		 5695 while (cleaned_count--) {
5696 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5697
6ec43fe6 5698 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5699 if (!buffer_info->page) {
42d0781a 5700 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5701 if (!buffer_info->page) {
04a5fcaa 5702 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5703 goto no_buffers;
5704 }
5705 buffer_info->page_offset = 0;
5706 } else {
5707 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5708 }
5709 buffer_info->page_dma =
59d71989 5710 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5711 buffer_info->page_offset,
5712 PAGE_SIZE / 2,
59d71989
AD		 5713 DMA_FROM_DEVICE);
5714 if (dma_mapping_error(rx_ring->dev,
5715 buffer_info->page_dma)) {
42d0781a
AD		 5716 buffer_info->page_dma = 0;
5717 rx_ring->rx_stats.alloc_failed++;
5718 goto no_buffers;
5719 }
9d5c8243
AK		 5720 }
5721
42d0781a
AD		 5722 skb = buffer_info->skb;
5723 if (!skb) {
89d71a66 5724 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5725 if (!skb) {
04a5fcaa 5726 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5727 goto no_buffers;
5728 }
5729
9d5c8243 5730 buffer_info->skb = skb;
42d0781a
AD		 5731 }
5732 if (!buffer_info->dma) {
59d71989 5733 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5734 skb->data,
9d5c8243 5735 bufsz,
59d71989
AD		 5736 DMA_FROM_DEVICE);
5737 if (dma_mapping_error(rx_ring->dev,
5738 buffer_info->dma)) {
42d0781a
AD		 5739 buffer_info->dma = 0;
5740 rx_ring->rx_stats.alloc_failed++;
5741 goto no_buffers;
5742 }
9d5c8243
AK		 5743 }
5744 /* Refresh the desc even if buffer_addrs didn't change because
5745 * each write-back erases this info. */
6ec43fe6 5746 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5747 rx_desc->read.pkt_addr =
5748 cpu_to_le64(buffer_info->page_dma);
5749 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5750 } else {
42d0781a 5751 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5752 rx_desc->read.hdr_addr = 0;
5753 }
5754
5755 i++;
5756 if (i == rx_ring->count)
5757 i = 0;
5758 buffer_info = &rx_ring->buffer_info[i];
5759 }
5760
5761no_buffers:
5762 if (rx_ring->next_to_use != i) {
5763 rx_ring->next_to_use = i;
5764 if (i == 0)
5765 i = (rx_ring->count - 1);
5766 else
5767 i--;
5768
5769 /* Force memory writes to complete before letting h/w
5770 * know there are new descriptors to fetch. (Only
5771 * applicable for weak-ordered memory model archs,
5772 * such as IA-64). */
5773 wmb();
fce99e34 5774 writel(i, rx_ring->tail);
9d5c8243
AK		 5775 }
5776}
5777
5778/**
5779 * igb_mii_ioctl -
5780 * @netdev:
5781 * @ifreq:
5782 * @cmd:
5783 **/
5784static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5785{
5786 struct igb_adapter *adapter = netdev_priv(netdev);
5787 struct mii_ioctl_data *data = if_mii(ifr);
5788
5789 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5790 return -EOPNOTSUPP;
5791
5792 switch (cmd) {
5793 case SIOCGMIIPHY:
5794 data->phy_id = adapter->hw.phy.addr;
5795 break;
5796 case SIOCGMIIREG:
f5f4cf08
AD		 5797 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5798 &data->val_out))
9d5c8243
AK		 5799 return -EIO;
5800 break;
5801 case SIOCSMIIREG:
5802 default:
5803 return -EOPNOTSUPP;
5804 }
5805 return 0;
5806}
5807
c6cb090b
PO		 5808/**
5809 * igb_hwtstamp_ioctl - control hardware time stamping
5810 * @netdev:
5811 * @ifreq:
5812 * @cmd:
5813 *
33af6bcc
PO		 5814 * Outgoing time stamping can be enabled and disabled. Play nice and
5815 * disable it when requested, although it shouldn't case any overhead
5816 * when no packet needs it. At most one packet in the queue may be
5817 * marked for time stamping, otherwise it would be impossible to tell
5818 * for sure to which packet the hardware time stamp belongs.
5819 *
5820 * Incoming time stamping has to be configured via the hardware
5821 * filters. Not all combinations are supported, in particular event
5822 * type has to be specified. Matching the kind of event packet is
5823 * not supported, with the exception of "all V2 events regardless of
5824 * level 2 or 4".
5825 *
c6cb090b
PO		 5826 **/
5827static int igb_hwtstamp_ioctl(struct net_device *netdev,
5828 struct ifreq *ifr, int cmd)
5829{
33af6bcc
PO		 5830 struct igb_adapter *adapter = netdev_priv(netdev);
5831 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5832 struct hwtstamp_config config;
c5b9bd5e
AD		 5833 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5834 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5835 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5836 bool is_l4 = false;
5837 bool is_l2 = false;
33af6bcc 5838 u32 regval;
c6cb090b
PO		 5839
5840 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5841 return -EFAULT;
5842
5843 /* reserved for future extensions */
5844 if (config.flags)
5845 return -EINVAL;
5846
33af6bcc
PO		 5847 switch (config.tx_type) {
5848 case HWTSTAMP_TX_OFF:
c5b9bd5e 5849 tsync_tx_ctl = 0;
33af6bcc 5850 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5851 break;
5852 default:
5853 return -ERANGE;
5854 }
5855
5856 switch (config.rx_filter) {
5857 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5858 tsync_rx_ctl = 0;
33af6bcc
PO		 5859 break;
5860 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5861 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5862 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5863 case HWTSTAMP_FILTER_ALL:
5864 /*
5865 * register TSYNCRXCFG must be set, therefore it is not
5866 * possible to time stamp both Sync and Delay_Req messages
5867 * => fall back to time stamping all packets
5868 */
c5b9bd5e 5869 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5870 config.rx_filter = HWTSTAMP_FILTER_ALL;
5871 break;
5872 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5873 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5874 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5875 is_l4 = true;
33af6bcc
PO		 5876 break;
5877 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5878 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5879 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5880 is_l4 = true;
33af6bcc
PO		 5881 break;
5882 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5883 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5884 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5885 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5886 is_l2 = true;
5887 is_l4 = true;
33af6bcc
PO		 5888 config.rx_filter = HWTSTAMP_FILTER_SOME;
5889 break;
5890 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5891 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5892 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5893 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5894 is_l2 = true;
5895 is_l4 = true;
33af6bcc
PO		 5896 config.rx_filter = HWTSTAMP_FILTER_SOME;
5897 break;
5898 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5899 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5900 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5901 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5902 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5903 is_l2 = true;
33af6bcc
PO		 5904 break;
5905 default:
5906 return -ERANGE;
5907 }
5908
c5b9bd5e
AD		 5909 if (hw->mac.type == e1000_82575) {
5910 if (tsync_rx_ctl | tsync_tx_ctl)
5911 return -EINVAL;
5912 return 0;
5913 }
5914
757b77e2
NN		 5915 /*
5916 * Per-packet timestamping only works if all packets are
5917 * timestamped, so enable timestamping in all packets as
5918 * long as one rx filter was configured.
5919 */
5920 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5921 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5922 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5923 }
5924
33af6bcc
PO		 5925 /* enable/disable TX */
5926 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5927 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5928 regval |= tsync_tx_ctl;
33af6bcc
PO		 5929 wr32(E1000_TSYNCTXCTL, regval);
5930
c5b9bd5e 5931 /* enable/disable RX */
33af6bcc 5932 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5933 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5934 regval |= tsync_rx_ctl;
33af6bcc 5935 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5936
c5b9bd5e
AD		 5937 /* define which PTP packets are time stamped */
5938 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5939
c5b9bd5e
AD		 5940 /* define ethertype filter for timestamped packets */
5941 if (is_l2)
5942 wr32(E1000_ETQF(3),
5943 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5944 E1000_ETQF_1588 | /* enable timestamping */
5945 ETH_P_1588)); /* 1588 eth protocol type */
5946 else
5947 wr32(E1000_ETQF(3), 0);
5948
5949#define PTP_PORT 319
5950 /* L4 Queue Filter[3]: filter by destination port and protocol */
5951 if (is_l4) {
5952 u32 ftqf = (IPPROTO_UDP /* UDP */
5953 | E1000_FTQF_VF_BP /* VF not compared */
5954 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5955 | E1000_FTQF_MASK); /* mask all inputs */
5956 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5957
5958 wr32(E1000_IMIR(3), htons(PTP_PORT));
5959 wr32(E1000_IMIREXT(3),
5960 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5961 if (hw->mac.type == e1000_82576) {
5962 /* enable source port check */
5963 wr32(E1000_SPQF(3), htons(PTP_PORT));
5964 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5965 }
5966 wr32(E1000_FTQF(3), ftqf);
5967 } else {
5968 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5969 }
33af6bcc
PO		 5970 wrfl();
5971
5972 adapter->hwtstamp_config = config;
5973
5974 /* clear TX/RX time stamp registers, just to be sure */
5975 regval = rd32(E1000_TXSTMPH);
5976 regval = rd32(E1000_RXSTMPH);
c6cb090b 5977
33af6bcc
PO		 5978 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5979 -EFAULT : 0;
c6cb090b
PO		 5980}
5981
9d5c8243
AK		 5982/**
5983 * igb_ioctl -
5984 * @netdev:
5985 * @ifreq:
5986 * @cmd:
5987 **/
5988static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5989{
5990 switch (cmd) {
5991 case SIOCGMIIPHY:
5992 case SIOCGMIIREG:
5993 case SIOCSMIIREG:
5994 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5995 case SIOCSHWTSTAMP:
5996 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5997 default:
5998 return -EOPNOTSUPP;
5999 }
6000}
6001
009bc06e
AD		 6002s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6003{
6004 struct igb_adapter *adapter = hw->back;
6005 u16 cap_offset;
6006
6007 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6008 if (!cap_offset)
6009 return -E1000_ERR_CONFIG;
6010
6011 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6012
6013 return 0;
6014}
6015
6016s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6017{
6018 struct igb_adapter *adapter = hw->back;
6019 u16 cap_offset;
6020
6021 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6022 if (!cap_offset)
6023 return -E1000_ERR_CONFIG;
6024
6025 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6026
6027 return 0;
6028}
6029
9d5c8243
AK		 6030static void igb_vlan_rx_register(struct net_device *netdev,
6031 struct vlan_group *grp)
6032{
6033 struct igb_adapter *adapter = netdev_priv(netdev);
6034 struct e1000_hw *hw = &adapter->hw;
6035 u32 ctrl, rctl;
6036
6037 igb_irq_disable(adapter);
6038 adapter->vlgrp = grp;
6039
6040 if (grp) {
6041 /* enable VLAN tag insert/strip */
6042 ctrl = rd32(E1000_CTRL);
6043 ctrl |= E1000_CTRL_VME;
6044 wr32(E1000_CTRL, ctrl);
6045
51466239 6046 /* Disable CFI check */
9d5c8243 6047 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6048 rctl &= ~E1000_RCTL_CFIEN;
6049 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6050 } else {
6051 /* disable VLAN tag insert/strip */
6052 ctrl = rd32(E1000_CTRL);
6053 ctrl &= ~E1000_CTRL_VME;
6054 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6055 }
6056
e1739522
AD		 6057 igb_rlpml_set(adapter);
6058
9d5c8243
AK		 6059 if (!test_bit(__IGB_DOWN, &adapter->state))
6060 igb_irq_enable(adapter);
6061}
6062
6063static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6064{
6065 struct igb_adapter *adapter = netdev_priv(netdev);
6066 struct e1000_hw *hw = &adapter->hw;
4ae196df 6067 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6068
51466239
AD		 6069 /* attempt to add filter to vlvf array */
6070 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6071
51466239
AD		 6072 /* add the filter since PF can receive vlans w/o entry in vlvf */
6073 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 6074}
6075
6076static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6077{
6078 struct igb_adapter *adapter = netdev_priv(netdev);
6079 struct e1000_hw *hw = &adapter->hw;
4ae196df 6080 int pf_id = adapter->vfs_allocated_count;
51466239 6081 s32 err;
9d5c8243
AK		 6082
6083 igb_irq_disable(adapter);
6084 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6085
6086 if (!test_bit(__IGB_DOWN, &adapter->state))
6087 igb_irq_enable(adapter);
6088
51466239
AD		 6089 /* remove vlan from VLVF table array */
6090 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6091
51466239
AD		 6092 /* if vid was not present in VLVF just remove it from table */
6093 if (err)
4ae196df 6094 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 6095}
6096
6097static void igb_restore_vlan(struct igb_adapter *adapter)
6098{
6099 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6100
6101 if (adapter->vlgrp) {
6102 u16 vid;
6103 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6104 if (!vlan_group_get_device(adapter->vlgrp, vid))
6105 continue;
6106 igb_vlan_rx_add_vid(adapter->netdev, vid);
6107 }
6108 }
6109}
6110
6111int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6112{
090b1795 6113 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6114 struct e1000_mac_info *mac = &adapter->hw.mac;
6115
6116 mac->autoneg = 0;
6117
cd2638a8
CW		 6118 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6119 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6120 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
6121 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6122 return -EINVAL;
6123 }
6124
9d5c8243
AK		 6125 switch (spddplx) {
6126 case SPEED_10 + DUPLEX_HALF:
6127 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6128 break;
6129 case SPEED_10 + DUPLEX_FULL:
6130 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6131 break;
6132 case SPEED_100 + DUPLEX_HALF:
6133 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6134 break;
6135 case SPEED_100 + DUPLEX_FULL:
6136 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6137 break;
6138 case SPEED_1000 + DUPLEX_FULL:
6139 mac->autoneg = 1;
6140 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6141 break;
6142 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6143 default:
090b1795 6144 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 6145 return -EINVAL;
6146 }
6147 return 0;
6148}
6149
3fe7c4c9 6150static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6151{
6152 struct net_device *netdev = pci_get_drvdata(pdev);
6153 struct igb_adapter *adapter = netdev_priv(netdev);
6154 struct e1000_hw *hw = &adapter->hw;
2d064c06 6155 u32 ctrl, rctl, status;
9d5c8243
AK		 6156 u32 wufc = adapter->wol;
6157#ifdef CONFIG_PM
6158 int retval = 0;
6159#endif
6160
6161 netif_device_detach(netdev);
6162
a88f10ec
AD		 6163 if (netif_running(netdev))
6164 igb_close(netdev);
6165
047e0030 6166 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6167
6168#ifdef CONFIG_PM
6169 retval = pci_save_state(pdev);
6170 if (retval)
6171 return retval;
6172#endif
6173
6174 status = rd32(E1000_STATUS);
6175 if (status & E1000_STATUS_LU)
6176 wufc &= ~E1000_WUFC_LNKC;
6177
6178 if (wufc) {
6179 igb_setup_rctl(adapter);
ff41f8dc 6180 igb_set_rx_mode(netdev);
9d5c8243
AK		 6181
6182 /* turn on all-multi mode if wake on multicast is enabled */
6183 if (wufc & E1000_WUFC_MC) {
6184 rctl = rd32(E1000_RCTL);
6185 rctl |= E1000_RCTL_MPE;
6186 wr32(E1000_RCTL, rctl);
6187 }
6188
6189 ctrl = rd32(E1000_CTRL);
6190 /* advertise wake from D3Cold */
6191 #define E1000_CTRL_ADVD3WUC 0x00100000
6192 /* phy power management enable */
6193 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6194 ctrl |= E1000_CTRL_ADVD3WUC;
6195 wr32(E1000_CTRL, ctrl);
6196
9d5c8243 6197 /* Allow time for pending master requests to run */
330a6d6a 6198 igb_disable_pcie_master(hw);
9d5c8243
AK		 6199
6200 wr32(E1000_WUC, E1000_WUC_PME_EN);
6201 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6202 } else {
6203 wr32(E1000_WUC, 0);
6204 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6205 }
6206
3fe7c4c9
RW		 6207 *enable_wake = wufc || adapter->en_mng_pt;
6208 if (!*enable_wake)
88a268c1
NN		 6209 igb_power_down_link(adapter);
6210 else
6211 igb_power_up_link(adapter);
9d5c8243
AK		 6212
6213 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6214 * would have already happened in close and is redundant. */
6215 igb_release_hw_control(adapter);
6216
6217 pci_disable_device(pdev);
6218
9d5c8243
AK		 6219 return 0;
6220}
6221
6222#ifdef CONFIG_PM
3fe7c4c9
RW		 6223static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6224{
6225 int retval;
6226 bool wake;
6227
6228 retval = __igb_shutdown(pdev, &wake);
6229 if (retval)
6230 return retval;
6231
6232 if (wake) {
6233 pci_prepare_to_sleep(pdev);
6234 } else {
6235 pci_wake_from_d3(pdev, false);
6236 pci_set_power_state(pdev, PCI_D3hot);
6237 }
6238
6239 return 0;
6240}
6241
9d5c8243
AK		 6242static int igb_resume(struct pci_dev *pdev)
6243{
6244 struct net_device *netdev = pci_get_drvdata(pdev);
6245 struct igb_adapter *adapter = netdev_priv(netdev);
6246 struct e1000_hw *hw = &adapter->hw;
6247 u32 err;
6248
6249 pci_set_power_state(pdev, PCI_D0);
6250 pci_restore_state(pdev);
b94f2d77 6251 pci_save_state(pdev);
42bfd33a 6252
aed5dec3 6253 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6254 if (err) {
6255 dev_err(&pdev->dev,
6256 "igb: Cannot enable PCI device from suspend\n");
6257 return err;
6258 }
6259 pci_set_master(pdev);
6260
6261 pci_enable_wake(pdev, PCI_D3hot, 0);
6262 pci_enable_wake(pdev, PCI_D3cold, 0);
6263
047e0030 6264 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6265 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6266 return -ENOMEM;
9d5c8243
AK		 6267 }
6268
9d5c8243 6269 igb_reset(adapter);
a8564f03
AD		 6270
6271 /* let the f/w know that the h/w is now under the control of the
6272 * driver. */
6273 igb_get_hw_control(adapter);
6274
9d5c8243
AK		 6275 wr32(E1000_WUS, ~0);
6276
a88f10ec
AD		 6277 if (netif_running(netdev)) {
6278 err = igb_open(netdev);
6279 if (err)
6280 return err;
6281 }
9d5c8243
AK		 6282
6283 netif_device_attach(netdev);
6284
9d5c8243
AK		 6285 return 0;
6286}
6287#endif
6288
6289static void igb_shutdown(struct pci_dev *pdev)
6290{
3fe7c4c9
RW		 6291 bool wake;
6292
6293 __igb_shutdown(pdev, &wake);
6294
6295 if (system_state == SYSTEM_POWER_OFF) {
6296 pci_wake_from_d3(pdev, wake);
6297 pci_set_power_state(pdev, PCI_D3hot);
6298 }
9d5c8243
AK		 6299}
6300
6301#ifdef CONFIG_NET_POLL_CONTROLLER
6302/*
6303 * Polling 'interrupt' - used by things like netconsole to send skbs
6304 * without having to re-enable interrupts. It's not called while
6305 * the interrupt routine is executing.
6306 */
6307static void igb_netpoll(struct net_device *netdev)
6308{
6309 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6310 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6311 int i;
9d5c8243 6312
eebbbdba 6313 if (!adapter->msix_entries) {
047e0030 6314 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6315 igb_irq_disable(adapter);
047e0030 6316 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6317 return;
6318 }
9d5c8243 6319
047e0030
AD		 6320 for (i = 0; i < adapter->num_q_vectors; i++) {
6321 struct igb_q_vector *q_vector = adapter->q_vector[i];
6322 wr32(E1000_EIMC, q_vector->eims_value);
6323 napi_schedule(&q_vector->napi);
eebbbdba 6324 }
9d5c8243
AK		 6325}
6326#endif /* CONFIG_NET_POLL_CONTROLLER */
6327
6328/**
6329 * igb_io_error_detected - called when PCI error is detected
6330 * @pdev: Pointer to PCI device
6331 * @state: The current pci connection state
6332 *
6333 * This function is called after a PCI bus error affecting
6334 * this device has been detected.
6335 */
6336static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6337 pci_channel_state_t state)
6338{
6339 struct net_device *netdev = pci_get_drvdata(pdev);
6340 struct igb_adapter *adapter = netdev_priv(netdev);
6341
6342 netif_device_detach(netdev);
6343
59ed6eec
AD		 6344 if (state == pci_channel_io_perm_failure)
6345 return PCI_ERS_RESULT_DISCONNECT;
6346
9d5c8243
AK		 6347 if (netif_running(netdev))
6348 igb_down(adapter);
6349 pci_disable_device(pdev);
6350
6351 /* Request a slot slot reset. */
6352 return PCI_ERS_RESULT_NEED_RESET;
6353}
6354
6355/**
6356 * igb_io_slot_reset - called after the pci bus has been reset.
6357 * @pdev: Pointer to PCI device
6358 *
6359 * Restart the card from scratch, as if from a cold-boot. Implementation
6360 * resembles the first-half of the igb_resume routine.
6361 */
6362static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6363{
6364 struct net_device *netdev = pci_get_drvdata(pdev);
6365 struct igb_adapter *adapter = netdev_priv(netdev);
6366 struct e1000_hw *hw = &adapter->hw;
40a914fa 6367 pci_ers_result_t result;
42bfd33a 6368 int err;
9d5c8243 6369
aed5dec3 6370 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6371 dev_err(&pdev->dev,
6372 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6373 result = PCI_ERS_RESULT_DISCONNECT;
6374 } else {
6375 pci_set_master(pdev);
6376 pci_restore_state(pdev);
b94f2d77 6377 pci_save_state(pdev);
9d5c8243 6378
40a914fa
AD		 6379 pci_enable_wake(pdev, PCI_D3hot, 0);
6380 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6381
40a914fa
AD		 6382 igb_reset(adapter);
6383 wr32(E1000_WUS, ~0);
6384 result = PCI_ERS_RESULT_RECOVERED;
6385 }
9d5c8243 6386
ea943d41
JK		 6387 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6388 if (err) {
6389 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6390 "failed 0x%0x\n", err);
6391 /* non-fatal, continue */
6392 }
40a914fa
AD		 6393
6394 return result;
9d5c8243
AK		 6395}
6396
6397/**
6398 * igb_io_resume - called when traffic can start flowing again.
6399 * @pdev: Pointer to PCI device
6400 *
6401 * This callback is called when the error recovery driver tells us that
6402 * its OK to resume normal operation. Implementation resembles the
6403 * second-half of the igb_resume routine.
6404 */
6405static void igb_io_resume(struct pci_dev *pdev)
6406{
6407 struct net_device *netdev = pci_get_drvdata(pdev);
6408 struct igb_adapter *adapter = netdev_priv(netdev);
6409
9d5c8243
AK		 6410 if (netif_running(netdev)) {
6411 if (igb_up(adapter)) {
6412 dev_err(&pdev->dev, "igb_up failed after reset\n");
6413 return;
6414 }
6415 }
6416
6417 netif_device_attach(netdev);
6418
6419 /* let the f/w know that the h/w is now under the control of the
6420 * driver. */
6421 igb_get_hw_control(adapter);
9d5c8243
AK		 6422}
6423
26ad9178
AD		 6424static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6425 u8 qsel)
6426{
6427 u32 rar_low, rar_high;
6428 struct e1000_hw *hw = &adapter->hw;
6429
6430 /* HW expects these in little endian so we reverse the byte order
6431 * from network order (big endian) to little endian
6432 */
6433 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6434 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6435 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6436
6437 /* Indicate to hardware the Address is Valid. */
6438 rar_high |= E1000_RAH_AV;
6439
6440 if (hw->mac.type == e1000_82575)
6441 rar_high |= E1000_RAH_POOL_1 * qsel;
6442 else
6443 rar_high |= E1000_RAH_POOL_1 << qsel;
6444
6445 wr32(E1000_RAL(index), rar_low);
6446 wrfl();
6447 wr32(E1000_RAH(index), rar_high);
6448 wrfl();
6449}
6450
4ae196df
AD		 6451static int igb_set_vf_mac(struct igb_adapter *adapter,
6452 int vf, unsigned char *mac_addr)
6453{
6454 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6455 /* VF MAC addresses start at end of receive addresses and moves
6456 * torwards the first, as a result a collision should not be possible */
6457 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6458
37680117 6459 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6460
26ad9178 6461 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6462
6463 return 0;
6464}
6465
8151d294
WM		 6466static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6467{
6468 struct igb_adapter *adapter = netdev_priv(netdev);
6469 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6470 return -EINVAL;
6471 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6472 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6473 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6474 " change effective.");
6475 if (test_bit(__IGB_DOWN, &adapter->state)) {
6476 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6477 " but the PF device is not up.\n");
6478 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6479 " attempting to use the VF device.\n");
6480 }
6481 return igb_set_vf_mac(adapter, vf, mac);
6482}
6483
6484static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6485{
6486 return -EOPNOTSUPP;
6487}
6488
6489static int igb_ndo_get_vf_config(struct net_device *netdev,
6490 int vf, struct ifla_vf_info *ivi)
6491{
6492 struct igb_adapter *adapter = netdev_priv(netdev);
6493 if (vf >= adapter->vfs_allocated_count)
6494 return -EINVAL;
6495 ivi->vf = vf;
6496 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6497 ivi->tx_rate = 0;
6498 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6499 ivi->qos = adapter->vf_data[vf].pf_qos;
6500 return 0;
6501}
6502
4ae196df
AD		 6503static void igb_vmm_control(struct igb_adapter *adapter)
6504{
6505 struct e1000_hw *hw = &adapter->hw;
10d8e907 6506 u32 reg;
4ae196df 6507
52a1dd4d
AD		 6508 switch (hw->mac.type) {
6509 case e1000_82575:
6510 default:
6511 /* replication is not supported for 82575 */
4ae196df 6512 return;
52a1dd4d
AD		 6513 case e1000_82576:
6514 /* notify HW that the MAC is adding vlan tags */
6515 reg = rd32(E1000_DTXCTL);
6516 reg |= E1000_DTXCTL_VLAN_ADDED;
6517 wr32(E1000_DTXCTL, reg);
6518 case e1000_82580:
6519 /* enable replication vlan tag stripping */
6520 reg = rd32(E1000_RPLOLR);
6521 reg |= E1000_RPLOLR_STRVLAN;
6522 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6523 case e1000_i350:
6524 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6525 break;
6526 }
10d8e907 6527
d4960307
AD		 6528 if (adapter->vfs_allocated_count) {
6529 igb_vmdq_set_loopback_pf(hw, true);
6530 igb_vmdq_set_replication_pf(hw, true);
6531 } else {
6532 igb_vmdq_set_loopback_pf(hw, false);
6533 igb_vmdq_set_replication_pf(hw, false);
6534 }
4ae196df
AD		 6535}
6536
9d5c8243 6537/* igb_main.c */
kernel source for telekom puls (alcatel/mediatek)