projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xfrm6: Don't call icmpv6_send on local error
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4297f99b 4 Copyright(c) 2007-2011 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
b2cb09b1 31#include <linux/bitops.h>
9d5c8243
AK		 32#include <linux/vmalloc.h>
33#include <linux/pagemap.h>
34#include <linux/netdevice.h>
9d5c8243 35#include <linux/ipv6.h>
5a0e3ad6 36#include <linux/slab.h>
9d5c8243
AK		 37#include <net/checksum.h>
38#include <net/ip6_checksum.h>
c6cb090b 39#include <linux/net_tstamp.h>
9d5c8243
AK		 40#include <linux/mii.h>
41#include <linux/ethtool.h>
01789349 42#include <linux/if.h>
9d5c8243
AK		 43#include <linux/if_vlan.h>
44#include <linux/pci.h>
c54106bb 45#include <linux/pci-aspm.h>
9d5c8243
AK		 46#include <linux/delay.h>
47#include <linux/interrupt.h>
7d13a7d0
AD		 48#include <linux/ip.h>
49#include <linux/tcp.h>
50#include <linux/sctp.h>
9d5c8243 51#include <linux/if_ether.h>
40a914fa 52#include <linux/aer.h>
70c71606 53#include <linux/prefetch.h>
421e02f0 54#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 55#include <linux/dca.h>
56#endif
9d5c8243
AK		 57#include "igb.h"
58
0d1fe82d 59#define MAJ 3
a28dc43f
CW		 60#define MIN 2
61#define BUILD 10
0d1fe82d 62#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 63__stringify(BUILD) "-k"
9d5c8243
AK		 64char igb_driver_name[] = "igb";
65char igb_driver_version[] = DRV_VERSION;
66static const char igb_driver_string[] =
67 "Intel(R) Gigabit Ethernet Network Driver";
4c4b42cb 68static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
9d5c8243 69
9d5c8243
AK		 70static const struct e1000_info *igb_info_tbl[] = {
71 [board_82575] = &e1000_82575_info,
72};
73
a3aa1884 74static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
100 /* required last entry */
101 {0, }
102};
103
104MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
105
106void igb_reset(struct igb_adapter *);
107static int igb_setup_all_tx_resources(struct igb_adapter *);
108static int igb_setup_all_rx_resources(struct igb_adapter *);
109static void igb_free_all_tx_resources(struct igb_adapter *);
110static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 111static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 112static int igb_probe(struct pci_dev *, const struct pci_device_id *);
113static void __devexit igb_remove(struct pci_dev *pdev);
673b8b70 114static void igb_init_hw_timer(struct igb_adapter *adapter);
9d5c8243
AK		 115static int igb_sw_init(struct igb_adapter *);
116static int igb_open(struct net_device *);
117static int igb_close(struct net_device *);
118static void igb_configure_tx(struct igb_adapter *);
119static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 120static void igb_clean_all_tx_rings(struct igb_adapter *);
121static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 122static void igb_clean_tx_ring(struct igb_ring *);
123static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 124static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 125static void igb_update_phy_info(unsigned long);
126static void igb_watchdog(unsigned long);
127static void igb_watchdog_task(struct work_struct *);
cd392f5c 128static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 129static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
130 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 131static int igb_change_mtu(struct net_device *, int);
132static int igb_set_mac(struct net_device *, void *);
68d480c4 133static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 134static irqreturn_t igb_intr(int irq, void *);
135static irqreturn_t igb_intr_msi(int irq, void *);
136static irqreturn_t igb_msix_other(int irq, void *);
047e0030 137static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 138#ifdef CONFIG_IGB_DCA
047e0030 139static void igb_update_dca(struct igb_q_vector *);
fe4506b6 140static void igb_setup_dca(struct igb_adapter *);
421e02f0 141#endif /* CONFIG_IGB_DCA */
661086df 142static int igb_poll(struct napi_struct *, int);
13fde97a 143static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 144static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 145static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
146static void igb_tx_timeout(struct net_device *);
147static void igb_reset_task(struct work_struct *);
b2cb09b1 148static void igb_vlan_mode(struct net_device *netdev, u32 features);
9d5c8243
AK		 149static void igb_vlan_rx_add_vid(struct net_device *, u16);
150static void igb_vlan_rx_kill_vid(struct net_device *, u16);
151static void igb_restore_vlan(struct igb_adapter *);
26ad9178 152static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 153static void igb_ping_all_vfs(struct igb_adapter *);
154static void igb_msg_task(struct igb_adapter *);
4ae196df 155static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 156static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 157static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 158static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
159static int igb_ndo_set_vf_vlan(struct net_device *netdev,
160 int vf, u16 vlan, u8 qos);
161static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
162static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
163 struct ifla_vf_info *ivi);
17dc566c 164static void igb_check_vf_rate_limit(struct igb_adapter *);
0224d663
GR		 165static int igb_vf_configure(struct igb_adapter *adapter, int vf);
166static int igb_find_enabled_vfs(struct igb_adapter *adapter);
167static int igb_check_vf_assignment(struct igb_adapter *adapter);
9d5c8243 168
9d5c8243 169#ifdef CONFIG_PM
3fe7c4c9 170static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 171static int igb_resume(struct pci_dev *);
172#endif
173static void igb_shutdown(struct pci_dev *);
421e02f0 174#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 175static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
176static struct notifier_block dca_notifier = {
177 .notifier_call = igb_notify_dca,
178 .next = NULL,
179 .priority = 0
180};
181#endif
9d5c8243
AK		 182#ifdef CONFIG_NET_POLL_CONTROLLER
183/* for netdump / net console */
184static void igb_netpoll(struct net_device *);
185#endif
37680117 186#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 187static unsigned int max_vfs = 0;
188module_param(max_vfs, uint, 0);
189MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
190 "per physical function");
191#endif /* CONFIG_PCI_IOV */
192
9d5c8243
AK		 193static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
194 pci_channel_state_t);
195static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
196static void igb_io_resume(struct pci_dev *);
197
198static struct pci_error_handlers igb_err_handler = {
199 .error_detected = igb_io_error_detected,
200 .slot_reset = igb_io_slot_reset,
201 .resume = igb_io_resume,
202};
203
204
205static struct pci_driver igb_driver = {
206 .name = igb_driver_name,
207 .id_table = igb_pci_tbl,
208 .probe = igb_probe,
209 .remove = __devexit_p(igb_remove),
210#ifdef CONFIG_PM
25985edc 211 /* Power Management Hooks */
9d5c8243
AK		 212 .suspend = igb_suspend,
213 .resume = igb_resume,
214#endif
215 .shutdown = igb_shutdown,
216 .err_handler = &igb_err_handler
217};
218
219MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
220MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
221MODULE_LICENSE("GPL");
222MODULE_VERSION(DRV_VERSION);
223
c97ec42a
TI		 224struct igb_reg_info {
225 u32 ofs;
226 char *name;
227};
228
229static const struct igb_reg_info igb_reg_info_tbl[] = {
230
231 /* General Registers */
232 {E1000_CTRL, "CTRL"},
233 {E1000_STATUS, "STATUS"},
234 {E1000_CTRL_EXT, "CTRL_EXT"},
235
236 /* Interrupt Registers */
237 {E1000_ICR, "ICR"},
238
239 /* RX Registers */
240 {E1000_RCTL, "RCTL"},
241 {E1000_RDLEN(0), "RDLEN"},
242 {E1000_RDH(0), "RDH"},
243 {E1000_RDT(0), "RDT"},
244 {E1000_RXDCTL(0), "RXDCTL"},
245 {E1000_RDBAL(0), "RDBAL"},
246 {E1000_RDBAH(0), "RDBAH"},
247
248 /* TX Registers */
249 {E1000_TCTL, "TCTL"},
250 {E1000_TDBAL(0), "TDBAL"},
251 {E1000_TDBAH(0), "TDBAH"},
252 {E1000_TDLEN(0), "TDLEN"},
253 {E1000_TDH(0), "TDH"},
254 {E1000_TDT(0), "TDT"},
255 {E1000_TXDCTL(0), "TXDCTL"},
256 {E1000_TDFH, "TDFH"},
257 {E1000_TDFT, "TDFT"},
258 {E1000_TDFHS, "TDFHS"},
259 {E1000_TDFPC, "TDFPC"},
260
261 /* List Terminator */
262 {}
263};
264
265/*
266 * igb_regdump - register printout routine
267 */
268static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
269{
270 int n = 0;
271 char rname[16];
272 u32 regs[8];
273
274 switch (reginfo->ofs) {
275 case E1000_RDLEN(0):
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDLEN(n));
278 break;
279 case E1000_RDH(0):
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDH(n));
282 break;
283 case E1000_RDT(0):
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_RDT(n));
286 break;
287 case E1000_RXDCTL(0):
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_RXDCTL(n));
290 break;
291 case E1000_RDBAL(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_RDBAL(n));
294 break;
295 case E1000_RDBAH(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_RDBAH(n));
298 break;
299 case E1000_TDBAL(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_RDBAL(n));
302 break;
303 case E1000_TDBAH(0):
304 for (n = 0; n < 4; n++)
305 regs[n] = rd32(E1000_TDBAH(n));
306 break;
307 case E1000_TDLEN(0):
308 for (n = 0; n < 4; n++)
309 regs[n] = rd32(E1000_TDLEN(n));
310 break;
311 case E1000_TDH(0):
312 for (n = 0; n < 4; n++)
313 regs[n] = rd32(E1000_TDH(n));
314 break;
315 case E1000_TDT(0):
316 for (n = 0; n < 4; n++)
317 regs[n] = rd32(E1000_TDT(n));
318 break;
319 case E1000_TXDCTL(0):
320 for (n = 0; n < 4; n++)
321 regs[n] = rd32(E1000_TXDCTL(n));
322 break;
323 default:
324 printk(KERN_INFO "%-15s %08x\n",
325 reginfo->name, rd32(reginfo->ofs));
326 return;
327 }
328
329 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
330 printk(KERN_INFO "%-15s ", rname);
331 for (n = 0; n < 4; n++)
332 printk(KERN_CONT "%08x ", regs[n]);
333 printk(KERN_CONT "\n");
334}
335
336/*
337 * igb_dump - Print registers, tx-rings and rx-rings
338 */
339static void igb_dump(struct igb_adapter *adapter)
340{
341 struct net_device *netdev = adapter->netdev;
342 struct e1000_hw *hw = &adapter->hw;
343 struct igb_reg_info *reginfo;
c97ec42a
TI		 344 struct igb_ring *tx_ring;
345 union e1000_adv_tx_desc *tx_desc;
346 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 347 struct igb_ring *rx_ring;
348 union e1000_adv_rx_desc *rx_desc;
349 u32 staterr;
6ad4edfc 350 u16 i, n;
c97ec42a
TI		 351
352 if (!netif_msg_hw(adapter))
353 return;
354
355 /* Print netdevice Info */
356 if (netdev) {
357 dev_info(&adapter->pdev->dev, "Net device Info\n");
358 printk(KERN_INFO "Device Name state "
359 "trans_start last_rx\n");
360 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
361 netdev->name,
362 netdev->state,
363 netdev->trans_start,
364 netdev->last_rx);
365 }
366
367 /* Print Registers */
368 dev_info(&adapter->pdev->dev, "Register Dump\n");
369 printk(KERN_INFO " Register Name Value\n");
370 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
371 reginfo->name; reginfo++) {
372 igb_regdump(hw, reginfo);
373 }
374
375 /* Print TX Ring Summary */
376 if (!netdev || !netif_running(netdev))
377 goto exit;
378
379 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
380 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
381 " leng ntw timestamp\n");
382 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 383 struct igb_tx_buffer *buffer_info;
c97ec42a 384 tx_ring = adapter->tx_ring[n];
06034649 385 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
8542db05 386 printk(KERN_INFO " %5d %5X %5X %016llX %04X %p %016llX\n",
c97ec42a
TI		 387 n, tx_ring->next_to_use, tx_ring->next_to_clean,
388 (u64)buffer_info->dma,
389 buffer_info->length,
390 buffer_info->next_to_watch,
391 (u64)buffer_info->time_stamp);
392 }
393
394 /* Print TX Rings */
395 if (!netif_msg_tx_done(adapter))
396 goto rx_ring_summary;
397
398 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
399
400 /* Transmit Descriptor Formats
401 *
402 * Advanced Transmit Descriptor
403 * +--------------------------------------------------------------+
404 * 0 | Buffer Address [63:0] |
405 * +--------------------------------------------------------------+
406 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
407 * +--------------------------------------------------------------+
408 * 63 46 45 40 39 38 36 35 32 31 24 15 0
409 */
410
411 for (n = 0; n < adapter->num_tx_queues; n++) {
412 tx_ring = adapter->tx_ring[n];
413 printk(KERN_INFO "------------------------------------\n");
414 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
415 printk(KERN_INFO "------------------------------------\n");
416 printk(KERN_INFO "T [desc] [address 63:0 ] "
417 "[PlPOCIStDDM Ln] [bi->dma ] "
418 "leng ntw timestamp bi->skb\n");
419
420 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
06034649 421 struct igb_tx_buffer *buffer_info;
60136906 422 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 423 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a
TI		 424 u0 = (struct my_u0 *)tx_desc;
425 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
8542db05 426 " %04X %p %016llX %p", i,
c97ec42a
TI		 427 le64_to_cpu(u0->a),
428 le64_to_cpu(u0->b),
429 (u64)buffer_info->dma,
430 buffer_info->length,
431 buffer_info->next_to_watch,
432 (u64)buffer_info->time_stamp,
433 buffer_info->skb);
434 if (i == tx_ring->next_to_use &&
435 i == tx_ring->next_to_clean)
436 printk(KERN_CONT " NTC/U\n");
437 else if (i == tx_ring->next_to_use)
438 printk(KERN_CONT " NTU\n");
439 else if (i == tx_ring->next_to_clean)
440 printk(KERN_CONT " NTC\n");
441 else
442 printk(KERN_CONT "\n");
443
444 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
445 print_hex_dump(KERN_INFO, "",
446 DUMP_PREFIX_ADDRESS,
447 16, 1, phys_to_virt(buffer_info->dma),
448 buffer_info->length, true);
449 }
450 }
451
452 /* Print RX Rings Summary */
453rx_ring_summary:
454 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
455 printk(KERN_INFO "Queue [NTU] [NTC]\n");
456 for (n = 0; n < adapter->num_rx_queues; n++) {
457 rx_ring = adapter->rx_ring[n];
458 printk(KERN_INFO " %5d %5X %5X\n", n,
459 rx_ring->next_to_use, rx_ring->next_to_clean);
460 }
461
462 /* Print RX Rings */
463 if (!netif_msg_rx_status(adapter))
464 goto exit;
465
466 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
467
468 /* Advanced Receive Descriptor (Read) Format
469 * 63 1 0
470 * +-----------------------------------------------------+
471 * 0 | Packet Buffer Address [63:1] |A0/NSE|
472 * +----------------------------------------------+------+
473 * 8 | Header Buffer Address [63:1] | DD |
474 * +-----------------------------------------------------+
475 *
476 *
477 * Advanced Receive Descriptor (Write-Back) Format
478 *
479 * 63 48 47 32 31 30 21 20 17 16 4 3 0
480 * +------------------------------------------------------+
481 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
482 * | Checksum Ident | | | | Type | Type |
483 * +------------------------------------------------------+
484 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
485 * +------------------------------------------------------+
486 * 63 48 47 32 31 20 19 0
487 */
488
489 for (n = 0; n < adapter->num_rx_queues; n++) {
490 rx_ring = adapter->rx_ring[n];
491 printk(KERN_INFO "------------------------------------\n");
492 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
493 printk(KERN_INFO "------------------------------------\n");
494 printk(KERN_INFO "R [desc] [ PktBuf A0] "
495 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
496 "<-- Adv Rx Read format\n");
497 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
498 "[vl er S cks ln] ---------------- [bi->skb] "
499 "<-- Adv Rx Write-Back format\n");
500
501 for (i = 0; i < rx_ring->count; i++) {
06034649
AD		 502 struct igb_rx_buffer *buffer_info;
503 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 504 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 505 u0 = (struct my_u0 *)rx_desc;
506 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
507 if (staterr & E1000_RXD_STAT_DD) {
508 /* Descriptor Done */
509 printk(KERN_INFO "RWB[0x%03X] %016llX "
510 "%016llX ---------------- %p", i,
511 le64_to_cpu(u0->a),
512 le64_to_cpu(u0->b),
513 buffer_info->skb);
514 } else {
515 printk(KERN_INFO "R [0x%03X] %016llX "
516 "%016llX %016llX %p", i,
517 le64_to_cpu(u0->a),
518 le64_to_cpu(u0->b),
519 (u64)buffer_info->dma,
520 buffer_info->skb);
521
522 if (netif_msg_pktdata(adapter)) {
523 print_hex_dump(KERN_INFO, "",
524 DUMP_PREFIX_ADDRESS,
525 16, 1,
526 phys_to_virt(buffer_info->dma),
44390ca6
AD		 527 IGB_RX_HDR_LEN, true);
528 print_hex_dump(KERN_INFO, "",
529 DUMP_PREFIX_ADDRESS,
530 16, 1,
531 phys_to_virt(
532 buffer_info->page_dma +
533 buffer_info->page_offset),
534 PAGE_SIZE/2, true);
c97ec42a
TI		 535 }
536 }
537
538 if (i == rx_ring->next_to_use)
539 printk(KERN_CONT " NTU\n");
540 else if (i == rx_ring->next_to_clean)
541 printk(KERN_CONT " NTC\n");
542 else
543 printk(KERN_CONT "\n");
544
545 }
546 }
547
548exit:
549 return;
550}
551
552
38c845c7
PO		 553/**
554 * igb_read_clock - read raw cycle counter (to be used by time counter)
555 */
556static cycle_t igb_read_clock(const struct cyclecounter *tc)
557{
558 struct igb_adapter *adapter =
559 container_of(tc, struct igb_adapter, cycles);
560 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 561 u64 stamp = 0;
562 int shift = 0;
38c845c7 563
55cac248
AD		 564 /*
565 * The timestamp latches on lowest register read. For the 82580
566 * the lowest register is SYSTIMR instead of SYSTIML. However we never
567 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
568 */
06218a8d 569 if (hw->mac.type >= e1000_82580) {
55cac248
AD		 570 stamp = rd32(E1000_SYSTIMR) >> 8;
571 shift = IGB_82580_TSYNC_SHIFT;
572 }
573
c5b9bd5e
AD		 574 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
575 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 576 return stamp;
577}
578
9d5c8243 579/**
c041076a 580 * igb_get_hw_dev - return device
9d5c8243
AK		 581 * used by hardware layer to print debugging information
582 **/
c041076a 583struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 584{
585 struct igb_adapter *adapter = hw->back;
c041076a 586 return adapter->netdev;
9d5c8243 587}
38c845c7 588
9d5c8243
AK		 589/**
590 * igb_init_module - Driver Registration Routine
591 *
592 * igb_init_module is the first routine called when the driver is
593 * loaded. All it does is register with the PCI subsystem.
594 **/
595static int __init igb_init_module(void)
596{
597 int ret;
598 printk(KERN_INFO "%s - version %s\n",
599 igb_driver_string, igb_driver_version);
600
601 printk(KERN_INFO "%s\n", igb_copyright);
602
421e02f0 603#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 604 dca_register_notify(&dca_notifier);
605#endif
bbd98fe4 606 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 607 return ret;
608}
609
610module_init(igb_init_module);
611
612/**
613 * igb_exit_module - Driver Exit Cleanup Routine
614 *
615 * igb_exit_module is called just before the driver is removed
616 * from memory.
617 **/
618static void __exit igb_exit_module(void)
619{
421e02f0 620#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 621 dca_unregister_notify(&dca_notifier);
622#endif
9d5c8243
AK		 623 pci_unregister_driver(&igb_driver);
624}
625
626module_exit(igb_exit_module);
627
26bc19ec
AD		 628#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
629/**
630 * igb_cache_ring_register - Descriptor ring to register mapping
631 * @adapter: board private structure to initialize
632 *
633 * Once we know the feature-set enabled for the device, we'll cache
634 * the register offset the descriptor ring is assigned to.
635 **/
636static void igb_cache_ring_register(struct igb_adapter *adapter)
637{
ee1b9f06 638 int i = 0, j = 0;
047e0030 639 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 640
641 switch (adapter->hw.mac.type) {
642 case e1000_82576:
643 /* The queues are allocated for virtualization such that VF 0
644 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
645 * In order to avoid collision we start at the first free queue
646 * and continue consuming queues in the same sequence
647 */
ee1b9f06 648 if (adapter->vfs_allocated_count) {
a99955fc 649 for (; i < adapter->rss_queues; i++)
3025a446
AD		 650 adapter->rx_ring[i]->reg_idx = rbase_offset +
651 Q_IDX_82576(i);
ee1b9f06 652 }
26bc19ec 653 case e1000_82575:
55cac248 654 case e1000_82580:
d2ba2ed8 655 case e1000_i350:
26bc19ec 656 default:
ee1b9f06 657 for (; i < adapter->num_rx_queues; i++)
3025a446 658 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 659 for (; j < adapter->num_tx_queues; j++)
3025a446 660 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 661 break;
662 }
663}
664
047e0030
AD		 665static void igb_free_queues(struct igb_adapter *adapter)
666{
3025a446 667 int i;
047e0030 668
3025a446
AD		 669 for (i = 0; i < adapter->num_tx_queues; i++) {
670 kfree(adapter->tx_ring[i]);
671 adapter->tx_ring[i] = NULL;
672 }
673 for (i = 0; i < adapter->num_rx_queues; i++) {
674 kfree(adapter->rx_ring[i]);
675 adapter->rx_ring[i] = NULL;
676 }
047e0030
AD		 677 adapter->num_rx_queues = 0;
678 adapter->num_tx_queues = 0;
679}
680
9d5c8243
AK		 681/**
682 * igb_alloc_queues - Allocate memory for all rings
683 * @adapter: board private structure to initialize
684 *
685 * We allocate one ring per queue at run-time since we don't know the
686 * number of queues at compile-time.
687 **/
688static int igb_alloc_queues(struct igb_adapter *adapter)
689{
3025a446 690 struct igb_ring *ring;
9d5c8243 691 int i;
81c2fc22 692 int orig_node = adapter->node;
9d5c8243 693
661086df 694 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 695 if (orig_node == -1) {
696 int cur_node = next_online_node(adapter->node);
697 if (cur_node == MAX_NUMNODES)
698 cur_node = first_online_node;
699 adapter->node = cur_node;
700 }
701 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
702 adapter->node);
703 if (!ring)
704 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 705 if (!ring)
706 goto err;
68fd9910 707 ring->count = adapter->tx_ring_count;
661086df 708 ring->queue_index = i;
59d71989 709 ring->dev = &adapter->pdev->dev;
e694e964 710 ring->netdev = adapter->netdev;
81c2fc22 711 ring->numa_node = adapter->node;
85ad76b2
AD		 712 /* For 82575, context index must be unique per ring. */
713 if (adapter->hw.mac.type == e1000_82575)
866cff06 714 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
3025a446 715 adapter->tx_ring[i] = ring;
661086df 716 }
81c2fc22
AD		 717 /* Restore the adapter's original node */
718 adapter->node = orig_node;
85ad76b2 719
9d5c8243 720 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 721 if (orig_node == -1) {
722 int cur_node = next_online_node(adapter->node);
723 if (cur_node == MAX_NUMNODES)
724 cur_node = first_online_node;
725 adapter->node = cur_node;
726 }
727 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
728 adapter->node);
729 if (!ring)
730 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 731 if (!ring)
732 goto err;
68fd9910 733 ring->count = adapter->rx_ring_count;
844290e5 734 ring->queue_index = i;
59d71989 735 ring->dev = &adapter->pdev->dev;
e694e964 736 ring->netdev = adapter->netdev;
81c2fc22 737 ring->numa_node = adapter->node;
85ad76b2
AD		 738 /* set flag indicating ring supports SCTP checksum offload */
739 if (adapter->hw.mac.type >= e1000_82576)
866cff06 740 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
8be10e91
AD		 741
742 /* On i350, loopback VLAN packets have the tag byte-swapped. */
743 if (adapter->hw.mac.type == e1000_i350)
744 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
745
3025a446 746 adapter->rx_ring[i] = ring;
9d5c8243 747 }
81c2fc22
AD		 748 /* Restore the adapter's original node */
749 adapter->node = orig_node;
26bc19ec
AD		 750
751 igb_cache_ring_register(adapter);
9d5c8243 752
047e0030 753 return 0;
a88f10ec 754
047e0030 755err:
81c2fc22
AD		 756 /* Restore the adapter's original node */
757 adapter->node = orig_node;
047e0030 758 igb_free_queues(adapter);
d1a8c9e1 759
047e0030 760 return -ENOMEM;
a88f10ec
AD		 761}
762
4be000c8
AD		 763/**
764 * igb_write_ivar - configure ivar for given MSI-X vector
765 * @hw: pointer to the HW structure
766 * @msix_vector: vector number we are allocating to a given ring
767 * @index: row index of IVAR register to write within IVAR table
768 * @offset: column offset of in IVAR, should be multiple of 8
769 *
770 * This function is intended to handle the writing of the IVAR register
771 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
772 * each containing an cause allocation for an Rx and Tx ring, and a
773 * variable number of rows depending on the number of queues supported.
774 **/
775static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
776 int index, int offset)
777{
778 u32 ivar = array_rd32(E1000_IVAR0, index);
779
780 /* clear any bits that are currently set */
781 ivar &= ~((u32)0xFF << offset);
782
783 /* write vector and valid bit */
784 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
785
786 array_wr32(E1000_IVAR0, index, ivar);
787}
788
9d5c8243 789#define IGB_N0_QUEUE -1
047e0030 790static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243 791{
047e0030 792 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 793 struct e1000_hw *hw = &adapter->hw;
047e0030
AD		 794 int rx_queue = IGB_N0_QUEUE;
795 int tx_queue = IGB_N0_QUEUE;
4be000c8 796 u32 msixbm = 0;
047e0030 797
0ba82994
AD		 798 if (q_vector->rx.ring)
799 rx_queue = q_vector->rx.ring->reg_idx;
800 if (q_vector->tx.ring)
801 tx_queue = q_vector->tx.ring->reg_idx;
2d064c06
AD		 802
803 switch (hw->mac.type) {
804 case e1000_82575:
9d5c8243
AK		 805 /* The 82575 assigns vectors using a bitmask, which matches the
806 bitmask for the EICR/EIMS/EIMC registers. To assign one
807 or more queues to a vector, we write the appropriate bits
808 into the MSIXBM register for that vector. */
047e0030 809 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 810 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 811 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 812 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 813 if (!adapter->msix_entries && msix_vector == 0)
814 msixbm |= E1000_EIMS_OTHER;
9d5c8243 815 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 816 q_vector->eims_value = msixbm;
2d064c06
AD		 817 break;
818 case e1000_82576:
4be000c8
AD		 819 /*
820 * 82576 uses a table that essentially consists of 2 columns
821 * with 8 rows. The ordering is column-major so we use the
822 * lower 3 bits as the row index, and the 4th bit as the
823 * column offset.
824 */
825 if (rx_queue > IGB_N0_QUEUE)
826 igb_write_ivar(hw, msix_vector,
827 rx_queue & 0x7,
828 (rx_queue & 0x8) << 1);
829 if (tx_queue > IGB_N0_QUEUE)
830 igb_write_ivar(hw, msix_vector,
831 tx_queue & 0x7,
832 ((tx_queue & 0x8) << 1) + 8);
047e0030 833 q_vector->eims_value = 1 << msix_vector;
2d064c06 834 break;
55cac248 835 case e1000_82580:
d2ba2ed8 836 case e1000_i350:
4be000c8
AD		 837 /*
838 * On 82580 and newer adapters the scheme is similar to 82576
839 * however instead of ordering column-major we have things
840 * ordered row-major. So we traverse the table by using
841 * bit 0 as the column offset, and the remaining bits as the
842 * row index.
843 */
844 if (rx_queue > IGB_N0_QUEUE)
845 igb_write_ivar(hw, msix_vector,
846 rx_queue >> 1,
847 (rx_queue & 0x1) << 4);
848 if (tx_queue > IGB_N0_QUEUE)
849 igb_write_ivar(hw, msix_vector,
850 tx_queue >> 1,
851 ((tx_queue & 0x1) << 4) + 8);
55cac248
AD		 852 q_vector->eims_value = 1 << msix_vector;
853 break;
2d064c06
AD		 854 default:
855 BUG();
856 break;
857 }
26b39276
AD		 858
859 /* add q_vector eims value to global eims_enable_mask */
860 adapter->eims_enable_mask |= q_vector->eims_value;
861
862 /* configure q_vector to set itr on first interrupt */
863 q_vector->set_itr = 1;
9d5c8243
AK		 864}
865
866/**
867 * igb_configure_msix - Configure MSI-X hardware
868 *
869 * igb_configure_msix sets up the hardware to properly
870 * generate MSI-X interrupts.
871 **/
872static void igb_configure_msix(struct igb_adapter *adapter)
873{
874 u32 tmp;
875 int i, vector = 0;
876 struct e1000_hw *hw = &adapter->hw;
877
878 adapter->eims_enable_mask = 0;
9d5c8243
AK		 879
880 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 881 switch (hw->mac.type) {
882 case e1000_82575:
9d5c8243
AK		 883 tmp = rd32(E1000_CTRL_EXT);
884 /* enable MSI-X PBA support*/
885 tmp |= E1000_CTRL_EXT_PBA_CLR;
886
887 /* Auto-Mask interrupts upon ICR read. */
888 tmp |= E1000_CTRL_EXT_EIAME;
889 tmp |= E1000_CTRL_EXT_IRCA;
890
891 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 892
893 /* enable msix_other interrupt */
894 array_wr32(E1000_MSIXBM(0), vector++,
895 E1000_EIMS_OTHER);
844290e5 896 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 897
2d064c06
AD		 898 break;
899
900 case e1000_82576:
55cac248 901 case e1000_82580:
d2ba2ed8 902 case e1000_i350:
047e0030
AD		 903 /* Turn on MSI-X capability first, or our settings
904 * won't stick. And it will take days to debug. */
905 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
906 E1000_GPIE_PBA | E1000_GPIE_EIAME |
907 E1000_GPIE_NSICR);
908
909 /* enable msix_other interrupt */
910 adapter->eims_other = 1 << vector;
2d064c06 911 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 912
047e0030 913 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 914 break;
915 default:
916 /* do nothing, since nothing else supports MSI-X */
917 break;
918 } /* switch (hw->mac.type) */
047e0030
AD		 919
920 adapter->eims_enable_mask |= adapter->eims_other;
921
26b39276
AD		 922 for (i = 0; i < adapter->num_q_vectors; i++)
923 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 924
9d5c8243
AK		 925 wrfl();
926}
927
928/**
929 * igb_request_msix - Initialize MSI-X interrupts
930 *
931 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
932 * kernel.
933 **/
934static int igb_request_msix(struct igb_adapter *adapter)
935{
936 struct net_device *netdev = adapter->netdev;
047e0030 937 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 938 int i, err = 0, vector = 0;
939
047e0030 940 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 941 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 942 if (err)
943 goto out;
944 vector++;
945
946 for (i = 0; i < adapter->num_q_vectors; i++) {
947 struct igb_q_vector *q_vector = adapter->q_vector[i];
948
949 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
950
0ba82994 951 if (q_vector->rx.ring && q_vector->tx.ring)
047e0030 952 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
0ba82994
AD		 953 q_vector->rx.ring->queue_index);
954 else if (q_vector->tx.ring)
047e0030 955 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
0ba82994
AD		 956 q_vector->tx.ring->queue_index);
957 else if (q_vector->rx.ring)
047e0030 958 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
0ba82994 959 q_vector->rx.ring->queue_index);
9d5c8243 960 else
047e0030
AD		 961 sprintf(q_vector->name, "%s-unused", netdev->name);
962
9d5c8243 963 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 964 igb_msix_ring, 0, q_vector->name,
047e0030 965 q_vector);
9d5c8243
AK		 966 if (err)
967 goto out;
9d5c8243
AK		 968 vector++;
969 }
970
9d5c8243
AK		 971 igb_configure_msix(adapter);
972 return 0;
973out:
974 return err;
975}
976
977static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
978{
979 if (adapter->msix_entries) {
980 pci_disable_msix(adapter->pdev);
981 kfree(adapter->msix_entries);
982 adapter->msix_entries = NULL;
047e0030 983 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 984 pci_disable_msi(adapter->pdev);
047e0030 985 }
9d5c8243
AK		 986}
987
047e0030
AD		 988/**
989 * igb_free_q_vectors - Free memory allocated for interrupt vectors
990 * @adapter: board private structure to initialize
991 *
992 * This function frees the memory allocated to the q_vectors. In addition if
993 * NAPI is enabled it will delete any references to the NAPI struct prior
994 * to freeing the q_vector.
995 **/
996static void igb_free_q_vectors(struct igb_adapter *adapter)
997{
998 int v_idx;
999
1000 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1001 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1002 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1003 if (!q_vector)
1004 continue;
047e0030
AD		 1005 netif_napi_del(&q_vector->napi);
1006 kfree(q_vector);
1007 }
1008 adapter->num_q_vectors = 0;
1009}
1010
1011/**
1012 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1013 *
1014 * This function resets the device so that it has 0 rx queues, tx queues, and
1015 * MSI-X interrupts allocated.
1016 */
1017static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1018{
1019 igb_free_queues(adapter);
1020 igb_free_q_vectors(adapter);
1021 igb_reset_interrupt_capability(adapter);
1022}
9d5c8243
AK		 1023
1024/**
1025 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1026 *
1027 * Attempt to configure interrupts using the best available
1028 * capabilities of the hardware and kernel.
1029 **/
21adef3e 1030static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1031{
1032 int err;
1033 int numvecs, i;
1034
83b7180d 1035 /* Number of supported queues. */
a99955fc 1036 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1037 if (adapter->vfs_allocated_count)
1038 adapter->num_tx_queues = 1;
1039 else
1040 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1041
047e0030
AD		 1042 /* start with one vector for every rx queue */
1043 numvecs = adapter->num_rx_queues;
1044
3ad2f3fb 1045 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1046 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1047 numvecs += adapter->num_tx_queues;
047e0030
AD		 1048
1049 /* store the number of vectors reserved for queues */
1050 adapter->num_q_vectors = numvecs;
1051
1052 /* add 1 vector for link status interrupts */
1053 numvecs++;
9d5c8243
AK		 1054 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1055 GFP_KERNEL);
1056 if (!adapter->msix_entries)
1057 goto msi_only;
1058
1059 for (i = 0; i < numvecs; i++)
1060 adapter->msix_entries[i].entry = i;
1061
1062 err = pci_enable_msix(adapter->pdev,
1063 adapter->msix_entries,
1064 numvecs);
1065 if (err == 0)
34a20e89 1066 goto out;
9d5c8243
AK		 1067
1068 igb_reset_interrupt_capability(adapter);
1069
1070 /* If we can't do MSI-X, try MSI */
1071msi_only:
2a3abf6d
AD		 1072#ifdef CONFIG_PCI_IOV
1073 /* disable SR-IOV for non MSI-X configurations */
1074 if (adapter->vf_data) {
1075 struct e1000_hw *hw = &adapter->hw;
1076 /* disable iov and allow time for transactions to clear */
1077 pci_disable_sriov(adapter->pdev);
1078 msleep(500);
1079
1080 kfree(adapter->vf_data);
1081 adapter->vf_data = NULL;
1082 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1083 wrfl();
2a3abf6d
AD		 1084 msleep(100);
1085 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1086 }
1087#endif
4fc82adf 1088 adapter->vfs_allocated_count = 0;
a99955fc 1089 adapter->rss_queues = 1;
4fc82adf 1090 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1091 adapter->num_rx_queues = 1;
661086df 1092 adapter->num_tx_queues = 1;
047e0030 1093 adapter->num_q_vectors = 1;
9d5c8243 1094 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1095 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1096out:
21adef3e
BH		 1097 /* Notify the stack of the (possibly) reduced queue counts. */
1098 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1099 return netif_set_real_num_rx_queues(adapter->netdev,
1100 adapter->num_rx_queues);
9d5c8243
AK		 1101}
1102
047e0030
AD		 1103/**
1104 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1105 * @adapter: board private structure to initialize
1106 *
1107 * We allocate one q_vector per queue interrupt. If allocation fails we
1108 * return -ENOMEM.
1109 **/
1110static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1111{
1112 struct igb_q_vector *q_vector;
1113 struct e1000_hw *hw = &adapter->hw;
1114 int v_idx;
81c2fc22 1115 int orig_node = adapter->node;
047e0030
AD		 1116
1117 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1118 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1119 adapter->num_tx_queues)) &&
1120 (adapter->num_rx_queues == v_idx))
1121 adapter->node = orig_node;
1122 if (orig_node == -1) {
1123 int cur_node = next_online_node(adapter->node);
1124 if (cur_node == MAX_NUMNODES)
1125 cur_node = first_online_node;
1126 adapter->node = cur_node;
1127 }
1128 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1129 adapter->node);
1130 if (!q_vector)
1131 q_vector = kzalloc(sizeof(struct igb_q_vector),
1132 GFP_KERNEL);
047e0030
AD		 1133 if (!q_vector)
1134 goto err_out;
1135 q_vector->adapter = adapter;
047e0030
AD		 1136 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1137 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1138 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1139 adapter->q_vector[v_idx] = q_vector;
1140 }
81c2fc22
AD		 1141 /* Restore the adapter's original node */
1142 adapter->node = orig_node;
1143
047e0030
AD		 1144 return 0;
1145
1146err_out:
81c2fc22
AD		 1147 /* Restore the adapter's original node */
1148 adapter->node = orig_node;
fe0592b4 1149 igb_free_q_vectors(adapter);
047e0030
AD		 1150 return -ENOMEM;
1151}
1152
1153static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1154 int ring_idx, int v_idx)
1155{
3025a446 1156 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1157
0ba82994
AD		 1158 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1159 q_vector->rx.ring->q_vector = q_vector;
1160 q_vector->rx.count++;
4fc82adf
AD		 1161 q_vector->itr_val = adapter->rx_itr_setting;
1162 if (q_vector->itr_val && q_vector->itr_val <= 3)
1163 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1164}
1165
1166static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1167 int ring_idx, int v_idx)
1168{
3025a446 1169 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1170
0ba82994
AD		 1171 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1172 q_vector->tx.ring->q_vector = q_vector;
1173 q_vector->tx.count++;
4fc82adf 1174 q_vector->itr_val = adapter->tx_itr_setting;
0ba82994 1175 q_vector->tx.work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1176 if (q_vector->itr_val && q_vector->itr_val <= 3)
1177 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1178}
1179
1180/**
1181 * igb_map_ring_to_vector - maps allocated queues to vectors
1182 *
1183 * This function maps the recently allocated queues to vectors.
1184 **/
1185static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1186{
1187 int i;
1188 int v_idx = 0;
1189
1190 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1191 (adapter->num_q_vectors < adapter->num_tx_queues))
1192 return -ENOMEM;
1193
1194 if (adapter->num_q_vectors >=
1195 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1196 for (i = 0; i < adapter->num_rx_queues; i++)
1197 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1198 for (i = 0; i < adapter->num_tx_queues; i++)
1199 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1200 } else {
1201 for (i = 0; i < adapter->num_rx_queues; i++) {
1202 if (i < adapter->num_tx_queues)
1203 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1204 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1205 }
1206 for (; i < adapter->num_tx_queues; i++)
1207 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1208 }
1209 return 0;
1210}
1211
1212/**
1213 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1214 *
1215 * This function initializes the interrupts and allocates all of the queues.
1216 **/
1217static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1218{
1219 struct pci_dev *pdev = adapter->pdev;
1220 int err;
1221
21adef3e
BH		 1222 err = igb_set_interrupt_capability(adapter);
1223 if (err)
1224 return err;
047e0030
AD		 1225
1226 err = igb_alloc_q_vectors(adapter);
1227 if (err) {
1228 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1229 goto err_alloc_q_vectors;
1230 }
1231
1232 err = igb_alloc_queues(adapter);
1233 if (err) {
1234 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1235 goto err_alloc_queues;
1236 }
1237
1238 err = igb_map_ring_to_vector(adapter);
1239 if (err) {
1240 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1241 goto err_map_queues;
1242 }
1243
1244
1245 return 0;
1246err_map_queues:
1247 igb_free_queues(adapter);
1248err_alloc_queues:
1249 igb_free_q_vectors(adapter);
1250err_alloc_q_vectors:
1251 igb_reset_interrupt_capability(adapter);
1252 return err;
1253}
1254
9d5c8243
AK		 1255/**
1256 * igb_request_irq - initialize interrupts
1257 *
1258 * Attempts to configure interrupts using the best available
1259 * capabilities of the hardware and kernel.
1260 **/
1261static int igb_request_irq(struct igb_adapter *adapter)
1262{
1263 struct net_device *netdev = adapter->netdev;
047e0030 1264 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1265 int err = 0;
1266
1267 if (adapter->msix_entries) {
1268 err = igb_request_msix(adapter);
844290e5 1269 if (!err)
9d5c8243 1270 goto request_done;
9d5c8243 1271 /* fall back to MSI */
047e0030 1272 igb_clear_interrupt_scheme(adapter);
c74d588e 1273 if (!pci_enable_msi(pdev))
7dfc16fa 1274 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1275 igb_free_all_tx_resources(adapter);
1276 igb_free_all_rx_resources(adapter);
047e0030 1277 adapter->num_tx_queues = 1;
9d5c8243 1278 adapter->num_rx_queues = 1;
047e0030
AD		 1279 adapter->num_q_vectors = 1;
1280 err = igb_alloc_q_vectors(adapter);
1281 if (err) {
1282 dev_err(&pdev->dev,
1283 "Unable to allocate memory for vectors\n");
1284 goto request_done;
1285 }
1286 err = igb_alloc_queues(adapter);
1287 if (err) {
1288 dev_err(&pdev->dev,
1289 "Unable to allocate memory for queues\n");
1290 igb_free_q_vectors(adapter);
1291 goto request_done;
1292 }
1293 igb_setup_all_tx_resources(adapter);
1294 igb_setup_all_rx_resources(adapter);
9d5c8243 1295 }
844290e5 1296
c74d588e
AD		 1297 igb_assign_vector(adapter->q_vector[0], 0);
1298
7dfc16fa 1299 if (adapter->flags & IGB_FLAG_HAS_MSI) {
c74d588e 1300 err = request_irq(pdev->irq, igb_intr_msi, 0,
047e0030 1301 netdev->name, adapter);
9d5c8243
AK		 1302 if (!err)
1303 goto request_done;
047e0030 1304
9d5c8243
AK		 1305 /* fall back to legacy interrupts */
1306 igb_reset_interrupt_capability(adapter);
7dfc16fa 1307 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1308 }
1309
c74d588e 1310 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1311 netdev->name, adapter);
9d5c8243 1312
6cb5e577 1313 if (err)
c74d588e 1314 dev_err(&pdev->dev, "Error %d getting interrupt\n",
9d5c8243 1315 err);
9d5c8243
AK		 1316
1317request_done:
1318 return err;
1319}
1320
1321static void igb_free_irq(struct igb_adapter *adapter)
1322{
9d5c8243
AK		 1323 if (adapter->msix_entries) {
1324 int vector = 0, i;
1325
047e0030 1326 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1327
0d1ae7f4 1328 for (i = 0; i < adapter->num_q_vectors; i++)
047e0030 1329 free_irq(adapter->msix_entries[vector++].vector,
0d1ae7f4 1330 adapter->q_vector[i]);
047e0030
AD		 1331 } else {
1332 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1333 }
9d5c8243
AK		 1334}
1335
1336/**
1337 * igb_irq_disable - Mask off interrupt generation on the NIC
1338 * @adapter: board private structure
1339 **/
1340static void igb_irq_disable(struct igb_adapter *adapter)
1341{
1342 struct e1000_hw *hw = &adapter->hw;
1343
25568a53
AD		 1344 /*
1345 * we need to be careful when disabling interrupts. The VFs are also
1346 * mapped into these registers and so clearing the bits can cause
1347 * issues on the VF drivers so we only need to clear what we set
1348 */
9d5c8243 1349 if (adapter->msix_entries) {
2dfd1212
AD		 1350 u32 regval = rd32(E1000_EIAM);
1351 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1352 wr32(E1000_EIMC, adapter->eims_enable_mask);
1353 regval = rd32(E1000_EIAC);
1354 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1355 }
844290e5
PW		 1356
1357 wr32(E1000_IAM, 0);
9d5c8243
AK		 1358 wr32(E1000_IMC, ~0);
1359 wrfl();
81a61859
ET		 1360 if (adapter->msix_entries) {
1361 int i;
1362 for (i = 0; i < adapter->num_q_vectors; i++)
1363 synchronize_irq(adapter->msix_entries[i].vector);
1364 } else {
1365 synchronize_irq(adapter->pdev->irq);
1366 }
9d5c8243
AK		 1367}
1368
1369/**
1370 * igb_irq_enable - Enable default interrupt generation settings
1371 * @adapter: board private structure
1372 **/
1373static void igb_irq_enable(struct igb_adapter *adapter)
1374{
1375 struct e1000_hw *hw = &adapter->hw;
1376
1377 if (adapter->msix_entries) {
06218a8d 1378 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
2dfd1212
AD		 1379 u32 regval = rd32(E1000_EIAC);
1380 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1381 regval = rd32(E1000_EIAM);
1382 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1383 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1384 if (adapter->vfs_allocated_count) {
4ae196df 1385 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1386 ims |= E1000_IMS_VMMB;
1387 }
1388 wr32(E1000_IMS, ims);
844290e5 1389 } else {
55cac248
AD		 1390 wr32(E1000_IMS, IMS_ENABLE_MASK |
1391 E1000_IMS_DRSTA);
1392 wr32(E1000_IAM, IMS_ENABLE_MASK |
1393 E1000_IMS_DRSTA);
844290e5 1394 }
9d5c8243
AK		 1395}
1396
1397static void igb_update_mng_vlan(struct igb_adapter *adapter)
1398{
51466239 1399 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1400 u16 vid = adapter->hw.mng_cookie.vlan_id;
1401 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1402
1403 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1404 /* add VID to filter table */
1405 igb_vfta_set(hw, vid, true);
1406 adapter->mng_vlan_id = vid;
1407 } else {
1408 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1409 }
1410
1411 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1412 (vid != old_vid) &&
b2cb09b1 1413 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1414 /* remove VID from filter table */
1415 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1416 }
1417}
1418
1419/**
1420 * igb_release_hw_control - release control of the h/w to f/w
1421 * @adapter: address of board private structure
1422 *
1423 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1424 * For ASF and Pass Through versions of f/w this means that the
1425 * driver is no longer loaded.
1426 *
1427 **/
1428static void igb_release_hw_control(struct igb_adapter *adapter)
1429{
1430 struct e1000_hw *hw = &adapter->hw;
1431 u32 ctrl_ext;
1432
1433 /* Let firmware take over control of h/w */
1434 ctrl_ext = rd32(E1000_CTRL_EXT);
1435 wr32(E1000_CTRL_EXT,
1436 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1437}
1438
9d5c8243
AK		 1439/**
1440 * igb_get_hw_control - get control of the h/w from f/w
1441 * @adapter: address of board private structure
1442 *
1443 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1444 * For ASF and Pass Through versions of f/w this means that
1445 * the driver is loaded.
1446 *
1447 **/
1448static void igb_get_hw_control(struct igb_adapter *adapter)
1449{
1450 struct e1000_hw *hw = &adapter->hw;
1451 u32 ctrl_ext;
1452
1453 /* Let firmware know the driver has taken over */
1454 ctrl_ext = rd32(E1000_CTRL_EXT);
1455 wr32(E1000_CTRL_EXT,
1456 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1457}
1458
9d5c8243
AK		 1459/**
1460 * igb_configure - configure the hardware for RX and TX
1461 * @adapter: private board structure
1462 **/
1463static void igb_configure(struct igb_adapter *adapter)
1464{
1465 struct net_device *netdev = adapter->netdev;
1466 int i;
1467
1468 igb_get_hw_control(adapter);
ff41f8dc 1469 igb_set_rx_mode(netdev);
9d5c8243
AK		 1470
1471 igb_restore_vlan(adapter);
9d5c8243 1472
85b430b4 1473 igb_setup_tctl(adapter);
06cf2666 1474 igb_setup_mrqc(adapter);
9d5c8243 1475 igb_setup_rctl(adapter);
85b430b4
AD		 1476
1477 igb_configure_tx(adapter);
9d5c8243 1478 igb_configure_rx(adapter);
662d7205
AD		 1479
1480 igb_rx_fifo_flush_82575(&adapter->hw);
1481
c493ea45 1482 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1483 * at least 1 descriptor unused to make sure
1484 * next_to_use != next_to_clean */
1485 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1486 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1487 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1488 }
9d5c8243
AK		 1489}
1490
88a268c1
NN		 1491/**
1492 * igb_power_up_link - Power up the phy/serdes link
1493 * @adapter: address of board private structure
1494 **/
1495void igb_power_up_link(struct igb_adapter *adapter)
1496{
1497 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1498 igb_power_up_phy_copper(&adapter->hw);
1499 else
1500 igb_power_up_serdes_link_82575(&adapter->hw);
1501}
1502
1503/**
1504 * igb_power_down_link - Power down the phy/serdes link
1505 * @adapter: address of board private structure
1506 */
1507static void igb_power_down_link(struct igb_adapter *adapter)
1508{
1509 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1510 igb_power_down_phy_copper_82575(&adapter->hw);
1511 else
1512 igb_shutdown_serdes_link_82575(&adapter->hw);
1513}
9d5c8243
AK		 1514
1515/**
1516 * igb_up - Open the interface and prepare it to handle traffic
1517 * @adapter: board private structure
1518 **/
9d5c8243
AK		 1519int igb_up(struct igb_adapter *adapter)
1520{
1521 struct e1000_hw *hw = &adapter->hw;
1522 int i;
1523
1524 /* hardware has been reset, we need to reload some things */
1525 igb_configure(adapter);
1526
1527 clear_bit(__IGB_DOWN, &adapter->state);
1528
0d1ae7f4
AD		 1529 for (i = 0; i < adapter->num_q_vectors; i++)
1530 napi_enable(&(adapter->q_vector[i]->napi));
1531
844290e5 1532 if (adapter->msix_entries)
9d5c8243 1533 igb_configure_msix(adapter);
feeb2721
AD		 1534 else
1535 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1536
1537 /* Clear any pending interrupts. */
1538 rd32(E1000_ICR);
1539 igb_irq_enable(adapter);
1540
d4960307
AD		 1541 /* notify VFs that reset has been completed */
1542 if (adapter->vfs_allocated_count) {
1543 u32 reg_data = rd32(E1000_CTRL_EXT);
1544 reg_data |= E1000_CTRL_EXT_PFRSTD;
1545 wr32(E1000_CTRL_EXT, reg_data);
1546 }
1547
4cb9be7a
JB		 1548 netif_tx_start_all_queues(adapter->netdev);
1549
25568a53
AD		 1550 /* start the watchdog. */
1551 hw->mac.get_link_status = 1;
1552 schedule_work(&adapter->watchdog_task);
1553
9d5c8243
AK		 1554 return 0;
1555}
1556
1557void igb_down(struct igb_adapter *adapter)
1558{
9d5c8243 1559 struct net_device *netdev = adapter->netdev;
330a6d6a 1560 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1561 u32 tctl, rctl;
1562 int i;
1563
1564 /* signal that we're down so the interrupt handler does not
1565 * reschedule our watchdog timer */
1566 set_bit(__IGB_DOWN, &adapter->state);
1567
1568 /* disable receives in the hardware */
1569 rctl = rd32(E1000_RCTL);
1570 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1571 /* flush and sleep below */
1572
fd2ea0a7 1573 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1574
1575 /* disable transmits in the hardware */
1576 tctl = rd32(E1000_TCTL);
1577 tctl &= ~E1000_TCTL_EN;
1578 wr32(E1000_TCTL, tctl);
1579 /* flush both disables and wait for them to finish */
1580 wrfl();
1581 msleep(10);
1582
0d1ae7f4
AD		 1583 for (i = 0; i < adapter->num_q_vectors; i++)
1584 napi_disable(&(adapter->q_vector[i]->napi));
9d5c8243 1585
9d5c8243
AK		 1586 igb_irq_disable(adapter);
1587
1588 del_timer_sync(&adapter->watchdog_timer);
1589 del_timer_sync(&adapter->phy_info_timer);
1590
9d5c8243 1591 netif_carrier_off(netdev);
04fe6358
AD		 1592
1593 /* record the stats before reset*/
12dcd86b
ED		 1594 spin_lock(&adapter->stats64_lock);
1595 igb_update_stats(adapter, &adapter->stats64);
1596 spin_unlock(&adapter->stats64_lock);
04fe6358 1597
9d5c8243
AK		 1598 adapter->link_speed = 0;
1599 adapter->link_duplex = 0;
1600
3023682e
JK		 1601 if (!pci_channel_offline(adapter->pdev))
1602 igb_reset(adapter);
9d5c8243
AK		 1603 igb_clean_all_tx_rings(adapter);
1604 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1605#ifdef CONFIG_IGB_DCA
1606
1607 /* since we reset the hardware DCA settings were cleared */
1608 igb_setup_dca(adapter);
1609#endif
9d5c8243
AK		 1610}
1611
1612void igb_reinit_locked(struct igb_adapter *adapter)
1613{
1614 WARN_ON(in_interrupt());
1615 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1616 msleep(1);
1617 igb_down(adapter);
1618 igb_up(adapter);
1619 clear_bit(__IGB_RESETTING, &adapter->state);
1620}
1621
1622void igb_reset(struct igb_adapter *adapter)
1623{
090b1795 1624 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1625 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1626 struct e1000_mac_info *mac = &hw->mac;
1627 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1628 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1629 u16 hwm;
1630
1631 /* Repartition Pba for greater than 9k mtu
1632 * To take effect CTRL.RST is required.
1633 */
fa4dfae0 1634 switch (mac->type) {
d2ba2ed8 1635 case e1000_i350:
55cac248
AD		 1636 case e1000_82580:
1637 pba = rd32(E1000_RXPBS);
1638 pba = igb_rxpbs_adjust_82580(pba);
1639 break;
fa4dfae0 1640 case e1000_82576:
d249be54
AD		 1641 pba = rd32(E1000_RXPBS);
1642 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1643 break;
1644 case e1000_82575:
1645 default:
1646 pba = E1000_PBA_34K;
1647 break;
2d064c06 1648 }
9d5c8243 1649
2d064c06
AD		 1650 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1651 (mac->type < e1000_82576)) {
9d5c8243
AK		 1652 /* adjust PBA for jumbo frames */
1653 wr32(E1000_PBA, pba);
1654
1655 /* To maintain wire speed transmits, the Tx FIFO should be
1656 * large enough to accommodate two full transmit packets,
1657 * rounded up to the next 1KB and expressed in KB. Likewise,
1658 * the Rx FIFO should be large enough to accommodate at least
1659 * one full receive packet and is similarly rounded up and
1660 * expressed in KB. */
1661 pba = rd32(E1000_PBA);
1662 /* upper 16 bits has Tx packet buffer allocation size in KB */
1663 tx_space = pba >> 16;
1664 /* lower 16 bits has Rx packet buffer allocation size in KB */
1665 pba &= 0xffff;
1666 /* the tx fifo also stores 16 bytes of information about the tx
1667 * but don't include ethernet FCS because hardware appends it */
1668 min_tx_space = (adapter->max_frame_size +
85e8d004 1669 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1670 ETH_FCS_LEN) * 2;
1671 min_tx_space = ALIGN(min_tx_space, 1024);
1672 min_tx_space >>= 10;
1673 /* software strips receive CRC, so leave room for it */
1674 min_rx_space = adapter->max_frame_size;
1675 min_rx_space = ALIGN(min_rx_space, 1024);
1676 min_rx_space >>= 10;
1677
1678 /* If current Tx allocation is less than the min Tx FIFO size,
1679 * and the min Tx FIFO size is less than the current Rx FIFO
1680 * allocation, take space away from current Rx allocation */
1681 if (tx_space < min_tx_space &&
1682 ((min_tx_space - tx_space) < pba)) {
1683 pba = pba - (min_tx_space - tx_space);
1684
1685 /* if short on rx space, rx wins and must trump tx
1686 * adjustment */
1687 if (pba < min_rx_space)
1688 pba = min_rx_space;
1689 }
2d064c06 1690 wr32(E1000_PBA, pba);
9d5c8243 1691 }
9d5c8243
AK		 1692
1693 /* flow control settings */
1694 /* The high water mark must be low enough to fit one full frame
1695 * (or the size used for early receive) above it in the Rx FIFO.
1696 * Set it to the lower of:
1697 * - 90% of the Rx FIFO size, or
1698 * - the full Rx FIFO size minus one full frame */
1699 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1700 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1701
d405ea3e
AD		 1702 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1703 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1704 fc->pause_time = 0xFFFF;
1705 fc->send_xon = 1;
0cce119a 1706 fc->current_mode = fc->requested_mode;
9d5c8243 1707
4ae196df
AD		 1708 /* disable receive for all VFs and wait one second */
1709 if (adapter->vfs_allocated_count) {
1710 int i;
1711 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1712 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1713
1714 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1715 igb_ping_all_vfs(adapter);
4ae196df
AD		 1716
1717 /* disable transmits and receives */
1718 wr32(E1000_VFRE, 0);
1719 wr32(E1000_VFTE, 0);
1720 }
1721
9d5c8243 1722 /* Allow time for pending master requests to run */
330a6d6a 1723 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1724 wr32(E1000_WUC, 0);
1725
330a6d6a 1726 if (hw->mac.ops.init_hw(hw))
090b1795 1727 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4
CW		 1728 if (hw->mac.type > e1000_82580) {
1729 if (adapter->flags & IGB_FLAG_DMAC) {
1730 u32 reg;
1731
1732 /*
1733 * DMA Coalescing high water mark needs to be higher
1734 * than * the * Rx threshold. The Rx threshold is
1735 * currently * pba - 6, so we * should use a high water
1736 * mark of pba * - 4. */
1737 hwm = (pba - 4) << 10;
1738
1739 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
1740 & E1000_DMACR_DMACTHR_MASK);
1741
1742 /* transition to L0x or L1 if available..*/
1743 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
1744
1745 /* watchdog timer= +-1000 usec in 32usec intervals */
1746 reg |= (1000 >> 5);
1747 wr32(E1000_DMACR, reg);
1748
1749 /* no lower threshold to disable coalescing(smart fifb)
1750 * -UTRESH=0*/
1751 wr32(E1000_DMCRTRH, 0);
1752
1753 /* set hwm to PBA - 2 * max frame size */
1754 wr32(E1000_FCRTC, hwm);
1755
1756 /*
1757 * This sets the time to wait before requesting tran-
1758 * sition to * low power state to number of usecs needed
1759 * to receive 1 512 * byte frame at gigabit line rate
1760 */
1761 reg = rd32(E1000_DMCTLX);
1762 reg |= IGB_DMCTLX_DCFLUSH_DIS;
1763
1764 /* Delay 255 usec before entering Lx state. */
1765 reg |= 0xFF;
1766 wr32(E1000_DMCTLX, reg);
1767
1768 /* free space in Tx packet buffer to wake from DMAC */
1769 wr32(E1000_DMCTXTH,
1770 (IGB_MIN_TXPBSIZE -
1771 (IGB_TX_BUF_4096 + adapter->max_frame_size))
1772 >> 6);
1773
1774 /* make low power state decision controlled by DMAC */
1775 reg = rd32(E1000_PCIEMISC);
1776 reg |= E1000_PCIEMISC_LX_DECISION;
1777 wr32(E1000_PCIEMISC, reg);
1778 } /* end if IGB_FLAG_DMAC set */
1779 }
55cac248
AD		 1780 if (hw->mac.type == e1000_82580) {
1781 u32 reg = rd32(E1000_PCIEMISC);
1782 wr32(E1000_PCIEMISC,
1783 reg & ~E1000_PCIEMISC_LX_DECISION);
1784 }
88a268c1
NN		 1785 if (!netif_running(adapter->netdev))
1786 igb_power_down_link(adapter);
1787
9d5c8243
AK		 1788 igb_update_mng_vlan(adapter);
1789
1790 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1791 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1792
330a6d6a 1793 igb_get_phy_info(hw);
9d5c8243
AK		 1794}
1795
b2cb09b1
JP		 1796static u32 igb_fix_features(struct net_device *netdev, u32 features)
1797{
1798 /*
1799 * Since there is no support for separate rx/tx vlan accel
1800 * enable/disable make sure tx flag is always in same state as rx.
1801 */
1802 if (features & NETIF_F_HW_VLAN_RX)
1803 features |= NETIF_F_HW_VLAN_TX;
1804 else
1805 features &= ~NETIF_F_HW_VLAN_TX;
1806
1807 return features;
1808}
1809
ac52caa3
MM		 1810static int igb_set_features(struct net_device *netdev, u32 features)
1811{
b2cb09b1 1812 u32 changed = netdev->features ^ features;
ac52caa3 1813
b2cb09b1
JP		 1814 if (changed & NETIF_F_HW_VLAN_RX)
1815 igb_vlan_mode(netdev, features);
1816
ac52caa3
MM		 1817 return 0;
1818}
1819
2e5c6922 1820static const struct net_device_ops igb_netdev_ops = {
559e9c49 1821 .ndo_open = igb_open,
2e5c6922 1822 .ndo_stop = igb_close,
cd392f5c 1823 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1824 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1825 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1826 .ndo_set_mac_address = igb_set_mac,
1827 .ndo_change_mtu = igb_change_mtu,
1828 .ndo_do_ioctl = igb_ioctl,
1829 .ndo_tx_timeout = igb_tx_timeout,
1830 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1831 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1832 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1833 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1834 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1835 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1836 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1837#ifdef CONFIG_NET_POLL_CONTROLLER
1838 .ndo_poll_controller = igb_netpoll,
1839#endif
b2cb09b1
JP		 1840 .ndo_fix_features = igb_fix_features,
1841 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1842};
1843
9d5c8243
AK		 1844/**
1845 * igb_probe - Device Initialization Routine
1846 * @pdev: PCI device information struct
1847 * @ent: entry in igb_pci_tbl
1848 *
1849 * Returns 0 on success, negative on failure
1850 *
1851 * igb_probe initializes an adapter identified by a pci_dev structure.
1852 * The OS initialization, configuring of the adapter private structure,
1853 * and a hardware reset occur.
1854 **/
1855static int __devinit igb_probe(struct pci_dev *pdev,
1856 const struct pci_device_id *ent)
1857{
1858 struct net_device *netdev;
1859 struct igb_adapter *adapter;
1860 struct e1000_hw *hw;
4337e993 1861 u16 eeprom_data = 0;
9835fd73 1862 s32 ret_val;
4337e993 1863 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1864 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1865 unsigned long mmio_start, mmio_len;
2d6a5e95 1866 int err, pci_using_dac;
9d5c8243 1867 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1868 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1869
bded64a7
AG		 1870 /* Catch broken hardware that put the wrong VF device ID in
1871 * the PCIe SR-IOV capability.
1872 */
1873 if (pdev->is_virtfn) {
1874 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1875 pci_name(pdev), pdev->vendor, pdev->device);
1876 return -EINVAL;
1877 }
1878
aed5dec3 1879 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1880 if (err)
1881 return err;
1882
1883 pci_using_dac = 0;
59d71989 1884 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1885 if (!err) {
59d71989 1886 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1887 if (!err)
1888 pci_using_dac = 1;
1889 } else {
59d71989 1890 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1891 if (err) {
59d71989 1892 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1893 if (err) {
1894 dev_err(&pdev->dev, "No usable DMA "
1895 "configuration, aborting\n");
1896 goto err_dma;
1897 }
1898 }
1899 }
1900
aed5dec3
AD		 1901 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1902 IORESOURCE_MEM),
1903 igb_driver_name);
9d5c8243
AK		 1904 if (err)
1905 goto err_pci_reg;
1906
19d5afd4 1907 pci_enable_pcie_error_reporting(pdev);
40a914fa 1908
9d5c8243 1909 pci_set_master(pdev);
c682fc23 1910 pci_save_state(pdev);
9d5c8243
AK		 1911
1912 err = -ENOMEM;
1bfaf07b 1913 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1914 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1915 if (!netdev)
1916 goto err_alloc_etherdev;
1917
1918 SET_NETDEV_DEV(netdev, &pdev->dev);
1919
1920 pci_set_drvdata(pdev, netdev);
1921 adapter = netdev_priv(netdev);
1922 adapter->netdev = netdev;
1923 adapter->pdev = pdev;
1924 hw = &adapter->hw;
1925 hw->back = adapter;
1926 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1927
1928 mmio_start = pci_resource_start(pdev, 0);
1929 mmio_len = pci_resource_len(pdev, 0);
1930
1931 err = -EIO;
28b0759c
AD		 1932 hw->hw_addr = ioremap(mmio_start, mmio_len);
1933 if (!hw->hw_addr)
9d5c8243
AK		 1934 goto err_ioremap;
1935
2e5c6922 1936 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1937 igb_set_ethtool_ops(netdev);
9d5c8243 1938 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1939
1940 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1941
1942 netdev->mem_start = mmio_start;
1943 netdev->mem_end = mmio_start + mmio_len;
1944
9d5c8243
AK		 1945 /* PCI config space info */
1946 hw->vendor_id = pdev->vendor;
1947 hw->device_id = pdev->device;
1948 hw->revision_id = pdev->revision;
1949 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1950 hw->subsystem_device_id = pdev->subsystem_device;
1951
9d5c8243
AK		 1952 /* Copy the default MAC, PHY and NVM function pointers */
1953 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1954 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1955 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1956 /* Initialize skew-specific constants */
1957 err = ei->get_invariants(hw);
1958 if (err)
450c87c8 1959 goto err_sw_init;
9d5c8243 1960
450c87c8 1961 /* setup the private structure */
9d5c8243
AK		 1962 err = igb_sw_init(adapter);
1963 if (err)
1964 goto err_sw_init;
1965
1966 igb_get_bus_info_pcie(hw);
1967
1968 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1969
1970 /* Copper options */
1971 if (hw->phy.media_type == e1000_media_type_copper) {
1972 hw->phy.mdix = AUTO_ALL_MODES;
1973 hw->phy.disable_polarity_correction = false;
1974 hw->phy.ms_type = e1000_ms_hw_default;
1975 }
1976
1977 if (igb_check_reset_block(hw))
1978 dev_info(&pdev->dev,
1979 "PHY reset is blocked due to SOL/IDER session.\n");
1980
077887c3
AD		 1981 /*
1982 * features is initialized to 0 in allocation, it might have bits
1983 * set by igb_sw_init so we should use an or instead of an
1984 * assignment.
1985 */
1986 netdev->features |= NETIF_F_SG |
1987 NETIF_F_IP_CSUM |
1988 NETIF_F_IPV6_CSUM |
1989 NETIF_F_TSO |
1990 NETIF_F_TSO6 |
1991 NETIF_F_RXHASH |
1992 NETIF_F_RXCSUM |
1993 NETIF_F_HW_VLAN_RX |
1994 NETIF_F_HW_VLAN_TX;
1995
1996 /* copy netdev features into list of user selectable features */
1997 netdev->hw_features |= netdev->features;
1998
1999 /* set this bit last since it cannot be part of hw_features */
2000 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2001
2002 netdev->vlan_features |= NETIF_F_TSO |
2003 NETIF_F_TSO6 |
2004 NETIF_F_IP_CSUM |
2005 NETIF_F_IPV6_CSUM |
2006 NETIF_F_SG;
48f29ffc 2007
7b872a55 2008 if (pci_using_dac) {
9d5c8243 2009 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 2010 netdev->vlan_features |= NETIF_F_HIGHDMA;
2011 }
9d5c8243 2012
ac52caa3
MM		 2013 if (hw->mac.type >= e1000_82576) {
2014 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 2015 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 2016 }
b9473560 2017
01789349
JP		 2018 netdev->priv_flags |= IFF_UNICAST_FLT;
2019
330a6d6a 2020 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 2021
2022 /* before reading the NVM, reset the controller to put the device in a
2023 * known good starting state */
2024 hw->mac.ops.reset_hw(hw);
2025
2026 /* make sure the NVM is good */
4322e561 2027 if (hw->nvm.ops.validate(hw) < 0) {
9d5c8243
AK		 2028 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2029 err = -EIO;
2030 goto err_eeprom;
2031 }
2032
2033 /* copy the MAC address out of the NVM */
2034 if (hw->mac.ops.read_mac_addr(hw))
2035 dev_err(&pdev->dev, "NVM Read Error\n");
2036
2037 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2038 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2039
2040 if (!is_valid_ether_addr(netdev->perm_addr)) {
2041 dev_err(&pdev->dev, "Invalid MAC Address\n");
2042 err = -EIO;
2043 goto err_eeprom;
2044 }
2045
c061b18d 2046 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2047 (unsigned long) adapter);
c061b18d 2048 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2049 (unsigned long) adapter);
9d5c8243
AK		 2050
2051 INIT_WORK(&adapter->reset_task, igb_reset_task);
2052 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2053
450c87c8 2054 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2055 adapter->fc_autoneg = true;
2056 hw->mac.autoneg = true;
2057 hw->phy.autoneg_advertised = 0x2f;
2058
0cce119a
AD		 2059 hw->fc.requested_mode = e1000_fc_default;
2060 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2061
9d5c8243
AK		 2062 igb_validate_mdi_setting(hw);
2063
9d5c8243
AK		 2064 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2065 * enable the ACPI Magic Packet filter
2066 */
2067
a2cf8b6c 2068 if (hw->bus.func == 0)
312c75ae 2069 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2070 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2071 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2072 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2073 &eeprom_data);
a2cf8b6c
AD		 2074 else if (hw->bus.func == 1)
2075 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2076
2077 if (eeprom_data & eeprom_apme_mask)
2078 adapter->eeprom_wol |= E1000_WUFC_MAG;
2079
2080 /* now that we have the eeprom settings, apply the special cases where
2081 * the eeprom may be wrong or the board simply won't support wake on
2082 * lan on a particular port */
2083 switch (pdev->device) {
2084 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2085 adapter->eeprom_wol = 0;
2086 break;
2087 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2088 case E1000_DEV_ID_82576_FIBER:
2089 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2090 /* Wake events only supported on port A for dual fiber
2091 * regardless of eeprom setting */
2092 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2093 adapter->eeprom_wol = 0;
2094 break;
c8ea5ea9 2095 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2096 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2097 /* if quad port adapter, disable WoL on all but port A */
2098 if (global_quad_port_a != 0)
2099 adapter->eeprom_wol = 0;
2100 else
2101 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2102 /* Reset for multiple quad port adapters */
2103 if (++global_quad_port_a == 4)
2104 global_quad_port_a = 0;
2105 break;
9d5c8243
AK		 2106 }
2107
2108 /* initialize the wol settings based on the eeprom settings */
2109 adapter->wol = adapter->eeprom_wol;
e1b86d84 2110 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2111
2112 /* reset the hardware with the new settings */
2113 igb_reset(adapter);
2114
2115 /* let the f/w know that the h/w is now under the control of the
2116 * driver. */
2117 igb_get_hw_control(adapter);
2118
9d5c8243
AK		 2119 strcpy(netdev->name, "eth%d");
2120 err = register_netdev(netdev);
2121 if (err)
2122 goto err_register;
2123
b168dfc5
JB		 2124 /* carrier off reporting is important to ethtool even BEFORE open */
2125 netif_carrier_off(netdev);
2126
421e02f0 2127#ifdef CONFIG_IGB_DCA
bbd98fe4 2128 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2129 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2130 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2131 igb_setup_dca(adapter);
2132 }
fe4506b6 2133
38c845c7 2134#endif
673b8b70
AB		 2135 /* do hw tstamp init after resetting */
2136 igb_init_hw_timer(adapter);
2137
9d5c8243
AK		 2138 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2139 /* print bus type/speed/width info */
7c510e4b 2140 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2141 netdev->name,
559e9c49 2142 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2143 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2144 "unknown"),
59c3de89
AD		 2145 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2146 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2147 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2148 "unknown"),
7c510e4b 2149 netdev->dev_addr);
9d5c8243 2150
9835fd73
CW		 2151 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2152 if (ret_val)
2153 strcpy(part_str, "Unknown");
2154 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2155 dev_info(&pdev->dev,
2156 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2157 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2158 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2159 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2160 switch (hw->mac.type) {
2161 case e1000_i350:
2162 igb_set_eee_i350(hw);
2163 break;
2164 default:
2165 break;
2166 }
9d5c8243
AK		 2167 return 0;
2168
2169err_register:
2170 igb_release_hw_control(adapter);
2171err_eeprom:
2172 if (!igb_check_reset_block(hw))
f5f4cf08 2173 igb_reset_phy(hw);
9d5c8243
AK		 2174
2175 if (hw->flash_address)
2176 iounmap(hw->flash_address);
9d5c8243 2177err_sw_init:
047e0030 2178 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2179 iounmap(hw->hw_addr);
2180err_ioremap:
2181 free_netdev(netdev);
2182err_alloc_etherdev:
559e9c49
AD		 2183 pci_release_selected_regions(pdev,
2184 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2185err_pci_reg:
2186err_dma:
2187 pci_disable_device(pdev);
2188 return err;
2189}
2190
2191/**
2192 * igb_remove - Device Removal Routine
2193 * @pdev: PCI device information struct
2194 *
2195 * igb_remove is called by the PCI subsystem to alert the driver
2196 * that it should release a PCI device. The could be caused by a
2197 * Hot-Plug event, or because the driver is going to be removed from
2198 * memory.
2199 **/
2200static void __devexit igb_remove(struct pci_dev *pdev)
2201{
2202 struct net_device *netdev = pci_get_drvdata(pdev);
2203 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2204 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2205
760141a5
TH		 2206 /*
2207 * The watchdog timer may be rescheduled, so explicitly
2208 * disable watchdog from being rescheduled.
2209 */
9d5c8243
AK		 2210 set_bit(__IGB_DOWN, &adapter->state);
2211 del_timer_sync(&adapter->watchdog_timer);
2212 del_timer_sync(&adapter->phy_info_timer);
2213
760141a5
TH		 2214 cancel_work_sync(&adapter->reset_task);
2215 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2216
421e02f0 2217#ifdef CONFIG_IGB_DCA
7dfc16fa 2218 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2219 dev_info(&pdev->dev, "DCA disabled\n");
2220 dca_remove_requester(&pdev->dev);
7dfc16fa 2221 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2222 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2223 }
2224#endif
2225
9d5c8243
AK		 2226 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2227 * would have already happened in close and is redundant. */
2228 igb_release_hw_control(adapter);
2229
2230 unregister_netdev(netdev);
2231
047e0030 2232 igb_clear_interrupt_scheme(adapter);
9d5c8243 2233
37680117
AD		 2234#ifdef CONFIG_PCI_IOV
2235 /* reclaim resources allocated to VFs */
2236 if (adapter->vf_data) {
2237 /* disable iov and allow time for transactions to clear */
0224d663
GR		 2238 if (!igb_check_vf_assignment(adapter)) {
2239 pci_disable_sriov(pdev);
2240 msleep(500);
2241 } else {
2242 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2243 }
37680117
AD		 2244
2245 kfree(adapter->vf_data);
2246 adapter->vf_data = NULL;
2247 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2248 wrfl();
37680117
AD		 2249 msleep(100);
2250 dev_info(&pdev->dev, "IOV Disabled\n");
2251 }
2252#endif
559e9c49 2253
28b0759c
AD		 2254 iounmap(hw->hw_addr);
2255 if (hw->flash_address)
2256 iounmap(hw->flash_address);
559e9c49
AD		 2257 pci_release_selected_regions(pdev,
2258 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2259
2260 free_netdev(netdev);
2261
19d5afd4 2262 pci_disable_pcie_error_reporting(pdev);
40a914fa 2263
9d5c8243
AK		 2264 pci_disable_device(pdev);
2265}
2266
a6b623e0
AD		 2267/**
2268 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2269 * @adapter: board private structure to initialize
2270 *
2271 * This function initializes the vf specific data storage and then attempts to
2272 * allocate the VFs. The reason for ordering it this way is because it is much
2273 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2274 * the memory for the VFs.
2275 **/
2276static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2277{
2278#ifdef CONFIG_PCI_IOV
2279 struct pci_dev *pdev = adapter->pdev;
0224d663
GR		 2280 int old_vfs = igb_find_enabled_vfs(adapter);
2281 int i;
a6b623e0 2282
0224d663
GR		 2283 if (old_vfs) {
2284 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2285 "max_vfs setting of %d\n", old_vfs, max_vfs);
2286 adapter->vfs_allocated_count = old_vfs;
a6b623e0
AD		 2287 }
2288
0224d663
GR		 2289 if (!adapter->vfs_allocated_count)
2290 return;
2291
2292 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2293 sizeof(struct vf_data_storage), GFP_KERNEL);
2294 /* if allocation failed then we do not support SR-IOV */
2295 if (!adapter->vf_data) {
a6b623e0 2296 adapter->vfs_allocated_count = 0;
0224d663
GR		 2297 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2298 "Data Storage\n");
2299 goto out;
a6b623e0 2300 }
0224d663
GR		 2301
2302 if (!old_vfs) {
2303 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2304 goto err_out;
2305 }
2306 dev_info(&pdev->dev, "%d VFs allocated\n",
2307 adapter->vfs_allocated_count);
2308 for (i = 0; i < adapter->vfs_allocated_count; i++)
2309 igb_vf_configure(adapter, i);
2310
2311 /* DMA Coalescing is not supported in IOV mode. */
2312 adapter->flags &= ~IGB_FLAG_DMAC;
2313 goto out;
2314err_out:
2315 kfree(adapter->vf_data);
2316 adapter->vf_data = NULL;
2317 adapter->vfs_allocated_count = 0;
2318out:
2319 return;
a6b623e0
AD		 2320#endif /* CONFIG_PCI_IOV */
2321}
2322
115f459a
AD		 2323/**
2324 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2325 * @adapter: board private structure to initialize
2326 *
2327 * igb_init_hw_timer initializes the function pointer and values for the hw
2328 * timer found in hardware.
2329 **/
2330static void igb_init_hw_timer(struct igb_adapter *adapter)
2331{
2332 struct e1000_hw *hw = &adapter->hw;
2333
2334 switch (hw->mac.type) {
d2ba2ed8 2335 case e1000_i350:
55cac248
AD		 2336 case e1000_82580:
2337 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2338 adapter->cycles.read = igb_read_clock;
2339 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2340 adapter->cycles.mult = 1;
2341 /*
2342 * The 82580 timesync updates the system timer every 8ns by 8ns
2343 * and the value cannot be shifted. Instead we need to shift
2344 * the registers to generate a 64bit timer value. As a result
2345 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2346 * 24 in order to generate a larger value for synchronization.
2347 */
2348 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2349 /* disable system timer temporarily by setting bit 31 */
2350 wr32(E1000_TSAUXC, 0x80000000);
2351 wrfl();
2352
2353 /* Set registers so that rollover occurs soon to test this. */
2354 wr32(E1000_SYSTIMR, 0x00000000);
2355 wr32(E1000_SYSTIML, 0x80000000);
2356 wr32(E1000_SYSTIMH, 0x000000FF);
2357 wrfl();
2358
2359 /* enable system timer by clearing bit 31 */
2360 wr32(E1000_TSAUXC, 0x0);
2361 wrfl();
2362
2363 timecounter_init(&adapter->clock,
2364 &adapter->cycles,
2365 ktime_to_ns(ktime_get_real()));
2366 /*
2367 * Synchronize our NIC clock against system wall clock. NIC
2368 * time stamp reading requires ~3us per sample, each sample
2369 * was pretty stable even under load => only require 10
2370 * samples for each offset comparison.
2371 */
2372 memset(&adapter->compare, 0, sizeof(adapter->compare));
2373 adapter->compare.source = &adapter->clock;
2374 adapter->compare.target = ktime_get_real;
2375 adapter->compare.num_samples = 10;
2376 timecompare_update(&adapter->compare, 0);
2377 break;
115f459a
AD		 2378 case e1000_82576:
2379 /*
2380 * Initialize hardware timer: we keep it running just in case
2381 * that some program needs it later on.
2382 */
2383 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2384 adapter->cycles.read = igb_read_clock;
2385 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2386 adapter->cycles.mult = 1;
2387 /**
2388 * Scale the NIC clock cycle by a large factor so that
2389 * relatively small clock corrections can be added or
25985edc 2390 * subtracted at each clock tick. The drawbacks of a large
115f459a
AD		 2391 * factor are a) that the clock register overflows more quickly
2392 * (not such a big deal) and b) that the increment per tick has
2393 * to fit into 24 bits. As a result we need to use a shift of
2394 * 19 so we can fit a value of 16 into the TIMINCA register.
2395 */
2396 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2397 wr32(E1000_TIMINCA,
2398 (1 << E1000_TIMINCA_16NS_SHIFT) |
2399 (16 << IGB_82576_TSYNC_SHIFT));
2400
2401 /* Set registers so that rollover occurs soon to test this. */
2402 wr32(E1000_SYSTIML, 0x00000000);
2403 wr32(E1000_SYSTIMH, 0xFF800000);
2404 wrfl();
2405
2406 timecounter_init(&adapter->clock,
2407 &adapter->cycles,
2408 ktime_to_ns(ktime_get_real()));
2409 /*
2410 * Synchronize our NIC clock against system wall clock. NIC
2411 * time stamp reading requires ~3us per sample, each sample
2412 * was pretty stable even under load => only require 10
2413 * samples for each offset comparison.
2414 */
2415 memset(&adapter->compare, 0, sizeof(adapter->compare));
2416 adapter->compare.source = &adapter->clock;
2417 adapter->compare.target = ktime_get_real;
2418 adapter->compare.num_samples = 10;
2419 timecompare_update(&adapter->compare, 0);
2420 break;
2421 case e1000_82575:
2422 /* 82575 does not support timesync */
2423 default:
2424 break;
2425 }
2426
2427}
2428
9d5c8243
AK		 2429/**
2430 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2431 * @adapter: board private structure to initialize
2432 *
2433 * igb_sw_init initializes the Adapter private data structure.
2434 * Fields are initialized based on PCI device information and
2435 * OS network device settings (MTU size).
2436 **/
2437static int __devinit igb_sw_init(struct igb_adapter *adapter)
2438{
2439 struct e1000_hw *hw = &adapter->hw;
2440 struct net_device *netdev = adapter->netdev;
2441 struct pci_dev *pdev = adapter->pdev;
2442
2443 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2444
13fde97a 2445 /* set default ring sizes */
68fd9910
AD		 2446 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2447 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2448
2449 /* set default ITR values */
4fc82adf
AD		 2450 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2451 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2452
13fde97a
AD		 2453 /* set default work limits */
2454 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2455
153285f9
AD		 2456 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2457 VLAN_HLEN;
9d5c8243
AK		 2458 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2459
81c2fc22
AD		 2460 adapter->node = -1;
2461
12dcd86b 2462 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2463#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2464 switch (hw->mac.type) {
2465 case e1000_82576:
2466 case e1000_i350:
9b082d73
SA		 2467 if (max_vfs > 7) {
2468 dev_warn(&pdev->dev,
2469 "Maximum of 7 VFs per PF, using max\n");
2470 adapter->vfs_allocated_count = 7;
2471 } else
2472 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2473 break;
2474 default:
2475 break;
2476 }
a6b623e0 2477#endif /* CONFIG_PCI_IOV */
a99955fc 2478 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
665c8c8e
WM		 2479 /* i350 cannot do RSS and SR-IOV at the same time */
2480 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2481 adapter->rss_queues = 1;
a99955fc
AD		 2482
2483 /*
2484 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2485 * then we should combine the queues into a queue pair in order to
2486 * conserve interrupts due to limited supply
2487 */
2488 if ((adapter->rss_queues > 4) ||
2489 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2490 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2491
a6b623e0 2492 /* This call may decrease the number of queues */
047e0030 2493 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2494 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2495 return -ENOMEM;
2496 }
2497
a6b623e0
AD		 2498 igb_probe_vfs(adapter);
2499
9d5c8243
AK		 2500 /* Explicitly disable IRQ since the NIC can be in any state. */
2501 igb_irq_disable(adapter);
2502
831ec0b4
CW		 2503 if (hw->mac.type == e1000_i350)
2504 adapter->flags &= ~IGB_FLAG_DMAC;
2505
9d5c8243
AK		 2506 set_bit(__IGB_DOWN, &adapter->state);
2507 return 0;
2508}
2509
2510/**
2511 * igb_open - Called when a network interface is made active
2512 * @netdev: network interface device structure
2513 *
2514 * Returns 0 on success, negative value on failure
2515 *
2516 * The open entry point is called when a network interface is made
2517 * active by the system (IFF_UP). At this point all resources needed
2518 * for transmit and receive operations are allocated, the interrupt
2519 * handler is registered with the OS, the watchdog timer is started,
2520 * and the stack is notified that the interface is ready.
2521 **/
2522static int igb_open(struct net_device *netdev)
2523{
2524 struct igb_adapter *adapter = netdev_priv(netdev);
2525 struct e1000_hw *hw = &adapter->hw;
2526 int err;
2527 int i;
2528
2529 /* disallow open during test */
2530 if (test_bit(__IGB_TESTING, &adapter->state))
2531 return -EBUSY;
2532
b168dfc5
JB		 2533 netif_carrier_off(netdev);
2534
9d5c8243
AK		 2535 /* allocate transmit descriptors */
2536 err = igb_setup_all_tx_resources(adapter);
2537 if (err)
2538 goto err_setup_tx;
2539
2540 /* allocate receive descriptors */
2541 err = igb_setup_all_rx_resources(adapter);
2542 if (err)
2543 goto err_setup_rx;
2544
88a268c1 2545 igb_power_up_link(adapter);
9d5c8243 2546
9d5c8243
AK		 2547 /* before we allocate an interrupt, we must be ready to handle it.
2548 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2549 * as soon as we call pci_request_irq, so we have to setup our
2550 * clean_rx handler before we do so. */
2551 igb_configure(adapter);
2552
2553 err = igb_request_irq(adapter);
2554 if (err)
2555 goto err_req_irq;
2556
2557 /* From here on the code is the same as igb_up() */
2558 clear_bit(__IGB_DOWN, &adapter->state);
2559
0d1ae7f4
AD		 2560 for (i = 0; i < adapter->num_q_vectors; i++)
2561 napi_enable(&(adapter->q_vector[i]->napi));
9d5c8243
AK		 2562
2563 /* Clear any pending interrupts. */
2564 rd32(E1000_ICR);
844290e5
PW		 2565
2566 igb_irq_enable(adapter);
2567
d4960307
AD		 2568 /* notify VFs that reset has been completed */
2569 if (adapter->vfs_allocated_count) {
2570 u32 reg_data = rd32(E1000_CTRL_EXT);
2571 reg_data |= E1000_CTRL_EXT_PFRSTD;
2572 wr32(E1000_CTRL_EXT, reg_data);
2573 }
2574
d55b53ff
JK		 2575 netif_tx_start_all_queues(netdev);
2576
25568a53
AD		 2577 /* start the watchdog. */
2578 hw->mac.get_link_status = 1;
2579 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2580
2581 return 0;
2582
2583err_req_irq:
2584 igb_release_hw_control(adapter);
88a268c1 2585 igb_power_down_link(adapter);
9d5c8243
AK		 2586 igb_free_all_rx_resources(adapter);
2587err_setup_rx:
2588 igb_free_all_tx_resources(adapter);
2589err_setup_tx:
2590 igb_reset(adapter);
2591
2592 return err;
2593}
2594
2595/**
2596 * igb_close - Disables a network interface
2597 * @netdev: network interface device structure
2598 *
2599 * Returns 0, this is not allowed to fail
2600 *
2601 * The close entry point is called when an interface is de-activated
2602 * by the OS. The hardware is still under the driver's control, but
2603 * needs to be disabled. A global MAC reset is issued to stop the
2604 * hardware, and all transmit and receive resources are freed.
2605 **/
2606static int igb_close(struct net_device *netdev)
2607{
2608 struct igb_adapter *adapter = netdev_priv(netdev);
2609
2610 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2611 igb_down(adapter);
2612
2613 igb_free_irq(adapter);
2614
2615 igb_free_all_tx_resources(adapter);
2616 igb_free_all_rx_resources(adapter);
2617
9d5c8243
AK		 2618 return 0;
2619}
2620
2621/**
2622 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2623 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2624 *
2625 * Return 0 on success, negative on failure
2626 **/
80785298 2627int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2628{
59d71989 2629 struct device *dev = tx_ring->dev;
81c2fc22 2630 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2631 int size;
2632
06034649 2633 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2634 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2635 if (!tx_ring->tx_buffer_info)
2636 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2637 if (!tx_ring->tx_buffer_info)
9d5c8243 2638 goto err;
9d5c8243
AK		 2639
2640 /* round up to nearest 4K */
85e8d004 2641 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2642 tx_ring->size = ALIGN(tx_ring->size, 4096);
2643
81c2fc22 2644 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2645 tx_ring->desc = dma_alloc_coherent(dev,
2646 tx_ring->size,
2647 &tx_ring->dma,
2648 GFP_KERNEL);
81c2fc22
AD		 2649 set_dev_node(dev, orig_node);
2650 if (!tx_ring->desc)
2651 tx_ring->desc = dma_alloc_coherent(dev,
2652 tx_ring->size,
2653 &tx_ring->dma,
2654 GFP_KERNEL);
9d5c8243
AK		 2655
2656 if (!tx_ring->desc)
2657 goto err;
2658
9d5c8243
AK		 2659 tx_ring->next_to_use = 0;
2660 tx_ring->next_to_clean = 0;
81c2fc22 2661
9d5c8243
AK		 2662 return 0;
2663
2664err:
06034649 2665 vfree(tx_ring->tx_buffer_info);
59d71989 2666 dev_err(dev,
9d5c8243
AK		 2667 "Unable to allocate memory for the transmit descriptor ring\n");
2668 return -ENOMEM;
2669}
2670
2671/**
2672 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2673 * (Descriptors) for all queues
2674 * @adapter: board private structure
2675 *
2676 * Return 0 on success, negative on failure
2677 **/
2678static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2679{
439705e1 2680 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2681 int i, err = 0;
2682
2683 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2684 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2685 if (err) {
439705e1 2686 dev_err(&pdev->dev,
9d5c8243
AK		 2687 "Allocation for Tx Queue %u failed\n", i);
2688 for (i--; i >= 0; i--)
3025a446 2689 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2690 break;
2691 }
2692 }
2693
2694 return err;
2695}
2696
2697/**
85b430b4
AD		 2698 * igb_setup_tctl - configure the transmit control registers
2699 * @adapter: Board private structure
9d5c8243 2700 **/
d7ee5b3a 2701void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2702{
9d5c8243
AK		 2703 struct e1000_hw *hw = &adapter->hw;
2704 u32 tctl;
9d5c8243 2705
85b430b4
AD		 2706 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2707 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2708
2709 /* Program the Transmit Control Register */
9d5c8243
AK		 2710 tctl = rd32(E1000_TCTL);
2711 tctl &= ~E1000_TCTL_CT;
2712 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2713 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2714
2715 igb_config_collision_dist(hw);
2716
9d5c8243
AK		 2717 /* Enable transmits */
2718 tctl |= E1000_TCTL_EN;
2719
2720 wr32(E1000_TCTL, tctl);
2721}
2722
85b430b4
AD		 2723/**
2724 * igb_configure_tx_ring - Configure transmit ring after Reset
2725 * @adapter: board private structure
2726 * @ring: tx ring to configure
2727 *
2728 * Configure a transmit ring after a reset.
2729 **/
d7ee5b3a
AD		 2730void igb_configure_tx_ring(struct igb_adapter *adapter,
2731 struct igb_ring *ring)
85b430b4
AD		 2732{
2733 struct e1000_hw *hw = &adapter->hw;
a74420e0 2734 u32 txdctl = 0;
85b430b4
AD		 2735 u64 tdba = ring->dma;
2736 int reg_idx = ring->reg_idx;
2737
2738 /* disable the queue */
a74420e0 2739 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2740 wrfl();
2741 mdelay(10);
2742
2743 wr32(E1000_TDLEN(reg_idx),
2744 ring->count * sizeof(union e1000_adv_tx_desc));
2745 wr32(E1000_TDBAL(reg_idx),
2746 tdba & 0x00000000ffffffffULL);
2747 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2748
fce99e34 2749 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2750 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2751 writel(0, ring->tail);
85b430b4
AD		 2752
2753 txdctl |= IGB_TX_PTHRESH;
2754 txdctl |= IGB_TX_HTHRESH << 8;
2755 txdctl |= IGB_TX_WTHRESH << 16;
2756
2757 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2758 wr32(E1000_TXDCTL(reg_idx), txdctl);
2759}
2760
2761/**
2762 * igb_configure_tx - Configure transmit Unit after Reset
2763 * @adapter: board private structure
2764 *
2765 * Configure the Tx unit of the MAC after a reset.
2766 **/
2767static void igb_configure_tx(struct igb_adapter *adapter)
2768{
2769 int i;
2770
2771 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2772 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2773}
2774
9d5c8243
AK		 2775/**
2776 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2777 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2778 *
2779 * Returns 0 on success, negative on failure
2780 **/
80785298 2781int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2782{
59d71989 2783 struct device *dev = rx_ring->dev;
81c2fc22 2784 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2785 int size, desc_len;
2786
06034649 2787 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2788 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2789 if (!rx_ring->rx_buffer_info)
2790 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2791 if (!rx_ring->rx_buffer_info)
9d5c8243 2792 goto err;
9d5c8243
AK		 2793
2794 desc_len = sizeof(union e1000_adv_rx_desc);
2795
2796 /* Round up to nearest 4K */
2797 rx_ring->size = rx_ring->count * desc_len;
2798 rx_ring->size = ALIGN(rx_ring->size, 4096);
2799
81c2fc22 2800 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2801 rx_ring->desc = dma_alloc_coherent(dev,
2802 rx_ring->size,
2803 &rx_ring->dma,
2804 GFP_KERNEL);
81c2fc22
AD		 2805 set_dev_node(dev, orig_node);
2806 if (!rx_ring->desc)
2807 rx_ring->desc = dma_alloc_coherent(dev,
2808 rx_ring->size,
2809 &rx_ring->dma,
2810 GFP_KERNEL);
9d5c8243
AK		 2811
2812 if (!rx_ring->desc)
2813 goto err;
2814
2815 rx_ring->next_to_clean = 0;
2816 rx_ring->next_to_use = 0;
9d5c8243 2817
9d5c8243
AK		 2818 return 0;
2819
2820err:
06034649
AD		 2821 vfree(rx_ring->rx_buffer_info);
2822 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2823 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2824 " ring\n");
9d5c8243
AK		 2825 return -ENOMEM;
2826}
2827
2828/**
2829 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2830 * (Descriptors) for all queues
2831 * @adapter: board private structure
2832 *
2833 * Return 0 on success, negative on failure
2834 **/
2835static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2836{
439705e1 2837 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2838 int i, err = 0;
2839
2840 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2841 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2842 if (err) {
439705e1 2843 dev_err(&pdev->dev,
9d5c8243
AK		 2844 "Allocation for Rx Queue %u failed\n", i);
2845 for (i--; i >= 0; i--)
3025a446 2846 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2847 break;
2848 }
2849 }
2850
2851 return err;
2852}
2853
06cf2666
AD		 2854/**
2855 * igb_setup_mrqc - configure the multiple receive queue control registers
2856 * @adapter: Board private structure
2857 **/
2858static void igb_setup_mrqc(struct igb_adapter *adapter)
2859{
2860 struct e1000_hw *hw = &adapter->hw;
2861 u32 mrqc, rxcsum;
2862 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2863 union e1000_reta {
2864 u32 dword;
2865 u8 bytes[4];
2866 } reta;
2867 static const u8 rsshash[40] = {
2868 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2869 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2870 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2871 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2872
2873 /* Fill out hash function seeds */
2874 for (j = 0; j < 10; j++) {
2875 u32 rsskey = rsshash[(j * 4)];
2876 rsskey |= rsshash[(j * 4) + 1] << 8;
2877 rsskey |= rsshash[(j * 4) + 2] << 16;
2878 rsskey |= rsshash[(j * 4) + 3] << 24;
2879 array_wr32(E1000_RSSRK(0), j, rsskey);
2880 }
2881
a99955fc 2882 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2883
2884 if (adapter->vfs_allocated_count) {
2885 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2886 switch (hw->mac.type) {
d2ba2ed8 2887 case e1000_i350:
55cac248
AD		 2888 case e1000_82580:
2889 num_rx_queues = 1;
2890 shift = 0;
2891 break;
06cf2666
AD		 2892 case e1000_82576:
2893 shift = 3;
2894 num_rx_queues = 2;
2895 break;
2896 case e1000_82575:
2897 shift = 2;
2898 shift2 = 6;
2899 default:
2900 break;
2901 }
2902 } else {
2903 if (hw->mac.type == e1000_82575)
2904 shift = 6;
2905 }
2906
2907 for (j = 0; j < (32 * 4); j++) {
2908 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2909 if (shift2)
2910 reta.bytes[j & 3] |= num_rx_queues << shift2;
2911 if ((j & 3) == 3)
2912 wr32(E1000_RETA(j >> 2), reta.dword);
2913 }
2914
2915 /*
2916 * Disable raw packet checksumming so that RSS hash is placed in
2917 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2918 * offloads as they are enabled by default
2919 */
2920 rxcsum = rd32(E1000_RXCSUM);
2921 rxcsum |= E1000_RXCSUM_PCSD;
2922
2923 if (adapter->hw.mac.type >= e1000_82576)
2924 /* Enable Receive Checksum Offload for SCTP */
2925 rxcsum |= E1000_RXCSUM_CRCOFL;
2926
2927 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2928 wr32(E1000_RXCSUM, rxcsum);
2929
2930 /* If VMDq is enabled then we set the appropriate mode for that, else
2931 * we default to RSS so that an RSS hash is calculated per packet even
2932 * if we are only using one queue */
2933 if (adapter->vfs_allocated_count) {
2934 if (hw->mac.type > e1000_82575) {
2935 /* Set the default pool for the PF's first queue */
2936 u32 vtctl = rd32(E1000_VT_CTL);
2937 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2938 E1000_VT_CTL_DISABLE_DEF_POOL);
2939 vtctl |= adapter->vfs_allocated_count <<
2940 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2941 wr32(E1000_VT_CTL, vtctl);
2942 }
a99955fc 2943 if (adapter->rss_queues > 1)
06cf2666
AD		 2944 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2945 else
2946 mrqc = E1000_MRQC_ENABLE_VMDQ;
2947 } else {
2948 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2949 }
2950 igb_vmm_control(adapter);
2951
4478a9cd
AD		 2952 /*
2953 * Generate RSS hash based on TCP port numbers and/or
2954 * IPv4/v6 src and dst addresses since UDP cannot be
2955 * hashed reliably due to IP fragmentation
2956 */
2957 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2958 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2959 E1000_MRQC_RSS_FIELD_IPV6 |
2960 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2961 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2962
2963 wr32(E1000_MRQC, mrqc);
2964}
2965
9d5c8243
AK		 2966/**
2967 * igb_setup_rctl - configure the receive control registers
2968 * @adapter: Board private structure
2969 **/
d7ee5b3a 2970void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2971{
2972 struct e1000_hw *hw = &adapter->hw;
2973 u32 rctl;
9d5c8243
AK		 2974
2975 rctl = rd32(E1000_RCTL);
2976
2977 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2978 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2979
69d728ba 2980 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2981 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2982
87cb7e8c
AK		 2983 /*
2984 * enable stripping of CRC. It's unlikely this will break BMC
2985 * redirection as it did with e1000. Newer features require
2986 * that the HW strips the CRC.
73cd78f1 2987 */
87cb7e8c 2988 rctl |= E1000_RCTL_SECRC;
9d5c8243 2989
559e9c49 2990 /* disable store bad packets and clear size bits. */
ec54d7d6 2991 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2992
6ec43fe6
AD		 2993 /* enable LPE to prevent packets larger than max_frame_size */
2994 rctl |= E1000_RCTL_LPE;
9d5c8243 2995
952f72a8
AD		 2996 /* disable queue 0 to prevent tail write w/o re-config */
2997 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2998
e1739522
AD		 2999 /* Attention!!! For SR-IOV PF driver operations you must enable
3000 * queue drop for all VF and PF queues to prevent head of line blocking
3001 * if an un-trusted VF does not provide descriptors to hardware.
3002 */
3003 if (adapter->vfs_allocated_count) {
e1739522
AD		 3004 /* set all queue drop enable bits */
3005 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 3006 }
3007
9d5c8243
AK		 3008 wr32(E1000_RCTL, rctl);
3009}
3010
7d5753f0
AD		 3011static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3012 int vfn)
3013{
3014 struct e1000_hw *hw = &adapter->hw;
3015 u32 vmolr;
3016
3017 /* if it isn't the PF check to see if VFs are enabled and
3018 * increase the size to support vlan tags */
3019 if (vfn < adapter->vfs_allocated_count &&
3020 adapter->vf_data[vfn].vlans_enabled)
3021 size += VLAN_TAG_SIZE;
3022
3023 vmolr = rd32(E1000_VMOLR(vfn));
3024 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3025 vmolr |= size | E1000_VMOLR_LPE;
3026 wr32(E1000_VMOLR(vfn), vmolr);
3027
3028 return 0;
3029}
3030
e1739522
AD		 3031/**
3032 * igb_rlpml_set - set maximum receive packet size
3033 * @adapter: board private structure
3034 *
3035 * Configure maximum receivable packet size.
3036 **/
3037static void igb_rlpml_set(struct igb_adapter *adapter)
3038{
153285f9 3039 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 3040 struct e1000_hw *hw = &adapter->hw;
3041 u16 pf_id = adapter->vfs_allocated_count;
3042
e1739522
AD		 3043 if (pf_id) {
3044 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3045 /*
3046 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3047 * to our max jumbo frame size, in case we need to enable
3048 * jumbo frames on one of the rings later.
3049 * This will not pass over-length frames into the default
3050 * queue because it's gated by the VMOLR.RLPML.
3051 */
7d5753f0 3052 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3053 }
3054
3055 wr32(E1000_RLPML, max_frame_size);
3056}
3057
8151d294
WM		 3058static inline void igb_set_vmolr(struct igb_adapter *adapter,
3059 int vfn, bool aupe)
7d5753f0
AD		 3060{
3061 struct e1000_hw *hw = &adapter->hw;
3062 u32 vmolr;
3063
3064 /*
3065 * This register exists only on 82576 and newer so if we are older then
3066 * we should exit and do nothing
3067 */
3068 if (hw->mac.type < e1000_82576)
3069 return;
3070
3071 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3072 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3073 if (aupe)
3074 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3075 else
3076 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3077
3078 /* clear all bits that might not be set */
3079 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3080
a99955fc 3081 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3082 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3083 /*
3084 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3085 * multicast packets
3086 */
3087 if (vfn <= adapter->vfs_allocated_count)
3088 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3089
3090 wr32(E1000_VMOLR(vfn), vmolr);
3091}
3092
85b430b4
AD		 3093/**
3094 * igb_configure_rx_ring - Configure a receive ring after Reset
3095 * @adapter: board private structure
3096 * @ring: receive ring to be configured
3097 *
3098 * Configure the Rx unit of the MAC after a reset.
3099 **/
d7ee5b3a
AD		 3100void igb_configure_rx_ring(struct igb_adapter *adapter,
3101 struct igb_ring *ring)
85b430b4
AD		 3102{
3103 struct e1000_hw *hw = &adapter->hw;
3104 u64 rdba = ring->dma;
3105 int reg_idx = ring->reg_idx;
a74420e0 3106 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3107
3108 /* disable the queue */
a74420e0 3109 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3110
3111 /* Set DMA base address registers */
3112 wr32(E1000_RDBAL(reg_idx),
3113 rdba & 0x00000000ffffffffULL);
3114 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3115 wr32(E1000_RDLEN(reg_idx),
3116 ring->count * sizeof(union e1000_adv_rx_desc));
3117
3118 /* initialize head and tail */
fce99e34 3119 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3120 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3121 writel(0, ring->tail);
85b430b4 3122
952f72a8 3123 /* set descriptor configuration */
44390ca6 3124 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3125#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3126 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3127#else
44390ca6 3128 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3129#endif
44390ca6 3130 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
06218a8d 3131 if (hw->mac.type >= e1000_82580)
757b77e2 3132 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3133 /* Only set Drop Enable if we are supporting multiple queues */
3134 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3135 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3136
3137 wr32(E1000_SRRCTL(reg_idx), srrctl);
3138
7d5753f0 3139 /* set filtering for VMDQ pools */
8151d294 3140 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3141
85b430b4
AD		 3142 rxdctl |= IGB_RX_PTHRESH;
3143 rxdctl |= IGB_RX_HTHRESH << 8;
3144 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3145
3146 /* enable receive descriptor fetching */
3147 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3148 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3149}
3150
9d5c8243
AK		 3151/**
3152 * igb_configure_rx - Configure receive Unit after Reset
3153 * @adapter: board private structure
3154 *
3155 * Configure the Rx unit of the MAC after a reset.
3156 **/
3157static void igb_configure_rx(struct igb_adapter *adapter)
3158{
9107584e 3159 int i;
9d5c8243 3160
68d480c4
AD		 3161 /* set UTA to appropriate mode */
3162 igb_set_uta(adapter);
3163
26ad9178
AD		 3164 /* set the correct pool for the PF default MAC address in entry 0 */
3165 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3166 adapter->vfs_allocated_count);
3167
06cf2666
AD		 3168 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3169 * the Base and Length of the Rx Descriptor Ring */
3170 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3171 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3172}
3173
3174/**
3175 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3176 * @tx_ring: Tx descriptor ring for a specific queue
3177 *
3178 * Free all transmit software resources
3179 **/
68fd9910 3180void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3181{
3b644cf6 3182 igb_clean_tx_ring(tx_ring);
9d5c8243 3183
06034649
AD		 3184 vfree(tx_ring->tx_buffer_info);
3185 tx_ring->tx_buffer_info = NULL;
9d5c8243 3186
439705e1
AD		 3187 /* if not set, then don't free */
3188 if (!tx_ring->desc)
3189 return;
3190
59d71989
AD		 3191 dma_free_coherent(tx_ring->dev, tx_ring->size,
3192 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3193
3194 tx_ring->desc = NULL;
3195}
3196
3197/**
3198 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3199 * @adapter: board private structure
3200 *
3201 * Free all transmit software resources
3202 **/
3203static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3204{
3205 int i;
3206
3207 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3208 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3209}
3210
ebe42d16
AD		 3211void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3212 struct igb_tx_buffer *tx_buffer)
3213{
3214 if (tx_buffer->skb) {
3215 dev_kfree_skb_any(tx_buffer->skb);
3216 if (tx_buffer->dma)
3217 dma_unmap_single(ring->dev,
3218 tx_buffer->dma,
3219 tx_buffer->length,
3220 DMA_TO_DEVICE);
3221 } else if (tx_buffer->dma) {
3222 dma_unmap_page(ring->dev,
3223 tx_buffer->dma,
3224 tx_buffer->length,
3225 DMA_TO_DEVICE);
3226 }
3227 tx_buffer->next_to_watch = NULL;
3228 tx_buffer->skb = NULL;
3229 tx_buffer->dma = 0;
3230 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3231}
3232
3233/**
3234 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3235 * @tx_ring: ring to be cleaned
3236 **/
3b644cf6 3237static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3238{
06034649 3239 struct igb_tx_buffer *buffer_info;
9d5c8243 3240 unsigned long size;
6ad4edfc 3241 u16 i;
9d5c8243 3242
06034649 3243 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3244 return;
3245 /* Free all the Tx ring sk_buffs */
3246
3247 for (i = 0; i < tx_ring->count; i++) {
06034649 3248 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3249 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3250 }
3251
06034649
AD		 3252 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3253 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3254
3255 /* Zero out the descriptor ring */
9d5c8243
AK		 3256 memset(tx_ring->desc, 0, tx_ring->size);
3257
3258 tx_ring->next_to_use = 0;
3259 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3260}
3261
3262/**
3263 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3264 * @adapter: board private structure
3265 **/
3266static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3267{
3268 int i;
3269
3270 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3271 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3272}
3273
3274/**
3275 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3276 * @rx_ring: ring to clean the resources from
3277 *
3278 * Free all receive software resources
3279 **/
68fd9910 3280void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3281{
3b644cf6 3282 igb_clean_rx_ring(rx_ring);
9d5c8243 3283
06034649
AD		 3284 vfree(rx_ring->rx_buffer_info);
3285 rx_ring->rx_buffer_info = NULL;
9d5c8243 3286
439705e1
AD		 3287 /* if not set, then don't free */
3288 if (!rx_ring->desc)
3289 return;
3290
59d71989
AD		 3291 dma_free_coherent(rx_ring->dev, rx_ring->size,
3292 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3293
3294 rx_ring->desc = NULL;
3295}
3296
3297/**
3298 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3299 * @adapter: board private structure
3300 *
3301 * Free all receive software resources
3302 **/
3303static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3304{
3305 int i;
3306
3307 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3308 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3309}
3310
3311/**
3312 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3313 * @rx_ring: ring to free buffers from
3314 **/
3b644cf6 3315static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3316{
9d5c8243 3317 unsigned long size;
c023cd88 3318 u16 i;
9d5c8243 3319
06034649 3320 if (!rx_ring->rx_buffer_info)
9d5c8243 3321 return;
439705e1 3322
9d5c8243
AK		 3323 /* Free all the Rx ring sk_buffs */
3324 for (i = 0; i < rx_ring->count; i++) {
06034649 3325 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3326 if (buffer_info->dma) {
59d71989 3327 dma_unmap_single(rx_ring->dev,
80785298 3328 buffer_info->dma,
44390ca6 3329 IGB_RX_HDR_LEN,
59d71989 3330 DMA_FROM_DEVICE);
9d5c8243
AK		 3331 buffer_info->dma = 0;
3332 }
3333
3334 if (buffer_info->skb) {
3335 dev_kfree_skb(buffer_info->skb);
3336 buffer_info->skb = NULL;
3337 }
6ec43fe6 3338 if (buffer_info->page_dma) {
59d71989 3339 dma_unmap_page(rx_ring->dev,
80785298 3340 buffer_info->page_dma,
6ec43fe6 3341 PAGE_SIZE / 2,
59d71989 3342 DMA_FROM_DEVICE);
6ec43fe6
AD		 3343 buffer_info->page_dma = 0;
3344 }
9d5c8243 3345 if (buffer_info->page) {
9d5c8243
AK		 3346 put_page(buffer_info->page);
3347 buffer_info->page = NULL;
bf36c1a0 3348 buffer_info->page_offset = 0;
9d5c8243
AK		 3349 }
3350 }
3351
06034649
AD		 3352 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3353 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3354
3355 /* Zero out the descriptor ring */
3356 memset(rx_ring->desc, 0, rx_ring->size);
3357
3358 rx_ring->next_to_clean = 0;
3359 rx_ring->next_to_use = 0;
9d5c8243
AK		 3360}
3361
3362/**
3363 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3364 * @adapter: board private structure
3365 **/
3366static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3367{
3368 int i;
3369
3370 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3371 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3372}
3373
3374/**
3375 * igb_set_mac - Change the Ethernet Address of the NIC
3376 * @netdev: network interface device structure
3377 * @p: pointer to an address structure
3378 *
3379 * Returns 0 on success, negative on failure
3380 **/
3381static int igb_set_mac(struct net_device *netdev, void *p)
3382{
3383 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3384 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3385 struct sockaddr *addr = p;
3386
3387 if (!is_valid_ether_addr(addr->sa_data))
3388 return -EADDRNOTAVAIL;
3389
3390 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3391 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3392
26ad9178
AD		 3393 /* set the correct pool for the new PF MAC address in entry 0 */
3394 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3395 adapter->vfs_allocated_count);
e1739522 3396
9d5c8243
AK		 3397 return 0;
3398}
3399
3400/**
68d480c4 3401 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3402 * @netdev: network interface device structure
3403 *
68d480c4
AD		 3404 * Writes multicast address list to the MTA hash table.
3405 * Returns: -ENOMEM on failure
3406 * 0 on no addresses written
3407 * X on writing X addresses to MTA
9d5c8243 3408 **/
68d480c4 3409static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3410{
3411 struct igb_adapter *adapter = netdev_priv(netdev);
3412 struct e1000_hw *hw = &adapter->hw;
22bedad3 3413 struct netdev_hw_addr *ha;
68d480c4 3414 u8 *mta_list;
9d5c8243
AK		 3415 int i;
3416
4cd24eaf 3417 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3418 /* nothing to program, so clear mc list */
3419 igb_update_mc_addr_list(hw, NULL, 0);
3420 igb_restore_vf_multicasts(adapter);
3421 return 0;
3422 }
9d5c8243 3423
4cd24eaf 3424 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3425 if (!mta_list)
3426 return -ENOMEM;
ff41f8dc 3427
68d480c4 3428 /* The shared function expects a packed array of only addresses. */
48e2f183 3429 i = 0;
22bedad3
JP		 3430 netdev_for_each_mc_addr(ha, netdev)
3431 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3432
68d480c4
AD		 3433 igb_update_mc_addr_list(hw, mta_list, i);
3434 kfree(mta_list);
3435
4cd24eaf 3436 return netdev_mc_count(netdev);
68d480c4
AD		 3437}
3438
3439/**
3440 * igb_write_uc_addr_list - write unicast addresses to RAR table
3441 * @netdev: network interface device structure
3442 *
3443 * Writes unicast address list to the RAR table.
3444 * Returns: -ENOMEM on failure/insufficient address space
3445 * 0 on no addresses written
3446 * X on writing X addresses to the RAR table
3447 **/
3448static int igb_write_uc_addr_list(struct net_device *netdev)
3449{
3450 struct igb_adapter *adapter = netdev_priv(netdev);
3451 struct e1000_hw *hw = &adapter->hw;
3452 unsigned int vfn = adapter->vfs_allocated_count;
3453 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3454 int count = 0;
3455
3456 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3457 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3458 return -ENOMEM;
9d5c8243 3459
32e7bfc4 3460 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3461 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3462
3463 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3464 if (!rar_entries)
3465 break;
26ad9178
AD		 3466 igb_rar_set_qsel(adapter, ha->addr,
3467 rar_entries--,
68d480c4
AD		 3468 vfn);
3469 count++;
ff41f8dc
AD		 3470 }
3471 }
3472 /* write the addresses in reverse order to avoid write combining */
3473 for (; rar_entries > 0 ; rar_entries--) {
3474 wr32(E1000_RAH(rar_entries), 0);
3475 wr32(E1000_RAL(rar_entries), 0);
3476 }
3477 wrfl();
3478
68d480c4
AD		 3479 return count;
3480}
3481
3482/**
3483 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3484 * @netdev: network interface device structure
3485 *
3486 * The set_rx_mode entry point is called whenever the unicast or multicast
3487 * address lists or the network interface flags are updated. This routine is
3488 * responsible for configuring the hardware for proper unicast, multicast,
3489 * promiscuous mode, and all-multi behavior.
3490 **/
3491static void igb_set_rx_mode(struct net_device *netdev)
3492{
3493 struct igb_adapter *adapter = netdev_priv(netdev);
3494 struct e1000_hw *hw = &adapter->hw;
3495 unsigned int vfn = adapter->vfs_allocated_count;
3496 u32 rctl, vmolr = 0;
3497 int count;
3498
3499 /* Check for Promiscuous and All Multicast modes */
3500 rctl = rd32(E1000_RCTL);
3501
3502 /* clear the effected bits */
3503 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3504
3505 if (netdev->flags & IFF_PROMISC) {
3506 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3507 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3508 } else {
3509 if (netdev->flags & IFF_ALLMULTI) {
3510 rctl |= E1000_RCTL_MPE;
3511 vmolr |= E1000_VMOLR_MPME;
3512 } else {
3513 /*
3514 * Write addresses to the MTA, if the attempt fails
25985edc 3515 * then we should just turn on promiscuous mode so
68d480c4
AD		 3516 * that we can at least receive multicast traffic
3517 */
3518 count = igb_write_mc_addr_list(netdev);
3519 if (count < 0) {
3520 rctl |= E1000_RCTL_MPE;
3521 vmolr |= E1000_VMOLR_MPME;
3522 } else if (count) {
3523 vmolr |= E1000_VMOLR_ROMPE;
3524 }
3525 }
3526 /*
3527 * Write addresses to available RAR registers, if there is not
3528 * sufficient space to store all the addresses then enable
25985edc 3529 * unicast promiscuous mode
68d480c4
AD		 3530 */
3531 count = igb_write_uc_addr_list(netdev);
3532 if (count < 0) {
3533 rctl |= E1000_RCTL_UPE;
3534 vmolr |= E1000_VMOLR_ROPE;
3535 }
3536 rctl |= E1000_RCTL_VFE;
28fc06f5 3537 }
68d480c4 3538 wr32(E1000_RCTL, rctl);
28fc06f5 3539
68d480c4
AD		 3540 /*
3541 * In order to support SR-IOV and eventually VMDq it is necessary to set
3542 * the VMOLR to enable the appropriate modes. Without this workaround
3543 * we will have issues with VLAN tag stripping not being done for frames
3544 * that are only arriving because we are the default pool
3545 */
3546 if (hw->mac.type < e1000_82576)
28fc06f5 3547 return;
9d5c8243 3548
68d480c4
AD		 3549 vmolr |= rd32(E1000_VMOLR(vfn)) &
3550 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3551 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3552 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3553}
3554
13800469
GR		 3555static void igb_check_wvbr(struct igb_adapter *adapter)
3556{
3557 struct e1000_hw *hw = &adapter->hw;
3558 u32 wvbr = 0;
3559
3560 switch (hw->mac.type) {
3561 case e1000_82576:
3562 case e1000_i350:
3563 if (!(wvbr = rd32(E1000_WVBR)))
3564 return;
3565 break;
3566 default:
3567 break;
3568 }
3569
3570 adapter->wvbr |= wvbr;
3571}
3572
3573#define IGB_STAGGERED_QUEUE_OFFSET 8
3574
3575static void igb_spoof_check(struct igb_adapter *adapter)
3576{
3577 int j;
3578
3579 if (!adapter->wvbr)
3580 return;
3581
3582 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3583 if (adapter->wvbr & (1 << j) ||
3584 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3585 dev_warn(&adapter->pdev->dev,
3586 "Spoof event(s) detected on VF %d\n", j);
3587 adapter->wvbr &=
3588 ~((1 << j) |
3589 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3590 }
3591 }
3592}
3593
9d5c8243
AK		 3594/* Need to wait a few seconds after link up to get diagnostic information from
3595 * the phy */
3596static void igb_update_phy_info(unsigned long data)
3597{
3598 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3599 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3600}
3601
4d6b725e
AD		 3602/**
3603 * igb_has_link - check shared code for link and determine up/down
3604 * @adapter: pointer to driver private info
3605 **/
3145535a 3606bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3607{
3608 struct e1000_hw *hw = &adapter->hw;
3609 bool link_active = false;
3610 s32 ret_val = 0;
3611
3612 /* get_link_status is set on LSC (link status) interrupt or
3613 * rx sequence error interrupt. get_link_status will stay
3614 * false until the e1000_check_for_link establishes link
3615 * for copper adapters ONLY
3616 */
3617 switch (hw->phy.media_type) {
3618 case e1000_media_type_copper:
3619 if (hw->mac.get_link_status) {
3620 ret_val = hw->mac.ops.check_for_link(hw);
3621 link_active = !hw->mac.get_link_status;
3622 } else {
3623 link_active = true;
3624 }
3625 break;
4d6b725e
AD		 3626 case e1000_media_type_internal_serdes:
3627 ret_val = hw->mac.ops.check_for_link(hw);
3628 link_active = hw->mac.serdes_has_link;
3629 break;
3630 default:
3631 case e1000_media_type_unknown:
3632 break;
3633 }
3634
3635 return link_active;
3636}
3637
563988dc
SA		 3638static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3639{
3640 bool ret = false;
3641 u32 ctrl_ext, thstat;
3642
3643 /* check for thermal sensor event on i350, copper only */
3644 if (hw->mac.type == e1000_i350) {
3645 thstat = rd32(E1000_THSTAT);
3646 ctrl_ext = rd32(E1000_CTRL_EXT);
3647
3648 if ((hw->phy.media_type == e1000_media_type_copper) &&
3649 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3650 ret = !!(thstat & event);
3651 }
3652 }
3653
3654 return ret;
3655}
3656
9d5c8243
AK		 3657/**
3658 * igb_watchdog - Timer Call-back
3659 * @data: pointer to adapter cast into an unsigned long
3660 **/
3661static void igb_watchdog(unsigned long data)
3662{
3663 struct igb_adapter *adapter = (struct igb_adapter *)data;
3664 /* Do the rest outside of interrupt context */
3665 schedule_work(&adapter->watchdog_task);
3666}
3667
3668static void igb_watchdog_task(struct work_struct *work)
3669{
3670 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3671 struct igb_adapter,
3672 watchdog_task);
9d5c8243 3673 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3674 struct net_device *netdev = adapter->netdev;
563988dc 3675 u32 link;
7a6ea550 3676 int i;
9d5c8243 3677
4d6b725e 3678 link = igb_has_link(adapter);
9d5c8243
AK		 3679 if (link) {
3680 if (!netif_carrier_ok(netdev)) {
3681 u32 ctrl;
330a6d6a
AD		 3682 hw->mac.ops.get_speed_and_duplex(hw,
3683 &adapter->link_speed,
3684 &adapter->link_duplex);
9d5c8243
AK		 3685
3686 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3687 /* Links status message must follow this format */
3688 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3689 "Flow Control: %s\n",
559e9c49
AD		 3690 netdev->name,
3691 adapter->link_speed,
3692 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3693 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3694 ((ctrl & E1000_CTRL_TFCE) &&
3695 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3696 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3697 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3698
563988dc
SA		 3699 /* check for thermal sensor event */
3700 if (igb_thermal_sensor_event(hw, E1000_THSTAT_LINK_THROTTLE)) {
3701 printk(KERN_INFO "igb: %s The network adapter "
3702 "link speed was downshifted "
3703 "because it overheated.\n",
3704 netdev->name);
7ef5ed1c 3705 }
563988dc 3706
d07f3e37 3707 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3708 adapter->tx_timeout_factor = 1;
3709 switch (adapter->link_speed) {
3710 case SPEED_10:
9d5c8243
AK		 3711 adapter->tx_timeout_factor = 14;
3712 break;
3713 case SPEED_100:
9d5c8243
AK		 3714 /* maybe add some timeout factor ? */
3715 break;
3716 }
3717
3718 netif_carrier_on(netdev);
9d5c8243 3719
4ae196df 3720 igb_ping_all_vfs(adapter);
17dc566c 3721 igb_check_vf_rate_limit(adapter);
4ae196df 3722
4b1a9877 3723 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3724 if (!test_bit(__IGB_DOWN, &adapter->state))
3725 mod_timer(&adapter->phy_info_timer,
3726 round_jiffies(jiffies + 2 * HZ));
3727 }
3728 } else {
3729 if (netif_carrier_ok(netdev)) {
3730 adapter->link_speed = 0;
3731 adapter->link_duplex = 0;
563988dc
SA		 3732
3733 /* check for thermal sensor event */
3734 if (igb_thermal_sensor_event(hw, E1000_THSTAT_PWR_DOWN)) {
3735 printk(KERN_ERR "igb: %s The network adapter "
3736 "was stopped because it "
3737 "overheated.\n",
7ef5ed1c 3738 netdev->name);
7ef5ed1c 3739 }
563988dc 3740
527d47c1
AD		 3741 /* Links status message must follow this format */
3742 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3743 netdev->name);
9d5c8243 3744 netif_carrier_off(netdev);
4b1a9877 3745
4ae196df
AD		 3746 igb_ping_all_vfs(adapter);
3747
4b1a9877 3748 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3749 if (!test_bit(__IGB_DOWN, &adapter->state))
3750 mod_timer(&adapter->phy_info_timer,
3751 round_jiffies(jiffies + 2 * HZ));
3752 }
3753 }
3754
12dcd86b
ED		 3755 spin_lock(&adapter->stats64_lock);
3756 igb_update_stats(adapter, &adapter->stats64);
3757 spin_unlock(&adapter->stats64_lock);
9d5c8243 3758
dbabb065 3759 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3760 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3761 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3762 /* We've lost link, so the controller stops DMA,
3763 * but we've got queued Tx work that's never going
3764 * to get done, so reset controller to flush Tx.
3765 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3766 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3767 adapter->tx_timeout_count++;
3768 schedule_work(&adapter->reset_task);
3769 /* return immediately since reset is imminent */
3770 return;
3771 }
9d5c8243 3772 }
9d5c8243 3773
dbabb065 3774 /* Force detection of hung controller every watchdog period */
6d095fa8 3775 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
dbabb065 3776 }
f7ba205e 3777
9d5c8243 3778 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3779 if (adapter->msix_entries) {
047e0030 3780 u32 eics = 0;
0d1ae7f4
AD		 3781 for (i = 0; i < adapter->num_q_vectors; i++)
3782 eics |= adapter->q_vector[i]->eims_value;
7a6ea550
AD		 3783 wr32(E1000_EICS, eics);
3784 } else {
3785 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3786 }
9d5c8243 3787
13800469
GR		 3788 igb_spoof_check(adapter);
3789
9d5c8243
AK		 3790 /* Reset the timer */
3791 if (!test_bit(__IGB_DOWN, &adapter->state))
3792 mod_timer(&adapter->watchdog_timer,
3793 round_jiffies(jiffies + 2 * HZ));
3794}
3795
3796enum latency_range {
3797 lowest_latency = 0,
3798 low_latency = 1,
3799 bulk_latency = 2,
3800 latency_invalid = 255
3801};
3802
6eb5a7f1
AD		 3803/**
3804 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3805 *
3806 * Stores a new ITR value based on strictly on packet size. This
3807 * algorithm is less sophisticated than that used in igb_update_itr,
3808 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3809 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3810 * were determined based on theoretical maximum wire speed and testing
3811 * data, in order to minimize response time while increasing bulk
3812 * throughput.
3813 * This functionality is controlled by the InterruptThrottleRate module
3814 * parameter (see igb_param.c)
3815 * NOTE: This function is called only when operating in a multiqueue
3816 * receive environment.
047e0030 3817 * @q_vector: pointer to q_vector
6eb5a7f1 3818 **/
047e0030 3819static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3820{
047e0030 3821 int new_val = q_vector->itr_val;
6eb5a7f1 3822 int avg_wire_size = 0;
047e0030 3823 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b 3824 unsigned int packets;
9d5c8243 3825
6eb5a7f1
AD		 3826 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3827 * ints/sec - ITR timer value of 120 ticks.
3828 */
3829 if (adapter->link_speed != SPEED_1000) {
0ba82994 3830 new_val = IGB_4K_ITR;
6eb5a7f1 3831 goto set_itr_val;
9d5c8243 3832 }
047e0030 3833
0ba82994
AD		 3834 packets = q_vector->rx.total_packets;
3835 if (packets)
3836 avg_wire_size = q_vector->rx.total_bytes / packets;
047e0030 3837
0ba82994
AD		 3838 packets = q_vector->tx.total_packets;
3839 if (packets)
3840 avg_wire_size = max_t(u32, avg_wire_size,
3841 q_vector->tx.total_bytes / packets);
047e0030
AD		 3842
3843 /* if avg_wire_size isn't set no work was done */
3844 if (!avg_wire_size)
3845 goto clear_counts;
9d5c8243 3846
6eb5a7f1
AD		 3847 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3848 avg_wire_size += 24;
3849
3850 /* Don't starve jumbo frames */
3851 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3852
6eb5a7f1
AD		 3853 /* Give a little boost to mid-size frames */
3854 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3855 new_val = avg_wire_size / 3;
3856 else
3857 new_val = avg_wire_size / 2;
9d5c8243 3858
0ba82994
AD		 3859 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3860 if (new_val < IGB_20K_ITR &&
3861 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3862 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3863 new_val = IGB_20K_ITR;
abe1c363 3864
6eb5a7f1 3865set_itr_val:
047e0030
AD		 3866 if (new_val != q_vector->itr_val) {
3867 q_vector->itr_val = new_val;
3868 q_vector->set_itr = 1;
9d5c8243 3869 }
6eb5a7f1 3870clear_counts:
0ba82994
AD		 3871 q_vector->rx.total_bytes = 0;
3872 q_vector->rx.total_packets = 0;
3873 q_vector->tx.total_bytes = 0;
3874 q_vector->tx.total_packets = 0;
9d5c8243
AK		 3875}
3876
3877/**
3878 * igb_update_itr - update the dynamic ITR value based on statistics
3879 * Stores a new ITR value based on packets and byte
3880 * counts during the last interrupt. The advantage of per interrupt
3881 * computation is faster updates and more accurate ITR for the current
3882 * traffic pattern. Constants in this function were computed
3883 * based on theoretical maximum wire speed and thresholds were set based
3884 * on testing data as well as attempting to minimize response time
3885 * while increasing bulk throughput.
3886 * this functionality is controlled by the InterruptThrottleRate module
3887 * parameter (see igb_param.c)
3888 * NOTE: These calculations are only valid when operating in a single-
3889 * queue environment.
0ba82994
AD		 3890 * @q_vector: pointer to q_vector
3891 * @ring_container: ring info to update the itr for
9d5c8243 3892 **/
0ba82994
AD		 3893static void igb_update_itr(struct igb_q_vector *q_vector,
3894 struct igb_ring_container *ring_container)
9d5c8243 3895{
0ba82994
AD		 3896 unsigned int packets = ring_container->total_packets;
3897 unsigned int bytes = ring_container->total_bytes;
3898 u8 itrval = ring_container->itr;
9d5c8243 3899
0ba82994 3900 /* no packets, exit with status unchanged */
9d5c8243 3901 if (packets == 0)
0ba82994 3902 return;
9d5c8243 3903
0ba82994 3904 switch (itrval) {
9d5c8243
AK		 3905 case lowest_latency:
3906 /* handle TSO and jumbo frames */
3907 if (bytes/packets > 8000)
0ba82994 3908 itrval = bulk_latency;
9d5c8243 3909 else if ((packets < 5) && (bytes > 512))
0ba82994 3910 itrval = low_latency;
9d5c8243
AK		 3911 break;
3912 case low_latency: /* 50 usec aka 20000 ints/s */
3913 if (bytes > 10000) {
3914 /* this if handles the TSO accounting */
3915 if (bytes/packets > 8000) {
0ba82994 3916 itrval = bulk_latency;
9d5c8243 3917 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
0ba82994 3918 itrval = bulk_latency;
9d5c8243 3919 } else if ((packets > 35)) {
0ba82994 3920 itrval = lowest_latency;
9d5c8243
AK		 3921 }
3922 } else if (bytes/packets > 2000) {
0ba82994 3923 itrval = bulk_latency;
9d5c8243 3924 } else if (packets <= 2 && bytes < 512) {
0ba82994 3925 itrval = lowest_latency;
9d5c8243
AK		 3926 }
3927 break;
3928 case bulk_latency: /* 250 usec aka 4000 ints/s */
3929 if (bytes > 25000) {
3930 if (packets > 35)
0ba82994 3931 itrval = low_latency;
1e5c3d21 3932 } else if (bytes < 1500) {
0ba82994 3933 itrval = low_latency;
9d5c8243
AK		 3934 }
3935 break;
3936 }
3937
0ba82994
AD		 3938 /* clear work counters since we have the values we need */
3939 ring_container->total_bytes = 0;
3940 ring_container->total_packets = 0;
3941
3942 /* write updated itr to ring container */
3943 ring_container->itr = itrval;
9d5c8243
AK		 3944}
3945
0ba82994 3946static void igb_set_itr(struct igb_q_vector *q_vector)
9d5c8243 3947{
0ba82994 3948 struct igb_adapter *adapter = q_vector->adapter;
047e0030 3949 u32 new_itr = q_vector->itr_val;
0ba82994 3950 u8 current_itr = 0;
9d5c8243
AK		 3951
3952 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3953 if (adapter->link_speed != SPEED_1000) {
3954 current_itr = 0;
0ba82994 3955 new_itr = IGB_4K_ITR;
9d5c8243
AK		 3956 goto set_itr_now;
3957 }
3958
0ba82994
AD		 3959 igb_update_itr(q_vector, &q_vector->tx);
3960 igb_update_itr(q_vector, &q_vector->rx);
9d5c8243 3961
0ba82994 3962 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
9d5c8243 3963
6eb5a7f1 3964 /* conservative mode (itr 3) eliminates the lowest_latency setting */
0ba82994
AD		 3965 if (current_itr == lowest_latency &&
3966 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3967 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
6eb5a7f1
AD		 3968 current_itr = low_latency;
3969
9d5c8243
AK		 3970 switch (current_itr) {
3971 /* counts and packets in update_itr are dependent on these numbers */
3972 case lowest_latency:
0ba82994 3973 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
9d5c8243
AK		 3974 break;
3975 case low_latency:
0ba82994 3976 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
9d5c8243
AK		 3977 break;
3978 case bulk_latency:
0ba82994 3979 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
9d5c8243
AK		 3980 break;
3981 default:
3982 break;
3983 }
3984
3985set_itr_now:
047e0030 3986 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3987 /* this attempts to bias the interrupt rate towards Bulk
3988 * by adding intermediate steps when interrupt rate is
3989 * increasing */
047e0030
AD		 3990 new_itr = new_itr > q_vector->itr_val ?
3991 max((new_itr * q_vector->itr_val) /
3992 (new_itr + (q_vector->itr_val >> 2)),
0ba82994 3993 new_itr) :
9d5c8243
AK		 3994 new_itr;
3995 /* Don't write the value here; it resets the adapter's
3996 * internal timer, and causes us to delay far longer than
3997 * we should between interrupts. Instead, we write the ITR
3998 * value at the beginning of the next interrupt so the timing
3999 * ends up being correct.
4000 */
047e0030
AD		 4001 q_vector->itr_val = new_itr;
4002 q_vector->set_itr = 1;
9d5c8243 4003 }
9d5c8243
AK		 4004}
4005
7d13a7d0
AD		 4006void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4007 u32 type_tucmd, u32 mss_l4len_idx)
4008{
4009 struct e1000_adv_tx_context_desc *context_desc;
4010 u16 i = tx_ring->next_to_use;
4011
4012 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4013
4014 i++;
4015 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4016
4017 /* set bits to identify this as an advanced context descriptor */
4018 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4019
4020 /* For 82575, context index must be unique per ring. */
866cff06 4021 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
7d13a7d0
AD		 4022 mss_l4len_idx |= tx_ring->reg_idx << 4;
4023
4024 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4025 context_desc->seqnum_seed = 0;
4026 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4027 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4028}
4029
7af40ad9
AD		 4030static int igb_tso(struct igb_ring *tx_ring,
4031 struct igb_tx_buffer *first,
4032 u8 *hdr_len)
9d5c8243 4033{
7af40ad9 4034 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4035 u32 vlan_macip_lens, type_tucmd;
4036 u32 mss_l4len_idx, l4len;
4037
4038 if (!skb_is_gso(skb))
4039 return 0;
9d5c8243
AK		 4040
4041 if (skb_header_cloned(skb)) {
7af40ad9 4042 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4043 if (err)
4044 return err;
4045 }
4046
7d13a7d0
AD		 4047 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4048 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4049
7af40ad9 4050 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4051 struct iphdr *iph = ip_hdr(skb);
4052 iph->tot_len = 0;
4053 iph->check = 0;
4054 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4055 iph->daddr, 0,
4056 IPPROTO_TCP,
4057 0);
7d13a7d0 4058 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4059 first->tx_flags |= IGB_TX_FLAGS_TSO |
4060 IGB_TX_FLAGS_CSUM |
4061 IGB_TX_FLAGS_IPV4;
8e1e8a47 4062 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4063 ipv6_hdr(skb)->payload_len = 0;
4064 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4065 &ipv6_hdr(skb)->daddr,
4066 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4067 first->tx_flags |= IGB_TX_FLAGS_TSO |
4068 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4069 }
4070
7af40ad9 4071 /* compute header lengths */
7d13a7d0
AD		 4072 l4len = tcp_hdrlen(skb);
4073 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4074
7af40ad9
AD		 4075 /* update gso size and bytecount with header size */
4076 first->gso_segs = skb_shinfo(skb)->gso_segs;
4077 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4078
9d5c8243 4079 /* MSS L4LEN IDX */
7d13a7d0
AD		 4080 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4081 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4082
7d13a7d0
AD		 4083 /* VLAN MACLEN IPLEN */
4084 vlan_macip_lens = skb_network_header_len(skb);
4085 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4086 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4087
7d13a7d0 4088 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4089
7d13a7d0 4090 return 1;
9d5c8243
AK		 4091}
4092
7af40ad9 4093static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4094{
7af40ad9 4095 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4096 u32 vlan_macip_lens = 0;
4097 u32 mss_l4len_idx = 0;
4098 u32 type_tucmd = 0;
9d5c8243 4099
7d13a7d0 4100 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4101 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4102 return;
7d13a7d0
AD		 4103 } else {
4104 u8 l4_hdr = 0;
7af40ad9 4105 switch (first->protocol) {
7d13a7d0
AD		 4106 case __constant_htons(ETH_P_IP):
4107 vlan_macip_lens |= skb_network_header_len(skb);
4108 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4109 l4_hdr = ip_hdr(skb)->protocol;
4110 break;
4111 case __constant_htons(ETH_P_IPV6):
4112 vlan_macip_lens |= skb_network_header_len(skb);
4113 l4_hdr = ipv6_hdr(skb)->nexthdr;
4114 break;
4115 default:
4116 if (unlikely(net_ratelimit())) {
4117 dev_warn(tx_ring->dev,
4118 "partial checksum but proto=%x!\n",
7af40ad9 4119 first->protocol);
fa4a7ef3 4120 }
7d13a7d0
AD		 4121 break;
4122 }
fa4a7ef3 4123
7d13a7d0
AD		 4124 switch (l4_hdr) {
4125 case IPPROTO_TCP:
4126 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4127 mss_l4len_idx = tcp_hdrlen(skb) <<
4128 E1000_ADVTXD_L4LEN_SHIFT;
4129 break;
4130 case IPPROTO_SCTP:
4131 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4132 mss_l4len_idx = sizeof(struct sctphdr) <<
4133 E1000_ADVTXD_L4LEN_SHIFT;
4134 break;
4135 case IPPROTO_UDP:
4136 mss_l4len_idx = sizeof(struct udphdr) <<
4137 E1000_ADVTXD_L4LEN_SHIFT;
4138 break;
4139 default:
4140 if (unlikely(net_ratelimit())) {
4141 dev_warn(tx_ring->dev,
4142 "partial checksum but l4 proto=%x!\n",
4143 l4_hdr);
44b0cda3 4144 }
7d13a7d0 4145 break;
9d5c8243 4146 }
7af40ad9
AD		 4147
4148 /* update TX checksum flag */
4149 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4150 }
9d5c8243 4151
7d13a7d0 4152 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4153 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4154
7d13a7d0 4155 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4156}
4157
e032afc8
AD		 4158static __le32 igb_tx_cmd_type(u32 tx_flags)
4159{
4160 /* set type for advanced descriptor with frame checksum insertion */
4161 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4162 E1000_ADVTXD_DCMD_IFCS |
4163 E1000_ADVTXD_DCMD_DEXT);
4164
4165 /* set HW vlan bit if vlan is present */
4166 if (tx_flags & IGB_TX_FLAGS_VLAN)
4167 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4168
4169 /* set timestamp bit if present */
4170 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4171 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4172
4173 /* set segmentation bits for TSO */
4174 if (tx_flags & IGB_TX_FLAGS_TSO)
4175 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4176
4177 return cmd_type;
4178}
4179
7af40ad9
AD		 4180static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4181 union e1000_adv_tx_desc *tx_desc,
4182 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4183{
4184 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4185
4186 /* 82575 requires a unique index per ring if any offload is enabled */
4187 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
866cff06 4188 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
e032afc8
AD		 4189 olinfo_status |= tx_ring->reg_idx << 4;
4190
4191 /* insert L4 checksum */
4192 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4193 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4194
4195 /* insert IPv4 checksum */
4196 if (tx_flags & IGB_TX_FLAGS_IPV4)
4197 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4198 }
4199
7af40ad9 4200 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4201}
4202
ebe42d16
AD		 4203/*
4204 * The largest size we can write to the descriptor is 65535. In order to
4205 * maintain a power of two alignment we have to limit ourselves to 32K.
4206 */
4207#define IGB_MAX_TXD_PWR 15
7af40ad9 4208#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4209
7af40ad9
AD		 4210static void igb_tx_map(struct igb_ring *tx_ring,
4211 struct igb_tx_buffer *first,
ebe42d16 4212 const u8 hdr_len)
9d5c8243 4213{
7af40ad9 4214 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4215 struct igb_tx_buffer *tx_buffer_info;
4216 union e1000_adv_tx_desc *tx_desc;
4217 dma_addr_t dma;
4218 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4219 unsigned int data_len = skb->data_len;
4220 unsigned int size = skb_headlen(skb);
4221 unsigned int paylen = skb->len - hdr_len;
4222 __le32 cmd_type;
7af40ad9 4223 u32 tx_flags = first->tx_flags;
ebe42d16 4224 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4225
4226 tx_desc = IGB_TX_DESC(tx_ring, i);
4227
7af40ad9 4228 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4229 cmd_type = igb_tx_cmd_type(tx_flags);
4230
4231 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4232 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4233 goto dma_error;
9d5c8243 4234
ebe42d16
AD		 4235 /* record length, and DMA address */
4236 first->length = size;
4237 first->dma = dma;
ebe42d16
AD		 4238 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4239
4240 for (;;) {
4241 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4242 tx_desc->read.cmd_type_len =
4243 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4244
4245 i++;
4246 tx_desc++;
4247 if (i == tx_ring->count) {
4248 tx_desc = IGB_TX_DESC(tx_ring, 0);
4249 i = 0;
4250 }
4251
4252 dma += IGB_MAX_DATA_PER_TXD;
4253 size -= IGB_MAX_DATA_PER_TXD;
4254
4255 tx_desc->read.olinfo_status = 0;
4256 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4257 }
4258
4259 if (likely(!data_len))
4260 break;
2bbfebe2 4261
ebe42d16 4262 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4263
65689fef 4264 i++;
ebe42d16
AD		 4265 tx_desc++;
4266 if (i == tx_ring->count) {
4267 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4268 i = 0;
ebe42d16 4269 }
65689fef 4270
ebe42d16
AD		 4271 size = frag->size;
4272 data_len -= size;
4273
4274 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4275 size, DMA_TO_DEVICE);
4276 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4277 goto dma_error;
4278
ebe42d16
AD		 4279 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4280 tx_buffer_info->length = size;
4281 tx_buffer_info->dma = dma;
4282
4283 tx_desc->read.olinfo_status = 0;
4284 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4285
4286 frag++;
9d5c8243
AK		 4287 }
4288
ebe42d16
AD		 4289 /* write last descriptor with RS and EOP bits */
4290 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4291 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4292
4293 /* set the timestamp */
4294 first->time_stamp = jiffies;
4295
ebe42d16
AD		 4296 /*
4297 * Force memory writes to complete before letting h/w know there
4298 * are new descriptors to fetch. (Only applicable for weak-ordered
4299 * memory model archs, such as IA-64).
4300 *
4301 * We also need this memory barrier to make certain all of the
4302 * status bits have been updated before next_to_watch is written.
4303 */
4304 wmb();
4305
8542db05 4306 /* set next_to_watch value indicating a packet is present */
ebe42d16 4307 first->next_to_watch = tx_desc;
9d5c8243 4308
ebe42d16
AD		 4309 i++;
4310 if (i == tx_ring->count)
4311 i = 0;
6366ad33 4312
ebe42d16 4313 tx_ring->next_to_use = i;
6366ad33 4314
ebe42d16 4315 writel(i, tx_ring->tail);
6366ad33 4316
ebe42d16
AD		 4317 /* we need this if more than one processor can write to our tail
4318 * at a time, it syncronizes IO on IA64/Altix systems */
4319 mmiowb();
4320
4321 return;
4322
4323dma_error:
4324 dev_err(tx_ring->dev, "TX DMA map failed\n");
4325
4326 /* clear dma mappings for failed tx_buffer_info map */
4327 for (;;) {
4328 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4329 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4330 if (tx_buffer_info == first)
4331 break;
a77ff709
NN		 4332 if (i == 0)
4333 i = tx_ring->count;
6366ad33 4334 i--;
6366ad33
AD		 4335 }
4336
9d5c8243 4337 tx_ring->next_to_use = i;
9d5c8243
AK		 4338}
4339
6ad4edfc 4340static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4341{
e694e964
AD		 4342 struct net_device *netdev = tx_ring->netdev;
4343
661086df 4344 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4345
9d5c8243
AK		 4346 /* Herbert's original patch had:
4347 * smp_mb__after_netif_stop_queue();
4348 * but since that doesn't exist yet, just open code it. */
4349 smp_mb();
4350
4351 /* We need to check again in a case another CPU has just
4352 * made room available. */
c493ea45 4353 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4354 return -EBUSY;
4355
4356 /* A reprieve! */
661086df 4357 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4358
4359 u64_stats_update_begin(&tx_ring->tx_syncp2);
4360 tx_ring->tx_stats.restart_queue2++;
4361 u64_stats_update_end(&tx_ring->tx_syncp2);
4362
9d5c8243
AK		 4363 return 0;
4364}
4365
6ad4edfc 4366static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4367{
c493ea45 4368 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4369 return 0;
e694e964 4370 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4371}
4372
cd392f5c
AD		 4373netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4374 struct igb_ring *tx_ring)
9d5c8243 4375{
8542db05 4376 struct igb_tx_buffer *first;
ebe42d16 4377 int tso;
91d4ee33 4378 u32 tx_flags = 0;
31f6adbb 4379 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4380 u8 hdr_len = 0;
9d5c8243 4381
9d5c8243
AK		 4382 /* need: 1 descriptor per page,
4383 * + 2 desc gap to keep tail from touching head,
4384 * + 1 desc for skb->data,
4385 * + 1 desc for context descriptor,
4386 * otherwise try next time */
e694e964 4387 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4388 /* this is a hard error */
9d5c8243
AK		 4389 return NETDEV_TX_BUSY;
4390 }
33af6bcc 4391
7af40ad9
AD		 4392 /* record the location of the first descriptor for this packet */
4393 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4394 first->skb = skb;
4395 first->bytecount = skb->len;
4396 first->gso_segs = 1;
4397
2244d07b
OH		 4398 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4399 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4400 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4401 }
9d5c8243 4402
eab6d18d 4403 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4404 tx_flags |= IGB_TX_FLAGS_VLAN;
4405 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4406 }
4407
7af40ad9
AD		 4408 /* record initial flags and protocol */
4409 first->tx_flags = tx_flags;
4410 first->protocol = protocol;
cdfd01fc 4411
7af40ad9
AD		 4412 tso = igb_tso(tx_ring, first, &hdr_len);
4413 if (tso < 0)
7d13a7d0 4414 goto out_drop;
7af40ad9
AD		 4415 else if (!tso)
4416 igb_tx_csum(tx_ring, first);
9d5c8243 4417
7af40ad9 4418 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4419
4420 /* Make sure there is space in the ring for the next send. */
e694e964 4421 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4422
9d5c8243 4423 return NETDEV_TX_OK;
7d13a7d0
AD		 4424
4425out_drop:
7af40ad9
AD		 4426 igb_unmap_and_free_tx_resource(tx_ring, first);
4427
7d13a7d0 4428 return NETDEV_TX_OK;
9d5c8243
AK		 4429}
4430
1cc3bd87
AD		 4431static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4432 struct sk_buff *skb)
4433{
4434 unsigned int r_idx = skb->queue_mapping;
4435
4436 if (r_idx >= adapter->num_tx_queues)
4437 r_idx = r_idx % adapter->num_tx_queues;
4438
4439 return adapter->tx_ring[r_idx];
4440}
4441
cd392f5c
AD		 4442static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4443 struct net_device *netdev)
9d5c8243
AK		 4444{
4445 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4446
4447 if (test_bit(__IGB_DOWN, &adapter->state)) {
4448 dev_kfree_skb_any(skb);
4449 return NETDEV_TX_OK;
4450 }
4451
4452 if (skb->len <= 0) {
4453 dev_kfree_skb_any(skb);
4454 return NETDEV_TX_OK;
4455 }
4456
1cc3bd87
AD		 4457 /*
4458 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4459 * in order to meet this minimum size requirement.
4460 */
4461 if (skb->len < 17) {
4462 if (skb_padto(skb, 17))
4463 return NETDEV_TX_OK;
4464 skb->len = 17;
4465 }
9d5c8243 4466
1cc3bd87 4467 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4468}
4469
4470/**
4471 * igb_tx_timeout - Respond to a Tx Hang
4472 * @netdev: network interface device structure
4473 **/
4474static void igb_tx_timeout(struct net_device *netdev)
4475{
4476 struct igb_adapter *adapter = netdev_priv(netdev);
4477 struct e1000_hw *hw = &adapter->hw;
4478
4479 /* Do the reset outside of interrupt context */
4480 adapter->tx_timeout_count++;
f7ba205e 4481
06218a8d 4482 if (hw->mac.type >= e1000_82580)
55cac248
AD		 4483 hw->dev_spec._82575.global_device_reset = true;
4484
9d5c8243 4485 schedule_work(&adapter->reset_task);
265de409
AD		 4486 wr32(E1000_EICS,
4487 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4488}
4489
4490static void igb_reset_task(struct work_struct *work)
4491{
4492 struct igb_adapter *adapter;
4493 adapter = container_of(work, struct igb_adapter, reset_task);
4494
c97ec42a
TI		 4495 igb_dump(adapter);
4496 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4497 igb_reinit_locked(adapter);
4498}
4499
4500/**
12dcd86b 4501 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4502 * @netdev: network interface device structure
12dcd86b 4503 * @stats: rtnl_link_stats64 pointer
9d5c8243 4504 *
9d5c8243 4505 **/
12dcd86b
ED		 4506static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4507 struct rtnl_link_stats64 *stats)
9d5c8243 4508{
12dcd86b
ED		 4509 struct igb_adapter *adapter = netdev_priv(netdev);
4510
4511 spin_lock(&adapter->stats64_lock);
4512 igb_update_stats(adapter, &adapter->stats64);
4513 memcpy(stats, &adapter->stats64, sizeof(*stats));
4514 spin_unlock(&adapter->stats64_lock);
4515
4516 return stats;
9d5c8243
AK		 4517}
4518
4519/**
4520 * igb_change_mtu - Change the Maximum Transfer Unit
4521 * @netdev: network interface device structure
4522 * @new_mtu: new value for maximum frame size
4523 *
4524 * Returns 0 on success, negative on failure
4525 **/
4526static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4527{
4528 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4529 struct pci_dev *pdev = adapter->pdev;
153285f9 4530 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4531
c809d227 4532 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4533 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4534 return -EINVAL;
4535 }
4536
153285f9 4537#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4538 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4539 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4540 return -EINVAL;
4541 }
4542
4543 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4544 msleep(1);
73cd78f1 4545
9d5c8243
AK		 4546 /* igb_down has a dependency on max_frame_size */
4547 adapter->max_frame_size = max_frame;
559e9c49 4548
4c844851
AD		 4549 if (netif_running(netdev))
4550 igb_down(adapter);
9d5c8243 4551
090b1795 4552 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4553 netdev->mtu, new_mtu);
4554 netdev->mtu = new_mtu;
4555
4556 if (netif_running(netdev))
4557 igb_up(adapter);
4558 else
4559 igb_reset(adapter);
4560
4561 clear_bit(__IGB_RESETTING, &adapter->state);
4562
4563 return 0;
4564}
4565
4566/**
4567 * igb_update_stats - Update the board statistics counters
4568 * @adapter: board private structure
4569 **/
4570
12dcd86b
ED		 4571void igb_update_stats(struct igb_adapter *adapter,
4572 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4573{
4574 struct e1000_hw *hw = &adapter->hw;
4575 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4576 u32 reg, mpc;
9d5c8243 4577 u16 phy_tmp;
3f9c0164
AD		 4578 int i;
4579 u64 bytes, packets;
12dcd86b
ED		 4580 unsigned int start;
4581 u64 _bytes, _packets;
9d5c8243
AK		 4582
4583#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4584
4585 /*
4586 * Prevent stats update while adapter is being reset, or if the pci
4587 * connection is down.
4588 */
4589 if (adapter->link_speed == 0)
4590 return;
4591 if (pci_channel_offline(pdev))
4592 return;
4593
3f9c0164
AD		 4594 bytes = 0;
4595 packets = 0;
4596 for (i = 0; i < adapter->num_rx_queues; i++) {
4597 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4598 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4599
3025a446 4600 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4601 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4602
4603 do {
4604 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4605 _bytes = ring->rx_stats.bytes;
4606 _packets = ring->rx_stats.packets;
4607 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4608 bytes += _bytes;
4609 packets += _packets;
3f9c0164
AD		 4610 }
4611
128e45eb
AD		 4612 net_stats->rx_bytes = bytes;
4613 net_stats->rx_packets = packets;
3f9c0164
AD		 4614
4615 bytes = 0;
4616 packets = 0;
4617 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4618 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4619 do {
4620 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4621 _bytes = ring->tx_stats.bytes;
4622 _packets = ring->tx_stats.packets;
4623 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4624 bytes += _bytes;
4625 packets += _packets;
3f9c0164 4626 }
128e45eb
AD		 4627 net_stats->tx_bytes = bytes;
4628 net_stats->tx_packets = packets;
3f9c0164
AD		 4629
4630 /* read stats registers */
9d5c8243
AK		 4631 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4632 adapter->stats.gprc += rd32(E1000_GPRC);
4633 adapter->stats.gorc += rd32(E1000_GORCL);
4634 rd32(E1000_GORCH); /* clear GORCL */
4635 adapter->stats.bprc += rd32(E1000_BPRC);
4636 adapter->stats.mprc += rd32(E1000_MPRC);
4637 adapter->stats.roc += rd32(E1000_ROC);
4638
4639 adapter->stats.prc64 += rd32(E1000_PRC64);
4640 adapter->stats.prc127 += rd32(E1000_PRC127);
4641 adapter->stats.prc255 += rd32(E1000_PRC255);
4642 adapter->stats.prc511 += rd32(E1000_PRC511);
4643 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4644 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4645 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4646 adapter->stats.sec += rd32(E1000_SEC);
4647
fa3d9a6d
MW		 4648 mpc = rd32(E1000_MPC);
4649 adapter->stats.mpc += mpc;
4650 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4651 adapter->stats.scc += rd32(E1000_SCC);
4652 adapter->stats.ecol += rd32(E1000_ECOL);
4653 adapter->stats.mcc += rd32(E1000_MCC);
4654 adapter->stats.latecol += rd32(E1000_LATECOL);
4655 adapter->stats.dc += rd32(E1000_DC);
4656 adapter->stats.rlec += rd32(E1000_RLEC);
4657 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4658 adapter->stats.xontxc += rd32(E1000_XONTXC);
4659 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4660 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4661 adapter->stats.fcruc += rd32(E1000_FCRUC);
4662 adapter->stats.gptc += rd32(E1000_GPTC);
4663 adapter->stats.gotc += rd32(E1000_GOTCL);
4664 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4665 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4666 adapter->stats.ruc += rd32(E1000_RUC);
4667 adapter->stats.rfc += rd32(E1000_RFC);
4668 adapter->stats.rjc += rd32(E1000_RJC);
4669 adapter->stats.tor += rd32(E1000_TORH);
4670 adapter->stats.tot += rd32(E1000_TOTH);
4671 adapter->stats.tpr += rd32(E1000_TPR);
4672
4673 adapter->stats.ptc64 += rd32(E1000_PTC64);
4674 adapter->stats.ptc127 += rd32(E1000_PTC127);
4675 adapter->stats.ptc255 += rd32(E1000_PTC255);
4676 adapter->stats.ptc511 += rd32(E1000_PTC511);
4677 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4678 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4679
4680 adapter->stats.mptc += rd32(E1000_MPTC);
4681 adapter->stats.bptc += rd32(E1000_BPTC);
4682
2d0b0f69
NN		 4683 adapter->stats.tpt += rd32(E1000_TPT);
4684 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4685
4686 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4687 /* read internal phy specific stats */
4688 reg = rd32(E1000_CTRL_EXT);
4689 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4690 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4691 adapter->stats.tncrs += rd32(E1000_TNCRS);
4692 }
4693
9d5c8243
AK		 4694 adapter->stats.tsctc += rd32(E1000_TSCTC);
4695 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4696
4697 adapter->stats.iac += rd32(E1000_IAC);
4698 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4699 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4700 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4701 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4702 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4703 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4704 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4705 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4706
4707 /* Fill out the OS statistics structure */
128e45eb
AD		 4708 net_stats->multicast = adapter->stats.mprc;
4709 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4710
4711 /* Rx Errors */
4712
4713 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4714 * our own version based on RUC and ROC */
128e45eb 4715 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4716 adapter->stats.crcerrs + adapter->stats.algnerrc +
4717 adapter->stats.ruc + adapter->stats.roc +
4718 adapter->stats.cexterr;
128e45eb
AD		 4719 net_stats->rx_length_errors = adapter->stats.ruc +
4720 adapter->stats.roc;
4721 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4722 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4723 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4724
4725 /* Tx Errors */
128e45eb
AD		 4726 net_stats->tx_errors = adapter->stats.ecol +
4727 adapter->stats.latecol;
4728 net_stats->tx_aborted_errors = adapter->stats.ecol;
4729 net_stats->tx_window_errors = adapter->stats.latecol;
4730 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4731
4732 /* Tx Dropped needs to be maintained elsewhere */
4733
4734 /* Phy Stats */
4735 if (hw->phy.media_type == e1000_media_type_copper) {
4736 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4737 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4738 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4739 adapter->phy_stats.idle_errors += phy_tmp;
4740 }
4741 }
4742
4743 /* Management Stats */
4744 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4745 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4746 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4747
4748 /* OS2BMC Stats */
4749 reg = rd32(E1000_MANC);
4750 if (reg & E1000_MANC_EN_BMC2OS) {
4751 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4752 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4753 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4754 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4755 }
9d5c8243
AK		 4756}
4757
9d5c8243
AK		 4758static irqreturn_t igb_msix_other(int irq, void *data)
4759{
047e0030 4760 struct igb_adapter *adapter = data;
9d5c8243 4761 struct e1000_hw *hw = &adapter->hw;
844290e5 4762 u32 icr = rd32(E1000_ICR);
844290e5 4763 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4764
7f081d40
AD		 4765 if (icr & E1000_ICR_DRSTA)
4766 schedule_work(&adapter->reset_task);
4767
047e0030 4768 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4769 /* HW is reporting DMA is out of sync */
4770 adapter->stats.doosync++;
13800469
GR		 4771 /* The DMA Out of Sync is also indication of a spoof event
4772 * in IOV mode. Check the Wrong VM Behavior register to
4773 * see if it is really a spoof event. */
4774 igb_check_wvbr(adapter);
dda0e083 4775 }
eebbbdba 4776
4ae196df
AD		 4777 /* Check for a mailbox event */
4778 if (icr & E1000_ICR_VMMB)
4779 igb_msg_task(adapter);
4780
4781 if (icr & E1000_ICR_LSC) {
4782 hw->mac.get_link_status = 1;
4783 /* guard against interrupt when we're going down */
4784 if (!test_bit(__IGB_DOWN, &adapter->state))
4785 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4786 }
4787
844290e5 4788 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4789
4790 return IRQ_HANDLED;
4791}
4792
047e0030 4793static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4794{
26b39276 4795 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4796 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4797
047e0030
AD		 4798 if (!q_vector->set_itr)
4799 return;
73cd78f1 4800
047e0030
AD		 4801 if (!itr_val)
4802 itr_val = 0x4;
661086df 4803
26b39276
AD		 4804 if (adapter->hw.mac.type == e1000_82575)
4805 itr_val |= itr_val << 16;
661086df 4806 else
0ba82994 4807 itr_val |= E1000_EITR_CNT_IGNR;
661086df 4808
047e0030
AD		 4809 writel(itr_val, q_vector->itr_register);
4810 q_vector->set_itr = 0;
6eb5a7f1
AD		 4811}
4812
047e0030 4813static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4814{
047e0030 4815 struct igb_q_vector *q_vector = data;
9d5c8243 4816
047e0030
AD		 4817 /* Write the ITR value calculated from the previous interrupt. */
4818 igb_write_itr(q_vector);
9d5c8243 4819
047e0030 4820 napi_schedule(&q_vector->napi);
844290e5 4821
047e0030 4822 return IRQ_HANDLED;
fe4506b6
JC		 4823}
4824
421e02f0 4825#ifdef CONFIG_IGB_DCA
047e0030 4826static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4827{
047e0030 4828 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4829 struct e1000_hw *hw = &adapter->hw;
4830 int cpu = get_cpu();
fe4506b6 4831
047e0030
AD		 4832 if (q_vector->cpu == cpu)
4833 goto out_no_update;
4834
0ba82994
AD		 4835 if (q_vector->tx.ring) {
4836 int q = q_vector->tx.ring->reg_idx;
047e0030
AD		 4837 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4838 if (hw->mac.type == e1000_82575) {
4839 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4840 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4841 } else {
047e0030
AD		 4842 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4843 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4844 E1000_DCA_TXCTRL_CPUID_SHIFT;
4845 }
4846 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4847 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4848 }
0ba82994
AD		 4849 if (q_vector->rx.ring) {
4850 int q = q_vector->rx.ring->reg_idx;
047e0030
AD		 4851 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4852 if (hw->mac.type == e1000_82575) {
2d064c06 4853 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4854 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4855 } else {
4856 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4857 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4858 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4859 }
fe4506b6
JC		 4860 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4861 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4862 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4863 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4864 }
047e0030
AD		 4865 q_vector->cpu = cpu;
4866out_no_update:
fe4506b6
JC		 4867 put_cpu();
4868}
4869
4870static void igb_setup_dca(struct igb_adapter *adapter)
4871{
7e0e99ef 4872 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4873 int i;
4874
7dfc16fa 4875 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4876 return;
4877
7e0e99ef
AD		 4878 /* Always use CB2 mode, difference is masked in the CB driver. */
4879 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4880
047e0030 4881 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4882 adapter->q_vector[i]->cpu = -1;
4883 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4884 }
4885}
4886
4887static int __igb_notify_dca(struct device *dev, void *data)
4888{
4889 struct net_device *netdev = dev_get_drvdata(dev);
4890 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4891 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4892 struct e1000_hw *hw = &adapter->hw;
4893 unsigned long event = *(unsigned long *)data;
4894
4895 switch (event) {
4896 case DCA_PROVIDER_ADD:
4897 /* if already enabled, don't do it again */
7dfc16fa 4898 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4899 break;
fe4506b6 4900 if (dca_add_requester(dev) == 0) {
bbd98fe4 4901 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4902 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4903 igb_setup_dca(adapter);
4904 break;
4905 }
4906 /* Fall Through since DCA is disabled. */
4907 case DCA_PROVIDER_REMOVE:
7dfc16fa 4908 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4909 /* without this a class_device is left
047e0030 4910 * hanging around in the sysfs model */
fe4506b6 4911 dca_remove_requester(dev);
090b1795 4912 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4913 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4914 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4915 }
4916 break;
4917 }
bbd98fe4 4918
fe4506b6 4919 return 0;
9d5c8243
AK		 4920}
4921
fe4506b6
JC		 4922static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4923 void *p)
4924{
4925 int ret_val;
4926
4927 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4928 __igb_notify_dca);
4929
4930 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4931}
421e02f0 4932#endif /* CONFIG_IGB_DCA */
9d5c8243 4933
0224d663
GR		 4934#ifdef CONFIG_PCI_IOV
4935static int igb_vf_configure(struct igb_adapter *adapter, int vf)
4936{
4937 unsigned char mac_addr[ETH_ALEN];
4938 struct pci_dev *pdev = adapter->pdev;
4939 struct e1000_hw *hw = &adapter->hw;
4940 struct pci_dev *pvfdev;
4941 unsigned int device_id;
4942 u16 thisvf_devfn;
4943
4944 random_ether_addr(mac_addr);
4945 igb_set_vf_mac(adapter, vf, mac_addr);
4946
4947 switch (adapter->hw.mac.type) {
4948 case e1000_82576:
4949 device_id = IGB_82576_VF_DEV_ID;
4950 /* VF Stride for 82576 is 2 */
4951 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
4952 (pdev->devfn & 1);
4953 break;
4954 case e1000_i350:
4955 device_id = IGB_I350_VF_DEV_ID;
4956 /* VF Stride for I350 is 4 */
4957 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
4958 (pdev->devfn & 3);
4959 break;
4960 default:
4961 device_id = 0;
4962 thisvf_devfn = 0;
4963 break;
4964 }
4965
4966 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4967 while (pvfdev) {
4968 if (pvfdev->devfn == thisvf_devfn)
4969 break;
4970 pvfdev = pci_get_device(hw->vendor_id,
4971 device_id, pvfdev);
4972 }
4973
4974 if (pvfdev)
4975 adapter->vf_data[vf].vfdev = pvfdev;
4976 else
4977 dev_err(&pdev->dev,
4978 "Couldn't find pci dev ptr for VF %4.4x\n",
4979 thisvf_devfn);
4980 return pvfdev != NULL;
4981}
4982
4983static int igb_find_enabled_vfs(struct igb_adapter *adapter)
4984{
4985 struct e1000_hw *hw = &adapter->hw;
4986 struct pci_dev *pdev = adapter->pdev;
4987 struct pci_dev *pvfdev;
4988 u16 vf_devfn = 0;
4989 u16 vf_stride;
4990 unsigned int device_id;
4991 int vfs_found = 0;
4992
4993 switch (adapter->hw.mac.type) {
4994 case e1000_82576:
4995 device_id = IGB_82576_VF_DEV_ID;
4996 /* VF Stride for 82576 is 2 */
4997 vf_stride = 2;
4998 break;
4999 case e1000_i350:
5000 device_id = IGB_I350_VF_DEV_ID;
5001 /* VF Stride for I350 is 4 */
5002 vf_stride = 4;
5003 break;
5004 default:
5005 device_id = 0;
5006 vf_stride = 0;
5007 break;
5008 }
5009
5010 vf_devfn = pdev->devfn + 0x80;
5011 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5012 while (pvfdev) {
5013 if (pvfdev->devfn == vf_devfn)
5014 vfs_found++;
5015 vf_devfn += vf_stride;
5016 pvfdev = pci_get_device(hw->vendor_id,
5017 device_id, pvfdev);
5018 }
5019
5020 return vfs_found;
5021}
5022
5023static int igb_check_vf_assignment(struct igb_adapter *adapter)
5024{
5025 int i;
5026 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5027 if (adapter->vf_data[i].vfdev) {
5028 if (adapter->vf_data[i].vfdev->dev_flags &
5029 PCI_DEV_FLAGS_ASSIGNED)
5030 return true;
5031 }
5032 }
5033 return false;
5034}
5035
5036#endif
4ae196df
AD		 5037static void igb_ping_all_vfs(struct igb_adapter *adapter)
5038{
5039 struct e1000_hw *hw = &adapter->hw;
5040 u32 ping;
5041 int i;
5042
5043 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5044 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 5045 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 5046 ping |= E1000_VT_MSGTYPE_CTS;
5047 igb_write_mbx(hw, &ping, 1, i);
5048 }
5049}
5050
7d5753f0
AD		 5051static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5052{
5053 struct e1000_hw *hw = &adapter->hw;
5054 u32 vmolr = rd32(E1000_VMOLR(vf));
5055 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5056
d85b9004 5057 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 5058 IGB_VF_FLAG_MULTI_PROMISC);
5059 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5060
5061 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5062 vmolr |= E1000_VMOLR_MPME;
d85b9004 5063 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 5064 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5065 } else {
5066 /*
5067 * if we have hashes and we are clearing a multicast promisc
5068 * flag we need to write the hashes to the MTA as this step
5069 * was previously skipped
5070 */
5071 if (vf_data->num_vf_mc_hashes > 30) {
5072 vmolr |= E1000_VMOLR_MPME;
5073 } else if (vf_data->num_vf_mc_hashes) {
5074 int j;
5075 vmolr |= E1000_VMOLR_ROMPE;
5076 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5077 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5078 }
5079 }
5080
5081 wr32(E1000_VMOLR(vf), vmolr);
5082
5083 /* there are flags left unprocessed, likely not supported */
5084 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5085 return -EINVAL;
5086
5087 return 0;
5088
5089}
5090
4ae196df
AD		 5091static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5092 u32 *msgbuf, u32 vf)
5093{
5094 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5095 u16 *hash_list = (u16 *)&msgbuf[1];
5096 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5097 int i;
5098
7d5753f0 5099 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5100 * to this VF for later use to restore when the PF multi cast
5101 * list changes
5102 */
5103 vf_data->num_vf_mc_hashes = n;
5104
7d5753f0
AD		 5105 /* only up to 30 hash values supported */
5106 if (n > 30)
5107 n = 30;
5108
5109 /* store the hashes for later use */
4ae196df 5110 for (i = 0; i < n; i++)
a419aef8 5111 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5112
5113 /* Flush and reset the mta with the new values */
ff41f8dc 5114 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5115
5116 return 0;
5117}
5118
5119static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5120{
5121 struct e1000_hw *hw = &adapter->hw;
5122 struct vf_data_storage *vf_data;
5123 int i, j;
5124
5125 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5126 u32 vmolr = rd32(E1000_VMOLR(i));
5127 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5128
4ae196df 5129 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5130
5131 if ((vf_data->num_vf_mc_hashes > 30) ||
5132 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5133 vmolr |= E1000_VMOLR_MPME;
5134 } else if (vf_data->num_vf_mc_hashes) {
5135 vmolr |= E1000_VMOLR_ROMPE;
5136 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5137 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5138 }
5139 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5140 }
5141}
5142
5143static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5144{
5145 struct e1000_hw *hw = &adapter->hw;
5146 u32 pool_mask, reg, vid;
5147 int i;
5148
5149 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5150
5151 /* Find the vlan filter for this id */
5152 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5153 reg = rd32(E1000_VLVF(i));
5154
5155 /* remove the vf from the pool */
5156 reg &= ~pool_mask;
5157
5158 /* if pool is empty then remove entry from vfta */
5159 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5160 (reg & E1000_VLVF_VLANID_ENABLE)) {
5161 reg = 0;
5162 vid = reg & E1000_VLVF_VLANID_MASK;
5163 igb_vfta_set(hw, vid, false);
5164 }
5165
5166 wr32(E1000_VLVF(i), reg);
5167 }
ae641bdc
AD		 5168
5169 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5170}
5171
5172static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5173{
5174 struct e1000_hw *hw = &adapter->hw;
5175 u32 reg, i;
5176
51466239
AD		 5177 /* The vlvf table only exists on 82576 hardware and newer */
5178 if (hw->mac.type < e1000_82576)
5179 return -1;
5180
5181 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5182 if (!adapter->vfs_allocated_count)
5183 return -1;
5184
5185 /* Find the vlan filter for this id */
5186 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5187 reg = rd32(E1000_VLVF(i));
5188 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5189 vid == (reg & E1000_VLVF_VLANID_MASK))
5190 break;
5191 }
5192
5193 if (add) {
5194 if (i == E1000_VLVF_ARRAY_SIZE) {
5195 /* Did not find a matching VLAN ID entry that was
5196 * enabled. Search for a free filter entry, i.e.
5197 * one without the enable bit set
5198 */
5199 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5200 reg = rd32(E1000_VLVF(i));
5201 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5202 break;
5203 }
5204 }
5205 if (i < E1000_VLVF_ARRAY_SIZE) {
5206 /* Found an enabled/available entry */
5207 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5208
5209 /* if !enabled we need to set this up in vfta */
5210 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5211 /* add VID to filter table */
5212 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5213 reg |= E1000_VLVF_VLANID_ENABLE;
5214 }
cad6d05f
AD		 5215 reg &= ~E1000_VLVF_VLANID_MASK;
5216 reg |= vid;
4ae196df 5217 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5218
5219 /* do not modify RLPML for PF devices */
5220 if (vf >= adapter->vfs_allocated_count)
5221 return 0;
5222
5223 if (!adapter->vf_data[vf].vlans_enabled) {
5224 u32 size;
5225 reg = rd32(E1000_VMOLR(vf));
5226 size = reg & E1000_VMOLR_RLPML_MASK;
5227 size += 4;
5228 reg &= ~E1000_VMOLR_RLPML_MASK;
5229 reg |= size;
5230 wr32(E1000_VMOLR(vf), reg);
5231 }
ae641bdc 5232
51466239 5233 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5234 }
5235 } else {
5236 if (i < E1000_VLVF_ARRAY_SIZE) {
5237 /* remove vf from the pool */
5238 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5239 /* if pool is empty then remove entry from vfta */
5240 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5241 reg = 0;
5242 igb_vfta_set(hw, vid, false);
5243 }
5244 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5245
5246 /* do not modify RLPML for PF devices */
5247 if (vf >= adapter->vfs_allocated_count)
5248 return 0;
5249
5250 adapter->vf_data[vf].vlans_enabled--;
5251 if (!adapter->vf_data[vf].vlans_enabled) {
5252 u32 size;
5253 reg = rd32(E1000_VMOLR(vf));
5254 size = reg & E1000_VMOLR_RLPML_MASK;
5255 size -= 4;
5256 reg &= ~E1000_VMOLR_RLPML_MASK;
5257 reg |= size;
5258 wr32(E1000_VMOLR(vf), reg);
5259 }
4ae196df
AD		 5260 }
5261 }
8151d294
WM		 5262 return 0;
5263}
5264
5265static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5266{
5267 struct e1000_hw *hw = &adapter->hw;
5268
5269 if (vid)
5270 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5271 else
5272 wr32(E1000_VMVIR(vf), 0);
5273}
5274
5275static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5276 int vf, u16 vlan, u8 qos)
5277{
5278 int err = 0;
5279 struct igb_adapter *adapter = netdev_priv(netdev);
5280
5281 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5282 return -EINVAL;
5283 if (vlan || qos) {
5284 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5285 if (err)
5286 goto out;
5287 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5288 igb_set_vmolr(adapter, vf, !vlan);
5289 adapter->vf_data[vf].pf_vlan = vlan;
5290 adapter->vf_data[vf].pf_qos = qos;
5291 dev_info(&adapter->pdev->dev,
5292 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5293 if (test_bit(__IGB_DOWN, &adapter->state)) {
5294 dev_warn(&adapter->pdev->dev,
5295 "The VF VLAN has been set,"
5296 " but the PF device is not up.\n");
5297 dev_warn(&adapter->pdev->dev,
5298 "Bring the PF device up before"
5299 " attempting to use the VF device.\n");
5300 }
5301 } else {
5302 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5303 false, vf);
5304 igb_set_vmvir(adapter, vlan, vf);
5305 igb_set_vmolr(adapter, vf, true);
5306 adapter->vf_data[vf].pf_vlan = 0;
5307 adapter->vf_data[vf].pf_qos = 0;
5308 }
5309out:
5310 return err;
4ae196df
AD		 5311}
5312
5313static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5314{
5315 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5316 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5317
5318 return igb_vlvf_set(adapter, vid, add, vf);
5319}
5320
f2ca0dbe 5321static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5322{
8fa7e0f7
GR		 5323 /* clear flags - except flag that indicates PF has set the MAC */
5324 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5325 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5326
5327 /* reset offloads to defaults */
8151d294 5328 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5329
5330 /* reset vlans for device */
5331 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5332 if (adapter->vf_data[vf].pf_vlan)
5333 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5334 adapter->vf_data[vf].pf_vlan,
5335 adapter->vf_data[vf].pf_qos);
5336 else
5337 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5338
5339 /* reset multicast table array for vf */
5340 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5341
5342 /* Flush and reset the mta with the new values */
ff41f8dc 5343 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5344}
5345
f2ca0dbe
AD		 5346static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5347{
5348 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5349
5350 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5351 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5352 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5353
5354 /* process remaining reset events */
5355 igb_vf_reset(adapter, vf);
5356}
5357
5358static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5359{
5360 struct e1000_hw *hw = &adapter->hw;
5361 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5362 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5363 u32 reg, msgbuf[3];
5364 u8 *addr = (u8 *)(&msgbuf[1]);
5365
5366 /* process all the same items cleared in a function level reset */
f2ca0dbe 5367 igb_vf_reset(adapter, vf);
4ae196df
AD		 5368
5369 /* set vf mac address */
26ad9178 5370 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5371
5372 /* enable transmit and receive for vf */
5373 reg = rd32(E1000_VFTE);
5374 wr32(E1000_VFTE, reg | (1 << vf));
5375 reg = rd32(E1000_VFRE);
5376 wr32(E1000_VFRE, reg | (1 << vf));
5377
8fa7e0f7 5378 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5379
5380 /* reply to reset with ack and vf mac address */
5381 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5382 memcpy(addr, vf_mac, 6);
5383 igb_write_mbx(hw, msgbuf, 3, vf);
5384}
5385
5386static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5387{
de42edde
GR		 5388 /*
5389 * The VF MAC Address is stored in a packed array of bytes
5390 * starting at the second 32 bit word of the msg array
5391 */
f2ca0dbe
AD		 5392 unsigned char *addr = (char *)&msg[1];
5393 int err = -1;
4ae196df 5394
f2ca0dbe
AD		 5395 if (is_valid_ether_addr(addr))
5396 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5397
f2ca0dbe 5398 return err;
4ae196df
AD		 5399}
5400
5401static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5402{
5403 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5404 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5405 u32 msg = E1000_VT_MSGTYPE_NACK;
5406
5407 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5408 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5409 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5410 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5411 vf_data->last_nack = jiffies;
4ae196df
AD		 5412 }
5413}
5414
f2ca0dbe 5415static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5416{
f2ca0dbe
AD		 5417 struct pci_dev *pdev = adapter->pdev;
5418 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5419 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5420 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5421 s32 retval;
5422
f2ca0dbe 5423 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5424
fef45f4c
AD		 5425 if (retval) {
5426 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5427 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5428 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5429 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5430 return;
5431 goto out;
5432 }
4ae196df
AD		 5433
5434 /* this is a message we already processed, do nothing */
5435 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5436 return;
4ae196df
AD		 5437
5438 /*
5439 * until the vf completes a reset it should not be
5440 * allowed to start any configuration.
5441 */
5442
5443 if (msgbuf[0] == E1000_VF_RESET) {
5444 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5445 return;
4ae196df
AD		 5446 }
5447
f2ca0dbe 5448 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5449 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5450 return;
5451 retval = -1;
5452 goto out;
4ae196df
AD		 5453 }
5454
5455 switch ((msgbuf[0] & 0xFFFF)) {
5456 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5457 retval = -EINVAL;
5458 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5459 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5460 else
5461 dev_warn(&pdev->dev,
5462 "VF %d attempted to override administratively "
5463 "set MAC address\nReload the VF driver to "
5464 "resume operations\n", vf);
4ae196df 5465 break;
7d5753f0
AD		 5466 case E1000_VF_SET_PROMISC:
5467 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5468 break;
4ae196df
AD		 5469 case E1000_VF_SET_MULTICAST:
5470 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5471 break;
5472 case E1000_VF_SET_LPE:
5473 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5474 break;
5475 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5476 retval = -1;
5477 if (vf_data->pf_vlan)
5478 dev_warn(&pdev->dev,
5479 "VF %d attempted to override administratively "
5480 "set VLAN tag\nReload the VF driver to "
5481 "resume operations\n", vf);
8151d294
WM		 5482 else
5483 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5484 break;
5485 default:
090b1795 5486 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5487 retval = -1;
5488 break;
5489 }
5490
fef45f4c
AD		 5491 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5492out:
4ae196df
AD		 5493 /* notify the VF of the results of what it sent us */
5494 if (retval)
5495 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5496 else
5497 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5498
4ae196df 5499 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5500}
4ae196df 5501
f2ca0dbe
AD		 5502static void igb_msg_task(struct igb_adapter *adapter)
5503{
5504 struct e1000_hw *hw = &adapter->hw;
5505 u32 vf;
5506
5507 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5508 /* process any reset requests */
5509 if (!igb_check_for_rst(hw, vf))
5510 igb_vf_reset_event(adapter, vf);
5511
5512 /* process any messages pending */
5513 if (!igb_check_for_msg(hw, vf))
5514 igb_rcv_msg_from_vf(adapter, vf);
5515
5516 /* process any acks */
5517 if (!igb_check_for_ack(hw, vf))
5518 igb_rcv_ack_from_vf(adapter, vf);
5519 }
4ae196df
AD		 5520}
5521
68d480c4
AD		 5522/**
5523 * igb_set_uta - Set unicast filter table address
5524 * @adapter: board private structure
5525 *
5526 * The unicast table address is a register array of 32-bit registers.
5527 * The table is meant to be used in a way similar to how the MTA is used
5528 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5529 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5530 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5531 **/
5532static void igb_set_uta(struct igb_adapter *adapter)
5533{
5534 struct e1000_hw *hw = &adapter->hw;
5535 int i;
5536
5537 /* The UTA table only exists on 82576 hardware and newer */
5538 if (hw->mac.type < e1000_82576)
5539 return;
5540
5541 /* we only need to do this if VMDq is enabled */
5542 if (!adapter->vfs_allocated_count)
5543 return;
5544
5545 for (i = 0; i < hw->mac.uta_reg_count; i++)
5546 array_wr32(E1000_UTA, i, ~0);
5547}
5548
9d5c8243
AK		 5549/**
5550 * igb_intr_msi - Interrupt Handler
5551 * @irq: interrupt number
5552 * @data: pointer to a network interface device structure
5553 **/
5554static irqreturn_t igb_intr_msi(int irq, void *data)
5555{
047e0030
AD		 5556 struct igb_adapter *adapter = data;
5557 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5558 struct e1000_hw *hw = &adapter->hw;
5559 /* read ICR disables interrupts using IAM */
5560 u32 icr = rd32(E1000_ICR);
5561
047e0030 5562 igb_write_itr(q_vector);
9d5c8243 5563
7f081d40
AD		 5564 if (icr & E1000_ICR_DRSTA)
5565 schedule_work(&adapter->reset_task);
5566
047e0030 5567 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5568 /* HW is reporting DMA is out of sync */
5569 adapter->stats.doosync++;
5570 }
5571
9d5c8243
AK		 5572 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5573 hw->mac.get_link_status = 1;
5574 if (!test_bit(__IGB_DOWN, &adapter->state))
5575 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5576 }
5577
047e0030 5578 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5579
5580 return IRQ_HANDLED;
5581}
5582
5583/**
4a3c6433 5584 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5585 * @irq: interrupt number
5586 * @data: pointer to a network interface device structure
5587 **/
5588static irqreturn_t igb_intr(int irq, void *data)
5589{
047e0030
AD		 5590 struct igb_adapter *adapter = data;
5591 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5592 struct e1000_hw *hw = &adapter->hw;
5593 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5594 * need for the IMC write */
5595 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5596
5597 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5598 * not set, then the adapter didn't send an interrupt */
5599 if (!(icr & E1000_ICR_INT_ASSERTED))
5600 return IRQ_NONE;
5601
0ba82994
AD		 5602 igb_write_itr(q_vector);
5603
7f081d40
AD		 5604 if (icr & E1000_ICR_DRSTA)
5605 schedule_work(&adapter->reset_task);
5606
047e0030 5607 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5608 /* HW is reporting DMA is out of sync */
5609 adapter->stats.doosync++;
5610 }
5611
9d5c8243
AK		 5612 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5613 hw->mac.get_link_status = 1;
5614 /* guard against interrupt when we're going down */
5615 if (!test_bit(__IGB_DOWN, &adapter->state))
5616 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5617 }
5618
047e0030 5619 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5620
5621 return IRQ_HANDLED;
5622}
5623
0ba82994 5624void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5625{
047e0030 5626 struct igb_adapter *adapter = q_vector->adapter;
46544258 5627 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5628
0ba82994
AD		 5629 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5630 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5631 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5632 igb_set_itr(q_vector);
46544258 5633 else
047e0030 5634 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5635 }
5636
46544258
AD		 5637 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5638 if (adapter->msix_entries)
047e0030 5639 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5640 else
5641 igb_irq_enable(adapter);
5642 }
9d5c8243
AK		 5643}
5644
46544258
AD		 5645/**
5646 * igb_poll - NAPI Rx polling callback
5647 * @napi: napi polling structure
5648 * @budget: count of how many packets we should handle
5649 **/
5650static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5651{
047e0030
AD		 5652 struct igb_q_vector *q_vector = container_of(napi,
5653 struct igb_q_vector,
5654 napi);
16eb8815 5655 bool clean_complete = true;
9d5c8243 5656
421e02f0 5657#ifdef CONFIG_IGB_DCA
047e0030
AD		 5658 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5659 igb_update_dca(q_vector);
fe4506b6 5660#endif
0ba82994 5661 if (q_vector->tx.ring)
13fde97a 5662 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5663
0ba82994 5664 if (q_vector->rx.ring)
cd392f5c 5665 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5666
16eb8815
AD		 5667 /* If all work not completed, return budget and keep polling */
5668 if (!clean_complete)
5669 return budget;
46544258 5670
9d5c8243 5671 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5672 napi_complete(napi);
5673 igb_ring_irq_enable(q_vector);
9d5c8243 5674
16eb8815 5675 return 0;
9d5c8243 5676}
6d8126f9 5677
33af6bcc 5678/**
c5b9bd5e 5679 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5680 * @adapter: board private structure
c5b9bd5e
AD		 5681 * @shhwtstamps: timestamp structure to update
5682 * @regval: unsigned 64bit system time value.
5683 *
5684 * We need to convert the system time value stored in the RX/TXSTMP registers
5685 * into a hwtstamp which can be used by the upper level timestamping functions
5686 */
5687static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5688 struct skb_shared_hwtstamps *shhwtstamps,
5689 u64 regval)
5690{
5691 u64 ns;
5692
55cac248
AD		 5693 /*
5694 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5695 * 24 to match clock shift we setup earlier.
5696 */
06218a8d 5697 if (adapter->hw.mac.type >= e1000_82580)
55cac248
AD		 5698 regval <<= IGB_82580_TSYNC_SHIFT;
5699
c5b9bd5e
AD		 5700 ns = timecounter_cyc2time(&adapter->clock, regval);
5701 timecompare_update(&adapter->compare, ns);
5702 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5703 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5704 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5705}
5706
5707/**
5708 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5709 * @q_vector: pointer to q_vector containing needed info
06034649 5710 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5711 *
5712 * If we were asked to do hardware stamping and such a time stamp is
5713 * available, then it must have been for this skb here because we only
5714 * allow only one such packet into the queue.
5715 */
06034649
AD		 5716static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5717 struct igb_tx_buffer *buffer_info)
33af6bcc 5718{
c5b9bd5e 5719 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5720 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5721 struct skb_shared_hwtstamps shhwtstamps;
5722 u64 regval;
33af6bcc 5723
c5b9bd5e 5724 /* if skb does not support hw timestamp or TX stamp not valid exit */
2bbfebe2 5725 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
c5b9bd5e
AD		 5726 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5727 return;
5728
5729 regval = rd32(E1000_TXSTMPL);
5730 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5731
5732 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5733 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5734}
5735
9d5c8243
AK		 5736/**
5737 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5738 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5739 * returns true if ring is completely cleaned
5740 **/
047e0030 5741static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5742{
047e0030 5743 struct igb_adapter *adapter = q_vector->adapter;
0ba82994 5744 struct igb_ring *tx_ring = q_vector->tx.ring;
06034649 5745 struct igb_tx_buffer *tx_buffer;
8542db05 5746 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5747 unsigned int total_bytes = 0, total_packets = 0;
0ba82994 5748 unsigned int budget = q_vector->tx.work_limit;
8542db05 5749 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5750
13fde97a
AD		 5751 if (test_bit(__IGB_DOWN, &adapter->state))
5752 return true;
0e014cb1 5753
06034649 5754 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5755 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5756 i -= tx_ring->count;
9d5c8243 5757
13fde97a 5758 for (; budget; budget--) {
8542db05 5759 eop_desc = tx_buffer->next_to_watch;
13fde97a 5760
8542db05
AD		 5761 /* prevent any other reads prior to eop_desc */
5762 rmb();
5763
5764 /* if next_to_watch is not set then there is no work pending */
5765 if (!eop_desc)
5766 break;
13fde97a
AD		 5767
5768 /* if DD is not set pending work has not been completed */
5769 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5770 break;
5771
8542db05
AD		 5772 /* clear next_to_watch to prevent false hangs */
5773 tx_buffer->next_to_watch = NULL;
9d5c8243 5774
ebe42d16
AD		 5775 /* update the statistics for this packet */
5776 total_bytes += tx_buffer->bytecount;
5777 total_packets += tx_buffer->gso_segs;
13fde97a 5778
ebe42d16
AD		 5779 /* retrieve hardware timestamp */
5780 igb_tx_hwtstamp(q_vector, tx_buffer);
5781
5782 /* free the skb */
5783 dev_kfree_skb_any(tx_buffer->skb);
5784 tx_buffer->skb = NULL;
13fde97a 5785
ebe42d16
AD		 5786 /* unmap skb header data */
5787 dma_unmap_single(tx_ring->dev,
5788 tx_buffer->dma,
5789 tx_buffer->length,
5790 DMA_TO_DEVICE);
5791
5792 /* clear last DMA location and unmap remaining buffers */
5793 while (tx_desc != eop_desc) {
5794 tx_buffer->dma = 0;
9d5c8243 5795
13fde97a
AD		 5796 tx_buffer++;
5797 tx_desc++;
9d5c8243 5798 i++;
8542db05
AD		 5799 if (unlikely(!i)) {
5800 i -= tx_ring->count;
06034649 5801 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5802 tx_desc = IGB_TX_DESC(tx_ring, 0);
5803 }
ebe42d16
AD		 5804
5805 /* unmap any remaining paged data */
5806 if (tx_buffer->dma) {
5807 dma_unmap_page(tx_ring->dev,
5808 tx_buffer->dma,
5809 tx_buffer->length,
5810 DMA_TO_DEVICE);
5811 }
5812 }
5813
5814 /* clear last DMA location */
5815 tx_buffer->dma = 0;
5816
5817 /* move us one more past the eop_desc for start of next pkt */
5818 tx_buffer++;
5819 tx_desc++;
5820 i++;
5821 if (unlikely(!i)) {
5822 i -= tx_ring->count;
5823 tx_buffer = tx_ring->tx_buffer_info;
5824 tx_desc = IGB_TX_DESC(tx_ring, 0);
5825 }
0e014cb1
AD		 5826 }
5827
8542db05 5828 i += tx_ring->count;
9d5c8243 5829 tx_ring->next_to_clean = i;
13fde97a
AD		 5830 u64_stats_update_begin(&tx_ring->tx_syncp);
5831 tx_ring->tx_stats.bytes += total_bytes;
5832 tx_ring->tx_stats.packets += total_packets;
5833 u64_stats_update_end(&tx_ring->tx_syncp);
0ba82994
AD		 5834 q_vector->tx.total_bytes += total_bytes;
5835 q_vector->tx.total_packets += total_packets;
9d5c8243 5836
6d095fa8 5837 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
13fde97a 5838 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5839
8542db05 5840 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5841
9d5c8243
AK		 5842 /* Detect a transmit hang in hardware, this serializes the
5843 * check with the clearing of time_stamp and movement of i */
6d095fa8 5844 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8542db05
AD		 5845 if (eop_desc &&
5846 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5847 (adapter->tx_timeout_factor * HZ)) &&
5848 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5849
9d5c8243 5850 /* detected Tx unit hang */
59d71989 5851 dev_err(tx_ring->dev,
9d5c8243 5852 "Detected Tx Unit Hang\n"
2d064c06 5853 " Tx Queue <%d>\n"
9d5c8243
AK		 5854 " TDH <%x>\n"
5855 " TDT <%x>\n"
5856 " next_to_use <%x>\n"
5857 " next_to_clean <%x>\n"
9d5c8243
AK		 5858 "buffer_info[next_to_clean]\n"
5859 " time_stamp <%lx>\n"
8542db05 5860 " next_to_watch <%p>\n"
9d5c8243
AK		 5861 " jiffies <%lx>\n"
5862 " desc.status <%x>\n",
2d064c06 5863 tx_ring->queue_index,
238ac817 5864 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5865 readl(tx_ring->tail),
9d5c8243
AK		 5866 tx_ring->next_to_use,
5867 tx_ring->next_to_clean,
8542db05
AD		 5868 tx_buffer->time_stamp,
5869 eop_desc,
9d5c8243 5870 jiffies,
0e014cb1 5871 eop_desc->wb.status);
13fde97a
AD		 5872 netif_stop_subqueue(tx_ring->netdev,
5873 tx_ring->queue_index);
5874
5875 /* we are about to reset, no point in enabling stuff */
5876 return true;
9d5c8243
AK		 5877 }
5878 }
13fde97a
AD		 5879
5880 if (unlikely(total_packets &&
5881 netif_carrier_ok(tx_ring->netdev) &&
5882 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5883 /* Make sure that anybody stopping the queue after this
5884 * sees the new next_to_clean.
5885 */
5886 smp_mb();
5887 if (__netif_subqueue_stopped(tx_ring->netdev,
5888 tx_ring->queue_index) &&
5889 !(test_bit(__IGB_DOWN, &adapter->state))) {
5890 netif_wake_subqueue(tx_ring->netdev,
5891 tx_ring->queue_index);
5892
5893 u64_stats_update_begin(&tx_ring->tx_syncp);
5894 tx_ring->tx_stats.restart_queue++;
5895 u64_stats_update_end(&tx_ring->tx_syncp);
5896 }
5897 }
5898
5899 return !!budget;
9d5c8243
AK		 5900}
5901
cd392f5c 5902static inline void igb_rx_checksum(struct igb_ring *ring,
3ceb90fd
AD		 5903 union e1000_adv_rx_desc *rx_desc,
5904 struct sk_buff *skb)
9d5c8243 5905{
bc8acf2c 5906 skb_checksum_none_assert(skb);
9d5c8243 5907
294e7d78 5908 /* Ignore Checksum bit is set */
3ceb90fd 5909 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
294e7d78
AD		 5910 return;
5911
5912 /* Rx checksum disabled via ethtool */
5913 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9d5c8243 5914 return;
85ad76b2 5915
9d5c8243 5916 /* TCP/UDP checksum error bit is set */
3ceb90fd
AD		 5917 if (igb_test_staterr(rx_desc,
5918 E1000_RXDEXT_STATERR_TCPE |
5919 E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5920 /*
5921 * work around errata with sctp packets where the TCPE aka
5922 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5923 * packets, (aka let the stack check the crc32c)
5924 */
866cff06
AD		 5925 if (!((skb->len == 60) &&
5926 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
12dcd86b 5927 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5928 ring->rx_stats.csum_err++;
12dcd86b
ED		 5929 u64_stats_update_end(&ring->rx_syncp);
5930 }
9d5c8243 5931 /* let the stack verify checksum errors */
9d5c8243
AK		 5932 return;
5933 }
5934 /* It must be a TCP or UDP packet with a valid checksum */
3ceb90fd
AD		 5935 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5936 E1000_RXD_STAT_UDPCS))
9d5c8243
AK		 5937 skb->ip_summed = CHECKSUM_UNNECESSARY;
5938
3ceb90fd
AD		 5939 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5940 le32_to_cpu(rx_desc->wb.upper.status_error));
9d5c8243
AK		 5941}
5942
077887c3
AD		 5943static inline void igb_rx_hash(struct igb_ring *ring,
5944 union e1000_adv_rx_desc *rx_desc,
5945 struct sk_buff *skb)
5946{
5947 if (ring->netdev->features & NETIF_F_RXHASH)
5948 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5949}
5950
3ceb90fd
AD		 5951static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
5952 union e1000_adv_rx_desc *rx_desc,
5953 struct sk_buff *skb)
c5b9bd5e
AD		 5954{
5955 struct igb_adapter *adapter = q_vector->adapter;
5956 struct e1000_hw *hw = &adapter->hw;
5957 u64 regval;
5958
3ceb90fd
AD		 5959 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
5960 E1000_RXDADV_STAT_TS))
5961 return;
5962
c5b9bd5e
AD		 5963 /*
5964 * If this bit is set, then the RX registers contain the time stamp. No
5965 * other packet will be time stamped until we read these registers, so
5966 * read the registers to make them available again. Because only one
5967 * packet can be time stamped at a time, we know that the register
5968 * values must belong to this one here and therefore we don't need to
5969 * compare any of the additional attributes stored for it.
5970 *
2244d07b 5971 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5972 * can turn into a skb_shared_hwtstamps.
5973 */
3ceb90fd 5974 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
757b77e2
NN		 5975 u32 *stamp = (u32 *)skb->data;
5976 regval = le32_to_cpu(*(stamp + 2));
5977 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5978 skb_pull(skb, IGB_TS_HDR_LEN);
5979 } else {
5980 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5981 return;
c5b9bd5e 5982
757b77e2
NN		 5983 regval = rd32(E1000_RXSTMPL);
5984 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5985 }
c5b9bd5e
AD		 5986
5987 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5988}
8be10e91
AD		 5989
5990static void igb_rx_vlan(struct igb_ring *ring,
5991 union e1000_adv_rx_desc *rx_desc,
5992 struct sk_buff *skb)
5993{
5994 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
5995 u16 vid;
5996 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
5997 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
5998 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
5999 else
6000 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6001
6002 __vlan_hwaccel_put_tag(skb, vid);
6003 }
6004}
6005
44390ca6 6006static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 6007{
6008 /* HW will not DMA in data larger than the given buffer, even if it
6009 * parses the (NFS, of course) header to be larger. In that case, it
6010 * fills the header buffer and spills the rest into the page.
6011 */
6012 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
6013 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 6014 if (hlen > IGB_RX_HDR_LEN)
6015 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 6016 return hlen;
6017}
6018
cd392f5c 6019static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 6020{
0ba82994 6021 struct igb_ring *rx_ring = q_vector->rx.ring;
16eb8815
AD		 6022 union e1000_adv_rx_desc *rx_desc;
6023 const int current_node = numa_node_id();
9d5c8243 6024 unsigned int total_bytes = 0, total_packets = 0;
16eb8815
AD		 6025 u16 cleaned_count = igb_desc_unused(rx_ring);
6026 u16 i = rx_ring->next_to_clean;
9d5c8243 6027
60136906 6028 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243 6029
3ceb90fd 6030 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
06034649 6031 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 6032 struct sk_buff *skb = buffer_info->skb;
6033 union e1000_adv_rx_desc *next_rxd;
9d5c8243 6034
69d3ca53 6035 buffer_info->skb = NULL;
16eb8815 6036 prefetch(skb->data);
69d3ca53
AD		 6037
6038 i++;
6039 if (i == rx_ring->count)
6040 i = 0;
42d0781a 6041
60136906 6042 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 6043 prefetch(next_rxd);
9d5c8243 6044
16eb8815
AD		 6045 /*
6046 * This memory barrier is needed to keep us from reading
6047 * any other fields out of the rx_desc until we know the
6048 * RXD_STAT_DD bit is set
6049 */
6050 rmb();
9d5c8243 6051
16eb8815
AD		 6052 if (!skb_is_nonlinear(skb)) {
6053 __skb_put(skb, igb_get_hlen(rx_desc));
6054 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 6055 IGB_RX_HDR_LEN,
59d71989 6056 DMA_FROM_DEVICE);
91615f76 6057 buffer_info->dma = 0;
bf36c1a0
AD		 6058 }
6059
16eb8815
AD		 6060 if (rx_desc->wb.upper.length) {
6061 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 6062
aa913403 6063 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 6064 buffer_info->page,
6065 buffer_info->page_offset,
6066 length);
6067
16eb8815
AD		 6068 skb->len += length;
6069 skb->data_len += length;
95b9c1df 6070 skb->truesize += PAGE_SIZE / 2;
16eb8815 6071
d1eff350
AD		 6072 if ((page_count(buffer_info->page) != 1) ||
6073 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 6074 buffer_info->page = NULL;
6075 else
6076 get_page(buffer_info->page);
9d5c8243 6077
16eb8815
AD		 6078 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6079 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6080 buffer_info->page_dma = 0;
9d5c8243 6081 }
9d5c8243 6082
3ceb90fd 6083 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
06034649
AD		 6084 struct igb_rx_buffer *next_buffer;
6085 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 6086 buffer_info->skb = next_buffer->skb;
6087 buffer_info->dma = next_buffer->dma;
6088 next_buffer->skb = skb;
6089 next_buffer->dma = 0;
bf36c1a0
AD		 6090 goto next_desc;
6091 }
44390ca6 6092
3ceb90fd
AD		 6093 if (igb_test_staterr(rx_desc,
6094 E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
16eb8815 6095 dev_kfree_skb_any(skb);
9d5c8243
AK		 6096 goto next_desc;
6097 }
9d5c8243 6098
3ceb90fd 6099 igb_rx_hwtstamp(q_vector, rx_desc, skb);
077887c3 6100 igb_rx_hash(rx_ring, rx_desc, skb);
3ceb90fd 6101 igb_rx_checksum(rx_ring, rx_desc, skb);
8be10e91 6102 igb_rx_vlan(rx_ring, rx_desc, skb);
3ceb90fd
AD		 6103
6104 total_bytes += skb->len;
6105 total_packets++;
6106
6107 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6108
b2cb09b1 6109 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6110
16eb8815 6111 budget--;
9d5c8243 6112next_desc:
16eb8815
AD		 6113 if (!budget)
6114 break;
6115
6116 cleaned_count++;
9d5c8243
AK		 6117 /* return some buffers to hardware, one at a time is too slow */
6118 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6119 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6120 cleaned_count = 0;
6121 }
6122
6123 /* use prefetched values */
6124 rx_desc = next_rxd;
9d5c8243 6125 }
bf36c1a0 6126
9d5c8243 6127 rx_ring->next_to_clean = i;
12dcd86b 6128 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6129 rx_ring->rx_stats.packets += total_packets;
6130 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6131 u64_stats_update_end(&rx_ring->rx_syncp);
0ba82994
AD		 6132 q_vector->rx.total_packets += total_packets;
6133 q_vector->rx.total_bytes += total_bytes;
c023cd88
AD		 6134
6135 if (cleaned_count)
cd392f5c 6136 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6137
16eb8815 6138 return !!budget;
9d5c8243
AK		 6139}
6140
c023cd88 6141static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6142 struct igb_rx_buffer *bi)
c023cd88
AD		 6143{
6144 struct sk_buff *skb = bi->skb;
6145 dma_addr_t dma = bi->dma;
6146
6147 if (dma)
6148 return true;
6149
6150 if (likely(!skb)) {
6151 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6152 IGB_RX_HDR_LEN);
6153 bi->skb = skb;
6154 if (!skb) {
6155 rx_ring->rx_stats.alloc_failed++;
6156 return false;
6157 }
6158
6159 /* initialize skb for ring */
6160 skb_record_rx_queue(skb, rx_ring->queue_index);
6161 }
6162
6163 dma = dma_map_single(rx_ring->dev, skb->data,
6164 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6165
6166 if (dma_mapping_error(rx_ring->dev, dma)) {
6167 rx_ring->rx_stats.alloc_failed++;
6168 return false;
6169 }
6170
6171 bi->dma = dma;
6172 return true;
6173}
6174
6175static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6176 struct igb_rx_buffer *bi)
c023cd88
AD		 6177{
6178 struct page *page = bi->page;
6179 dma_addr_t page_dma = bi->page_dma;
6180 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6181
6182 if (page_dma)
6183 return true;
6184
6185 if (!page) {
6186 page = netdev_alloc_page(rx_ring->netdev);
6187 bi->page = page;
6188 if (unlikely(!page)) {
6189 rx_ring->rx_stats.alloc_failed++;
6190 return false;
6191 }
6192 }
6193
6194 page_dma = dma_map_page(rx_ring->dev, page,
6195 page_offset, PAGE_SIZE / 2,
6196 DMA_FROM_DEVICE);
6197
6198 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6199 rx_ring->rx_stats.alloc_failed++;
6200 return false;
6201 }
6202
6203 bi->page_dma = page_dma;
6204 bi->page_offset = page_offset;
6205 return true;
6206}
6207
9d5c8243 6208/**
cd392f5c 6209 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6210 * @adapter: address of board private structure
6211 **/
cd392f5c 6212void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6213{
9d5c8243 6214 union e1000_adv_rx_desc *rx_desc;
06034649 6215 struct igb_rx_buffer *bi;
c023cd88 6216 u16 i = rx_ring->next_to_use;
9d5c8243 6217
60136906 6218 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6219 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6220 i -= rx_ring->count;
9d5c8243
AK		 6221
6222 while (cleaned_count--) {
c023cd88
AD		 6223 if (!igb_alloc_mapped_skb(rx_ring, bi))
6224 break;
9d5c8243 6225
c023cd88
AD		 6226 /* Refresh the desc even if buffer_addrs didn't change
6227 * because each write-back erases this info. */
6228 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6229
c023cd88
AD		 6230 if (!igb_alloc_mapped_page(rx_ring, bi))
6231 break;
6232
6233 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6234
c023cd88
AD		 6235 rx_desc++;
6236 bi++;
9d5c8243 6237 i++;
c023cd88 6238 if (unlikely(!i)) {
60136906 6239 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6240 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6241 i -= rx_ring->count;
6242 }
6243
6244 /* clear the hdr_addr for the next_to_use descriptor */
6245 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6246 }
6247
c023cd88
AD		 6248 i += rx_ring->count;
6249
9d5c8243
AK		 6250 if (rx_ring->next_to_use != i) {
6251 rx_ring->next_to_use = i;
9d5c8243
AK		 6252
6253 /* Force memory writes to complete before letting h/w
6254 * know there are new descriptors to fetch. (Only
6255 * applicable for weak-ordered memory model archs,
6256 * such as IA-64). */
6257 wmb();
fce99e34 6258 writel(i, rx_ring->tail);
9d5c8243
AK		 6259 }
6260}
6261
6262/**
6263 * igb_mii_ioctl -
6264 * @netdev:
6265 * @ifreq:
6266 * @cmd:
6267 **/
6268static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6269{
6270 struct igb_adapter *adapter = netdev_priv(netdev);
6271 struct mii_ioctl_data *data = if_mii(ifr);
6272
6273 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6274 return -EOPNOTSUPP;
6275
6276 switch (cmd) {
6277 case SIOCGMIIPHY:
6278 data->phy_id = adapter->hw.phy.addr;
6279 break;
6280 case SIOCGMIIREG:
f5f4cf08
AD		 6281 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6282 &data->val_out))
9d5c8243
AK		 6283 return -EIO;
6284 break;
6285 case SIOCSMIIREG:
6286 default:
6287 return -EOPNOTSUPP;
6288 }
6289 return 0;
6290}
6291
c6cb090b
PO		 6292/**
6293 * igb_hwtstamp_ioctl - control hardware time stamping
6294 * @netdev:
6295 * @ifreq:
6296 * @cmd:
6297 *
33af6bcc
PO		 6298 * Outgoing time stamping can be enabled and disabled. Play nice and
6299 * disable it when requested, although it shouldn't case any overhead
6300 * when no packet needs it. At most one packet in the queue may be
6301 * marked for time stamping, otherwise it would be impossible to tell
6302 * for sure to which packet the hardware time stamp belongs.
6303 *
6304 * Incoming time stamping has to be configured via the hardware
6305 * filters. Not all combinations are supported, in particular event
6306 * type has to be specified. Matching the kind of event packet is
6307 * not supported, with the exception of "all V2 events regardless of
6308 * level 2 or 4".
6309 *
c6cb090b
PO		 6310 **/
6311static int igb_hwtstamp_ioctl(struct net_device *netdev,
6312 struct ifreq *ifr, int cmd)
6313{
33af6bcc
PO		 6314 struct igb_adapter *adapter = netdev_priv(netdev);
6315 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6316 struct hwtstamp_config config;
c5b9bd5e
AD		 6317 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6318 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6319 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6320 bool is_l4 = false;
6321 bool is_l2 = false;
33af6bcc 6322 u32 regval;
c6cb090b
PO		 6323
6324 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6325 return -EFAULT;
6326
6327 /* reserved for future extensions */
6328 if (config.flags)
6329 return -EINVAL;
6330
33af6bcc
PO		 6331 switch (config.tx_type) {
6332 case HWTSTAMP_TX_OFF:
c5b9bd5e 6333 tsync_tx_ctl = 0;
33af6bcc 6334 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6335 break;
6336 default:
6337 return -ERANGE;
6338 }
6339
6340 switch (config.rx_filter) {
6341 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6342 tsync_rx_ctl = 0;
33af6bcc
PO		 6343 break;
6344 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6345 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6346 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6347 case HWTSTAMP_FILTER_ALL:
6348 /*
6349 * register TSYNCRXCFG must be set, therefore it is not
6350 * possible to time stamp both Sync and Delay_Req messages
6351 * => fall back to time stamping all packets
6352 */
c5b9bd5e 6353 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6354 config.rx_filter = HWTSTAMP_FILTER_ALL;
6355 break;
6356 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6357 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6358 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6359 is_l4 = true;
33af6bcc
PO		 6360 break;
6361 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6362 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6363 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6364 is_l4 = true;
33af6bcc
PO		 6365 break;
6366 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6367 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6368 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6369 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6370 is_l2 = true;
6371 is_l4 = true;
33af6bcc
PO		 6372 config.rx_filter = HWTSTAMP_FILTER_SOME;
6373 break;
6374 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6375 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6376 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6377 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6378 is_l2 = true;
6379 is_l4 = true;
33af6bcc
PO		 6380 config.rx_filter = HWTSTAMP_FILTER_SOME;
6381 break;
6382 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6383 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6384 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6385 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6386 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6387 is_l2 = true;
11ba69e8 6388 is_l4 = true;
33af6bcc
PO		 6389 break;
6390 default:
6391 return -ERANGE;
6392 }
6393
c5b9bd5e
AD		 6394 if (hw->mac.type == e1000_82575) {
6395 if (tsync_rx_ctl | tsync_tx_ctl)
6396 return -EINVAL;
6397 return 0;
6398 }
6399
757b77e2
NN		 6400 /*
6401 * Per-packet timestamping only works if all packets are
6402 * timestamped, so enable timestamping in all packets as
6403 * long as one rx filter was configured.
6404 */
06218a8d 6405 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
757b77e2
NN		 6406 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6407 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6408 }
6409
33af6bcc
PO		 6410 /* enable/disable TX */
6411 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6412 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6413 regval |= tsync_tx_ctl;
33af6bcc
PO		 6414 wr32(E1000_TSYNCTXCTL, regval);
6415
c5b9bd5e 6416 /* enable/disable RX */
33af6bcc 6417 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6418 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6419 regval |= tsync_rx_ctl;
33af6bcc 6420 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6421
c5b9bd5e
AD		 6422 /* define which PTP packets are time stamped */
6423 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6424
c5b9bd5e
AD		 6425 /* define ethertype filter for timestamped packets */
6426 if (is_l2)
6427 wr32(E1000_ETQF(3),
6428 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6429 E1000_ETQF_1588 | /* enable timestamping */
6430 ETH_P_1588)); /* 1588 eth protocol type */
6431 else
6432 wr32(E1000_ETQF(3), 0);
6433
6434#define PTP_PORT 319
6435 /* L4 Queue Filter[3]: filter by destination port and protocol */
6436 if (is_l4) {
6437 u32 ftqf = (IPPROTO_UDP /* UDP */
6438 | E1000_FTQF_VF_BP /* VF not compared */
6439 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6440 | E1000_FTQF_MASK); /* mask all inputs */
6441 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6442
6443 wr32(E1000_IMIR(3), htons(PTP_PORT));
6444 wr32(E1000_IMIREXT(3),
6445 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6446 if (hw->mac.type == e1000_82576) {
6447 /* enable source port check */
6448 wr32(E1000_SPQF(3), htons(PTP_PORT));
6449 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6450 }
6451 wr32(E1000_FTQF(3), ftqf);
6452 } else {
6453 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6454 }
33af6bcc
PO		 6455 wrfl();
6456
6457 adapter->hwtstamp_config = config;
6458
6459 /* clear TX/RX time stamp registers, just to be sure */
6460 regval = rd32(E1000_TXSTMPH);
6461 regval = rd32(E1000_RXSTMPH);
c6cb090b 6462
33af6bcc
PO		 6463 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6464 -EFAULT : 0;
c6cb090b
PO		 6465}
6466
9d5c8243
AK		 6467/**
6468 * igb_ioctl -
6469 * @netdev:
6470 * @ifreq:
6471 * @cmd:
6472 **/
6473static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6474{
6475 switch (cmd) {
6476 case SIOCGMIIPHY:
6477 case SIOCGMIIREG:
6478 case SIOCSMIIREG:
6479 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6480 case SIOCSHWTSTAMP:
6481 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6482 default:
6483 return -EOPNOTSUPP;
6484 }
6485}
6486
009bc06e
AD		 6487s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6488{
6489 struct igb_adapter *adapter = hw->back;
6490 u16 cap_offset;
6491
bdaae04c 6492 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6493 if (!cap_offset)
6494 return -E1000_ERR_CONFIG;
6495
6496 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6497
6498 return 0;
6499}
6500
6501s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6502{
6503 struct igb_adapter *adapter = hw->back;
6504 u16 cap_offset;
6505
bdaae04c 6506 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6507 if (!cap_offset)
6508 return -E1000_ERR_CONFIG;
6509
6510 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6511
6512 return 0;
6513}
6514
b2cb09b1 6515static void igb_vlan_mode(struct net_device *netdev, u32 features)
9d5c8243
AK		 6516{
6517 struct igb_adapter *adapter = netdev_priv(netdev);
6518 struct e1000_hw *hw = &adapter->hw;
6519 u32 ctrl, rctl;
5faf030c 6520 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
9d5c8243 6521
5faf030c 6522 if (enable) {
9d5c8243
AK		 6523 /* enable VLAN tag insert/strip */
6524 ctrl = rd32(E1000_CTRL);
6525 ctrl |= E1000_CTRL_VME;
6526 wr32(E1000_CTRL, ctrl);
6527
51466239 6528 /* Disable CFI check */
9d5c8243 6529 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6530 rctl &= ~E1000_RCTL_CFIEN;
6531 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6532 } else {
6533 /* disable VLAN tag insert/strip */
6534 ctrl = rd32(E1000_CTRL);
6535 ctrl &= ~E1000_CTRL_VME;
6536 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6537 }
6538
e1739522 6539 igb_rlpml_set(adapter);
9d5c8243
AK		 6540}
6541
6542static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6543{
6544 struct igb_adapter *adapter = netdev_priv(netdev);
6545 struct e1000_hw *hw = &adapter->hw;
4ae196df 6546 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6547
51466239
AD		 6548 /* attempt to add filter to vlvf array */
6549 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6550
51466239
AD		 6551 /* add the filter since PF can receive vlans w/o entry in vlvf */
6552 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6553
6554 set_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6555}
6556
6557static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6558{
6559 struct igb_adapter *adapter = netdev_priv(netdev);
6560 struct e1000_hw *hw = &adapter->hw;
4ae196df 6561 int pf_id = adapter->vfs_allocated_count;
51466239 6562 s32 err;
9d5c8243 6563
51466239
AD		 6564 /* remove vlan from VLVF table array */
6565 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6566
51466239
AD		 6567 /* if vid was not present in VLVF just remove it from table */
6568 if (err)
4ae196df 6569 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6570
6571 clear_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6572}
6573
6574static void igb_restore_vlan(struct igb_adapter *adapter)
6575{
b2cb09b1 6576 u16 vid;
9d5c8243 6577
5faf030c
AD		 6578 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6579
b2cb09b1
JP		 6580 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6581 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6582}
6583
14ad2513 6584int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6585{
090b1795 6586 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6587 struct e1000_mac_info *mac = &adapter->hw.mac;
6588
6589 mac->autoneg = 0;
6590
14ad2513
DD		 6591 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6592 * for the switch() below to work */
6593 if ((spd & 1) || (dplx & ~1))
6594 goto err_inval;
6595
cd2638a8
CW		 6596 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6597 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6598 spd != SPEED_1000 &&
6599 dplx != DUPLEX_FULL)
6600 goto err_inval;
cd2638a8 6601
14ad2513 6602 switch (spd + dplx) {
9d5c8243
AK		 6603 case SPEED_10 + DUPLEX_HALF:
6604 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6605 break;
6606 case SPEED_10 + DUPLEX_FULL:
6607 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6608 break;
6609 case SPEED_100 + DUPLEX_HALF:
6610 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6611 break;
6612 case SPEED_100 + DUPLEX_FULL:
6613 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6614 break;
6615 case SPEED_1000 + DUPLEX_FULL:
6616 mac->autoneg = 1;
6617 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6618 break;
6619 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6620 default:
14ad2513 6621 goto err_inval;
9d5c8243
AK		 6622 }
6623 return 0;
14ad2513
DD		 6624
6625err_inval:
6626 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6627 return -EINVAL;
9d5c8243
AK		 6628}
6629
3fe7c4c9 6630static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6631{
6632 struct net_device *netdev = pci_get_drvdata(pdev);
6633 struct igb_adapter *adapter = netdev_priv(netdev);
6634 struct e1000_hw *hw = &adapter->hw;
2d064c06 6635 u32 ctrl, rctl, status;
9d5c8243
AK		 6636 u32 wufc = adapter->wol;
6637#ifdef CONFIG_PM
6638 int retval = 0;
6639#endif
6640
6641 netif_device_detach(netdev);
6642
a88f10ec
AD		 6643 if (netif_running(netdev))
6644 igb_close(netdev);
6645
047e0030 6646 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6647
6648#ifdef CONFIG_PM
6649 retval = pci_save_state(pdev);
6650 if (retval)
6651 return retval;
6652#endif
6653
6654 status = rd32(E1000_STATUS);
6655 if (status & E1000_STATUS_LU)
6656 wufc &= ~E1000_WUFC_LNKC;
6657
6658 if (wufc) {
6659 igb_setup_rctl(adapter);
ff41f8dc 6660 igb_set_rx_mode(netdev);
9d5c8243
AK		 6661
6662 /* turn on all-multi mode if wake on multicast is enabled */
6663 if (wufc & E1000_WUFC_MC) {
6664 rctl = rd32(E1000_RCTL);
6665 rctl |= E1000_RCTL_MPE;
6666 wr32(E1000_RCTL, rctl);
6667 }
6668
6669 ctrl = rd32(E1000_CTRL);
6670 /* advertise wake from D3Cold */
6671 #define E1000_CTRL_ADVD3WUC 0x00100000
6672 /* phy power management enable */
6673 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6674 ctrl |= E1000_CTRL_ADVD3WUC;
6675 wr32(E1000_CTRL, ctrl);
6676
9d5c8243 6677 /* Allow time for pending master requests to run */
330a6d6a 6678 igb_disable_pcie_master(hw);
9d5c8243
AK		 6679
6680 wr32(E1000_WUC, E1000_WUC_PME_EN);
6681 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6682 } else {
6683 wr32(E1000_WUC, 0);
6684 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6685 }
6686
3fe7c4c9
RW		 6687 *enable_wake = wufc || adapter->en_mng_pt;
6688 if (!*enable_wake)
88a268c1
NN		 6689 igb_power_down_link(adapter);
6690 else
6691 igb_power_up_link(adapter);
9d5c8243
AK		 6692
6693 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6694 * would have already happened in close and is redundant. */
6695 igb_release_hw_control(adapter);
6696
6697 pci_disable_device(pdev);
6698
9d5c8243
AK		 6699 return 0;
6700}
6701
6702#ifdef CONFIG_PM
3fe7c4c9
RW		 6703static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6704{
6705 int retval;
6706 bool wake;
6707
6708 retval = __igb_shutdown(pdev, &wake);
6709 if (retval)
6710 return retval;
6711
6712 if (wake) {
6713 pci_prepare_to_sleep(pdev);
6714 } else {
6715 pci_wake_from_d3(pdev, false);
6716 pci_set_power_state(pdev, PCI_D3hot);
6717 }
6718
6719 return 0;
6720}
6721
9d5c8243
AK		 6722static int igb_resume(struct pci_dev *pdev)
6723{
6724 struct net_device *netdev = pci_get_drvdata(pdev);
6725 struct igb_adapter *adapter = netdev_priv(netdev);
6726 struct e1000_hw *hw = &adapter->hw;
6727 u32 err;
6728
6729 pci_set_power_state(pdev, PCI_D0);
6730 pci_restore_state(pdev);
b94f2d77 6731 pci_save_state(pdev);
42bfd33a 6732
aed5dec3 6733 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6734 if (err) {
6735 dev_err(&pdev->dev,
6736 "igb: Cannot enable PCI device from suspend\n");
6737 return err;
6738 }
6739 pci_set_master(pdev);
6740
6741 pci_enable_wake(pdev, PCI_D3hot, 0);
6742 pci_enable_wake(pdev, PCI_D3cold, 0);
6743
047e0030 6744 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6745 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6746 return -ENOMEM;
9d5c8243
AK		 6747 }
6748
9d5c8243 6749 igb_reset(adapter);
a8564f03
AD		 6750
6751 /* let the f/w know that the h/w is now under the control of the
6752 * driver. */
6753 igb_get_hw_control(adapter);
6754
9d5c8243
AK		 6755 wr32(E1000_WUS, ~0);
6756
a88f10ec
AD		 6757 if (netif_running(netdev)) {
6758 err = igb_open(netdev);
6759 if (err)
6760 return err;
6761 }
9d5c8243
AK		 6762
6763 netif_device_attach(netdev);
6764
9d5c8243
AK		 6765 return 0;
6766}
6767#endif
6768
6769static void igb_shutdown(struct pci_dev *pdev)
6770{
3fe7c4c9
RW		 6771 bool wake;
6772
6773 __igb_shutdown(pdev, &wake);
6774
6775 if (system_state == SYSTEM_POWER_OFF) {
6776 pci_wake_from_d3(pdev, wake);
6777 pci_set_power_state(pdev, PCI_D3hot);
6778 }
9d5c8243
AK		 6779}
6780
6781#ifdef CONFIG_NET_POLL_CONTROLLER
6782/*
6783 * Polling 'interrupt' - used by things like netconsole to send skbs
6784 * without having to re-enable interrupts. It's not called while
6785 * the interrupt routine is executing.
6786 */
6787static void igb_netpoll(struct net_device *netdev)
6788{
6789 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6790 struct e1000_hw *hw = &adapter->hw;
0d1ae7f4 6791 struct igb_q_vector *q_vector;
9d5c8243 6792 int i;
9d5c8243 6793
047e0030 6794 for (i = 0; i < adapter->num_q_vectors; i++) {
0d1ae7f4
AD		 6795 q_vector = adapter->q_vector[i];
6796 if (adapter->msix_entries)
6797 wr32(E1000_EIMC, q_vector->eims_value);
6798 else
6799 igb_irq_disable(adapter);
047e0030 6800 napi_schedule(&q_vector->napi);
eebbbdba 6801 }
9d5c8243
AK		 6802}
6803#endif /* CONFIG_NET_POLL_CONTROLLER */
6804
6805/**
6806 * igb_io_error_detected - called when PCI error is detected
6807 * @pdev: Pointer to PCI device
6808 * @state: The current pci connection state
6809 *
6810 * This function is called after a PCI bus error affecting
6811 * this device has been detected.
6812 */
6813static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6814 pci_channel_state_t state)
6815{
6816 struct net_device *netdev = pci_get_drvdata(pdev);
6817 struct igb_adapter *adapter = netdev_priv(netdev);
6818
6819 netif_device_detach(netdev);
6820
59ed6eec
AD		 6821 if (state == pci_channel_io_perm_failure)
6822 return PCI_ERS_RESULT_DISCONNECT;
6823
9d5c8243
AK		 6824 if (netif_running(netdev))
6825 igb_down(adapter);
6826 pci_disable_device(pdev);
6827
6828 /* Request a slot slot reset. */
6829 return PCI_ERS_RESULT_NEED_RESET;
6830}
6831
6832/**
6833 * igb_io_slot_reset - called after the pci bus has been reset.
6834 * @pdev: Pointer to PCI device
6835 *
6836 * Restart the card from scratch, as if from a cold-boot. Implementation
6837 * resembles the first-half of the igb_resume routine.
6838 */
6839static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6840{
6841 struct net_device *netdev = pci_get_drvdata(pdev);
6842 struct igb_adapter *adapter = netdev_priv(netdev);
6843 struct e1000_hw *hw = &adapter->hw;
40a914fa 6844 pci_ers_result_t result;
42bfd33a 6845 int err;
9d5c8243 6846
aed5dec3 6847 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6848 dev_err(&pdev->dev,
6849 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6850 result = PCI_ERS_RESULT_DISCONNECT;
6851 } else {
6852 pci_set_master(pdev);
6853 pci_restore_state(pdev);
b94f2d77 6854 pci_save_state(pdev);
9d5c8243 6855
40a914fa
AD		 6856 pci_enable_wake(pdev, PCI_D3hot, 0);
6857 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6858
40a914fa
AD		 6859 igb_reset(adapter);
6860 wr32(E1000_WUS, ~0);
6861 result = PCI_ERS_RESULT_RECOVERED;
6862 }
9d5c8243 6863
ea943d41
JK		 6864 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6865 if (err) {
6866 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6867 "failed 0x%0x\n", err);
6868 /* non-fatal, continue */
6869 }
40a914fa
AD		 6870
6871 return result;
9d5c8243
AK		 6872}
6873
6874/**
6875 * igb_io_resume - called when traffic can start flowing again.
6876 * @pdev: Pointer to PCI device
6877 *
6878 * This callback is called when the error recovery driver tells us that
6879 * its OK to resume normal operation. Implementation resembles the
6880 * second-half of the igb_resume routine.
6881 */
6882static void igb_io_resume(struct pci_dev *pdev)
6883{
6884 struct net_device *netdev = pci_get_drvdata(pdev);
6885 struct igb_adapter *adapter = netdev_priv(netdev);
6886
9d5c8243
AK		 6887 if (netif_running(netdev)) {
6888 if (igb_up(adapter)) {
6889 dev_err(&pdev->dev, "igb_up failed after reset\n");
6890 return;
6891 }
6892 }
6893
6894 netif_device_attach(netdev);
6895
6896 /* let the f/w know that the h/w is now under the control of the
6897 * driver. */
6898 igb_get_hw_control(adapter);
9d5c8243
AK		 6899}
6900
26ad9178
AD		 6901static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6902 u8 qsel)
6903{
6904 u32 rar_low, rar_high;
6905 struct e1000_hw *hw = &adapter->hw;
6906
6907 /* HW expects these in little endian so we reverse the byte order
6908 * from network order (big endian) to little endian
6909 */
6910 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6911 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6912 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6913
6914 /* Indicate to hardware the Address is Valid. */
6915 rar_high |= E1000_RAH_AV;
6916
6917 if (hw->mac.type == e1000_82575)
6918 rar_high |= E1000_RAH_POOL_1 * qsel;
6919 else
6920 rar_high |= E1000_RAH_POOL_1 << qsel;
6921
6922 wr32(E1000_RAL(index), rar_low);
6923 wrfl();
6924 wr32(E1000_RAH(index), rar_high);
6925 wrfl();
6926}
6927
4ae196df
AD		 6928static int igb_set_vf_mac(struct igb_adapter *adapter,
6929 int vf, unsigned char *mac_addr)
6930{
6931 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6932 /* VF MAC addresses start at end of receive addresses and moves
6933 * torwards the first, as a result a collision should not be possible */
6934 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6935
37680117 6936 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6937
26ad9178 6938 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6939
6940 return 0;
6941}
6942
8151d294
WM		 6943static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6944{
6945 struct igb_adapter *adapter = netdev_priv(netdev);
6946 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6947 return -EINVAL;
6948 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6949 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6950 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6951 " change effective.");
6952 if (test_bit(__IGB_DOWN, &adapter->state)) {
6953 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6954 " but the PF device is not up.\n");
6955 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6956 " attempting to use the VF device.\n");
6957 }
6958 return igb_set_vf_mac(adapter, vf, mac);
6959}
6960
17dc566c
LL		 6961static int igb_link_mbps(int internal_link_speed)
6962{
6963 switch (internal_link_speed) {
6964 case SPEED_100:
6965 return 100;
6966 case SPEED_1000:
6967 return 1000;
6968 default:
6969 return 0;
6970 }
6971}
6972
6973static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6974 int link_speed)
6975{
6976 int rf_dec, rf_int;
6977 u32 bcnrc_val;
6978
6979 if (tx_rate != 0) {
6980 /* Calculate the rate factor values to set */
6981 rf_int = link_speed / tx_rate;
6982 rf_dec = (link_speed - (rf_int * tx_rate));
6983 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6984
6985 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6986 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6987 E1000_RTTBCNRC_RF_INT_MASK);
6988 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6989 } else {
6990 bcnrc_val = 0;
6991 }
6992
6993 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6994 wr32(E1000_RTTBCNRC, bcnrc_val);
6995}
6996
6997static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6998{
6999 int actual_link_speed, i;
7000 bool reset_rate = false;
7001
7002 /* VF TX rate limit was not set or not supported */
7003 if ((adapter->vf_rate_link_speed == 0) ||
7004 (adapter->hw.mac.type != e1000_82576))
7005 return;
7006
7007 actual_link_speed = igb_link_mbps(adapter->link_speed);
7008 if (actual_link_speed != adapter->vf_rate_link_speed) {
7009 reset_rate = true;
7010 adapter->vf_rate_link_speed = 0;
7011 dev_info(&adapter->pdev->dev,
7012 "Link speed has been changed. VF Transmit "
7013 "rate is disabled\n");
7014 }
7015
7016 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7017 if (reset_rate)
7018 adapter->vf_data[i].tx_rate = 0;
7019
7020 igb_set_vf_rate_limit(&adapter->hw, i,
7021 adapter->vf_data[i].tx_rate,
7022 actual_link_speed);
7023 }
7024}
7025
8151d294
WM		 7026static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7027{
17dc566c
LL		 7028 struct igb_adapter *adapter = netdev_priv(netdev);
7029 struct e1000_hw *hw = &adapter->hw;
7030 int actual_link_speed;
7031
7032 if (hw->mac.type != e1000_82576)
7033 return -EOPNOTSUPP;
7034
7035 actual_link_speed = igb_link_mbps(adapter->link_speed);
7036 if ((vf >= adapter->vfs_allocated_count) ||
7037 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7038 (tx_rate < 0) || (tx_rate > actual_link_speed))
7039 return -EINVAL;
7040
7041 adapter->vf_rate_link_speed = actual_link_speed;
7042 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7043 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7044
7045 return 0;
8151d294
WM		 7046}
7047
7048static int igb_ndo_get_vf_config(struct net_device *netdev,
7049 int vf, struct ifla_vf_info *ivi)
7050{
7051 struct igb_adapter *adapter = netdev_priv(netdev);
7052 if (vf >= adapter->vfs_allocated_count)
7053 return -EINVAL;
7054 ivi->vf = vf;
7055 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 7056 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 7057 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7058 ivi->qos = adapter->vf_data[vf].pf_qos;
7059 return 0;
7060}
7061
4ae196df
AD		 7062static void igb_vmm_control(struct igb_adapter *adapter)
7063{
7064 struct e1000_hw *hw = &adapter->hw;
10d8e907 7065 u32 reg;
4ae196df 7066
52a1dd4d
AD		 7067 switch (hw->mac.type) {
7068 case e1000_82575:
7069 default:
7070 /* replication is not supported for 82575 */
4ae196df 7071 return;
52a1dd4d
AD		 7072 case e1000_82576:
7073 /* notify HW that the MAC is adding vlan tags */
7074 reg = rd32(E1000_DTXCTL);
7075 reg |= E1000_DTXCTL_VLAN_ADDED;
7076 wr32(E1000_DTXCTL, reg);
7077 case e1000_82580:
7078 /* enable replication vlan tag stripping */
7079 reg = rd32(E1000_RPLOLR);
7080 reg |= E1000_RPLOLR_STRVLAN;
7081 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 7082 case e1000_i350:
7083 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 7084 break;
7085 }
10d8e907 7086
d4960307
AD		 7087 if (adapter->vfs_allocated_count) {
7088 igb_vmdq_set_loopback_pf(hw, true);
7089 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 7090 igb_vmdq_set_anti_spoofing_pf(hw, true,
7091 adapter->vfs_allocated_count);
d4960307
AD		 7092 } else {
7093 igb_vmdq_set_loopback_pf(hw, false);
7094 igb_vmdq_set_replication_pf(hw, false);
7095 }
4ae196df
AD		 7096}
7097
9d5c8243 7098/* igb_main.c */
kernel source for telekom puls (alcatel/mediatek)