projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: rework handling of the vfta and vlvf registers in relation to mng_vlan
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
86d5d38f 52#define DRV_VERSION "1.3.16-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
63static struct pci_device_id igb_pci_tbl[] = {
2d064c06 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
75 {0, }
76};
77
78MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
79
80void igb_reset(struct igb_adapter *);
81static int igb_setup_all_tx_resources(struct igb_adapter *);
82static int igb_setup_all_rx_resources(struct igb_adapter *);
83static void igb_free_all_tx_resources(struct igb_adapter *);
84static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 85static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 86void igb_update_stats(struct igb_adapter *);
87static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88static void __devexit igb_remove(struct pci_dev *pdev);
89static int igb_sw_init(struct igb_adapter *);
90static int igb_open(struct net_device *);
91static int igb_close(struct net_device *);
92static void igb_configure_tx(struct igb_adapter *);
93static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 94static void igb_clean_all_tx_rings(struct igb_adapter *);
95static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 96static void igb_clean_tx_ring(struct igb_ring *);
97static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 98static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 99static void igb_update_phy_info(unsigned long);
100static void igb_watchdog(unsigned long);
101static void igb_watchdog_task(struct work_struct *);
b1a436c3 102static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 103static struct net_device_stats *igb_get_stats(struct net_device *);
104static int igb_change_mtu(struct net_device *, int);
105static int igb_set_mac(struct net_device *, void *);
68d480c4 106static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 107static irqreturn_t igb_intr(int irq, void *);
108static irqreturn_t igb_intr_msi(int irq, void *);
109static irqreturn_t igb_msix_other(int irq, void *);
047e0030 110static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 111#ifdef CONFIG_IGB_DCA
047e0030 112static void igb_update_dca(struct igb_q_vector *);
fe4506b6 113static void igb_setup_dca(struct igb_adapter *);
421e02f0 114#endif /* CONFIG_IGB_DCA */
047e0030 115static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 116static int igb_poll(struct napi_struct *, int);
047e0030 117static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 118static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
119static void igb_tx_timeout(struct net_device *);
120static void igb_reset_task(struct work_struct *);
121static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
122static void igb_vlan_rx_add_vid(struct net_device *, u16);
123static void igb_vlan_rx_kill_vid(struct net_device *, u16);
124static void igb_restore_vlan(struct igb_adapter *);
26ad9178 125static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 126static void igb_ping_all_vfs(struct igb_adapter *);
127static void igb_msg_task(struct igb_adapter *);
4ae196df 128static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 129static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 130static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
9d5c8243 131
9d5c8243 132#ifdef CONFIG_PM
3fe7c4c9 133static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 134static int igb_resume(struct pci_dev *);
135#endif
136static void igb_shutdown(struct pci_dev *);
421e02f0 137#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 138static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
139static struct notifier_block dca_notifier = {
140 .notifier_call = igb_notify_dca,
141 .next = NULL,
142 .priority = 0
143};
144#endif
9d5c8243
AK		 145#ifdef CONFIG_NET_POLL_CONTROLLER
146/* for netdump / net console */
147static void igb_netpoll(struct net_device *);
148#endif
37680117 149#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 150static unsigned int max_vfs = 0;
151module_param(max_vfs, uint, 0);
152MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
153 "per physical function");
154#endif /* CONFIG_PCI_IOV */
155
9d5c8243
AK		 156static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
157 pci_channel_state_t);
158static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
159static void igb_io_resume(struct pci_dev *);
160
161static struct pci_error_handlers igb_err_handler = {
162 .error_detected = igb_io_error_detected,
163 .slot_reset = igb_io_slot_reset,
164 .resume = igb_io_resume,
165};
166
167
168static struct pci_driver igb_driver = {
169 .name = igb_driver_name,
170 .id_table = igb_pci_tbl,
171 .probe = igb_probe,
172 .remove = __devexit_p(igb_remove),
173#ifdef CONFIG_PM
174 /* Power Managment Hooks */
175 .suspend = igb_suspend,
176 .resume = igb_resume,
177#endif
178 .shutdown = igb_shutdown,
179 .err_handler = &igb_err_handler
180};
181
7dfc16fa
AD		 182static int global_quad_port_a; /* global quad port a indication */
183
9d5c8243
AK		 184MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
185MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
186MODULE_LICENSE("GPL");
187MODULE_VERSION(DRV_VERSION);
188
38c845c7
PO		 189/**
190 * igb_read_clock - read raw cycle counter (to be used by time counter)
191 */
192static cycle_t igb_read_clock(const struct cyclecounter *tc)
193{
194 struct igb_adapter *adapter =
195 container_of(tc, struct igb_adapter, cycles);
196 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 197 u64 stamp = 0;
198 int shift = 0;
38c845c7 199
c5b9bd5e
AD		 200 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
201 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 202 return stamp;
203}
204
9d5c8243
AK		 205#ifdef DEBUG
206/**
207 * igb_get_hw_dev_name - return device name string
208 * used by hardware layer to print debugging information
209 **/
210char *igb_get_hw_dev_name(struct e1000_hw *hw)
211{
212 struct igb_adapter *adapter = hw->back;
213 return adapter->netdev->name;
214}
38c845c7
PO		 215
216/**
217 * igb_get_time_str - format current NIC and system time as string
218 */
219static char *igb_get_time_str(struct igb_adapter *adapter,
220 char buffer[160])
221{
222 cycle_t hw = adapter->cycles.read(&adapter->cycles);
223 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
224 struct timespec sys;
225 struct timespec delta;
226 getnstimeofday(&sys);
227
228 delta = timespec_sub(nic, sys);
229
230 sprintf(buffer,
33af6bcc
PO		 231 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
232 hw,
38c845c7
PO		 233 (long)nic.tv_sec, nic.tv_nsec,
234 (long)sys.tv_sec, sys.tv_nsec,
235 (long)delta.tv_sec, delta.tv_nsec);
236
237 return buffer;
238}
9d5c8243
AK		 239#endif
240
241/**
242 * igb_init_module - Driver Registration Routine
243 *
244 * igb_init_module is the first routine called when the driver is
245 * loaded. All it does is register with the PCI subsystem.
246 **/
247static int __init igb_init_module(void)
248{
249 int ret;
250 printk(KERN_INFO "%s - version %s\n",
251 igb_driver_string, igb_driver_version);
252
253 printk(KERN_INFO "%s\n", igb_copyright);
254
7dfc16fa
AD		 255 global_quad_port_a = 0;
256
421e02f0 257#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 258 dca_register_notify(&dca_notifier);
259#endif
bbd98fe4
AD		 260
261 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 262 return ret;
263}
264
265module_init(igb_init_module);
266
267/**
268 * igb_exit_module - Driver Exit Cleanup Routine
269 *
270 * igb_exit_module is called just before the driver is removed
271 * from memory.
272 **/
273static void __exit igb_exit_module(void)
274{
421e02f0 275#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 276 dca_unregister_notify(&dca_notifier);
277#endif
9d5c8243
AK		 278 pci_unregister_driver(&igb_driver);
279}
280
281module_exit(igb_exit_module);
282
26bc19ec
AD		 283#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
284/**
285 * igb_cache_ring_register - Descriptor ring to register mapping
286 * @adapter: board private structure to initialize
287 *
288 * Once we know the feature-set enabled for the device, we'll cache
289 * the register offset the descriptor ring is assigned to.
290 **/
291static void igb_cache_ring_register(struct igb_adapter *adapter)
292{
293 int i;
047e0030 294 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 295
296 switch (adapter->hw.mac.type) {
297 case e1000_82576:
298 /* The queues are allocated for virtualization such that VF 0
299 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
300 * In order to avoid collision we start at the first free queue
301 * and continue consuming queues in the same sequence
302 */
303 for (i = 0; i < adapter->num_rx_queues; i++)
1bfaf07b
AD		 304 adapter->rx_ring[i].reg_idx = rbase_offset +
305 Q_IDX_82576(i);
26bc19ec 306 for (i = 0; i < adapter->num_tx_queues; i++)
1bfaf07b
AD		 307 adapter->tx_ring[i].reg_idx = rbase_offset +
308 Q_IDX_82576(i);
26bc19ec
AD		 309 break;
310 case e1000_82575:
311 default:
312 for (i = 0; i < adapter->num_rx_queues; i++)
313 adapter->rx_ring[i].reg_idx = i;
314 for (i = 0; i < adapter->num_tx_queues; i++)
315 adapter->tx_ring[i].reg_idx = i;
316 break;
317 }
318}
319
047e0030
AD		 320static void igb_free_queues(struct igb_adapter *adapter)
321{
322 kfree(adapter->tx_ring);
323 kfree(adapter->rx_ring);
324
325 adapter->tx_ring = NULL;
326 adapter->rx_ring = NULL;
327
328 adapter->num_rx_queues = 0;
329 adapter->num_tx_queues = 0;
330}
331
9d5c8243
AK		 332/**
333 * igb_alloc_queues - Allocate memory for all rings
334 * @adapter: board private structure to initialize
335 *
336 * We allocate one ring per queue at run-time since we don't know the
337 * number of queues at compile-time.
338 **/
339static int igb_alloc_queues(struct igb_adapter *adapter)
340{
341 int i;
342
343 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
344 sizeof(struct igb_ring), GFP_KERNEL);
345 if (!adapter->tx_ring)
047e0030 346 goto err;
9d5c8243
AK		 347
348 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
349 sizeof(struct igb_ring), GFP_KERNEL);
047e0030
AD		 350 if (!adapter->rx_ring)
351 goto err;
6eb5a7f1 352
661086df
PWJ		 353 for (i = 0; i < adapter->num_tx_queues; i++) {
354 struct igb_ring *ring = &(adapter->tx_ring[i]);
68fd9910 355 ring->count = adapter->tx_ring_count;
661086df 356 ring->queue_index = i;
80785298 357 ring->pdev = adapter->pdev;
e694e964 358 ring->netdev = adapter->netdev;
85ad76b2
AD		 359 /* For 82575, context index must be unique per ring. */
360 if (adapter->hw.mac.type == e1000_82575)
361 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
661086df 362 }
85ad76b2 363
9d5c8243
AK		 364 for (i = 0; i < adapter->num_rx_queues; i++) {
365 struct igb_ring *ring = &(adapter->rx_ring[i]);
68fd9910 366 ring->count = adapter->rx_ring_count;
844290e5 367 ring->queue_index = i;
80785298 368 ring->pdev = adapter->pdev;
e694e964 369 ring->netdev = adapter->netdev;
4c844851 370 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 371 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
372 /* set flag indicating ring supports SCTP checksum offload */
373 if (adapter->hw.mac.type >= e1000_82576)
374 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
9d5c8243 375 }
26bc19ec
AD		 376
377 igb_cache_ring_register(adapter);
9d5c8243 378
047e0030 379 return 0;
a88f10ec 380
047e0030
AD		 381err:
382 igb_free_queues(adapter);
d1a8c9e1 383
047e0030 384 return -ENOMEM;
a88f10ec
AD		 385}
386
9d5c8243 387#define IGB_N0_QUEUE -1
047e0030 388static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 389{
390 u32 msixbm = 0;
047e0030 391 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 392 struct e1000_hw *hw = &adapter->hw;
2d064c06 393 u32 ivar, index;
047e0030
AD		 394 int rx_queue = IGB_N0_QUEUE;
395 int tx_queue = IGB_N0_QUEUE;
396
397 if (q_vector->rx_ring)
398 rx_queue = q_vector->rx_ring->reg_idx;
399 if (q_vector->tx_ring)
400 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 401
402 switch (hw->mac.type) {
403 case e1000_82575:
9d5c8243
AK		 404 /* The 82575 assigns vectors using a bitmask, which matches the
405 bitmask for the EICR/EIMS/EIMC registers. To assign one
406 or more queues to a vector, we write the appropriate bits
407 into the MSIXBM register for that vector. */
047e0030 408 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 409 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 410 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 411 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
9d5c8243 412 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 413 q_vector->eims_value = msixbm;
2d064c06
AD		 414 break;
415 case e1000_82576:
26bc19ec 416 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 417 Each queue has a single entry in the table to which we write
418 a vector number along with a "valid" bit. Sadly, the layout
419 of the table is somewhat counterintuitive. */
420 if (rx_queue > IGB_N0_QUEUE) {
047e0030 421 index = (rx_queue & 0x7);
2d064c06 422 ivar = array_rd32(E1000_IVAR0, index);
047e0030 423 if (rx_queue < 8) {
26bc19ec
AD		 424 /* vector goes into low byte of register */
425 ivar = ivar & 0xFFFFFF00;
426 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 427 } else {
428 /* vector goes into third byte of register */
429 ivar = ivar & 0xFF00FFFF;
430 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 431 }
2d064c06
AD		 432 array_wr32(E1000_IVAR0, index, ivar);
433 }
434 if (tx_queue > IGB_N0_QUEUE) {
047e0030 435 index = (tx_queue & 0x7);
2d064c06 436 ivar = array_rd32(E1000_IVAR0, index);
047e0030 437 if (tx_queue < 8) {
26bc19ec
AD		 438 /* vector goes into second byte of register */
439 ivar = ivar & 0xFFFF00FF;
440 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 441 } else {
442 /* vector goes into high byte of register */
443 ivar = ivar & 0x00FFFFFF;
444 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 445 }
2d064c06
AD		 446 array_wr32(E1000_IVAR0, index, ivar);
447 }
047e0030 448 q_vector->eims_value = 1 << msix_vector;
2d064c06
AD		 449 break;
450 default:
451 BUG();
452 break;
453 }
9d5c8243
AK		 454}
455
456/**
457 * igb_configure_msix - Configure MSI-X hardware
458 *
459 * igb_configure_msix sets up the hardware to properly
460 * generate MSI-X interrupts.
461 **/
462static void igb_configure_msix(struct igb_adapter *adapter)
463{
464 u32 tmp;
465 int i, vector = 0;
466 struct e1000_hw *hw = &adapter->hw;
467
468 adapter->eims_enable_mask = 0;
9d5c8243
AK		 469
470 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 471 switch (hw->mac.type) {
472 case e1000_82575:
9d5c8243
AK		 473 tmp = rd32(E1000_CTRL_EXT);
474 /* enable MSI-X PBA support*/
475 tmp |= E1000_CTRL_EXT_PBA_CLR;
476
477 /* Auto-Mask interrupts upon ICR read. */
478 tmp |= E1000_CTRL_EXT_EIAME;
479 tmp |= E1000_CTRL_EXT_IRCA;
480
481 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 482
483 /* enable msix_other interrupt */
484 array_wr32(E1000_MSIXBM(0), vector++,
485 E1000_EIMS_OTHER);
844290e5 486 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 487
2d064c06
AD		 488 break;
489
490 case e1000_82576:
047e0030
AD		 491 /* Turn on MSI-X capability first, or our settings
492 * won't stick. And it will take days to debug. */
493 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
494 E1000_GPIE_PBA | E1000_GPIE_EIAME |
495 E1000_GPIE_NSICR);
496
497 /* enable msix_other interrupt */
498 adapter->eims_other = 1 << vector;
2d064c06 499 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 500
047e0030 501 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 502 break;
503 default:
504 /* do nothing, since nothing else supports MSI-X */
505 break;
506 } /* switch (hw->mac.type) */
047e0030
AD		 507
508 adapter->eims_enable_mask |= adapter->eims_other;
509
510 for (i = 0; i < adapter->num_q_vectors; i++) {
511 struct igb_q_vector *q_vector = adapter->q_vector[i];
512 igb_assign_vector(q_vector, vector++);
513 adapter->eims_enable_mask |= q_vector->eims_value;
514 }
515
9d5c8243
AK		 516 wrfl();
517}
518
519/**
520 * igb_request_msix - Initialize MSI-X interrupts
521 *
522 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
523 * kernel.
524 **/
525static int igb_request_msix(struct igb_adapter *adapter)
526{
527 struct net_device *netdev = adapter->netdev;
047e0030 528 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 529 int i, err = 0, vector = 0;
530
047e0030
AD		 531 err = request_irq(adapter->msix_entries[vector].vector,
532 &igb_msix_other, 0, netdev->name, adapter);
533 if (err)
534 goto out;
535 vector++;
536
537 for (i = 0; i < adapter->num_q_vectors; i++) {
538 struct igb_q_vector *q_vector = adapter->q_vector[i];
539
540 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
541
542 if (q_vector->rx_ring && q_vector->tx_ring)
543 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
544 q_vector->rx_ring->queue_index);
545 else if (q_vector->tx_ring)
546 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
547 q_vector->tx_ring->queue_index);
548 else if (q_vector->rx_ring)
549 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
550 q_vector->rx_ring->queue_index);
9d5c8243 551 else
047e0030
AD		 552 sprintf(q_vector->name, "%s-unused", netdev->name);
553
9d5c8243 554 err = request_irq(adapter->msix_entries[vector].vector,
047e0030
AD		 555 &igb_msix_ring, 0, q_vector->name,
556 q_vector);
9d5c8243
AK		 557 if (err)
558 goto out;
9d5c8243
AK		 559 vector++;
560 }
561
9d5c8243
AK		 562 igb_configure_msix(adapter);
563 return 0;
564out:
565 return err;
566}
567
568static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
569{
570 if (adapter->msix_entries) {
571 pci_disable_msix(adapter->pdev);
572 kfree(adapter->msix_entries);
573 adapter->msix_entries = NULL;
047e0030 574 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 575 pci_disable_msi(adapter->pdev);
047e0030 576 }
9d5c8243
AK		 577}
578
047e0030
AD		 579/**
580 * igb_free_q_vectors - Free memory allocated for interrupt vectors
581 * @adapter: board private structure to initialize
582 *
583 * This function frees the memory allocated to the q_vectors. In addition if
584 * NAPI is enabled it will delete any references to the NAPI struct prior
585 * to freeing the q_vector.
586 **/
587static void igb_free_q_vectors(struct igb_adapter *adapter)
588{
589 int v_idx;
590
591 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
592 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
593 adapter->q_vector[v_idx] = NULL;
594 netif_napi_del(&q_vector->napi);
595 kfree(q_vector);
596 }
597 adapter->num_q_vectors = 0;
598}
599
600/**
601 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
602 *
603 * This function resets the device so that it has 0 rx queues, tx queues, and
604 * MSI-X interrupts allocated.
605 */
606static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
607{
608 igb_free_queues(adapter);
609 igb_free_q_vectors(adapter);
610 igb_reset_interrupt_capability(adapter);
611}
9d5c8243
AK		 612
613/**
614 * igb_set_interrupt_capability - set MSI or MSI-X if supported
615 *
616 * Attempt to configure interrupts using the best available
617 * capabilities of the hardware and kernel.
618 **/
619static void igb_set_interrupt_capability(struct igb_adapter *adapter)
620{
621 int err;
622 int numvecs, i;
623
83b7180d 624 /* Number of supported queues. */
83b7180d
AD		 625 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
626 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
627
047e0030
AD		 628 /* start with one vector for every rx queue */
629 numvecs = adapter->num_rx_queues;
630
631 /* if tx handler is seperate add 1 for every tx queue */
632 numvecs += adapter->num_tx_queues;
633
634 /* store the number of vectors reserved for queues */
635 adapter->num_q_vectors = numvecs;
636
637 /* add 1 vector for link status interrupts */
638 numvecs++;
9d5c8243
AK		 639 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
640 GFP_KERNEL);
641 if (!adapter->msix_entries)
642 goto msi_only;
643
644 for (i = 0; i < numvecs; i++)
645 adapter->msix_entries[i].entry = i;
646
647 err = pci_enable_msix(adapter->pdev,
648 adapter->msix_entries,
649 numvecs);
650 if (err == 0)
34a20e89 651 goto out;
9d5c8243
AK		 652
653 igb_reset_interrupt_capability(adapter);
654
655 /* If we can't do MSI-X, try MSI */
656msi_only:
2a3abf6d
AD		 657#ifdef CONFIG_PCI_IOV
658 /* disable SR-IOV for non MSI-X configurations */
659 if (adapter->vf_data) {
660 struct e1000_hw *hw = &adapter->hw;
661 /* disable iov and allow time for transactions to clear */
662 pci_disable_sriov(adapter->pdev);
663 msleep(500);
664
665 kfree(adapter->vf_data);
666 adapter->vf_data = NULL;
667 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
668 msleep(100);
669 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
670 }
671#endif
4fc82adf
AD		 672 adapter->vfs_allocated_count = 0;
673 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 674 adapter->num_rx_queues = 1;
661086df 675 adapter->num_tx_queues = 1;
047e0030 676 adapter->num_q_vectors = 1;
9d5c8243 677 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 678 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 679out:
661086df 680 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 681 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 682 return;
683}
684
047e0030
AD		 685/**
686 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
687 * @adapter: board private structure to initialize
688 *
689 * We allocate one q_vector per queue interrupt. If allocation fails we
690 * return -ENOMEM.
691 **/
692static int igb_alloc_q_vectors(struct igb_adapter *adapter)
693{
694 struct igb_q_vector *q_vector;
695 struct e1000_hw *hw = &adapter->hw;
696 int v_idx;
697
698 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
699 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
700 if (!q_vector)
701 goto err_out;
702 q_vector->adapter = adapter;
703 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
704 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
705 q_vector->itr_val = IGB_START_ITR;
706 q_vector->set_itr = 1;
707 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
708 adapter->q_vector[v_idx] = q_vector;
709 }
710 return 0;
711
712err_out:
713 while (v_idx) {
714 v_idx--;
715 q_vector = adapter->q_vector[v_idx];
716 netif_napi_del(&q_vector->napi);
717 kfree(q_vector);
718 adapter->q_vector[v_idx] = NULL;
719 }
720 return -ENOMEM;
721}
722
723static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
724 int ring_idx, int v_idx)
725{
726 struct igb_q_vector *q_vector;
727
728 q_vector = adapter->q_vector[v_idx];
729 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
730 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 731 q_vector->itr_val = adapter->rx_itr_setting;
732 if (q_vector->itr_val && q_vector->itr_val <= 3)
733 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 734}
735
736static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
737 int ring_idx, int v_idx)
738{
739 struct igb_q_vector *q_vector;
740
741 q_vector = adapter->q_vector[v_idx];
742 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
743 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 744 q_vector->itr_val = adapter->tx_itr_setting;
745 if (q_vector->itr_val && q_vector->itr_val <= 3)
746 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 747}
748
749/**
750 * igb_map_ring_to_vector - maps allocated queues to vectors
751 *
752 * This function maps the recently allocated queues to vectors.
753 **/
754static int igb_map_ring_to_vector(struct igb_adapter *adapter)
755{
756 int i;
757 int v_idx = 0;
758
759 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
760 (adapter->num_q_vectors < adapter->num_tx_queues))
761 return -ENOMEM;
762
763 if (adapter->num_q_vectors >=
764 (adapter->num_rx_queues + adapter->num_tx_queues)) {
765 for (i = 0; i < adapter->num_rx_queues; i++)
766 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
767 for (i = 0; i < adapter->num_tx_queues; i++)
768 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
769 } else {
770 for (i = 0; i < adapter->num_rx_queues; i++) {
771 if (i < adapter->num_tx_queues)
772 igb_map_tx_ring_to_vector(adapter, i, v_idx);
773 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
774 }
775 for (; i < adapter->num_tx_queues; i++)
776 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
777 }
778 return 0;
779}
780
781/**
782 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
783 *
784 * This function initializes the interrupts and allocates all of the queues.
785 **/
786static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
787{
788 struct pci_dev *pdev = adapter->pdev;
789 int err;
790
791 igb_set_interrupt_capability(adapter);
792
793 err = igb_alloc_q_vectors(adapter);
794 if (err) {
795 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
796 goto err_alloc_q_vectors;
797 }
798
799 err = igb_alloc_queues(adapter);
800 if (err) {
801 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
802 goto err_alloc_queues;
803 }
804
805 err = igb_map_ring_to_vector(adapter);
806 if (err) {
807 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
808 goto err_map_queues;
809 }
810
811
812 return 0;
813err_map_queues:
814 igb_free_queues(adapter);
815err_alloc_queues:
816 igb_free_q_vectors(adapter);
817err_alloc_q_vectors:
818 igb_reset_interrupt_capability(adapter);
819 return err;
820}
821
9d5c8243
AK		 822/**
823 * igb_request_irq - initialize interrupts
824 *
825 * Attempts to configure interrupts using the best available
826 * capabilities of the hardware and kernel.
827 **/
828static int igb_request_irq(struct igb_adapter *adapter)
829{
830 struct net_device *netdev = adapter->netdev;
047e0030 831 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 832 struct e1000_hw *hw = &adapter->hw;
833 int err = 0;
834
835 if (adapter->msix_entries) {
836 err = igb_request_msix(adapter);
844290e5 837 if (!err)
9d5c8243 838 goto request_done;
9d5c8243 839 /* fall back to MSI */
047e0030 840 igb_clear_interrupt_scheme(adapter);
9d5c8243 841 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 842 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 843 igb_free_all_tx_resources(adapter);
844 igb_free_all_rx_resources(adapter);
047e0030 845 adapter->num_tx_queues = 1;
9d5c8243 846 adapter->num_rx_queues = 1;
047e0030
AD		 847 adapter->num_q_vectors = 1;
848 err = igb_alloc_q_vectors(adapter);
849 if (err) {
850 dev_err(&pdev->dev,
851 "Unable to allocate memory for vectors\n");
852 goto request_done;
853 }
854 err = igb_alloc_queues(adapter);
855 if (err) {
856 dev_err(&pdev->dev,
857 "Unable to allocate memory for queues\n");
858 igb_free_q_vectors(adapter);
859 goto request_done;
860 }
861 igb_setup_all_tx_resources(adapter);
862 igb_setup_all_rx_resources(adapter);
844290e5 863 } else {
2d064c06
AD		 864 switch (hw->mac.type) {
865 case e1000_82575:
866 wr32(E1000_MSIXBM(0),
047e0030
AD		 867 (E1000_EICR_RX_QUEUE0 |
868 E1000_EICR_TX_QUEUE0 |
869 E1000_EIMS_OTHER));
2d064c06
AD		 870 break;
871 case e1000_82576:
872 wr32(E1000_IVAR0, E1000_IVAR_VALID);
873 break;
874 default:
875 break;
876 }
9d5c8243 877 }
844290e5 878
7dfc16fa 879 if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 880 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
047e0030 881 netdev->name, adapter);
9d5c8243
AK		 882 if (!err)
883 goto request_done;
047e0030 884
9d5c8243
AK		 885 /* fall back to legacy interrupts */
886 igb_reset_interrupt_capability(adapter);
7dfc16fa 887 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 888 }
889
890 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
047e0030 891 netdev->name, adapter);
9d5c8243 892
6cb5e577 893 if (err)
9d5c8243
AK		 894 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
895 err);
9d5c8243
AK		 896
897request_done:
898 return err;
899}
900
901static void igb_free_irq(struct igb_adapter *adapter)
902{
9d5c8243
AK		 903 if (adapter->msix_entries) {
904 int vector = 0, i;
905
047e0030 906 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 907
047e0030
AD		 908 for (i = 0; i < adapter->num_q_vectors; i++) {
909 struct igb_q_vector *q_vector = adapter->q_vector[i];
910 free_irq(adapter->msix_entries[vector++].vector,
911 q_vector);
912 }
913 } else {
914 free_irq(adapter->pdev->irq, adapter);
9d5c8243 915 }
9d5c8243
AK		 916}
917
918/**
919 * igb_irq_disable - Mask off interrupt generation on the NIC
920 * @adapter: board private structure
921 **/
922static void igb_irq_disable(struct igb_adapter *adapter)
923{
924 struct e1000_hw *hw = &adapter->hw;
925
926 if (adapter->msix_entries) {
2dfd1212
AD		 927 u32 regval = rd32(E1000_EIAM);
928 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
929 wr32(E1000_EIMC, adapter->eims_enable_mask);
930 regval = rd32(E1000_EIAC);
931 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 932 }
844290e5
PW		 933
934 wr32(E1000_IAM, 0);
9d5c8243
AK		 935 wr32(E1000_IMC, ~0);
936 wrfl();
937 synchronize_irq(adapter->pdev->irq);
938}
939
940/**
941 * igb_irq_enable - Enable default interrupt generation settings
942 * @adapter: board private structure
943 **/
944static void igb_irq_enable(struct igb_adapter *adapter)
945{
946 struct e1000_hw *hw = &adapter->hw;
947
948 if (adapter->msix_entries) {
2dfd1212
AD		 949 u32 regval = rd32(E1000_EIAC);
950 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
951 regval = rd32(E1000_EIAM);
952 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 953 wr32(E1000_EIMS, adapter->eims_enable_mask);
4ae196df
AD		 954 if (adapter->vfs_allocated_count)
955 wr32(E1000_MBVFIMR, 0xFF);
956 wr32(E1000_IMS, (E1000_IMS_LSC | E1000_IMS_VMMB |
957 E1000_IMS_DOUTSYNC));
844290e5
PW		 958 } else {
959 wr32(E1000_IMS, IMS_ENABLE_MASK);
960 wr32(E1000_IAM, IMS_ENABLE_MASK);
961 }
9d5c8243
AK		 962}
963
964static void igb_update_mng_vlan(struct igb_adapter *adapter)
965{
51466239 966 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 967 u16 vid = adapter->hw.mng_cookie.vlan_id;
968 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 969
970 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
971 /* add VID to filter table */
972 igb_vfta_set(hw, vid, true);
973 adapter->mng_vlan_id = vid;
974 } else {
975 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
976 }
977
978 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
979 (vid != old_vid) &&
980 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
981 /* remove VID from filter table */
982 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 983 }
984}
985
986/**
987 * igb_release_hw_control - release control of the h/w to f/w
988 * @adapter: address of board private structure
989 *
990 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
991 * For ASF and Pass Through versions of f/w this means that the
992 * driver is no longer loaded.
993 *
994 **/
995static void igb_release_hw_control(struct igb_adapter *adapter)
996{
997 struct e1000_hw *hw = &adapter->hw;
998 u32 ctrl_ext;
999
1000 /* Let firmware take over control of h/w */
1001 ctrl_ext = rd32(E1000_CTRL_EXT);
1002 wr32(E1000_CTRL_EXT,
1003 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1004}
1005
1006
1007/**
1008 * igb_get_hw_control - get control of the h/w from f/w
1009 * @adapter: address of board private structure
1010 *
1011 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1012 * For ASF and Pass Through versions of f/w this means that
1013 * the driver is loaded.
1014 *
1015 **/
1016static void igb_get_hw_control(struct igb_adapter *adapter)
1017{
1018 struct e1000_hw *hw = &adapter->hw;
1019 u32 ctrl_ext;
1020
1021 /* Let firmware know the driver has taken over */
1022 ctrl_ext = rd32(E1000_CTRL_EXT);
1023 wr32(E1000_CTRL_EXT,
1024 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1025}
1026
9d5c8243
AK		 1027/**
1028 * igb_configure - configure the hardware for RX and TX
1029 * @adapter: private board structure
1030 **/
1031static void igb_configure(struct igb_adapter *adapter)
1032{
1033 struct net_device *netdev = adapter->netdev;
1034 int i;
1035
1036 igb_get_hw_control(adapter);
ff41f8dc 1037 igb_set_rx_mode(netdev);
9d5c8243
AK		 1038
1039 igb_restore_vlan(adapter);
9d5c8243 1040
85b430b4 1041 igb_setup_tctl(adapter);
06cf2666 1042 igb_setup_mrqc(adapter);
9d5c8243 1043 igb_setup_rctl(adapter);
85b430b4
AD		 1044
1045 igb_configure_tx(adapter);
9d5c8243 1046 igb_configure_rx(adapter);
662d7205
AD		 1047
1048 igb_rx_fifo_flush_82575(&adapter->hw);
1049
c493ea45 1050 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1051 * at least 1 descriptor unused to make sure
1052 * next_to_use != next_to_clean */
1053 for (i = 0; i < adapter->num_rx_queues; i++) {
1054 struct igb_ring *ring = &adapter->rx_ring[i];
c493ea45 1055 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1056 }
1057
1058
1059 adapter->tx_queue_len = netdev->tx_queue_len;
1060}
1061
1062
1063/**
1064 * igb_up - Open the interface and prepare it to handle traffic
1065 * @adapter: board private structure
1066 **/
1067
1068int igb_up(struct igb_adapter *adapter)
1069{
1070 struct e1000_hw *hw = &adapter->hw;
1071 int i;
1072
1073 /* hardware has been reset, we need to reload some things */
1074 igb_configure(adapter);
1075
1076 clear_bit(__IGB_DOWN, &adapter->state);
1077
047e0030
AD		 1078 for (i = 0; i < adapter->num_q_vectors; i++) {
1079 struct igb_q_vector *q_vector = adapter->q_vector[i];
1080 napi_enable(&q_vector->napi);
1081 }
844290e5 1082 if (adapter->msix_entries)
9d5c8243 1083 igb_configure_msix(adapter);
9d5c8243
AK		 1084
1085 /* Clear any pending interrupts. */
1086 rd32(E1000_ICR);
1087 igb_irq_enable(adapter);
1088
d4960307
AD		 1089 /* notify VFs that reset has been completed */
1090 if (adapter->vfs_allocated_count) {
1091 u32 reg_data = rd32(E1000_CTRL_EXT);
1092 reg_data |= E1000_CTRL_EXT_PFRSTD;
1093 wr32(E1000_CTRL_EXT, reg_data);
1094 }
1095
4cb9be7a
JB		 1096 netif_tx_start_all_queues(adapter->netdev);
1097
9d5c8243
AK		 1098 /* Fire a link change interrupt to start the watchdog. */
1099 wr32(E1000_ICS, E1000_ICS_LSC);
1100 return 0;
1101}
1102
1103void igb_down(struct igb_adapter *adapter)
1104{
1105 struct e1000_hw *hw = &adapter->hw;
1106 struct net_device *netdev = adapter->netdev;
1107 u32 tctl, rctl;
1108 int i;
1109
1110 /* signal that we're down so the interrupt handler does not
1111 * reschedule our watchdog timer */
1112 set_bit(__IGB_DOWN, &adapter->state);
1113
1114 /* disable receives in the hardware */
1115 rctl = rd32(E1000_RCTL);
1116 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1117 /* flush and sleep below */
1118
fd2ea0a7 1119 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1120
1121 /* disable transmits in the hardware */
1122 tctl = rd32(E1000_TCTL);
1123 tctl &= ~E1000_TCTL_EN;
1124 wr32(E1000_TCTL, tctl);
1125 /* flush both disables and wait for them to finish */
1126 wrfl();
1127 msleep(10);
1128
047e0030
AD		 1129 for (i = 0; i < adapter->num_q_vectors; i++) {
1130 struct igb_q_vector *q_vector = adapter->q_vector[i];
1131 napi_disable(&q_vector->napi);
1132 }
9d5c8243 1133
9d5c8243
AK		 1134 igb_irq_disable(adapter);
1135
1136 del_timer_sync(&adapter->watchdog_timer);
1137 del_timer_sync(&adapter->phy_info_timer);
1138
1139 netdev->tx_queue_len = adapter->tx_queue_len;
1140 netif_carrier_off(netdev);
04fe6358
AD		 1141
1142 /* record the stats before reset*/
1143 igb_update_stats(adapter);
1144
9d5c8243
AK		 1145 adapter->link_speed = 0;
1146 adapter->link_duplex = 0;
1147
3023682e
JK		 1148 if (!pci_channel_offline(adapter->pdev))
1149 igb_reset(adapter);
9d5c8243
AK		 1150 igb_clean_all_tx_rings(adapter);
1151 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1152#ifdef CONFIG_IGB_DCA
1153
1154 /* since we reset the hardware DCA settings were cleared */
1155 igb_setup_dca(adapter);
1156#endif
9d5c8243
AK		 1157}
1158
1159void igb_reinit_locked(struct igb_adapter *adapter)
1160{
1161 WARN_ON(in_interrupt());
1162 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1163 msleep(1);
1164 igb_down(adapter);
1165 igb_up(adapter);
1166 clear_bit(__IGB_RESETTING, &adapter->state);
1167}
1168
1169void igb_reset(struct igb_adapter *adapter)
1170{
1171 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1172 struct e1000_mac_info *mac = &hw->mac;
1173 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1174 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1175 u16 hwm;
1176
1177 /* Repartition Pba for greater than 9k mtu
1178 * To take effect CTRL.RST is required.
1179 */
fa4dfae0
AD		 1180 switch (mac->type) {
1181 case e1000_82576:
d249be54
AD		 1182 pba = rd32(E1000_RXPBS);
1183 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1184 break;
1185 case e1000_82575:
1186 default:
1187 pba = E1000_PBA_34K;
1188 break;
2d064c06 1189 }
9d5c8243 1190
2d064c06
AD		 1191 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1192 (mac->type < e1000_82576)) {
9d5c8243
AK		 1193 /* adjust PBA for jumbo frames */
1194 wr32(E1000_PBA, pba);
1195
1196 /* To maintain wire speed transmits, the Tx FIFO should be
1197 * large enough to accommodate two full transmit packets,
1198 * rounded up to the next 1KB and expressed in KB. Likewise,
1199 * the Rx FIFO should be large enough to accommodate at least
1200 * one full receive packet and is similarly rounded up and
1201 * expressed in KB. */
1202 pba = rd32(E1000_PBA);
1203 /* upper 16 bits has Tx packet buffer allocation size in KB */
1204 tx_space = pba >> 16;
1205 /* lower 16 bits has Rx packet buffer allocation size in KB */
1206 pba &= 0xffff;
1207 /* the tx fifo also stores 16 bytes of information about the tx
1208 * but don't include ethernet FCS because hardware appends it */
1209 min_tx_space = (adapter->max_frame_size +
85e8d004 1210 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1211 ETH_FCS_LEN) * 2;
1212 min_tx_space = ALIGN(min_tx_space, 1024);
1213 min_tx_space >>= 10;
1214 /* software strips receive CRC, so leave room for it */
1215 min_rx_space = adapter->max_frame_size;
1216 min_rx_space = ALIGN(min_rx_space, 1024);
1217 min_rx_space >>= 10;
1218
1219 /* If current Tx allocation is less than the min Tx FIFO size,
1220 * and the min Tx FIFO size is less than the current Rx FIFO
1221 * allocation, take space away from current Rx allocation */
1222 if (tx_space < min_tx_space &&
1223 ((min_tx_space - tx_space) < pba)) {
1224 pba = pba - (min_tx_space - tx_space);
1225
1226 /* if short on rx space, rx wins and must trump tx
1227 * adjustment */
1228 if (pba < min_rx_space)
1229 pba = min_rx_space;
1230 }
2d064c06 1231 wr32(E1000_PBA, pba);
9d5c8243 1232 }
9d5c8243
AK		 1233
1234 /* flow control settings */
1235 /* The high water mark must be low enough to fit one full frame
1236 * (or the size used for early receive) above it in the Rx FIFO.
1237 * Set it to the lower of:
1238 * - 90% of the Rx FIFO size, or
1239 * - the full Rx FIFO size minus one full frame */
1240 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1241 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1242
2d064c06
AD		 1243 if (mac->type < e1000_82576) {
1244 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1245 fc->low_water = fc->high_water - 8;
1246 } else {
1247 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1248 fc->low_water = fc->high_water - 16;
1249 }
9d5c8243
AK		 1250 fc->pause_time = 0xFFFF;
1251 fc->send_xon = 1;
0cce119a 1252 fc->current_mode = fc->requested_mode;
9d5c8243 1253
4ae196df
AD		 1254 /* disable receive for all VFs and wait one second */
1255 if (adapter->vfs_allocated_count) {
1256 int i;
1257 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1258 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1259
1260 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1261 igb_ping_all_vfs(adapter);
4ae196df
AD		 1262
1263 /* disable transmits and receives */
1264 wr32(E1000_VFRE, 0);
1265 wr32(E1000_VFTE, 0);
1266 }
1267
9d5c8243
AK		 1268 /* Allow time for pending master requests to run */
1269 adapter->hw.mac.ops.reset_hw(&adapter->hw);
1270 wr32(E1000_WUC, 0);
1271
1272 if (adapter->hw.mac.ops.init_hw(&adapter->hw))
1273 dev_err(&adapter->pdev->dev, "Hardware Error\n");
1274
1275 igb_update_mng_vlan(adapter);
1276
1277 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1278 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1279
1280 igb_reset_adaptive(&adapter->hw);
f5f4cf08 1281 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 1282}
1283
2e5c6922
SH		 1284static const struct net_device_ops igb_netdev_ops = {
1285 .ndo_open = igb_open,
1286 .ndo_stop = igb_close,
00829823 1287 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1288 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1289 .ndo_set_rx_mode = igb_set_rx_mode,
1290 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1291 .ndo_set_mac_address = igb_set_mac,
1292 .ndo_change_mtu = igb_change_mtu,
1293 .ndo_do_ioctl = igb_ioctl,
1294 .ndo_tx_timeout = igb_tx_timeout,
1295 .ndo_validate_addr = eth_validate_addr,
1296 .ndo_vlan_rx_register = igb_vlan_rx_register,
1297 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1298 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1299#ifdef CONFIG_NET_POLL_CONTROLLER
1300 .ndo_poll_controller = igb_netpoll,
1301#endif
1302};
1303
9d5c8243
AK		 1304/**
1305 * igb_probe - Device Initialization Routine
1306 * @pdev: PCI device information struct
1307 * @ent: entry in igb_pci_tbl
1308 *
1309 * Returns 0 on success, negative on failure
1310 *
1311 * igb_probe initializes an adapter identified by a pci_dev structure.
1312 * The OS initialization, configuring of the adapter private structure,
1313 * and a hardware reset occur.
1314 **/
1315static int __devinit igb_probe(struct pci_dev *pdev,
1316 const struct pci_device_id *ent)
1317{
1318 struct net_device *netdev;
1319 struct igb_adapter *adapter;
1320 struct e1000_hw *hw;
1321 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1322 unsigned long mmio_start, mmio_len;
2d6a5e95 1323 int err, pci_using_dac;
682337fe 1324 u16 eeprom_data = 0;
9d5c8243
AK		 1325 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1326 u32 part_num;
1327
aed5dec3 1328 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1329 if (err)
1330 return err;
1331
1332 pci_using_dac = 0;
6a35528a 1333 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1334 if (!err) {
6a35528a 1335 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1336 if (!err)
1337 pci_using_dac = 1;
1338 } else {
284901a9 1339 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1340 if (err) {
284901a9 1341 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1342 if (err) {
1343 dev_err(&pdev->dev, "No usable DMA "
1344 "configuration, aborting\n");
1345 goto err_dma;
1346 }
1347 }
1348 }
1349
aed5dec3
AD		 1350 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1351 IORESOURCE_MEM),
1352 igb_driver_name);
9d5c8243
AK		 1353 if (err)
1354 goto err_pci_reg;
1355
19d5afd4 1356 pci_enable_pcie_error_reporting(pdev);
40a914fa 1357
9d5c8243 1358 pci_set_master(pdev);
c682fc23 1359 pci_save_state(pdev);
9d5c8243
AK		 1360
1361 err = -ENOMEM;
1bfaf07b
AD		 1362 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1363 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1364 if (!netdev)
1365 goto err_alloc_etherdev;
1366
1367 SET_NETDEV_DEV(netdev, &pdev->dev);
1368
1369 pci_set_drvdata(pdev, netdev);
1370 adapter = netdev_priv(netdev);
1371 adapter->netdev = netdev;
1372 adapter->pdev = pdev;
1373 hw = &adapter->hw;
1374 hw->back = adapter;
1375 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1376
1377 mmio_start = pci_resource_start(pdev, 0);
1378 mmio_len = pci_resource_len(pdev, 0);
1379
1380 err = -EIO;
28b0759c
AD		 1381 hw->hw_addr = ioremap(mmio_start, mmio_len);
1382 if (!hw->hw_addr)
9d5c8243
AK		 1383 goto err_ioremap;
1384
2e5c6922 1385 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1386 igb_set_ethtool_ops(netdev);
9d5c8243 1387 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1388
1389 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1390
1391 netdev->mem_start = mmio_start;
1392 netdev->mem_end = mmio_start + mmio_len;
1393
9d5c8243
AK		 1394 /* PCI config space info */
1395 hw->vendor_id = pdev->vendor;
1396 hw->device_id = pdev->device;
1397 hw->revision_id = pdev->revision;
1398 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1399 hw->subsystem_device_id = pdev->subsystem_device;
1400
1401 /* setup the private structure */
1402 hw->back = adapter;
1403 /* Copy the default MAC, PHY and NVM function pointers */
1404 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1405 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1406 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1407 /* Initialize skew-specific constants */
1408 err = ei->get_invariants(hw);
1409 if (err)
450c87c8 1410 goto err_sw_init;
9d5c8243 1411
2a3abf6d
AD		 1412#ifdef CONFIG_PCI_IOV
1413 /* since iov functionality isn't critical to base device function we
1414 * can accept failure. If it fails we don't allow iov to be enabled */
1415 if (hw->mac.type == e1000_82576) {
1416 /* 82576 supports a maximum of 7 VFs in addition to the PF */
1417 unsigned int num_vfs = (max_vfs > 7) ? 7 : max_vfs;
1418 int i;
1419 unsigned char mac_addr[ETH_ALEN];
1420
9ca046d5 1421 if (num_vfs) {
2a3abf6d
AD		 1422 adapter->vf_data = kcalloc(num_vfs,
1423 sizeof(struct vf_data_storage),
1424 GFP_KERNEL);
9ca046d5
AD		 1425 if (!adapter->vf_data) {
1426 dev_err(&pdev->dev,
1427 "Could not allocate VF private data - "
1428 "IOV enable failed\n");
2a3abf6d 1429 } else {
9ca046d5
AD		 1430 err = pci_enable_sriov(pdev, num_vfs);
1431 if (!err) {
1432 adapter->vfs_allocated_count = num_vfs;
1433 dev_info(&pdev->dev,
1434 "%d vfs allocated\n",
1435 num_vfs);
1436 for (i = 0;
1437 i < adapter->vfs_allocated_count;
1438 i++) {
1439 random_ether_addr(mac_addr);
1440 igb_set_vf_mac(adapter, i,
1441 mac_addr);
1442 }
1443 } else {
1444 kfree(adapter->vf_data);
1445 adapter->vf_data = NULL;
1446 }
2a3abf6d
AD		 1447 }
1448 }
1449 }
1450
1451#endif
450c87c8 1452 /* setup the private structure */
9d5c8243
AK		 1453 err = igb_sw_init(adapter);
1454 if (err)
1455 goto err_sw_init;
1456
1457 igb_get_bus_info_pcie(hw);
1458
1459 hw->phy.autoneg_wait_to_complete = false;
1460 hw->mac.adaptive_ifs = true;
1461
1462 /* Copper options */
1463 if (hw->phy.media_type == e1000_media_type_copper) {
1464 hw->phy.mdix = AUTO_ALL_MODES;
1465 hw->phy.disable_polarity_correction = false;
1466 hw->phy.ms_type = e1000_ms_hw_default;
1467 }
1468
1469 if (igb_check_reset_block(hw))
1470 dev_info(&pdev->dev,
1471 "PHY reset is blocked due to SOL/IDER session.\n");
1472
1473 netdev->features = NETIF_F_SG |
7d8eb29e 1474 NETIF_F_IP_CSUM |
9d5c8243
AK		 1475 NETIF_F_HW_VLAN_TX |
1476 NETIF_F_HW_VLAN_RX |
1477 NETIF_F_HW_VLAN_FILTER;
1478
7d8eb29e 1479 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1480 netdev->features |= NETIF_F_TSO;
9d5c8243 1481 netdev->features |= NETIF_F_TSO6;
48f29ffc 1482
5c0999b7 1483 netdev->features |= NETIF_F_GRO;
d3352520 1484
48f29ffc
JK		 1485 netdev->vlan_features |= NETIF_F_TSO;
1486 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1487 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1488 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1489 netdev->vlan_features |= NETIF_F_SG;
1490
9d5c8243
AK		 1491 if (pci_using_dac)
1492 netdev->features |= NETIF_F_HIGHDMA;
1493
b9473560
JB		 1494 if (adapter->hw.mac.type == e1000_82576)
1495 netdev->features |= NETIF_F_SCTP_CSUM;
1496
9d5c8243
AK		 1497 adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
1498
1499 /* before reading the NVM, reset the controller to put the device in a
1500 * known good starting state */
1501 hw->mac.ops.reset_hw(hw);
1502
1503 /* make sure the NVM is good */
1504 if (igb_validate_nvm_checksum(hw) < 0) {
1505 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1506 err = -EIO;
1507 goto err_eeprom;
1508 }
1509
1510 /* copy the MAC address out of the NVM */
1511 if (hw->mac.ops.read_mac_addr(hw))
1512 dev_err(&pdev->dev, "NVM Read Error\n");
1513
1514 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1515 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1516
1517 if (!is_valid_ether_addr(netdev->perm_addr)) {
1518 dev_err(&pdev->dev, "Invalid MAC Address\n");
1519 err = -EIO;
1520 goto err_eeprom;
1521 }
1522
0e340485
AD		 1523 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1524 (unsigned long) adapter);
1525 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1526 (unsigned long) adapter);
9d5c8243
AK		 1527
1528 INIT_WORK(&adapter->reset_task, igb_reset_task);
1529 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1530
450c87c8 1531 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1532 adapter->fc_autoneg = true;
1533 hw->mac.autoneg = true;
1534 hw->phy.autoneg_advertised = 0x2f;
1535
0cce119a
AD		 1536 hw->fc.requested_mode = e1000_fc_default;
1537 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1538
9d5c8243
AK		 1539 igb_validate_mdi_setting(hw);
1540
9d5c8243
AK		 1541 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1542 * enable the ACPI Magic Packet filter
1543 */
1544
a2cf8b6c 1545 if (hw->bus.func == 0)
312c75ae 1546 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
a2cf8b6c
AD		 1547 else if (hw->bus.func == 1)
1548 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1549
1550 if (eeprom_data & eeprom_apme_mask)
1551 adapter->eeprom_wol |= E1000_WUFC_MAG;
1552
1553 /* now that we have the eeprom settings, apply the special cases where
1554 * the eeprom may be wrong or the board simply won't support wake on
1555 * lan on a particular port */
1556 switch (pdev->device) {
1557 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1558 adapter->eeprom_wol = 0;
1559 break;
1560 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1561 case E1000_DEV_ID_82576_FIBER:
1562 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1563 /* Wake events only supported on port A for dual fiber
1564 * regardless of eeprom setting */
1565 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1566 adapter->eeprom_wol = 0;
1567 break;
c8ea5ea9
AD		 1568 case E1000_DEV_ID_82576_QUAD_COPPER:
1569 /* if quad port adapter, disable WoL on all but port A */
1570 if (global_quad_port_a != 0)
1571 adapter->eeprom_wol = 0;
1572 else
1573 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1574 /* Reset for multiple quad port adapters */
1575 if (++global_quad_port_a == 4)
1576 global_quad_port_a = 0;
1577 break;
9d5c8243
AK		 1578 }
1579
1580 /* initialize the wol settings based on the eeprom settings */
1581 adapter->wol = adapter->eeprom_wol;
e1b86d84 1582 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1583
1584 /* reset the hardware with the new settings */
1585 igb_reset(adapter);
1586
1587 /* let the f/w know that the h/w is now under the control of the
1588 * driver. */
1589 igb_get_hw_control(adapter);
1590
9d5c8243
AK		 1591 strcpy(netdev->name, "eth%d");
1592 err = register_netdev(netdev);
1593 if (err)
1594 goto err_register;
1595
b168dfc5
JB		 1596 /* carrier off reporting is important to ethtool even BEFORE open */
1597 netif_carrier_off(netdev);
1598
421e02f0 1599#ifdef CONFIG_IGB_DCA
bbd98fe4 1600 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1601 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1602 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1603 igb_setup_dca(adapter);
1604 }
fe4506b6 1605
38c845c7 1606#endif
38c845c7 1607
c5b9bd5e
AD		 1608 switch (hw->mac.type) {
1609 case e1000_82576:
1610 /*
1611 * Initialize hardware timer: we keep it running just in case
1612 * that some program needs it later on.
1613 */
1614 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1615 adapter->cycles.read = igb_read_clock;
1616 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1617 adapter->cycles.mult = 1;
1618 /**
1619 * Scale the NIC clock cycle by a large factor so that
1620 * relatively small clock corrections can be added or
1621 * substracted at each clock tick. The drawbacks of a large
1622 * factor are a) that the clock register overflows more quickly
1623 * (not such a big deal) and b) that the increment per tick has
1624 * to fit into 24 bits. As a result we need to use a shift of
1625 * 19 so we can fit a value of 16 into the TIMINCA register.
1626 */
1627 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1628 wr32(E1000_TIMINCA,
1629 (1 << E1000_TIMINCA_16NS_SHIFT) |
1630 (16 << IGB_82576_TSYNC_SHIFT));
1631
1632 /* Set registers so that rollover occurs soon to test this. */
1633 wr32(E1000_SYSTIML, 0x00000000);
1634 wr32(E1000_SYSTIMH, 0xFF800000);
1635 wrfl();
1636
1637 timecounter_init(&adapter->clock,
1638 &adapter->cycles,
1639 ktime_to_ns(ktime_get_real()));
1640 /*
1641 * Synchronize our NIC clock against system wall clock. NIC
1642 * time stamp reading requires ~3us per sample, each sample
1643 * was pretty stable even under load => only require 10
1644 * samples for each offset comparison.
1645 */
1646 memset(&adapter->compare, 0, sizeof(adapter->compare));
1647 adapter->compare.source = &adapter->clock;
1648 adapter->compare.target = ktime_get_real;
1649 adapter->compare.num_samples = 10;
1650 timecompare_update(&adapter->compare, 0);
1651 break;
1652 case e1000_82575:
1653 /* 82575 does not support timesync */
1654 default:
1655 break;
38c845c7 1656 }
38c845c7 1657
9d5c8243
AK		 1658 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1659 /* print bus type/speed/width info */
7c510e4b 1660 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243
AK		 1661 netdev->name,
1662 ((hw->bus.speed == e1000_bus_speed_2500)
1663 ? "2.5Gb/s" : "unknown"),
59c3de89
AD		 1664 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1665 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1666 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1667 "unknown"),
7c510e4b 1668 netdev->dev_addr);
9d5c8243
AK		 1669
1670 igb_read_part_num(hw, &part_num);
1671 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1672 (part_num >> 8), (part_num & 0xff));
1673
1674 dev_info(&pdev->dev,
1675 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1676 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1677 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1678 adapter->num_rx_queues, adapter->num_tx_queues);
1679
9d5c8243
AK		 1680 return 0;
1681
1682err_register:
1683 igb_release_hw_control(adapter);
1684err_eeprom:
1685 if (!igb_check_reset_block(hw))
f5f4cf08 1686 igb_reset_phy(hw);
9d5c8243
AK		 1687
1688 if (hw->flash_address)
1689 iounmap(hw->flash_address);
9d5c8243 1690err_sw_init:
047e0030 1691 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1692 iounmap(hw->hw_addr);
1693err_ioremap:
1694 free_netdev(netdev);
1695err_alloc_etherdev:
aed5dec3
AD		 1696 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1697 IORESOURCE_MEM));
9d5c8243
AK		 1698err_pci_reg:
1699err_dma:
1700 pci_disable_device(pdev);
1701 return err;
1702}
1703
1704/**
1705 * igb_remove - Device Removal Routine
1706 * @pdev: PCI device information struct
1707 *
1708 * igb_remove is called by the PCI subsystem to alert the driver
1709 * that it should release a PCI device. The could be caused by a
1710 * Hot-Plug event, or because the driver is going to be removed from
1711 * memory.
1712 **/
1713static void __devexit igb_remove(struct pci_dev *pdev)
1714{
1715 struct net_device *netdev = pci_get_drvdata(pdev);
1716 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1717 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1718
1719 /* flush_scheduled work may reschedule our watchdog task, so
1720 * explicitly disable watchdog tasks from being rescheduled */
1721 set_bit(__IGB_DOWN, &adapter->state);
1722 del_timer_sync(&adapter->watchdog_timer);
1723 del_timer_sync(&adapter->phy_info_timer);
1724
1725 flush_scheduled_work();
1726
421e02f0 1727#ifdef CONFIG_IGB_DCA
7dfc16fa 1728 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1729 dev_info(&pdev->dev, "DCA disabled\n");
1730 dca_remove_requester(&pdev->dev);
7dfc16fa 1731 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1732 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1733 }
1734#endif
1735
9d5c8243
AK		 1736 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1737 * would have already happened in close and is redundant. */
1738 igb_release_hw_control(adapter);
1739
1740 unregister_netdev(netdev);
1741
f5f4cf08
AD		 1742 if (!igb_check_reset_block(&adapter->hw))
1743 igb_reset_phy(&adapter->hw);
9d5c8243 1744
047e0030 1745 igb_clear_interrupt_scheme(adapter);
9d5c8243 1746
37680117
AD		 1747#ifdef CONFIG_PCI_IOV
1748 /* reclaim resources allocated to VFs */
1749 if (adapter->vf_data) {
1750 /* disable iov and allow time for transactions to clear */
1751 pci_disable_sriov(pdev);
1752 msleep(500);
1753
1754 kfree(adapter->vf_data);
1755 adapter->vf_data = NULL;
1756 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1757 msleep(100);
1758 dev_info(&pdev->dev, "IOV Disabled\n");
1759 }
1760#endif
28b0759c
AD		 1761 iounmap(hw->hw_addr);
1762 if (hw->flash_address)
1763 iounmap(hw->flash_address);
aed5dec3
AD		 1764 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1765 IORESOURCE_MEM));
9d5c8243
AK		 1766
1767 free_netdev(netdev);
1768
19d5afd4 1769 pci_disable_pcie_error_reporting(pdev);
40a914fa 1770
9d5c8243
AK		 1771 pci_disable_device(pdev);
1772}
1773
1774/**
1775 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1776 * @adapter: board private structure to initialize
1777 *
1778 * igb_sw_init initializes the Adapter private data structure.
1779 * Fields are initialized based on PCI device information and
1780 * OS network device settings (MTU size).
1781 **/
1782static int __devinit igb_sw_init(struct igb_adapter *adapter)
1783{
1784 struct e1000_hw *hw = &adapter->hw;
1785 struct net_device *netdev = adapter->netdev;
1786 struct pci_dev *pdev = adapter->pdev;
1787
1788 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1789
68fd9910
AD		 1790 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1791 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1792 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1793 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1794
9d5c8243
AK		 1795 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1796 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1797
661086df
PWJ		 1798 /* This call may decrease the number of queues depending on
1799 * interrupt mode. */
047e0030 1800 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1801 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1802 return -ENOMEM;
1803 }
1804
1805 /* Explicitly disable IRQ since the NIC can be in any state. */
1806 igb_irq_disable(adapter);
1807
1808 set_bit(__IGB_DOWN, &adapter->state);
1809 return 0;
1810}
1811
1812/**
1813 * igb_open - Called when a network interface is made active
1814 * @netdev: network interface device structure
1815 *
1816 * Returns 0 on success, negative value on failure
1817 *
1818 * The open entry point is called when a network interface is made
1819 * active by the system (IFF_UP). At this point all resources needed
1820 * for transmit and receive operations are allocated, the interrupt
1821 * handler is registered with the OS, the watchdog timer is started,
1822 * and the stack is notified that the interface is ready.
1823 **/
1824static int igb_open(struct net_device *netdev)
1825{
1826 struct igb_adapter *adapter = netdev_priv(netdev);
1827 struct e1000_hw *hw = &adapter->hw;
1828 int err;
1829 int i;
1830
1831 /* disallow open during test */
1832 if (test_bit(__IGB_TESTING, &adapter->state))
1833 return -EBUSY;
1834
b168dfc5
JB		 1835 netif_carrier_off(netdev);
1836
9d5c8243
AK		 1837 /* allocate transmit descriptors */
1838 err = igb_setup_all_tx_resources(adapter);
1839 if (err)
1840 goto err_setup_tx;
1841
1842 /* allocate receive descriptors */
1843 err = igb_setup_all_rx_resources(adapter);
1844 if (err)
1845 goto err_setup_rx;
1846
1847 /* e1000_power_up_phy(adapter); */
1848
9d5c8243
AK		 1849 /* before we allocate an interrupt, we must be ready to handle it.
1850 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1851 * as soon as we call pci_request_irq, so we have to setup our
1852 * clean_rx handler before we do so. */
1853 igb_configure(adapter);
1854
1855 err = igb_request_irq(adapter);
1856 if (err)
1857 goto err_req_irq;
1858
1859 /* From here on the code is the same as igb_up() */
1860 clear_bit(__IGB_DOWN, &adapter->state);
1861
047e0030
AD		 1862 for (i = 0; i < adapter->num_q_vectors; i++) {
1863 struct igb_q_vector *q_vector = adapter->q_vector[i];
1864 napi_enable(&q_vector->napi);
1865 }
9d5c8243
AK		 1866
1867 /* Clear any pending interrupts. */
1868 rd32(E1000_ICR);
844290e5
PW		 1869
1870 igb_irq_enable(adapter);
1871
d4960307
AD		 1872 /* notify VFs that reset has been completed */
1873 if (adapter->vfs_allocated_count) {
1874 u32 reg_data = rd32(E1000_CTRL_EXT);
1875 reg_data |= E1000_CTRL_EXT_PFRSTD;
1876 wr32(E1000_CTRL_EXT, reg_data);
1877 }
1878
d55b53ff
JK		 1879 netif_tx_start_all_queues(netdev);
1880
9d5c8243
AK		 1881 /* Fire a link status change interrupt to start the watchdog. */
1882 wr32(E1000_ICS, E1000_ICS_LSC);
1883
1884 return 0;
1885
1886err_req_irq:
1887 igb_release_hw_control(adapter);
1888 /* e1000_power_down_phy(adapter); */
1889 igb_free_all_rx_resources(adapter);
1890err_setup_rx:
1891 igb_free_all_tx_resources(adapter);
1892err_setup_tx:
1893 igb_reset(adapter);
1894
1895 return err;
1896}
1897
1898/**
1899 * igb_close - Disables a network interface
1900 * @netdev: network interface device structure
1901 *
1902 * Returns 0, this is not allowed to fail
1903 *
1904 * The close entry point is called when an interface is de-activated
1905 * by the OS. The hardware is still under the driver's control, but
1906 * needs to be disabled. A global MAC reset is issued to stop the
1907 * hardware, and all transmit and receive resources are freed.
1908 **/
1909static int igb_close(struct net_device *netdev)
1910{
1911 struct igb_adapter *adapter = netdev_priv(netdev);
1912
1913 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1914 igb_down(adapter);
1915
1916 igb_free_irq(adapter);
1917
1918 igb_free_all_tx_resources(adapter);
1919 igb_free_all_rx_resources(adapter);
1920
9d5c8243
AK		 1921 return 0;
1922}
1923
1924/**
1925 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 1926 * @tx_ring: tx descriptor ring (for a specific queue) to setup
1927 *
1928 * Return 0 on success, negative on failure
1929 **/
80785298 1930int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 1931{
80785298 1932 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 1933 int size;
1934
1935 size = sizeof(struct igb_buffer) * tx_ring->count;
1936 tx_ring->buffer_info = vmalloc(size);
1937 if (!tx_ring->buffer_info)
1938 goto err;
1939 memset(tx_ring->buffer_info, 0, size);
1940
1941 /* round up to nearest 4K */
85e8d004 1942 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 1943 tx_ring->size = ALIGN(tx_ring->size, 4096);
1944
1945 tx_ring->desc = pci_alloc_consistent(pdev, tx_ring->size,
1946 &tx_ring->dma);
1947
1948 if (!tx_ring->desc)
1949 goto err;
1950
9d5c8243
AK		 1951 tx_ring->next_to_use = 0;
1952 tx_ring->next_to_clean = 0;
9d5c8243
AK		 1953 return 0;
1954
1955err:
1956 vfree(tx_ring->buffer_info);
047e0030 1957 dev_err(&pdev->dev,
9d5c8243
AK		 1958 "Unable to allocate memory for the transmit descriptor ring\n");
1959 return -ENOMEM;
1960}
1961
1962/**
1963 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
1964 * (Descriptors) for all queues
1965 * @adapter: board private structure
1966 *
1967 * Return 0 on success, negative on failure
1968 **/
1969static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
1970{
1971 int i, err = 0;
661086df 1972 int r_idx;
9d5c8243
AK		 1973
1974 for (i = 0; i < adapter->num_tx_queues; i++) {
80785298 1975 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 1976 if (err) {
1977 dev_err(&adapter->pdev->dev,
1978 "Allocation for Tx Queue %u failed\n", i);
1979 for (i--; i >= 0; i--)
3b644cf6 1980 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 1981 break;
1982 }
1983 }
1984
661086df
PWJ		 1985 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
1986 r_idx = i % adapter->num_tx_queues;
1987 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
eebbbdba 1988 }
9d5c8243
AK		 1989 return err;
1990}
1991
1992/**
85b430b4
AD		 1993 * igb_setup_tctl - configure the transmit control registers
1994 * @adapter: Board private structure
9d5c8243 1995 **/
d7ee5b3a 1996void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 1997{
9d5c8243
AK		 1998 struct e1000_hw *hw = &adapter->hw;
1999 u32 tctl;
9d5c8243 2000
85b430b4
AD		 2001 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2002 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2003
2004 /* Program the Transmit Control Register */
9d5c8243
AK		 2005 tctl = rd32(E1000_TCTL);
2006 tctl &= ~E1000_TCTL_CT;
2007 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2008 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2009
2010 igb_config_collision_dist(hw);
2011
9d5c8243
AK		 2012 /* Enable transmits */
2013 tctl |= E1000_TCTL_EN;
2014
2015 wr32(E1000_TCTL, tctl);
2016}
2017
85b430b4
AD		 2018/**
2019 * igb_configure_tx_ring - Configure transmit ring after Reset
2020 * @adapter: board private structure
2021 * @ring: tx ring to configure
2022 *
2023 * Configure a transmit ring after a reset.
2024 **/
d7ee5b3a
AD		 2025void igb_configure_tx_ring(struct igb_adapter *adapter,
2026 struct igb_ring *ring)
85b430b4
AD		 2027{
2028 struct e1000_hw *hw = &adapter->hw;
2029 u32 txdctl;
2030 u64 tdba = ring->dma;
2031 int reg_idx = ring->reg_idx;
2032
2033 /* disable the queue */
2034 txdctl = rd32(E1000_TXDCTL(reg_idx));
2035 wr32(E1000_TXDCTL(reg_idx),
2036 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2037 wrfl();
2038 mdelay(10);
2039
2040 wr32(E1000_TDLEN(reg_idx),
2041 ring->count * sizeof(union e1000_adv_tx_desc));
2042 wr32(E1000_TDBAL(reg_idx),
2043 tdba & 0x00000000ffffffffULL);
2044 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2045
fce99e34
AD		 2046 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2047 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2048 writel(0, ring->head);
2049 writel(0, ring->tail);
85b430b4
AD		 2050
2051 txdctl |= IGB_TX_PTHRESH;
2052 txdctl |= IGB_TX_HTHRESH << 8;
2053 txdctl |= IGB_TX_WTHRESH << 16;
2054
2055 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2056 wr32(E1000_TXDCTL(reg_idx), txdctl);
2057}
2058
2059/**
2060 * igb_configure_tx - Configure transmit Unit after Reset
2061 * @adapter: board private structure
2062 *
2063 * Configure the Tx unit of the MAC after a reset.
2064 **/
2065static void igb_configure_tx(struct igb_adapter *adapter)
2066{
2067 int i;
2068
2069 for (i = 0; i < adapter->num_tx_queues; i++)
2070 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
85b430b4
AD		 2071}
2072
9d5c8243
AK		 2073/**
2074 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2075 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2076 *
2077 * Returns 0 on success, negative on failure
2078 **/
80785298 2079int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2080{
80785298 2081 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2082 int size, desc_len;
2083
2084 size = sizeof(struct igb_buffer) * rx_ring->count;
2085 rx_ring->buffer_info = vmalloc(size);
2086 if (!rx_ring->buffer_info)
2087 goto err;
2088 memset(rx_ring->buffer_info, 0, size);
2089
2090 desc_len = sizeof(union e1000_adv_rx_desc);
2091
2092 /* Round up to nearest 4K */
2093 rx_ring->size = rx_ring->count * desc_len;
2094 rx_ring->size = ALIGN(rx_ring->size, 4096);
2095
2096 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2097 &rx_ring->dma);
2098
2099 if (!rx_ring->desc)
2100 goto err;
2101
2102 rx_ring->next_to_clean = 0;
2103 rx_ring->next_to_use = 0;
9d5c8243 2104
9d5c8243
AK		 2105 return 0;
2106
2107err:
2108 vfree(rx_ring->buffer_info);
80785298 2109 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2110 "the receive descriptor ring\n");
2111 return -ENOMEM;
2112}
2113
2114/**
2115 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2116 * (Descriptors) for all queues
2117 * @adapter: board private structure
2118 *
2119 * Return 0 on success, negative on failure
2120 **/
2121static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2122{
2123 int i, err = 0;
2124
2125 for (i = 0; i < adapter->num_rx_queues; i++) {
80785298 2126 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2127 if (err) {
2128 dev_err(&adapter->pdev->dev,
2129 "Allocation for Rx Queue %u failed\n", i);
2130 for (i--; i >= 0; i--)
3b644cf6 2131 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2132 break;
2133 }
2134 }
2135
2136 return err;
2137}
2138
06cf2666
AD		 2139/**
2140 * igb_setup_mrqc - configure the multiple receive queue control registers
2141 * @adapter: Board private structure
2142 **/
2143static void igb_setup_mrqc(struct igb_adapter *adapter)
2144{
2145 struct e1000_hw *hw = &adapter->hw;
2146 u32 mrqc, rxcsum;
2147 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2148 union e1000_reta {
2149 u32 dword;
2150 u8 bytes[4];
2151 } reta;
2152 static const u8 rsshash[40] = {
2153 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2154 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2155 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2156 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2157
2158 /* Fill out hash function seeds */
2159 for (j = 0; j < 10; j++) {
2160 u32 rsskey = rsshash[(j * 4)];
2161 rsskey |= rsshash[(j * 4) + 1] << 8;
2162 rsskey |= rsshash[(j * 4) + 2] << 16;
2163 rsskey |= rsshash[(j * 4) + 3] << 24;
2164 array_wr32(E1000_RSSRK(0), j, rsskey);
2165 }
2166
2167 num_rx_queues = adapter->num_rx_queues;
2168
2169 if (adapter->vfs_allocated_count) {
2170 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2171 switch (hw->mac.type) {
2172 case e1000_82576:
2173 shift = 3;
2174 num_rx_queues = 2;
2175 break;
2176 case e1000_82575:
2177 shift = 2;
2178 shift2 = 6;
2179 default:
2180 break;
2181 }
2182 } else {
2183 if (hw->mac.type == e1000_82575)
2184 shift = 6;
2185 }
2186
2187 for (j = 0; j < (32 * 4); j++) {
2188 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2189 if (shift2)
2190 reta.bytes[j & 3] |= num_rx_queues << shift2;
2191 if ((j & 3) == 3)
2192 wr32(E1000_RETA(j >> 2), reta.dword);
2193 }
2194
2195 /*
2196 * Disable raw packet checksumming so that RSS hash is placed in
2197 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2198 * offloads as they are enabled by default
2199 */
2200 rxcsum = rd32(E1000_RXCSUM);
2201 rxcsum |= E1000_RXCSUM_PCSD;
2202
2203 if (adapter->hw.mac.type >= e1000_82576)
2204 /* Enable Receive Checksum Offload for SCTP */
2205 rxcsum |= E1000_RXCSUM_CRCOFL;
2206
2207 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2208 wr32(E1000_RXCSUM, rxcsum);
2209
2210 /* If VMDq is enabled then we set the appropriate mode for that, else
2211 * we default to RSS so that an RSS hash is calculated per packet even
2212 * if we are only using one queue */
2213 if (adapter->vfs_allocated_count) {
2214 if (hw->mac.type > e1000_82575) {
2215 /* Set the default pool for the PF's first queue */
2216 u32 vtctl = rd32(E1000_VT_CTL);
2217 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2218 E1000_VT_CTL_DISABLE_DEF_POOL);
2219 vtctl |= adapter->vfs_allocated_count <<
2220 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2221 wr32(E1000_VT_CTL, vtctl);
2222 }
2223 if (adapter->num_rx_queues > 1)
2224 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2225 else
2226 mrqc = E1000_MRQC_ENABLE_VMDQ;
2227 } else {
2228 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2229 }
2230 igb_vmm_control(adapter);
2231
2232 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2233 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2234 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2235 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2236 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2237 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2238 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2239 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2240
2241 wr32(E1000_MRQC, mrqc);
2242}
2243
9d5c8243
AK		 2244/**
2245 * igb_setup_rctl - configure the receive control registers
2246 * @adapter: Board private structure
2247 **/
d7ee5b3a 2248void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2249{
2250 struct e1000_hw *hw = &adapter->hw;
2251 u32 rctl;
9d5c8243
AK		 2252
2253 rctl = rd32(E1000_RCTL);
2254
2255 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2256 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2257
69d728ba 2258 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2259 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2260
87cb7e8c
AK		 2261 /*
2262 * enable stripping of CRC. It's unlikely this will break BMC
2263 * redirection as it did with e1000. Newer features require
2264 * that the HW strips the CRC.
73cd78f1 2265 */
87cb7e8c 2266 rctl |= E1000_RCTL_SECRC;
9d5c8243 2267
9b07f3d3 2268 /*
ec54d7d6 2269 * disable store bad packets and clear size bits.
9b07f3d3 2270 */
ec54d7d6 2271 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2272
6ec43fe6
AD		 2273 /* enable LPE to prevent packets larger than max_frame_size */
2274 rctl |= E1000_RCTL_LPE;
9d5c8243 2275
952f72a8
AD		 2276 /* disable queue 0 to prevent tail write w/o re-config */
2277 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2278
e1739522
AD		 2279 /* Attention!!! For SR-IOV PF driver operations you must enable
2280 * queue drop for all VF and PF queues to prevent head of line blocking
2281 * if an un-trusted VF does not provide descriptors to hardware.
2282 */
2283 if (adapter->vfs_allocated_count) {
e1739522
AD		 2284 /* set all queue drop enable bits */
2285 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2286 }
2287
9d5c8243
AK		 2288 wr32(E1000_RCTL, rctl);
2289}
2290
7d5753f0
AD		 2291static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2292 int vfn)
2293{
2294 struct e1000_hw *hw = &adapter->hw;
2295 u32 vmolr;
2296
2297 /* if it isn't the PF check to see if VFs are enabled and
2298 * increase the size to support vlan tags */
2299 if (vfn < adapter->vfs_allocated_count &&
2300 adapter->vf_data[vfn].vlans_enabled)
2301 size += VLAN_TAG_SIZE;
2302
2303 vmolr = rd32(E1000_VMOLR(vfn));
2304 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2305 vmolr |= size | E1000_VMOLR_LPE;
2306 wr32(E1000_VMOLR(vfn), vmolr);
2307
2308 return 0;
2309}
2310
e1739522
AD		 2311/**
2312 * igb_rlpml_set - set maximum receive packet size
2313 * @adapter: board private structure
2314 *
2315 * Configure maximum receivable packet size.
2316 **/
2317static void igb_rlpml_set(struct igb_adapter *adapter)
2318{
2319 u32 max_frame_size = adapter->max_frame_size;
2320 struct e1000_hw *hw = &adapter->hw;
2321 u16 pf_id = adapter->vfs_allocated_count;
2322
2323 if (adapter->vlgrp)
2324 max_frame_size += VLAN_TAG_SIZE;
2325
2326 /* if vfs are enabled we set RLPML to the largest possible request
2327 * size and set the VMOLR RLPML to the size we need */
2328 if (pf_id) {
2329 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2330 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2331 }
2332
2333 wr32(E1000_RLPML, max_frame_size);
2334}
2335
7d5753f0
AD		 2336static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2337{
2338 struct e1000_hw *hw = &adapter->hw;
2339 u32 vmolr;
2340
2341 /*
2342 * This register exists only on 82576 and newer so if we are older then
2343 * we should exit and do nothing
2344 */
2345 if (hw->mac.type < e1000_82576)
2346 return;
2347
2348 vmolr = rd32(E1000_VMOLR(vfn));
2349 vmolr |= E1000_VMOLR_AUPE | /* Accept untagged packets */
2350 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2351
2352 /* clear all bits that might not be set */
2353 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2354
2355 if (adapter->num_rx_queues > 1 && vfn == adapter->vfs_allocated_count)
2356 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2357 /*
2358 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2359 * multicast packets
2360 */
2361 if (vfn <= adapter->vfs_allocated_count)
2362 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2363
2364 wr32(E1000_VMOLR(vfn), vmolr);
2365}
2366
85b430b4
AD		 2367/**
2368 * igb_configure_rx_ring - Configure a receive ring after Reset
2369 * @adapter: board private structure
2370 * @ring: receive ring to be configured
2371 *
2372 * Configure the Rx unit of the MAC after a reset.
2373 **/
d7ee5b3a
AD		 2374void igb_configure_rx_ring(struct igb_adapter *adapter,
2375 struct igb_ring *ring)
85b430b4
AD		 2376{
2377 struct e1000_hw *hw = &adapter->hw;
2378 u64 rdba = ring->dma;
2379 int reg_idx = ring->reg_idx;
952f72a8 2380 u32 srrctl, rxdctl;
85b430b4
AD		 2381
2382 /* disable the queue */
2383 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2384 wr32(E1000_RXDCTL(reg_idx),
2385 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2386
2387 /* Set DMA base address registers */
2388 wr32(E1000_RDBAL(reg_idx),
2389 rdba & 0x00000000ffffffffULL);
2390 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2391 wr32(E1000_RDLEN(reg_idx),
2392 ring->count * sizeof(union e1000_adv_rx_desc));
2393
2394 /* initialize head and tail */
fce99e34
AD		 2395 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2396 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2397 writel(0, ring->head);
2398 writel(0, ring->tail);
85b430b4 2399
952f72a8 2400 /* set descriptor configuration */
4c844851
AD		 2401 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2402 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2403 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2404#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2405 srrctl |= IGB_RXBUFFER_16384 >>
2406 E1000_SRRCTL_BSIZEPKT_SHIFT;
2407#else
2408 srrctl |= (PAGE_SIZE / 2) >>
2409 E1000_SRRCTL_BSIZEPKT_SHIFT;
2410#endif
2411 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2412 } else {
4c844851 2413 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2414 E1000_SRRCTL_BSIZEPKT_SHIFT;
2415 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2416 }
2417
2418 wr32(E1000_SRRCTL(reg_idx), srrctl);
2419
7d5753f0
AD		 2420 /* set filtering for VMDQ pools */
2421 igb_set_vmolr(adapter, reg_idx & 0x7);
2422
85b430b4
AD		 2423 /* enable receive descriptor fetching */
2424 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2425 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2426 rxdctl &= 0xFFF00000;
2427 rxdctl |= IGB_RX_PTHRESH;
2428 rxdctl |= IGB_RX_HTHRESH << 8;
2429 rxdctl |= IGB_RX_WTHRESH << 16;
2430 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2431}
2432
9d5c8243
AK		 2433/**
2434 * igb_configure_rx - Configure receive Unit after Reset
2435 * @adapter: board private structure
2436 *
2437 * Configure the Rx unit of the MAC after a reset.
2438 **/
2439static void igb_configure_rx(struct igb_adapter *adapter)
2440{
9107584e 2441 int i;
9d5c8243 2442
68d480c4
AD		 2443 /* set UTA to appropriate mode */
2444 igb_set_uta(adapter);
2445
26ad9178
AD		 2446 /* set the correct pool for the PF default MAC address in entry 0 */
2447 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2448 adapter->vfs_allocated_count);
2449
06cf2666
AD		 2450 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2451 * the Base and Length of the Rx Descriptor Ring */
2452 for (i = 0; i < adapter->num_rx_queues; i++)
2453 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
9d5c8243
AK		 2454}
2455
2456/**
2457 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2458 * @tx_ring: Tx descriptor ring for a specific queue
2459 *
2460 * Free all transmit software resources
2461 **/
68fd9910 2462void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2463{
3b644cf6 2464 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2465
2466 vfree(tx_ring->buffer_info);
2467 tx_ring->buffer_info = NULL;
2468
80785298
AD		 2469 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2470 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2471
2472 tx_ring->desc = NULL;
2473}
2474
2475/**
2476 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2477 * @adapter: board private structure
2478 *
2479 * Free all transmit software resources
2480 **/
2481static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2482{
2483 int i;
2484
2485 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2486 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2487}
2488
b1a436c3
AD		 2489void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2490 struct igb_buffer *buffer_info)
9d5c8243 2491{
65689fef 2492 buffer_info->dma = 0;
9d5c8243 2493 if (buffer_info->skb) {
80785298
AD		 2494 skb_dma_unmap(&tx_ring->pdev->dev,
2495 buffer_info->skb,
65689fef 2496 DMA_TO_DEVICE);
9d5c8243
AK		 2497 dev_kfree_skb_any(buffer_info->skb);
2498 buffer_info->skb = NULL;
2499 }
2500 buffer_info->time_stamp = 0;
2501 /* buffer_info must be completely set up in the transmit path */
2502}
2503
2504/**
2505 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2506 * @tx_ring: ring to be cleaned
2507 **/
3b644cf6 2508static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2509{
2510 struct igb_buffer *buffer_info;
2511 unsigned long size;
2512 unsigned int i;
2513
2514 if (!tx_ring->buffer_info)
2515 return;
2516 /* Free all the Tx ring sk_buffs */
2517
2518 for (i = 0; i < tx_ring->count; i++) {
2519 buffer_info = &tx_ring->buffer_info[i];
80785298 2520 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2521 }
2522
2523 size = sizeof(struct igb_buffer) * tx_ring->count;
2524 memset(tx_ring->buffer_info, 0, size);
2525
2526 /* Zero out the descriptor ring */
2527
2528 memset(tx_ring->desc, 0, tx_ring->size);
2529
2530 tx_ring->next_to_use = 0;
2531 tx_ring->next_to_clean = 0;
2532
fce99e34
AD		 2533 writel(0, tx_ring->head);
2534 writel(0, tx_ring->tail);
9d5c8243
AK		 2535}
2536
2537/**
2538 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2539 * @adapter: board private structure
2540 **/
2541static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2542{
2543 int i;
2544
2545 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2546 igb_clean_tx_ring(&adapter->tx_ring[i]);
9d5c8243
AK		 2547}
2548
2549/**
2550 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2551 * @rx_ring: ring to clean the resources from
2552 *
2553 * Free all receive software resources
2554 **/
68fd9910 2555void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2556{
3b644cf6 2557 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2558
2559 vfree(rx_ring->buffer_info);
2560 rx_ring->buffer_info = NULL;
2561
80785298
AD		 2562 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2563 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2564
2565 rx_ring->desc = NULL;
2566}
2567
2568/**
2569 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2570 * @adapter: board private structure
2571 *
2572 * Free all receive software resources
2573 **/
2574static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2575{
2576 int i;
2577
2578 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2579 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2580}
2581
2582/**
2583 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2584 * @rx_ring: ring to free buffers from
2585 **/
3b644cf6 2586static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2587{
2588 struct igb_buffer *buffer_info;
9d5c8243
AK		 2589 unsigned long size;
2590 unsigned int i;
2591
2592 if (!rx_ring->buffer_info)
2593 return;
2594 /* Free all the Rx ring sk_buffs */
2595 for (i = 0; i < rx_ring->count; i++) {
2596 buffer_info = &rx_ring->buffer_info[i];
2597 if (buffer_info->dma) {
80785298
AD		 2598 pci_unmap_single(rx_ring->pdev,
2599 buffer_info->dma,
4c844851 2600 rx_ring->rx_buffer_len,
6ec43fe6 2601 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2602 buffer_info->dma = 0;
2603 }
2604
2605 if (buffer_info->skb) {
2606 dev_kfree_skb(buffer_info->skb);
2607 buffer_info->skb = NULL;
2608 }
6ec43fe6 2609 if (buffer_info->page_dma) {
80785298
AD		 2610 pci_unmap_page(rx_ring->pdev,
2611 buffer_info->page_dma,
6ec43fe6
AD		 2612 PAGE_SIZE / 2,
2613 PCI_DMA_FROMDEVICE);
2614 buffer_info->page_dma = 0;
2615 }
9d5c8243 2616 if (buffer_info->page) {
9d5c8243
AK		 2617 put_page(buffer_info->page);
2618 buffer_info->page = NULL;
bf36c1a0 2619 buffer_info->page_offset = 0;
9d5c8243
AK		 2620 }
2621 }
2622
9d5c8243
AK		 2623 size = sizeof(struct igb_buffer) * rx_ring->count;
2624 memset(rx_ring->buffer_info, 0, size);
2625
2626 /* Zero out the descriptor ring */
2627 memset(rx_ring->desc, 0, rx_ring->size);
2628
2629 rx_ring->next_to_clean = 0;
2630 rx_ring->next_to_use = 0;
2631
fce99e34
AD		 2632 writel(0, rx_ring->head);
2633 writel(0, rx_ring->tail);
9d5c8243
AK		 2634}
2635
2636/**
2637 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2638 * @adapter: board private structure
2639 **/
2640static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2641{
2642 int i;
2643
2644 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2645 igb_clean_rx_ring(&adapter->rx_ring[i]);
9d5c8243
AK		 2646}
2647
2648/**
2649 * igb_set_mac - Change the Ethernet Address of the NIC
2650 * @netdev: network interface device structure
2651 * @p: pointer to an address structure
2652 *
2653 * Returns 0 on success, negative on failure
2654 **/
2655static int igb_set_mac(struct net_device *netdev, void *p)
2656{
2657 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2658 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2659 struct sockaddr *addr = p;
2660
2661 if (!is_valid_ether_addr(addr->sa_data))
2662 return -EADDRNOTAVAIL;
2663
2664 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2665 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2666
26ad9178
AD		 2667 /* set the correct pool for the new PF MAC address in entry 0 */
2668 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2669 adapter->vfs_allocated_count);
e1739522 2670
9d5c8243
AK		 2671 return 0;
2672}
2673
2674/**
68d480c4 2675 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2676 * @netdev: network interface device structure
2677 *
68d480c4
AD		 2678 * Writes multicast address list to the MTA hash table.
2679 * Returns: -ENOMEM on failure
2680 * 0 on no addresses written
2681 * X on writing X addresses to MTA
9d5c8243 2682 **/
68d480c4 2683static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2684{
2685 struct igb_adapter *adapter = netdev_priv(netdev);
2686 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2687 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4
AD		 2688 u8 *mta_list;
2689 u32 vmolr = 0;
9d5c8243
AK		 2690 int i;
2691
68d480c4
AD		 2692 if (!netdev->mc_count) {
2693 /* nothing to program, so clear mc list */
2694 igb_update_mc_addr_list(hw, NULL, 0);
2695 igb_restore_vf_multicasts(adapter);
2696 return 0;
2697 }
9d5c8243 2698
68d480c4
AD		 2699 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2700 if (!mta_list)
2701 return -ENOMEM;
ff41f8dc 2702
68d480c4
AD		 2703 /* set vmolr receive overflow multicast bit */
2704 vmolr |= E1000_VMOLR_ROMPE;
2705
2706 /* The shared function expects a packed array of only addresses. */
2707 mc_ptr = netdev->mc_list;
2708
2709 for (i = 0; i < netdev->mc_count; i++) {
2710 if (!mc_ptr)
2711 break;
2712 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2713 mc_ptr = mc_ptr->next;
746b9f02 2714 }
68d480c4
AD		 2715 igb_update_mc_addr_list(hw, mta_list, i);
2716 kfree(mta_list);
2717
2718 return netdev->mc_count;
2719}
2720
2721/**
2722 * igb_write_uc_addr_list - write unicast addresses to RAR table
2723 * @netdev: network interface device structure
2724 *
2725 * Writes unicast address list to the RAR table.
2726 * Returns: -ENOMEM on failure/insufficient address space
2727 * 0 on no addresses written
2728 * X on writing X addresses to the RAR table
2729 **/
2730static int igb_write_uc_addr_list(struct net_device *netdev)
2731{
2732 struct igb_adapter *adapter = netdev_priv(netdev);
2733 struct e1000_hw *hw = &adapter->hw;
2734 unsigned int vfn = adapter->vfs_allocated_count;
2735 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2736 int count = 0;
2737
2738 /* return ENOMEM indicating insufficient memory for addresses */
2739 if (netdev->uc.count > rar_entries)
2740 return -ENOMEM;
9d5c8243 2741
ff41f8dc
AD		 2742 if (netdev->uc.count && rar_entries) {
2743 struct netdev_hw_addr *ha;
2744 list_for_each_entry(ha, &netdev->uc.list, list) {
2745 if (!rar_entries)
2746 break;
26ad9178
AD		 2747 igb_rar_set_qsel(adapter, ha->addr,
2748 rar_entries--,
68d480c4
AD		 2749 vfn);
2750 count++;
ff41f8dc
AD		 2751 }
2752 }
2753 /* write the addresses in reverse order to avoid write combining */
2754 for (; rar_entries > 0 ; rar_entries--) {
2755 wr32(E1000_RAH(rar_entries), 0);
2756 wr32(E1000_RAL(rar_entries), 0);
2757 }
2758 wrfl();
2759
68d480c4
AD		 2760 return count;
2761}
2762
2763/**
2764 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2765 * @netdev: network interface device structure
2766 *
2767 * The set_rx_mode entry point is called whenever the unicast or multicast
2768 * address lists or the network interface flags are updated. This routine is
2769 * responsible for configuring the hardware for proper unicast, multicast,
2770 * promiscuous mode, and all-multi behavior.
2771 **/
2772static void igb_set_rx_mode(struct net_device *netdev)
2773{
2774 struct igb_adapter *adapter = netdev_priv(netdev);
2775 struct e1000_hw *hw = &adapter->hw;
2776 unsigned int vfn = adapter->vfs_allocated_count;
2777 u32 rctl, vmolr = 0;
2778 int count;
2779
2780 /* Check for Promiscuous and All Multicast modes */
2781 rctl = rd32(E1000_RCTL);
2782
2783 /* clear the effected bits */
2784 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2785
2786 if (netdev->flags & IFF_PROMISC) {
2787 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2788 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2789 } else {
2790 if (netdev->flags & IFF_ALLMULTI) {
2791 rctl |= E1000_RCTL_MPE;
2792 vmolr |= E1000_VMOLR_MPME;
2793 } else {
2794 /*
2795 * Write addresses to the MTA, if the attempt fails
2796 * then we should just turn on promiscous mode so
2797 * that we can at least receive multicast traffic
2798 */
2799 count = igb_write_mc_addr_list(netdev);
2800 if (count < 0) {
2801 rctl |= E1000_RCTL_MPE;
2802 vmolr |= E1000_VMOLR_MPME;
2803 } else if (count) {
2804 vmolr |= E1000_VMOLR_ROMPE;
2805 }
2806 }
2807 /*
2808 * Write addresses to available RAR registers, if there is not
2809 * sufficient space to store all the addresses then enable
2810 * unicast promiscous mode
2811 */
2812 count = igb_write_uc_addr_list(netdev);
2813 if (count < 0) {
2814 rctl |= E1000_RCTL_UPE;
2815 vmolr |= E1000_VMOLR_ROPE;
2816 }
2817 rctl |= E1000_RCTL_VFE;
28fc06f5 2818 }
68d480c4 2819 wr32(E1000_RCTL, rctl);
28fc06f5 2820
68d480c4
AD		 2821 /*
2822 * In order to support SR-IOV and eventually VMDq it is necessary to set
2823 * the VMOLR to enable the appropriate modes. Without this workaround
2824 * we will have issues with VLAN tag stripping not being done for frames
2825 * that are only arriving because we are the default pool
2826 */
2827 if (hw->mac.type < e1000_82576)
28fc06f5 2828 return;
9d5c8243 2829
68d480c4
AD		 2830 vmolr |= rd32(E1000_VMOLR(vfn)) &
2831 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2832 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 2833 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 2834}
2835
2836/* Need to wait a few seconds after link up to get diagnostic information from
2837 * the phy */
2838static void igb_update_phy_info(unsigned long data)
2839{
2840 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 2841 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 2842}
2843
4d6b725e
AD		 2844/**
2845 * igb_has_link - check shared code for link and determine up/down
2846 * @adapter: pointer to driver private info
2847 **/
2848static bool igb_has_link(struct igb_adapter *adapter)
2849{
2850 struct e1000_hw *hw = &adapter->hw;
2851 bool link_active = false;
2852 s32 ret_val = 0;
2853
2854 /* get_link_status is set on LSC (link status) interrupt or
2855 * rx sequence error interrupt. get_link_status will stay
2856 * false until the e1000_check_for_link establishes link
2857 * for copper adapters ONLY
2858 */
2859 switch (hw->phy.media_type) {
2860 case e1000_media_type_copper:
2861 if (hw->mac.get_link_status) {
2862 ret_val = hw->mac.ops.check_for_link(hw);
2863 link_active = !hw->mac.get_link_status;
2864 } else {
2865 link_active = true;
2866 }
2867 break;
4d6b725e
AD		 2868 case e1000_media_type_internal_serdes:
2869 ret_val = hw->mac.ops.check_for_link(hw);
2870 link_active = hw->mac.serdes_has_link;
2871 break;
2872 default:
2873 case e1000_media_type_unknown:
2874 break;
2875 }
2876
2877 return link_active;
2878}
2879
9d5c8243
AK		 2880/**
2881 * igb_watchdog - Timer Call-back
2882 * @data: pointer to adapter cast into an unsigned long
2883 **/
2884static void igb_watchdog(unsigned long data)
2885{
2886 struct igb_adapter *adapter = (struct igb_adapter *)data;
2887 /* Do the rest outside of interrupt context */
2888 schedule_work(&adapter->watchdog_task);
2889}
2890
2891static void igb_watchdog_task(struct work_struct *work)
2892{
2893 struct igb_adapter *adapter = container_of(work,
2894 struct igb_adapter, watchdog_task);
2895 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2896 struct net_device *netdev = adapter->netdev;
2897 struct igb_ring *tx_ring = adapter->tx_ring;
9d5c8243 2898 u32 link;
7a6ea550 2899 int i;
9d5c8243 2900
4d6b725e
AD		 2901 link = igb_has_link(adapter);
2902 if ((netif_carrier_ok(netdev)) && link)
9d5c8243
AK		 2903 goto link_up;
2904
9d5c8243
AK		 2905 if (link) {
2906 if (!netif_carrier_ok(netdev)) {
2907 u32 ctrl;
2908 hw->mac.ops.get_speed_and_duplex(&adapter->hw,
2909 &adapter->link_speed,
2910 &adapter->link_duplex);
2911
2912 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 2913 /* Links status message must follow this format */
2914 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 2915 "Flow Control: %s\n",
527d47c1 2916 netdev->name,
9d5c8243
AK		 2917 adapter->link_speed,
2918 adapter->link_duplex == FULL_DUPLEX ?
2919 "Full Duplex" : "Half Duplex",
2920 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2921 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2922 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2923 E1000_CTRL_TFCE) ? "TX" : "None")));
2924
2925 /* tweak tx_queue_len according to speed/duplex and
2926 * adjust the timeout factor */
2927 netdev->tx_queue_len = adapter->tx_queue_len;
2928 adapter->tx_timeout_factor = 1;
2929 switch (adapter->link_speed) {
2930 case SPEED_10:
2931 netdev->tx_queue_len = 10;
2932 adapter->tx_timeout_factor = 14;
2933 break;
2934 case SPEED_100:
2935 netdev->tx_queue_len = 100;
2936 /* maybe add some timeout factor ? */
2937 break;
2938 }
2939
2940 netif_carrier_on(netdev);
9d5c8243 2941
4ae196df
AD		 2942 igb_ping_all_vfs(adapter);
2943
4b1a9877 2944 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2945 if (!test_bit(__IGB_DOWN, &adapter->state))
2946 mod_timer(&adapter->phy_info_timer,
2947 round_jiffies(jiffies + 2 * HZ));
2948 }
2949 } else {
2950 if (netif_carrier_ok(netdev)) {
2951 adapter->link_speed = 0;
2952 adapter->link_duplex = 0;
527d47c1
AD		 2953 /* Links status message must follow this format */
2954 printk(KERN_INFO "igb: %s NIC Link is Down\n",
2955 netdev->name);
9d5c8243 2956 netif_carrier_off(netdev);
4b1a9877 2957
4ae196df
AD		 2958 igb_ping_all_vfs(adapter);
2959
4b1a9877 2960 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2961 if (!test_bit(__IGB_DOWN, &adapter->state))
2962 mod_timer(&adapter->phy_info_timer,
2963 round_jiffies(jiffies + 2 * HZ));
2964 }
2965 }
2966
2967link_up:
2968 igb_update_stats(adapter);
2969
4b1a9877 2970 hw->mac.tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
9d5c8243 2971 adapter->tpt_old = adapter->stats.tpt;
4b1a9877 2972 hw->mac.collision_delta = adapter->stats.colc - adapter->colc_old;
9d5c8243
AK		 2973 adapter->colc_old = adapter->stats.colc;
2974
2975 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
2976 adapter->gorc_old = adapter->stats.gorc;
2977 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
2978 adapter->gotc_old = adapter->stats.gotc;
2979
2980 igb_update_adaptive(&adapter->hw);
2981
2982 if (!netif_carrier_ok(netdev)) {
c493ea45 2983 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
9d5c8243
AK		 2984 /* We've lost link, so the controller stops DMA,
2985 * but we've got queued Tx work that's never going
2986 * to get done, so reset controller to flush Tx.
2987 * (Do the reset outside of interrupt context). */
2988 adapter->tx_timeout_count++;
2989 schedule_work(&adapter->reset_task);
c2d5ab49
JB		 2990 /* return immediately since reset is imminent */
2991 return;
9d5c8243
AK		 2992 }
2993 }
2994
2995 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 2996 if (adapter->msix_entries) {
047e0030
AD		 2997 u32 eics = 0;
2998 for (i = 0; i < adapter->num_q_vectors; i++) {
2999 struct igb_q_vector *q_vector = adapter->q_vector[i];
3000 eics |= q_vector->eims_value;
3001 }
7a6ea550
AD		 3002 wr32(E1000_EICS, eics);
3003 } else {
3004 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3005 }
9d5c8243
AK		 3006
3007 /* Force detection of hung controller every watchdog period */
3008 tx_ring->detect_tx_hung = true;
3009
3010 /* Reset the timer */
3011 if (!test_bit(__IGB_DOWN, &adapter->state))
3012 mod_timer(&adapter->watchdog_timer,
3013 round_jiffies(jiffies + 2 * HZ));
3014}
3015
3016enum latency_range {
3017 lowest_latency = 0,
3018 low_latency = 1,
3019 bulk_latency = 2,
3020 latency_invalid = 255
3021};
3022
6eb5a7f1
AD		 3023/**
3024 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3025 *
3026 * Stores a new ITR value based on strictly on packet size. This
3027 * algorithm is less sophisticated than that used in igb_update_itr,
3028 * due to the difficulty of synchronizing statistics across multiple
3029 * receive rings. The divisors and thresholds used by this fuction
3030 * were determined based on theoretical maximum wire speed and testing
3031 * data, in order to minimize response time while increasing bulk
3032 * throughput.
3033 * This functionality is controlled by the InterruptThrottleRate module
3034 * parameter (see igb_param.c)
3035 * NOTE: This function is called only when operating in a multiqueue
3036 * receive environment.
047e0030 3037 * @q_vector: pointer to q_vector
6eb5a7f1 3038 **/
047e0030 3039static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3040{
047e0030 3041 int new_val = q_vector->itr_val;
6eb5a7f1 3042 int avg_wire_size = 0;
047e0030 3043 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3044
6eb5a7f1
AD		 3045 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3046 * ints/sec - ITR timer value of 120 ticks.
3047 */
3048 if (adapter->link_speed != SPEED_1000) {
047e0030 3049 new_val = 976;
6eb5a7f1 3050 goto set_itr_val;
9d5c8243 3051 }
047e0030
AD		 3052
3053 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3054 struct igb_ring *ring = q_vector->rx_ring;
3055 avg_wire_size = ring->total_bytes / ring->total_packets;
3056 }
3057
3058 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3059 struct igb_ring *ring = q_vector->tx_ring;
3060 avg_wire_size = max_t(u32, avg_wire_size,
3061 (ring->total_bytes /
3062 ring->total_packets));
3063 }
3064
3065 /* if avg_wire_size isn't set no work was done */
3066 if (!avg_wire_size)
3067 goto clear_counts;
9d5c8243 3068
6eb5a7f1
AD		 3069 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3070 avg_wire_size += 24;
3071
3072 /* Don't starve jumbo frames */
3073 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3074
6eb5a7f1
AD		 3075 /* Give a little boost to mid-size frames */
3076 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3077 new_val = avg_wire_size / 3;
3078 else
3079 new_val = avg_wire_size / 2;
9d5c8243 3080
6eb5a7f1 3081set_itr_val:
047e0030
AD		 3082 if (new_val != q_vector->itr_val) {
3083 q_vector->itr_val = new_val;
3084 q_vector->set_itr = 1;
9d5c8243 3085 }
6eb5a7f1 3086clear_counts:
047e0030
AD		 3087 if (q_vector->rx_ring) {
3088 q_vector->rx_ring->total_bytes = 0;
3089 q_vector->rx_ring->total_packets = 0;
3090 }
3091 if (q_vector->tx_ring) {
3092 q_vector->tx_ring->total_bytes = 0;
3093 q_vector->tx_ring->total_packets = 0;
3094 }
9d5c8243
AK		 3095}
3096
3097/**
3098 * igb_update_itr - update the dynamic ITR value based on statistics
3099 * Stores a new ITR value based on packets and byte
3100 * counts during the last interrupt. The advantage of per interrupt
3101 * computation is faster updates and more accurate ITR for the current
3102 * traffic pattern. Constants in this function were computed
3103 * based on theoretical maximum wire speed and thresholds were set based
3104 * on testing data as well as attempting to minimize response time
3105 * while increasing bulk throughput.
3106 * this functionality is controlled by the InterruptThrottleRate module
3107 * parameter (see igb_param.c)
3108 * NOTE: These calculations are only valid when operating in a single-
3109 * queue environment.
3110 * @adapter: pointer to adapter
047e0030 3111 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3112 * @packets: the number of packets during this measurement interval
3113 * @bytes: the number of bytes during this measurement interval
3114 **/
3115static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3116 int packets, int bytes)
3117{
3118 unsigned int retval = itr_setting;
3119
3120 if (packets == 0)
3121 goto update_itr_done;
3122
3123 switch (itr_setting) {
3124 case lowest_latency:
3125 /* handle TSO and jumbo frames */
3126 if (bytes/packets > 8000)
3127 retval = bulk_latency;
3128 else if ((packets < 5) && (bytes > 512))
3129 retval = low_latency;
3130 break;
3131 case low_latency: /* 50 usec aka 20000 ints/s */
3132 if (bytes > 10000) {
3133 /* this if handles the TSO accounting */
3134 if (bytes/packets > 8000) {
3135 retval = bulk_latency;
3136 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3137 retval = bulk_latency;
3138 } else if ((packets > 35)) {
3139 retval = lowest_latency;
3140 }
3141 } else if (bytes/packets > 2000) {
3142 retval = bulk_latency;
3143 } else if (packets <= 2 && bytes < 512) {
3144 retval = lowest_latency;
3145 }
3146 break;
3147 case bulk_latency: /* 250 usec aka 4000 ints/s */
3148 if (bytes > 25000) {
3149 if (packets > 35)
3150 retval = low_latency;
1e5c3d21 3151 } else if (bytes < 1500) {
9d5c8243
AK		 3152 retval = low_latency;
3153 }
3154 break;
3155 }
3156
3157update_itr_done:
3158 return retval;
3159}
3160
6eb5a7f1 3161static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3162{
047e0030 3163 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3164 u16 current_itr;
047e0030 3165 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3166
3167 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3168 if (adapter->link_speed != SPEED_1000) {
3169 current_itr = 0;
3170 new_itr = 4000;
3171 goto set_itr_now;
3172 }
3173
3174 adapter->rx_itr = igb_update_itr(adapter,
3175 adapter->rx_itr,
3176 adapter->rx_ring->total_packets,
3177 adapter->rx_ring->total_bytes);
9d5c8243 3178
047e0030
AD		 3179 adapter->tx_itr = igb_update_itr(adapter,
3180 adapter->tx_itr,
3181 adapter->tx_ring->total_packets,
3182 adapter->tx_ring->total_bytes);
3183 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3184
6eb5a7f1 3185 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3186 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3187 current_itr = low_latency;
3188
9d5c8243
AK		 3189 switch (current_itr) {
3190 /* counts and packets in update_itr are dependent on these numbers */
3191 case lowest_latency:
78b1f607 3192 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3193 break;
3194 case low_latency:
78b1f607 3195 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3196 break;
3197 case bulk_latency:
78b1f607 3198 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3199 break;
3200 default:
3201 break;
3202 }
3203
3204set_itr_now:
6eb5a7f1
AD		 3205 adapter->rx_ring->total_bytes = 0;
3206 adapter->rx_ring->total_packets = 0;
047e0030
AD		 3207 adapter->tx_ring->total_bytes = 0;
3208 adapter->tx_ring->total_packets = 0;
6eb5a7f1 3209
047e0030 3210 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3211 /* this attempts to bias the interrupt rate towards Bulk
3212 * by adding intermediate steps when interrupt rate is
3213 * increasing */
047e0030
AD		 3214 new_itr = new_itr > q_vector->itr_val ?
3215 max((new_itr * q_vector->itr_val) /
3216 (new_itr + (q_vector->itr_val >> 2)),
3217 new_itr) :
9d5c8243
AK		 3218 new_itr;
3219 /* Don't write the value here; it resets the adapter's
3220 * internal timer, and causes us to delay far longer than
3221 * we should between interrupts. Instead, we write the ITR
3222 * value at the beginning of the next interrupt so the timing
3223 * ends up being correct.
3224 */
047e0030
AD		 3225 q_vector->itr_val = new_itr;
3226 q_vector->set_itr = 1;
9d5c8243
AK		 3227 }
3228
3229 return;
3230}
3231
9d5c8243
AK		 3232#define IGB_TX_FLAGS_CSUM 0x00000001
3233#define IGB_TX_FLAGS_VLAN 0x00000002
3234#define IGB_TX_FLAGS_TSO 0x00000004
3235#define IGB_TX_FLAGS_IPV4 0x00000008
33af6bcc 3236#define IGB_TX_FLAGS_TSTAMP 0x00000010
9d5c8243
AK		 3237#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3238#define IGB_TX_FLAGS_VLAN_SHIFT 16
3239
85ad76b2 3240static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3241 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3242{
3243 struct e1000_adv_tx_context_desc *context_desc;
3244 unsigned int i;
3245 int err;
3246 struct igb_buffer *buffer_info;
3247 u32 info = 0, tu_cmd = 0;
3248 u32 mss_l4len_idx, l4len;
3249 *hdr_len = 0;
3250
3251 if (skb_header_cloned(skb)) {
3252 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3253 if (err)
3254 return err;
3255 }
3256
3257 l4len = tcp_hdrlen(skb);
3258 *hdr_len += l4len;
3259
3260 if (skb->protocol == htons(ETH_P_IP)) {
3261 struct iphdr *iph = ip_hdr(skb);
3262 iph->tot_len = 0;
3263 iph->check = 0;
3264 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3265 iph->daddr, 0,
3266 IPPROTO_TCP,
3267 0);
3268 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3269 ipv6_hdr(skb)->payload_len = 0;
3270 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3271 &ipv6_hdr(skb)->daddr,
3272 0, IPPROTO_TCP, 0);
3273 }
3274
3275 i = tx_ring->next_to_use;
3276
3277 buffer_info = &tx_ring->buffer_info[i];
3278 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3279 /* VLAN MACLEN IPLEN */
3280 if (tx_flags & IGB_TX_FLAGS_VLAN)
3281 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3282 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3283 *hdr_len += skb_network_offset(skb);
3284 info |= skb_network_header_len(skb);
3285 *hdr_len += skb_network_header_len(skb);
3286 context_desc->vlan_macip_lens = cpu_to_le32(info);
3287
3288 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3289 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3290
3291 if (skb->protocol == htons(ETH_P_IP))
3292 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3293 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3294
3295 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3296
3297 /* MSS L4LEN IDX */
3298 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3299 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3300
73cd78f1 3301 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3302 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3303 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3304
3305 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3306 context_desc->seqnum_seed = 0;
3307
3308 buffer_info->time_stamp = jiffies;
0e014cb1 3309 buffer_info->next_to_watch = i;
9d5c8243
AK		 3310 buffer_info->dma = 0;
3311 i++;
3312 if (i == tx_ring->count)
3313 i = 0;
3314
3315 tx_ring->next_to_use = i;
3316
3317 return true;
3318}
3319
85ad76b2
AD		 3320static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3321 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3322{
3323 struct e1000_adv_tx_context_desc *context_desc;
80785298 3324 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3325 struct igb_buffer *buffer_info;
3326 u32 info = 0, tu_cmd = 0;
80785298 3327 unsigned int i;
9d5c8243
AK		 3328
3329 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3330 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3331 i = tx_ring->next_to_use;
3332 buffer_info = &tx_ring->buffer_info[i];
3333 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3334
3335 if (tx_flags & IGB_TX_FLAGS_VLAN)
3336 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3337 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3338 if (skb->ip_summed == CHECKSUM_PARTIAL)
3339 info |= skb_network_header_len(skb);
3340
3341 context_desc->vlan_macip_lens = cpu_to_le32(info);
3342
3343 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3344
3345 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3346 __be16 protocol;
3347
3348 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3349 const struct vlan_ethhdr *vhdr =
3350 (const struct vlan_ethhdr*)skb->data;
3351
3352 protocol = vhdr->h_vlan_encapsulated_proto;
3353 } else {
3354 protocol = skb->protocol;
3355 }
3356
3357 switch (protocol) {
09640e63 3358 case cpu_to_be16(ETH_P_IP):
9d5c8243 3359 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3360 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3361 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3362 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3363 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3364 break;
09640e63 3365 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3366 /* XXX what about other V6 headers?? */
3367 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3368 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3369 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3370 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3371 break;
3372 default:
3373 if (unlikely(net_ratelimit()))
80785298 3374 dev_warn(&pdev->dev,
44b0cda3
MW		 3375 "partial checksum but proto=%x!\n",
3376 skb->protocol);
3377 break;
3378 }
9d5c8243
AK		 3379 }
3380
3381 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3382 context_desc->seqnum_seed = 0;
85ad76b2 3383 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3384 context_desc->mss_l4len_idx =
85ad76b2 3385 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3386
3387 buffer_info->time_stamp = jiffies;
0e014cb1 3388 buffer_info->next_to_watch = i;
9d5c8243
AK		 3389 buffer_info->dma = 0;
3390
3391 i++;
3392 if (i == tx_ring->count)
3393 i = 0;
3394 tx_ring->next_to_use = i;
3395
3396 return true;
3397 }
9d5c8243
AK		 3398 return false;
3399}
3400
3401#define IGB_MAX_TXD_PWR 16
3402#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3403
80785298 3404static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3405 unsigned int first)
9d5c8243
AK		 3406{
3407 struct igb_buffer *buffer_info;
80785298 3408 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3409 unsigned int len = skb_headlen(skb);
3410 unsigned int count = 0, i;
3411 unsigned int f;
65689fef 3412 dma_addr_t *map;
9d5c8243
AK		 3413
3414 i = tx_ring->next_to_use;
3415
80785298
AD		 3416 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3417 dev_err(&pdev->dev, "TX DMA map failed\n");
65689fef
AD		 3418 return 0;
3419 }
3420
3421 map = skb_shinfo(skb)->dma_maps;
3422
9d5c8243
AK		 3423 buffer_info = &tx_ring->buffer_info[i];
3424 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3425 buffer_info->length = len;
3426 /* set time_stamp *before* dma to help avoid a possible race */
3427 buffer_info->time_stamp = jiffies;
0e014cb1 3428 buffer_info->next_to_watch = i;
042a53a9 3429 buffer_info->dma = skb_shinfo(skb)->dma_head;
9d5c8243
AK		 3430
3431 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3432 struct skb_frag_struct *frag;
3433
65689fef
AD		 3434 i++;
3435 if (i == tx_ring->count)
3436 i = 0;
3437
9d5c8243
AK		 3438 frag = &skb_shinfo(skb)->frags[f];
3439 len = frag->size;
3440
3441 buffer_info = &tx_ring->buffer_info[i];
3442 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3443 buffer_info->length = len;
3444 buffer_info->time_stamp = jiffies;
0e014cb1 3445 buffer_info->next_to_watch = i;
65689fef 3446 buffer_info->dma = map[count];
9d5c8243 3447 count++;
9d5c8243
AK		 3448 }
3449
9d5c8243 3450 tx_ring->buffer_info[i].skb = skb;
0e014cb1 3451 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3452
042a53a9 3453 return count + 1;
9d5c8243
AK		 3454}
3455
85ad76b2 3456static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3457 int tx_flags, int count, u32 paylen,
3458 u8 hdr_len)
3459{
3460 union e1000_adv_tx_desc *tx_desc = NULL;
3461 struct igb_buffer *buffer_info;
3462 u32 olinfo_status = 0, cmd_type_len;
3463 unsigned int i;
3464
3465 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3466 E1000_ADVTXD_DCMD_DEXT);
3467
3468 if (tx_flags & IGB_TX_FLAGS_VLAN)
3469 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3470
33af6bcc
PO		 3471 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3472 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3473
9d5c8243
AK		 3474 if (tx_flags & IGB_TX_FLAGS_TSO) {
3475 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3476
3477 /* insert tcp checksum */
3478 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3479
3480 /* insert ip checksum */
3481 if (tx_flags & IGB_TX_FLAGS_IPV4)
3482 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3483
3484 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3485 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3486 }
3487
85ad76b2
AD		 3488 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3489 (tx_flags & (IGB_TX_FLAGS_CSUM |
3490 IGB_TX_FLAGS_TSO |
7dfc16fa 3491 IGB_TX_FLAGS_VLAN)))
85ad76b2 3492 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3493
3494 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3495
3496 i = tx_ring->next_to_use;
3497 while (count--) {
3498 buffer_info = &tx_ring->buffer_info[i];
3499 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3500 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3501 tx_desc->read.cmd_type_len =
3502 cpu_to_le32(cmd_type_len | buffer_info->length);
3503 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3504 i++;
3505 if (i == tx_ring->count)
3506 i = 0;
3507 }
3508
85ad76b2 3509 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3510 /* Force memory writes to complete before letting h/w
3511 * know there are new descriptors to fetch. (Only
3512 * applicable for weak-ordered memory model archs,
3513 * such as IA-64). */
3514 wmb();
3515
3516 tx_ring->next_to_use = i;
fce99e34 3517 writel(i, tx_ring->tail);
9d5c8243
AK		 3518 /* we need this if more than one processor can write to our tail
3519 * at a time, it syncronizes IO on IA64/Altix systems */
3520 mmiowb();
3521}
3522
e694e964 3523static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3524{
e694e964
AD		 3525 struct net_device *netdev = tx_ring->netdev;
3526
661086df 3527 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3528
9d5c8243
AK		 3529 /* Herbert's original patch had:
3530 * smp_mb__after_netif_stop_queue();
3531 * but since that doesn't exist yet, just open code it. */
3532 smp_mb();
3533
3534 /* We need to check again in a case another CPU has just
3535 * made room available. */
c493ea45 3536 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3537 return -EBUSY;
3538
3539 /* A reprieve! */
661086df 3540 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3541 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3542 return 0;
3543}
3544
e694e964 3545static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3546{
c493ea45 3547 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3548 return 0;
e694e964 3549 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3550}
3551
b1a436c3
AD		 3552netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3553 struct igb_ring *tx_ring)
9d5c8243 3554{
e694e964 3555 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
0e014cb1 3556 unsigned int first;
9d5c8243 3557 unsigned int tx_flags = 0;
9d5c8243 3558 u8 hdr_len = 0;
65689fef 3559 int count = 0;
9d5c8243 3560 int tso = 0;
c5b9bd5e 3561 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3562
9d5c8243
AK		 3563 /* need: 1 descriptor per page,
3564 * + 2 desc gap to keep tail from touching head,
3565 * + 1 desc for skb->data,
3566 * + 1 desc for context descriptor,
3567 * otherwise try next time */
e694e964 3568 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3569 /* this is a hard error */
9d5c8243
AK		 3570 return NETDEV_TX_BUSY;
3571 }
33af6bcc 3572
33af6bcc
PO		 3573 if (unlikely(shtx->hardware)) {
3574 shtx->in_progress = 1;
3575 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3576 }
9d5c8243
AK		 3577
3578 if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
3579 tx_flags |= IGB_TX_FLAGS_VLAN;
3580 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3581 }
3582
661086df
PWJ		 3583 if (skb->protocol == htons(ETH_P_IP))
3584 tx_flags |= IGB_TX_FLAGS_IPV4;
3585
0e014cb1 3586 first = tx_ring->next_to_use;
85ad76b2
AD		 3587 if (skb_is_gso(skb)) {
3588 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3589 if (tso < 0) {
3590 dev_kfree_skb_any(skb);
3591 return NETDEV_TX_OK;
3592 }
9d5c8243
AK		 3593 }
3594
3595 if (tso)
3596 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3597 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3598 (skb->ip_summed == CHECKSUM_PARTIAL))
3599 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3600
65689fef
AD		 3601 /*
3602 * count reflects descriptors mapped, if 0 then mapping error
3603 * has occured and we need to rewind the descriptor queue
3604 */
80785298 3605 count = igb_tx_map_adv(tx_ring, skb, first);
65689fef 3606
85ad76b2 3607 if (!count) {
65689fef
AD		 3608 dev_kfree_skb_any(skb);
3609 tx_ring->buffer_info[first].time_stamp = 0;
3610 tx_ring->next_to_use = first;
85ad76b2 3611 return NETDEV_TX_OK;
65689fef 3612 }
9d5c8243 3613
85ad76b2
AD		 3614 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3615
3616 /* Make sure there is space in the ring for the next send. */
e694e964 3617 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3618
9d5c8243
AK		 3619 return NETDEV_TX_OK;
3620}
3621
3b29a56d
SH		 3622static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3623 struct net_device *netdev)
9d5c8243
AK		 3624{
3625 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3626 struct igb_ring *tx_ring;
661086df 3627 int r_idx = 0;
b1a436c3
AD		 3628
3629 if (test_bit(__IGB_DOWN, &adapter->state)) {
3630 dev_kfree_skb_any(skb);
3631 return NETDEV_TX_OK;
3632 }
3633
3634 if (skb->len <= 0) {
3635 dev_kfree_skb_any(skb);
3636 return NETDEV_TX_OK;
3637 }
3638
1bfaf07b 3639 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3640 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3641
3642 /* This goes back to the question of how to logically map a tx queue
3643 * to a flow. Right now, performance is impacted slightly negatively
3644 * if using multiple tx queues. If the stack breaks away from a
3645 * single qdisc implementation, we can look at this again. */
e694e964 3646 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3647}
3648
3649/**
3650 * igb_tx_timeout - Respond to a Tx Hang
3651 * @netdev: network interface device structure
3652 **/
3653static void igb_tx_timeout(struct net_device *netdev)
3654{
3655 struct igb_adapter *adapter = netdev_priv(netdev);
3656 struct e1000_hw *hw = &adapter->hw;
3657
3658 /* Do the reset outside of interrupt context */
3659 adapter->tx_timeout_count++;
3660 schedule_work(&adapter->reset_task);
265de409
AD		 3661 wr32(E1000_EICS,
3662 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3663}
3664
3665static void igb_reset_task(struct work_struct *work)
3666{
3667 struct igb_adapter *adapter;
3668 adapter = container_of(work, struct igb_adapter, reset_task);
3669
3670 igb_reinit_locked(adapter);
3671}
3672
3673/**
3674 * igb_get_stats - Get System Network Statistics
3675 * @netdev: network interface device structure
3676 *
3677 * Returns the address of the device statistics structure.
3678 * The statistics are actually updated from the timer callback.
3679 **/
73cd78f1 3680static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3681{
9d5c8243 3682 /* only return the current stats */
8d24e933 3683 return &netdev->stats;
9d5c8243
AK		 3684}
3685
3686/**
3687 * igb_change_mtu - Change the Maximum Transfer Unit
3688 * @netdev: network interface device structure
3689 * @new_mtu: new value for maximum frame size
3690 *
3691 * Returns 0 on success, negative on failure
3692 **/
3693static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3694{
3695 struct igb_adapter *adapter = netdev_priv(netdev);
3696 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3697 u32 rx_buffer_len, i;
9d5c8243
AK		 3698
3699 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3700 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3701 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
3702 return -EINVAL;
3703 }
3704
9d5c8243
AK		 3705 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3706 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
3707 return -EINVAL;
3708 }
3709
3710 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3711 msleep(1);
73cd78f1 3712
9d5c8243
AK		 3713 /* igb_down has a dependency on max_frame_size */
3714 adapter->max_frame_size = max_frame;
9d5c8243
AK		 3715 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3716 * means we reserve 2 more, this pushes us to allocate from the next
3717 * larger slab size.
3718 * i.e. RXBUFFER_2048 --> size-4096 slab
3719 */
3720
7d95b717 3721 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3722 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3723 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3724 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3725 else
4c844851
AD		 3726 rx_buffer_len = IGB_RXBUFFER_128;
3727
3728 if (netif_running(netdev))
3729 igb_down(adapter);
9d5c8243
AK		 3730
3731 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
3732 netdev->mtu, new_mtu);
3733 netdev->mtu = new_mtu;
3734
4c844851
AD		 3735 for (i = 0; i < adapter->num_rx_queues; i++)
3736 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3737
9d5c8243
AK		 3738 if (netif_running(netdev))
3739 igb_up(adapter);
3740 else
3741 igb_reset(adapter);
3742
3743 clear_bit(__IGB_RESETTING, &adapter->state);
3744
3745 return 0;
3746}
3747
3748/**
3749 * igb_update_stats - Update the board statistics counters
3750 * @adapter: board private structure
3751 **/
3752
3753void igb_update_stats(struct igb_adapter *adapter)
3754{
8d24e933 3755 struct net_device *netdev = adapter->netdev;
9d5c8243
AK		 3756 struct e1000_hw *hw = &adapter->hw;
3757 struct pci_dev *pdev = adapter->pdev;
3758 u16 phy_tmp;
3759
3760#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3761
3762 /*
3763 * Prevent stats update while adapter is being reset, or if the pci
3764 * connection is down.
3765 */
3766 if (adapter->link_speed == 0)
3767 return;
3768 if (pci_channel_offline(pdev))
3769 return;
3770
3771 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3772 adapter->stats.gprc += rd32(E1000_GPRC);
3773 adapter->stats.gorc += rd32(E1000_GORCL);
3774 rd32(E1000_GORCH); /* clear GORCL */
3775 adapter->stats.bprc += rd32(E1000_BPRC);
3776 adapter->stats.mprc += rd32(E1000_MPRC);
3777 adapter->stats.roc += rd32(E1000_ROC);
3778
3779 adapter->stats.prc64 += rd32(E1000_PRC64);
3780 adapter->stats.prc127 += rd32(E1000_PRC127);
3781 adapter->stats.prc255 += rd32(E1000_PRC255);
3782 adapter->stats.prc511 += rd32(E1000_PRC511);
3783 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3784 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3785 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3786 adapter->stats.sec += rd32(E1000_SEC);
3787
3788 adapter->stats.mpc += rd32(E1000_MPC);
3789 adapter->stats.scc += rd32(E1000_SCC);
3790 adapter->stats.ecol += rd32(E1000_ECOL);
3791 adapter->stats.mcc += rd32(E1000_MCC);
3792 adapter->stats.latecol += rd32(E1000_LATECOL);
3793 adapter->stats.dc += rd32(E1000_DC);
3794 adapter->stats.rlec += rd32(E1000_RLEC);
3795 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3796 adapter->stats.xontxc += rd32(E1000_XONTXC);
3797 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3798 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3799 adapter->stats.fcruc += rd32(E1000_FCRUC);
3800 adapter->stats.gptc += rd32(E1000_GPTC);
3801 adapter->stats.gotc += rd32(E1000_GOTCL);
3802 rd32(E1000_GOTCH); /* clear GOTCL */
3803 adapter->stats.rnbc += rd32(E1000_RNBC);
3804 adapter->stats.ruc += rd32(E1000_RUC);
3805 adapter->stats.rfc += rd32(E1000_RFC);
3806 adapter->stats.rjc += rd32(E1000_RJC);
3807 adapter->stats.tor += rd32(E1000_TORH);
3808 adapter->stats.tot += rd32(E1000_TOTH);
3809 adapter->stats.tpr += rd32(E1000_TPR);
3810
3811 adapter->stats.ptc64 += rd32(E1000_PTC64);
3812 adapter->stats.ptc127 += rd32(E1000_PTC127);
3813 adapter->stats.ptc255 += rd32(E1000_PTC255);
3814 adapter->stats.ptc511 += rd32(E1000_PTC511);
3815 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3816 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3817
3818 adapter->stats.mptc += rd32(E1000_MPTC);
3819 adapter->stats.bptc += rd32(E1000_BPTC);
3820
3821 /* used for adaptive IFS */
3822
3823 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3824 adapter->stats.tpt += hw->mac.tx_packet_delta;
3825 hw->mac.collision_delta = rd32(E1000_COLC);
3826 adapter->stats.colc += hw->mac.collision_delta;
3827
3828 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3829 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3830 adapter->stats.tncrs += rd32(E1000_TNCRS);
3831 adapter->stats.tsctc += rd32(E1000_TSCTC);
3832 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3833
3834 adapter->stats.iac += rd32(E1000_IAC);
3835 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3836 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3837 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3838 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3839 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3840 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3841 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3842 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3843
3844 /* Fill out the OS statistics structure */
8d24e933
AK		 3845 netdev->stats.multicast = adapter->stats.mprc;
3846 netdev->stats.collisions = adapter->stats.colc;
9d5c8243
AK		 3847
3848 /* Rx Errors */
3849
8c0ab70a
JDB		 3850 if (hw->mac.type != e1000_82575) {
3851 u32 rqdpc_tmp;
3ea73afa 3852 u64 rqdpc_total = 0;
8c0ab70a
JDB		 3853 int i;
3854 /* Read out drops stats per RX queue. Notice RQDPC (Receive
3855 * Queue Drop Packet Count) stats only gets incremented, if
3856 * the DROP_EN but it set (in the SRRCTL register for that
3857 * queue). If DROP_EN bit is NOT set, then the some what
3858 * equivalent count is stored in RNBC (not per queue basis).
3859 * Also note the drop count is due to lack of available
3860 * descriptors.
3861 */
3862 for (i = 0; i < adapter->num_rx_queues; i++) {
3863 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0xFFF;
3864 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3ea73afa 3865 rqdpc_total += adapter->rx_ring[i].rx_stats.drops;
8c0ab70a 3866 }
8d24e933 3867 netdev->stats.rx_fifo_errors = rqdpc_total;
8c0ab70a
JDB		 3868 }
3869
3ea73afa
JDB		 3870 /* Note RNBC (Receive No Buffers Count) is an not an exact
3871 * drop count as the hardware FIFO might save the day. Thats
3872 * one of the reason for saving it in rx_fifo_errors, as its
3873 * potentially not a true drop.
3874 */
8d24e933 3875 netdev->stats.rx_fifo_errors += adapter->stats.rnbc;
3ea73afa 3876
9d5c8243 3877 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 3878 * our own version based on RUC and ROC */
8d24e933 3879 netdev->stats.rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 3880 adapter->stats.crcerrs + adapter->stats.algnerrc +
3881 adapter->stats.ruc + adapter->stats.roc +
3882 adapter->stats.cexterr;
8d24e933 3883 netdev->stats.rx_length_errors = adapter->stats.ruc +
9d5c8243 3884 adapter->stats.roc;
8d24e933
AK		 3885 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3886 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3887 netdev->stats.rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 3888
3889 /* Tx Errors */
8d24e933 3890 netdev->stats.tx_errors = adapter->stats.ecol +
9d5c8243 3891 adapter->stats.latecol;
8d24e933
AK		 3892 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3893 netdev->stats.tx_window_errors = adapter->stats.latecol;
3894 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 3895
3896 /* Tx Dropped needs to be maintained elsewhere */
3897
3898 /* Phy Stats */
3899 if (hw->phy.media_type == e1000_media_type_copper) {
3900 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 3901 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 3902 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3903 adapter->phy_stats.idle_errors += phy_tmp;
3904 }
3905 }
3906
3907 /* Management Stats */
3908 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3909 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3910 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3911}
3912
9d5c8243
AK		 3913static irqreturn_t igb_msix_other(int irq, void *data)
3914{
047e0030 3915 struct igb_adapter *adapter = data;
9d5c8243 3916 struct e1000_hw *hw = &adapter->hw;
844290e5 3917 u32 icr = rd32(E1000_ICR);
844290e5 3918 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 3919
047e0030 3920 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 3921 /* HW is reporting DMA is out of sync */
3922 adapter->stats.doosync++;
3923 }
eebbbdba 3924
4ae196df
AD		 3925 /* Check for a mailbox event */
3926 if (icr & E1000_ICR_VMMB)
3927 igb_msg_task(adapter);
3928
3929 if (icr & E1000_ICR_LSC) {
3930 hw->mac.get_link_status = 1;
3931 /* guard against interrupt when we're going down */
3932 if (!test_bit(__IGB_DOWN, &adapter->state))
3933 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3934 }
3935
3936 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_VMMB);
844290e5 3937 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 3938
3939 return IRQ_HANDLED;
3940}
3941
047e0030 3942static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 3943{
047e0030 3944 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 3945
047e0030
AD		 3946 if (!q_vector->set_itr)
3947 return;
73cd78f1 3948
047e0030
AD		 3949 if (!itr_val)
3950 itr_val = 0x4;
661086df 3951
047e0030
AD		 3952 if (q_vector->itr_shift)
3953 itr_val |= itr_val << q_vector->itr_shift;
661086df 3954 else
047e0030 3955 itr_val |= 0x8000000;
661086df 3956
047e0030
AD		 3957 writel(itr_val, q_vector->itr_register);
3958 q_vector->set_itr = 0;
6eb5a7f1
AD		 3959}
3960
047e0030 3961static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 3962{
047e0030 3963 struct igb_q_vector *q_vector = data;
9d5c8243 3964
047e0030
AD		 3965 /* Write the ITR value calculated from the previous interrupt. */
3966 igb_write_itr(q_vector);
9d5c8243 3967
047e0030 3968 napi_schedule(&q_vector->napi);
844290e5 3969
047e0030 3970 return IRQ_HANDLED;
fe4506b6
JC		 3971}
3972
421e02f0 3973#ifdef CONFIG_IGB_DCA
047e0030 3974static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 3975{
047e0030 3976 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 3977 struct e1000_hw *hw = &adapter->hw;
3978 int cpu = get_cpu();
fe4506b6 3979
047e0030
AD		 3980 if (q_vector->cpu == cpu)
3981 goto out_no_update;
3982
3983 if (q_vector->tx_ring) {
3984 int q = q_vector->tx_ring->reg_idx;
3985 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
3986 if (hw->mac.type == e1000_82575) {
3987 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
3988 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 3989 } else {
047e0030
AD		 3990 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
3991 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
3992 E1000_DCA_TXCTRL_CPUID_SHIFT;
3993 }
3994 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
3995 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
3996 }
3997 if (q_vector->rx_ring) {
3998 int q = q_vector->rx_ring->reg_idx;
3999 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4000 if (hw->mac.type == e1000_82575) {
2d064c06 4001 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4002 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4003 } else {
4004 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4005 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4006 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4007 }
fe4506b6
JC		 4008 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4009 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4010 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4011 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4012 }
047e0030
AD		 4013 q_vector->cpu = cpu;
4014out_no_update:
fe4506b6
JC		 4015 put_cpu();
4016}
4017
4018static void igb_setup_dca(struct igb_adapter *adapter)
4019{
7e0e99ef 4020 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4021 int i;
4022
7dfc16fa 4023 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4024 return;
4025
7e0e99ef
AD		 4026 /* Always use CB2 mode, difference is masked in the CB driver. */
4027 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4028
047e0030
AD		 4029 for (i = 0; i < adapter->num_q_vectors; i++) {
4030 struct igb_q_vector *q_vector = adapter->q_vector[i];
4031 q_vector->cpu = -1;
4032 igb_update_dca(q_vector);
fe4506b6
JC		 4033 }
4034}
4035
4036static int __igb_notify_dca(struct device *dev, void *data)
4037{
4038 struct net_device *netdev = dev_get_drvdata(dev);
4039 struct igb_adapter *adapter = netdev_priv(netdev);
4040 struct e1000_hw *hw = &adapter->hw;
4041 unsigned long event = *(unsigned long *)data;
4042
4043 switch (event) {
4044 case DCA_PROVIDER_ADD:
4045 /* if already enabled, don't do it again */
7dfc16fa 4046 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4047 break;
fe4506b6
JC		 4048 /* Always use CB2 mode, difference is masked
4049 * in the CB driver. */
cbd347ad 4050 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
fe4506b6 4051 if (dca_add_requester(dev) == 0) {
bbd98fe4 4052 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6
JC		 4053 dev_info(&adapter->pdev->dev, "DCA enabled\n");
4054 igb_setup_dca(adapter);
4055 break;
4056 }
4057 /* Fall Through since DCA is disabled. */
4058 case DCA_PROVIDER_REMOVE:
7dfc16fa 4059 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4060 /* without this a class_device is left
047e0030 4061 * hanging around in the sysfs model */
fe4506b6
JC		 4062 dca_remove_requester(dev);
4063 dev_info(&adapter->pdev->dev, "DCA disabled\n");
7dfc16fa 4064 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4065 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4066 }
4067 break;
4068 }
bbd98fe4 4069
fe4506b6 4070 return 0;
9d5c8243
AK		 4071}
4072
fe4506b6
JC		 4073static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4074 void *p)
4075{
4076 int ret_val;
4077
4078 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4079 __igb_notify_dca);
4080
4081 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4082}
421e02f0 4083#endif /* CONFIG_IGB_DCA */
9d5c8243 4084
4ae196df
AD		 4085static void igb_ping_all_vfs(struct igb_adapter *adapter)
4086{
4087 struct e1000_hw *hw = &adapter->hw;
4088 u32 ping;
4089 int i;
4090
4091 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4092 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4093 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4094 ping |= E1000_VT_MSGTYPE_CTS;
4095 igb_write_mbx(hw, &ping, 1, i);
4096 }
4097}
4098
7d5753f0
AD		 4099static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4100{
4101 struct e1000_hw *hw = &adapter->hw;
4102 u32 vmolr = rd32(E1000_VMOLR(vf));
4103 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4104
4105 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4106 IGB_VF_FLAG_MULTI_PROMISC);
4107 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4108
4109 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4110 vmolr |= E1000_VMOLR_MPME;
4111 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4112 } else {
4113 /*
4114 * if we have hashes and we are clearing a multicast promisc
4115 * flag we need to write the hashes to the MTA as this step
4116 * was previously skipped
4117 */
4118 if (vf_data->num_vf_mc_hashes > 30) {
4119 vmolr |= E1000_VMOLR_MPME;
4120 } else if (vf_data->num_vf_mc_hashes) {
4121 int j;
4122 vmolr |= E1000_VMOLR_ROMPE;
4123 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4124 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4125 }
4126 }
4127
4128 wr32(E1000_VMOLR(vf), vmolr);
4129
4130 /* there are flags left unprocessed, likely not supported */
4131 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4132 return -EINVAL;
4133
4134 return 0;
4135
4136}
4137
4ae196df
AD		 4138static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4139 u32 *msgbuf, u32 vf)
4140{
4141 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4142 u16 *hash_list = (u16 *)&msgbuf[1];
4143 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4144 int i;
4145
7d5753f0 4146 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4147 * to this VF for later use to restore when the PF multi cast
4148 * list changes
4149 */
4150 vf_data->num_vf_mc_hashes = n;
4151
7d5753f0
AD		 4152 /* only up to 30 hash values supported */
4153 if (n > 30)
4154 n = 30;
4155
4156 /* store the hashes for later use */
4ae196df 4157 for (i = 0; i < n; i++)
a419aef8 4158 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4159
4160 /* Flush and reset the mta with the new values */
ff41f8dc 4161 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4162
4163 return 0;
4164}
4165
4166static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4167{
4168 struct e1000_hw *hw = &adapter->hw;
4169 struct vf_data_storage *vf_data;
4170 int i, j;
4171
4172 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4173 u32 vmolr = rd32(E1000_VMOLR(i));
4174 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4175
4ae196df 4176 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4177
4178 if ((vf_data->num_vf_mc_hashes > 30) ||
4179 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4180 vmolr |= E1000_VMOLR_MPME;
4181 } else if (vf_data->num_vf_mc_hashes) {
4182 vmolr |= E1000_VMOLR_ROMPE;
4183 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4184 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4185 }
4186 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4187 }
4188}
4189
4190static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4191{
4192 struct e1000_hw *hw = &adapter->hw;
4193 u32 pool_mask, reg, vid;
4194 int i;
4195
4196 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4197
4198 /* Find the vlan filter for this id */
4199 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4200 reg = rd32(E1000_VLVF(i));
4201
4202 /* remove the vf from the pool */
4203 reg &= ~pool_mask;
4204
4205 /* if pool is empty then remove entry from vfta */
4206 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4207 (reg & E1000_VLVF_VLANID_ENABLE)) {
4208 reg = 0;
4209 vid = reg & E1000_VLVF_VLANID_MASK;
4210 igb_vfta_set(hw, vid, false);
4211 }
4212
4213 wr32(E1000_VLVF(i), reg);
4214 }
ae641bdc
AD		 4215
4216 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4217}
4218
4219static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4220{
4221 struct e1000_hw *hw = &adapter->hw;
4222 u32 reg, i;
4223
51466239
AD		 4224 /* The vlvf table only exists on 82576 hardware and newer */
4225 if (hw->mac.type < e1000_82576)
4226 return -1;
4227
4228 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4229 if (!adapter->vfs_allocated_count)
4230 return -1;
4231
4232 /* Find the vlan filter for this id */
4233 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4234 reg = rd32(E1000_VLVF(i));
4235 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4236 vid == (reg & E1000_VLVF_VLANID_MASK))
4237 break;
4238 }
4239
4240 if (add) {
4241 if (i == E1000_VLVF_ARRAY_SIZE) {
4242 /* Did not find a matching VLAN ID entry that was
4243 * enabled. Search for a free filter entry, i.e.
4244 * one without the enable bit set
4245 */
4246 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4247 reg = rd32(E1000_VLVF(i));
4248 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4249 break;
4250 }
4251 }
4252 if (i < E1000_VLVF_ARRAY_SIZE) {
4253 /* Found an enabled/available entry */
4254 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4255
4256 /* if !enabled we need to set this up in vfta */
4257 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4258 /* add VID to filter table */
4259 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4260 reg |= E1000_VLVF_VLANID_ENABLE;
4261 }
cad6d05f
AD		 4262 reg &= ~E1000_VLVF_VLANID_MASK;
4263 reg |= vid;
4ae196df 4264 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4265
4266 /* do not modify RLPML for PF devices */
4267 if (vf >= adapter->vfs_allocated_count)
4268 return 0;
4269
4270 if (!adapter->vf_data[vf].vlans_enabled) {
4271 u32 size;
4272 reg = rd32(E1000_VMOLR(vf));
4273 size = reg & E1000_VMOLR_RLPML_MASK;
4274 size += 4;
4275 reg &= ~E1000_VMOLR_RLPML_MASK;
4276 reg |= size;
4277 wr32(E1000_VMOLR(vf), reg);
4278 }
ae641bdc 4279
51466239 4280 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4281 return 0;
4282 }
4283 } else {
4284 if (i < E1000_VLVF_ARRAY_SIZE) {
4285 /* remove vf from the pool */
4286 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4287 /* if pool is empty then remove entry from vfta */
4288 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4289 reg = 0;
4290 igb_vfta_set(hw, vid, false);
4291 }
4292 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4293
4294 /* do not modify RLPML for PF devices */
4295 if (vf >= adapter->vfs_allocated_count)
4296 return 0;
4297
4298 adapter->vf_data[vf].vlans_enabled--;
4299 if (!adapter->vf_data[vf].vlans_enabled) {
4300 u32 size;
4301 reg = rd32(E1000_VMOLR(vf));
4302 size = reg & E1000_VMOLR_RLPML_MASK;
4303 size -= 4;
4304 reg &= ~E1000_VMOLR_RLPML_MASK;
4305 reg |= size;
4306 wr32(E1000_VMOLR(vf), reg);
4307 }
4ae196df
AD		 4308 return 0;
4309 }
4310 }
4311 return -1;
4312}
4313
4314static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4315{
4316 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4317 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4318
4319 return igb_vlvf_set(adapter, vid, add, vf);
4320}
4321
f2ca0dbe 4322static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4323{
f2ca0dbe
AD		 4324 /* clear all flags */
4325 adapter->vf_data[vf].flags = 0;
4326 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4327
4328 /* reset offloads to defaults */
7d5753f0 4329 igb_set_vmolr(adapter, vf);
4ae196df
AD		 4330
4331 /* reset vlans for device */
4332 igb_clear_vf_vfta(adapter, vf);
4333
4334 /* reset multicast table array for vf */
4335 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4336
4337 /* Flush and reset the mta with the new values */
ff41f8dc 4338 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4339}
4340
f2ca0dbe
AD		 4341static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4342{
4343 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4344
4345 /* generate a new mac address as we were hotplug removed/added */
4346 random_ether_addr(vf_mac);
4347
4348 /* process remaining reset events */
4349 igb_vf_reset(adapter, vf);
4350}
4351
4352static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4353{
4354 struct e1000_hw *hw = &adapter->hw;
4355 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4356 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4357 u32 reg, msgbuf[3];
4358 u8 *addr = (u8 *)(&msgbuf[1]);
4359
4360 /* process all the same items cleared in a function level reset */
f2ca0dbe 4361 igb_vf_reset(adapter, vf);
4ae196df
AD		 4362
4363 /* set vf mac address */
26ad9178 4364 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4365
4366 /* enable transmit and receive for vf */
4367 reg = rd32(E1000_VFTE);
4368 wr32(E1000_VFTE, reg | (1 << vf));
4369 reg = rd32(E1000_VFRE);
4370 wr32(E1000_VFRE, reg | (1 << vf));
4371
f2ca0dbe 4372 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4373
4374 /* reply to reset with ack and vf mac address */
4375 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4376 memcpy(addr, vf_mac, 6);
4377 igb_write_mbx(hw, msgbuf, 3, vf);
4378}
4379
4380static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4381{
f2ca0dbe
AD		 4382 unsigned char *addr = (char *)&msg[1];
4383 int err = -1;
4ae196df 4384
f2ca0dbe
AD		 4385 if (is_valid_ether_addr(addr))
4386 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4387
f2ca0dbe 4388 return err;
4ae196df
AD		 4389}
4390
4391static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4392{
4393 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4394 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4395 u32 msg = E1000_VT_MSGTYPE_NACK;
4396
4397 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4398 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4399 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4400 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4401 vf_data->last_nack = jiffies;
4ae196df
AD		 4402 }
4403}
4404
f2ca0dbe 4405static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4406{
f2ca0dbe
AD		 4407 struct pci_dev *pdev = adapter->pdev;
4408 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4409 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4410 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4411 s32 retval;
4412
f2ca0dbe 4413 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df
AD		 4414
4415 if (retval)
f2ca0dbe 4416 dev_err(&pdev->dev, "Error receiving message from VF\n");
4ae196df
AD		 4417
4418 /* this is a message we already processed, do nothing */
4419 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4420 return;
4ae196df
AD		 4421
4422 /*
4423 * until the vf completes a reset it should not be
4424 * allowed to start any configuration.
4425 */
4426
4427 if (msgbuf[0] == E1000_VF_RESET) {
4428 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4429 return;
4ae196df
AD		 4430 }
4431
f2ca0dbe
AD		 4432 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4433 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
4434 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4435 igb_write_mbx(hw, msgbuf, 1, vf);
4436 vf_data->last_nack = jiffies;
4437 }
4438 return;
4ae196df
AD		 4439 }
4440
4441 switch ((msgbuf[0] & 0xFFFF)) {
4442 case E1000_VF_SET_MAC_ADDR:
4443 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4444 break;
7d5753f0
AD		 4445 case E1000_VF_SET_PROMISC:
4446 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4447 break;
4ae196df
AD		 4448 case E1000_VF_SET_MULTICAST:
4449 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4450 break;
4451 case E1000_VF_SET_LPE:
4452 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4453 break;
4454 case E1000_VF_SET_VLAN:
4455 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4456 break;
4457 default:
4458 dev_err(&adapter->pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4459 retval = -1;
4460 break;
4461 }
4462
4463 /* notify the VF of the results of what it sent us */
4464 if (retval)
4465 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4466 else
4467 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4468
4469 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4470
4471 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4472}
4ae196df 4473
f2ca0dbe
AD		 4474static void igb_msg_task(struct igb_adapter *adapter)
4475{
4476 struct e1000_hw *hw = &adapter->hw;
4477 u32 vf;
4478
4479 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4480 /* process any reset requests */
4481 if (!igb_check_for_rst(hw, vf))
4482 igb_vf_reset_event(adapter, vf);
4483
4484 /* process any messages pending */
4485 if (!igb_check_for_msg(hw, vf))
4486 igb_rcv_msg_from_vf(adapter, vf);
4487
4488 /* process any acks */
4489 if (!igb_check_for_ack(hw, vf))
4490 igb_rcv_ack_from_vf(adapter, vf);
4491 }
4ae196df
AD		 4492}
4493
68d480c4
AD		 4494/**
4495 * igb_set_uta - Set unicast filter table address
4496 * @adapter: board private structure
4497 *
4498 * The unicast table address is a register array of 32-bit registers.
4499 * The table is meant to be used in a way similar to how the MTA is used
4500 * however due to certain limitations in the hardware it is necessary to
4501 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4502 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4503 **/
4504static void igb_set_uta(struct igb_adapter *adapter)
4505{
4506 struct e1000_hw *hw = &adapter->hw;
4507 int i;
4508
4509 /* The UTA table only exists on 82576 hardware and newer */
4510 if (hw->mac.type < e1000_82576)
4511 return;
4512
4513 /* we only need to do this if VMDq is enabled */
4514 if (!adapter->vfs_allocated_count)
4515 return;
4516
4517 for (i = 0; i < hw->mac.uta_reg_count; i++)
4518 array_wr32(E1000_UTA, i, ~0);
4519}
4520
9d5c8243
AK		 4521/**
4522 * igb_intr_msi - Interrupt Handler
4523 * @irq: interrupt number
4524 * @data: pointer to a network interface device structure
4525 **/
4526static irqreturn_t igb_intr_msi(int irq, void *data)
4527{
047e0030
AD		 4528 struct igb_adapter *adapter = data;
4529 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4530 struct e1000_hw *hw = &adapter->hw;
4531 /* read ICR disables interrupts using IAM */
4532 u32 icr = rd32(E1000_ICR);
4533
047e0030 4534 igb_write_itr(q_vector);
9d5c8243 4535
047e0030 4536 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4537 /* HW is reporting DMA is out of sync */
4538 adapter->stats.doosync++;
4539 }
4540
9d5c8243
AK		 4541 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4542 hw->mac.get_link_status = 1;
4543 if (!test_bit(__IGB_DOWN, &adapter->state))
4544 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4545 }
4546
047e0030 4547 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4548
4549 return IRQ_HANDLED;
4550}
4551
4552/**
4a3c6433 4553 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4554 * @irq: interrupt number
4555 * @data: pointer to a network interface device structure
4556 **/
4557static irqreturn_t igb_intr(int irq, void *data)
4558{
047e0030
AD		 4559 struct igb_adapter *adapter = data;
4560 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4561 struct e1000_hw *hw = &adapter->hw;
4562 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4563 * need for the IMC write */
4564 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4565 if (!icr)
4566 return IRQ_NONE; /* Not our interrupt */
4567
047e0030 4568 igb_write_itr(q_vector);
9d5c8243
AK		 4569
4570 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4571 * not set, then the adapter didn't send an interrupt */
4572 if (!(icr & E1000_ICR_INT_ASSERTED))
4573 return IRQ_NONE;
4574
047e0030 4575 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4576 /* HW is reporting DMA is out of sync */
4577 adapter->stats.doosync++;
4578 }
4579
9d5c8243
AK		 4580 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4581 hw->mac.get_link_status = 1;
4582 /* guard against interrupt when we're going down */
4583 if (!test_bit(__IGB_DOWN, &adapter->state))
4584 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4585 }
4586
047e0030 4587 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4588
4589 return IRQ_HANDLED;
4590}
4591
047e0030 4592static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4593{
047e0030 4594 struct igb_adapter *adapter = q_vector->adapter;
46544258 4595 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4596
4fc82adf
AD		 4597 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4598 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4599 if (!adapter->msix_entries)
6eb5a7f1 4600 igb_set_itr(adapter);
46544258 4601 else
047e0030 4602 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4603 }
4604
46544258
AD		 4605 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4606 if (adapter->msix_entries)
047e0030 4607 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4608 else
4609 igb_irq_enable(adapter);
4610 }
9d5c8243
AK		 4611}
4612
46544258
AD		 4613/**
4614 * igb_poll - NAPI Rx polling callback
4615 * @napi: napi polling structure
4616 * @budget: count of how many packets we should handle
4617 **/
4618static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4619{
047e0030
AD		 4620 struct igb_q_vector *q_vector = container_of(napi,
4621 struct igb_q_vector,
4622 napi);
4623 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4624
421e02f0 4625#ifdef CONFIG_IGB_DCA
047e0030
AD		 4626 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4627 igb_update_dca(q_vector);
fe4506b6 4628#endif
047e0030
AD		 4629 if (q_vector->tx_ring)
4630 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4631
047e0030
AD		 4632 if (q_vector->rx_ring)
4633 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4634
4635 if (!tx_clean_complete)
4636 work_done = budget;
46544258 4637
9d5c8243 4638 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4639 if (work_done < budget) {
288379f0 4640 napi_complete(napi);
047e0030 4641 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4642 }
4643
46544258 4644 return work_done;
9d5c8243 4645}
6d8126f9 4646
33af6bcc 4647/**
c5b9bd5e 4648 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4649 * @adapter: board private structure
c5b9bd5e
AD		 4650 * @shhwtstamps: timestamp structure to update
4651 * @regval: unsigned 64bit system time value.
4652 *
4653 * We need to convert the system time value stored in the RX/TXSTMP registers
4654 * into a hwtstamp which can be used by the upper level timestamping functions
4655 */
4656static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4657 struct skb_shared_hwtstamps *shhwtstamps,
4658 u64 regval)
4659{
4660 u64 ns;
4661
4662 ns = timecounter_cyc2time(&adapter->clock, regval);
4663 timecompare_update(&adapter->compare, ns);
4664 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4665 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4666 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4667}
4668
4669/**
4670 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4671 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4672 * @skb: packet that was just sent
4673 *
4674 * If we were asked to do hardware stamping and such a time stamp is
4675 * available, then it must have been for this skb here because we only
4676 * allow only one such packet into the queue.
4677 */
c5b9bd5e 4678static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4679{
c5b9bd5e 4680 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4681 union skb_shared_tx *shtx = skb_tx(skb);
4682 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4683 struct skb_shared_hwtstamps shhwtstamps;
4684 u64 regval;
33af6bcc 4685
c5b9bd5e
AD		 4686 /* if skb does not support hw timestamp or TX stamp not valid exit */
4687 if (likely(!shtx->hardware) ||
4688 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4689 return;
4690
4691 regval = rd32(E1000_TXSTMPL);
4692 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4693
4694 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4695 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4696}
4697
9d5c8243
AK		 4698/**
4699 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4700 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4701 * returns true if ring is completely cleaned
4702 **/
047e0030 4703static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4704{
047e0030
AD		 4705 struct igb_adapter *adapter = q_vector->adapter;
4706 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4707 struct net_device *netdev = tx_ring->netdev;
0e014cb1 4708 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 4709 struct igb_buffer *buffer_info;
4710 struct sk_buff *skb;
0e014cb1 4711 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 4712 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 4713 unsigned int i, eop, count = 0;
4714 bool cleaned = false;
9d5c8243 4715
9d5c8243 4716 i = tx_ring->next_to_clean;
0e014cb1
AD		 4717 eop = tx_ring->buffer_info[i].next_to_watch;
4718 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4719
4720 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4721 (count < tx_ring->count)) {
4722 for (cleaned = false; !cleaned; count++) {
4723 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 4724 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 4725 cleaned = (i == eop);
9d5c8243
AK		 4726 skb = buffer_info->skb;
4727
4728 if (skb) {
4729 unsigned int segs, bytecount;
4730 /* gso_segs is currently only valid for tcp */
4731 segs = skb_shinfo(skb)->gso_segs ?: 1;
4732 /* multiply data chunks by size of headers */
4733 bytecount = ((segs - 1) * skb_headlen(skb)) +
4734 skb->len;
4735 total_packets += segs;
4736 total_bytes += bytecount;
33af6bcc 4737
c5b9bd5e 4738 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 4739 }
4740
80785298 4741 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 4742 tx_desc->wb.status = 0;
9d5c8243
AK		 4743
4744 i++;
4745 if (i == tx_ring->count)
4746 i = 0;
9d5c8243 4747 }
0e014cb1
AD		 4748 eop = tx_ring->buffer_info[i].next_to_watch;
4749 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4750 }
4751
9d5c8243
AK		 4752 tx_ring->next_to_clean = i;
4753
fc7d345d 4754 if (unlikely(count &&
9d5c8243 4755 netif_carrier_ok(netdev) &&
c493ea45 4756 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 4757 /* Make sure that anybody stopping the queue after this
4758 * sees the new next_to_clean.
4759 */
4760 smp_mb();
661086df
PWJ		 4761 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4762 !(test_bit(__IGB_DOWN, &adapter->state))) {
4763 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4764 tx_ring->tx_stats.restart_queue++;
661086df 4765 }
9d5c8243
AK		 4766 }
4767
4768 if (tx_ring->detect_tx_hung) {
4769 /* Detect a transmit hang in hardware, this serializes the
4770 * check with the clearing of time_stamp and movement of i */
4771 tx_ring->detect_tx_hung = false;
4772 if (tx_ring->buffer_info[i].time_stamp &&
4773 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4774 (adapter->tx_timeout_factor * HZ))
4775 && !(rd32(E1000_STATUS) &
4776 E1000_STATUS_TXOFF)) {
4777
9d5c8243 4778 /* detected Tx unit hang */
80785298 4779 dev_err(&tx_ring->pdev->dev,
9d5c8243 4780 "Detected Tx Unit Hang\n"
2d064c06 4781 " Tx Queue <%d>\n"
9d5c8243
AK		 4782 " TDH <%x>\n"
4783 " TDT <%x>\n"
4784 " next_to_use <%x>\n"
4785 " next_to_clean <%x>\n"
9d5c8243
AK		 4786 "buffer_info[next_to_clean]\n"
4787 " time_stamp <%lx>\n"
0e014cb1 4788 " next_to_watch <%x>\n"
9d5c8243
AK		 4789 " jiffies <%lx>\n"
4790 " desc.status <%x>\n",
2d064c06 4791 tx_ring->queue_index,
fce99e34
AD		 4792 readl(tx_ring->head),
4793 readl(tx_ring->tail),
9d5c8243
AK		 4794 tx_ring->next_to_use,
4795 tx_ring->next_to_clean,
9d5c8243 4796 tx_ring->buffer_info[i].time_stamp,
0e014cb1 4797 eop,
9d5c8243 4798 jiffies,
0e014cb1 4799 eop_desc->wb.status);
661086df 4800 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 4801 }
4802 }
4803 tx_ring->total_bytes += total_bytes;
4804 tx_ring->total_packets += total_packets;
e21ed353
AD		 4805 tx_ring->tx_stats.bytes += total_bytes;
4806 tx_ring->tx_stats.packets += total_packets;
8d24e933
AK		 4807 netdev->stats.tx_bytes += total_bytes;
4808 netdev->stats.tx_packets += total_packets;
0e014cb1 4809 return (count < tx_ring->count);
9d5c8243
AK		 4810}
4811
9d5c8243
AK		 4812/**
4813 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 4814 * @q_vector: structure containing interrupt and ring information
4815 * @skb: packet to send up
4816 * @vlan_tag: vlan tag for packet
9d5c8243 4817 **/
047e0030
AD		 4818static void igb_receive_skb(struct igb_q_vector *q_vector,
4819 struct sk_buff *skb,
4820 u16 vlan_tag)
4821{
4822 struct igb_adapter *adapter = q_vector->adapter;
4823
4824 if (vlan_tag)
4825 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4826 vlan_tag, skb);
182ff8df 4827 else
047e0030 4828 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 4829}
4830
04a5fcaa 4831static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 4832 u32 status_err, struct sk_buff *skb)
4833{
4834 skb->ip_summed = CHECKSUM_NONE;
4835
4836 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 4837 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4838 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 4839 return;
85ad76b2 4840
9d5c8243
AK		 4841 /* TCP/UDP checksum error bit is set */
4842 if (status_err &
4843 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 4844 /*
4845 * work around errata with sctp packets where the TCPE aka
4846 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4847 * packets, (aka let the stack check the crc32c)
4848 */
85ad76b2
AD		 4849 if ((skb->len == 60) &&
4850 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 4851 ring->rx_stats.csum_err++;
85ad76b2 4852
9d5c8243 4853 /* let the stack verify checksum errors */
9d5c8243
AK		 4854 return;
4855 }
4856 /* It must be a TCP or UDP packet with a valid checksum */
4857 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4858 skb->ip_summed = CHECKSUM_UNNECESSARY;
4859
85ad76b2 4860 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 4861}
4862
c5b9bd5e
AD		 4863static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
4864 struct sk_buff *skb)
4865{
4866 struct igb_adapter *adapter = q_vector->adapter;
4867 struct e1000_hw *hw = &adapter->hw;
4868 u64 regval;
4869
4870 /*
4871 * If this bit is set, then the RX registers contain the time stamp. No
4872 * other packet will be time stamped until we read these registers, so
4873 * read the registers to make them available again. Because only one
4874 * packet can be time stamped at a time, we know that the register
4875 * values must belong to this one here and therefore we don't need to
4876 * compare any of the additional attributes stored for it.
4877 *
4878 * If nothing went wrong, then it should have a skb_shared_tx that we
4879 * can turn into a skb_shared_hwtstamps.
4880 */
4881 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
4882 return;
4883 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
4884 return;
4885
4886 regval = rd32(E1000_RXSTMPL);
4887 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4888
4889 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
4890}
4c844851 4891static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 4892 union e1000_adv_rx_desc *rx_desc)
4893{
4894 /* HW will not DMA in data larger than the given buffer, even if it
4895 * parses the (NFS, of course) header to be larger. In that case, it
4896 * fills the header buffer and spills the rest into the page.
4897 */
4898 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4899 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 4900 if (hlen > rx_ring->rx_buffer_len)
4901 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 4902 return hlen;
4903}
4904
047e0030
AD		 4905static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4906 int *work_done, int budget)
9d5c8243 4907{
047e0030 4908 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 4909 struct net_device *netdev = rx_ring->netdev;
80785298 4910 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 4911 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4912 struct igb_buffer *buffer_info , *next_buffer;
4913 struct sk_buff *skb;
9d5c8243
AK		 4914 bool cleaned = false;
4915 int cleaned_count = 0;
4916 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 4917 unsigned int i;
2d94d8ab
AD		 4918 u32 staterr;
4919 u16 length;
047e0030 4920 u16 vlan_tag;
9d5c8243
AK		 4921
4922 i = rx_ring->next_to_clean;
69d3ca53 4923 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 4924 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4925 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4926
4927 while (staterr & E1000_RXD_STAT_DD) {
4928 if (*work_done >= budget)
4929 break;
4930 (*work_done)++;
9d5c8243 4931
69d3ca53
AD		 4932 skb = buffer_info->skb;
4933 prefetch(skb->data - NET_IP_ALIGN);
4934 buffer_info->skb = NULL;
4935
4936 i++;
4937 if (i == rx_ring->count)
4938 i = 0;
4939 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4940 prefetch(next_rxd);
4941 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 4942
4943 length = le16_to_cpu(rx_desc->wb.upper.length);
4944 cleaned = true;
4945 cleaned_count++;
4946
2d94d8ab 4947 if (buffer_info->dma) {
bf36c1a0 4948 pci_unmap_single(pdev, buffer_info->dma,
4c844851 4949 rx_ring->rx_buffer_len,
bf36c1a0 4950 PCI_DMA_FROMDEVICE);
91615f76 4951 buffer_info->dma = 0;
4c844851 4952 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 4953 skb_put(skb, length);
4954 goto send_up;
4955 }
4c844851 4956 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 4957 }
4958
4959 if (length) {
9d5c8243 4960 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 4961 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 4962 buffer_info->page_dma = 0;
bf36c1a0
AD		 4963
4964 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4965 buffer_info->page,
4966 buffer_info->page_offset,
4967 length);
4968
6ec43fe6 4969 if (page_count(buffer_info->page) != 1)
bf36c1a0
AD		 4970 buffer_info->page = NULL;
4971 else
4972 get_page(buffer_info->page);
9d5c8243
AK		 4973
4974 skb->len += length;
4975 skb->data_len += length;
9d5c8243 4976
bf36c1a0 4977 skb->truesize += length;
9d5c8243 4978 }
9d5c8243 4979
bf36c1a0 4980 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 4981 buffer_info->skb = next_buffer->skb;
4982 buffer_info->dma = next_buffer->dma;
4983 next_buffer->skb = skb;
4984 next_buffer->dma = 0;
bf36c1a0
AD		 4985 goto next_desc;
4986 }
69d3ca53 4987send_up:
9d5c8243
AK		 4988 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
4989 dev_kfree_skb_irq(skb);
4990 goto next_desc;
4991 }
9d5c8243 4992
c5b9bd5e 4993 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 4994 total_bytes += skb->len;
4995 total_packets++;
4996
85ad76b2 4997 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 4998
4999 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5000 skb_record_rx_queue(skb, rx_ring->queue_index);
5001
5002 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5003 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5004
047e0030 5005 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5006
9d5c8243
AK		 5007next_desc:
5008 rx_desc->wb.upper.status_error = 0;
5009
5010 /* return some buffers to hardware, one at a time is too slow */
5011 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5012 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5013 cleaned_count = 0;
5014 }
5015
5016 /* use prefetched values */
5017 rx_desc = next_rxd;
5018 buffer_info = next_buffer;
9d5c8243
AK		 5019 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5020 }
bf36c1a0 5021
9d5c8243 5022 rx_ring->next_to_clean = i;
c493ea45 5023 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5024
5025 if (cleaned_count)
3b644cf6 5026 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5027
5028 rx_ring->total_packets += total_packets;
5029 rx_ring->total_bytes += total_bytes;
5030 rx_ring->rx_stats.packets += total_packets;
5031 rx_ring->rx_stats.bytes += total_bytes;
8d24e933
AK		 5032 netdev->stats.rx_bytes += total_bytes;
5033 netdev->stats.rx_packets += total_packets;
9d5c8243
AK		 5034 return cleaned;
5035}
5036
9d5c8243
AK		 5037/**
5038 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5039 * @adapter: address of board private structure
5040 **/
d7ee5b3a 5041void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5042{
e694e964 5043 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5044 union e1000_adv_rx_desc *rx_desc;
5045 struct igb_buffer *buffer_info;
5046 struct sk_buff *skb;
5047 unsigned int i;
db761762 5048 int bufsz;
9d5c8243
AK		 5049
5050 i = rx_ring->next_to_use;
5051 buffer_info = &rx_ring->buffer_info[i];
5052
4c844851 5053 bufsz = rx_ring->rx_buffer_len;
db761762 5054
9d5c8243
AK		 5055 while (cleaned_count--) {
5056 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5057
6ec43fe6 5058 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5059 if (!buffer_info->page) {
bf36c1a0
AD		 5060 buffer_info->page = alloc_page(GFP_ATOMIC);
5061 if (!buffer_info->page) {
04a5fcaa 5062 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5063 goto no_buffers;
5064 }
5065 buffer_info->page_offset = 0;
5066 } else {
5067 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5068 }
5069 buffer_info->page_dma =
80785298 5070 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5071 buffer_info->page_offset,
5072 PAGE_SIZE / 2,
9d5c8243
AK		 5073 PCI_DMA_FROMDEVICE);
5074 }
5075
5076 if (!buffer_info->skb) {
89d71a66 5077 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5078 if (!skb) {
04a5fcaa 5079 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5080 goto no_buffers;
5081 }
5082
9d5c8243 5083 buffer_info->skb = skb;
80785298
AD		 5084 buffer_info->dma = pci_map_single(rx_ring->pdev,
5085 skb->data,
9d5c8243
AK		 5086 bufsz,
5087 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 5088 }
5089 /* Refresh the desc even if buffer_addrs didn't change because
5090 * each write-back erases this info. */
6ec43fe6 5091 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5092 rx_desc->read.pkt_addr =
5093 cpu_to_le64(buffer_info->page_dma);
5094 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5095 } else {
5096 rx_desc->read.pkt_addr =
5097 cpu_to_le64(buffer_info->dma);
5098 rx_desc->read.hdr_addr = 0;
5099 }
5100
5101 i++;
5102 if (i == rx_ring->count)
5103 i = 0;
5104 buffer_info = &rx_ring->buffer_info[i];
5105 }
5106
5107no_buffers:
5108 if (rx_ring->next_to_use != i) {
5109 rx_ring->next_to_use = i;
5110 if (i == 0)
5111 i = (rx_ring->count - 1);
5112 else
5113 i--;
5114
5115 /* Force memory writes to complete before letting h/w
5116 * know there are new descriptors to fetch. (Only
5117 * applicable for weak-ordered memory model archs,
5118 * such as IA-64). */
5119 wmb();
fce99e34 5120 writel(i, rx_ring->tail);
9d5c8243
AK		 5121 }
5122}
5123
5124/**
5125 * igb_mii_ioctl -
5126 * @netdev:
5127 * @ifreq:
5128 * @cmd:
5129 **/
5130static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5131{
5132 struct igb_adapter *adapter = netdev_priv(netdev);
5133 struct mii_ioctl_data *data = if_mii(ifr);
5134
5135 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5136 return -EOPNOTSUPP;
5137
5138 switch (cmd) {
5139 case SIOCGMIIPHY:
5140 data->phy_id = adapter->hw.phy.addr;
5141 break;
5142 case SIOCGMIIREG:
f5f4cf08
AD		 5143 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5144 &data->val_out))
9d5c8243
AK		 5145 return -EIO;
5146 break;
5147 case SIOCSMIIREG:
5148 default:
5149 return -EOPNOTSUPP;
5150 }
5151 return 0;
5152}
5153
c6cb090b
PO		 5154/**
5155 * igb_hwtstamp_ioctl - control hardware time stamping
5156 * @netdev:
5157 * @ifreq:
5158 * @cmd:
5159 *
33af6bcc
PO		 5160 * Outgoing time stamping can be enabled and disabled. Play nice and
5161 * disable it when requested, although it shouldn't case any overhead
5162 * when no packet needs it. At most one packet in the queue may be
5163 * marked for time stamping, otherwise it would be impossible to tell
5164 * for sure to which packet the hardware time stamp belongs.
5165 *
5166 * Incoming time stamping has to be configured via the hardware
5167 * filters. Not all combinations are supported, in particular event
5168 * type has to be specified. Matching the kind of event packet is
5169 * not supported, with the exception of "all V2 events regardless of
5170 * level 2 or 4".
5171 *
c6cb090b
PO		 5172 **/
5173static int igb_hwtstamp_ioctl(struct net_device *netdev,
5174 struct ifreq *ifr, int cmd)
5175{
33af6bcc
PO		 5176 struct igb_adapter *adapter = netdev_priv(netdev);
5177 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5178 struct hwtstamp_config config;
c5b9bd5e
AD		 5179 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5180 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5181 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5182 bool is_l4 = false;
5183 bool is_l2 = false;
33af6bcc 5184 u32 regval;
c6cb090b
PO		 5185
5186 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5187 return -EFAULT;
5188
5189 /* reserved for future extensions */
5190 if (config.flags)
5191 return -EINVAL;
5192
33af6bcc
PO		 5193 switch (config.tx_type) {
5194 case HWTSTAMP_TX_OFF:
c5b9bd5e 5195 tsync_tx_ctl = 0;
33af6bcc 5196 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5197 break;
5198 default:
5199 return -ERANGE;
5200 }
5201
5202 switch (config.rx_filter) {
5203 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5204 tsync_rx_ctl = 0;
33af6bcc
PO		 5205 break;
5206 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5207 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5208 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5209 case HWTSTAMP_FILTER_ALL:
5210 /*
5211 * register TSYNCRXCFG must be set, therefore it is not
5212 * possible to time stamp both Sync and Delay_Req messages
5213 * => fall back to time stamping all packets
5214 */
c5b9bd5e 5215 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5216 config.rx_filter = HWTSTAMP_FILTER_ALL;
5217 break;
5218 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5219 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5220 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5221 is_l4 = true;
33af6bcc
PO		 5222 break;
5223 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5224 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5225 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5226 is_l4 = true;
33af6bcc
PO		 5227 break;
5228 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5229 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5230 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5231 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5232 is_l2 = true;
5233 is_l4 = true;
33af6bcc
PO		 5234 config.rx_filter = HWTSTAMP_FILTER_SOME;
5235 break;
5236 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5237 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5238 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5239 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5240 is_l2 = true;
5241 is_l4 = true;
33af6bcc
PO		 5242 config.rx_filter = HWTSTAMP_FILTER_SOME;
5243 break;
5244 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5245 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5246 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5247 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5248 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5249 is_l2 = true;
33af6bcc
PO		 5250 break;
5251 default:
5252 return -ERANGE;
5253 }
5254
c5b9bd5e
AD		 5255 if (hw->mac.type == e1000_82575) {
5256 if (tsync_rx_ctl | tsync_tx_ctl)
5257 return -EINVAL;
5258 return 0;
5259 }
5260
33af6bcc
PO		 5261 /* enable/disable TX */
5262 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5263 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5264 regval |= tsync_tx_ctl;
33af6bcc
PO		 5265 wr32(E1000_TSYNCTXCTL, regval);
5266
c5b9bd5e 5267 /* enable/disable RX */
33af6bcc 5268 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5269 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5270 regval |= tsync_rx_ctl;
33af6bcc 5271 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5272
c5b9bd5e
AD		 5273 /* define which PTP packets are time stamped */
5274 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5275
c5b9bd5e
AD		 5276 /* define ethertype filter for timestamped packets */
5277 if (is_l2)
5278 wr32(E1000_ETQF(3),
5279 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5280 E1000_ETQF_1588 | /* enable timestamping */
5281 ETH_P_1588)); /* 1588 eth protocol type */
5282 else
5283 wr32(E1000_ETQF(3), 0);
5284
5285#define PTP_PORT 319
5286 /* L4 Queue Filter[3]: filter by destination port and protocol */
5287 if (is_l4) {
5288 u32 ftqf = (IPPROTO_UDP /* UDP */
5289 | E1000_FTQF_VF_BP /* VF not compared */
5290 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5291 | E1000_FTQF_MASK); /* mask all inputs */
5292 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5293
5294 wr32(E1000_IMIR(3), htons(PTP_PORT));
5295 wr32(E1000_IMIREXT(3),
5296 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5297 if (hw->mac.type == e1000_82576) {
5298 /* enable source port check */
5299 wr32(E1000_SPQF(3), htons(PTP_PORT));
5300 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5301 }
5302 wr32(E1000_FTQF(3), ftqf);
5303 } else {
5304 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5305 }
33af6bcc
PO		 5306 wrfl();
5307
5308 adapter->hwtstamp_config = config;
5309
5310 /* clear TX/RX time stamp registers, just to be sure */
5311 regval = rd32(E1000_TXSTMPH);
5312 regval = rd32(E1000_RXSTMPH);
c6cb090b 5313
33af6bcc
PO		 5314 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5315 -EFAULT : 0;
c6cb090b
PO		 5316}
5317
9d5c8243
AK		 5318/**
5319 * igb_ioctl -
5320 * @netdev:
5321 * @ifreq:
5322 * @cmd:
5323 **/
5324static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5325{
5326 switch (cmd) {
5327 case SIOCGMIIPHY:
5328 case SIOCGMIIREG:
5329 case SIOCSMIIREG:
5330 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5331 case SIOCSHWTSTAMP:
5332 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5333 default:
5334 return -EOPNOTSUPP;
5335 }
5336}
5337
009bc06e
AD		 5338s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5339{
5340 struct igb_adapter *adapter = hw->back;
5341 u16 cap_offset;
5342
5343 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5344 if (!cap_offset)
5345 return -E1000_ERR_CONFIG;
5346
5347 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5348
5349 return 0;
5350}
5351
5352s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5353{
5354 struct igb_adapter *adapter = hw->back;
5355 u16 cap_offset;
5356
5357 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5358 if (!cap_offset)
5359 return -E1000_ERR_CONFIG;
5360
5361 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5362
5363 return 0;
5364}
5365
9d5c8243
AK		 5366static void igb_vlan_rx_register(struct net_device *netdev,
5367 struct vlan_group *grp)
5368{
5369 struct igb_adapter *adapter = netdev_priv(netdev);
5370 struct e1000_hw *hw = &adapter->hw;
5371 u32 ctrl, rctl;
5372
5373 igb_irq_disable(adapter);
5374 adapter->vlgrp = grp;
5375
5376 if (grp) {
5377 /* enable VLAN tag insert/strip */
5378 ctrl = rd32(E1000_CTRL);
5379 ctrl |= E1000_CTRL_VME;
5380 wr32(E1000_CTRL, ctrl);
5381
51466239 5382 /* Disable CFI check */
9d5c8243 5383 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5384 rctl &= ~E1000_RCTL_CFIEN;
5385 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5386 } else {
5387 /* disable VLAN tag insert/strip */
5388 ctrl = rd32(E1000_CTRL);
5389 ctrl &= ~E1000_CTRL_VME;
5390 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5391 }
5392
e1739522
AD		 5393 igb_rlpml_set(adapter);
5394
9d5c8243
AK		 5395 if (!test_bit(__IGB_DOWN, &adapter->state))
5396 igb_irq_enable(adapter);
5397}
5398
5399static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5400{
5401 struct igb_adapter *adapter = netdev_priv(netdev);
5402 struct e1000_hw *hw = &adapter->hw;
4ae196df 5403 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5404
51466239
AD		 5405 /* attempt to add filter to vlvf array */
5406 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5407
51466239
AD		 5408 /* add the filter since PF can receive vlans w/o entry in vlvf */
5409 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5410}
5411
5412static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5413{
5414 struct igb_adapter *adapter = netdev_priv(netdev);
5415 struct e1000_hw *hw = &adapter->hw;
4ae196df 5416 int pf_id = adapter->vfs_allocated_count;
51466239 5417 s32 err;
9d5c8243
AK		 5418
5419 igb_irq_disable(adapter);
5420 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5421
5422 if (!test_bit(__IGB_DOWN, &adapter->state))
5423 igb_irq_enable(adapter);
5424
51466239
AD		 5425 /* remove vlan from VLVF table array */
5426 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5427
51466239
AD		 5428 /* if vid was not present in VLVF just remove it from table */
5429 if (err)
4ae196df 5430 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5431}
5432
5433static void igb_restore_vlan(struct igb_adapter *adapter)
5434{
5435 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5436
5437 if (adapter->vlgrp) {
5438 u16 vid;
5439 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5440 if (!vlan_group_get_device(adapter->vlgrp, vid))
5441 continue;
5442 igb_vlan_rx_add_vid(adapter->netdev, vid);
5443 }
5444 }
5445}
5446
5447int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5448{
5449 struct e1000_mac_info *mac = &adapter->hw.mac;
5450
5451 mac->autoneg = 0;
5452
9d5c8243
AK		 5453 switch (spddplx) {
5454 case SPEED_10 + DUPLEX_HALF:
5455 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5456 break;
5457 case SPEED_10 + DUPLEX_FULL:
5458 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5459 break;
5460 case SPEED_100 + DUPLEX_HALF:
5461 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5462 break;
5463 case SPEED_100 + DUPLEX_FULL:
5464 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5465 break;
5466 case SPEED_1000 + DUPLEX_FULL:
5467 mac->autoneg = 1;
5468 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5469 break;
5470 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5471 default:
5472 dev_err(&adapter->pdev->dev,
5473 "Unsupported Speed/Duplex configuration\n");
5474 return -EINVAL;
5475 }
5476 return 0;
5477}
5478
3fe7c4c9 5479static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5480{
5481 struct net_device *netdev = pci_get_drvdata(pdev);
5482 struct igb_adapter *adapter = netdev_priv(netdev);
5483 struct e1000_hw *hw = &adapter->hw;
2d064c06 5484 u32 ctrl, rctl, status;
9d5c8243
AK		 5485 u32 wufc = adapter->wol;
5486#ifdef CONFIG_PM
5487 int retval = 0;
5488#endif
5489
5490 netif_device_detach(netdev);
5491
a88f10ec
AD		 5492 if (netif_running(netdev))
5493 igb_close(netdev);
5494
047e0030 5495 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5496
5497#ifdef CONFIG_PM
5498 retval = pci_save_state(pdev);
5499 if (retval)
5500 return retval;
5501#endif
5502
5503 status = rd32(E1000_STATUS);
5504 if (status & E1000_STATUS_LU)
5505 wufc &= ~E1000_WUFC_LNKC;
5506
5507 if (wufc) {
5508 igb_setup_rctl(adapter);
ff41f8dc 5509 igb_set_rx_mode(netdev);
9d5c8243
AK		 5510
5511 /* turn on all-multi mode if wake on multicast is enabled */
5512 if (wufc & E1000_WUFC_MC) {
5513 rctl = rd32(E1000_RCTL);
5514 rctl |= E1000_RCTL_MPE;
5515 wr32(E1000_RCTL, rctl);
5516 }
5517
5518 ctrl = rd32(E1000_CTRL);
5519 /* advertise wake from D3Cold */
5520 #define E1000_CTRL_ADVD3WUC 0x00100000
5521 /* phy power management enable */
5522 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5523 ctrl |= E1000_CTRL_ADVD3WUC;
5524 wr32(E1000_CTRL, ctrl);
5525
9d5c8243
AK		 5526 /* Allow time for pending master requests to run */
5527 igb_disable_pcie_master(&adapter->hw);
5528
5529 wr32(E1000_WUC, E1000_WUC_PME_EN);
5530 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5531 } else {
5532 wr32(E1000_WUC, 0);
5533 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5534 }
5535
3fe7c4c9
RW		 5536 *enable_wake = wufc || adapter->en_mng_pt;
5537 if (!*enable_wake)
2fb02a26 5538 igb_shutdown_serdes_link_82575(hw);
9d5c8243
AK		 5539
5540 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5541 * would have already happened in close and is redundant. */
5542 igb_release_hw_control(adapter);
5543
5544 pci_disable_device(pdev);
5545
9d5c8243
AK		 5546 return 0;
5547}
5548
5549#ifdef CONFIG_PM
3fe7c4c9
RW		 5550static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5551{
5552 int retval;
5553 bool wake;
5554
5555 retval = __igb_shutdown(pdev, &wake);
5556 if (retval)
5557 return retval;
5558
5559 if (wake) {
5560 pci_prepare_to_sleep(pdev);
5561 } else {
5562 pci_wake_from_d3(pdev, false);
5563 pci_set_power_state(pdev, PCI_D3hot);
5564 }
5565
5566 return 0;
5567}
5568
9d5c8243
AK		 5569static int igb_resume(struct pci_dev *pdev)
5570{
5571 struct net_device *netdev = pci_get_drvdata(pdev);
5572 struct igb_adapter *adapter = netdev_priv(netdev);
5573 struct e1000_hw *hw = &adapter->hw;
5574 u32 err;
5575
5576 pci_set_power_state(pdev, PCI_D0);
5577 pci_restore_state(pdev);
42bfd33a 5578
aed5dec3 5579 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5580 if (err) {
5581 dev_err(&pdev->dev,
5582 "igb: Cannot enable PCI device from suspend\n");
5583 return err;
5584 }
5585 pci_set_master(pdev);
5586
5587 pci_enable_wake(pdev, PCI_D3hot, 0);
5588 pci_enable_wake(pdev, PCI_D3cold, 0);
5589
047e0030 5590 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5591 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5592 return -ENOMEM;
9d5c8243
AK		 5593 }
5594
5595 /* e1000_power_up_phy(adapter); */
5596
5597 igb_reset(adapter);
a8564f03
AD		 5598
5599 /* let the f/w know that the h/w is now under the control of the
5600 * driver. */
5601 igb_get_hw_control(adapter);
5602
9d5c8243
AK		 5603 wr32(E1000_WUS, ~0);
5604
a88f10ec
AD		 5605 if (netif_running(netdev)) {
5606 err = igb_open(netdev);
5607 if (err)
5608 return err;
5609 }
9d5c8243
AK		 5610
5611 netif_device_attach(netdev);
5612
9d5c8243
AK		 5613 return 0;
5614}
5615#endif
5616
5617static void igb_shutdown(struct pci_dev *pdev)
5618{
3fe7c4c9
RW		 5619 bool wake;
5620
5621 __igb_shutdown(pdev, &wake);
5622
5623 if (system_state == SYSTEM_POWER_OFF) {
5624 pci_wake_from_d3(pdev, wake);
5625 pci_set_power_state(pdev, PCI_D3hot);
5626 }
9d5c8243
AK		 5627}
5628
5629#ifdef CONFIG_NET_POLL_CONTROLLER
5630/*
5631 * Polling 'interrupt' - used by things like netconsole to send skbs
5632 * without having to re-enable interrupts. It's not called while
5633 * the interrupt routine is executing.
5634 */
5635static void igb_netpoll(struct net_device *netdev)
5636{
5637 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5638 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5639 int i;
9d5c8243 5640
eebbbdba 5641 if (!adapter->msix_entries) {
047e0030 5642 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5643 igb_irq_disable(adapter);
047e0030 5644 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5645 return;
5646 }
9d5c8243 5647
047e0030
AD		 5648 for (i = 0; i < adapter->num_q_vectors; i++) {
5649 struct igb_q_vector *q_vector = adapter->q_vector[i];
5650 wr32(E1000_EIMC, q_vector->eims_value);
5651 napi_schedule(&q_vector->napi);
eebbbdba 5652 }
9d5c8243
AK		 5653}
5654#endif /* CONFIG_NET_POLL_CONTROLLER */
5655
5656/**
5657 * igb_io_error_detected - called when PCI error is detected
5658 * @pdev: Pointer to PCI device
5659 * @state: The current pci connection state
5660 *
5661 * This function is called after a PCI bus error affecting
5662 * this device has been detected.
5663 */
5664static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5665 pci_channel_state_t state)
5666{
5667 struct net_device *netdev = pci_get_drvdata(pdev);
5668 struct igb_adapter *adapter = netdev_priv(netdev);
5669
5670 netif_device_detach(netdev);
5671
59ed6eec
AD		 5672 if (state == pci_channel_io_perm_failure)
5673 return PCI_ERS_RESULT_DISCONNECT;
5674
9d5c8243
AK		 5675 if (netif_running(netdev))
5676 igb_down(adapter);
5677 pci_disable_device(pdev);
5678
5679 /* Request a slot slot reset. */
5680 return PCI_ERS_RESULT_NEED_RESET;
5681}
5682
5683/**
5684 * igb_io_slot_reset - called after the pci bus has been reset.
5685 * @pdev: Pointer to PCI device
5686 *
5687 * Restart the card from scratch, as if from a cold-boot. Implementation
5688 * resembles the first-half of the igb_resume routine.
5689 */
5690static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5691{
5692 struct net_device *netdev = pci_get_drvdata(pdev);
5693 struct igb_adapter *adapter = netdev_priv(netdev);
5694 struct e1000_hw *hw = &adapter->hw;
40a914fa 5695 pci_ers_result_t result;
42bfd33a 5696 int err;
9d5c8243 5697
aed5dec3 5698 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 5699 dev_err(&pdev->dev,
5700 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 5701 result = PCI_ERS_RESULT_DISCONNECT;
5702 } else {
5703 pci_set_master(pdev);
5704 pci_restore_state(pdev);
9d5c8243 5705
40a914fa
AD		 5706 pci_enable_wake(pdev, PCI_D3hot, 0);
5707 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 5708
40a914fa
AD		 5709 igb_reset(adapter);
5710 wr32(E1000_WUS, ~0);
5711 result = PCI_ERS_RESULT_RECOVERED;
5712 }
9d5c8243 5713
ea943d41
JK		 5714 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5715 if (err) {
5716 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5717 "failed 0x%0x\n", err);
5718 /* non-fatal, continue */
5719 }
40a914fa
AD		 5720
5721 return result;
9d5c8243
AK		 5722}
5723
5724/**
5725 * igb_io_resume - called when traffic can start flowing again.
5726 * @pdev: Pointer to PCI device
5727 *
5728 * This callback is called when the error recovery driver tells us that
5729 * its OK to resume normal operation. Implementation resembles the
5730 * second-half of the igb_resume routine.
5731 */
5732static void igb_io_resume(struct pci_dev *pdev)
5733{
5734 struct net_device *netdev = pci_get_drvdata(pdev);
5735 struct igb_adapter *adapter = netdev_priv(netdev);
5736
9d5c8243
AK		 5737 if (netif_running(netdev)) {
5738 if (igb_up(adapter)) {
5739 dev_err(&pdev->dev, "igb_up failed after reset\n");
5740 return;
5741 }
5742 }
5743
5744 netif_device_attach(netdev);
5745
5746 /* let the f/w know that the h/w is now under the control of the
5747 * driver. */
5748 igb_get_hw_control(adapter);
9d5c8243
AK		 5749}
5750
26ad9178
AD		 5751static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5752 u8 qsel)
5753{
5754 u32 rar_low, rar_high;
5755 struct e1000_hw *hw = &adapter->hw;
5756
5757 /* HW expects these in little endian so we reverse the byte order
5758 * from network order (big endian) to little endian
5759 */
5760 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5761 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5762 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5763
5764 /* Indicate to hardware the Address is Valid. */
5765 rar_high |= E1000_RAH_AV;
5766
5767 if (hw->mac.type == e1000_82575)
5768 rar_high |= E1000_RAH_POOL_1 * qsel;
5769 else
5770 rar_high |= E1000_RAH_POOL_1 << qsel;
5771
5772 wr32(E1000_RAL(index), rar_low);
5773 wrfl();
5774 wr32(E1000_RAH(index), rar_high);
5775 wrfl();
5776}
5777
4ae196df
AD		 5778static int igb_set_vf_mac(struct igb_adapter *adapter,
5779 int vf, unsigned char *mac_addr)
5780{
5781 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 5782 /* VF MAC addresses start at end of receive addresses and moves
5783 * torwards the first, as a result a collision should not be possible */
5784 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 5785
37680117 5786 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 5787
26ad9178 5788 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 5789
5790 return 0;
5791}
5792
5793static void igb_vmm_control(struct igb_adapter *adapter)
5794{
5795 struct e1000_hw *hw = &adapter->hw;
10d8e907 5796 u32 reg;
4ae196df 5797
d4960307
AD		 5798 /* replication is not supported for 82575 */
5799 if (hw->mac.type == e1000_82575)
4ae196df
AD		 5800 return;
5801
10d8e907
AD		 5802 /* enable replication vlan tag stripping */
5803 reg = rd32(E1000_RPLOLR);
5804 reg |= E1000_RPLOLR_STRVLAN;
5805 wr32(E1000_RPLOLR, reg);
5806
5807 /* notify HW that the MAC is adding vlan tags */
5808 reg = rd32(E1000_DTXCTL);
5809 reg |= E1000_DTXCTL_VLAN_ADDED;
5810 wr32(E1000_DTXCTL, reg);
5811
d4960307
AD		 5812 if (adapter->vfs_allocated_count) {
5813 igb_vmdq_set_loopback_pf(hw, true);
5814 igb_vmdq_set_replication_pf(hw, true);
5815 } else {
5816 igb_vmdq_set_loopback_pf(hw, false);
5817 igb_vmdq_set_replication_pf(hw, false);
5818 }
4ae196df
AD		 5819}
5820
9d5c8243 5821/* igb_main.c */
kernel source for telekom puls (alcatel/mediatek)