projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: change queue ordering for 82576 based adapters
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
86d5d38f 52#define DRV_VERSION "1.3.16-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
63static struct pci_device_id igb_pci_tbl[] = {
2d064c06 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
75 {0, }
76};
77
78MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
79
80void igb_reset(struct igb_adapter *);
81static int igb_setup_all_tx_resources(struct igb_adapter *);
82static int igb_setup_all_rx_resources(struct igb_adapter *);
83static void igb_free_all_tx_resources(struct igb_adapter *);
84static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 85static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 86void igb_update_stats(struct igb_adapter *);
87static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88static void __devexit igb_remove(struct pci_dev *pdev);
89static int igb_sw_init(struct igb_adapter *);
90static int igb_open(struct net_device *);
91static int igb_close(struct net_device *);
92static void igb_configure_tx(struct igb_adapter *);
93static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 94static void igb_clean_all_tx_rings(struct igb_adapter *);
95static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 96static void igb_clean_tx_ring(struct igb_ring *);
97static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 98static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 99static void igb_update_phy_info(unsigned long);
100static void igb_watchdog(unsigned long);
101static void igb_watchdog_task(struct work_struct *);
b1a436c3 102static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 103static struct net_device_stats *igb_get_stats(struct net_device *);
104static int igb_change_mtu(struct net_device *, int);
105static int igb_set_mac(struct net_device *, void *);
68d480c4 106static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 107static irqreturn_t igb_intr(int irq, void *);
108static irqreturn_t igb_intr_msi(int irq, void *);
109static irqreturn_t igb_msix_other(int irq, void *);
047e0030 110static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 111#ifdef CONFIG_IGB_DCA
047e0030 112static void igb_update_dca(struct igb_q_vector *);
fe4506b6 113static void igb_setup_dca(struct igb_adapter *);
421e02f0 114#endif /* CONFIG_IGB_DCA */
047e0030 115static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 116static int igb_poll(struct napi_struct *, int);
047e0030 117static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 118static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
119static void igb_tx_timeout(struct net_device *);
120static void igb_reset_task(struct work_struct *);
121static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
122static void igb_vlan_rx_add_vid(struct net_device *, u16);
123static void igb_vlan_rx_kill_vid(struct net_device *, u16);
124static void igb_restore_vlan(struct igb_adapter *);
26ad9178 125static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 126static void igb_ping_all_vfs(struct igb_adapter *);
127static void igb_msg_task(struct igb_adapter *);
4ae196df 128static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 129static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 130static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
9d5c8243 131
9d5c8243 132#ifdef CONFIG_PM
3fe7c4c9 133static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 134static int igb_resume(struct pci_dev *);
135#endif
136static void igb_shutdown(struct pci_dev *);
421e02f0 137#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 138static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
139static struct notifier_block dca_notifier = {
140 .notifier_call = igb_notify_dca,
141 .next = NULL,
142 .priority = 0
143};
144#endif
9d5c8243
AK		 145#ifdef CONFIG_NET_POLL_CONTROLLER
146/* for netdump / net console */
147static void igb_netpoll(struct net_device *);
148#endif
37680117 149#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 150static unsigned int max_vfs = 0;
151module_param(max_vfs, uint, 0);
152MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
153 "per physical function");
154#endif /* CONFIG_PCI_IOV */
155
9d5c8243
AK		 156static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
157 pci_channel_state_t);
158static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
159static void igb_io_resume(struct pci_dev *);
160
161static struct pci_error_handlers igb_err_handler = {
162 .error_detected = igb_io_error_detected,
163 .slot_reset = igb_io_slot_reset,
164 .resume = igb_io_resume,
165};
166
167
168static struct pci_driver igb_driver = {
169 .name = igb_driver_name,
170 .id_table = igb_pci_tbl,
171 .probe = igb_probe,
172 .remove = __devexit_p(igb_remove),
173#ifdef CONFIG_PM
174 /* Power Managment Hooks */
175 .suspend = igb_suspend,
176 .resume = igb_resume,
177#endif
178 .shutdown = igb_shutdown,
179 .err_handler = &igb_err_handler
180};
181
182MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
183MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
184MODULE_LICENSE("GPL");
185MODULE_VERSION(DRV_VERSION);
186
38c845c7
PO		 187/**
188 * igb_read_clock - read raw cycle counter (to be used by time counter)
189 */
190static cycle_t igb_read_clock(const struct cyclecounter *tc)
191{
192 struct igb_adapter *adapter =
193 container_of(tc, struct igb_adapter, cycles);
194 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 195 u64 stamp = 0;
196 int shift = 0;
38c845c7 197
c5b9bd5e
AD		 198 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
199 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 200 return stamp;
201}
202
9d5c8243
AK		 203#ifdef DEBUG
204/**
205 * igb_get_hw_dev_name - return device name string
206 * used by hardware layer to print debugging information
207 **/
208char *igb_get_hw_dev_name(struct e1000_hw *hw)
209{
210 struct igb_adapter *adapter = hw->back;
211 return adapter->netdev->name;
212}
38c845c7
PO		 213
214/**
215 * igb_get_time_str - format current NIC and system time as string
216 */
217static char *igb_get_time_str(struct igb_adapter *adapter,
218 char buffer[160])
219{
220 cycle_t hw = adapter->cycles.read(&adapter->cycles);
221 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
222 struct timespec sys;
223 struct timespec delta;
224 getnstimeofday(&sys);
225
226 delta = timespec_sub(nic, sys);
227
228 sprintf(buffer,
33af6bcc
PO		 229 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
230 hw,
38c845c7
PO		 231 (long)nic.tv_sec, nic.tv_nsec,
232 (long)sys.tv_sec, sys.tv_nsec,
233 (long)delta.tv_sec, delta.tv_nsec);
234
235 return buffer;
236}
9d5c8243
AK		 237#endif
238
239/**
240 * igb_init_module - Driver Registration Routine
241 *
242 * igb_init_module is the first routine called when the driver is
243 * loaded. All it does is register with the PCI subsystem.
244 **/
245static int __init igb_init_module(void)
246{
247 int ret;
248 printk(KERN_INFO "%s - version %s\n",
249 igb_driver_string, igb_driver_version);
250
251 printk(KERN_INFO "%s\n", igb_copyright);
252
421e02f0 253#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 254 dca_register_notify(&dca_notifier);
255#endif
bbd98fe4 256 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 257 return ret;
258}
259
260module_init(igb_init_module);
261
262/**
263 * igb_exit_module - Driver Exit Cleanup Routine
264 *
265 * igb_exit_module is called just before the driver is removed
266 * from memory.
267 **/
268static void __exit igb_exit_module(void)
269{
421e02f0 270#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 271 dca_unregister_notify(&dca_notifier);
272#endif
9d5c8243
AK		 273 pci_unregister_driver(&igb_driver);
274}
275
276module_exit(igb_exit_module);
277
26bc19ec
AD		 278#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
279/**
280 * igb_cache_ring_register - Descriptor ring to register mapping
281 * @adapter: board private structure to initialize
282 *
283 * Once we know the feature-set enabled for the device, we'll cache
284 * the register offset the descriptor ring is assigned to.
285 **/
286static void igb_cache_ring_register(struct igb_adapter *adapter)
287{
ee1b9f06 288 int i = 0, j = 0;
047e0030 289 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 290
291 switch (adapter->hw.mac.type) {
292 case e1000_82576:
293 /* The queues are allocated for virtualization such that VF 0
294 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
295 * In order to avoid collision we start at the first free queue
296 * and continue consuming queues in the same sequence
297 */
ee1b9f06
AD		 298 if (adapter->vfs_allocated_count) {
299 for (; i < adapter->num_rx_queues; i++)
300 adapter->rx_ring[i].reg_idx = rbase_offset +
301 Q_IDX_82576(i);
302 for (; j < adapter->num_tx_queues; j++)
303 adapter->tx_ring[j].reg_idx = rbase_offset +
304 Q_IDX_82576(j);
305 }
26bc19ec
AD		 306 case e1000_82575:
307 default:
ee1b9f06
AD		 308 for (; i < adapter->num_rx_queues; i++)
309 adapter->rx_ring[i].reg_idx = rbase_offset + i;
310 for (; j < adapter->num_tx_queues; j++)
311 adapter->tx_ring[j].reg_idx = rbase_offset + j;
26bc19ec
AD		 312 break;
313 }
314}
315
047e0030
AD		 316static void igb_free_queues(struct igb_adapter *adapter)
317{
318 kfree(adapter->tx_ring);
319 kfree(adapter->rx_ring);
320
321 adapter->tx_ring = NULL;
322 adapter->rx_ring = NULL;
323
324 adapter->num_rx_queues = 0;
325 adapter->num_tx_queues = 0;
326}
327
9d5c8243
AK		 328/**
329 * igb_alloc_queues - Allocate memory for all rings
330 * @adapter: board private structure to initialize
331 *
332 * We allocate one ring per queue at run-time since we don't know the
333 * number of queues at compile-time.
334 **/
335static int igb_alloc_queues(struct igb_adapter *adapter)
336{
337 int i;
338
339 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
340 sizeof(struct igb_ring), GFP_KERNEL);
341 if (!adapter->tx_ring)
047e0030 342 goto err;
9d5c8243
AK		 343
344 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
345 sizeof(struct igb_ring), GFP_KERNEL);
047e0030
AD		 346 if (!adapter->rx_ring)
347 goto err;
6eb5a7f1 348
661086df
PWJ		 349 for (i = 0; i < adapter->num_tx_queues; i++) {
350 struct igb_ring *ring = &(adapter->tx_ring[i]);
68fd9910 351 ring->count = adapter->tx_ring_count;
661086df 352 ring->queue_index = i;
80785298 353 ring->pdev = adapter->pdev;
e694e964 354 ring->netdev = adapter->netdev;
85ad76b2
AD		 355 /* For 82575, context index must be unique per ring. */
356 if (adapter->hw.mac.type == e1000_82575)
357 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
661086df 358 }
85ad76b2 359
9d5c8243
AK		 360 for (i = 0; i < adapter->num_rx_queues; i++) {
361 struct igb_ring *ring = &(adapter->rx_ring[i]);
68fd9910 362 ring->count = adapter->rx_ring_count;
844290e5 363 ring->queue_index = i;
80785298 364 ring->pdev = adapter->pdev;
e694e964 365 ring->netdev = adapter->netdev;
4c844851 366 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 367 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
368 /* set flag indicating ring supports SCTP checksum offload */
369 if (adapter->hw.mac.type >= e1000_82576)
370 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
9d5c8243 371 }
26bc19ec
AD		 372
373 igb_cache_ring_register(adapter);
9d5c8243 374
047e0030 375 return 0;
a88f10ec 376
047e0030
AD		 377err:
378 igb_free_queues(adapter);
d1a8c9e1 379
047e0030 380 return -ENOMEM;
a88f10ec
AD		 381}
382
9d5c8243 383#define IGB_N0_QUEUE -1
047e0030 384static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 385{
386 u32 msixbm = 0;
047e0030 387 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 388 struct e1000_hw *hw = &adapter->hw;
2d064c06 389 u32 ivar, index;
047e0030
AD		 390 int rx_queue = IGB_N0_QUEUE;
391 int tx_queue = IGB_N0_QUEUE;
392
393 if (q_vector->rx_ring)
394 rx_queue = q_vector->rx_ring->reg_idx;
395 if (q_vector->tx_ring)
396 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 397
398 switch (hw->mac.type) {
399 case e1000_82575:
9d5c8243
AK		 400 /* The 82575 assigns vectors using a bitmask, which matches the
401 bitmask for the EICR/EIMS/EIMC registers. To assign one
402 or more queues to a vector, we write the appropriate bits
403 into the MSIXBM register for that vector. */
047e0030 404 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 405 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 406 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 407 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
9d5c8243 408 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 409 q_vector->eims_value = msixbm;
2d064c06
AD		 410 break;
411 case e1000_82576:
26bc19ec 412 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 413 Each queue has a single entry in the table to which we write
414 a vector number along with a "valid" bit. Sadly, the layout
415 of the table is somewhat counterintuitive. */
416 if (rx_queue > IGB_N0_QUEUE) {
047e0030 417 index = (rx_queue & 0x7);
2d064c06 418 ivar = array_rd32(E1000_IVAR0, index);
047e0030 419 if (rx_queue < 8) {
26bc19ec
AD		 420 /* vector goes into low byte of register */
421 ivar = ivar & 0xFFFFFF00;
422 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 423 } else {
424 /* vector goes into third byte of register */
425 ivar = ivar & 0xFF00FFFF;
426 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 427 }
2d064c06
AD		 428 array_wr32(E1000_IVAR0, index, ivar);
429 }
430 if (tx_queue > IGB_N0_QUEUE) {
047e0030 431 index = (tx_queue & 0x7);
2d064c06 432 ivar = array_rd32(E1000_IVAR0, index);
047e0030 433 if (tx_queue < 8) {
26bc19ec
AD		 434 /* vector goes into second byte of register */
435 ivar = ivar & 0xFFFF00FF;
436 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 437 } else {
438 /* vector goes into high byte of register */
439 ivar = ivar & 0x00FFFFFF;
440 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 441 }
2d064c06
AD		 442 array_wr32(E1000_IVAR0, index, ivar);
443 }
047e0030 444 q_vector->eims_value = 1 << msix_vector;
2d064c06
AD		 445 break;
446 default:
447 BUG();
448 break;
449 }
9d5c8243
AK		 450}
451
452/**
453 * igb_configure_msix - Configure MSI-X hardware
454 *
455 * igb_configure_msix sets up the hardware to properly
456 * generate MSI-X interrupts.
457 **/
458static void igb_configure_msix(struct igb_adapter *adapter)
459{
460 u32 tmp;
461 int i, vector = 0;
462 struct e1000_hw *hw = &adapter->hw;
463
464 adapter->eims_enable_mask = 0;
9d5c8243
AK		 465
466 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 467 switch (hw->mac.type) {
468 case e1000_82575:
9d5c8243
AK		 469 tmp = rd32(E1000_CTRL_EXT);
470 /* enable MSI-X PBA support*/
471 tmp |= E1000_CTRL_EXT_PBA_CLR;
472
473 /* Auto-Mask interrupts upon ICR read. */
474 tmp |= E1000_CTRL_EXT_EIAME;
475 tmp |= E1000_CTRL_EXT_IRCA;
476
477 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 478
479 /* enable msix_other interrupt */
480 array_wr32(E1000_MSIXBM(0), vector++,
481 E1000_EIMS_OTHER);
844290e5 482 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 483
2d064c06
AD		 484 break;
485
486 case e1000_82576:
047e0030
AD		 487 /* Turn on MSI-X capability first, or our settings
488 * won't stick. And it will take days to debug. */
489 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
490 E1000_GPIE_PBA | E1000_GPIE_EIAME |
491 E1000_GPIE_NSICR);
492
493 /* enable msix_other interrupt */
494 adapter->eims_other = 1 << vector;
2d064c06 495 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 496
047e0030 497 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 498 break;
499 default:
500 /* do nothing, since nothing else supports MSI-X */
501 break;
502 } /* switch (hw->mac.type) */
047e0030
AD		 503
504 adapter->eims_enable_mask |= adapter->eims_other;
505
506 for (i = 0; i < adapter->num_q_vectors; i++) {
507 struct igb_q_vector *q_vector = adapter->q_vector[i];
508 igb_assign_vector(q_vector, vector++);
509 adapter->eims_enable_mask |= q_vector->eims_value;
510 }
511
9d5c8243
AK		 512 wrfl();
513}
514
515/**
516 * igb_request_msix - Initialize MSI-X interrupts
517 *
518 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
519 * kernel.
520 **/
521static int igb_request_msix(struct igb_adapter *adapter)
522{
523 struct net_device *netdev = adapter->netdev;
047e0030 524 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 525 int i, err = 0, vector = 0;
526
047e0030
AD		 527 err = request_irq(adapter->msix_entries[vector].vector,
528 &igb_msix_other, 0, netdev->name, adapter);
529 if (err)
530 goto out;
531 vector++;
532
533 for (i = 0; i < adapter->num_q_vectors; i++) {
534 struct igb_q_vector *q_vector = adapter->q_vector[i];
535
536 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
537
538 if (q_vector->rx_ring && q_vector->tx_ring)
539 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
540 q_vector->rx_ring->queue_index);
541 else if (q_vector->tx_ring)
542 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
543 q_vector->tx_ring->queue_index);
544 else if (q_vector->rx_ring)
545 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
546 q_vector->rx_ring->queue_index);
9d5c8243 547 else
047e0030
AD		 548 sprintf(q_vector->name, "%s-unused", netdev->name);
549
9d5c8243 550 err = request_irq(adapter->msix_entries[vector].vector,
047e0030
AD		 551 &igb_msix_ring, 0, q_vector->name,
552 q_vector);
9d5c8243
AK		 553 if (err)
554 goto out;
9d5c8243
AK		 555 vector++;
556 }
557
9d5c8243
AK		 558 igb_configure_msix(adapter);
559 return 0;
560out:
561 return err;
562}
563
564static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
565{
566 if (adapter->msix_entries) {
567 pci_disable_msix(adapter->pdev);
568 kfree(adapter->msix_entries);
569 adapter->msix_entries = NULL;
047e0030 570 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 571 pci_disable_msi(adapter->pdev);
047e0030 572 }
9d5c8243
AK		 573}
574
047e0030
AD		 575/**
576 * igb_free_q_vectors - Free memory allocated for interrupt vectors
577 * @adapter: board private structure to initialize
578 *
579 * This function frees the memory allocated to the q_vectors. In addition if
580 * NAPI is enabled it will delete any references to the NAPI struct prior
581 * to freeing the q_vector.
582 **/
583static void igb_free_q_vectors(struct igb_adapter *adapter)
584{
585 int v_idx;
586
587 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
588 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
589 adapter->q_vector[v_idx] = NULL;
590 netif_napi_del(&q_vector->napi);
591 kfree(q_vector);
592 }
593 adapter->num_q_vectors = 0;
594}
595
596/**
597 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
598 *
599 * This function resets the device so that it has 0 rx queues, tx queues, and
600 * MSI-X interrupts allocated.
601 */
602static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
603{
604 igb_free_queues(adapter);
605 igb_free_q_vectors(adapter);
606 igb_reset_interrupt_capability(adapter);
607}
9d5c8243
AK		 608
609/**
610 * igb_set_interrupt_capability - set MSI or MSI-X if supported
611 *
612 * Attempt to configure interrupts using the best available
613 * capabilities of the hardware and kernel.
614 **/
615static void igb_set_interrupt_capability(struct igb_adapter *adapter)
616{
617 int err;
618 int numvecs, i;
619
83b7180d 620 /* Number of supported queues. */
83b7180d
AD		 621 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
622 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
623
047e0030
AD		 624 /* start with one vector for every rx queue */
625 numvecs = adapter->num_rx_queues;
626
627 /* if tx handler is seperate add 1 for every tx queue */
628 numvecs += adapter->num_tx_queues;
629
630 /* store the number of vectors reserved for queues */
631 adapter->num_q_vectors = numvecs;
632
633 /* add 1 vector for link status interrupts */
634 numvecs++;
9d5c8243
AK		 635 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
636 GFP_KERNEL);
637 if (!adapter->msix_entries)
638 goto msi_only;
639
640 for (i = 0; i < numvecs; i++)
641 adapter->msix_entries[i].entry = i;
642
643 err = pci_enable_msix(adapter->pdev,
644 adapter->msix_entries,
645 numvecs);
646 if (err == 0)
34a20e89 647 goto out;
9d5c8243
AK		 648
649 igb_reset_interrupt_capability(adapter);
650
651 /* If we can't do MSI-X, try MSI */
652msi_only:
2a3abf6d
AD		 653#ifdef CONFIG_PCI_IOV
654 /* disable SR-IOV for non MSI-X configurations */
655 if (adapter->vf_data) {
656 struct e1000_hw *hw = &adapter->hw;
657 /* disable iov and allow time for transactions to clear */
658 pci_disable_sriov(adapter->pdev);
659 msleep(500);
660
661 kfree(adapter->vf_data);
662 adapter->vf_data = NULL;
663 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
664 msleep(100);
665 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
666 }
667#endif
4fc82adf
AD		 668 adapter->vfs_allocated_count = 0;
669 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 670 adapter->num_rx_queues = 1;
661086df 671 adapter->num_tx_queues = 1;
047e0030 672 adapter->num_q_vectors = 1;
9d5c8243 673 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 674 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 675out:
661086df 676 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 677 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 678 return;
679}
680
047e0030
AD		 681/**
682 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
683 * @adapter: board private structure to initialize
684 *
685 * We allocate one q_vector per queue interrupt. If allocation fails we
686 * return -ENOMEM.
687 **/
688static int igb_alloc_q_vectors(struct igb_adapter *adapter)
689{
690 struct igb_q_vector *q_vector;
691 struct e1000_hw *hw = &adapter->hw;
692 int v_idx;
693
694 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
695 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
696 if (!q_vector)
697 goto err_out;
698 q_vector->adapter = adapter;
699 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
700 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
701 q_vector->itr_val = IGB_START_ITR;
702 q_vector->set_itr = 1;
703 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
704 adapter->q_vector[v_idx] = q_vector;
705 }
706 return 0;
707
708err_out:
709 while (v_idx) {
710 v_idx--;
711 q_vector = adapter->q_vector[v_idx];
712 netif_napi_del(&q_vector->napi);
713 kfree(q_vector);
714 adapter->q_vector[v_idx] = NULL;
715 }
716 return -ENOMEM;
717}
718
719static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
720 int ring_idx, int v_idx)
721{
722 struct igb_q_vector *q_vector;
723
724 q_vector = adapter->q_vector[v_idx];
725 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
726 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 727 q_vector->itr_val = adapter->rx_itr_setting;
728 if (q_vector->itr_val && q_vector->itr_val <= 3)
729 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 730}
731
732static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
733 int ring_idx, int v_idx)
734{
735 struct igb_q_vector *q_vector;
736
737 q_vector = adapter->q_vector[v_idx];
738 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
739 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 740 q_vector->itr_val = adapter->tx_itr_setting;
741 if (q_vector->itr_val && q_vector->itr_val <= 3)
742 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 743}
744
745/**
746 * igb_map_ring_to_vector - maps allocated queues to vectors
747 *
748 * This function maps the recently allocated queues to vectors.
749 **/
750static int igb_map_ring_to_vector(struct igb_adapter *adapter)
751{
752 int i;
753 int v_idx = 0;
754
755 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
756 (adapter->num_q_vectors < adapter->num_tx_queues))
757 return -ENOMEM;
758
759 if (adapter->num_q_vectors >=
760 (adapter->num_rx_queues + adapter->num_tx_queues)) {
761 for (i = 0; i < adapter->num_rx_queues; i++)
762 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
763 for (i = 0; i < adapter->num_tx_queues; i++)
764 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
765 } else {
766 for (i = 0; i < adapter->num_rx_queues; i++) {
767 if (i < adapter->num_tx_queues)
768 igb_map_tx_ring_to_vector(adapter, i, v_idx);
769 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
770 }
771 for (; i < adapter->num_tx_queues; i++)
772 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
773 }
774 return 0;
775}
776
777/**
778 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
779 *
780 * This function initializes the interrupts and allocates all of the queues.
781 **/
782static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
783{
784 struct pci_dev *pdev = adapter->pdev;
785 int err;
786
787 igb_set_interrupt_capability(adapter);
788
789 err = igb_alloc_q_vectors(adapter);
790 if (err) {
791 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
792 goto err_alloc_q_vectors;
793 }
794
795 err = igb_alloc_queues(adapter);
796 if (err) {
797 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
798 goto err_alloc_queues;
799 }
800
801 err = igb_map_ring_to_vector(adapter);
802 if (err) {
803 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
804 goto err_map_queues;
805 }
806
807
808 return 0;
809err_map_queues:
810 igb_free_queues(adapter);
811err_alloc_queues:
812 igb_free_q_vectors(adapter);
813err_alloc_q_vectors:
814 igb_reset_interrupt_capability(adapter);
815 return err;
816}
817
9d5c8243
AK		 818/**
819 * igb_request_irq - initialize interrupts
820 *
821 * Attempts to configure interrupts using the best available
822 * capabilities of the hardware and kernel.
823 **/
824static int igb_request_irq(struct igb_adapter *adapter)
825{
826 struct net_device *netdev = adapter->netdev;
047e0030 827 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 828 struct e1000_hw *hw = &adapter->hw;
829 int err = 0;
830
831 if (adapter->msix_entries) {
832 err = igb_request_msix(adapter);
844290e5 833 if (!err)
9d5c8243 834 goto request_done;
9d5c8243 835 /* fall back to MSI */
047e0030 836 igb_clear_interrupt_scheme(adapter);
9d5c8243 837 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 838 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 839 igb_free_all_tx_resources(adapter);
840 igb_free_all_rx_resources(adapter);
047e0030 841 adapter->num_tx_queues = 1;
9d5c8243 842 adapter->num_rx_queues = 1;
047e0030
AD		 843 adapter->num_q_vectors = 1;
844 err = igb_alloc_q_vectors(adapter);
845 if (err) {
846 dev_err(&pdev->dev,
847 "Unable to allocate memory for vectors\n");
848 goto request_done;
849 }
850 err = igb_alloc_queues(adapter);
851 if (err) {
852 dev_err(&pdev->dev,
853 "Unable to allocate memory for queues\n");
854 igb_free_q_vectors(adapter);
855 goto request_done;
856 }
857 igb_setup_all_tx_resources(adapter);
858 igb_setup_all_rx_resources(adapter);
844290e5 859 } else {
2d064c06
AD		 860 switch (hw->mac.type) {
861 case e1000_82575:
862 wr32(E1000_MSIXBM(0),
047e0030
AD		 863 (E1000_EICR_RX_QUEUE0 |
864 E1000_EICR_TX_QUEUE0 |
865 E1000_EIMS_OTHER));
2d064c06
AD		 866 break;
867 case e1000_82576:
868 wr32(E1000_IVAR0, E1000_IVAR_VALID);
869 break;
870 default:
871 break;
872 }
9d5c8243 873 }
844290e5 874
7dfc16fa 875 if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 876 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
047e0030 877 netdev->name, adapter);
9d5c8243
AK		 878 if (!err)
879 goto request_done;
047e0030 880
9d5c8243
AK		 881 /* fall back to legacy interrupts */
882 igb_reset_interrupt_capability(adapter);
7dfc16fa 883 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 884 }
885
886 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
047e0030 887 netdev->name, adapter);
9d5c8243 888
6cb5e577 889 if (err)
9d5c8243
AK		 890 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
891 err);
9d5c8243
AK		 892
893request_done:
894 return err;
895}
896
897static void igb_free_irq(struct igb_adapter *adapter)
898{
9d5c8243
AK		 899 if (adapter->msix_entries) {
900 int vector = 0, i;
901
047e0030 902 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 903
047e0030
AD		 904 for (i = 0; i < adapter->num_q_vectors; i++) {
905 struct igb_q_vector *q_vector = adapter->q_vector[i];
906 free_irq(adapter->msix_entries[vector++].vector,
907 q_vector);
908 }
909 } else {
910 free_irq(adapter->pdev->irq, adapter);
9d5c8243 911 }
9d5c8243
AK		 912}
913
914/**
915 * igb_irq_disable - Mask off interrupt generation on the NIC
916 * @adapter: board private structure
917 **/
918static void igb_irq_disable(struct igb_adapter *adapter)
919{
920 struct e1000_hw *hw = &adapter->hw;
921
922 if (adapter->msix_entries) {
2dfd1212
AD		 923 u32 regval = rd32(E1000_EIAM);
924 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
925 wr32(E1000_EIMC, adapter->eims_enable_mask);
926 regval = rd32(E1000_EIAC);
927 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 928 }
844290e5
PW		 929
930 wr32(E1000_IAM, 0);
9d5c8243
AK		 931 wr32(E1000_IMC, ~0);
932 wrfl();
933 synchronize_irq(adapter->pdev->irq);
934}
935
936/**
937 * igb_irq_enable - Enable default interrupt generation settings
938 * @adapter: board private structure
939 **/
940static void igb_irq_enable(struct igb_adapter *adapter)
941{
942 struct e1000_hw *hw = &adapter->hw;
943
944 if (adapter->msix_entries) {
2dfd1212
AD		 945 u32 regval = rd32(E1000_EIAC);
946 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
947 regval = rd32(E1000_EIAM);
948 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 949 wr32(E1000_EIMS, adapter->eims_enable_mask);
4ae196df
AD		 950 if (adapter->vfs_allocated_count)
951 wr32(E1000_MBVFIMR, 0xFF);
952 wr32(E1000_IMS, (E1000_IMS_LSC | E1000_IMS_VMMB |
953 E1000_IMS_DOUTSYNC));
844290e5
PW		 954 } else {
955 wr32(E1000_IMS, IMS_ENABLE_MASK);
956 wr32(E1000_IAM, IMS_ENABLE_MASK);
957 }
9d5c8243
AK		 958}
959
960static void igb_update_mng_vlan(struct igb_adapter *adapter)
961{
51466239 962 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 963 u16 vid = adapter->hw.mng_cookie.vlan_id;
964 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 965
966 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
967 /* add VID to filter table */
968 igb_vfta_set(hw, vid, true);
969 adapter->mng_vlan_id = vid;
970 } else {
971 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
972 }
973
974 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
975 (vid != old_vid) &&
976 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
977 /* remove VID from filter table */
978 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 979 }
980}
981
982/**
983 * igb_release_hw_control - release control of the h/w to f/w
984 * @adapter: address of board private structure
985 *
986 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
987 * For ASF and Pass Through versions of f/w this means that the
988 * driver is no longer loaded.
989 *
990 **/
991static void igb_release_hw_control(struct igb_adapter *adapter)
992{
993 struct e1000_hw *hw = &adapter->hw;
994 u32 ctrl_ext;
995
996 /* Let firmware take over control of h/w */
997 ctrl_ext = rd32(E1000_CTRL_EXT);
998 wr32(E1000_CTRL_EXT,
999 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1000}
1001
1002
1003/**
1004 * igb_get_hw_control - get control of the h/w from f/w
1005 * @adapter: address of board private structure
1006 *
1007 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1008 * For ASF and Pass Through versions of f/w this means that
1009 * the driver is loaded.
1010 *
1011 **/
1012static void igb_get_hw_control(struct igb_adapter *adapter)
1013{
1014 struct e1000_hw *hw = &adapter->hw;
1015 u32 ctrl_ext;
1016
1017 /* Let firmware know the driver has taken over */
1018 ctrl_ext = rd32(E1000_CTRL_EXT);
1019 wr32(E1000_CTRL_EXT,
1020 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1021}
1022
9d5c8243
AK		 1023/**
1024 * igb_configure - configure the hardware for RX and TX
1025 * @adapter: private board structure
1026 **/
1027static void igb_configure(struct igb_adapter *adapter)
1028{
1029 struct net_device *netdev = adapter->netdev;
1030 int i;
1031
1032 igb_get_hw_control(adapter);
ff41f8dc 1033 igb_set_rx_mode(netdev);
9d5c8243
AK		 1034
1035 igb_restore_vlan(adapter);
9d5c8243 1036
85b430b4 1037 igb_setup_tctl(adapter);
06cf2666 1038 igb_setup_mrqc(adapter);
9d5c8243 1039 igb_setup_rctl(adapter);
85b430b4
AD		 1040
1041 igb_configure_tx(adapter);
9d5c8243 1042 igb_configure_rx(adapter);
662d7205
AD		 1043
1044 igb_rx_fifo_flush_82575(&adapter->hw);
1045
c493ea45 1046 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1047 * at least 1 descriptor unused to make sure
1048 * next_to_use != next_to_clean */
1049 for (i = 0; i < adapter->num_rx_queues; i++) {
1050 struct igb_ring *ring = &adapter->rx_ring[i];
c493ea45 1051 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1052 }
1053
1054
1055 adapter->tx_queue_len = netdev->tx_queue_len;
1056}
1057
1058
1059/**
1060 * igb_up - Open the interface and prepare it to handle traffic
1061 * @adapter: board private structure
1062 **/
1063
1064int igb_up(struct igb_adapter *adapter)
1065{
1066 struct e1000_hw *hw = &adapter->hw;
1067 int i;
1068
1069 /* hardware has been reset, we need to reload some things */
1070 igb_configure(adapter);
1071
1072 clear_bit(__IGB_DOWN, &adapter->state);
1073
047e0030
AD		 1074 for (i = 0; i < adapter->num_q_vectors; i++) {
1075 struct igb_q_vector *q_vector = adapter->q_vector[i];
1076 napi_enable(&q_vector->napi);
1077 }
844290e5 1078 if (adapter->msix_entries)
9d5c8243 1079 igb_configure_msix(adapter);
9d5c8243
AK		 1080
1081 /* Clear any pending interrupts. */
1082 rd32(E1000_ICR);
1083 igb_irq_enable(adapter);
1084
d4960307
AD		 1085 /* notify VFs that reset has been completed */
1086 if (adapter->vfs_allocated_count) {
1087 u32 reg_data = rd32(E1000_CTRL_EXT);
1088 reg_data |= E1000_CTRL_EXT_PFRSTD;
1089 wr32(E1000_CTRL_EXT, reg_data);
1090 }
1091
4cb9be7a
JB		 1092 netif_tx_start_all_queues(adapter->netdev);
1093
9d5c8243
AK		 1094 /* Fire a link change interrupt to start the watchdog. */
1095 wr32(E1000_ICS, E1000_ICS_LSC);
1096 return 0;
1097}
1098
1099void igb_down(struct igb_adapter *adapter)
1100{
1101 struct e1000_hw *hw = &adapter->hw;
1102 struct net_device *netdev = adapter->netdev;
1103 u32 tctl, rctl;
1104 int i;
1105
1106 /* signal that we're down so the interrupt handler does not
1107 * reschedule our watchdog timer */
1108 set_bit(__IGB_DOWN, &adapter->state);
1109
1110 /* disable receives in the hardware */
1111 rctl = rd32(E1000_RCTL);
1112 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1113 /* flush and sleep below */
1114
fd2ea0a7 1115 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1116
1117 /* disable transmits in the hardware */
1118 tctl = rd32(E1000_TCTL);
1119 tctl &= ~E1000_TCTL_EN;
1120 wr32(E1000_TCTL, tctl);
1121 /* flush both disables and wait for them to finish */
1122 wrfl();
1123 msleep(10);
1124
047e0030
AD		 1125 for (i = 0; i < adapter->num_q_vectors; i++) {
1126 struct igb_q_vector *q_vector = adapter->q_vector[i];
1127 napi_disable(&q_vector->napi);
1128 }
9d5c8243 1129
9d5c8243
AK		 1130 igb_irq_disable(adapter);
1131
1132 del_timer_sync(&adapter->watchdog_timer);
1133 del_timer_sync(&adapter->phy_info_timer);
1134
1135 netdev->tx_queue_len = adapter->tx_queue_len;
1136 netif_carrier_off(netdev);
04fe6358
AD		 1137
1138 /* record the stats before reset*/
1139 igb_update_stats(adapter);
1140
9d5c8243
AK		 1141 adapter->link_speed = 0;
1142 adapter->link_duplex = 0;
1143
3023682e
JK		 1144 if (!pci_channel_offline(adapter->pdev))
1145 igb_reset(adapter);
9d5c8243
AK		 1146 igb_clean_all_tx_rings(adapter);
1147 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1148#ifdef CONFIG_IGB_DCA
1149
1150 /* since we reset the hardware DCA settings were cleared */
1151 igb_setup_dca(adapter);
1152#endif
9d5c8243
AK		 1153}
1154
1155void igb_reinit_locked(struct igb_adapter *adapter)
1156{
1157 WARN_ON(in_interrupt());
1158 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1159 msleep(1);
1160 igb_down(adapter);
1161 igb_up(adapter);
1162 clear_bit(__IGB_RESETTING, &adapter->state);
1163}
1164
1165void igb_reset(struct igb_adapter *adapter)
1166{
1167 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1168 struct e1000_mac_info *mac = &hw->mac;
1169 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1170 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1171 u16 hwm;
1172
1173 /* Repartition Pba for greater than 9k mtu
1174 * To take effect CTRL.RST is required.
1175 */
fa4dfae0
AD		 1176 switch (mac->type) {
1177 case e1000_82576:
d249be54
AD		 1178 pba = rd32(E1000_RXPBS);
1179 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1180 break;
1181 case e1000_82575:
1182 default:
1183 pba = E1000_PBA_34K;
1184 break;
2d064c06 1185 }
9d5c8243 1186
2d064c06
AD		 1187 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1188 (mac->type < e1000_82576)) {
9d5c8243
AK		 1189 /* adjust PBA for jumbo frames */
1190 wr32(E1000_PBA, pba);
1191
1192 /* To maintain wire speed transmits, the Tx FIFO should be
1193 * large enough to accommodate two full transmit packets,
1194 * rounded up to the next 1KB and expressed in KB. Likewise,
1195 * the Rx FIFO should be large enough to accommodate at least
1196 * one full receive packet and is similarly rounded up and
1197 * expressed in KB. */
1198 pba = rd32(E1000_PBA);
1199 /* upper 16 bits has Tx packet buffer allocation size in KB */
1200 tx_space = pba >> 16;
1201 /* lower 16 bits has Rx packet buffer allocation size in KB */
1202 pba &= 0xffff;
1203 /* the tx fifo also stores 16 bytes of information about the tx
1204 * but don't include ethernet FCS because hardware appends it */
1205 min_tx_space = (adapter->max_frame_size +
85e8d004 1206 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1207 ETH_FCS_LEN) * 2;
1208 min_tx_space = ALIGN(min_tx_space, 1024);
1209 min_tx_space >>= 10;
1210 /* software strips receive CRC, so leave room for it */
1211 min_rx_space = adapter->max_frame_size;
1212 min_rx_space = ALIGN(min_rx_space, 1024);
1213 min_rx_space >>= 10;
1214
1215 /* If current Tx allocation is less than the min Tx FIFO size,
1216 * and the min Tx FIFO size is less than the current Rx FIFO
1217 * allocation, take space away from current Rx allocation */
1218 if (tx_space < min_tx_space &&
1219 ((min_tx_space - tx_space) < pba)) {
1220 pba = pba - (min_tx_space - tx_space);
1221
1222 /* if short on rx space, rx wins and must trump tx
1223 * adjustment */
1224 if (pba < min_rx_space)
1225 pba = min_rx_space;
1226 }
2d064c06 1227 wr32(E1000_PBA, pba);
9d5c8243 1228 }
9d5c8243
AK		 1229
1230 /* flow control settings */
1231 /* The high water mark must be low enough to fit one full frame
1232 * (or the size used for early receive) above it in the Rx FIFO.
1233 * Set it to the lower of:
1234 * - 90% of the Rx FIFO size, or
1235 * - the full Rx FIFO size minus one full frame */
1236 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1237 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1238
2d064c06
AD		 1239 if (mac->type < e1000_82576) {
1240 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1241 fc->low_water = fc->high_water - 8;
1242 } else {
1243 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1244 fc->low_water = fc->high_water - 16;
1245 }
9d5c8243
AK		 1246 fc->pause_time = 0xFFFF;
1247 fc->send_xon = 1;
0cce119a 1248 fc->current_mode = fc->requested_mode;
9d5c8243 1249
4ae196df
AD		 1250 /* disable receive for all VFs and wait one second */
1251 if (adapter->vfs_allocated_count) {
1252 int i;
1253 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1254 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1255
1256 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1257 igb_ping_all_vfs(adapter);
4ae196df
AD		 1258
1259 /* disable transmits and receives */
1260 wr32(E1000_VFRE, 0);
1261 wr32(E1000_VFTE, 0);
1262 }
1263
9d5c8243
AK		 1264 /* Allow time for pending master requests to run */
1265 adapter->hw.mac.ops.reset_hw(&adapter->hw);
1266 wr32(E1000_WUC, 0);
1267
1268 if (adapter->hw.mac.ops.init_hw(&adapter->hw))
1269 dev_err(&adapter->pdev->dev, "Hardware Error\n");
1270
1271 igb_update_mng_vlan(adapter);
1272
1273 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1274 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1275
1276 igb_reset_adaptive(&adapter->hw);
f5f4cf08 1277 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 1278}
1279
2e5c6922
SH		 1280static const struct net_device_ops igb_netdev_ops = {
1281 .ndo_open = igb_open,
1282 .ndo_stop = igb_close,
00829823 1283 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1284 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1285 .ndo_set_rx_mode = igb_set_rx_mode,
1286 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1287 .ndo_set_mac_address = igb_set_mac,
1288 .ndo_change_mtu = igb_change_mtu,
1289 .ndo_do_ioctl = igb_ioctl,
1290 .ndo_tx_timeout = igb_tx_timeout,
1291 .ndo_validate_addr = eth_validate_addr,
1292 .ndo_vlan_rx_register = igb_vlan_rx_register,
1293 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1294 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1295#ifdef CONFIG_NET_POLL_CONTROLLER
1296 .ndo_poll_controller = igb_netpoll,
1297#endif
1298};
1299
9d5c8243
AK		 1300/**
1301 * igb_probe - Device Initialization Routine
1302 * @pdev: PCI device information struct
1303 * @ent: entry in igb_pci_tbl
1304 *
1305 * Returns 0 on success, negative on failure
1306 *
1307 * igb_probe initializes an adapter identified by a pci_dev structure.
1308 * The OS initialization, configuring of the adapter private structure,
1309 * and a hardware reset occur.
1310 **/
1311static int __devinit igb_probe(struct pci_dev *pdev,
1312 const struct pci_device_id *ent)
1313{
1314 struct net_device *netdev;
1315 struct igb_adapter *adapter;
1316 struct e1000_hw *hw;
4337e993
AD		 1317 u16 eeprom_data = 0;
1318 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1319 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1320 unsigned long mmio_start, mmio_len;
2d6a5e95 1321 int err, pci_using_dac;
9d5c8243
AK		 1322 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1323 u32 part_num;
1324
aed5dec3 1325 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1326 if (err)
1327 return err;
1328
1329 pci_using_dac = 0;
6a35528a 1330 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1331 if (!err) {
6a35528a 1332 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1333 if (!err)
1334 pci_using_dac = 1;
1335 } else {
284901a9 1336 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1337 if (err) {
284901a9 1338 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1339 if (err) {
1340 dev_err(&pdev->dev, "No usable DMA "
1341 "configuration, aborting\n");
1342 goto err_dma;
1343 }
1344 }
1345 }
1346
aed5dec3
AD		 1347 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1348 IORESOURCE_MEM),
1349 igb_driver_name);
9d5c8243
AK		 1350 if (err)
1351 goto err_pci_reg;
1352
19d5afd4 1353 pci_enable_pcie_error_reporting(pdev);
40a914fa 1354
9d5c8243 1355 pci_set_master(pdev);
c682fc23 1356 pci_save_state(pdev);
9d5c8243
AK		 1357
1358 err = -ENOMEM;
1bfaf07b
AD		 1359 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1360 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1361 if (!netdev)
1362 goto err_alloc_etherdev;
1363
1364 SET_NETDEV_DEV(netdev, &pdev->dev);
1365
1366 pci_set_drvdata(pdev, netdev);
1367 adapter = netdev_priv(netdev);
1368 adapter->netdev = netdev;
1369 adapter->pdev = pdev;
1370 hw = &adapter->hw;
1371 hw->back = adapter;
1372 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1373
1374 mmio_start = pci_resource_start(pdev, 0);
1375 mmio_len = pci_resource_len(pdev, 0);
1376
1377 err = -EIO;
28b0759c
AD		 1378 hw->hw_addr = ioremap(mmio_start, mmio_len);
1379 if (!hw->hw_addr)
9d5c8243
AK		 1380 goto err_ioremap;
1381
2e5c6922 1382 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1383 igb_set_ethtool_ops(netdev);
9d5c8243 1384 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1385
1386 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1387
1388 netdev->mem_start = mmio_start;
1389 netdev->mem_end = mmio_start + mmio_len;
1390
9d5c8243
AK		 1391 /* PCI config space info */
1392 hw->vendor_id = pdev->vendor;
1393 hw->device_id = pdev->device;
1394 hw->revision_id = pdev->revision;
1395 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1396 hw->subsystem_device_id = pdev->subsystem_device;
1397
1398 /* setup the private structure */
1399 hw->back = adapter;
1400 /* Copy the default MAC, PHY and NVM function pointers */
1401 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1402 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1403 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1404 /* Initialize skew-specific constants */
1405 err = ei->get_invariants(hw);
1406 if (err)
450c87c8 1407 goto err_sw_init;
9d5c8243 1408
450c87c8 1409 /* setup the private structure */
9d5c8243
AK		 1410 err = igb_sw_init(adapter);
1411 if (err)
1412 goto err_sw_init;
1413
1414 igb_get_bus_info_pcie(hw);
1415
1416 hw->phy.autoneg_wait_to_complete = false;
1417 hw->mac.adaptive_ifs = true;
1418
1419 /* Copper options */
1420 if (hw->phy.media_type == e1000_media_type_copper) {
1421 hw->phy.mdix = AUTO_ALL_MODES;
1422 hw->phy.disable_polarity_correction = false;
1423 hw->phy.ms_type = e1000_ms_hw_default;
1424 }
1425
1426 if (igb_check_reset_block(hw))
1427 dev_info(&pdev->dev,
1428 "PHY reset is blocked due to SOL/IDER session.\n");
1429
1430 netdev->features = NETIF_F_SG |
7d8eb29e 1431 NETIF_F_IP_CSUM |
9d5c8243
AK		 1432 NETIF_F_HW_VLAN_TX |
1433 NETIF_F_HW_VLAN_RX |
1434 NETIF_F_HW_VLAN_FILTER;
1435
7d8eb29e 1436 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1437 netdev->features |= NETIF_F_TSO;
9d5c8243 1438 netdev->features |= NETIF_F_TSO6;
48f29ffc 1439
5c0999b7 1440 netdev->features |= NETIF_F_GRO;
d3352520 1441
48f29ffc
JK		 1442 netdev->vlan_features |= NETIF_F_TSO;
1443 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1444 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1445 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1446 netdev->vlan_features |= NETIF_F_SG;
1447
9d5c8243
AK		 1448 if (pci_using_dac)
1449 netdev->features |= NETIF_F_HIGHDMA;
1450
b9473560
JB		 1451 if (adapter->hw.mac.type == e1000_82576)
1452 netdev->features |= NETIF_F_SCTP_CSUM;
1453
9d5c8243
AK		 1454 adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
1455
1456 /* before reading the NVM, reset the controller to put the device in a
1457 * known good starting state */
1458 hw->mac.ops.reset_hw(hw);
1459
1460 /* make sure the NVM is good */
1461 if (igb_validate_nvm_checksum(hw) < 0) {
1462 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1463 err = -EIO;
1464 goto err_eeprom;
1465 }
1466
1467 /* copy the MAC address out of the NVM */
1468 if (hw->mac.ops.read_mac_addr(hw))
1469 dev_err(&pdev->dev, "NVM Read Error\n");
1470
1471 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1472 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1473
1474 if (!is_valid_ether_addr(netdev->perm_addr)) {
1475 dev_err(&pdev->dev, "Invalid MAC Address\n");
1476 err = -EIO;
1477 goto err_eeprom;
1478 }
1479
0e340485
AD		 1480 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1481 (unsigned long) adapter);
1482 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1483 (unsigned long) adapter);
9d5c8243
AK		 1484
1485 INIT_WORK(&adapter->reset_task, igb_reset_task);
1486 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1487
450c87c8 1488 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1489 adapter->fc_autoneg = true;
1490 hw->mac.autoneg = true;
1491 hw->phy.autoneg_advertised = 0x2f;
1492
0cce119a
AD		 1493 hw->fc.requested_mode = e1000_fc_default;
1494 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1495
9d5c8243
AK		 1496 igb_validate_mdi_setting(hw);
1497
9d5c8243
AK		 1498 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1499 * enable the ACPI Magic Packet filter
1500 */
1501
a2cf8b6c 1502 if (hw->bus.func == 0)
312c75ae 1503 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
a2cf8b6c
AD		 1504 else if (hw->bus.func == 1)
1505 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1506
1507 if (eeprom_data & eeprom_apme_mask)
1508 adapter->eeprom_wol |= E1000_WUFC_MAG;
1509
1510 /* now that we have the eeprom settings, apply the special cases where
1511 * the eeprom may be wrong or the board simply won't support wake on
1512 * lan on a particular port */
1513 switch (pdev->device) {
1514 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1515 adapter->eeprom_wol = 0;
1516 break;
1517 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1518 case E1000_DEV_ID_82576_FIBER:
1519 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1520 /* Wake events only supported on port A for dual fiber
1521 * regardless of eeprom setting */
1522 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1523 adapter->eeprom_wol = 0;
1524 break;
c8ea5ea9
AD		 1525 case E1000_DEV_ID_82576_QUAD_COPPER:
1526 /* if quad port adapter, disable WoL on all but port A */
1527 if (global_quad_port_a != 0)
1528 adapter->eeprom_wol = 0;
1529 else
1530 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1531 /* Reset for multiple quad port adapters */
1532 if (++global_quad_port_a == 4)
1533 global_quad_port_a = 0;
1534 break;
9d5c8243
AK		 1535 }
1536
1537 /* initialize the wol settings based on the eeprom settings */
1538 adapter->wol = adapter->eeprom_wol;
e1b86d84 1539 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1540
1541 /* reset the hardware with the new settings */
1542 igb_reset(adapter);
1543
1544 /* let the f/w know that the h/w is now under the control of the
1545 * driver. */
1546 igb_get_hw_control(adapter);
1547
9d5c8243
AK		 1548 strcpy(netdev->name, "eth%d");
1549 err = register_netdev(netdev);
1550 if (err)
1551 goto err_register;
1552
b168dfc5
JB		 1553 /* carrier off reporting is important to ethtool even BEFORE open */
1554 netif_carrier_off(netdev);
1555
421e02f0 1556#ifdef CONFIG_IGB_DCA
bbd98fe4 1557 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1558 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1559 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1560 igb_setup_dca(adapter);
1561 }
fe4506b6 1562
38c845c7 1563#endif
38c845c7 1564
c5b9bd5e
AD		 1565 switch (hw->mac.type) {
1566 case e1000_82576:
1567 /*
1568 * Initialize hardware timer: we keep it running just in case
1569 * that some program needs it later on.
1570 */
1571 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1572 adapter->cycles.read = igb_read_clock;
1573 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1574 adapter->cycles.mult = 1;
1575 /**
1576 * Scale the NIC clock cycle by a large factor so that
1577 * relatively small clock corrections can be added or
1578 * substracted at each clock tick. The drawbacks of a large
1579 * factor are a) that the clock register overflows more quickly
1580 * (not such a big deal) and b) that the increment per tick has
1581 * to fit into 24 bits. As a result we need to use a shift of
1582 * 19 so we can fit a value of 16 into the TIMINCA register.
1583 */
1584 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1585 wr32(E1000_TIMINCA,
1586 (1 << E1000_TIMINCA_16NS_SHIFT) |
1587 (16 << IGB_82576_TSYNC_SHIFT));
1588
1589 /* Set registers so that rollover occurs soon to test this. */
1590 wr32(E1000_SYSTIML, 0x00000000);
1591 wr32(E1000_SYSTIMH, 0xFF800000);
1592 wrfl();
1593
1594 timecounter_init(&adapter->clock,
1595 &adapter->cycles,
1596 ktime_to_ns(ktime_get_real()));
1597 /*
1598 * Synchronize our NIC clock against system wall clock. NIC
1599 * time stamp reading requires ~3us per sample, each sample
1600 * was pretty stable even under load => only require 10
1601 * samples for each offset comparison.
1602 */
1603 memset(&adapter->compare, 0, sizeof(adapter->compare));
1604 adapter->compare.source = &adapter->clock;
1605 adapter->compare.target = ktime_get_real;
1606 adapter->compare.num_samples = 10;
1607 timecompare_update(&adapter->compare, 0);
1608 break;
1609 case e1000_82575:
1610 /* 82575 does not support timesync */
1611 default:
1612 break;
38c845c7 1613 }
38c845c7 1614
9d5c8243
AK		 1615 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1616 /* print bus type/speed/width info */
7c510e4b 1617 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243
AK		 1618 netdev->name,
1619 ((hw->bus.speed == e1000_bus_speed_2500)
1620 ? "2.5Gb/s" : "unknown"),
59c3de89
AD		 1621 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1622 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1623 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1624 "unknown"),
7c510e4b 1625 netdev->dev_addr);
9d5c8243
AK		 1626
1627 igb_read_part_num(hw, &part_num);
1628 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1629 (part_num >> 8), (part_num & 0xff));
1630
1631 dev_info(&pdev->dev,
1632 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1633 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1634 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1635 adapter->num_rx_queues, adapter->num_tx_queues);
1636
9d5c8243
AK		 1637 return 0;
1638
1639err_register:
1640 igb_release_hw_control(adapter);
1641err_eeprom:
1642 if (!igb_check_reset_block(hw))
f5f4cf08 1643 igb_reset_phy(hw);
9d5c8243
AK		 1644
1645 if (hw->flash_address)
1646 iounmap(hw->flash_address);
9d5c8243 1647err_sw_init:
047e0030 1648 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1649 iounmap(hw->hw_addr);
1650err_ioremap:
1651 free_netdev(netdev);
1652err_alloc_etherdev:
aed5dec3
AD		 1653 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1654 IORESOURCE_MEM));
9d5c8243
AK		 1655err_pci_reg:
1656err_dma:
1657 pci_disable_device(pdev);
1658 return err;
1659}
1660
1661/**
1662 * igb_remove - Device Removal Routine
1663 * @pdev: PCI device information struct
1664 *
1665 * igb_remove is called by the PCI subsystem to alert the driver
1666 * that it should release a PCI device. The could be caused by a
1667 * Hot-Plug event, or because the driver is going to be removed from
1668 * memory.
1669 **/
1670static void __devexit igb_remove(struct pci_dev *pdev)
1671{
1672 struct net_device *netdev = pci_get_drvdata(pdev);
1673 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1674 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1675
1676 /* flush_scheduled work may reschedule our watchdog task, so
1677 * explicitly disable watchdog tasks from being rescheduled */
1678 set_bit(__IGB_DOWN, &adapter->state);
1679 del_timer_sync(&adapter->watchdog_timer);
1680 del_timer_sync(&adapter->phy_info_timer);
1681
1682 flush_scheduled_work();
1683
421e02f0 1684#ifdef CONFIG_IGB_DCA
7dfc16fa 1685 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1686 dev_info(&pdev->dev, "DCA disabled\n");
1687 dca_remove_requester(&pdev->dev);
7dfc16fa 1688 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1689 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1690 }
1691#endif
1692
9d5c8243
AK		 1693 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1694 * would have already happened in close and is redundant. */
1695 igb_release_hw_control(adapter);
1696
1697 unregister_netdev(netdev);
1698
f5f4cf08
AD		 1699 if (!igb_check_reset_block(&adapter->hw))
1700 igb_reset_phy(&adapter->hw);
9d5c8243 1701
047e0030 1702 igb_clear_interrupt_scheme(adapter);
9d5c8243 1703
37680117
AD		 1704#ifdef CONFIG_PCI_IOV
1705 /* reclaim resources allocated to VFs */
1706 if (adapter->vf_data) {
1707 /* disable iov and allow time for transactions to clear */
1708 pci_disable_sriov(pdev);
1709 msleep(500);
1710
1711 kfree(adapter->vf_data);
1712 adapter->vf_data = NULL;
1713 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1714 msleep(100);
1715 dev_info(&pdev->dev, "IOV Disabled\n");
1716 }
1717#endif
28b0759c
AD		 1718 iounmap(hw->hw_addr);
1719 if (hw->flash_address)
1720 iounmap(hw->flash_address);
aed5dec3
AD		 1721 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1722 IORESOURCE_MEM));
9d5c8243
AK		 1723
1724 free_netdev(netdev);
1725
19d5afd4 1726 pci_disable_pcie_error_reporting(pdev);
40a914fa 1727
9d5c8243
AK		 1728 pci_disable_device(pdev);
1729}
1730
a6b623e0
AD		 1731/**
1732 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1733 * @adapter: board private structure to initialize
1734 *
1735 * This function initializes the vf specific data storage and then attempts to
1736 * allocate the VFs. The reason for ordering it this way is because it is much
1737 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1738 * the memory for the VFs.
1739 **/
1740static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1741{
1742#ifdef CONFIG_PCI_IOV
1743 struct pci_dev *pdev = adapter->pdev;
1744
1745 if (adapter->vfs_allocated_count > 7)
1746 adapter->vfs_allocated_count = 7;
1747
1748 if (adapter->vfs_allocated_count) {
1749 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1750 sizeof(struct vf_data_storage),
1751 GFP_KERNEL);
1752 /* if allocation failed then we do not support SR-IOV */
1753 if (!adapter->vf_data) {
1754 adapter->vfs_allocated_count = 0;
1755 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1756 "Data Storage\n");
1757 }
1758 }
1759
1760 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1761 kfree(adapter->vf_data);
1762 adapter->vf_data = NULL;
1763#endif /* CONFIG_PCI_IOV */
1764 adapter->vfs_allocated_count = 0;
1765#ifdef CONFIG_PCI_IOV
1766 } else {
1767 unsigned char mac_addr[ETH_ALEN];
1768 int i;
1769 dev_info(&pdev->dev, "%d vfs allocated\n",
1770 adapter->vfs_allocated_count);
1771 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1772 random_ether_addr(mac_addr);
1773 igb_set_vf_mac(adapter, i, mac_addr);
1774 }
1775 }
1776#endif /* CONFIG_PCI_IOV */
1777}
1778
9d5c8243
AK		 1779/**
1780 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1781 * @adapter: board private structure to initialize
1782 *
1783 * igb_sw_init initializes the Adapter private data structure.
1784 * Fields are initialized based on PCI device information and
1785 * OS network device settings (MTU size).
1786 **/
1787static int __devinit igb_sw_init(struct igb_adapter *adapter)
1788{
1789 struct e1000_hw *hw = &adapter->hw;
1790 struct net_device *netdev = adapter->netdev;
1791 struct pci_dev *pdev = adapter->pdev;
1792
1793 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1794
68fd9910
AD		 1795 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1796 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1797 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1798 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1799
9d5c8243
AK		 1800 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1801 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1802
a6b623e0
AD		 1803#ifdef CONFIG_PCI_IOV
1804 if (hw->mac.type == e1000_82576)
1805 adapter->vfs_allocated_count = max_vfs;
1806
1807#endif /* CONFIG_PCI_IOV */
1808 /* This call may decrease the number of queues */
047e0030 1809 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1810 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1811 return -ENOMEM;
1812 }
1813
a6b623e0
AD		 1814 igb_probe_vfs(adapter);
1815
9d5c8243
AK		 1816 /* Explicitly disable IRQ since the NIC can be in any state. */
1817 igb_irq_disable(adapter);
1818
1819 set_bit(__IGB_DOWN, &adapter->state);
1820 return 0;
1821}
1822
1823/**
1824 * igb_open - Called when a network interface is made active
1825 * @netdev: network interface device structure
1826 *
1827 * Returns 0 on success, negative value on failure
1828 *
1829 * The open entry point is called when a network interface is made
1830 * active by the system (IFF_UP). At this point all resources needed
1831 * for transmit and receive operations are allocated, the interrupt
1832 * handler is registered with the OS, the watchdog timer is started,
1833 * and the stack is notified that the interface is ready.
1834 **/
1835static int igb_open(struct net_device *netdev)
1836{
1837 struct igb_adapter *adapter = netdev_priv(netdev);
1838 struct e1000_hw *hw = &adapter->hw;
1839 int err;
1840 int i;
1841
1842 /* disallow open during test */
1843 if (test_bit(__IGB_TESTING, &adapter->state))
1844 return -EBUSY;
1845
b168dfc5
JB		 1846 netif_carrier_off(netdev);
1847
9d5c8243
AK		 1848 /* allocate transmit descriptors */
1849 err = igb_setup_all_tx_resources(adapter);
1850 if (err)
1851 goto err_setup_tx;
1852
1853 /* allocate receive descriptors */
1854 err = igb_setup_all_rx_resources(adapter);
1855 if (err)
1856 goto err_setup_rx;
1857
1858 /* e1000_power_up_phy(adapter); */
1859
9d5c8243
AK		 1860 /* before we allocate an interrupt, we must be ready to handle it.
1861 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1862 * as soon as we call pci_request_irq, so we have to setup our
1863 * clean_rx handler before we do so. */
1864 igb_configure(adapter);
1865
1866 err = igb_request_irq(adapter);
1867 if (err)
1868 goto err_req_irq;
1869
1870 /* From here on the code is the same as igb_up() */
1871 clear_bit(__IGB_DOWN, &adapter->state);
1872
047e0030
AD		 1873 for (i = 0; i < adapter->num_q_vectors; i++) {
1874 struct igb_q_vector *q_vector = adapter->q_vector[i];
1875 napi_enable(&q_vector->napi);
1876 }
9d5c8243
AK		 1877
1878 /* Clear any pending interrupts. */
1879 rd32(E1000_ICR);
844290e5
PW		 1880
1881 igb_irq_enable(adapter);
1882
d4960307
AD		 1883 /* notify VFs that reset has been completed */
1884 if (adapter->vfs_allocated_count) {
1885 u32 reg_data = rd32(E1000_CTRL_EXT);
1886 reg_data |= E1000_CTRL_EXT_PFRSTD;
1887 wr32(E1000_CTRL_EXT, reg_data);
1888 }
1889
d55b53ff
JK		 1890 netif_tx_start_all_queues(netdev);
1891
9d5c8243
AK		 1892 /* Fire a link status change interrupt to start the watchdog. */
1893 wr32(E1000_ICS, E1000_ICS_LSC);
1894
1895 return 0;
1896
1897err_req_irq:
1898 igb_release_hw_control(adapter);
1899 /* e1000_power_down_phy(adapter); */
1900 igb_free_all_rx_resources(adapter);
1901err_setup_rx:
1902 igb_free_all_tx_resources(adapter);
1903err_setup_tx:
1904 igb_reset(adapter);
1905
1906 return err;
1907}
1908
1909/**
1910 * igb_close - Disables a network interface
1911 * @netdev: network interface device structure
1912 *
1913 * Returns 0, this is not allowed to fail
1914 *
1915 * The close entry point is called when an interface is de-activated
1916 * by the OS. The hardware is still under the driver's control, but
1917 * needs to be disabled. A global MAC reset is issued to stop the
1918 * hardware, and all transmit and receive resources are freed.
1919 **/
1920static int igb_close(struct net_device *netdev)
1921{
1922 struct igb_adapter *adapter = netdev_priv(netdev);
1923
1924 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1925 igb_down(adapter);
1926
1927 igb_free_irq(adapter);
1928
1929 igb_free_all_tx_resources(adapter);
1930 igb_free_all_rx_resources(adapter);
1931
9d5c8243
AK		 1932 return 0;
1933}
1934
1935/**
1936 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 1937 * @tx_ring: tx descriptor ring (for a specific queue) to setup
1938 *
1939 * Return 0 on success, negative on failure
1940 **/
80785298 1941int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 1942{
80785298 1943 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 1944 int size;
1945
1946 size = sizeof(struct igb_buffer) * tx_ring->count;
1947 tx_ring->buffer_info = vmalloc(size);
1948 if (!tx_ring->buffer_info)
1949 goto err;
1950 memset(tx_ring->buffer_info, 0, size);
1951
1952 /* round up to nearest 4K */
85e8d004 1953 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 1954 tx_ring->size = ALIGN(tx_ring->size, 4096);
1955
439705e1
AD		 1956 tx_ring->desc = pci_alloc_consistent(pdev,
1957 tx_ring->size,
9d5c8243
AK		 1958 &tx_ring->dma);
1959
1960 if (!tx_ring->desc)
1961 goto err;
1962
9d5c8243
AK		 1963 tx_ring->next_to_use = 0;
1964 tx_ring->next_to_clean = 0;
9d5c8243
AK		 1965 return 0;
1966
1967err:
1968 vfree(tx_ring->buffer_info);
047e0030 1969 dev_err(&pdev->dev,
9d5c8243
AK		 1970 "Unable to allocate memory for the transmit descriptor ring\n");
1971 return -ENOMEM;
1972}
1973
1974/**
1975 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
1976 * (Descriptors) for all queues
1977 * @adapter: board private structure
1978 *
1979 * Return 0 on success, negative on failure
1980 **/
1981static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
1982{
439705e1 1983 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1984 int i, err = 0;
1985
1986 for (i = 0; i < adapter->num_tx_queues; i++) {
80785298 1987 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
9d5c8243 1988 if (err) {
439705e1 1989 dev_err(&pdev->dev,
9d5c8243
AK		 1990 "Allocation for Tx Queue %u failed\n", i);
1991 for (i--; i >= 0; i--)
3b644cf6 1992 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 1993 break;
1994 }
1995 }
1996
661086df 1997 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
439705e1 1998 int r_idx = i % adapter->num_tx_queues;
661086df 1999 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
eebbbdba 2000 }
9d5c8243
AK		 2001 return err;
2002}
2003
2004/**
85b430b4
AD		 2005 * igb_setup_tctl - configure the transmit control registers
2006 * @adapter: Board private structure
9d5c8243 2007 **/
d7ee5b3a 2008void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2009{
9d5c8243
AK		 2010 struct e1000_hw *hw = &adapter->hw;
2011 u32 tctl;
9d5c8243 2012
85b430b4
AD		 2013 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2014 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2015
2016 /* Program the Transmit Control Register */
9d5c8243
AK		 2017 tctl = rd32(E1000_TCTL);
2018 tctl &= ~E1000_TCTL_CT;
2019 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2020 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2021
2022 igb_config_collision_dist(hw);
2023
9d5c8243
AK		 2024 /* Enable transmits */
2025 tctl |= E1000_TCTL_EN;
2026
2027 wr32(E1000_TCTL, tctl);
2028}
2029
85b430b4
AD		 2030/**
2031 * igb_configure_tx_ring - Configure transmit ring after Reset
2032 * @adapter: board private structure
2033 * @ring: tx ring to configure
2034 *
2035 * Configure a transmit ring after a reset.
2036 **/
d7ee5b3a
AD		 2037void igb_configure_tx_ring(struct igb_adapter *adapter,
2038 struct igb_ring *ring)
85b430b4
AD		 2039{
2040 struct e1000_hw *hw = &adapter->hw;
2041 u32 txdctl;
2042 u64 tdba = ring->dma;
2043 int reg_idx = ring->reg_idx;
2044
2045 /* disable the queue */
2046 txdctl = rd32(E1000_TXDCTL(reg_idx));
2047 wr32(E1000_TXDCTL(reg_idx),
2048 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2049 wrfl();
2050 mdelay(10);
2051
2052 wr32(E1000_TDLEN(reg_idx),
2053 ring->count * sizeof(union e1000_adv_tx_desc));
2054 wr32(E1000_TDBAL(reg_idx),
2055 tdba & 0x00000000ffffffffULL);
2056 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2057
fce99e34
AD		 2058 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2059 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2060 writel(0, ring->head);
2061 writel(0, ring->tail);
85b430b4
AD		 2062
2063 txdctl |= IGB_TX_PTHRESH;
2064 txdctl |= IGB_TX_HTHRESH << 8;
2065 txdctl |= IGB_TX_WTHRESH << 16;
2066
2067 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2068 wr32(E1000_TXDCTL(reg_idx), txdctl);
2069}
2070
2071/**
2072 * igb_configure_tx - Configure transmit Unit after Reset
2073 * @adapter: board private structure
2074 *
2075 * Configure the Tx unit of the MAC after a reset.
2076 **/
2077static void igb_configure_tx(struct igb_adapter *adapter)
2078{
2079 int i;
2080
2081 for (i = 0; i < adapter->num_tx_queues; i++)
2082 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
85b430b4
AD		 2083}
2084
9d5c8243
AK		 2085/**
2086 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2087 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2088 *
2089 * Returns 0 on success, negative on failure
2090 **/
80785298 2091int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2092{
80785298 2093 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2094 int size, desc_len;
2095
2096 size = sizeof(struct igb_buffer) * rx_ring->count;
2097 rx_ring->buffer_info = vmalloc(size);
2098 if (!rx_ring->buffer_info)
2099 goto err;
2100 memset(rx_ring->buffer_info, 0, size);
2101
2102 desc_len = sizeof(union e1000_adv_rx_desc);
2103
2104 /* Round up to nearest 4K */
2105 rx_ring->size = rx_ring->count * desc_len;
2106 rx_ring->size = ALIGN(rx_ring->size, 4096);
2107
2108 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2109 &rx_ring->dma);
2110
2111 if (!rx_ring->desc)
2112 goto err;
2113
2114 rx_ring->next_to_clean = 0;
2115 rx_ring->next_to_use = 0;
9d5c8243 2116
9d5c8243
AK		 2117 return 0;
2118
2119err:
2120 vfree(rx_ring->buffer_info);
439705e1 2121 rx_ring->buffer_info = NULL;
80785298 2122 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2123 "the receive descriptor ring\n");
2124 return -ENOMEM;
2125}
2126
2127/**
2128 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2129 * (Descriptors) for all queues
2130 * @adapter: board private structure
2131 *
2132 * Return 0 on success, negative on failure
2133 **/
2134static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2135{
439705e1 2136 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2137 int i, err = 0;
2138
2139 for (i = 0; i < adapter->num_rx_queues; i++) {
80785298 2140 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
9d5c8243 2141 if (err) {
439705e1 2142 dev_err(&pdev->dev,
9d5c8243
AK		 2143 "Allocation for Rx Queue %u failed\n", i);
2144 for (i--; i >= 0; i--)
3b644cf6 2145 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2146 break;
2147 }
2148 }
2149
2150 return err;
2151}
2152
06cf2666
AD		 2153/**
2154 * igb_setup_mrqc - configure the multiple receive queue control registers
2155 * @adapter: Board private structure
2156 **/
2157static void igb_setup_mrqc(struct igb_adapter *adapter)
2158{
2159 struct e1000_hw *hw = &adapter->hw;
2160 u32 mrqc, rxcsum;
2161 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2162 union e1000_reta {
2163 u32 dword;
2164 u8 bytes[4];
2165 } reta;
2166 static const u8 rsshash[40] = {
2167 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2168 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2169 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2170 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2171
2172 /* Fill out hash function seeds */
2173 for (j = 0; j < 10; j++) {
2174 u32 rsskey = rsshash[(j * 4)];
2175 rsskey |= rsshash[(j * 4) + 1] << 8;
2176 rsskey |= rsshash[(j * 4) + 2] << 16;
2177 rsskey |= rsshash[(j * 4) + 3] << 24;
2178 array_wr32(E1000_RSSRK(0), j, rsskey);
2179 }
2180
2181 num_rx_queues = adapter->num_rx_queues;
2182
2183 if (adapter->vfs_allocated_count) {
2184 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2185 switch (hw->mac.type) {
2186 case e1000_82576:
2187 shift = 3;
2188 num_rx_queues = 2;
2189 break;
2190 case e1000_82575:
2191 shift = 2;
2192 shift2 = 6;
2193 default:
2194 break;
2195 }
2196 } else {
2197 if (hw->mac.type == e1000_82575)
2198 shift = 6;
2199 }
2200
2201 for (j = 0; j < (32 * 4); j++) {
2202 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2203 if (shift2)
2204 reta.bytes[j & 3] |= num_rx_queues << shift2;
2205 if ((j & 3) == 3)
2206 wr32(E1000_RETA(j >> 2), reta.dword);
2207 }
2208
2209 /*
2210 * Disable raw packet checksumming so that RSS hash is placed in
2211 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2212 * offloads as they are enabled by default
2213 */
2214 rxcsum = rd32(E1000_RXCSUM);
2215 rxcsum |= E1000_RXCSUM_PCSD;
2216
2217 if (adapter->hw.mac.type >= e1000_82576)
2218 /* Enable Receive Checksum Offload for SCTP */
2219 rxcsum |= E1000_RXCSUM_CRCOFL;
2220
2221 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2222 wr32(E1000_RXCSUM, rxcsum);
2223
2224 /* If VMDq is enabled then we set the appropriate mode for that, else
2225 * we default to RSS so that an RSS hash is calculated per packet even
2226 * if we are only using one queue */
2227 if (adapter->vfs_allocated_count) {
2228 if (hw->mac.type > e1000_82575) {
2229 /* Set the default pool for the PF's first queue */
2230 u32 vtctl = rd32(E1000_VT_CTL);
2231 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2232 E1000_VT_CTL_DISABLE_DEF_POOL);
2233 vtctl |= adapter->vfs_allocated_count <<
2234 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2235 wr32(E1000_VT_CTL, vtctl);
2236 }
2237 if (adapter->num_rx_queues > 1)
2238 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2239 else
2240 mrqc = E1000_MRQC_ENABLE_VMDQ;
2241 } else {
2242 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2243 }
2244 igb_vmm_control(adapter);
2245
2246 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2247 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2248 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2249 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2250 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2251 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2252 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2253 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2254
2255 wr32(E1000_MRQC, mrqc);
2256}
2257
9d5c8243
AK		 2258/**
2259 * igb_setup_rctl - configure the receive control registers
2260 * @adapter: Board private structure
2261 **/
d7ee5b3a 2262void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2263{
2264 struct e1000_hw *hw = &adapter->hw;
2265 u32 rctl;
9d5c8243
AK		 2266
2267 rctl = rd32(E1000_RCTL);
2268
2269 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2270 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2271
69d728ba 2272 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2273 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2274
87cb7e8c
AK		 2275 /*
2276 * enable stripping of CRC. It's unlikely this will break BMC
2277 * redirection as it did with e1000. Newer features require
2278 * that the HW strips the CRC.
73cd78f1 2279 */
87cb7e8c 2280 rctl |= E1000_RCTL_SECRC;
9d5c8243 2281
9b07f3d3 2282 /*
ec54d7d6 2283 * disable store bad packets and clear size bits.
9b07f3d3 2284 */
ec54d7d6 2285 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2286
6ec43fe6
AD		 2287 /* enable LPE to prevent packets larger than max_frame_size */
2288 rctl |= E1000_RCTL_LPE;
9d5c8243 2289
952f72a8
AD		 2290 /* disable queue 0 to prevent tail write w/o re-config */
2291 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2292
e1739522
AD		 2293 /* Attention!!! For SR-IOV PF driver operations you must enable
2294 * queue drop for all VF and PF queues to prevent head of line blocking
2295 * if an un-trusted VF does not provide descriptors to hardware.
2296 */
2297 if (adapter->vfs_allocated_count) {
e1739522
AD		 2298 /* set all queue drop enable bits */
2299 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2300 }
2301
9d5c8243
AK		 2302 wr32(E1000_RCTL, rctl);
2303}
2304
7d5753f0
AD		 2305static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2306 int vfn)
2307{
2308 struct e1000_hw *hw = &adapter->hw;
2309 u32 vmolr;
2310
2311 /* if it isn't the PF check to see if VFs are enabled and
2312 * increase the size to support vlan tags */
2313 if (vfn < adapter->vfs_allocated_count &&
2314 adapter->vf_data[vfn].vlans_enabled)
2315 size += VLAN_TAG_SIZE;
2316
2317 vmolr = rd32(E1000_VMOLR(vfn));
2318 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2319 vmolr |= size | E1000_VMOLR_LPE;
2320 wr32(E1000_VMOLR(vfn), vmolr);
2321
2322 return 0;
2323}
2324
e1739522
AD		 2325/**
2326 * igb_rlpml_set - set maximum receive packet size
2327 * @adapter: board private structure
2328 *
2329 * Configure maximum receivable packet size.
2330 **/
2331static void igb_rlpml_set(struct igb_adapter *adapter)
2332{
2333 u32 max_frame_size = adapter->max_frame_size;
2334 struct e1000_hw *hw = &adapter->hw;
2335 u16 pf_id = adapter->vfs_allocated_count;
2336
2337 if (adapter->vlgrp)
2338 max_frame_size += VLAN_TAG_SIZE;
2339
2340 /* if vfs are enabled we set RLPML to the largest possible request
2341 * size and set the VMOLR RLPML to the size we need */
2342 if (pf_id) {
2343 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2344 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2345 }
2346
2347 wr32(E1000_RLPML, max_frame_size);
2348}
2349
7d5753f0
AD		 2350static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2351{
2352 struct e1000_hw *hw = &adapter->hw;
2353 u32 vmolr;
2354
2355 /*
2356 * This register exists only on 82576 and newer so if we are older then
2357 * we should exit and do nothing
2358 */
2359 if (hw->mac.type < e1000_82576)
2360 return;
2361
2362 vmolr = rd32(E1000_VMOLR(vfn));
2363 vmolr |= E1000_VMOLR_AUPE | /* Accept untagged packets */
2364 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2365
2366 /* clear all bits that might not be set */
2367 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2368
2369 if (adapter->num_rx_queues > 1 && vfn == adapter->vfs_allocated_count)
2370 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2371 /*
2372 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2373 * multicast packets
2374 */
2375 if (vfn <= adapter->vfs_allocated_count)
2376 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2377
2378 wr32(E1000_VMOLR(vfn), vmolr);
2379}
2380
85b430b4
AD		 2381/**
2382 * igb_configure_rx_ring - Configure a receive ring after Reset
2383 * @adapter: board private structure
2384 * @ring: receive ring to be configured
2385 *
2386 * Configure the Rx unit of the MAC after a reset.
2387 **/
d7ee5b3a
AD		 2388void igb_configure_rx_ring(struct igb_adapter *adapter,
2389 struct igb_ring *ring)
85b430b4
AD		 2390{
2391 struct e1000_hw *hw = &adapter->hw;
2392 u64 rdba = ring->dma;
2393 int reg_idx = ring->reg_idx;
952f72a8 2394 u32 srrctl, rxdctl;
85b430b4
AD		 2395
2396 /* disable the queue */
2397 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2398 wr32(E1000_RXDCTL(reg_idx),
2399 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2400
2401 /* Set DMA base address registers */
2402 wr32(E1000_RDBAL(reg_idx),
2403 rdba & 0x00000000ffffffffULL);
2404 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2405 wr32(E1000_RDLEN(reg_idx),
2406 ring->count * sizeof(union e1000_adv_rx_desc));
2407
2408 /* initialize head and tail */
fce99e34
AD		 2409 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2410 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2411 writel(0, ring->head);
2412 writel(0, ring->tail);
85b430b4 2413
952f72a8 2414 /* set descriptor configuration */
4c844851
AD		 2415 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2416 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2417 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2418#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2419 srrctl |= IGB_RXBUFFER_16384 >>
2420 E1000_SRRCTL_BSIZEPKT_SHIFT;
2421#else
2422 srrctl |= (PAGE_SIZE / 2) >>
2423 E1000_SRRCTL_BSIZEPKT_SHIFT;
2424#endif
2425 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2426 } else {
4c844851 2427 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2428 E1000_SRRCTL_BSIZEPKT_SHIFT;
2429 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2430 }
2431
2432 wr32(E1000_SRRCTL(reg_idx), srrctl);
2433
7d5753f0
AD		 2434 /* set filtering for VMDQ pools */
2435 igb_set_vmolr(adapter, reg_idx & 0x7);
2436
85b430b4
AD		 2437 /* enable receive descriptor fetching */
2438 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2439 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2440 rxdctl &= 0xFFF00000;
2441 rxdctl |= IGB_RX_PTHRESH;
2442 rxdctl |= IGB_RX_HTHRESH << 8;
2443 rxdctl |= IGB_RX_WTHRESH << 16;
2444 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2445}
2446
9d5c8243
AK		 2447/**
2448 * igb_configure_rx - Configure receive Unit after Reset
2449 * @adapter: board private structure
2450 *
2451 * Configure the Rx unit of the MAC after a reset.
2452 **/
2453static void igb_configure_rx(struct igb_adapter *adapter)
2454{
9107584e 2455 int i;
9d5c8243 2456
68d480c4
AD		 2457 /* set UTA to appropriate mode */
2458 igb_set_uta(adapter);
2459
26ad9178
AD		 2460 /* set the correct pool for the PF default MAC address in entry 0 */
2461 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2462 adapter->vfs_allocated_count);
2463
06cf2666
AD		 2464 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2465 * the Base and Length of the Rx Descriptor Ring */
2466 for (i = 0; i < adapter->num_rx_queues; i++)
2467 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
9d5c8243
AK		 2468}
2469
2470/**
2471 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2472 * @tx_ring: Tx descriptor ring for a specific queue
2473 *
2474 * Free all transmit software resources
2475 **/
68fd9910 2476void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2477{
3b644cf6 2478 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2479
2480 vfree(tx_ring->buffer_info);
2481 tx_ring->buffer_info = NULL;
2482
439705e1
AD		 2483 /* if not set, then don't free */
2484 if (!tx_ring->desc)
2485 return;
2486
80785298
AD		 2487 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2488 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2489
2490 tx_ring->desc = NULL;
2491}
2492
2493/**
2494 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2495 * @adapter: board private structure
2496 *
2497 * Free all transmit software resources
2498 **/
2499static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2500{
2501 int i;
2502
2503 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2504 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2505}
2506
b1a436c3
AD		 2507void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2508 struct igb_buffer *buffer_info)
9d5c8243 2509{
65689fef 2510 buffer_info->dma = 0;
9d5c8243 2511 if (buffer_info->skb) {
80785298
AD		 2512 skb_dma_unmap(&tx_ring->pdev->dev,
2513 buffer_info->skb,
65689fef 2514 DMA_TO_DEVICE);
9d5c8243
AK		 2515 dev_kfree_skb_any(buffer_info->skb);
2516 buffer_info->skb = NULL;
2517 }
2518 buffer_info->time_stamp = 0;
2519 /* buffer_info must be completely set up in the transmit path */
2520}
2521
2522/**
2523 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2524 * @tx_ring: ring to be cleaned
2525 **/
3b644cf6 2526static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2527{
2528 struct igb_buffer *buffer_info;
2529 unsigned long size;
2530 unsigned int i;
2531
2532 if (!tx_ring->buffer_info)
2533 return;
2534 /* Free all the Tx ring sk_buffs */
2535
2536 for (i = 0; i < tx_ring->count; i++) {
2537 buffer_info = &tx_ring->buffer_info[i];
80785298 2538 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2539 }
2540
2541 size = sizeof(struct igb_buffer) * tx_ring->count;
2542 memset(tx_ring->buffer_info, 0, size);
2543
2544 /* Zero out the descriptor ring */
9d5c8243
AK		 2545 memset(tx_ring->desc, 0, tx_ring->size);
2546
2547 tx_ring->next_to_use = 0;
2548 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2549}
2550
2551/**
2552 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2553 * @adapter: board private structure
2554 **/
2555static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2556{
2557 int i;
2558
2559 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2560 igb_clean_tx_ring(&adapter->tx_ring[i]);
9d5c8243
AK		 2561}
2562
2563/**
2564 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2565 * @rx_ring: ring to clean the resources from
2566 *
2567 * Free all receive software resources
2568 **/
68fd9910 2569void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2570{
3b644cf6 2571 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2572
2573 vfree(rx_ring->buffer_info);
2574 rx_ring->buffer_info = NULL;
2575
439705e1
AD		 2576 /* if not set, then don't free */
2577 if (!rx_ring->desc)
2578 return;
2579
80785298
AD		 2580 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2581 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2582
2583 rx_ring->desc = NULL;
2584}
2585
2586/**
2587 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2588 * @adapter: board private structure
2589 *
2590 * Free all receive software resources
2591 **/
2592static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2593{
2594 int i;
2595
2596 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2597 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2598}
2599
2600/**
2601 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2602 * @rx_ring: ring to free buffers from
2603 **/
3b644cf6 2604static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2605{
2606 struct igb_buffer *buffer_info;
9d5c8243
AK		 2607 unsigned long size;
2608 unsigned int i;
2609
2610 if (!rx_ring->buffer_info)
2611 return;
439705e1 2612
9d5c8243
AK		 2613 /* Free all the Rx ring sk_buffs */
2614 for (i = 0; i < rx_ring->count; i++) {
2615 buffer_info = &rx_ring->buffer_info[i];
2616 if (buffer_info->dma) {
80785298
AD		 2617 pci_unmap_single(rx_ring->pdev,
2618 buffer_info->dma,
4c844851 2619 rx_ring->rx_buffer_len,
6ec43fe6 2620 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2621 buffer_info->dma = 0;
2622 }
2623
2624 if (buffer_info->skb) {
2625 dev_kfree_skb(buffer_info->skb);
2626 buffer_info->skb = NULL;
2627 }
6ec43fe6 2628 if (buffer_info->page_dma) {
80785298
AD		 2629 pci_unmap_page(rx_ring->pdev,
2630 buffer_info->page_dma,
6ec43fe6
AD		 2631 PAGE_SIZE / 2,
2632 PCI_DMA_FROMDEVICE);
2633 buffer_info->page_dma = 0;
2634 }
9d5c8243 2635 if (buffer_info->page) {
9d5c8243
AK		 2636 put_page(buffer_info->page);
2637 buffer_info->page = NULL;
bf36c1a0 2638 buffer_info->page_offset = 0;
9d5c8243
AK		 2639 }
2640 }
2641
9d5c8243
AK		 2642 size = sizeof(struct igb_buffer) * rx_ring->count;
2643 memset(rx_ring->buffer_info, 0, size);
2644
2645 /* Zero out the descriptor ring */
2646 memset(rx_ring->desc, 0, rx_ring->size);
2647
2648 rx_ring->next_to_clean = 0;
2649 rx_ring->next_to_use = 0;
9d5c8243
AK		 2650}
2651
2652/**
2653 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2654 * @adapter: board private structure
2655 **/
2656static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2657{
2658 int i;
2659
2660 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2661 igb_clean_rx_ring(&adapter->rx_ring[i]);
9d5c8243
AK		 2662}
2663
2664/**
2665 * igb_set_mac - Change the Ethernet Address of the NIC
2666 * @netdev: network interface device structure
2667 * @p: pointer to an address structure
2668 *
2669 * Returns 0 on success, negative on failure
2670 **/
2671static int igb_set_mac(struct net_device *netdev, void *p)
2672{
2673 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2674 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2675 struct sockaddr *addr = p;
2676
2677 if (!is_valid_ether_addr(addr->sa_data))
2678 return -EADDRNOTAVAIL;
2679
2680 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2681 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2682
26ad9178
AD		 2683 /* set the correct pool for the new PF MAC address in entry 0 */
2684 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2685 adapter->vfs_allocated_count);
e1739522 2686
9d5c8243
AK		 2687 return 0;
2688}
2689
2690/**
68d480c4 2691 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2692 * @netdev: network interface device structure
2693 *
68d480c4
AD		 2694 * Writes multicast address list to the MTA hash table.
2695 * Returns: -ENOMEM on failure
2696 * 0 on no addresses written
2697 * X on writing X addresses to MTA
9d5c8243 2698 **/
68d480c4 2699static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2700{
2701 struct igb_adapter *adapter = netdev_priv(netdev);
2702 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2703 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4
AD		 2704 u8 *mta_list;
2705 u32 vmolr = 0;
9d5c8243
AK		 2706 int i;
2707
68d480c4
AD		 2708 if (!netdev->mc_count) {
2709 /* nothing to program, so clear mc list */
2710 igb_update_mc_addr_list(hw, NULL, 0);
2711 igb_restore_vf_multicasts(adapter);
2712 return 0;
2713 }
9d5c8243 2714
68d480c4
AD		 2715 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2716 if (!mta_list)
2717 return -ENOMEM;
ff41f8dc 2718
68d480c4
AD		 2719 /* set vmolr receive overflow multicast bit */
2720 vmolr |= E1000_VMOLR_ROMPE;
2721
2722 /* The shared function expects a packed array of only addresses. */
2723 mc_ptr = netdev->mc_list;
2724
2725 for (i = 0; i < netdev->mc_count; i++) {
2726 if (!mc_ptr)
2727 break;
2728 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2729 mc_ptr = mc_ptr->next;
746b9f02 2730 }
68d480c4
AD		 2731 igb_update_mc_addr_list(hw, mta_list, i);
2732 kfree(mta_list);
2733
2734 return netdev->mc_count;
2735}
2736
2737/**
2738 * igb_write_uc_addr_list - write unicast addresses to RAR table
2739 * @netdev: network interface device structure
2740 *
2741 * Writes unicast address list to the RAR table.
2742 * Returns: -ENOMEM on failure/insufficient address space
2743 * 0 on no addresses written
2744 * X on writing X addresses to the RAR table
2745 **/
2746static int igb_write_uc_addr_list(struct net_device *netdev)
2747{
2748 struct igb_adapter *adapter = netdev_priv(netdev);
2749 struct e1000_hw *hw = &adapter->hw;
2750 unsigned int vfn = adapter->vfs_allocated_count;
2751 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2752 int count = 0;
2753
2754 /* return ENOMEM indicating insufficient memory for addresses */
2755 if (netdev->uc.count > rar_entries)
2756 return -ENOMEM;
9d5c8243 2757
ff41f8dc
AD		 2758 if (netdev->uc.count && rar_entries) {
2759 struct netdev_hw_addr *ha;
2760 list_for_each_entry(ha, &netdev->uc.list, list) {
2761 if (!rar_entries)
2762 break;
26ad9178
AD		 2763 igb_rar_set_qsel(adapter, ha->addr,
2764 rar_entries--,
68d480c4
AD		 2765 vfn);
2766 count++;
ff41f8dc
AD		 2767 }
2768 }
2769 /* write the addresses in reverse order to avoid write combining */
2770 for (; rar_entries > 0 ; rar_entries--) {
2771 wr32(E1000_RAH(rar_entries), 0);
2772 wr32(E1000_RAL(rar_entries), 0);
2773 }
2774 wrfl();
2775
68d480c4
AD		 2776 return count;
2777}
2778
2779/**
2780 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2781 * @netdev: network interface device structure
2782 *
2783 * The set_rx_mode entry point is called whenever the unicast or multicast
2784 * address lists or the network interface flags are updated. This routine is
2785 * responsible for configuring the hardware for proper unicast, multicast,
2786 * promiscuous mode, and all-multi behavior.
2787 **/
2788static void igb_set_rx_mode(struct net_device *netdev)
2789{
2790 struct igb_adapter *adapter = netdev_priv(netdev);
2791 struct e1000_hw *hw = &adapter->hw;
2792 unsigned int vfn = adapter->vfs_allocated_count;
2793 u32 rctl, vmolr = 0;
2794 int count;
2795
2796 /* Check for Promiscuous and All Multicast modes */
2797 rctl = rd32(E1000_RCTL);
2798
2799 /* clear the effected bits */
2800 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2801
2802 if (netdev->flags & IFF_PROMISC) {
2803 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2804 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2805 } else {
2806 if (netdev->flags & IFF_ALLMULTI) {
2807 rctl |= E1000_RCTL_MPE;
2808 vmolr |= E1000_VMOLR_MPME;
2809 } else {
2810 /*
2811 * Write addresses to the MTA, if the attempt fails
2812 * then we should just turn on promiscous mode so
2813 * that we can at least receive multicast traffic
2814 */
2815 count = igb_write_mc_addr_list(netdev);
2816 if (count < 0) {
2817 rctl |= E1000_RCTL_MPE;
2818 vmolr |= E1000_VMOLR_MPME;
2819 } else if (count) {
2820 vmolr |= E1000_VMOLR_ROMPE;
2821 }
2822 }
2823 /*
2824 * Write addresses to available RAR registers, if there is not
2825 * sufficient space to store all the addresses then enable
2826 * unicast promiscous mode
2827 */
2828 count = igb_write_uc_addr_list(netdev);
2829 if (count < 0) {
2830 rctl |= E1000_RCTL_UPE;
2831 vmolr |= E1000_VMOLR_ROPE;
2832 }
2833 rctl |= E1000_RCTL_VFE;
28fc06f5 2834 }
68d480c4 2835 wr32(E1000_RCTL, rctl);
28fc06f5 2836
68d480c4
AD		 2837 /*
2838 * In order to support SR-IOV and eventually VMDq it is necessary to set
2839 * the VMOLR to enable the appropriate modes. Without this workaround
2840 * we will have issues with VLAN tag stripping not being done for frames
2841 * that are only arriving because we are the default pool
2842 */
2843 if (hw->mac.type < e1000_82576)
28fc06f5 2844 return;
9d5c8243 2845
68d480c4
AD		 2846 vmolr |= rd32(E1000_VMOLR(vfn)) &
2847 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2848 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 2849 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 2850}
2851
2852/* Need to wait a few seconds after link up to get diagnostic information from
2853 * the phy */
2854static void igb_update_phy_info(unsigned long data)
2855{
2856 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 2857 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 2858}
2859
4d6b725e
AD		 2860/**
2861 * igb_has_link - check shared code for link and determine up/down
2862 * @adapter: pointer to driver private info
2863 **/
2864static bool igb_has_link(struct igb_adapter *adapter)
2865{
2866 struct e1000_hw *hw = &adapter->hw;
2867 bool link_active = false;
2868 s32 ret_val = 0;
2869
2870 /* get_link_status is set on LSC (link status) interrupt or
2871 * rx sequence error interrupt. get_link_status will stay
2872 * false until the e1000_check_for_link establishes link
2873 * for copper adapters ONLY
2874 */
2875 switch (hw->phy.media_type) {
2876 case e1000_media_type_copper:
2877 if (hw->mac.get_link_status) {
2878 ret_val = hw->mac.ops.check_for_link(hw);
2879 link_active = !hw->mac.get_link_status;
2880 } else {
2881 link_active = true;
2882 }
2883 break;
4d6b725e
AD		 2884 case e1000_media_type_internal_serdes:
2885 ret_val = hw->mac.ops.check_for_link(hw);
2886 link_active = hw->mac.serdes_has_link;
2887 break;
2888 default:
2889 case e1000_media_type_unknown:
2890 break;
2891 }
2892
2893 return link_active;
2894}
2895
9d5c8243
AK		 2896/**
2897 * igb_watchdog - Timer Call-back
2898 * @data: pointer to adapter cast into an unsigned long
2899 **/
2900static void igb_watchdog(unsigned long data)
2901{
2902 struct igb_adapter *adapter = (struct igb_adapter *)data;
2903 /* Do the rest outside of interrupt context */
2904 schedule_work(&adapter->watchdog_task);
2905}
2906
2907static void igb_watchdog_task(struct work_struct *work)
2908{
2909 struct igb_adapter *adapter = container_of(work,
2910 struct igb_adapter, watchdog_task);
2911 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2912 struct net_device *netdev = adapter->netdev;
2913 struct igb_ring *tx_ring = adapter->tx_ring;
9d5c8243 2914 u32 link;
7a6ea550 2915 int i;
9d5c8243 2916
4d6b725e
AD		 2917 link = igb_has_link(adapter);
2918 if ((netif_carrier_ok(netdev)) && link)
9d5c8243
AK		 2919 goto link_up;
2920
9d5c8243
AK		 2921 if (link) {
2922 if (!netif_carrier_ok(netdev)) {
2923 u32 ctrl;
2924 hw->mac.ops.get_speed_and_duplex(&adapter->hw,
2925 &adapter->link_speed,
2926 &adapter->link_duplex);
2927
2928 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 2929 /* Links status message must follow this format */
2930 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 2931 "Flow Control: %s\n",
527d47c1 2932 netdev->name,
9d5c8243
AK		 2933 adapter->link_speed,
2934 adapter->link_duplex == FULL_DUPLEX ?
2935 "Full Duplex" : "Half Duplex",
2936 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2937 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2938 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2939 E1000_CTRL_TFCE) ? "TX" : "None")));
2940
2941 /* tweak tx_queue_len according to speed/duplex and
2942 * adjust the timeout factor */
2943 netdev->tx_queue_len = adapter->tx_queue_len;
2944 adapter->tx_timeout_factor = 1;
2945 switch (adapter->link_speed) {
2946 case SPEED_10:
2947 netdev->tx_queue_len = 10;
2948 adapter->tx_timeout_factor = 14;
2949 break;
2950 case SPEED_100:
2951 netdev->tx_queue_len = 100;
2952 /* maybe add some timeout factor ? */
2953 break;
2954 }
2955
2956 netif_carrier_on(netdev);
9d5c8243 2957
4ae196df
AD		 2958 igb_ping_all_vfs(adapter);
2959
4b1a9877 2960 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2961 if (!test_bit(__IGB_DOWN, &adapter->state))
2962 mod_timer(&adapter->phy_info_timer,
2963 round_jiffies(jiffies + 2 * HZ));
2964 }
2965 } else {
2966 if (netif_carrier_ok(netdev)) {
2967 adapter->link_speed = 0;
2968 adapter->link_duplex = 0;
527d47c1
AD		 2969 /* Links status message must follow this format */
2970 printk(KERN_INFO "igb: %s NIC Link is Down\n",
2971 netdev->name);
9d5c8243 2972 netif_carrier_off(netdev);
4b1a9877 2973
4ae196df
AD		 2974 igb_ping_all_vfs(adapter);
2975
4b1a9877 2976 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2977 if (!test_bit(__IGB_DOWN, &adapter->state))
2978 mod_timer(&adapter->phy_info_timer,
2979 round_jiffies(jiffies + 2 * HZ));
2980 }
2981 }
2982
2983link_up:
2984 igb_update_stats(adapter);
2985
4b1a9877 2986 hw->mac.tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
9d5c8243 2987 adapter->tpt_old = adapter->stats.tpt;
4b1a9877 2988 hw->mac.collision_delta = adapter->stats.colc - adapter->colc_old;
9d5c8243
AK		 2989 adapter->colc_old = adapter->stats.colc;
2990
2991 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
2992 adapter->gorc_old = adapter->stats.gorc;
2993 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
2994 adapter->gotc_old = adapter->stats.gotc;
2995
2996 igb_update_adaptive(&adapter->hw);
2997
2998 if (!netif_carrier_ok(netdev)) {
c493ea45 2999 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
9d5c8243
AK		 3000 /* We've lost link, so the controller stops DMA,
3001 * but we've got queued Tx work that's never going
3002 * to get done, so reset controller to flush Tx.
3003 * (Do the reset outside of interrupt context). */
3004 adapter->tx_timeout_count++;
3005 schedule_work(&adapter->reset_task);
c2d5ab49
JB		 3006 /* return immediately since reset is imminent */
3007 return;
9d5c8243
AK		 3008 }
3009 }
3010
f7ba205e
AD		 3011 /* Force detection of hung controller every watchdog period */
3012 for (i = 0; i < adapter->num_tx_queues; i++)
3013 adapter->tx_ring[i].detect_tx_hung = true;
3014
9d5c8243 3015 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3016 if (adapter->msix_entries) {
047e0030
AD		 3017 u32 eics = 0;
3018 for (i = 0; i < adapter->num_q_vectors; i++) {
3019 struct igb_q_vector *q_vector = adapter->q_vector[i];
3020 eics |= q_vector->eims_value;
3021 }
7a6ea550
AD		 3022 wr32(E1000_EICS, eics);
3023 } else {
3024 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3025 }
9d5c8243 3026
9d5c8243
AK		 3027 /* Reset the timer */
3028 if (!test_bit(__IGB_DOWN, &adapter->state))
3029 mod_timer(&adapter->watchdog_timer,
3030 round_jiffies(jiffies + 2 * HZ));
3031}
3032
3033enum latency_range {
3034 lowest_latency = 0,
3035 low_latency = 1,
3036 bulk_latency = 2,
3037 latency_invalid = 255
3038};
3039
6eb5a7f1
AD		 3040/**
3041 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3042 *
3043 * Stores a new ITR value based on strictly on packet size. This
3044 * algorithm is less sophisticated than that used in igb_update_itr,
3045 * due to the difficulty of synchronizing statistics across multiple
3046 * receive rings. The divisors and thresholds used by this fuction
3047 * were determined based on theoretical maximum wire speed and testing
3048 * data, in order to minimize response time while increasing bulk
3049 * throughput.
3050 * This functionality is controlled by the InterruptThrottleRate module
3051 * parameter (see igb_param.c)
3052 * NOTE: This function is called only when operating in a multiqueue
3053 * receive environment.
047e0030 3054 * @q_vector: pointer to q_vector
6eb5a7f1 3055 **/
047e0030 3056static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3057{
047e0030 3058 int new_val = q_vector->itr_val;
6eb5a7f1 3059 int avg_wire_size = 0;
047e0030 3060 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3061
6eb5a7f1
AD		 3062 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3063 * ints/sec - ITR timer value of 120 ticks.
3064 */
3065 if (adapter->link_speed != SPEED_1000) {
047e0030 3066 new_val = 976;
6eb5a7f1 3067 goto set_itr_val;
9d5c8243 3068 }
047e0030
AD		 3069
3070 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3071 struct igb_ring *ring = q_vector->rx_ring;
3072 avg_wire_size = ring->total_bytes / ring->total_packets;
3073 }
3074
3075 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3076 struct igb_ring *ring = q_vector->tx_ring;
3077 avg_wire_size = max_t(u32, avg_wire_size,
3078 (ring->total_bytes /
3079 ring->total_packets));
3080 }
3081
3082 /* if avg_wire_size isn't set no work was done */
3083 if (!avg_wire_size)
3084 goto clear_counts;
9d5c8243 3085
6eb5a7f1
AD		 3086 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3087 avg_wire_size += 24;
3088
3089 /* Don't starve jumbo frames */
3090 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3091
6eb5a7f1
AD		 3092 /* Give a little boost to mid-size frames */
3093 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3094 new_val = avg_wire_size / 3;
3095 else
3096 new_val = avg_wire_size / 2;
9d5c8243 3097
6eb5a7f1 3098set_itr_val:
047e0030
AD		 3099 if (new_val != q_vector->itr_val) {
3100 q_vector->itr_val = new_val;
3101 q_vector->set_itr = 1;
9d5c8243 3102 }
6eb5a7f1 3103clear_counts:
047e0030
AD		 3104 if (q_vector->rx_ring) {
3105 q_vector->rx_ring->total_bytes = 0;
3106 q_vector->rx_ring->total_packets = 0;
3107 }
3108 if (q_vector->tx_ring) {
3109 q_vector->tx_ring->total_bytes = 0;
3110 q_vector->tx_ring->total_packets = 0;
3111 }
9d5c8243
AK		 3112}
3113
3114/**
3115 * igb_update_itr - update the dynamic ITR value based on statistics
3116 * Stores a new ITR value based on packets and byte
3117 * counts during the last interrupt. The advantage of per interrupt
3118 * computation is faster updates and more accurate ITR for the current
3119 * traffic pattern. Constants in this function were computed
3120 * based on theoretical maximum wire speed and thresholds were set based
3121 * on testing data as well as attempting to minimize response time
3122 * while increasing bulk throughput.
3123 * this functionality is controlled by the InterruptThrottleRate module
3124 * parameter (see igb_param.c)
3125 * NOTE: These calculations are only valid when operating in a single-
3126 * queue environment.
3127 * @adapter: pointer to adapter
047e0030 3128 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3129 * @packets: the number of packets during this measurement interval
3130 * @bytes: the number of bytes during this measurement interval
3131 **/
3132static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3133 int packets, int bytes)
3134{
3135 unsigned int retval = itr_setting;
3136
3137 if (packets == 0)
3138 goto update_itr_done;
3139
3140 switch (itr_setting) {
3141 case lowest_latency:
3142 /* handle TSO and jumbo frames */
3143 if (bytes/packets > 8000)
3144 retval = bulk_latency;
3145 else if ((packets < 5) && (bytes > 512))
3146 retval = low_latency;
3147 break;
3148 case low_latency: /* 50 usec aka 20000 ints/s */
3149 if (bytes > 10000) {
3150 /* this if handles the TSO accounting */
3151 if (bytes/packets > 8000) {
3152 retval = bulk_latency;
3153 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3154 retval = bulk_latency;
3155 } else if ((packets > 35)) {
3156 retval = lowest_latency;
3157 }
3158 } else if (bytes/packets > 2000) {
3159 retval = bulk_latency;
3160 } else if (packets <= 2 && bytes < 512) {
3161 retval = lowest_latency;
3162 }
3163 break;
3164 case bulk_latency: /* 250 usec aka 4000 ints/s */
3165 if (bytes > 25000) {
3166 if (packets > 35)
3167 retval = low_latency;
1e5c3d21 3168 } else if (bytes < 1500) {
9d5c8243
AK		 3169 retval = low_latency;
3170 }
3171 break;
3172 }
3173
3174update_itr_done:
3175 return retval;
3176}
3177
6eb5a7f1 3178static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3179{
047e0030 3180 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3181 u16 current_itr;
047e0030 3182 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3183
3184 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3185 if (adapter->link_speed != SPEED_1000) {
3186 current_itr = 0;
3187 new_itr = 4000;
3188 goto set_itr_now;
3189 }
3190
3191 adapter->rx_itr = igb_update_itr(adapter,
3192 adapter->rx_itr,
3193 adapter->rx_ring->total_packets,
3194 adapter->rx_ring->total_bytes);
9d5c8243 3195
047e0030
AD		 3196 adapter->tx_itr = igb_update_itr(adapter,
3197 adapter->tx_itr,
3198 adapter->tx_ring->total_packets,
3199 adapter->tx_ring->total_bytes);
3200 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3201
6eb5a7f1 3202 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3203 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3204 current_itr = low_latency;
3205
9d5c8243
AK		 3206 switch (current_itr) {
3207 /* counts and packets in update_itr are dependent on these numbers */
3208 case lowest_latency:
78b1f607 3209 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3210 break;
3211 case low_latency:
78b1f607 3212 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3213 break;
3214 case bulk_latency:
78b1f607 3215 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3216 break;
3217 default:
3218 break;
3219 }
3220
3221set_itr_now:
6eb5a7f1
AD		 3222 adapter->rx_ring->total_bytes = 0;
3223 adapter->rx_ring->total_packets = 0;
047e0030
AD		 3224 adapter->tx_ring->total_bytes = 0;
3225 adapter->tx_ring->total_packets = 0;
6eb5a7f1 3226
047e0030 3227 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3228 /* this attempts to bias the interrupt rate towards Bulk
3229 * by adding intermediate steps when interrupt rate is
3230 * increasing */
047e0030
AD		 3231 new_itr = new_itr > q_vector->itr_val ?
3232 max((new_itr * q_vector->itr_val) /
3233 (new_itr + (q_vector->itr_val >> 2)),
3234 new_itr) :
9d5c8243
AK		 3235 new_itr;
3236 /* Don't write the value here; it resets the adapter's
3237 * internal timer, and causes us to delay far longer than
3238 * we should between interrupts. Instead, we write the ITR
3239 * value at the beginning of the next interrupt so the timing
3240 * ends up being correct.
3241 */
047e0030
AD		 3242 q_vector->itr_val = new_itr;
3243 q_vector->set_itr = 1;
9d5c8243
AK		 3244 }
3245
3246 return;
3247}
3248
9d5c8243
AK		 3249#define IGB_TX_FLAGS_CSUM 0x00000001
3250#define IGB_TX_FLAGS_VLAN 0x00000002
3251#define IGB_TX_FLAGS_TSO 0x00000004
3252#define IGB_TX_FLAGS_IPV4 0x00000008
33af6bcc 3253#define IGB_TX_FLAGS_TSTAMP 0x00000010
9d5c8243
AK		 3254#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3255#define IGB_TX_FLAGS_VLAN_SHIFT 16
3256
85ad76b2 3257static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3258 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3259{
3260 struct e1000_adv_tx_context_desc *context_desc;
3261 unsigned int i;
3262 int err;
3263 struct igb_buffer *buffer_info;
3264 u32 info = 0, tu_cmd = 0;
3265 u32 mss_l4len_idx, l4len;
3266 *hdr_len = 0;
3267
3268 if (skb_header_cloned(skb)) {
3269 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3270 if (err)
3271 return err;
3272 }
3273
3274 l4len = tcp_hdrlen(skb);
3275 *hdr_len += l4len;
3276
3277 if (skb->protocol == htons(ETH_P_IP)) {
3278 struct iphdr *iph = ip_hdr(skb);
3279 iph->tot_len = 0;
3280 iph->check = 0;
3281 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3282 iph->daddr, 0,
3283 IPPROTO_TCP,
3284 0);
3285 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3286 ipv6_hdr(skb)->payload_len = 0;
3287 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3288 &ipv6_hdr(skb)->daddr,
3289 0, IPPROTO_TCP, 0);
3290 }
3291
3292 i = tx_ring->next_to_use;
3293
3294 buffer_info = &tx_ring->buffer_info[i];
3295 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3296 /* VLAN MACLEN IPLEN */
3297 if (tx_flags & IGB_TX_FLAGS_VLAN)
3298 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3299 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3300 *hdr_len += skb_network_offset(skb);
3301 info |= skb_network_header_len(skb);
3302 *hdr_len += skb_network_header_len(skb);
3303 context_desc->vlan_macip_lens = cpu_to_le32(info);
3304
3305 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3306 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3307
3308 if (skb->protocol == htons(ETH_P_IP))
3309 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3310 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3311
3312 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3313
3314 /* MSS L4LEN IDX */
3315 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3316 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3317
73cd78f1 3318 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3319 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3320 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3321
3322 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3323 context_desc->seqnum_seed = 0;
3324
3325 buffer_info->time_stamp = jiffies;
0e014cb1 3326 buffer_info->next_to_watch = i;
9d5c8243
AK		 3327 buffer_info->dma = 0;
3328 i++;
3329 if (i == tx_ring->count)
3330 i = 0;
3331
3332 tx_ring->next_to_use = i;
3333
3334 return true;
3335}
3336
85ad76b2
AD		 3337static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3338 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3339{
3340 struct e1000_adv_tx_context_desc *context_desc;
80785298 3341 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3342 struct igb_buffer *buffer_info;
3343 u32 info = 0, tu_cmd = 0;
80785298 3344 unsigned int i;
9d5c8243
AK		 3345
3346 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3347 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3348 i = tx_ring->next_to_use;
3349 buffer_info = &tx_ring->buffer_info[i];
3350 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3351
3352 if (tx_flags & IGB_TX_FLAGS_VLAN)
3353 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3354 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3355 if (skb->ip_summed == CHECKSUM_PARTIAL)
3356 info |= skb_network_header_len(skb);
3357
3358 context_desc->vlan_macip_lens = cpu_to_le32(info);
3359
3360 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3361
3362 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3363 __be16 protocol;
3364
3365 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3366 const struct vlan_ethhdr *vhdr =
3367 (const struct vlan_ethhdr*)skb->data;
3368
3369 protocol = vhdr->h_vlan_encapsulated_proto;
3370 } else {
3371 protocol = skb->protocol;
3372 }
3373
3374 switch (protocol) {
09640e63 3375 case cpu_to_be16(ETH_P_IP):
9d5c8243 3376 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3377 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3378 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3379 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3380 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3381 break;
09640e63 3382 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3383 /* XXX what about other V6 headers?? */
3384 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3385 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3386 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3387 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3388 break;
3389 default:
3390 if (unlikely(net_ratelimit()))
80785298 3391 dev_warn(&pdev->dev,
44b0cda3
MW		 3392 "partial checksum but proto=%x!\n",
3393 skb->protocol);
3394 break;
3395 }
9d5c8243
AK		 3396 }
3397
3398 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3399 context_desc->seqnum_seed = 0;
85ad76b2 3400 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3401 context_desc->mss_l4len_idx =
85ad76b2 3402 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3403
3404 buffer_info->time_stamp = jiffies;
0e014cb1 3405 buffer_info->next_to_watch = i;
9d5c8243
AK		 3406 buffer_info->dma = 0;
3407
3408 i++;
3409 if (i == tx_ring->count)
3410 i = 0;
3411 tx_ring->next_to_use = i;
3412
3413 return true;
3414 }
9d5c8243
AK		 3415 return false;
3416}
3417
3418#define IGB_MAX_TXD_PWR 16
3419#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3420
80785298 3421static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3422 unsigned int first)
9d5c8243
AK		 3423{
3424 struct igb_buffer *buffer_info;
80785298 3425 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3426 unsigned int len = skb_headlen(skb);
3427 unsigned int count = 0, i;
3428 unsigned int f;
65689fef 3429 dma_addr_t *map;
9d5c8243
AK		 3430
3431 i = tx_ring->next_to_use;
3432
80785298
AD		 3433 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3434 dev_err(&pdev->dev, "TX DMA map failed\n");
65689fef
AD		 3435 return 0;
3436 }
3437
3438 map = skb_shinfo(skb)->dma_maps;
3439
9d5c8243
AK		 3440 buffer_info = &tx_ring->buffer_info[i];
3441 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3442 buffer_info->length = len;
3443 /* set time_stamp *before* dma to help avoid a possible race */
3444 buffer_info->time_stamp = jiffies;
0e014cb1 3445 buffer_info->next_to_watch = i;
042a53a9 3446 buffer_info->dma = skb_shinfo(skb)->dma_head;
9d5c8243
AK		 3447
3448 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3449 struct skb_frag_struct *frag;
3450
65689fef
AD		 3451 i++;
3452 if (i == tx_ring->count)
3453 i = 0;
3454
9d5c8243
AK		 3455 frag = &skb_shinfo(skb)->frags[f];
3456 len = frag->size;
3457
3458 buffer_info = &tx_ring->buffer_info[i];
3459 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3460 buffer_info->length = len;
3461 buffer_info->time_stamp = jiffies;
0e014cb1 3462 buffer_info->next_to_watch = i;
65689fef 3463 buffer_info->dma = map[count];
9d5c8243 3464 count++;
9d5c8243
AK		 3465 }
3466
9d5c8243 3467 tx_ring->buffer_info[i].skb = skb;
0e014cb1 3468 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3469
042a53a9 3470 return count + 1;
9d5c8243
AK		 3471}
3472
85ad76b2 3473static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3474 int tx_flags, int count, u32 paylen,
3475 u8 hdr_len)
3476{
3477 union e1000_adv_tx_desc *tx_desc = NULL;
3478 struct igb_buffer *buffer_info;
3479 u32 olinfo_status = 0, cmd_type_len;
3480 unsigned int i;
3481
3482 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3483 E1000_ADVTXD_DCMD_DEXT);
3484
3485 if (tx_flags & IGB_TX_FLAGS_VLAN)
3486 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3487
33af6bcc
PO		 3488 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3489 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3490
9d5c8243
AK		 3491 if (tx_flags & IGB_TX_FLAGS_TSO) {
3492 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3493
3494 /* insert tcp checksum */
3495 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3496
3497 /* insert ip checksum */
3498 if (tx_flags & IGB_TX_FLAGS_IPV4)
3499 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3500
3501 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3502 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3503 }
3504
85ad76b2
AD		 3505 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3506 (tx_flags & (IGB_TX_FLAGS_CSUM |
3507 IGB_TX_FLAGS_TSO |
7dfc16fa 3508 IGB_TX_FLAGS_VLAN)))
85ad76b2 3509 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3510
3511 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3512
3513 i = tx_ring->next_to_use;
3514 while (count--) {
3515 buffer_info = &tx_ring->buffer_info[i];
3516 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3517 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3518 tx_desc->read.cmd_type_len =
3519 cpu_to_le32(cmd_type_len | buffer_info->length);
3520 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3521 i++;
3522 if (i == tx_ring->count)
3523 i = 0;
3524 }
3525
85ad76b2 3526 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3527 /* Force memory writes to complete before letting h/w
3528 * know there are new descriptors to fetch. (Only
3529 * applicable for weak-ordered memory model archs,
3530 * such as IA-64). */
3531 wmb();
3532
3533 tx_ring->next_to_use = i;
fce99e34 3534 writel(i, tx_ring->tail);
9d5c8243
AK		 3535 /* we need this if more than one processor can write to our tail
3536 * at a time, it syncronizes IO on IA64/Altix systems */
3537 mmiowb();
3538}
3539
e694e964 3540static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3541{
e694e964
AD		 3542 struct net_device *netdev = tx_ring->netdev;
3543
661086df 3544 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3545
9d5c8243
AK		 3546 /* Herbert's original patch had:
3547 * smp_mb__after_netif_stop_queue();
3548 * but since that doesn't exist yet, just open code it. */
3549 smp_mb();
3550
3551 /* We need to check again in a case another CPU has just
3552 * made room available. */
c493ea45 3553 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3554 return -EBUSY;
3555
3556 /* A reprieve! */
661086df 3557 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3558 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3559 return 0;
3560}
3561
e694e964 3562static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3563{
c493ea45 3564 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3565 return 0;
e694e964 3566 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3567}
3568
b1a436c3
AD		 3569netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3570 struct igb_ring *tx_ring)
9d5c8243 3571{
e694e964 3572 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
0e014cb1 3573 unsigned int first;
9d5c8243 3574 unsigned int tx_flags = 0;
9d5c8243 3575 u8 hdr_len = 0;
65689fef 3576 int count = 0;
9d5c8243 3577 int tso = 0;
c5b9bd5e 3578 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3579
9d5c8243
AK		 3580 /* need: 1 descriptor per page,
3581 * + 2 desc gap to keep tail from touching head,
3582 * + 1 desc for skb->data,
3583 * + 1 desc for context descriptor,
3584 * otherwise try next time */
e694e964 3585 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3586 /* this is a hard error */
9d5c8243
AK		 3587 return NETDEV_TX_BUSY;
3588 }
33af6bcc 3589
33af6bcc
PO		 3590 if (unlikely(shtx->hardware)) {
3591 shtx->in_progress = 1;
3592 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3593 }
9d5c8243
AK		 3594
3595 if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
3596 tx_flags |= IGB_TX_FLAGS_VLAN;
3597 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3598 }
3599
661086df
PWJ		 3600 if (skb->protocol == htons(ETH_P_IP))
3601 tx_flags |= IGB_TX_FLAGS_IPV4;
3602
0e014cb1 3603 first = tx_ring->next_to_use;
85ad76b2
AD		 3604 if (skb_is_gso(skb)) {
3605 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3606 if (tso < 0) {
3607 dev_kfree_skb_any(skb);
3608 return NETDEV_TX_OK;
3609 }
9d5c8243
AK		 3610 }
3611
3612 if (tso)
3613 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3614 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3615 (skb->ip_summed == CHECKSUM_PARTIAL))
3616 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3617
65689fef
AD		 3618 /*
3619 * count reflects descriptors mapped, if 0 then mapping error
3620 * has occured and we need to rewind the descriptor queue
3621 */
80785298 3622 count = igb_tx_map_adv(tx_ring, skb, first);
65689fef 3623
85ad76b2 3624 if (!count) {
65689fef
AD		 3625 dev_kfree_skb_any(skb);
3626 tx_ring->buffer_info[first].time_stamp = 0;
3627 tx_ring->next_to_use = first;
85ad76b2 3628 return NETDEV_TX_OK;
65689fef 3629 }
9d5c8243 3630
85ad76b2
AD		 3631 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3632
3633 /* Make sure there is space in the ring for the next send. */
e694e964 3634 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3635
9d5c8243
AK		 3636 return NETDEV_TX_OK;
3637}
3638
3b29a56d
SH		 3639static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3640 struct net_device *netdev)
9d5c8243
AK		 3641{
3642 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3643 struct igb_ring *tx_ring;
661086df 3644 int r_idx = 0;
b1a436c3
AD		 3645
3646 if (test_bit(__IGB_DOWN, &adapter->state)) {
3647 dev_kfree_skb_any(skb);
3648 return NETDEV_TX_OK;
3649 }
3650
3651 if (skb->len <= 0) {
3652 dev_kfree_skb_any(skb);
3653 return NETDEV_TX_OK;
3654 }
3655
1bfaf07b 3656 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3657 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3658
3659 /* This goes back to the question of how to logically map a tx queue
3660 * to a flow. Right now, performance is impacted slightly negatively
3661 * if using multiple tx queues. If the stack breaks away from a
3662 * single qdisc implementation, we can look at this again. */
e694e964 3663 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3664}
3665
3666/**
3667 * igb_tx_timeout - Respond to a Tx Hang
3668 * @netdev: network interface device structure
3669 **/
3670static void igb_tx_timeout(struct net_device *netdev)
3671{
3672 struct igb_adapter *adapter = netdev_priv(netdev);
3673 struct e1000_hw *hw = &adapter->hw;
3674
3675 /* Do the reset outside of interrupt context */
3676 adapter->tx_timeout_count++;
f7ba205e 3677
9d5c8243 3678 schedule_work(&adapter->reset_task);
265de409
AD		 3679 wr32(E1000_EICS,
3680 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3681}
3682
3683static void igb_reset_task(struct work_struct *work)
3684{
3685 struct igb_adapter *adapter;
3686 adapter = container_of(work, struct igb_adapter, reset_task);
3687
3688 igb_reinit_locked(adapter);
3689}
3690
3691/**
3692 * igb_get_stats - Get System Network Statistics
3693 * @netdev: network interface device structure
3694 *
3695 * Returns the address of the device statistics structure.
3696 * The statistics are actually updated from the timer callback.
3697 **/
73cd78f1 3698static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3699{
9d5c8243 3700 /* only return the current stats */
8d24e933 3701 return &netdev->stats;
9d5c8243
AK		 3702}
3703
3704/**
3705 * igb_change_mtu - Change the Maximum Transfer Unit
3706 * @netdev: network interface device structure
3707 * @new_mtu: new value for maximum frame size
3708 *
3709 * Returns 0 on success, negative on failure
3710 **/
3711static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3712{
3713 struct igb_adapter *adapter = netdev_priv(netdev);
3714 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3715 u32 rx_buffer_len, i;
9d5c8243
AK		 3716
3717 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3718 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3719 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
3720 return -EINVAL;
3721 }
3722
9d5c8243
AK		 3723 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3724 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
3725 return -EINVAL;
3726 }
3727
3728 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3729 msleep(1);
73cd78f1 3730
9d5c8243
AK		 3731 /* igb_down has a dependency on max_frame_size */
3732 adapter->max_frame_size = max_frame;
9d5c8243
AK		 3733 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3734 * means we reserve 2 more, this pushes us to allocate from the next
3735 * larger slab size.
3736 * i.e. RXBUFFER_2048 --> size-4096 slab
3737 */
3738
7d95b717 3739 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3740 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3741 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3742 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3743 else
4c844851
AD		 3744 rx_buffer_len = IGB_RXBUFFER_128;
3745
3746 if (netif_running(netdev))
3747 igb_down(adapter);
9d5c8243
AK		 3748
3749 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
3750 netdev->mtu, new_mtu);
3751 netdev->mtu = new_mtu;
3752
4c844851
AD		 3753 for (i = 0; i < adapter->num_rx_queues; i++)
3754 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3755
9d5c8243
AK		 3756 if (netif_running(netdev))
3757 igb_up(adapter);
3758 else
3759 igb_reset(adapter);
3760
3761 clear_bit(__IGB_RESETTING, &adapter->state);
3762
3763 return 0;
3764}
3765
3766/**
3767 * igb_update_stats - Update the board statistics counters
3768 * @adapter: board private structure
3769 **/
3770
3771void igb_update_stats(struct igb_adapter *adapter)
3772{
8d24e933 3773 struct net_device *netdev = adapter->netdev;
9d5c8243
AK		 3774 struct e1000_hw *hw = &adapter->hw;
3775 struct pci_dev *pdev = adapter->pdev;
3f9c0164 3776 u32 rnbc;
9d5c8243 3777 u16 phy_tmp;
3f9c0164
AD		 3778 int i;
3779 u64 bytes, packets;
9d5c8243
AK		 3780
3781#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3782
3783 /*
3784 * Prevent stats update while adapter is being reset, or if the pci
3785 * connection is down.
3786 */
3787 if (adapter->link_speed == 0)
3788 return;
3789 if (pci_channel_offline(pdev))
3790 return;
3791
3f9c0164
AD		 3792 bytes = 0;
3793 packets = 0;
3794 for (i = 0; i < adapter->num_rx_queues; i++) {
3795 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3796 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3797 netdev->stats.rx_fifo_errors += rqdpc_tmp;
3798 bytes += adapter->rx_ring[i].rx_stats.bytes;
3799 packets += adapter->rx_ring[i].rx_stats.packets;
3800 }
3801
3802 netdev->stats.rx_bytes = bytes;
3803 netdev->stats.rx_packets = packets;
3804
3805 bytes = 0;
3806 packets = 0;
3807 for (i = 0; i < adapter->num_tx_queues; i++) {
3808 bytes += adapter->tx_ring[i].tx_stats.bytes;
3809 packets += adapter->tx_ring[i].tx_stats.packets;
3810 }
3811 netdev->stats.tx_bytes = bytes;
3812 netdev->stats.tx_packets = packets;
3813
3814 /* read stats registers */
9d5c8243
AK		 3815 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3816 adapter->stats.gprc += rd32(E1000_GPRC);
3817 adapter->stats.gorc += rd32(E1000_GORCL);
3818 rd32(E1000_GORCH); /* clear GORCL */
3819 adapter->stats.bprc += rd32(E1000_BPRC);
3820 adapter->stats.mprc += rd32(E1000_MPRC);
3821 adapter->stats.roc += rd32(E1000_ROC);
3822
3823 adapter->stats.prc64 += rd32(E1000_PRC64);
3824 adapter->stats.prc127 += rd32(E1000_PRC127);
3825 adapter->stats.prc255 += rd32(E1000_PRC255);
3826 adapter->stats.prc511 += rd32(E1000_PRC511);
3827 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3828 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3829 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3830 adapter->stats.sec += rd32(E1000_SEC);
3831
3832 adapter->stats.mpc += rd32(E1000_MPC);
3833 adapter->stats.scc += rd32(E1000_SCC);
3834 adapter->stats.ecol += rd32(E1000_ECOL);
3835 adapter->stats.mcc += rd32(E1000_MCC);
3836 adapter->stats.latecol += rd32(E1000_LATECOL);
3837 adapter->stats.dc += rd32(E1000_DC);
3838 adapter->stats.rlec += rd32(E1000_RLEC);
3839 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3840 adapter->stats.xontxc += rd32(E1000_XONTXC);
3841 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3842 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3843 adapter->stats.fcruc += rd32(E1000_FCRUC);
3844 adapter->stats.gptc += rd32(E1000_GPTC);
3845 adapter->stats.gotc += rd32(E1000_GOTCL);
3846 rd32(E1000_GOTCH); /* clear GOTCL */
3f9c0164
AD		 3847 rnbc = rd32(E1000_RNBC);
3848 adapter->stats.rnbc += rnbc;
3849 netdev->stats.rx_fifo_errors += rnbc;
9d5c8243
AK		 3850 adapter->stats.ruc += rd32(E1000_RUC);
3851 adapter->stats.rfc += rd32(E1000_RFC);
3852 adapter->stats.rjc += rd32(E1000_RJC);
3853 adapter->stats.tor += rd32(E1000_TORH);
3854 adapter->stats.tot += rd32(E1000_TOTH);
3855 adapter->stats.tpr += rd32(E1000_TPR);
3856
3857 adapter->stats.ptc64 += rd32(E1000_PTC64);
3858 adapter->stats.ptc127 += rd32(E1000_PTC127);
3859 adapter->stats.ptc255 += rd32(E1000_PTC255);
3860 adapter->stats.ptc511 += rd32(E1000_PTC511);
3861 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3862 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3863
3864 adapter->stats.mptc += rd32(E1000_MPTC);
3865 adapter->stats.bptc += rd32(E1000_BPTC);
3866
3867 /* used for adaptive IFS */
3868
3869 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3870 adapter->stats.tpt += hw->mac.tx_packet_delta;
3871 hw->mac.collision_delta = rd32(E1000_COLC);
3872 adapter->stats.colc += hw->mac.collision_delta;
3873
3874 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3875 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3876 adapter->stats.tncrs += rd32(E1000_TNCRS);
3877 adapter->stats.tsctc += rd32(E1000_TSCTC);
3878 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3879
3880 adapter->stats.iac += rd32(E1000_IAC);
3881 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3882 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3883 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3884 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3885 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3886 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3887 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3888 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3889
3890 /* Fill out the OS statistics structure */
8d24e933
AK		 3891 netdev->stats.multicast = adapter->stats.mprc;
3892 netdev->stats.collisions = adapter->stats.colc;
9d5c8243
AK		 3893
3894 /* Rx Errors */
3895
3896 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 3897 * our own version based on RUC and ROC */
8d24e933 3898 netdev->stats.rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 3899 adapter->stats.crcerrs + adapter->stats.algnerrc +
3900 adapter->stats.ruc + adapter->stats.roc +
3901 adapter->stats.cexterr;
8d24e933 3902 netdev->stats.rx_length_errors = adapter->stats.ruc +
9d5c8243 3903 adapter->stats.roc;
8d24e933
AK		 3904 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3905 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3906 netdev->stats.rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 3907
3908 /* Tx Errors */
8d24e933 3909 netdev->stats.tx_errors = adapter->stats.ecol +
9d5c8243 3910 adapter->stats.latecol;
8d24e933
AK		 3911 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3912 netdev->stats.tx_window_errors = adapter->stats.latecol;
3913 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 3914
3915 /* Tx Dropped needs to be maintained elsewhere */
3916
3917 /* Phy Stats */
3918 if (hw->phy.media_type == e1000_media_type_copper) {
3919 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 3920 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 3921 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3922 adapter->phy_stats.idle_errors += phy_tmp;
3923 }
3924 }
3925
3926 /* Management Stats */
3927 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3928 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3929 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3930}
3931
9d5c8243
AK		 3932static irqreturn_t igb_msix_other(int irq, void *data)
3933{
047e0030 3934 struct igb_adapter *adapter = data;
9d5c8243 3935 struct e1000_hw *hw = &adapter->hw;
844290e5 3936 u32 icr = rd32(E1000_ICR);
844290e5 3937 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 3938
047e0030 3939 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 3940 /* HW is reporting DMA is out of sync */
3941 adapter->stats.doosync++;
3942 }
eebbbdba 3943
4ae196df
AD		 3944 /* Check for a mailbox event */
3945 if (icr & E1000_ICR_VMMB)
3946 igb_msg_task(adapter);
3947
3948 if (icr & E1000_ICR_LSC) {
3949 hw->mac.get_link_status = 1;
3950 /* guard against interrupt when we're going down */
3951 if (!test_bit(__IGB_DOWN, &adapter->state))
3952 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3953 }
3954
3955 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_VMMB);
844290e5 3956 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 3957
3958 return IRQ_HANDLED;
3959}
3960
047e0030 3961static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 3962{
047e0030 3963 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 3964
047e0030
AD		 3965 if (!q_vector->set_itr)
3966 return;
73cd78f1 3967
047e0030
AD		 3968 if (!itr_val)
3969 itr_val = 0x4;
661086df 3970
047e0030
AD		 3971 if (q_vector->itr_shift)
3972 itr_val |= itr_val << q_vector->itr_shift;
661086df 3973 else
047e0030 3974 itr_val |= 0x8000000;
661086df 3975
047e0030
AD		 3976 writel(itr_val, q_vector->itr_register);
3977 q_vector->set_itr = 0;
6eb5a7f1
AD		 3978}
3979
047e0030 3980static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 3981{
047e0030 3982 struct igb_q_vector *q_vector = data;
9d5c8243 3983
047e0030
AD		 3984 /* Write the ITR value calculated from the previous interrupt. */
3985 igb_write_itr(q_vector);
9d5c8243 3986
047e0030 3987 napi_schedule(&q_vector->napi);
844290e5 3988
047e0030 3989 return IRQ_HANDLED;
fe4506b6
JC		 3990}
3991
421e02f0 3992#ifdef CONFIG_IGB_DCA
047e0030 3993static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 3994{
047e0030 3995 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 3996 struct e1000_hw *hw = &adapter->hw;
3997 int cpu = get_cpu();
fe4506b6 3998
047e0030
AD		 3999 if (q_vector->cpu == cpu)
4000 goto out_no_update;
4001
4002 if (q_vector->tx_ring) {
4003 int q = q_vector->tx_ring->reg_idx;
4004 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4005 if (hw->mac.type == e1000_82575) {
4006 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4007 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4008 } else {
047e0030
AD		 4009 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4010 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4011 E1000_DCA_TXCTRL_CPUID_SHIFT;
4012 }
4013 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4014 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4015 }
4016 if (q_vector->rx_ring) {
4017 int q = q_vector->rx_ring->reg_idx;
4018 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4019 if (hw->mac.type == e1000_82575) {
2d064c06 4020 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4021 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4022 } else {
4023 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4024 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4025 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4026 }
fe4506b6
JC		 4027 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4028 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4029 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4030 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4031 }
047e0030
AD		 4032 q_vector->cpu = cpu;
4033out_no_update:
fe4506b6
JC		 4034 put_cpu();
4035}
4036
4037static void igb_setup_dca(struct igb_adapter *adapter)
4038{
7e0e99ef 4039 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4040 int i;
4041
7dfc16fa 4042 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4043 return;
4044
7e0e99ef
AD		 4045 /* Always use CB2 mode, difference is masked in the CB driver. */
4046 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4047
047e0030
AD		 4048 for (i = 0; i < adapter->num_q_vectors; i++) {
4049 struct igb_q_vector *q_vector = adapter->q_vector[i];
4050 q_vector->cpu = -1;
4051 igb_update_dca(q_vector);
fe4506b6
JC		 4052 }
4053}
4054
4055static int __igb_notify_dca(struct device *dev, void *data)
4056{
4057 struct net_device *netdev = dev_get_drvdata(dev);
4058 struct igb_adapter *adapter = netdev_priv(netdev);
4059 struct e1000_hw *hw = &adapter->hw;
4060 unsigned long event = *(unsigned long *)data;
4061
4062 switch (event) {
4063 case DCA_PROVIDER_ADD:
4064 /* if already enabled, don't do it again */
7dfc16fa 4065 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4066 break;
fe4506b6
JC		 4067 /* Always use CB2 mode, difference is masked
4068 * in the CB driver. */
cbd347ad 4069 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
fe4506b6 4070 if (dca_add_requester(dev) == 0) {
bbd98fe4 4071 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6
JC		 4072 dev_info(&adapter->pdev->dev, "DCA enabled\n");
4073 igb_setup_dca(adapter);
4074 break;
4075 }
4076 /* Fall Through since DCA is disabled. */
4077 case DCA_PROVIDER_REMOVE:
7dfc16fa 4078 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4079 /* without this a class_device is left
047e0030 4080 * hanging around in the sysfs model */
fe4506b6
JC		 4081 dca_remove_requester(dev);
4082 dev_info(&adapter->pdev->dev, "DCA disabled\n");
7dfc16fa 4083 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4084 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4085 }
4086 break;
4087 }
bbd98fe4 4088
fe4506b6 4089 return 0;
9d5c8243
AK		 4090}
4091
fe4506b6
JC		 4092static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4093 void *p)
4094{
4095 int ret_val;
4096
4097 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4098 __igb_notify_dca);
4099
4100 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4101}
421e02f0 4102#endif /* CONFIG_IGB_DCA */
9d5c8243 4103
4ae196df
AD		 4104static void igb_ping_all_vfs(struct igb_adapter *adapter)
4105{
4106 struct e1000_hw *hw = &adapter->hw;
4107 u32 ping;
4108 int i;
4109
4110 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4111 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4112 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4113 ping |= E1000_VT_MSGTYPE_CTS;
4114 igb_write_mbx(hw, &ping, 1, i);
4115 }
4116}
4117
7d5753f0
AD		 4118static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4119{
4120 struct e1000_hw *hw = &adapter->hw;
4121 u32 vmolr = rd32(E1000_VMOLR(vf));
4122 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4123
4124 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4125 IGB_VF_FLAG_MULTI_PROMISC);
4126 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4127
4128 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4129 vmolr |= E1000_VMOLR_MPME;
4130 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4131 } else {
4132 /*
4133 * if we have hashes and we are clearing a multicast promisc
4134 * flag we need to write the hashes to the MTA as this step
4135 * was previously skipped
4136 */
4137 if (vf_data->num_vf_mc_hashes > 30) {
4138 vmolr |= E1000_VMOLR_MPME;
4139 } else if (vf_data->num_vf_mc_hashes) {
4140 int j;
4141 vmolr |= E1000_VMOLR_ROMPE;
4142 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4143 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4144 }
4145 }
4146
4147 wr32(E1000_VMOLR(vf), vmolr);
4148
4149 /* there are flags left unprocessed, likely not supported */
4150 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4151 return -EINVAL;
4152
4153 return 0;
4154
4155}
4156
4ae196df
AD		 4157static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4158 u32 *msgbuf, u32 vf)
4159{
4160 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4161 u16 *hash_list = (u16 *)&msgbuf[1];
4162 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4163 int i;
4164
7d5753f0 4165 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4166 * to this VF for later use to restore when the PF multi cast
4167 * list changes
4168 */
4169 vf_data->num_vf_mc_hashes = n;
4170
7d5753f0
AD		 4171 /* only up to 30 hash values supported */
4172 if (n > 30)
4173 n = 30;
4174
4175 /* store the hashes for later use */
4ae196df 4176 for (i = 0; i < n; i++)
a419aef8 4177 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4178
4179 /* Flush and reset the mta with the new values */
ff41f8dc 4180 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4181
4182 return 0;
4183}
4184
4185static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4186{
4187 struct e1000_hw *hw = &adapter->hw;
4188 struct vf_data_storage *vf_data;
4189 int i, j;
4190
4191 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4192 u32 vmolr = rd32(E1000_VMOLR(i));
4193 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4194
4ae196df 4195 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4196
4197 if ((vf_data->num_vf_mc_hashes > 30) ||
4198 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4199 vmolr |= E1000_VMOLR_MPME;
4200 } else if (vf_data->num_vf_mc_hashes) {
4201 vmolr |= E1000_VMOLR_ROMPE;
4202 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4203 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4204 }
4205 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4206 }
4207}
4208
4209static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4210{
4211 struct e1000_hw *hw = &adapter->hw;
4212 u32 pool_mask, reg, vid;
4213 int i;
4214
4215 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4216
4217 /* Find the vlan filter for this id */
4218 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4219 reg = rd32(E1000_VLVF(i));
4220
4221 /* remove the vf from the pool */
4222 reg &= ~pool_mask;
4223
4224 /* if pool is empty then remove entry from vfta */
4225 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4226 (reg & E1000_VLVF_VLANID_ENABLE)) {
4227 reg = 0;
4228 vid = reg & E1000_VLVF_VLANID_MASK;
4229 igb_vfta_set(hw, vid, false);
4230 }
4231
4232 wr32(E1000_VLVF(i), reg);
4233 }
ae641bdc
AD		 4234
4235 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4236}
4237
4238static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4239{
4240 struct e1000_hw *hw = &adapter->hw;
4241 u32 reg, i;
4242
51466239
AD		 4243 /* The vlvf table only exists on 82576 hardware and newer */
4244 if (hw->mac.type < e1000_82576)
4245 return -1;
4246
4247 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4248 if (!adapter->vfs_allocated_count)
4249 return -1;
4250
4251 /* Find the vlan filter for this id */
4252 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4253 reg = rd32(E1000_VLVF(i));
4254 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4255 vid == (reg & E1000_VLVF_VLANID_MASK))
4256 break;
4257 }
4258
4259 if (add) {
4260 if (i == E1000_VLVF_ARRAY_SIZE) {
4261 /* Did not find a matching VLAN ID entry that was
4262 * enabled. Search for a free filter entry, i.e.
4263 * one without the enable bit set
4264 */
4265 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4266 reg = rd32(E1000_VLVF(i));
4267 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4268 break;
4269 }
4270 }
4271 if (i < E1000_VLVF_ARRAY_SIZE) {
4272 /* Found an enabled/available entry */
4273 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4274
4275 /* if !enabled we need to set this up in vfta */
4276 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4277 /* add VID to filter table */
4278 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4279 reg |= E1000_VLVF_VLANID_ENABLE;
4280 }
cad6d05f
AD		 4281 reg &= ~E1000_VLVF_VLANID_MASK;
4282 reg |= vid;
4ae196df 4283 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4284
4285 /* do not modify RLPML for PF devices */
4286 if (vf >= adapter->vfs_allocated_count)
4287 return 0;
4288
4289 if (!adapter->vf_data[vf].vlans_enabled) {
4290 u32 size;
4291 reg = rd32(E1000_VMOLR(vf));
4292 size = reg & E1000_VMOLR_RLPML_MASK;
4293 size += 4;
4294 reg &= ~E1000_VMOLR_RLPML_MASK;
4295 reg |= size;
4296 wr32(E1000_VMOLR(vf), reg);
4297 }
ae641bdc 4298
51466239 4299 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4300 return 0;
4301 }
4302 } else {
4303 if (i < E1000_VLVF_ARRAY_SIZE) {
4304 /* remove vf from the pool */
4305 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4306 /* if pool is empty then remove entry from vfta */
4307 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4308 reg = 0;
4309 igb_vfta_set(hw, vid, false);
4310 }
4311 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4312
4313 /* do not modify RLPML for PF devices */
4314 if (vf >= adapter->vfs_allocated_count)
4315 return 0;
4316
4317 adapter->vf_data[vf].vlans_enabled--;
4318 if (!adapter->vf_data[vf].vlans_enabled) {
4319 u32 size;
4320 reg = rd32(E1000_VMOLR(vf));
4321 size = reg & E1000_VMOLR_RLPML_MASK;
4322 size -= 4;
4323 reg &= ~E1000_VMOLR_RLPML_MASK;
4324 reg |= size;
4325 wr32(E1000_VMOLR(vf), reg);
4326 }
4ae196df
AD		 4327 return 0;
4328 }
4329 }
4330 return -1;
4331}
4332
4333static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4334{
4335 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4336 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4337
4338 return igb_vlvf_set(adapter, vid, add, vf);
4339}
4340
f2ca0dbe 4341static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4342{
f2ca0dbe
AD		 4343 /* clear all flags */
4344 adapter->vf_data[vf].flags = 0;
4345 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4346
4347 /* reset offloads to defaults */
7d5753f0 4348 igb_set_vmolr(adapter, vf);
4ae196df
AD		 4349
4350 /* reset vlans for device */
4351 igb_clear_vf_vfta(adapter, vf);
4352
4353 /* reset multicast table array for vf */
4354 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4355
4356 /* Flush and reset the mta with the new values */
ff41f8dc 4357 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4358}
4359
f2ca0dbe
AD		 4360static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4361{
4362 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4363
4364 /* generate a new mac address as we were hotplug removed/added */
4365 random_ether_addr(vf_mac);
4366
4367 /* process remaining reset events */
4368 igb_vf_reset(adapter, vf);
4369}
4370
4371static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4372{
4373 struct e1000_hw *hw = &adapter->hw;
4374 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4375 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4376 u32 reg, msgbuf[3];
4377 u8 *addr = (u8 *)(&msgbuf[1]);
4378
4379 /* process all the same items cleared in a function level reset */
f2ca0dbe 4380 igb_vf_reset(adapter, vf);
4ae196df
AD		 4381
4382 /* set vf mac address */
26ad9178 4383 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4384
4385 /* enable transmit and receive for vf */
4386 reg = rd32(E1000_VFTE);
4387 wr32(E1000_VFTE, reg | (1 << vf));
4388 reg = rd32(E1000_VFRE);
4389 wr32(E1000_VFRE, reg | (1 << vf));
4390
f2ca0dbe 4391 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4392
4393 /* reply to reset with ack and vf mac address */
4394 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4395 memcpy(addr, vf_mac, 6);
4396 igb_write_mbx(hw, msgbuf, 3, vf);
4397}
4398
4399static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4400{
f2ca0dbe
AD		 4401 unsigned char *addr = (char *)&msg[1];
4402 int err = -1;
4ae196df 4403
f2ca0dbe
AD		 4404 if (is_valid_ether_addr(addr))
4405 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4406
f2ca0dbe 4407 return err;
4ae196df
AD		 4408}
4409
4410static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4411{
4412 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4413 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4414 u32 msg = E1000_VT_MSGTYPE_NACK;
4415
4416 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4417 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4418 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4419 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4420 vf_data->last_nack = jiffies;
4ae196df
AD		 4421 }
4422}
4423
f2ca0dbe 4424static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4425{
f2ca0dbe
AD		 4426 struct pci_dev *pdev = adapter->pdev;
4427 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4428 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4429 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4430 s32 retval;
4431
f2ca0dbe 4432 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df
AD		 4433
4434 if (retval)
f2ca0dbe 4435 dev_err(&pdev->dev, "Error receiving message from VF\n");
4ae196df
AD		 4436
4437 /* this is a message we already processed, do nothing */
4438 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4439 return;
4ae196df
AD		 4440
4441 /*
4442 * until the vf completes a reset it should not be
4443 * allowed to start any configuration.
4444 */
4445
4446 if (msgbuf[0] == E1000_VF_RESET) {
4447 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4448 return;
4ae196df
AD		 4449 }
4450
f2ca0dbe
AD		 4451 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4452 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
4453 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4454 igb_write_mbx(hw, msgbuf, 1, vf);
4455 vf_data->last_nack = jiffies;
4456 }
4457 return;
4ae196df
AD		 4458 }
4459
4460 switch ((msgbuf[0] & 0xFFFF)) {
4461 case E1000_VF_SET_MAC_ADDR:
4462 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4463 break;
7d5753f0
AD		 4464 case E1000_VF_SET_PROMISC:
4465 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4466 break;
4ae196df
AD		 4467 case E1000_VF_SET_MULTICAST:
4468 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4469 break;
4470 case E1000_VF_SET_LPE:
4471 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4472 break;
4473 case E1000_VF_SET_VLAN:
4474 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4475 break;
4476 default:
4477 dev_err(&adapter->pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4478 retval = -1;
4479 break;
4480 }
4481
4482 /* notify the VF of the results of what it sent us */
4483 if (retval)
4484 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4485 else
4486 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4487
4488 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4489
4490 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4491}
4ae196df 4492
f2ca0dbe
AD		 4493static void igb_msg_task(struct igb_adapter *adapter)
4494{
4495 struct e1000_hw *hw = &adapter->hw;
4496 u32 vf;
4497
4498 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4499 /* process any reset requests */
4500 if (!igb_check_for_rst(hw, vf))
4501 igb_vf_reset_event(adapter, vf);
4502
4503 /* process any messages pending */
4504 if (!igb_check_for_msg(hw, vf))
4505 igb_rcv_msg_from_vf(adapter, vf);
4506
4507 /* process any acks */
4508 if (!igb_check_for_ack(hw, vf))
4509 igb_rcv_ack_from_vf(adapter, vf);
4510 }
4ae196df
AD		 4511}
4512
68d480c4
AD		 4513/**
4514 * igb_set_uta - Set unicast filter table address
4515 * @adapter: board private structure
4516 *
4517 * The unicast table address is a register array of 32-bit registers.
4518 * The table is meant to be used in a way similar to how the MTA is used
4519 * however due to certain limitations in the hardware it is necessary to
4520 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4521 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4522 **/
4523static void igb_set_uta(struct igb_adapter *adapter)
4524{
4525 struct e1000_hw *hw = &adapter->hw;
4526 int i;
4527
4528 /* The UTA table only exists on 82576 hardware and newer */
4529 if (hw->mac.type < e1000_82576)
4530 return;
4531
4532 /* we only need to do this if VMDq is enabled */
4533 if (!adapter->vfs_allocated_count)
4534 return;
4535
4536 for (i = 0; i < hw->mac.uta_reg_count; i++)
4537 array_wr32(E1000_UTA, i, ~0);
4538}
4539
9d5c8243
AK		 4540/**
4541 * igb_intr_msi - Interrupt Handler
4542 * @irq: interrupt number
4543 * @data: pointer to a network interface device structure
4544 **/
4545static irqreturn_t igb_intr_msi(int irq, void *data)
4546{
047e0030
AD		 4547 struct igb_adapter *adapter = data;
4548 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4549 struct e1000_hw *hw = &adapter->hw;
4550 /* read ICR disables interrupts using IAM */
4551 u32 icr = rd32(E1000_ICR);
4552
047e0030 4553 igb_write_itr(q_vector);
9d5c8243 4554
047e0030 4555 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4556 /* HW is reporting DMA is out of sync */
4557 adapter->stats.doosync++;
4558 }
4559
9d5c8243
AK		 4560 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4561 hw->mac.get_link_status = 1;
4562 if (!test_bit(__IGB_DOWN, &adapter->state))
4563 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4564 }
4565
047e0030 4566 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4567
4568 return IRQ_HANDLED;
4569}
4570
4571/**
4a3c6433 4572 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4573 * @irq: interrupt number
4574 * @data: pointer to a network interface device structure
4575 **/
4576static irqreturn_t igb_intr(int irq, void *data)
4577{
047e0030
AD		 4578 struct igb_adapter *adapter = data;
4579 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4580 struct e1000_hw *hw = &adapter->hw;
4581 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4582 * need for the IMC write */
4583 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4584 if (!icr)
4585 return IRQ_NONE; /* Not our interrupt */
4586
047e0030 4587 igb_write_itr(q_vector);
9d5c8243
AK		 4588
4589 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4590 * not set, then the adapter didn't send an interrupt */
4591 if (!(icr & E1000_ICR_INT_ASSERTED))
4592 return IRQ_NONE;
4593
047e0030 4594 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4595 /* HW is reporting DMA is out of sync */
4596 adapter->stats.doosync++;
4597 }
4598
9d5c8243
AK		 4599 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4600 hw->mac.get_link_status = 1;
4601 /* guard against interrupt when we're going down */
4602 if (!test_bit(__IGB_DOWN, &adapter->state))
4603 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4604 }
4605
047e0030 4606 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4607
4608 return IRQ_HANDLED;
4609}
4610
047e0030 4611static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4612{
047e0030 4613 struct igb_adapter *adapter = q_vector->adapter;
46544258 4614 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4615
4fc82adf
AD		 4616 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4617 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4618 if (!adapter->msix_entries)
6eb5a7f1 4619 igb_set_itr(adapter);
46544258 4620 else
047e0030 4621 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4622 }
4623
46544258
AD		 4624 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4625 if (adapter->msix_entries)
047e0030 4626 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4627 else
4628 igb_irq_enable(adapter);
4629 }
9d5c8243
AK		 4630}
4631
46544258
AD		 4632/**
4633 * igb_poll - NAPI Rx polling callback
4634 * @napi: napi polling structure
4635 * @budget: count of how many packets we should handle
4636 **/
4637static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4638{
047e0030
AD		 4639 struct igb_q_vector *q_vector = container_of(napi,
4640 struct igb_q_vector,
4641 napi);
4642 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4643
421e02f0 4644#ifdef CONFIG_IGB_DCA
047e0030
AD		 4645 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4646 igb_update_dca(q_vector);
fe4506b6 4647#endif
047e0030
AD		 4648 if (q_vector->tx_ring)
4649 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4650
047e0030
AD		 4651 if (q_vector->rx_ring)
4652 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4653
4654 if (!tx_clean_complete)
4655 work_done = budget;
46544258 4656
9d5c8243 4657 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4658 if (work_done < budget) {
288379f0 4659 napi_complete(napi);
047e0030 4660 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4661 }
4662
46544258 4663 return work_done;
9d5c8243 4664}
6d8126f9 4665
33af6bcc 4666/**
c5b9bd5e 4667 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4668 * @adapter: board private structure
c5b9bd5e
AD		 4669 * @shhwtstamps: timestamp structure to update
4670 * @regval: unsigned 64bit system time value.
4671 *
4672 * We need to convert the system time value stored in the RX/TXSTMP registers
4673 * into a hwtstamp which can be used by the upper level timestamping functions
4674 */
4675static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4676 struct skb_shared_hwtstamps *shhwtstamps,
4677 u64 regval)
4678{
4679 u64 ns;
4680
4681 ns = timecounter_cyc2time(&adapter->clock, regval);
4682 timecompare_update(&adapter->compare, ns);
4683 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4684 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4685 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4686}
4687
4688/**
4689 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4690 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4691 * @skb: packet that was just sent
4692 *
4693 * If we were asked to do hardware stamping and such a time stamp is
4694 * available, then it must have been for this skb here because we only
4695 * allow only one such packet into the queue.
4696 */
c5b9bd5e 4697static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4698{
c5b9bd5e 4699 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4700 union skb_shared_tx *shtx = skb_tx(skb);
4701 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4702 struct skb_shared_hwtstamps shhwtstamps;
4703 u64 regval;
33af6bcc 4704
c5b9bd5e
AD		 4705 /* if skb does not support hw timestamp or TX stamp not valid exit */
4706 if (likely(!shtx->hardware) ||
4707 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4708 return;
4709
4710 regval = rd32(E1000_TXSTMPL);
4711 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4712
4713 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4714 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4715}
4716
9d5c8243
AK		 4717/**
4718 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4719 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4720 * returns true if ring is completely cleaned
4721 **/
047e0030 4722static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4723{
047e0030
AD		 4724 struct igb_adapter *adapter = q_vector->adapter;
4725 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4726 struct net_device *netdev = tx_ring->netdev;
0e014cb1 4727 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 4728 struct igb_buffer *buffer_info;
4729 struct sk_buff *skb;
0e014cb1 4730 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 4731 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 4732 unsigned int i, eop, count = 0;
4733 bool cleaned = false;
9d5c8243 4734
9d5c8243 4735 i = tx_ring->next_to_clean;
0e014cb1
AD		 4736 eop = tx_ring->buffer_info[i].next_to_watch;
4737 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4738
4739 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4740 (count < tx_ring->count)) {
4741 for (cleaned = false; !cleaned; count++) {
4742 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 4743 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 4744 cleaned = (i == eop);
9d5c8243
AK		 4745 skb = buffer_info->skb;
4746
4747 if (skb) {
4748 unsigned int segs, bytecount;
4749 /* gso_segs is currently only valid for tcp */
4750 segs = skb_shinfo(skb)->gso_segs ?: 1;
4751 /* multiply data chunks by size of headers */
4752 bytecount = ((segs - 1) * skb_headlen(skb)) +
4753 skb->len;
4754 total_packets += segs;
4755 total_bytes += bytecount;
33af6bcc 4756
c5b9bd5e 4757 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 4758 }
4759
80785298 4760 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 4761 tx_desc->wb.status = 0;
9d5c8243
AK		 4762
4763 i++;
4764 if (i == tx_ring->count)
4765 i = 0;
9d5c8243 4766 }
0e014cb1
AD		 4767 eop = tx_ring->buffer_info[i].next_to_watch;
4768 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4769 }
4770
9d5c8243
AK		 4771 tx_ring->next_to_clean = i;
4772
fc7d345d 4773 if (unlikely(count &&
9d5c8243 4774 netif_carrier_ok(netdev) &&
c493ea45 4775 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 4776 /* Make sure that anybody stopping the queue after this
4777 * sees the new next_to_clean.
4778 */
4779 smp_mb();
661086df
PWJ		 4780 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4781 !(test_bit(__IGB_DOWN, &adapter->state))) {
4782 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4783 tx_ring->tx_stats.restart_queue++;
661086df 4784 }
9d5c8243
AK		 4785 }
4786
4787 if (tx_ring->detect_tx_hung) {
4788 /* Detect a transmit hang in hardware, this serializes the
4789 * check with the clearing of time_stamp and movement of i */
4790 tx_ring->detect_tx_hung = false;
4791 if (tx_ring->buffer_info[i].time_stamp &&
4792 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4793 (adapter->tx_timeout_factor * HZ))
4794 && !(rd32(E1000_STATUS) &
4795 E1000_STATUS_TXOFF)) {
4796
9d5c8243 4797 /* detected Tx unit hang */
80785298 4798 dev_err(&tx_ring->pdev->dev,
9d5c8243 4799 "Detected Tx Unit Hang\n"
2d064c06 4800 " Tx Queue <%d>\n"
9d5c8243
AK		 4801 " TDH <%x>\n"
4802 " TDT <%x>\n"
4803 " next_to_use <%x>\n"
4804 " next_to_clean <%x>\n"
9d5c8243
AK		 4805 "buffer_info[next_to_clean]\n"
4806 " time_stamp <%lx>\n"
0e014cb1 4807 " next_to_watch <%x>\n"
9d5c8243
AK		 4808 " jiffies <%lx>\n"
4809 " desc.status <%x>\n",
2d064c06 4810 tx_ring->queue_index,
fce99e34
AD		 4811 readl(tx_ring->head),
4812 readl(tx_ring->tail),
9d5c8243
AK		 4813 tx_ring->next_to_use,
4814 tx_ring->next_to_clean,
f7ba205e 4815 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 4816 eop,
9d5c8243 4817 jiffies,
0e014cb1 4818 eop_desc->wb.status);
661086df 4819 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 4820 }
4821 }
4822 tx_ring->total_bytes += total_bytes;
4823 tx_ring->total_packets += total_packets;
e21ed353
AD		 4824 tx_ring->tx_stats.bytes += total_bytes;
4825 tx_ring->tx_stats.packets += total_packets;
0e014cb1 4826 return (count < tx_ring->count);
9d5c8243
AK		 4827}
4828
9d5c8243
AK		 4829/**
4830 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 4831 * @q_vector: structure containing interrupt and ring information
4832 * @skb: packet to send up
4833 * @vlan_tag: vlan tag for packet
9d5c8243 4834 **/
047e0030
AD		 4835static void igb_receive_skb(struct igb_q_vector *q_vector,
4836 struct sk_buff *skb,
4837 u16 vlan_tag)
4838{
4839 struct igb_adapter *adapter = q_vector->adapter;
4840
4841 if (vlan_tag)
4842 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4843 vlan_tag, skb);
182ff8df 4844 else
047e0030 4845 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 4846}
4847
04a5fcaa 4848static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 4849 u32 status_err, struct sk_buff *skb)
4850{
4851 skb->ip_summed = CHECKSUM_NONE;
4852
4853 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 4854 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4855 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 4856 return;
85ad76b2 4857
9d5c8243
AK		 4858 /* TCP/UDP checksum error bit is set */
4859 if (status_err &
4860 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 4861 /*
4862 * work around errata with sctp packets where the TCPE aka
4863 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4864 * packets, (aka let the stack check the crc32c)
4865 */
85ad76b2
AD		 4866 if ((skb->len == 60) &&
4867 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 4868 ring->rx_stats.csum_err++;
85ad76b2 4869
9d5c8243 4870 /* let the stack verify checksum errors */
9d5c8243
AK		 4871 return;
4872 }
4873 /* It must be a TCP or UDP packet with a valid checksum */
4874 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4875 skb->ip_summed = CHECKSUM_UNNECESSARY;
4876
85ad76b2 4877 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 4878}
4879
c5b9bd5e
AD		 4880static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
4881 struct sk_buff *skb)
4882{
4883 struct igb_adapter *adapter = q_vector->adapter;
4884 struct e1000_hw *hw = &adapter->hw;
4885 u64 regval;
4886
4887 /*
4888 * If this bit is set, then the RX registers contain the time stamp. No
4889 * other packet will be time stamped until we read these registers, so
4890 * read the registers to make them available again. Because only one
4891 * packet can be time stamped at a time, we know that the register
4892 * values must belong to this one here and therefore we don't need to
4893 * compare any of the additional attributes stored for it.
4894 *
4895 * If nothing went wrong, then it should have a skb_shared_tx that we
4896 * can turn into a skb_shared_hwtstamps.
4897 */
4898 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
4899 return;
4900 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
4901 return;
4902
4903 regval = rd32(E1000_RXSTMPL);
4904 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4905
4906 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
4907}
4c844851 4908static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 4909 union e1000_adv_rx_desc *rx_desc)
4910{
4911 /* HW will not DMA in data larger than the given buffer, even if it
4912 * parses the (NFS, of course) header to be larger. In that case, it
4913 * fills the header buffer and spills the rest into the page.
4914 */
4915 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4916 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 4917 if (hlen > rx_ring->rx_buffer_len)
4918 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 4919 return hlen;
4920}
4921
047e0030
AD		 4922static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4923 int *work_done, int budget)
9d5c8243 4924{
047e0030 4925 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 4926 struct net_device *netdev = rx_ring->netdev;
80785298 4927 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 4928 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4929 struct igb_buffer *buffer_info , *next_buffer;
4930 struct sk_buff *skb;
9d5c8243
AK		 4931 bool cleaned = false;
4932 int cleaned_count = 0;
4933 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 4934 unsigned int i;
2d94d8ab
AD		 4935 u32 staterr;
4936 u16 length;
047e0030 4937 u16 vlan_tag;
9d5c8243
AK		 4938
4939 i = rx_ring->next_to_clean;
69d3ca53 4940 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 4941 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4942 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4943
4944 while (staterr & E1000_RXD_STAT_DD) {
4945 if (*work_done >= budget)
4946 break;
4947 (*work_done)++;
9d5c8243 4948
69d3ca53
AD		 4949 skb = buffer_info->skb;
4950 prefetch(skb->data - NET_IP_ALIGN);
4951 buffer_info->skb = NULL;
4952
4953 i++;
4954 if (i == rx_ring->count)
4955 i = 0;
4956 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4957 prefetch(next_rxd);
4958 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 4959
4960 length = le16_to_cpu(rx_desc->wb.upper.length);
4961 cleaned = true;
4962 cleaned_count++;
4963
2d94d8ab 4964 if (buffer_info->dma) {
bf36c1a0 4965 pci_unmap_single(pdev, buffer_info->dma,
4c844851 4966 rx_ring->rx_buffer_len,
bf36c1a0 4967 PCI_DMA_FROMDEVICE);
91615f76 4968 buffer_info->dma = 0;
4c844851 4969 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 4970 skb_put(skb, length);
4971 goto send_up;
4972 }
4c844851 4973 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 4974 }
4975
4976 if (length) {
9d5c8243 4977 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 4978 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 4979 buffer_info->page_dma = 0;
bf36c1a0
AD		 4980
4981 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4982 buffer_info->page,
4983 buffer_info->page_offset,
4984 length);
4985
6ec43fe6 4986 if (page_count(buffer_info->page) != 1)
bf36c1a0
AD		 4987 buffer_info->page = NULL;
4988 else
4989 get_page(buffer_info->page);
9d5c8243
AK		 4990
4991 skb->len += length;
4992 skb->data_len += length;
9d5c8243 4993
bf36c1a0 4994 skb->truesize += length;
9d5c8243 4995 }
9d5c8243 4996
bf36c1a0 4997 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 4998 buffer_info->skb = next_buffer->skb;
4999 buffer_info->dma = next_buffer->dma;
5000 next_buffer->skb = skb;
5001 next_buffer->dma = 0;
bf36c1a0
AD		 5002 goto next_desc;
5003 }
69d3ca53 5004send_up:
9d5c8243
AK		 5005 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5006 dev_kfree_skb_irq(skb);
5007 goto next_desc;
5008 }
9d5c8243 5009
c5b9bd5e 5010 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5011 total_bytes += skb->len;
5012 total_packets++;
5013
85ad76b2 5014 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5015
5016 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5017 skb_record_rx_queue(skb, rx_ring->queue_index);
5018
5019 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5020 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5021
047e0030 5022 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5023
9d5c8243
AK		 5024next_desc:
5025 rx_desc->wb.upper.status_error = 0;
5026
5027 /* return some buffers to hardware, one at a time is too slow */
5028 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5029 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5030 cleaned_count = 0;
5031 }
5032
5033 /* use prefetched values */
5034 rx_desc = next_rxd;
5035 buffer_info = next_buffer;
9d5c8243
AK		 5036 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5037 }
bf36c1a0 5038
9d5c8243 5039 rx_ring->next_to_clean = i;
c493ea45 5040 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5041
5042 if (cleaned_count)
3b644cf6 5043 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5044
5045 rx_ring->total_packets += total_packets;
5046 rx_ring->total_bytes += total_bytes;
5047 rx_ring->rx_stats.packets += total_packets;
5048 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5049 return cleaned;
5050}
5051
9d5c8243
AK		 5052/**
5053 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5054 * @adapter: address of board private structure
5055 **/
d7ee5b3a 5056void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5057{
e694e964 5058 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5059 union e1000_adv_rx_desc *rx_desc;
5060 struct igb_buffer *buffer_info;
5061 struct sk_buff *skb;
5062 unsigned int i;
db761762 5063 int bufsz;
9d5c8243
AK		 5064
5065 i = rx_ring->next_to_use;
5066 buffer_info = &rx_ring->buffer_info[i];
5067
4c844851 5068 bufsz = rx_ring->rx_buffer_len;
db761762 5069
9d5c8243
AK		 5070 while (cleaned_count--) {
5071 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5072
6ec43fe6 5073 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5074 if (!buffer_info->page) {
bf36c1a0
AD		 5075 buffer_info->page = alloc_page(GFP_ATOMIC);
5076 if (!buffer_info->page) {
04a5fcaa 5077 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5078 goto no_buffers;
5079 }
5080 buffer_info->page_offset = 0;
5081 } else {
5082 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5083 }
5084 buffer_info->page_dma =
80785298 5085 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5086 buffer_info->page_offset,
5087 PAGE_SIZE / 2,
9d5c8243
AK		 5088 PCI_DMA_FROMDEVICE);
5089 }
5090
5091 if (!buffer_info->skb) {
89d71a66 5092 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5093 if (!skb) {
04a5fcaa 5094 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5095 goto no_buffers;
5096 }
5097
9d5c8243 5098 buffer_info->skb = skb;
80785298
AD		 5099 buffer_info->dma = pci_map_single(rx_ring->pdev,
5100 skb->data,
9d5c8243
AK		 5101 bufsz,
5102 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 5103 }
5104 /* Refresh the desc even if buffer_addrs didn't change because
5105 * each write-back erases this info. */
6ec43fe6 5106 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5107 rx_desc->read.pkt_addr =
5108 cpu_to_le64(buffer_info->page_dma);
5109 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5110 } else {
5111 rx_desc->read.pkt_addr =
5112 cpu_to_le64(buffer_info->dma);
5113 rx_desc->read.hdr_addr = 0;
5114 }
5115
5116 i++;
5117 if (i == rx_ring->count)
5118 i = 0;
5119 buffer_info = &rx_ring->buffer_info[i];
5120 }
5121
5122no_buffers:
5123 if (rx_ring->next_to_use != i) {
5124 rx_ring->next_to_use = i;
5125 if (i == 0)
5126 i = (rx_ring->count - 1);
5127 else
5128 i--;
5129
5130 /* Force memory writes to complete before letting h/w
5131 * know there are new descriptors to fetch. (Only
5132 * applicable for weak-ordered memory model archs,
5133 * such as IA-64). */
5134 wmb();
fce99e34 5135 writel(i, rx_ring->tail);
9d5c8243
AK		 5136 }
5137}
5138
5139/**
5140 * igb_mii_ioctl -
5141 * @netdev:
5142 * @ifreq:
5143 * @cmd:
5144 **/
5145static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5146{
5147 struct igb_adapter *adapter = netdev_priv(netdev);
5148 struct mii_ioctl_data *data = if_mii(ifr);
5149
5150 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5151 return -EOPNOTSUPP;
5152
5153 switch (cmd) {
5154 case SIOCGMIIPHY:
5155 data->phy_id = adapter->hw.phy.addr;
5156 break;
5157 case SIOCGMIIREG:
f5f4cf08
AD		 5158 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5159 &data->val_out))
9d5c8243
AK		 5160 return -EIO;
5161 break;
5162 case SIOCSMIIREG:
5163 default:
5164 return -EOPNOTSUPP;
5165 }
5166 return 0;
5167}
5168
c6cb090b
PO		 5169/**
5170 * igb_hwtstamp_ioctl - control hardware time stamping
5171 * @netdev:
5172 * @ifreq:
5173 * @cmd:
5174 *
33af6bcc
PO		 5175 * Outgoing time stamping can be enabled and disabled. Play nice and
5176 * disable it when requested, although it shouldn't case any overhead
5177 * when no packet needs it. At most one packet in the queue may be
5178 * marked for time stamping, otherwise it would be impossible to tell
5179 * for sure to which packet the hardware time stamp belongs.
5180 *
5181 * Incoming time stamping has to be configured via the hardware
5182 * filters. Not all combinations are supported, in particular event
5183 * type has to be specified. Matching the kind of event packet is
5184 * not supported, with the exception of "all V2 events regardless of
5185 * level 2 or 4".
5186 *
c6cb090b
PO		 5187 **/
5188static int igb_hwtstamp_ioctl(struct net_device *netdev,
5189 struct ifreq *ifr, int cmd)
5190{
33af6bcc
PO		 5191 struct igb_adapter *adapter = netdev_priv(netdev);
5192 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5193 struct hwtstamp_config config;
c5b9bd5e
AD		 5194 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5195 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5196 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5197 bool is_l4 = false;
5198 bool is_l2 = false;
33af6bcc 5199 u32 regval;
c6cb090b
PO		 5200
5201 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5202 return -EFAULT;
5203
5204 /* reserved for future extensions */
5205 if (config.flags)
5206 return -EINVAL;
5207
33af6bcc
PO		 5208 switch (config.tx_type) {
5209 case HWTSTAMP_TX_OFF:
c5b9bd5e 5210 tsync_tx_ctl = 0;
33af6bcc 5211 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5212 break;
5213 default:
5214 return -ERANGE;
5215 }
5216
5217 switch (config.rx_filter) {
5218 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5219 tsync_rx_ctl = 0;
33af6bcc
PO		 5220 break;
5221 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5222 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5223 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5224 case HWTSTAMP_FILTER_ALL:
5225 /*
5226 * register TSYNCRXCFG must be set, therefore it is not
5227 * possible to time stamp both Sync and Delay_Req messages
5228 * => fall back to time stamping all packets
5229 */
c5b9bd5e 5230 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5231 config.rx_filter = HWTSTAMP_FILTER_ALL;
5232 break;
5233 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5234 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5235 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5236 is_l4 = true;
33af6bcc
PO		 5237 break;
5238 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5239 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5240 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5241 is_l4 = true;
33af6bcc
PO		 5242 break;
5243 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5244 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5245 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5246 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5247 is_l2 = true;
5248 is_l4 = true;
33af6bcc
PO		 5249 config.rx_filter = HWTSTAMP_FILTER_SOME;
5250 break;
5251 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5252 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5253 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5254 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5255 is_l2 = true;
5256 is_l4 = true;
33af6bcc
PO		 5257 config.rx_filter = HWTSTAMP_FILTER_SOME;
5258 break;
5259 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5260 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5261 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5262 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5263 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5264 is_l2 = true;
33af6bcc
PO		 5265 break;
5266 default:
5267 return -ERANGE;
5268 }
5269
c5b9bd5e
AD		 5270 if (hw->mac.type == e1000_82575) {
5271 if (tsync_rx_ctl | tsync_tx_ctl)
5272 return -EINVAL;
5273 return 0;
5274 }
5275
33af6bcc
PO		 5276 /* enable/disable TX */
5277 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5278 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5279 regval |= tsync_tx_ctl;
33af6bcc
PO		 5280 wr32(E1000_TSYNCTXCTL, regval);
5281
c5b9bd5e 5282 /* enable/disable RX */
33af6bcc 5283 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5284 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5285 regval |= tsync_rx_ctl;
33af6bcc 5286 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5287
c5b9bd5e
AD		 5288 /* define which PTP packets are time stamped */
5289 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5290
c5b9bd5e
AD		 5291 /* define ethertype filter for timestamped packets */
5292 if (is_l2)
5293 wr32(E1000_ETQF(3),
5294 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5295 E1000_ETQF_1588 | /* enable timestamping */
5296 ETH_P_1588)); /* 1588 eth protocol type */
5297 else
5298 wr32(E1000_ETQF(3), 0);
5299
5300#define PTP_PORT 319
5301 /* L4 Queue Filter[3]: filter by destination port and protocol */
5302 if (is_l4) {
5303 u32 ftqf = (IPPROTO_UDP /* UDP */
5304 | E1000_FTQF_VF_BP /* VF not compared */
5305 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5306 | E1000_FTQF_MASK); /* mask all inputs */
5307 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5308
5309 wr32(E1000_IMIR(3), htons(PTP_PORT));
5310 wr32(E1000_IMIREXT(3),
5311 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5312 if (hw->mac.type == e1000_82576) {
5313 /* enable source port check */
5314 wr32(E1000_SPQF(3), htons(PTP_PORT));
5315 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5316 }
5317 wr32(E1000_FTQF(3), ftqf);
5318 } else {
5319 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5320 }
33af6bcc
PO		 5321 wrfl();
5322
5323 adapter->hwtstamp_config = config;
5324
5325 /* clear TX/RX time stamp registers, just to be sure */
5326 regval = rd32(E1000_TXSTMPH);
5327 regval = rd32(E1000_RXSTMPH);
c6cb090b 5328
33af6bcc
PO		 5329 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5330 -EFAULT : 0;
c6cb090b
PO		 5331}
5332
9d5c8243
AK		 5333/**
5334 * igb_ioctl -
5335 * @netdev:
5336 * @ifreq:
5337 * @cmd:
5338 **/
5339static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5340{
5341 switch (cmd) {
5342 case SIOCGMIIPHY:
5343 case SIOCGMIIREG:
5344 case SIOCSMIIREG:
5345 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5346 case SIOCSHWTSTAMP:
5347 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5348 default:
5349 return -EOPNOTSUPP;
5350 }
5351}
5352
009bc06e
AD		 5353s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5354{
5355 struct igb_adapter *adapter = hw->back;
5356 u16 cap_offset;
5357
5358 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5359 if (!cap_offset)
5360 return -E1000_ERR_CONFIG;
5361
5362 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5363
5364 return 0;
5365}
5366
5367s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5368{
5369 struct igb_adapter *adapter = hw->back;
5370 u16 cap_offset;
5371
5372 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5373 if (!cap_offset)
5374 return -E1000_ERR_CONFIG;
5375
5376 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5377
5378 return 0;
5379}
5380
9d5c8243
AK		 5381static void igb_vlan_rx_register(struct net_device *netdev,
5382 struct vlan_group *grp)
5383{
5384 struct igb_adapter *adapter = netdev_priv(netdev);
5385 struct e1000_hw *hw = &adapter->hw;
5386 u32 ctrl, rctl;
5387
5388 igb_irq_disable(adapter);
5389 adapter->vlgrp = grp;
5390
5391 if (grp) {
5392 /* enable VLAN tag insert/strip */
5393 ctrl = rd32(E1000_CTRL);
5394 ctrl |= E1000_CTRL_VME;
5395 wr32(E1000_CTRL, ctrl);
5396
51466239 5397 /* Disable CFI check */
9d5c8243 5398 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5399 rctl &= ~E1000_RCTL_CFIEN;
5400 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5401 } else {
5402 /* disable VLAN tag insert/strip */
5403 ctrl = rd32(E1000_CTRL);
5404 ctrl &= ~E1000_CTRL_VME;
5405 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5406 }
5407
e1739522
AD		 5408 igb_rlpml_set(adapter);
5409
9d5c8243
AK		 5410 if (!test_bit(__IGB_DOWN, &adapter->state))
5411 igb_irq_enable(adapter);
5412}
5413
5414static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5415{
5416 struct igb_adapter *adapter = netdev_priv(netdev);
5417 struct e1000_hw *hw = &adapter->hw;
4ae196df 5418 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5419
51466239
AD		 5420 /* attempt to add filter to vlvf array */
5421 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5422
51466239
AD		 5423 /* add the filter since PF can receive vlans w/o entry in vlvf */
5424 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5425}
5426
5427static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5428{
5429 struct igb_adapter *adapter = netdev_priv(netdev);
5430 struct e1000_hw *hw = &adapter->hw;
4ae196df 5431 int pf_id = adapter->vfs_allocated_count;
51466239 5432 s32 err;
9d5c8243
AK		 5433
5434 igb_irq_disable(adapter);
5435 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5436
5437 if (!test_bit(__IGB_DOWN, &adapter->state))
5438 igb_irq_enable(adapter);
5439
51466239
AD		 5440 /* remove vlan from VLVF table array */
5441 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5442
51466239
AD		 5443 /* if vid was not present in VLVF just remove it from table */
5444 if (err)
4ae196df 5445 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5446}
5447
5448static void igb_restore_vlan(struct igb_adapter *adapter)
5449{
5450 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5451
5452 if (adapter->vlgrp) {
5453 u16 vid;
5454 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5455 if (!vlan_group_get_device(adapter->vlgrp, vid))
5456 continue;
5457 igb_vlan_rx_add_vid(adapter->netdev, vid);
5458 }
5459 }
5460}
5461
5462int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5463{
5464 struct e1000_mac_info *mac = &adapter->hw.mac;
5465
5466 mac->autoneg = 0;
5467
9d5c8243
AK		 5468 switch (spddplx) {
5469 case SPEED_10 + DUPLEX_HALF:
5470 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5471 break;
5472 case SPEED_10 + DUPLEX_FULL:
5473 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5474 break;
5475 case SPEED_100 + DUPLEX_HALF:
5476 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5477 break;
5478 case SPEED_100 + DUPLEX_FULL:
5479 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5480 break;
5481 case SPEED_1000 + DUPLEX_FULL:
5482 mac->autoneg = 1;
5483 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5484 break;
5485 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5486 default:
5487 dev_err(&adapter->pdev->dev,
5488 "Unsupported Speed/Duplex configuration\n");
5489 return -EINVAL;
5490 }
5491 return 0;
5492}
5493
3fe7c4c9 5494static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5495{
5496 struct net_device *netdev = pci_get_drvdata(pdev);
5497 struct igb_adapter *adapter = netdev_priv(netdev);
5498 struct e1000_hw *hw = &adapter->hw;
2d064c06 5499 u32 ctrl, rctl, status;
9d5c8243
AK		 5500 u32 wufc = adapter->wol;
5501#ifdef CONFIG_PM
5502 int retval = 0;
5503#endif
5504
5505 netif_device_detach(netdev);
5506
a88f10ec
AD		 5507 if (netif_running(netdev))
5508 igb_close(netdev);
5509
047e0030 5510 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5511
5512#ifdef CONFIG_PM
5513 retval = pci_save_state(pdev);
5514 if (retval)
5515 return retval;
5516#endif
5517
5518 status = rd32(E1000_STATUS);
5519 if (status & E1000_STATUS_LU)
5520 wufc &= ~E1000_WUFC_LNKC;
5521
5522 if (wufc) {
5523 igb_setup_rctl(adapter);
ff41f8dc 5524 igb_set_rx_mode(netdev);
9d5c8243
AK		 5525
5526 /* turn on all-multi mode if wake on multicast is enabled */
5527 if (wufc & E1000_WUFC_MC) {
5528 rctl = rd32(E1000_RCTL);
5529 rctl |= E1000_RCTL_MPE;
5530 wr32(E1000_RCTL, rctl);
5531 }
5532
5533 ctrl = rd32(E1000_CTRL);
5534 /* advertise wake from D3Cold */
5535 #define E1000_CTRL_ADVD3WUC 0x00100000
5536 /* phy power management enable */
5537 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5538 ctrl |= E1000_CTRL_ADVD3WUC;
5539 wr32(E1000_CTRL, ctrl);
5540
9d5c8243
AK		 5541 /* Allow time for pending master requests to run */
5542 igb_disable_pcie_master(&adapter->hw);
5543
5544 wr32(E1000_WUC, E1000_WUC_PME_EN);
5545 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5546 } else {
5547 wr32(E1000_WUC, 0);
5548 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5549 }
5550
3fe7c4c9
RW		 5551 *enable_wake = wufc || adapter->en_mng_pt;
5552 if (!*enable_wake)
2fb02a26 5553 igb_shutdown_serdes_link_82575(hw);
9d5c8243
AK		 5554
5555 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5556 * would have already happened in close and is redundant. */
5557 igb_release_hw_control(adapter);
5558
5559 pci_disable_device(pdev);
5560
9d5c8243
AK		 5561 return 0;
5562}
5563
5564#ifdef CONFIG_PM
3fe7c4c9
RW		 5565static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5566{
5567 int retval;
5568 bool wake;
5569
5570 retval = __igb_shutdown(pdev, &wake);
5571 if (retval)
5572 return retval;
5573
5574 if (wake) {
5575 pci_prepare_to_sleep(pdev);
5576 } else {
5577 pci_wake_from_d3(pdev, false);
5578 pci_set_power_state(pdev, PCI_D3hot);
5579 }
5580
5581 return 0;
5582}
5583
9d5c8243
AK		 5584static int igb_resume(struct pci_dev *pdev)
5585{
5586 struct net_device *netdev = pci_get_drvdata(pdev);
5587 struct igb_adapter *adapter = netdev_priv(netdev);
5588 struct e1000_hw *hw = &adapter->hw;
5589 u32 err;
5590
5591 pci_set_power_state(pdev, PCI_D0);
5592 pci_restore_state(pdev);
42bfd33a 5593
aed5dec3 5594 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5595 if (err) {
5596 dev_err(&pdev->dev,
5597 "igb: Cannot enable PCI device from suspend\n");
5598 return err;
5599 }
5600 pci_set_master(pdev);
5601
5602 pci_enable_wake(pdev, PCI_D3hot, 0);
5603 pci_enable_wake(pdev, PCI_D3cold, 0);
5604
047e0030 5605 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5606 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5607 return -ENOMEM;
9d5c8243
AK		 5608 }
5609
5610 /* e1000_power_up_phy(adapter); */
5611
5612 igb_reset(adapter);
a8564f03
AD		 5613
5614 /* let the f/w know that the h/w is now under the control of the
5615 * driver. */
5616 igb_get_hw_control(adapter);
5617
9d5c8243
AK		 5618 wr32(E1000_WUS, ~0);
5619
a88f10ec
AD		 5620 if (netif_running(netdev)) {
5621 err = igb_open(netdev);
5622 if (err)
5623 return err;
5624 }
9d5c8243
AK		 5625
5626 netif_device_attach(netdev);
5627
9d5c8243
AK		 5628 return 0;
5629}
5630#endif
5631
5632static void igb_shutdown(struct pci_dev *pdev)
5633{
3fe7c4c9
RW		 5634 bool wake;
5635
5636 __igb_shutdown(pdev, &wake);
5637
5638 if (system_state == SYSTEM_POWER_OFF) {
5639 pci_wake_from_d3(pdev, wake);
5640 pci_set_power_state(pdev, PCI_D3hot);
5641 }
9d5c8243
AK		 5642}
5643
5644#ifdef CONFIG_NET_POLL_CONTROLLER
5645/*
5646 * Polling 'interrupt' - used by things like netconsole to send skbs
5647 * without having to re-enable interrupts. It's not called while
5648 * the interrupt routine is executing.
5649 */
5650static void igb_netpoll(struct net_device *netdev)
5651{
5652 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5653 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5654 int i;
9d5c8243 5655
eebbbdba 5656 if (!adapter->msix_entries) {
047e0030 5657 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5658 igb_irq_disable(adapter);
047e0030 5659 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5660 return;
5661 }
9d5c8243 5662
047e0030
AD		 5663 for (i = 0; i < adapter->num_q_vectors; i++) {
5664 struct igb_q_vector *q_vector = adapter->q_vector[i];
5665 wr32(E1000_EIMC, q_vector->eims_value);
5666 napi_schedule(&q_vector->napi);
eebbbdba 5667 }
9d5c8243
AK		 5668}
5669#endif /* CONFIG_NET_POLL_CONTROLLER */
5670
5671/**
5672 * igb_io_error_detected - called when PCI error is detected
5673 * @pdev: Pointer to PCI device
5674 * @state: The current pci connection state
5675 *
5676 * This function is called after a PCI bus error affecting
5677 * this device has been detected.
5678 */
5679static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5680 pci_channel_state_t state)
5681{
5682 struct net_device *netdev = pci_get_drvdata(pdev);
5683 struct igb_adapter *adapter = netdev_priv(netdev);
5684
5685 netif_device_detach(netdev);
5686
59ed6eec
AD		 5687 if (state == pci_channel_io_perm_failure)
5688 return PCI_ERS_RESULT_DISCONNECT;
5689
9d5c8243
AK		 5690 if (netif_running(netdev))
5691 igb_down(adapter);
5692 pci_disable_device(pdev);
5693
5694 /* Request a slot slot reset. */
5695 return PCI_ERS_RESULT_NEED_RESET;
5696}
5697
5698/**
5699 * igb_io_slot_reset - called after the pci bus has been reset.
5700 * @pdev: Pointer to PCI device
5701 *
5702 * Restart the card from scratch, as if from a cold-boot. Implementation
5703 * resembles the first-half of the igb_resume routine.
5704 */
5705static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5706{
5707 struct net_device *netdev = pci_get_drvdata(pdev);
5708 struct igb_adapter *adapter = netdev_priv(netdev);
5709 struct e1000_hw *hw = &adapter->hw;
40a914fa 5710 pci_ers_result_t result;
42bfd33a 5711 int err;
9d5c8243 5712
aed5dec3 5713 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 5714 dev_err(&pdev->dev,
5715 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 5716 result = PCI_ERS_RESULT_DISCONNECT;
5717 } else {
5718 pci_set_master(pdev);
5719 pci_restore_state(pdev);
9d5c8243 5720
40a914fa
AD		 5721 pci_enable_wake(pdev, PCI_D3hot, 0);
5722 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 5723
40a914fa
AD		 5724 igb_reset(adapter);
5725 wr32(E1000_WUS, ~0);
5726 result = PCI_ERS_RESULT_RECOVERED;
5727 }
9d5c8243 5728
ea943d41
JK		 5729 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5730 if (err) {
5731 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5732 "failed 0x%0x\n", err);
5733 /* non-fatal, continue */
5734 }
40a914fa
AD		 5735
5736 return result;
9d5c8243
AK		 5737}
5738
5739/**
5740 * igb_io_resume - called when traffic can start flowing again.
5741 * @pdev: Pointer to PCI device
5742 *
5743 * This callback is called when the error recovery driver tells us that
5744 * its OK to resume normal operation. Implementation resembles the
5745 * second-half of the igb_resume routine.
5746 */
5747static void igb_io_resume(struct pci_dev *pdev)
5748{
5749 struct net_device *netdev = pci_get_drvdata(pdev);
5750 struct igb_adapter *adapter = netdev_priv(netdev);
5751
9d5c8243
AK		 5752 if (netif_running(netdev)) {
5753 if (igb_up(adapter)) {
5754 dev_err(&pdev->dev, "igb_up failed after reset\n");
5755 return;
5756 }
5757 }
5758
5759 netif_device_attach(netdev);
5760
5761 /* let the f/w know that the h/w is now under the control of the
5762 * driver. */
5763 igb_get_hw_control(adapter);
9d5c8243
AK		 5764}
5765
26ad9178
AD		 5766static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5767 u8 qsel)
5768{
5769 u32 rar_low, rar_high;
5770 struct e1000_hw *hw = &adapter->hw;
5771
5772 /* HW expects these in little endian so we reverse the byte order
5773 * from network order (big endian) to little endian
5774 */
5775 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5776 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5777 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5778
5779 /* Indicate to hardware the Address is Valid. */
5780 rar_high |= E1000_RAH_AV;
5781
5782 if (hw->mac.type == e1000_82575)
5783 rar_high |= E1000_RAH_POOL_1 * qsel;
5784 else
5785 rar_high |= E1000_RAH_POOL_1 << qsel;
5786
5787 wr32(E1000_RAL(index), rar_low);
5788 wrfl();
5789 wr32(E1000_RAH(index), rar_high);
5790 wrfl();
5791}
5792
4ae196df
AD		 5793static int igb_set_vf_mac(struct igb_adapter *adapter,
5794 int vf, unsigned char *mac_addr)
5795{
5796 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 5797 /* VF MAC addresses start at end of receive addresses and moves
5798 * torwards the first, as a result a collision should not be possible */
5799 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 5800
37680117 5801 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 5802
26ad9178 5803 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 5804
5805 return 0;
5806}
5807
5808static void igb_vmm_control(struct igb_adapter *adapter)
5809{
5810 struct e1000_hw *hw = &adapter->hw;
10d8e907 5811 u32 reg;
4ae196df 5812
d4960307
AD		 5813 /* replication is not supported for 82575 */
5814 if (hw->mac.type == e1000_82575)
4ae196df
AD		 5815 return;
5816
10d8e907
AD		 5817 /* enable replication vlan tag stripping */
5818 reg = rd32(E1000_RPLOLR);
5819 reg |= E1000_RPLOLR_STRVLAN;
5820 wr32(E1000_RPLOLR, reg);
5821
5822 /* notify HW that the MAC is adding vlan tags */
5823 reg = rd32(E1000_DTXCTL);
5824 reg |= E1000_DTXCTL_VLAN_ADDED;
5825 wr32(E1000_DTXCTL, reg);
5826
d4960307
AD		 5827 if (adapter->vfs_allocated_count) {
5828 igb_vmdq_set_loopback_pf(hw, true);
5829 igb_vmdq_set_replication_pf(hw, true);
5830 } else {
5831 igb_vmdq_set_loopback_pf(hw, false);
5832 igb_vmdq_set_replication_pf(hw, false);
5833 }
4ae196df
AD		 5834}
5835
9d5c8243 5836/* igb_main.c */
kernel source for telekom puls (alcatel/mediatek)