projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: replace unecessary &adapter->hw with just hw where applicable
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
86d5d38f 52#define DRV_VERSION "1.3.16-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
63static struct pci_device_id igb_pci_tbl[] = {
2d064c06 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
75 {0, }
76};
77
78MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
79
80void igb_reset(struct igb_adapter *);
81static int igb_setup_all_tx_resources(struct igb_adapter *);
82static int igb_setup_all_rx_resources(struct igb_adapter *);
83static void igb_free_all_tx_resources(struct igb_adapter *);
84static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 85static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 86void igb_update_stats(struct igb_adapter *);
87static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88static void __devexit igb_remove(struct pci_dev *pdev);
89static int igb_sw_init(struct igb_adapter *);
90static int igb_open(struct net_device *);
91static int igb_close(struct net_device *);
92static void igb_configure_tx(struct igb_adapter *);
93static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 94static void igb_clean_all_tx_rings(struct igb_adapter *);
95static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 96static void igb_clean_tx_ring(struct igb_ring *);
97static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 98static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 99static void igb_update_phy_info(unsigned long);
100static void igb_watchdog(unsigned long);
101static void igb_watchdog_task(struct work_struct *);
b1a436c3 102static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 103static struct net_device_stats *igb_get_stats(struct net_device *);
104static int igb_change_mtu(struct net_device *, int);
105static int igb_set_mac(struct net_device *, void *);
68d480c4 106static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 107static irqreturn_t igb_intr(int irq, void *);
108static irqreturn_t igb_intr_msi(int irq, void *);
109static irqreturn_t igb_msix_other(int irq, void *);
047e0030 110static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 111#ifdef CONFIG_IGB_DCA
047e0030 112static void igb_update_dca(struct igb_q_vector *);
fe4506b6 113static void igb_setup_dca(struct igb_adapter *);
421e02f0 114#endif /* CONFIG_IGB_DCA */
047e0030 115static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 116static int igb_poll(struct napi_struct *, int);
047e0030 117static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 118static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
119static void igb_tx_timeout(struct net_device *);
120static void igb_reset_task(struct work_struct *);
121static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
122static void igb_vlan_rx_add_vid(struct net_device *, u16);
123static void igb_vlan_rx_kill_vid(struct net_device *, u16);
124static void igb_restore_vlan(struct igb_adapter *);
26ad9178 125static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 126static void igb_ping_all_vfs(struct igb_adapter *);
127static void igb_msg_task(struct igb_adapter *);
4ae196df 128static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 129static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 130static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
9d5c8243 131
9d5c8243 132#ifdef CONFIG_PM
3fe7c4c9 133static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 134static int igb_resume(struct pci_dev *);
135#endif
136static void igb_shutdown(struct pci_dev *);
421e02f0 137#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 138static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
139static struct notifier_block dca_notifier = {
140 .notifier_call = igb_notify_dca,
141 .next = NULL,
142 .priority = 0
143};
144#endif
9d5c8243
AK		 145#ifdef CONFIG_NET_POLL_CONTROLLER
146/* for netdump / net console */
147static void igb_netpoll(struct net_device *);
148#endif
37680117 149#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 150static unsigned int max_vfs = 0;
151module_param(max_vfs, uint, 0);
152MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
153 "per physical function");
154#endif /* CONFIG_PCI_IOV */
155
9d5c8243
AK		 156static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
157 pci_channel_state_t);
158static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
159static void igb_io_resume(struct pci_dev *);
160
161static struct pci_error_handlers igb_err_handler = {
162 .error_detected = igb_io_error_detected,
163 .slot_reset = igb_io_slot_reset,
164 .resume = igb_io_resume,
165};
166
167
168static struct pci_driver igb_driver = {
169 .name = igb_driver_name,
170 .id_table = igb_pci_tbl,
171 .probe = igb_probe,
172 .remove = __devexit_p(igb_remove),
173#ifdef CONFIG_PM
174 /* Power Managment Hooks */
175 .suspend = igb_suspend,
176 .resume = igb_resume,
177#endif
178 .shutdown = igb_shutdown,
179 .err_handler = &igb_err_handler
180};
181
182MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
183MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
184MODULE_LICENSE("GPL");
185MODULE_VERSION(DRV_VERSION);
186
38c845c7
PO		 187/**
188 * igb_read_clock - read raw cycle counter (to be used by time counter)
189 */
190static cycle_t igb_read_clock(const struct cyclecounter *tc)
191{
192 struct igb_adapter *adapter =
193 container_of(tc, struct igb_adapter, cycles);
194 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 195 u64 stamp = 0;
196 int shift = 0;
38c845c7 197
c5b9bd5e
AD		 198 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
199 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 200 return stamp;
201}
202
9d5c8243
AK		 203#ifdef DEBUG
204/**
205 * igb_get_hw_dev_name - return device name string
206 * used by hardware layer to print debugging information
207 **/
208char *igb_get_hw_dev_name(struct e1000_hw *hw)
209{
210 struct igb_adapter *adapter = hw->back;
211 return adapter->netdev->name;
212}
38c845c7
PO		 213
214/**
215 * igb_get_time_str - format current NIC and system time as string
216 */
217static char *igb_get_time_str(struct igb_adapter *adapter,
218 char buffer[160])
219{
220 cycle_t hw = adapter->cycles.read(&adapter->cycles);
221 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
222 struct timespec sys;
223 struct timespec delta;
224 getnstimeofday(&sys);
225
226 delta = timespec_sub(nic, sys);
227
228 sprintf(buffer,
33af6bcc
PO		 229 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
230 hw,
38c845c7
PO		 231 (long)nic.tv_sec, nic.tv_nsec,
232 (long)sys.tv_sec, sys.tv_nsec,
233 (long)delta.tv_sec, delta.tv_nsec);
234
235 return buffer;
236}
9d5c8243
AK		 237#endif
238
239/**
240 * igb_init_module - Driver Registration Routine
241 *
242 * igb_init_module is the first routine called when the driver is
243 * loaded. All it does is register with the PCI subsystem.
244 **/
245static int __init igb_init_module(void)
246{
247 int ret;
248 printk(KERN_INFO "%s - version %s\n",
249 igb_driver_string, igb_driver_version);
250
251 printk(KERN_INFO "%s\n", igb_copyright);
252
421e02f0 253#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 254 dca_register_notify(&dca_notifier);
255#endif
bbd98fe4 256 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 257 return ret;
258}
259
260module_init(igb_init_module);
261
262/**
263 * igb_exit_module - Driver Exit Cleanup Routine
264 *
265 * igb_exit_module is called just before the driver is removed
266 * from memory.
267 **/
268static void __exit igb_exit_module(void)
269{
421e02f0 270#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 271 dca_unregister_notify(&dca_notifier);
272#endif
9d5c8243
AK		 273 pci_unregister_driver(&igb_driver);
274}
275
276module_exit(igb_exit_module);
277
26bc19ec
AD		 278#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
279/**
280 * igb_cache_ring_register - Descriptor ring to register mapping
281 * @adapter: board private structure to initialize
282 *
283 * Once we know the feature-set enabled for the device, we'll cache
284 * the register offset the descriptor ring is assigned to.
285 **/
286static void igb_cache_ring_register(struct igb_adapter *adapter)
287{
ee1b9f06 288 int i = 0, j = 0;
047e0030 289 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 290
291 switch (adapter->hw.mac.type) {
292 case e1000_82576:
293 /* The queues are allocated for virtualization such that VF 0
294 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
295 * In order to avoid collision we start at the first free queue
296 * and continue consuming queues in the same sequence
297 */
ee1b9f06
AD		 298 if (adapter->vfs_allocated_count) {
299 for (; i < adapter->num_rx_queues; i++)
300 adapter->rx_ring[i].reg_idx = rbase_offset +
301 Q_IDX_82576(i);
302 for (; j < adapter->num_tx_queues; j++)
303 adapter->tx_ring[j].reg_idx = rbase_offset +
304 Q_IDX_82576(j);
305 }
26bc19ec
AD		 306 case e1000_82575:
307 default:
ee1b9f06
AD		 308 for (; i < adapter->num_rx_queues; i++)
309 adapter->rx_ring[i].reg_idx = rbase_offset + i;
310 for (; j < adapter->num_tx_queues; j++)
311 adapter->tx_ring[j].reg_idx = rbase_offset + j;
26bc19ec
AD		 312 break;
313 }
314}
315
047e0030
AD		 316static void igb_free_queues(struct igb_adapter *adapter)
317{
318 kfree(adapter->tx_ring);
319 kfree(adapter->rx_ring);
320
321 adapter->tx_ring = NULL;
322 adapter->rx_ring = NULL;
323
324 adapter->num_rx_queues = 0;
325 adapter->num_tx_queues = 0;
326}
327
9d5c8243
AK		 328/**
329 * igb_alloc_queues - Allocate memory for all rings
330 * @adapter: board private structure to initialize
331 *
332 * We allocate one ring per queue at run-time since we don't know the
333 * number of queues at compile-time.
334 **/
335static int igb_alloc_queues(struct igb_adapter *adapter)
336{
337 int i;
338
339 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
340 sizeof(struct igb_ring), GFP_KERNEL);
341 if (!adapter->tx_ring)
047e0030 342 goto err;
9d5c8243
AK		 343
344 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
345 sizeof(struct igb_ring), GFP_KERNEL);
047e0030
AD		 346 if (!adapter->rx_ring)
347 goto err;
6eb5a7f1 348
661086df
PWJ		 349 for (i = 0; i < adapter->num_tx_queues; i++) {
350 struct igb_ring *ring = &(adapter->tx_ring[i]);
68fd9910 351 ring->count = adapter->tx_ring_count;
661086df 352 ring->queue_index = i;
80785298 353 ring->pdev = adapter->pdev;
e694e964 354 ring->netdev = adapter->netdev;
85ad76b2
AD		 355 /* For 82575, context index must be unique per ring. */
356 if (adapter->hw.mac.type == e1000_82575)
357 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
661086df 358 }
85ad76b2 359
9d5c8243
AK		 360 for (i = 0; i < adapter->num_rx_queues; i++) {
361 struct igb_ring *ring = &(adapter->rx_ring[i]);
68fd9910 362 ring->count = adapter->rx_ring_count;
844290e5 363 ring->queue_index = i;
80785298 364 ring->pdev = adapter->pdev;
e694e964 365 ring->netdev = adapter->netdev;
4c844851 366 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 367 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
368 /* set flag indicating ring supports SCTP checksum offload */
369 if (adapter->hw.mac.type >= e1000_82576)
370 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
9d5c8243 371 }
26bc19ec
AD		 372
373 igb_cache_ring_register(adapter);
9d5c8243 374
047e0030 375 return 0;
a88f10ec 376
047e0030
AD		 377err:
378 igb_free_queues(adapter);
d1a8c9e1 379
047e0030 380 return -ENOMEM;
a88f10ec
AD		 381}
382
9d5c8243 383#define IGB_N0_QUEUE -1
047e0030 384static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 385{
386 u32 msixbm = 0;
047e0030 387 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 388 struct e1000_hw *hw = &adapter->hw;
2d064c06 389 u32 ivar, index;
047e0030
AD		 390 int rx_queue = IGB_N0_QUEUE;
391 int tx_queue = IGB_N0_QUEUE;
392
393 if (q_vector->rx_ring)
394 rx_queue = q_vector->rx_ring->reg_idx;
395 if (q_vector->tx_ring)
396 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 397
398 switch (hw->mac.type) {
399 case e1000_82575:
9d5c8243
AK		 400 /* The 82575 assigns vectors using a bitmask, which matches the
401 bitmask for the EICR/EIMS/EIMC registers. To assign one
402 or more queues to a vector, we write the appropriate bits
403 into the MSIXBM register for that vector. */
047e0030 404 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 405 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 406 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 407 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
9d5c8243 408 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 409 q_vector->eims_value = msixbm;
2d064c06
AD		 410 break;
411 case e1000_82576:
26bc19ec 412 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 413 Each queue has a single entry in the table to which we write
414 a vector number along with a "valid" bit. Sadly, the layout
415 of the table is somewhat counterintuitive. */
416 if (rx_queue > IGB_N0_QUEUE) {
047e0030 417 index = (rx_queue & 0x7);
2d064c06 418 ivar = array_rd32(E1000_IVAR0, index);
047e0030 419 if (rx_queue < 8) {
26bc19ec
AD		 420 /* vector goes into low byte of register */
421 ivar = ivar & 0xFFFFFF00;
422 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 423 } else {
424 /* vector goes into third byte of register */
425 ivar = ivar & 0xFF00FFFF;
426 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 427 }
2d064c06
AD		 428 array_wr32(E1000_IVAR0, index, ivar);
429 }
430 if (tx_queue > IGB_N0_QUEUE) {
047e0030 431 index = (tx_queue & 0x7);
2d064c06 432 ivar = array_rd32(E1000_IVAR0, index);
047e0030 433 if (tx_queue < 8) {
26bc19ec
AD		 434 /* vector goes into second byte of register */
435 ivar = ivar & 0xFFFF00FF;
436 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 437 } else {
438 /* vector goes into high byte of register */
439 ivar = ivar & 0x00FFFFFF;
440 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 441 }
2d064c06
AD		 442 array_wr32(E1000_IVAR0, index, ivar);
443 }
047e0030 444 q_vector->eims_value = 1 << msix_vector;
2d064c06
AD		 445 break;
446 default:
447 BUG();
448 break;
449 }
9d5c8243
AK		 450}
451
452/**
453 * igb_configure_msix - Configure MSI-X hardware
454 *
455 * igb_configure_msix sets up the hardware to properly
456 * generate MSI-X interrupts.
457 **/
458static void igb_configure_msix(struct igb_adapter *adapter)
459{
460 u32 tmp;
461 int i, vector = 0;
462 struct e1000_hw *hw = &adapter->hw;
463
464 adapter->eims_enable_mask = 0;
9d5c8243
AK		 465
466 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 467 switch (hw->mac.type) {
468 case e1000_82575:
9d5c8243
AK		 469 tmp = rd32(E1000_CTRL_EXT);
470 /* enable MSI-X PBA support*/
471 tmp |= E1000_CTRL_EXT_PBA_CLR;
472
473 /* Auto-Mask interrupts upon ICR read. */
474 tmp |= E1000_CTRL_EXT_EIAME;
475 tmp |= E1000_CTRL_EXT_IRCA;
476
477 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 478
479 /* enable msix_other interrupt */
480 array_wr32(E1000_MSIXBM(0), vector++,
481 E1000_EIMS_OTHER);
844290e5 482 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 483
2d064c06
AD		 484 break;
485
486 case e1000_82576:
047e0030
AD		 487 /* Turn on MSI-X capability first, or our settings
488 * won't stick. And it will take days to debug. */
489 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
490 E1000_GPIE_PBA | E1000_GPIE_EIAME |
491 E1000_GPIE_NSICR);
492
493 /* enable msix_other interrupt */
494 adapter->eims_other = 1 << vector;
2d064c06 495 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 496
047e0030 497 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 498 break;
499 default:
500 /* do nothing, since nothing else supports MSI-X */
501 break;
502 } /* switch (hw->mac.type) */
047e0030
AD		 503
504 adapter->eims_enable_mask |= adapter->eims_other;
505
506 for (i = 0; i < adapter->num_q_vectors; i++) {
507 struct igb_q_vector *q_vector = adapter->q_vector[i];
508 igb_assign_vector(q_vector, vector++);
509 adapter->eims_enable_mask |= q_vector->eims_value;
510 }
511
9d5c8243
AK		 512 wrfl();
513}
514
515/**
516 * igb_request_msix - Initialize MSI-X interrupts
517 *
518 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
519 * kernel.
520 **/
521static int igb_request_msix(struct igb_adapter *adapter)
522{
523 struct net_device *netdev = adapter->netdev;
047e0030 524 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 525 int i, err = 0, vector = 0;
526
047e0030
AD		 527 err = request_irq(adapter->msix_entries[vector].vector,
528 &igb_msix_other, 0, netdev->name, adapter);
529 if (err)
530 goto out;
531 vector++;
532
533 for (i = 0; i < adapter->num_q_vectors; i++) {
534 struct igb_q_vector *q_vector = adapter->q_vector[i];
535
536 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
537
538 if (q_vector->rx_ring && q_vector->tx_ring)
539 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
540 q_vector->rx_ring->queue_index);
541 else if (q_vector->tx_ring)
542 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
543 q_vector->tx_ring->queue_index);
544 else if (q_vector->rx_ring)
545 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
546 q_vector->rx_ring->queue_index);
9d5c8243 547 else
047e0030
AD		 548 sprintf(q_vector->name, "%s-unused", netdev->name);
549
9d5c8243 550 err = request_irq(adapter->msix_entries[vector].vector,
047e0030
AD		 551 &igb_msix_ring, 0, q_vector->name,
552 q_vector);
9d5c8243
AK		 553 if (err)
554 goto out;
9d5c8243
AK		 555 vector++;
556 }
557
9d5c8243
AK		 558 igb_configure_msix(adapter);
559 return 0;
560out:
561 return err;
562}
563
564static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
565{
566 if (adapter->msix_entries) {
567 pci_disable_msix(adapter->pdev);
568 kfree(adapter->msix_entries);
569 adapter->msix_entries = NULL;
047e0030 570 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 571 pci_disable_msi(adapter->pdev);
047e0030 572 }
9d5c8243
AK		 573}
574
047e0030
AD		 575/**
576 * igb_free_q_vectors - Free memory allocated for interrupt vectors
577 * @adapter: board private structure to initialize
578 *
579 * This function frees the memory allocated to the q_vectors. In addition if
580 * NAPI is enabled it will delete any references to the NAPI struct prior
581 * to freeing the q_vector.
582 **/
583static void igb_free_q_vectors(struct igb_adapter *adapter)
584{
585 int v_idx;
586
587 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
588 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
589 adapter->q_vector[v_idx] = NULL;
590 netif_napi_del(&q_vector->napi);
591 kfree(q_vector);
592 }
593 adapter->num_q_vectors = 0;
594}
595
596/**
597 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
598 *
599 * This function resets the device so that it has 0 rx queues, tx queues, and
600 * MSI-X interrupts allocated.
601 */
602static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
603{
604 igb_free_queues(adapter);
605 igb_free_q_vectors(adapter);
606 igb_reset_interrupt_capability(adapter);
607}
9d5c8243
AK		 608
609/**
610 * igb_set_interrupt_capability - set MSI or MSI-X if supported
611 *
612 * Attempt to configure interrupts using the best available
613 * capabilities of the hardware and kernel.
614 **/
615static void igb_set_interrupt_capability(struct igb_adapter *adapter)
616{
617 int err;
618 int numvecs, i;
619
83b7180d 620 /* Number of supported queues. */
83b7180d
AD		 621 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
622 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
623
047e0030
AD		 624 /* start with one vector for every rx queue */
625 numvecs = adapter->num_rx_queues;
626
627 /* if tx handler is seperate add 1 for every tx queue */
628 numvecs += adapter->num_tx_queues;
629
630 /* store the number of vectors reserved for queues */
631 adapter->num_q_vectors = numvecs;
632
633 /* add 1 vector for link status interrupts */
634 numvecs++;
9d5c8243
AK		 635 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
636 GFP_KERNEL);
637 if (!adapter->msix_entries)
638 goto msi_only;
639
640 for (i = 0; i < numvecs; i++)
641 adapter->msix_entries[i].entry = i;
642
643 err = pci_enable_msix(adapter->pdev,
644 adapter->msix_entries,
645 numvecs);
646 if (err == 0)
34a20e89 647 goto out;
9d5c8243
AK		 648
649 igb_reset_interrupt_capability(adapter);
650
651 /* If we can't do MSI-X, try MSI */
652msi_only:
2a3abf6d
AD		 653#ifdef CONFIG_PCI_IOV
654 /* disable SR-IOV for non MSI-X configurations */
655 if (adapter->vf_data) {
656 struct e1000_hw *hw = &adapter->hw;
657 /* disable iov and allow time for transactions to clear */
658 pci_disable_sriov(adapter->pdev);
659 msleep(500);
660
661 kfree(adapter->vf_data);
662 adapter->vf_data = NULL;
663 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
664 msleep(100);
665 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
666 }
667#endif
4fc82adf
AD		 668 adapter->vfs_allocated_count = 0;
669 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 670 adapter->num_rx_queues = 1;
661086df 671 adapter->num_tx_queues = 1;
047e0030 672 adapter->num_q_vectors = 1;
9d5c8243 673 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 674 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 675out:
661086df 676 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 677 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 678 return;
679}
680
047e0030
AD		 681/**
682 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
683 * @adapter: board private structure to initialize
684 *
685 * We allocate one q_vector per queue interrupt. If allocation fails we
686 * return -ENOMEM.
687 **/
688static int igb_alloc_q_vectors(struct igb_adapter *adapter)
689{
690 struct igb_q_vector *q_vector;
691 struct e1000_hw *hw = &adapter->hw;
692 int v_idx;
693
694 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
695 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
696 if (!q_vector)
697 goto err_out;
698 q_vector->adapter = adapter;
699 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
700 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
701 q_vector->itr_val = IGB_START_ITR;
702 q_vector->set_itr = 1;
703 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
704 adapter->q_vector[v_idx] = q_vector;
705 }
706 return 0;
707
708err_out:
709 while (v_idx) {
710 v_idx--;
711 q_vector = adapter->q_vector[v_idx];
712 netif_napi_del(&q_vector->napi);
713 kfree(q_vector);
714 adapter->q_vector[v_idx] = NULL;
715 }
716 return -ENOMEM;
717}
718
719static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
720 int ring_idx, int v_idx)
721{
722 struct igb_q_vector *q_vector;
723
724 q_vector = adapter->q_vector[v_idx];
725 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
726 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 727 q_vector->itr_val = adapter->rx_itr_setting;
728 if (q_vector->itr_val && q_vector->itr_val <= 3)
729 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 730}
731
732static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
733 int ring_idx, int v_idx)
734{
735 struct igb_q_vector *q_vector;
736
737 q_vector = adapter->q_vector[v_idx];
738 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
739 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 740 q_vector->itr_val = adapter->tx_itr_setting;
741 if (q_vector->itr_val && q_vector->itr_val <= 3)
742 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 743}
744
745/**
746 * igb_map_ring_to_vector - maps allocated queues to vectors
747 *
748 * This function maps the recently allocated queues to vectors.
749 **/
750static int igb_map_ring_to_vector(struct igb_adapter *adapter)
751{
752 int i;
753 int v_idx = 0;
754
755 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
756 (adapter->num_q_vectors < adapter->num_tx_queues))
757 return -ENOMEM;
758
759 if (adapter->num_q_vectors >=
760 (adapter->num_rx_queues + adapter->num_tx_queues)) {
761 for (i = 0; i < adapter->num_rx_queues; i++)
762 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
763 for (i = 0; i < adapter->num_tx_queues; i++)
764 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
765 } else {
766 for (i = 0; i < adapter->num_rx_queues; i++) {
767 if (i < adapter->num_tx_queues)
768 igb_map_tx_ring_to_vector(adapter, i, v_idx);
769 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
770 }
771 for (; i < adapter->num_tx_queues; i++)
772 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
773 }
774 return 0;
775}
776
777/**
778 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
779 *
780 * This function initializes the interrupts and allocates all of the queues.
781 **/
782static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
783{
784 struct pci_dev *pdev = adapter->pdev;
785 int err;
786
787 igb_set_interrupt_capability(adapter);
788
789 err = igb_alloc_q_vectors(adapter);
790 if (err) {
791 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
792 goto err_alloc_q_vectors;
793 }
794
795 err = igb_alloc_queues(adapter);
796 if (err) {
797 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
798 goto err_alloc_queues;
799 }
800
801 err = igb_map_ring_to_vector(adapter);
802 if (err) {
803 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
804 goto err_map_queues;
805 }
806
807
808 return 0;
809err_map_queues:
810 igb_free_queues(adapter);
811err_alloc_queues:
812 igb_free_q_vectors(adapter);
813err_alloc_q_vectors:
814 igb_reset_interrupt_capability(adapter);
815 return err;
816}
817
9d5c8243
AK		 818/**
819 * igb_request_irq - initialize interrupts
820 *
821 * Attempts to configure interrupts using the best available
822 * capabilities of the hardware and kernel.
823 **/
824static int igb_request_irq(struct igb_adapter *adapter)
825{
826 struct net_device *netdev = adapter->netdev;
047e0030 827 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 828 struct e1000_hw *hw = &adapter->hw;
829 int err = 0;
830
831 if (adapter->msix_entries) {
832 err = igb_request_msix(adapter);
844290e5 833 if (!err)
9d5c8243 834 goto request_done;
9d5c8243 835 /* fall back to MSI */
047e0030 836 igb_clear_interrupt_scheme(adapter);
9d5c8243 837 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 838 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 839 igb_free_all_tx_resources(adapter);
840 igb_free_all_rx_resources(adapter);
047e0030 841 adapter->num_tx_queues = 1;
9d5c8243 842 adapter->num_rx_queues = 1;
047e0030
AD		 843 adapter->num_q_vectors = 1;
844 err = igb_alloc_q_vectors(adapter);
845 if (err) {
846 dev_err(&pdev->dev,
847 "Unable to allocate memory for vectors\n");
848 goto request_done;
849 }
850 err = igb_alloc_queues(adapter);
851 if (err) {
852 dev_err(&pdev->dev,
853 "Unable to allocate memory for queues\n");
854 igb_free_q_vectors(adapter);
855 goto request_done;
856 }
857 igb_setup_all_tx_resources(adapter);
858 igb_setup_all_rx_resources(adapter);
844290e5 859 } else {
2d064c06
AD		 860 switch (hw->mac.type) {
861 case e1000_82575:
862 wr32(E1000_MSIXBM(0),
047e0030
AD		 863 (E1000_EICR_RX_QUEUE0 |
864 E1000_EICR_TX_QUEUE0 |
865 E1000_EIMS_OTHER));
2d064c06
AD		 866 break;
867 case e1000_82576:
868 wr32(E1000_IVAR0, E1000_IVAR_VALID);
869 break;
870 default:
871 break;
872 }
9d5c8243 873 }
844290e5 874
7dfc16fa 875 if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 876 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
047e0030 877 netdev->name, adapter);
9d5c8243
AK		 878 if (!err)
879 goto request_done;
047e0030 880
9d5c8243
AK		 881 /* fall back to legacy interrupts */
882 igb_reset_interrupt_capability(adapter);
7dfc16fa 883 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 884 }
885
886 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
047e0030 887 netdev->name, adapter);
9d5c8243 888
6cb5e577 889 if (err)
9d5c8243
AK		 890 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
891 err);
9d5c8243
AK		 892
893request_done:
894 return err;
895}
896
897static void igb_free_irq(struct igb_adapter *adapter)
898{
9d5c8243
AK		 899 if (adapter->msix_entries) {
900 int vector = 0, i;
901
047e0030 902 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 903
047e0030
AD		 904 for (i = 0; i < adapter->num_q_vectors; i++) {
905 struct igb_q_vector *q_vector = adapter->q_vector[i];
906 free_irq(adapter->msix_entries[vector++].vector,
907 q_vector);
908 }
909 } else {
910 free_irq(adapter->pdev->irq, adapter);
9d5c8243 911 }
9d5c8243
AK		 912}
913
914/**
915 * igb_irq_disable - Mask off interrupt generation on the NIC
916 * @adapter: board private structure
917 **/
918static void igb_irq_disable(struct igb_adapter *adapter)
919{
920 struct e1000_hw *hw = &adapter->hw;
921
25568a53
AD		 922 /*
923 * we need to be careful when disabling interrupts. The VFs are also
924 * mapped into these registers and so clearing the bits can cause
925 * issues on the VF drivers so we only need to clear what we set
926 */
9d5c8243 927 if (adapter->msix_entries) {
2dfd1212
AD		 928 u32 regval = rd32(E1000_EIAM);
929 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
930 wr32(E1000_EIMC, adapter->eims_enable_mask);
931 regval = rd32(E1000_EIAC);
932 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 933 }
844290e5
PW		 934
935 wr32(E1000_IAM, 0);
9d5c8243
AK		 936 wr32(E1000_IMC, ~0);
937 wrfl();
938 synchronize_irq(adapter->pdev->irq);
939}
940
941/**
942 * igb_irq_enable - Enable default interrupt generation settings
943 * @adapter: board private structure
944 **/
945static void igb_irq_enable(struct igb_adapter *adapter)
946{
947 struct e1000_hw *hw = &adapter->hw;
948
949 if (adapter->msix_entries) {
25568a53 950 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 951 u32 regval = rd32(E1000_EIAC);
952 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
953 regval = rd32(E1000_EIAM);
954 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 955 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 956 if (adapter->vfs_allocated_count) {
4ae196df 957 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 958 ims |= E1000_IMS_VMMB;
959 }
960 wr32(E1000_IMS, ims);
844290e5
PW		 961 } else {
962 wr32(E1000_IMS, IMS_ENABLE_MASK);
963 wr32(E1000_IAM, IMS_ENABLE_MASK);
964 }
9d5c8243
AK		 965}
966
967static void igb_update_mng_vlan(struct igb_adapter *adapter)
968{
51466239 969 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 970 u16 vid = adapter->hw.mng_cookie.vlan_id;
971 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 972
973 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
974 /* add VID to filter table */
975 igb_vfta_set(hw, vid, true);
976 adapter->mng_vlan_id = vid;
977 } else {
978 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
979 }
980
981 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
982 (vid != old_vid) &&
983 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
984 /* remove VID from filter table */
985 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 986 }
987}
988
989/**
990 * igb_release_hw_control - release control of the h/w to f/w
991 * @adapter: address of board private structure
992 *
993 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
994 * For ASF and Pass Through versions of f/w this means that the
995 * driver is no longer loaded.
996 *
997 **/
998static void igb_release_hw_control(struct igb_adapter *adapter)
999{
1000 struct e1000_hw *hw = &adapter->hw;
1001 u32 ctrl_ext;
1002
1003 /* Let firmware take over control of h/w */
1004 ctrl_ext = rd32(E1000_CTRL_EXT);
1005 wr32(E1000_CTRL_EXT,
1006 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1007}
1008
1009
1010/**
1011 * igb_get_hw_control - get control of the h/w from f/w
1012 * @adapter: address of board private structure
1013 *
1014 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1015 * For ASF and Pass Through versions of f/w this means that
1016 * the driver is loaded.
1017 *
1018 **/
1019static void igb_get_hw_control(struct igb_adapter *adapter)
1020{
1021 struct e1000_hw *hw = &adapter->hw;
1022 u32 ctrl_ext;
1023
1024 /* Let firmware know the driver has taken over */
1025 ctrl_ext = rd32(E1000_CTRL_EXT);
1026 wr32(E1000_CTRL_EXT,
1027 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1028}
1029
9d5c8243
AK		 1030/**
1031 * igb_configure - configure the hardware for RX and TX
1032 * @adapter: private board structure
1033 **/
1034static void igb_configure(struct igb_adapter *adapter)
1035{
1036 struct net_device *netdev = adapter->netdev;
1037 int i;
1038
1039 igb_get_hw_control(adapter);
ff41f8dc 1040 igb_set_rx_mode(netdev);
9d5c8243
AK		 1041
1042 igb_restore_vlan(adapter);
9d5c8243 1043
85b430b4 1044 igb_setup_tctl(adapter);
06cf2666 1045 igb_setup_mrqc(adapter);
9d5c8243 1046 igb_setup_rctl(adapter);
85b430b4
AD		 1047
1048 igb_configure_tx(adapter);
9d5c8243 1049 igb_configure_rx(adapter);
662d7205
AD		 1050
1051 igb_rx_fifo_flush_82575(&adapter->hw);
1052
c493ea45 1053 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1054 * at least 1 descriptor unused to make sure
1055 * next_to_use != next_to_clean */
1056 for (i = 0; i < adapter->num_rx_queues; i++) {
1057 struct igb_ring *ring = &adapter->rx_ring[i];
c493ea45 1058 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1059 }
1060
1061
1062 adapter->tx_queue_len = netdev->tx_queue_len;
1063}
1064
1065
1066/**
1067 * igb_up - Open the interface and prepare it to handle traffic
1068 * @adapter: board private structure
1069 **/
1070
1071int igb_up(struct igb_adapter *adapter)
1072{
1073 struct e1000_hw *hw = &adapter->hw;
1074 int i;
1075
1076 /* hardware has been reset, we need to reload some things */
1077 igb_configure(adapter);
1078
1079 clear_bit(__IGB_DOWN, &adapter->state);
1080
047e0030
AD		 1081 for (i = 0; i < adapter->num_q_vectors; i++) {
1082 struct igb_q_vector *q_vector = adapter->q_vector[i];
1083 napi_enable(&q_vector->napi);
1084 }
844290e5 1085 if (adapter->msix_entries)
9d5c8243 1086 igb_configure_msix(adapter);
9d5c8243
AK		 1087
1088 /* Clear any pending interrupts. */
1089 rd32(E1000_ICR);
1090 igb_irq_enable(adapter);
1091
d4960307
AD		 1092 /* notify VFs that reset has been completed */
1093 if (adapter->vfs_allocated_count) {
1094 u32 reg_data = rd32(E1000_CTRL_EXT);
1095 reg_data |= E1000_CTRL_EXT_PFRSTD;
1096 wr32(E1000_CTRL_EXT, reg_data);
1097 }
1098
4cb9be7a
JB		 1099 netif_tx_start_all_queues(adapter->netdev);
1100
25568a53
AD		 1101 /* start the watchdog. */
1102 hw->mac.get_link_status = 1;
1103 schedule_work(&adapter->watchdog_task);
1104
9d5c8243
AK		 1105 return 0;
1106}
1107
1108void igb_down(struct igb_adapter *adapter)
1109{
9d5c8243 1110 struct net_device *netdev = adapter->netdev;
330a6d6a 1111 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1112 u32 tctl, rctl;
1113 int i;
1114
1115 /* signal that we're down so the interrupt handler does not
1116 * reschedule our watchdog timer */
1117 set_bit(__IGB_DOWN, &adapter->state);
1118
1119 /* disable receives in the hardware */
1120 rctl = rd32(E1000_RCTL);
1121 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1122 /* flush and sleep below */
1123
fd2ea0a7 1124 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1125
1126 /* disable transmits in the hardware */
1127 tctl = rd32(E1000_TCTL);
1128 tctl &= ~E1000_TCTL_EN;
1129 wr32(E1000_TCTL, tctl);
1130 /* flush both disables and wait for them to finish */
1131 wrfl();
1132 msleep(10);
1133
047e0030
AD		 1134 for (i = 0; i < adapter->num_q_vectors; i++) {
1135 struct igb_q_vector *q_vector = adapter->q_vector[i];
1136 napi_disable(&q_vector->napi);
1137 }
9d5c8243 1138
9d5c8243
AK		 1139 igb_irq_disable(adapter);
1140
1141 del_timer_sync(&adapter->watchdog_timer);
1142 del_timer_sync(&adapter->phy_info_timer);
1143
1144 netdev->tx_queue_len = adapter->tx_queue_len;
1145 netif_carrier_off(netdev);
04fe6358
AD		 1146
1147 /* record the stats before reset*/
1148 igb_update_stats(adapter);
1149
9d5c8243
AK		 1150 adapter->link_speed = 0;
1151 adapter->link_duplex = 0;
1152
3023682e
JK		 1153 if (!pci_channel_offline(adapter->pdev))
1154 igb_reset(adapter);
9d5c8243
AK		 1155 igb_clean_all_tx_rings(adapter);
1156 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1157#ifdef CONFIG_IGB_DCA
1158
1159 /* since we reset the hardware DCA settings were cleared */
1160 igb_setup_dca(adapter);
1161#endif
9d5c8243
AK		 1162}
1163
1164void igb_reinit_locked(struct igb_adapter *adapter)
1165{
1166 WARN_ON(in_interrupt());
1167 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1168 msleep(1);
1169 igb_down(adapter);
1170 igb_up(adapter);
1171 clear_bit(__IGB_RESETTING, &adapter->state);
1172}
1173
1174void igb_reset(struct igb_adapter *adapter)
1175{
1176 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1177 struct e1000_mac_info *mac = &hw->mac;
1178 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1179 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1180 u16 hwm;
1181
1182 /* Repartition Pba for greater than 9k mtu
1183 * To take effect CTRL.RST is required.
1184 */
fa4dfae0
AD		 1185 switch (mac->type) {
1186 case e1000_82576:
d249be54
AD		 1187 pba = rd32(E1000_RXPBS);
1188 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1189 break;
1190 case e1000_82575:
1191 default:
1192 pba = E1000_PBA_34K;
1193 break;
2d064c06 1194 }
9d5c8243 1195
2d064c06
AD		 1196 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1197 (mac->type < e1000_82576)) {
9d5c8243
AK		 1198 /* adjust PBA for jumbo frames */
1199 wr32(E1000_PBA, pba);
1200
1201 /* To maintain wire speed transmits, the Tx FIFO should be
1202 * large enough to accommodate two full transmit packets,
1203 * rounded up to the next 1KB and expressed in KB. Likewise,
1204 * the Rx FIFO should be large enough to accommodate at least
1205 * one full receive packet and is similarly rounded up and
1206 * expressed in KB. */
1207 pba = rd32(E1000_PBA);
1208 /* upper 16 bits has Tx packet buffer allocation size in KB */
1209 tx_space = pba >> 16;
1210 /* lower 16 bits has Rx packet buffer allocation size in KB */
1211 pba &= 0xffff;
1212 /* the tx fifo also stores 16 bytes of information about the tx
1213 * but don't include ethernet FCS because hardware appends it */
1214 min_tx_space = (adapter->max_frame_size +
85e8d004 1215 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1216 ETH_FCS_LEN) * 2;
1217 min_tx_space = ALIGN(min_tx_space, 1024);
1218 min_tx_space >>= 10;
1219 /* software strips receive CRC, so leave room for it */
1220 min_rx_space = adapter->max_frame_size;
1221 min_rx_space = ALIGN(min_rx_space, 1024);
1222 min_rx_space >>= 10;
1223
1224 /* If current Tx allocation is less than the min Tx FIFO size,
1225 * and the min Tx FIFO size is less than the current Rx FIFO
1226 * allocation, take space away from current Rx allocation */
1227 if (tx_space < min_tx_space &&
1228 ((min_tx_space - tx_space) < pba)) {
1229 pba = pba - (min_tx_space - tx_space);
1230
1231 /* if short on rx space, rx wins and must trump tx
1232 * adjustment */
1233 if (pba < min_rx_space)
1234 pba = min_rx_space;
1235 }
2d064c06 1236 wr32(E1000_PBA, pba);
9d5c8243 1237 }
9d5c8243
AK		 1238
1239 /* flow control settings */
1240 /* The high water mark must be low enough to fit one full frame
1241 * (or the size used for early receive) above it in the Rx FIFO.
1242 * Set it to the lower of:
1243 * - 90% of the Rx FIFO size, or
1244 * - the full Rx FIFO size minus one full frame */
1245 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1246 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1247
2d064c06
AD		 1248 if (mac->type < e1000_82576) {
1249 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1250 fc->low_water = fc->high_water - 8;
1251 } else {
1252 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1253 fc->low_water = fc->high_water - 16;
1254 }
9d5c8243
AK		 1255 fc->pause_time = 0xFFFF;
1256 fc->send_xon = 1;
0cce119a 1257 fc->current_mode = fc->requested_mode;
9d5c8243 1258
4ae196df
AD		 1259 /* disable receive for all VFs and wait one second */
1260 if (adapter->vfs_allocated_count) {
1261 int i;
1262 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1263 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1264
1265 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1266 igb_ping_all_vfs(adapter);
4ae196df
AD		 1267
1268 /* disable transmits and receives */
1269 wr32(E1000_VFRE, 0);
1270 wr32(E1000_VFTE, 0);
1271 }
1272
9d5c8243 1273 /* Allow time for pending master requests to run */
330a6d6a 1274 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1275 wr32(E1000_WUC, 0);
1276
330a6d6a 1277 if (hw->mac.ops.init_hw(hw))
9d5c8243
AK		 1278 dev_err(&adapter->pdev->dev, "Hardware Error\n");
1279
1280 igb_update_mng_vlan(adapter);
1281
1282 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1283 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1284
330a6d6a
AD		 1285 igb_reset_adaptive(hw);
1286 igb_get_phy_info(hw);
9d5c8243
AK		 1287}
1288
2e5c6922
SH		 1289static const struct net_device_ops igb_netdev_ops = {
1290 .ndo_open = igb_open,
1291 .ndo_stop = igb_close,
00829823 1292 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1293 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1294 .ndo_set_rx_mode = igb_set_rx_mode,
1295 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1296 .ndo_set_mac_address = igb_set_mac,
1297 .ndo_change_mtu = igb_change_mtu,
1298 .ndo_do_ioctl = igb_ioctl,
1299 .ndo_tx_timeout = igb_tx_timeout,
1300 .ndo_validate_addr = eth_validate_addr,
1301 .ndo_vlan_rx_register = igb_vlan_rx_register,
1302 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1303 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1304#ifdef CONFIG_NET_POLL_CONTROLLER
1305 .ndo_poll_controller = igb_netpoll,
1306#endif
1307};
1308
9d5c8243
AK		 1309/**
1310 * igb_probe - Device Initialization Routine
1311 * @pdev: PCI device information struct
1312 * @ent: entry in igb_pci_tbl
1313 *
1314 * Returns 0 on success, negative on failure
1315 *
1316 * igb_probe initializes an adapter identified by a pci_dev structure.
1317 * The OS initialization, configuring of the adapter private structure,
1318 * and a hardware reset occur.
1319 **/
1320static int __devinit igb_probe(struct pci_dev *pdev,
1321 const struct pci_device_id *ent)
1322{
1323 struct net_device *netdev;
1324 struct igb_adapter *adapter;
1325 struct e1000_hw *hw;
4337e993
AD		 1326 u16 eeprom_data = 0;
1327 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1328 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1329 unsigned long mmio_start, mmio_len;
2d6a5e95 1330 int err, pci_using_dac;
9d5c8243
AK		 1331 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1332 u32 part_num;
1333
aed5dec3 1334 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1335 if (err)
1336 return err;
1337
1338 pci_using_dac = 0;
6a35528a 1339 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1340 if (!err) {
6a35528a 1341 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1342 if (!err)
1343 pci_using_dac = 1;
1344 } else {
284901a9 1345 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1346 if (err) {
284901a9 1347 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1348 if (err) {
1349 dev_err(&pdev->dev, "No usable DMA "
1350 "configuration, aborting\n");
1351 goto err_dma;
1352 }
1353 }
1354 }
1355
aed5dec3
AD		 1356 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1357 IORESOURCE_MEM),
1358 igb_driver_name);
9d5c8243
AK		 1359 if (err)
1360 goto err_pci_reg;
1361
19d5afd4 1362 pci_enable_pcie_error_reporting(pdev);
40a914fa 1363
9d5c8243 1364 pci_set_master(pdev);
c682fc23 1365 pci_save_state(pdev);
9d5c8243
AK		 1366
1367 err = -ENOMEM;
1bfaf07b
AD		 1368 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1369 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1370 if (!netdev)
1371 goto err_alloc_etherdev;
1372
1373 SET_NETDEV_DEV(netdev, &pdev->dev);
1374
1375 pci_set_drvdata(pdev, netdev);
1376 adapter = netdev_priv(netdev);
1377 adapter->netdev = netdev;
1378 adapter->pdev = pdev;
1379 hw = &adapter->hw;
1380 hw->back = adapter;
1381 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1382
1383 mmio_start = pci_resource_start(pdev, 0);
1384 mmio_len = pci_resource_len(pdev, 0);
1385
1386 err = -EIO;
28b0759c
AD		 1387 hw->hw_addr = ioremap(mmio_start, mmio_len);
1388 if (!hw->hw_addr)
9d5c8243
AK		 1389 goto err_ioremap;
1390
2e5c6922 1391 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1392 igb_set_ethtool_ops(netdev);
9d5c8243 1393 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1394
1395 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1396
1397 netdev->mem_start = mmio_start;
1398 netdev->mem_end = mmio_start + mmio_len;
1399
9d5c8243
AK		 1400 /* PCI config space info */
1401 hw->vendor_id = pdev->vendor;
1402 hw->device_id = pdev->device;
1403 hw->revision_id = pdev->revision;
1404 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1405 hw->subsystem_device_id = pdev->subsystem_device;
1406
9d5c8243
AK		 1407 /* Copy the default MAC, PHY and NVM function pointers */
1408 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1409 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1410 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1411 /* Initialize skew-specific constants */
1412 err = ei->get_invariants(hw);
1413 if (err)
450c87c8 1414 goto err_sw_init;
9d5c8243 1415
450c87c8 1416 /* setup the private structure */
9d5c8243
AK		 1417 err = igb_sw_init(adapter);
1418 if (err)
1419 goto err_sw_init;
1420
1421 igb_get_bus_info_pcie(hw);
1422
1423 hw->phy.autoneg_wait_to_complete = false;
1424 hw->mac.adaptive_ifs = true;
1425
1426 /* Copper options */
1427 if (hw->phy.media_type == e1000_media_type_copper) {
1428 hw->phy.mdix = AUTO_ALL_MODES;
1429 hw->phy.disable_polarity_correction = false;
1430 hw->phy.ms_type = e1000_ms_hw_default;
1431 }
1432
1433 if (igb_check_reset_block(hw))
1434 dev_info(&pdev->dev,
1435 "PHY reset is blocked due to SOL/IDER session.\n");
1436
1437 netdev->features = NETIF_F_SG |
7d8eb29e 1438 NETIF_F_IP_CSUM |
9d5c8243
AK		 1439 NETIF_F_HW_VLAN_TX |
1440 NETIF_F_HW_VLAN_RX |
1441 NETIF_F_HW_VLAN_FILTER;
1442
7d8eb29e 1443 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1444 netdev->features |= NETIF_F_TSO;
9d5c8243 1445 netdev->features |= NETIF_F_TSO6;
48f29ffc 1446
5c0999b7 1447 netdev->features |= NETIF_F_GRO;
d3352520 1448
48f29ffc
JK		 1449 netdev->vlan_features |= NETIF_F_TSO;
1450 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1451 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1452 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1453 netdev->vlan_features |= NETIF_F_SG;
1454
9d5c8243
AK		 1455 if (pci_using_dac)
1456 netdev->features |= NETIF_F_HIGHDMA;
1457
b9473560
JB		 1458 if (adapter->hw.mac.type == e1000_82576)
1459 netdev->features |= NETIF_F_SCTP_CSUM;
1460
330a6d6a 1461 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1462
1463 /* before reading the NVM, reset the controller to put the device in a
1464 * known good starting state */
1465 hw->mac.ops.reset_hw(hw);
1466
1467 /* make sure the NVM is good */
1468 if (igb_validate_nvm_checksum(hw) < 0) {
1469 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1470 err = -EIO;
1471 goto err_eeprom;
1472 }
1473
1474 /* copy the MAC address out of the NVM */
1475 if (hw->mac.ops.read_mac_addr(hw))
1476 dev_err(&pdev->dev, "NVM Read Error\n");
1477
1478 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1479 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1480
1481 if (!is_valid_ether_addr(netdev->perm_addr)) {
1482 dev_err(&pdev->dev, "Invalid MAC Address\n");
1483 err = -EIO;
1484 goto err_eeprom;
1485 }
1486
0e340485
AD		 1487 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1488 (unsigned long) adapter);
1489 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1490 (unsigned long) adapter);
9d5c8243
AK		 1491
1492 INIT_WORK(&adapter->reset_task, igb_reset_task);
1493 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1494
450c87c8 1495 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1496 adapter->fc_autoneg = true;
1497 hw->mac.autoneg = true;
1498 hw->phy.autoneg_advertised = 0x2f;
1499
0cce119a
AD		 1500 hw->fc.requested_mode = e1000_fc_default;
1501 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1502
9d5c8243
AK		 1503 igb_validate_mdi_setting(hw);
1504
9d5c8243
AK		 1505 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1506 * enable the ACPI Magic Packet filter
1507 */
1508
a2cf8b6c 1509 if (hw->bus.func == 0)
312c75ae 1510 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
a2cf8b6c
AD		 1511 else if (hw->bus.func == 1)
1512 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1513
1514 if (eeprom_data & eeprom_apme_mask)
1515 adapter->eeprom_wol |= E1000_WUFC_MAG;
1516
1517 /* now that we have the eeprom settings, apply the special cases where
1518 * the eeprom may be wrong or the board simply won't support wake on
1519 * lan on a particular port */
1520 switch (pdev->device) {
1521 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1522 adapter->eeprom_wol = 0;
1523 break;
1524 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1525 case E1000_DEV_ID_82576_FIBER:
1526 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1527 /* Wake events only supported on port A for dual fiber
1528 * regardless of eeprom setting */
1529 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1530 adapter->eeprom_wol = 0;
1531 break;
c8ea5ea9
AD		 1532 case E1000_DEV_ID_82576_QUAD_COPPER:
1533 /* if quad port adapter, disable WoL on all but port A */
1534 if (global_quad_port_a != 0)
1535 adapter->eeprom_wol = 0;
1536 else
1537 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1538 /* Reset for multiple quad port adapters */
1539 if (++global_quad_port_a == 4)
1540 global_quad_port_a = 0;
1541 break;
9d5c8243
AK		 1542 }
1543
1544 /* initialize the wol settings based on the eeprom settings */
1545 adapter->wol = adapter->eeprom_wol;
e1b86d84 1546 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1547
1548 /* reset the hardware with the new settings */
1549 igb_reset(adapter);
1550
1551 /* let the f/w know that the h/w is now under the control of the
1552 * driver. */
1553 igb_get_hw_control(adapter);
1554
9d5c8243
AK		 1555 strcpy(netdev->name, "eth%d");
1556 err = register_netdev(netdev);
1557 if (err)
1558 goto err_register;
1559
b168dfc5
JB		 1560 /* carrier off reporting is important to ethtool even BEFORE open */
1561 netif_carrier_off(netdev);
1562
421e02f0 1563#ifdef CONFIG_IGB_DCA
bbd98fe4 1564 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1565 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1566 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1567 igb_setup_dca(adapter);
1568 }
fe4506b6 1569
38c845c7 1570#endif
38c845c7 1571
c5b9bd5e
AD		 1572 switch (hw->mac.type) {
1573 case e1000_82576:
1574 /*
1575 * Initialize hardware timer: we keep it running just in case
1576 * that some program needs it later on.
1577 */
1578 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1579 adapter->cycles.read = igb_read_clock;
1580 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1581 adapter->cycles.mult = 1;
1582 /**
1583 * Scale the NIC clock cycle by a large factor so that
1584 * relatively small clock corrections can be added or
1585 * substracted at each clock tick. The drawbacks of a large
1586 * factor are a) that the clock register overflows more quickly
1587 * (not such a big deal) and b) that the increment per tick has
1588 * to fit into 24 bits. As a result we need to use a shift of
1589 * 19 so we can fit a value of 16 into the TIMINCA register.
1590 */
1591 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1592 wr32(E1000_TIMINCA,
1593 (1 << E1000_TIMINCA_16NS_SHIFT) |
1594 (16 << IGB_82576_TSYNC_SHIFT));
1595
1596 /* Set registers so that rollover occurs soon to test this. */
1597 wr32(E1000_SYSTIML, 0x00000000);
1598 wr32(E1000_SYSTIMH, 0xFF800000);
1599 wrfl();
1600
1601 timecounter_init(&adapter->clock,
1602 &adapter->cycles,
1603 ktime_to_ns(ktime_get_real()));
1604 /*
1605 * Synchronize our NIC clock against system wall clock. NIC
1606 * time stamp reading requires ~3us per sample, each sample
1607 * was pretty stable even under load => only require 10
1608 * samples for each offset comparison.
1609 */
1610 memset(&adapter->compare, 0, sizeof(adapter->compare));
1611 adapter->compare.source = &adapter->clock;
1612 adapter->compare.target = ktime_get_real;
1613 adapter->compare.num_samples = 10;
1614 timecompare_update(&adapter->compare, 0);
1615 break;
1616 case e1000_82575:
1617 /* 82575 does not support timesync */
1618 default:
1619 break;
38c845c7 1620 }
38c845c7 1621
9d5c8243
AK		 1622 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1623 /* print bus type/speed/width info */
7c510e4b 1624 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243
AK		 1625 netdev->name,
1626 ((hw->bus.speed == e1000_bus_speed_2500)
1627 ? "2.5Gb/s" : "unknown"),
59c3de89
AD		 1628 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1629 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1630 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1631 "unknown"),
7c510e4b 1632 netdev->dev_addr);
9d5c8243
AK		 1633
1634 igb_read_part_num(hw, &part_num);
1635 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1636 (part_num >> 8), (part_num & 0xff));
1637
1638 dev_info(&pdev->dev,
1639 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1640 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1641 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1642 adapter->num_rx_queues, adapter->num_tx_queues);
1643
9d5c8243
AK		 1644 return 0;
1645
1646err_register:
1647 igb_release_hw_control(adapter);
1648err_eeprom:
1649 if (!igb_check_reset_block(hw))
f5f4cf08 1650 igb_reset_phy(hw);
9d5c8243
AK		 1651
1652 if (hw->flash_address)
1653 iounmap(hw->flash_address);
9d5c8243 1654err_sw_init:
047e0030 1655 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1656 iounmap(hw->hw_addr);
1657err_ioremap:
1658 free_netdev(netdev);
1659err_alloc_etherdev:
aed5dec3
AD		 1660 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1661 IORESOURCE_MEM));
9d5c8243
AK		 1662err_pci_reg:
1663err_dma:
1664 pci_disable_device(pdev);
1665 return err;
1666}
1667
1668/**
1669 * igb_remove - Device Removal Routine
1670 * @pdev: PCI device information struct
1671 *
1672 * igb_remove is called by the PCI subsystem to alert the driver
1673 * that it should release a PCI device. The could be caused by a
1674 * Hot-Plug event, or because the driver is going to be removed from
1675 * memory.
1676 **/
1677static void __devexit igb_remove(struct pci_dev *pdev)
1678{
1679 struct net_device *netdev = pci_get_drvdata(pdev);
1680 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1681 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1682
1683 /* flush_scheduled work may reschedule our watchdog task, so
1684 * explicitly disable watchdog tasks from being rescheduled */
1685 set_bit(__IGB_DOWN, &adapter->state);
1686 del_timer_sync(&adapter->watchdog_timer);
1687 del_timer_sync(&adapter->phy_info_timer);
1688
1689 flush_scheduled_work();
1690
421e02f0 1691#ifdef CONFIG_IGB_DCA
7dfc16fa 1692 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1693 dev_info(&pdev->dev, "DCA disabled\n");
1694 dca_remove_requester(&pdev->dev);
7dfc16fa 1695 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1696 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1697 }
1698#endif
1699
9d5c8243
AK		 1700 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1701 * would have already happened in close and is redundant. */
1702 igb_release_hw_control(adapter);
1703
1704 unregister_netdev(netdev);
1705
330a6d6a
AD		 1706 if (!igb_check_reset_block(hw))
1707 igb_reset_phy(hw);
9d5c8243 1708
047e0030 1709 igb_clear_interrupt_scheme(adapter);
9d5c8243 1710
37680117
AD		 1711#ifdef CONFIG_PCI_IOV
1712 /* reclaim resources allocated to VFs */
1713 if (adapter->vf_data) {
1714 /* disable iov and allow time for transactions to clear */
1715 pci_disable_sriov(pdev);
1716 msleep(500);
1717
1718 kfree(adapter->vf_data);
1719 adapter->vf_data = NULL;
1720 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1721 msleep(100);
1722 dev_info(&pdev->dev, "IOV Disabled\n");
1723 }
1724#endif
28b0759c
AD		 1725 iounmap(hw->hw_addr);
1726 if (hw->flash_address)
1727 iounmap(hw->flash_address);
aed5dec3
AD		 1728 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1729 IORESOURCE_MEM));
9d5c8243
AK		 1730
1731 free_netdev(netdev);
1732
19d5afd4 1733 pci_disable_pcie_error_reporting(pdev);
40a914fa 1734
9d5c8243
AK		 1735 pci_disable_device(pdev);
1736}
1737
a6b623e0
AD		 1738/**
1739 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1740 * @adapter: board private structure to initialize
1741 *
1742 * This function initializes the vf specific data storage and then attempts to
1743 * allocate the VFs. The reason for ordering it this way is because it is much
1744 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1745 * the memory for the VFs.
1746 **/
1747static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1748{
1749#ifdef CONFIG_PCI_IOV
1750 struct pci_dev *pdev = adapter->pdev;
1751
1752 if (adapter->vfs_allocated_count > 7)
1753 adapter->vfs_allocated_count = 7;
1754
1755 if (adapter->vfs_allocated_count) {
1756 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1757 sizeof(struct vf_data_storage),
1758 GFP_KERNEL);
1759 /* if allocation failed then we do not support SR-IOV */
1760 if (!adapter->vf_data) {
1761 adapter->vfs_allocated_count = 0;
1762 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1763 "Data Storage\n");
1764 }
1765 }
1766
1767 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1768 kfree(adapter->vf_data);
1769 adapter->vf_data = NULL;
1770#endif /* CONFIG_PCI_IOV */
1771 adapter->vfs_allocated_count = 0;
1772#ifdef CONFIG_PCI_IOV
1773 } else {
1774 unsigned char mac_addr[ETH_ALEN];
1775 int i;
1776 dev_info(&pdev->dev, "%d vfs allocated\n",
1777 adapter->vfs_allocated_count);
1778 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1779 random_ether_addr(mac_addr);
1780 igb_set_vf_mac(adapter, i, mac_addr);
1781 }
1782 }
1783#endif /* CONFIG_PCI_IOV */
1784}
1785
9d5c8243
AK		 1786/**
1787 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1788 * @adapter: board private structure to initialize
1789 *
1790 * igb_sw_init initializes the Adapter private data structure.
1791 * Fields are initialized based on PCI device information and
1792 * OS network device settings (MTU size).
1793 **/
1794static int __devinit igb_sw_init(struct igb_adapter *adapter)
1795{
1796 struct e1000_hw *hw = &adapter->hw;
1797 struct net_device *netdev = adapter->netdev;
1798 struct pci_dev *pdev = adapter->pdev;
1799
1800 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1801
68fd9910
AD		 1802 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1803 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1804 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1805 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1806
9d5c8243
AK		 1807 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1808 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1809
a6b623e0
AD		 1810#ifdef CONFIG_PCI_IOV
1811 if (hw->mac.type == e1000_82576)
1812 adapter->vfs_allocated_count = max_vfs;
1813
1814#endif /* CONFIG_PCI_IOV */
1815 /* This call may decrease the number of queues */
047e0030 1816 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1817 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1818 return -ENOMEM;
1819 }
1820
a6b623e0
AD		 1821 igb_probe_vfs(adapter);
1822
9d5c8243
AK		 1823 /* Explicitly disable IRQ since the NIC can be in any state. */
1824 igb_irq_disable(adapter);
1825
1826 set_bit(__IGB_DOWN, &adapter->state);
1827 return 0;
1828}
1829
1830/**
1831 * igb_open - Called when a network interface is made active
1832 * @netdev: network interface device structure
1833 *
1834 * Returns 0 on success, negative value on failure
1835 *
1836 * The open entry point is called when a network interface is made
1837 * active by the system (IFF_UP). At this point all resources needed
1838 * for transmit and receive operations are allocated, the interrupt
1839 * handler is registered with the OS, the watchdog timer is started,
1840 * and the stack is notified that the interface is ready.
1841 **/
1842static int igb_open(struct net_device *netdev)
1843{
1844 struct igb_adapter *adapter = netdev_priv(netdev);
1845 struct e1000_hw *hw = &adapter->hw;
1846 int err;
1847 int i;
1848
1849 /* disallow open during test */
1850 if (test_bit(__IGB_TESTING, &adapter->state))
1851 return -EBUSY;
1852
b168dfc5
JB		 1853 netif_carrier_off(netdev);
1854
9d5c8243
AK		 1855 /* allocate transmit descriptors */
1856 err = igb_setup_all_tx_resources(adapter);
1857 if (err)
1858 goto err_setup_tx;
1859
1860 /* allocate receive descriptors */
1861 err = igb_setup_all_rx_resources(adapter);
1862 if (err)
1863 goto err_setup_rx;
1864
1865 /* e1000_power_up_phy(adapter); */
1866
9d5c8243
AK		 1867 /* before we allocate an interrupt, we must be ready to handle it.
1868 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1869 * as soon as we call pci_request_irq, so we have to setup our
1870 * clean_rx handler before we do so. */
1871 igb_configure(adapter);
1872
1873 err = igb_request_irq(adapter);
1874 if (err)
1875 goto err_req_irq;
1876
1877 /* From here on the code is the same as igb_up() */
1878 clear_bit(__IGB_DOWN, &adapter->state);
1879
047e0030
AD		 1880 for (i = 0; i < adapter->num_q_vectors; i++) {
1881 struct igb_q_vector *q_vector = adapter->q_vector[i];
1882 napi_enable(&q_vector->napi);
1883 }
9d5c8243
AK		 1884
1885 /* Clear any pending interrupts. */
1886 rd32(E1000_ICR);
844290e5
PW		 1887
1888 igb_irq_enable(adapter);
1889
d4960307
AD		 1890 /* notify VFs that reset has been completed */
1891 if (adapter->vfs_allocated_count) {
1892 u32 reg_data = rd32(E1000_CTRL_EXT);
1893 reg_data |= E1000_CTRL_EXT_PFRSTD;
1894 wr32(E1000_CTRL_EXT, reg_data);
1895 }
1896
d55b53ff
JK		 1897 netif_tx_start_all_queues(netdev);
1898
25568a53
AD		 1899 /* start the watchdog. */
1900 hw->mac.get_link_status = 1;
1901 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 1902
1903 return 0;
1904
1905err_req_irq:
1906 igb_release_hw_control(adapter);
1907 /* e1000_power_down_phy(adapter); */
1908 igb_free_all_rx_resources(adapter);
1909err_setup_rx:
1910 igb_free_all_tx_resources(adapter);
1911err_setup_tx:
1912 igb_reset(adapter);
1913
1914 return err;
1915}
1916
1917/**
1918 * igb_close - Disables a network interface
1919 * @netdev: network interface device structure
1920 *
1921 * Returns 0, this is not allowed to fail
1922 *
1923 * The close entry point is called when an interface is de-activated
1924 * by the OS. The hardware is still under the driver's control, but
1925 * needs to be disabled. A global MAC reset is issued to stop the
1926 * hardware, and all transmit and receive resources are freed.
1927 **/
1928static int igb_close(struct net_device *netdev)
1929{
1930 struct igb_adapter *adapter = netdev_priv(netdev);
1931
1932 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1933 igb_down(adapter);
1934
1935 igb_free_irq(adapter);
1936
1937 igb_free_all_tx_resources(adapter);
1938 igb_free_all_rx_resources(adapter);
1939
9d5c8243
AK		 1940 return 0;
1941}
1942
1943/**
1944 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 1945 * @tx_ring: tx descriptor ring (for a specific queue) to setup
1946 *
1947 * Return 0 on success, negative on failure
1948 **/
80785298 1949int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 1950{
80785298 1951 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 1952 int size;
1953
1954 size = sizeof(struct igb_buffer) * tx_ring->count;
1955 tx_ring->buffer_info = vmalloc(size);
1956 if (!tx_ring->buffer_info)
1957 goto err;
1958 memset(tx_ring->buffer_info, 0, size);
1959
1960 /* round up to nearest 4K */
85e8d004 1961 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 1962 tx_ring->size = ALIGN(tx_ring->size, 4096);
1963
439705e1
AD		 1964 tx_ring->desc = pci_alloc_consistent(pdev,
1965 tx_ring->size,
9d5c8243
AK		 1966 &tx_ring->dma);
1967
1968 if (!tx_ring->desc)
1969 goto err;
1970
9d5c8243
AK		 1971 tx_ring->next_to_use = 0;
1972 tx_ring->next_to_clean = 0;
9d5c8243
AK		 1973 return 0;
1974
1975err:
1976 vfree(tx_ring->buffer_info);
047e0030 1977 dev_err(&pdev->dev,
9d5c8243
AK		 1978 "Unable to allocate memory for the transmit descriptor ring\n");
1979 return -ENOMEM;
1980}
1981
1982/**
1983 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
1984 * (Descriptors) for all queues
1985 * @adapter: board private structure
1986 *
1987 * Return 0 on success, negative on failure
1988 **/
1989static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
1990{
439705e1 1991 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1992 int i, err = 0;
1993
1994 for (i = 0; i < adapter->num_tx_queues; i++) {
80785298 1995 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
9d5c8243 1996 if (err) {
439705e1 1997 dev_err(&pdev->dev,
9d5c8243
AK		 1998 "Allocation for Tx Queue %u failed\n", i);
1999 for (i--; i >= 0; i--)
3b644cf6 2000 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2001 break;
2002 }
2003 }
2004
661086df 2005 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
439705e1 2006 int r_idx = i % adapter->num_tx_queues;
661086df 2007 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
eebbbdba 2008 }
9d5c8243
AK		 2009 return err;
2010}
2011
2012/**
85b430b4
AD		 2013 * igb_setup_tctl - configure the transmit control registers
2014 * @adapter: Board private structure
9d5c8243 2015 **/
d7ee5b3a 2016void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2017{
9d5c8243
AK		 2018 struct e1000_hw *hw = &adapter->hw;
2019 u32 tctl;
9d5c8243 2020
85b430b4
AD		 2021 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2022 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2023
2024 /* Program the Transmit Control Register */
9d5c8243
AK		 2025 tctl = rd32(E1000_TCTL);
2026 tctl &= ~E1000_TCTL_CT;
2027 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2028 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2029
2030 igb_config_collision_dist(hw);
2031
9d5c8243
AK		 2032 /* Enable transmits */
2033 tctl |= E1000_TCTL_EN;
2034
2035 wr32(E1000_TCTL, tctl);
2036}
2037
85b430b4
AD		 2038/**
2039 * igb_configure_tx_ring - Configure transmit ring after Reset
2040 * @adapter: board private structure
2041 * @ring: tx ring to configure
2042 *
2043 * Configure a transmit ring after a reset.
2044 **/
d7ee5b3a
AD		 2045void igb_configure_tx_ring(struct igb_adapter *adapter,
2046 struct igb_ring *ring)
85b430b4
AD		 2047{
2048 struct e1000_hw *hw = &adapter->hw;
2049 u32 txdctl;
2050 u64 tdba = ring->dma;
2051 int reg_idx = ring->reg_idx;
2052
2053 /* disable the queue */
2054 txdctl = rd32(E1000_TXDCTL(reg_idx));
2055 wr32(E1000_TXDCTL(reg_idx),
2056 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2057 wrfl();
2058 mdelay(10);
2059
2060 wr32(E1000_TDLEN(reg_idx),
2061 ring->count * sizeof(union e1000_adv_tx_desc));
2062 wr32(E1000_TDBAL(reg_idx),
2063 tdba & 0x00000000ffffffffULL);
2064 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2065
fce99e34
AD		 2066 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2067 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2068 writel(0, ring->head);
2069 writel(0, ring->tail);
85b430b4
AD		 2070
2071 txdctl |= IGB_TX_PTHRESH;
2072 txdctl |= IGB_TX_HTHRESH << 8;
2073 txdctl |= IGB_TX_WTHRESH << 16;
2074
2075 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2076 wr32(E1000_TXDCTL(reg_idx), txdctl);
2077}
2078
2079/**
2080 * igb_configure_tx - Configure transmit Unit after Reset
2081 * @adapter: board private structure
2082 *
2083 * Configure the Tx unit of the MAC after a reset.
2084 **/
2085static void igb_configure_tx(struct igb_adapter *adapter)
2086{
2087 int i;
2088
2089 for (i = 0; i < adapter->num_tx_queues; i++)
2090 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
85b430b4
AD		 2091}
2092
9d5c8243
AK		 2093/**
2094 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2095 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2096 *
2097 * Returns 0 on success, negative on failure
2098 **/
80785298 2099int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2100{
80785298 2101 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2102 int size, desc_len;
2103
2104 size = sizeof(struct igb_buffer) * rx_ring->count;
2105 rx_ring->buffer_info = vmalloc(size);
2106 if (!rx_ring->buffer_info)
2107 goto err;
2108 memset(rx_ring->buffer_info, 0, size);
2109
2110 desc_len = sizeof(union e1000_adv_rx_desc);
2111
2112 /* Round up to nearest 4K */
2113 rx_ring->size = rx_ring->count * desc_len;
2114 rx_ring->size = ALIGN(rx_ring->size, 4096);
2115
2116 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2117 &rx_ring->dma);
2118
2119 if (!rx_ring->desc)
2120 goto err;
2121
2122 rx_ring->next_to_clean = 0;
2123 rx_ring->next_to_use = 0;
9d5c8243 2124
9d5c8243
AK		 2125 return 0;
2126
2127err:
2128 vfree(rx_ring->buffer_info);
439705e1 2129 rx_ring->buffer_info = NULL;
80785298 2130 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2131 "the receive descriptor ring\n");
2132 return -ENOMEM;
2133}
2134
2135/**
2136 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2137 * (Descriptors) for all queues
2138 * @adapter: board private structure
2139 *
2140 * Return 0 on success, negative on failure
2141 **/
2142static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2143{
439705e1 2144 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2145 int i, err = 0;
2146
2147 for (i = 0; i < adapter->num_rx_queues; i++) {
80785298 2148 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
9d5c8243 2149 if (err) {
439705e1 2150 dev_err(&pdev->dev,
9d5c8243
AK		 2151 "Allocation for Rx Queue %u failed\n", i);
2152 for (i--; i >= 0; i--)
3b644cf6 2153 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2154 break;
2155 }
2156 }
2157
2158 return err;
2159}
2160
06cf2666
AD		 2161/**
2162 * igb_setup_mrqc - configure the multiple receive queue control registers
2163 * @adapter: Board private structure
2164 **/
2165static void igb_setup_mrqc(struct igb_adapter *adapter)
2166{
2167 struct e1000_hw *hw = &adapter->hw;
2168 u32 mrqc, rxcsum;
2169 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2170 union e1000_reta {
2171 u32 dword;
2172 u8 bytes[4];
2173 } reta;
2174 static const u8 rsshash[40] = {
2175 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2176 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2177 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2178 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2179
2180 /* Fill out hash function seeds */
2181 for (j = 0; j < 10; j++) {
2182 u32 rsskey = rsshash[(j * 4)];
2183 rsskey |= rsshash[(j * 4) + 1] << 8;
2184 rsskey |= rsshash[(j * 4) + 2] << 16;
2185 rsskey |= rsshash[(j * 4) + 3] << 24;
2186 array_wr32(E1000_RSSRK(0), j, rsskey);
2187 }
2188
2189 num_rx_queues = adapter->num_rx_queues;
2190
2191 if (adapter->vfs_allocated_count) {
2192 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2193 switch (hw->mac.type) {
2194 case e1000_82576:
2195 shift = 3;
2196 num_rx_queues = 2;
2197 break;
2198 case e1000_82575:
2199 shift = 2;
2200 shift2 = 6;
2201 default:
2202 break;
2203 }
2204 } else {
2205 if (hw->mac.type == e1000_82575)
2206 shift = 6;
2207 }
2208
2209 for (j = 0; j < (32 * 4); j++) {
2210 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2211 if (shift2)
2212 reta.bytes[j & 3] |= num_rx_queues << shift2;
2213 if ((j & 3) == 3)
2214 wr32(E1000_RETA(j >> 2), reta.dword);
2215 }
2216
2217 /*
2218 * Disable raw packet checksumming so that RSS hash is placed in
2219 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2220 * offloads as they are enabled by default
2221 */
2222 rxcsum = rd32(E1000_RXCSUM);
2223 rxcsum |= E1000_RXCSUM_PCSD;
2224
2225 if (adapter->hw.mac.type >= e1000_82576)
2226 /* Enable Receive Checksum Offload for SCTP */
2227 rxcsum |= E1000_RXCSUM_CRCOFL;
2228
2229 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2230 wr32(E1000_RXCSUM, rxcsum);
2231
2232 /* If VMDq is enabled then we set the appropriate mode for that, else
2233 * we default to RSS so that an RSS hash is calculated per packet even
2234 * if we are only using one queue */
2235 if (adapter->vfs_allocated_count) {
2236 if (hw->mac.type > e1000_82575) {
2237 /* Set the default pool for the PF's first queue */
2238 u32 vtctl = rd32(E1000_VT_CTL);
2239 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2240 E1000_VT_CTL_DISABLE_DEF_POOL);
2241 vtctl |= adapter->vfs_allocated_count <<
2242 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2243 wr32(E1000_VT_CTL, vtctl);
2244 }
2245 if (adapter->num_rx_queues > 1)
2246 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2247 else
2248 mrqc = E1000_MRQC_ENABLE_VMDQ;
2249 } else {
2250 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2251 }
2252 igb_vmm_control(adapter);
2253
2254 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2255 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2256 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2257 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2258 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2259 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2260 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2261 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2262
2263 wr32(E1000_MRQC, mrqc);
2264}
2265
9d5c8243
AK		 2266/**
2267 * igb_setup_rctl - configure the receive control registers
2268 * @adapter: Board private structure
2269 **/
d7ee5b3a 2270void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2271{
2272 struct e1000_hw *hw = &adapter->hw;
2273 u32 rctl;
9d5c8243
AK		 2274
2275 rctl = rd32(E1000_RCTL);
2276
2277 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2278 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2279
69d728ba 2280 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2281 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2282
87cb7e8c
AK		 2283 /*
2284 * enable stripping of CRC. It's unlikely this will break BMC
2285 * redirection as it did with e1000. Newer features require
2286 * that the HW strips the CRC.
73cd78f1 2287 */
87cb7e8c 2288 rctl |= E1000_RCTL_SECRC;
9d5c8243 2289
9b07f3d3 2290 /*
ec54d7d6 2291 * disable store bad packets and clear size bits.
9b07f3d3 2292 */
ec54d7d6 2293 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2294
6ec43fe6
AD		 2295 /* enable LPE to prevent packets larger than max_frame_size */
2296 rctl |= E1000_RCTL_LPE;
9d5c8243 2297
952f72a8
AD		 2298 /* disable queue 0 to prevent tail write w/o re-config */
2299 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2300
e1739522
AD		 2301 /* Attention!!! For SR-IOV PF driver operations you must enable
2302 * queue drop for all VF and PF queues to prevent head of line blocking
2303 * if an un-trusted VF does not provide descriptors to hardware.
2304 */
2305 if (adapter->vfs_allocated_count) {
e1739522
AD		 2306 /* set all queue drop enable bits */
2307 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2308 }
2309
9d5c8243
AK		 2310 wr32(E1000_RCTL, rctl);
2311}
2312
7d5753f0
AD		 2313static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2314 int vfn)
2315{
2316 struct e1000_hw *hw = &adapter->hw;
2317 u32 vmolr;
2318
2319 /* if it isn't the PF check to see if VFs are enabled and
2320 * increase the size to support vlan tags */
2321 if (vfn < adapter->vfs_allocated_count &&
2322 adapter->vf_data[vfn].vlans_enabled)
2323 size += VLAN_TAG_SIZE;
2324
2325 vmolr = rd32(E1000_VMOLR(vfn));
2326 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2327 vmolr |= size | E1000_VMOLR_LPE;
2328 wr32(E1000_VMOLR(vfn), vmolr);
2329
2330 return 0;
2331}
2332
e1739522
AD		 2333/**
2334 * igb_rlpml_set - set maximum receive packet size
2335 * @adapter: board private structure
2336 *
2337 * Configure maximum receivable packet size.
2338 **/
2339static void igb_rlpml_set(struct igb_adapter *adapter)
2340{
2341 u32 max_frame_size = adapter->max_frame_size;
2342 struct e1000_hw *hw = &adapter->hw;
2343 u16 pf_id = adapter->vfs_allocated_count;
2344
2345 if (adapter->vlgrp)
2346 max_frame_size += VLAN_TAG_SIZE;
2347
2348 /* if vfs are enabled we set RLPML to the largest possible request
2349 * size and set the VMOLR RLPML to the size we need */
2350 if (pf_id) {
2351 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2352 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2353 }
2354
2355 wr32(E1000_RLPML, max_frame_size);
2356}
2357
7d5753f0
AD		 2358static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2359{
2360 struct e1000_hw *hw = &adapter->hw;
2361 u32 vmolr;
2362
2363 /*
2364 * This register exists only on 82576 and newer so if we are older then
2365 * we should exit and do nothing
2366 */
2367 if (hw->mac.type < e1000_82576)
2368 return;
2369
2370 vmolr = rd32(E1000_VMOLR(vfn));
2371 vmolr |= E1000_VMOLR_AUPE | /* Accept untagged packets */
2372 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2373
2374 /* clear all bits that might not be set */
2375 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2376
2377 if (adapter->num_rx_queues > 1 && vfn == adapter->vfs_allocated_count)
2378 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2379 /*
2380 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2381 * multicast packets
2382 */
2383 if (vfn <= adapter->vfs_allocated_count)
2384 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2385
2386 wr32(E1000_VMOLR(vfn), vmolr);
2387}
2388
85b430b4
AD		 2389/**
2390 * igb_configure_rx_ring - Configure a receive ring after Reset
2391 * @adapter: board private structure
2392 * @ring: receive ring to be configured
2393 *
2394 * Configure the Rx unit of the MAC after a reset.
2395 **/
d7ee5b3a
AD		 2396void igb_configure_rx_ring(struct igb_adapter *adapter,
2397 struct igb_ring *ring)
85b430b4
AD		 2398{
2399 struct e1000_hw *hw = &adapter->hw;
2400 u64 rdba = ring->dma;
2401 int reg_idx = ring->reg_idx;
952f72a8 2402 u32 srrctl, rxdctl;
85b430b4
AD		 2403
2404 /* disable the queue */
2405 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2406 wr32(E1000_RXDCTL(reg_idx),
2407 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2408
2409 /* Set DMA base address registers */
2410 wr32(E1000_RDBAL(reg_idx),
2411 rdba & 0x00000000ffffffffULL);
2412 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2413 wr32(E1000_RDLEN(reg_idx),
2414 ring->count * sizeof(union e1000_adv_rx_desc));
2415
2416 /* initialize head and tail */
fce99e34
AD		 2417 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2418 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2419 writel(0, ring->head);
2420 writel(0, ring->tail);
85b430b4 2421
952f72a8 2422 /* set descriptor configuration */
4c844851
AD		 2423 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2424 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2425 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2426#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2427 srrctl |= IGB_RXBUFFER_16384 >>
2428 E1000_SRRCTL_BSIZEPKT_SHIFT;
2429#else
2430 srrctl |= (PAGE_SIZE / 2) >>
2431 E1000_SRRCTL_BSIZEPKT_SHIFT;
2432#endif
2433 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2434 } else {
4c844851 2435 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2436 E1000_SRRCTL_BSIZEPKT_SHIFT;
2437 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2438 }
2439
2440 wr32(E1000_SRRCTL(reg_idx), srrctl);
2441
7d5753f0
AD		 2442 /* set filtering for VMDQ pools */
2443 igb_set_vmolr(adapter, reg_idx & 0x7);
2444
85b430b4
AD		 2445 /* enable receive descriptor fetching */
2446 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2447 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2448 rxdctl &= 0xFFF00000;
2449 rxdctl |= IGB_RX_PTHRESH;
2450 rxdctl |= IGB_RX_HTHRESH << 8;
2451 rxdctl |= IGB_RX_WTHRESH << 16;
2452 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2453}
2454
9d5c8243
AK		 2455/**
2456 * igb_configure_rx - Configure receive Unit after Reset
2457 * @adapter: board private structure
2458 *
2459 * Configure the Rx unit of the MAC after a reset.
2460 **/
2461static void igb_configure_rx(struct igb_adapter *adapter)
2462{
9107584e 2463 int i;
9d5c8243 2464
68d480c4
AD		 2465 /* set UTA to appropriate mode */
2466 igb_set_uta(adapter);
2467
26ad9178
AD		 2468 /* set the correct pool for the PF default MAC address in entry 0 */
2469 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2470 adapter->vfs_allocated_count);
2471
06cf2666
AD		 2472 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2473 * the Base and Length of the Rx Descriptor Ring */
2474 for (i = 0; i < adapter->num_rx_queues; i++)
2475 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
9d5c8243
AK		 2476}
2477
2478/**
2479 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2480 * @tx_ring: Tx descriptor ring for a specific queue
2481 *
2482 * Free all transmit software resources
2483 **/
68fd9910 2484void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2485{
3b644cf6 2486 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2487
2488 vfree(tx_ring->buffer_info);
2489 tx_ring->buffer_info = NULL;
2490
439705e1
AD		 2491 /* if not set, then don't free */
2492 if (!tx_ring->desc)
2493 return;
2494
80785298
AD		 2495 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2496 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2497
2498 tx_ring->desc = NULL;
2499}
2500
2501/**
2502 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2503 * @adapter: board private structure
2504 *
2505 * Free all transmit software resources
2506 **/
2507static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2508{
2509 int i;
2510
2511 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2512 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2513}
2514
b1a436c3
AD		 2515void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2516 struct igb_buffer *buffer_info)
9d5c8243 2517{
65689fef 2518 buffer_info->dma = 0;
9d5c8243 2519 if (buffer_info->skb) {
80785298
AD		 2520 skb_dma_unmap(&tx_ring->pdev->dev,
2521 buffer_info->skb,
65689fef 2522 DMA_TO_DEVICE);
9d5c8243
AK		 2523 dev_kfree_skb_any(buffer_info->skb);
2524 buffer_info->skb = NULL;
2525 }
2526 buffer_info->time_stamp = 0;
2527 /* buffer_info must be completely set up in the transmit path */
2528}
2529
2530/**
2531 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2532 * @tx_ring: ring to be cleaned
2533 **/
3b644cf6 2534static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2535{
2536 struct igb_buffer *buffer_info;
2537 unsigned long size;
2538 unsigned int i;
2539
2540 if (!tx_ring->buffer_info)
2541 return;
2542 /* Free all the Tx ring sk_buffs */
2543
2544 for (i = 0; i < tx_ring->count; i++) {
2545 buffer_info = &tx_ring->buffer_info[i];
80785298 2546 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2547 }
2548
2549 size = sizeof(struct igb_buffer) * tx_ring->count;
2550 memset(tx_ring->buffer_info, 0, size);
2551
2552 /* Zero out the descriptor ring */
9d5c8243
AK		 2553 memset(tx_ring->desc, 0, tx_ring->size);
2554
2555 tx_ring->next_to_use = 0;
2556 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2557}
2558
2559/**
2560 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2561 * @adapter: board private structure
2562 **/
2563static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2564{
2565 int i;
2566
2567 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2568 igb_clean_tx_ring(&adapter->tx_ring[i]);
9d5c8243
AK		 2569}
2570
2571/**
2572 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2573 * @rx_ring: ring to clean the resources from
2574 *
2575 * Free all receive software resources
2576 **/
68fd9910 2577void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2578{
3b644cf6 2579 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2580
2581 vfree(rx_ring->buffer_info);
2582 rx_ring->buffer_info = NULL;
2583
439705e1
AD		 2584 /* if not set, then don't free */
2585 if (!rx_ring->desc)
2586 return;
2587
80785298
AD		 2588 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2589 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2590
2591 rx_ring->desc = NULL;
2592}
2593
2594/**
2595 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2596 * @adapter: board private structure
2597 *
2598 * Free all receive software resources
2599 **/
2600static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2601{
2602 int i;
2603
2604 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2605 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2606}
2607
2608/**
2609 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2610 * @rx_ring: ring to free buffers from
2611 **/
3b644cf6 2612static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2613{
2614 struct igb_buffer *buffer_info;
9d5c8243
AK		 2615 unsigned long size;
2616 unsigned int i;
2617
2618 if (!rx_ring->buffer_info)
2619 return;
439705e1 2620
9d5c8243
AK		 2621 /* Free all the Rx ring sk_buffs */
2622 for (i = 0; i < rx_ring->count; i++) {
2623 buffer_info = &rx_ring->buffer_info[i];
2624 if (buffer_info->dma) {
80785298
AD		 2625 pci_unmap_single(rx_ring->pdev,
2626 buffer_info->dma,
4c844851 2627 rx_ring->rx_buffer_len,
6ec43fe6 2628 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2629 buffer_info->dma = 0;
2630 }
2631
2632 if (buffer_info->skb) {
2633 dev_kfree_skb(buffer_info->skb);
2634 buffer_info->skb = NULL;
2635 }
6ec43fe6 2636 if (buffer_info->page_dma) {
80785298
AD		 2637 pci_unmap_page(rx_ring->pdev,
2638 buffer_info->page_dma,
6ec43fe6
AD		 2639 PAGE_SIZE / 2,
2640 PCI_DMA_FROMDEVICE);
2641 buffer_info->page_dma = 0;
2642 }
9d5c8243 2643 if (buffer_info->page) {
9d5c8243
AK		 2644 put_page(buffer_info->page);
2645 buffer_info->page = NULL;
bf36c1a0 2646 buffer_info->page_offset = 0;
9d5c8243
AK		 2647 }
2648 }
2649
9d5c8243
AK		 2650 size = sizeof(struct igb_buffer) * rx_ring->count;
2651 memset(rx_ring->buffer_info, 0, size);
2652
2653 /* Zero out the descriptor ring */
2654 memset(rx_ring->desc, 0, rx_ring->size);
2655
2656 rx_ring->next_to_clean = 0;
2657 rx_ring->next_to_use = 0;
9d5c8243
AK		 2658}
2659
2660/**
2661 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2662 * @adapter: board private structure
2663 **/
2664static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2665{
2666 int i;
2667
2668 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2669 igb_clean_rx_ring(&adapter->rx_ring[i]);
9d5c8243
AK		 2670}
2671
2672/**
2673 * igb_set_mac - Change the Ethernet Address of the NIC
2674 * @netdev: network interface device structure
2675 * @p: pointer to an address structure
2676 *
2677 * Returns 0 on success, negative on failure
2678 **/
2679static int igb_set_mac(struct net_device *netdev, void *p)
2680{
2681 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2682 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2683 struct sockaddr *addr = p;
2684
2685 if (!is_valid_ether_addr(addr->sa_data))
2686 return -EADDRNOTAVAIL;
2687
2688 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2689 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2690
26ad9178
AD		 2691 /* set the correct pool for the new PF MAC address in entry 0 */
2692 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2693 adapter->vfs_allocated_count);
e1739522 2694
9d5c8243
AK		 2695 return 0;
2696}
2697
2698/**
68d480c4 2699 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2700 * @netdev: network interface device structure
2701 *
68d480c4
AD		 2702 * Writes multicast address list to the MTA hash table.
2703 * Returns: -ENOMEM on failure
2704 * 0 on no addresses written
2705 * X on writing X addresses to MTA
9d5c8243 2706 **/
68d480c4 2707static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2708{
2709 struct igb_adapter *adapter = netdev_priv(netdev);
2710 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2711 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4
AD		 2712 u8 *mta_list;
2713 u32 vmolr = 0;
9d5c8243
AK		 2714 int i;
2715
68d480c4
AD		 2716 if (!netdev->mc_count) {
2717 /* nothing to program, so clear mc list */
2718 igb_update_mc_addr_list(hw, NULL, 0);
2719 igb_restore_vf_multicasts(adapter);
2720 return 0;
2721 }
9d5c8243 2722
68d480c4
AD		 2723 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2724 if (!mta_list)
2725 return -ENOMEM;
ff41f8dc 2726
68d480c4
AD		 2727 /* set vmolr receive overflow multicast bit */
2728 vmolr |= E1000_VMOLR_ROMPE;
2729
2730 /* The shared function expects a packed array of only addresses. */
2731 mc_ptr = netdev->mc_list;
2732
2733 for (i = 0; i < netdev->mc_count; i++) {
2734 if (!mc_ptr)
2735 break;
2736 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2737 mc_ptr = mc_ptr->next;
746b9f02 2738 }
68d480c4
AD		 2739 igb_update_mc_addr_list(hw, mta_list, i);
2740 kfree(mta_list);
2741
2742 return netdev->mc_count;
2743}
2744
2745/**
2746 * igb_write_uc_addr_list - write unicast addresses to RAR table
2747 * @netdev: network interface device structure
2748 *
2749 * Writes unicast address list to the RAR table.
2750 * Returns: -ENOMEM on failure/insufficient address space
2751 * 0 on no addresses written
2752 * X on writing X addresses to the RAR table
2753 **/
2754static int igb_write_uc_addr_list(struct net_device *netdev)
2755{
2756 struct igb_adapter *adapter = netdev_priv(netdev);
2757 struct e1000_hw *hw = &adapter->hw;
2758 unsigned int vfn = adapter->vfs_allocated_count;
2759 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2760 int count = 0;
2761
2762 /* return ENOMEM indicating insufficient memory for addresses */
2763 if (netdev->uc.count > rar_entries)
2764 return -ENOMEM;
9d5c8243 2765
ff41f8dc
AD		 2766 if (netdev->uc.count && rar_entries) {
2767 struct netdev_hw_addr *ha;
2768 list_for_each_entry(ha, &netdev->uc.list, list) {
2769 if (!rar_entries)
2770 break;
26ad9178
AD		 2771 igb_rar_set_qsel(adapter, ha->addr,
2772 rar_entries--,
68d480c4
AD		 2773 vfn);
2774 count++;
ff41f8dc
AD		 2775 }
2776 }
2777 /* write the addresses in reverse order to avoid write combining */
2778 for (; rar_entries > 0 ; rar_entries--) {
2779 wr32(E1000_RAH(rar_entries), 0);
2780 wr32(E1000_RAL(rar_entries), 0);
2781 }
2782 wrfl();
2783
68d480c4
AD		 2784 return count;
2785}
2786
2787/**
2788 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2789 * @netdev: network interface device structure
2790 *
2791 * The set_rx_mode entry point is called whenever the unicast or multicast
2792 * address lists or the network interface flags are updated. This routine is
2793 * responsible for configuring the hardware for proper unicast, multicast,
2794 * promiscuous mode, and all-multi behavior.
2795 **/
2796static void igb_set_rx_mode(struct net_device *netdev)
2797{
2798 struct igb_adapter *adapter = netdev_priv(netdev);
2799 struct e1000_hw *hw = &adapter->hw;
2800 unsigned int vfn = adapter->vfs_allocated_count;
2801 u32 rctl, vmolr = 0;
2802 int count;
2803
2804 /* Check for Promiscuous and All Multicast modes */
2805 rctl = rd32(E1000_RCTL);
2806
2807 /* clear the effected bits */
2808 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2809
2810 if (netdev->flags & IFF_PROMISC) {
2811 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2812 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2813 } else {
2814 if (netdev->flags & IFF_ALLMULTI) {
2815 rctl |= E1000_RCTL_MPE;
2816 vmolr |= E1000_VMOLR_MPME;
2817 } else {
2818 /*
2819 * Write addresses to the MTA, if the attempt fails
2820 * then we should just turn on promiscous mode so
2821 * that we can at least receive multicast traffic
2822 */
2823 count = igb_write_mc_addr_list(netdev);
2824 if (count < 0) {
2825 rctl |= E1000_RCTL_MPE;
2826 vmolr |= E1000_VMOLR_MPME;
2827 } else if (count) {
2828 vmolr |= E1000_VMOLR_ROMPE;
2829 }
2830 }
2831 /*
2832 * Write addresses to available RAR registers, if there is not
2833 * sufficient space to store all the addresses then enable
2834 * unicast promiscous mode
2835 */
2836 count = igb_write_uc_addr_list(netdev);
2837 if (count < 0) {
2838 rctl |= E1000_RCTL_UPE;
2839 vmolr |= E1000_VMOLR_ROPE;
2840 }
2841 rctl |= E1000_RCTL_VFE;
28fc06f5 2842 }
68d480c4 2843 wr32(E1000_RCTL, rctl);
28fc06f5 2844
68d480c4
AD		 2845 /*
2846 * In order to support SR-IOV and eventually VMDq it is necessary to set
2847 * the VMOLR to enable the appropriate modes. Without this workaround
2848 * we will have issues with VLAN tag stripping not being done for frames
2849 * that are only arriving because we are the default pool
2850 */
2851 if (hw->mac.type < e1000_82576)
28fc06f5 2852 return;
9d5c8243 2853
68d480c4
AD		 2854 vmolr |= rd32(E1000_VMOLR(vfn)) &
2855 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2856 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 2857 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 2858}
2859
2860/* Need to wait a few seconds after link up to get diagnostic information from
2861 * the phy */
2862static void igb_update_phy_info(unsigned long data)
2863{
2864 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 2865 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 2866}
2867
4d6b725e
AD		 2868/**
2869 * igb_has_link - check shared code for link and determine up/down
2870 * @adapter: pointer to driver private info
2871 **/
2872static bool igb_has_link(struct igb_adapter *adapter)
2873{
2874 struct e1000_hw *hw = &adapter->hw;
2875 bool link_active = false;
2876 s32 ret_val = 0;
2877
2878 /* get_link_status is set on LSC (link status) interrupt or
2879 * rx sequence error interrupt. get_link_status will stay
2880 * false until the e1000_check_for_link establishes link
2881 * for copper adapters ONLY
2882 */
2883 switch (hw->phy.media_type) {
2884 case e1000_media_type_copper:
2885 if (hw->mac.get_link_status) {
2886 ret_val = hw->mac.ops.check_for_link(hw);
2887 link_active = !hw->mac.get_link_status;
2888 } else {
2889 link_active = true;
2890 }
2891 break;
4d6b725e
AD		 2892 case e1000_media_type_internal_serdes:
2893 ret_val = hw->mac.ops.check_for_link(hw);
2894 link_active = hw->mac.serdes_has_link;
2895 break;
2896 default:
2897 case e1000_media_type_unknown:
2898 break;
2899 }
2900
2901 return link_active;
2902}
2903
9d5c8243
AK		 2904/**
2905 * igb_watchdog - Timer Call-back
2906 * @data: pointer to adapter cast into an unsigned long
2907 **/
2908static void igb_watchdog(unsigned long data)
2909{
2910 struct igb_adapter *adapter = (struct igb_adapter *)data;
2911 /* Do the rest outside of interrupt context */
2912 schedule_work(&adapter->watchdog_task);
2913}
2914
2915static void igb_watchdog_task(struct work_struct *work)
2916{
2917 struct igb_adapter *adapter = container_of(work,
2918 struct igb_adapter, watchdog_task);
2919 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2920 struct net_device *netdev = adapter->netdev;
2921 struct igb_ring *tx_ring = adapter->tx_ring;
9d5c8243 2922 u32 link;
7a6ea550 2923 int i;
9d5c8243 2924
4d6b725e 2925 link = igb_has_link(adapter);
9d5c8243
AK		 2926 if (link) {
2927 if (!netif_carrier_ok(netdev)) {
2928 u32 ctrl;
330a6d6a
AD		 2929 hw->mac.ops.get_speed_and_duplex(hw,
2930 &adapter->link_speed,
2931 &adapter->link_duplex);
9d5c8243
AK		 2932
2933 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 2934 /* Links status message must follow this format */
2935 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 2936 "Flow Control: %s\n",
527d47c1 2937 netdev->name,
9d5c8243
AK		 2938 adapter->link_speed,
2939 adapter->link_duplex == FULL_DUPLEX ?
2940 "Full Duplex" : "Half Duplex",
2941 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2942 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2943 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2944 E1000_CTRL_TFCE) ? "TX" : "None")));
2945
2946 /* tweak tx_queue_len according to speed/duplex and
2947 * adjust the timeout factor */
2948 netdev->tx_queue_len = adapter->tx_queue_len;
2949 adapter->tx_timeout_factor = 1;
2950 switch (adapter->link_speed) {
2951 case SPEED_10:
2952 netdev->tx_queue_len = 10;
2953 adapter->tx_timeout_factor = 14;
2954 break;
2955 case SPEED_100:
2956 netdev->tx_queue_len = 100;
2957 /* maybe add some timeout factor ? */
2958 break;
2959 }
2960
2961 netif_carrier_on(netdev);
9d5c8243 2962
4ae196df
AD		 2963 igb_ping_all_vfs(adapter);
2964
4b1a9877 2965 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2966 if (!test_bit(__IGB_DOWN, &adapter->state))
2967 mod_timer(&adapter->phy_info_timer,
2968 round_jiffies(jiffies + 2 * HZ));
2969 }
2970 } else {
2971 if (netif_carrier_ok(netdev)) {
2972 adapter->link_speed = 0;
2973 adapter->link_duplex = 0;
527d47c1
AD		 2974 /* Links status message must follow this format */
2975 printk(KERN_INFO "igb: %s NIC Link is Down\n",
2976 netdev->name);
9d5c8243 2977 netif_carrier_off(netdev);
4b1a9877 2978
4ae196df
AD		 2979 igb_ping_all_vfs(adapter);
2980
4b1a9877 2981 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2982 if (!test_bit(__IGB_DOWN, &adapter->state))
2983 mod_timer(&adapter->phy_info_timer,
2984 round_jiffies(jiffies + 2 * HZ));
2985 }
2986 }
2987
9d5c8243 2988 igb_update_stats(adapter);
645a3abd 2989 igb_update_adaptive(hw);
9d5c8243
AK		 2990
2991 if (!netif_carrier_ok(netdev)) {
c493ea45 2992 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
9d5c8243
AK		 2993 /* We've lost link, so the controller stops DMA,
2994 * but we've got queued Tx work that's never going
2995 * to get done, so reset controller to flush Tx.
2996 * (Do the reset outside of interrupt context). */
2997 adapter->tx_timeout_count++;
2998 schedule_work(&adapter->reset_task);
c2d5ab49
JB		 2999 /* return immediately since reset is imminent */
3000 return;
9d5c8243
AK		 3001 }
3002 }
3003
f7ba205e
AD		 3004 /* Force detection of hung controller every watchdog period */
3005 for (i = 0; i < adapter->num_tx_queues; i++)
3006 adapter->tx_ring[i].detect_tx_hung = true;
3007
9d5c8243 3008 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3009 if (adapter->msix_entries) {
047e0030
AD		 3010 u32 eics = 0;
3011 for (i = 0; i < adapter->num_q_vectors; i++) {
3012 struct igb_q_vector *q_vector = adapter->q_vector[i];
3013 eics |= q_vector->eims_value;
3014 }
7a6ea550
AD		 3015 wr32(E1000_EICS, eics);
3016 } else {
3017 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3018 }
9d5c8243 3019
9d5c8243
AK		 3020 /* Reset the timer */
3021 if (!test_bit(__IGB_DOWN, &adapter->state))
3022 mod_timer(&adapter->watchdog_timer,
3023 round_jiffies(jiffies + 2 * HZ));
3024}
3025
3026enum latency_range {
3027 lowest_latency = 0,
3028 low_latency = 1,
3029 bulk_latency = 2,
3030 latency_invalid = 255
3031};
3032
6eb5a7f1
AD		 3033/**
3034 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3035 *
3036 * Stores a new ITR value based on strictly on packet size. This
3037 * algorithm is less sophisticated than that used in igb_update_itr,
3038 * due to the difficulty of synchronizing statistics across multiple
3039 * receive rings. The divisors and thresholds used by this fuction
3040 * were determined based on theoretical maximum wire speed and testing
3041 * data, in order to minimize response time while increasing bulk
3042 * throughput.
3043 * This functionality is controlled by the InterruptThrottleRate module
3044 * parameter (see igb_param.c)
3045 * NOTE: This function is called only when operating in a multiqueue
3046 * receive environment.
047e0030 3047 * @q_vector: pointer to q_vector
6eb5a7f1 3048 **/
047e0030 3049static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3050{
047e0030 3051 int new_val = q_vector->itr_val;
6eb5a7f1 3052 int avg_wire_size = 0;
047e0030 3053 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3054
6eb5a7f1
AD		 3055 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3056 * ints/sec - ITR timer value of 120 ticks.
3057 */
3058 if (adapter->link_speed != SPEED_1000) {
047e0030 3059 new_val = 976;
6eb5a7f1 3060 goto set_itr_val;
9d5c8243 3061 }
047e0030
AD		 3062
3063 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3064 struct igb_ring *ring = q_vector->rx_ring;
3065 avg_wire_size = ring->total_bytes / ring->total_packets;
3066 }
3067
3068 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3069 struct igb_ring *ring = q_vector->tx_ring;
3070 avg_wire_size = max_t(u32, avg_wire_size,
3071 (ring->total_bytes /
3072 ring->total_packets));
3073 }
3074
3075 /* if avg_wire_size isn't set no work was done */
3076 if (!avg_wire_size)
3077 goto clear_counts;
9d5c8243 3078
6eb5a7f1
AD		 3079 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3080 avg_wire_size += 24;
3081
3082 /* Don't starve jumbo frames */
3083 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3084
6eb5a7f1
AD		 3085 /* Give a little boost to mid-size frames */
3086 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3087 new_val = avg_wire_size / 3;
3088 else
3089 new_val = avg_wire_size / 2;
9d5c8243 3090
6eb5a7f1 3091set_itr_val:
047e0030
AD		 3092 if (new_val != q_vector->itr_val) {
3093 q_vector->itr_val = new_val;
3094 q_vector->set_itr = 1;
9d5c8243 3095 }
6eb5a7f1 3096clear_counts:
047e0030
AD		 3097 if (q_vector->rx_ring) {
3098 q_vector->rx_ring->total_bytes = 0;
3099 q_vector->rx_ring->total_packets = 0;
3100 }
3101 if (q_vector->tx_ring) {
3102 q_vector->tx_ring->total_bytes = 0;
3103 q_vector->tx_ring->total_packets = 0;
3104 }
9d5c8243
AK		 3105}
3106
3107/**
3108 * igb_update_itr - update the dynamic ITR value based on statistics
3109 * Stores a new ITR value based on packets and byte
3110 * counts during the last interrupt. The advantage of per interrupt
3111 * computation is faster updates and more accurate ITR for the current
3112 * traffic pattern. Constants in this function were computed
3113 * based on theoretical maximum wire speed and thresholds were set based
3114 * on testing data as well as attempting to minimize response time
3115 * while increasing bulk throughput.
3116 * this functionality is controlled by the InterruptThrottleRate module
3117 * parameter (see igb_param.c)
3118 * NOTE: These calculations are only valid when operating in a single-
3119 * queue environment.
3120 * @adapter: pointer to adapter
047e0030 3121 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3122 * @packets: the number of packets during this measurement interval
3123 * @bytes: the number of bytes during this measurement interval
3124 **/
3125static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3126 int packets, int bytes)
3127{
3128 unsigned int retval = itr_setting;
3129
3130 if (packets == 0)
3131 goto update_itr_done;
3132
3133 switch (itr_setting) {
3134 case lowest_latency:
3135 /* handle TSO and jumbo frames */
3136 if (bytes/packets > 8000)
3137 retval = bulk_latency;
3138 else if ((packets < 5) && (bytes > 512))
3139 retval = low_latency;
3140 break;
3141 case low_latency: /* 50 usec aka 20000 ints/s */
3142 if (bytes > 10000) {
3143 /* this if handles the TSO accounting */
3144 if (bytes/packets > 8000) {
3145 retval = bulk_latency;
3146 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3147 retval = bulk_latency;
3148 } else if ((packets > 35)) {
3149 retval = lowest_latency;
3150 }
3151 } else if (bytes/packets > 2000) {
3152 retval = bulk_latency;
3153 } else if (packets <= 2 && bytes < 512) {
3154 retval = lowest_latency;
3155 }
3156 break;
3157 case bulk_latency: /* 250 usec aka 4000 ints/s */
3158 if (bytes > 25000) {
3159 if (packets > 35)
3160 retval = low_latency;
1e5c3d21 3161 } else if (bytes < 1500) {
9d5c8243
AK		 3162 retval = low_latency;
3163 }
3164 break;
3165 }
3166
3167update_itr_done:
3168 return retval;
3169}
3170
6eb5a7f1 3171static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3172{
047e0030 3173 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3174 u16 current_itr;
047e0030 3175 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3176
3177 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3178 if (adapter->link_speed != SPEED_1000) {
3179 current_itr = 0;
3180 new_itr = 4000;
3181 goto set_itr_now;
3182 }
3183
3184 adapter->rx_itr = igb_update_itr(adapter,
3185 adapter->rx_itr,
3186 adapter->rx_ring->total_packets,
3187 adapter->rx_ring->total_bytes);
9d5c8243 3188
047e0030
AD		 3189 adapter->tx_itr = igb_update_itr(adapter,
3190 adapter->tx_itr,
3191 adapter->tx_ring->total_packets,
3192 adapter->tx_ring->total_bytes);
3193 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3194
6eb5a7f1 3195 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3196 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3197 current_itr = low_latency;
3198
9d5c8243
AK		 3199 switch (current_itr) {
3200 /* counts and packets in update_itr are dependent on these numbers */
3201 case lowest_latency:
78b1f607 3202 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3203 break;
3204 case low_latency:
78b1f607 3205 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3206 break;
3207 case bulk_latency:
78b1f607 3208 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3209 break;
3210 default:
3211 break;
3212 }
3213
3214set_itr_now:
6eb5a7f1
AD		 3215 adapter->rx_ring->total_bytes = 0;
3216 adapter->rx_ring->total_packets = 0;
047e0030
AD		 3217 adapter->tx_ring->total_bytes = 0;
3218 adapter->tx_ring->total_packets = 0;
6eb5a7f1 3219
047e0030 3220 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3221 /* this attempts to bias the interrupt rate towards Bulk
3222 * by adding intermediate steps when interrupt rate is
3223 * increasing */
047e0030
AD		 3224 new_itr = new_itr > q_vector->itr_val ?
3225 max((new_itr * q_vector->itr_val) /
3226 (new_itr + (q_vector->itr_val >> 2)),
3227 new_itr) :
9d5c8243
AK		 3228 new_itr;
3229 /* Don't write the value here; it resets the adapter's
3230 * internal timer, and causes us to delay far longer than
3231 * we should between interrupts. Instead, we write the ITR
3232 * value at the beginning of the next interrupt so the timing
3233 * ends up being correct.
3234 */
047e0030
AD		 3235 q_vector->itr_val = new_itr;
3236 q_vector->set_itr = 1;
9d5c8243
AK		 3237 }
3238
3239 return;
3240}
3241
9d5c8243
AK		 3242#define IGB_TX_FLAGS_CSUM 0x00000001
3243#define IGB_TX_FLAGS_VLAN 0x00000002
3244#define IGB_TX_FLAGS_TSO 0x00000004
3245#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3246#define IGB_TX_FLAGS_TSTAMP 0x00000010
3247#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3248#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3249
85ad76b2 3250static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3251 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3252{
3253 struct e1000_adv_tx_context_desc *context_desc;
3254 unsigned int i;
3255 int err;
3256 struct igb_buffer *buffer_info;
3257 u32 info = 0, tu_cmd = 0;
3258 u32 mss_l4len_idx, l4len;
3259 *hdr_len = 0;
3260
3261 if (skb_header_cloned(skb)) {
3262 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3263 if (err)
3264 return err;
3265 }
3266
3267 l4len = tcp_hdrlen(skb);
3268 *hdr_len += l4len;
3269
3270 if (skb->protocol == htons(ETH_P_IP)) {
3271 struct iphdr *iph = ip_hdr(skb);
3272 iph->tot_len = 0;
3273 iph->check = 0;
3274 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3275 iph->daddr, 0,
3276 IPPROTO_TCP,
3277 0);
3278 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3279 ipv6_hdr(skb)->payload_len = 0;
3280 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3281 &ipv6_hdr(skb)->daddr,
3282 0, IPPROTO_TCP, 0);
3283 }
3284
3285 i = tx_ring->next_to_use;
3286
3287 buffer_info = &tx_ring->buffer_info[i];
3288 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3289 /* VLAN MACLEN IPLEN */
3290 if (tx_flags & IGB_TX_FLAGS_VLAN)
3291 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3292 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3293 *hdr_len += skb_network_offset(skb);
3294 info |= skb_network_header_len(skb);
3295 *hdr_len += skb_network_header_len(skb);
3296 context_desc->vlan_macip_lens = cpu_to_le32(info);
3297
3298 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3299 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3300
3301 if (skb->protocol == htons(ETH_P_IP))
3302 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3303 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3304
3305 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3306
3307 /* MSS L4LEN IDX */
3308 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3309 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3310
73cd78f1 3311 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3312 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3313 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3314
3315 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3316 context_desc->seqnum_seed = 0;
3317
3318 buffer_info->time_stamp = jiffies;
0e014cb1 3319 buffer_info->next_to_watch = i;
9d5c8243
AK		 3320 buffer_info->dma = 0;
3321 i++;
3322 if (i == tx_ring->count)
3323 i = 0;
3324
3325 tx_ring->next_to_use = i;
3326
3327 return true;
3328}
3329
85ad76b2
AD		 3330static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3331 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3332{
3333 struct e1000_adv_tx_context_desc *context_desc;
80785298 3334 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3335 struct igb_buffer *buffer_info;
3336 u32 info = 0, tu_cmd = 0;
80785298 3337 unsigned int i;
9d5c8243
AK		 3338
3339 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3340 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3341 i = tx_ring->next_to_use;
3342 buffer_info = &tx_ring->buffer_info[i];
3343 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3344
3345 if (tx_flags & IGB_TX_FLAGS_VLAN)
3346 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3347
9d5c8243
AK		 3348 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3349 if (skb->ip_summed == CHECKSUM_PARTIAL)
3350 info |= skb_network_header_len(skb);
3351
3352 context_desc->vlan_macip_lens = cpu_to_le32(info);
3353
3354 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3355
3356 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3357 __be16 protocol;
3358
3359 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3360 const struct vlan_ethhdr *vhdr =
3361 (const struct vlan_ethhdr*)skb->data;
3362
3363 protocol = vhdr->h_vlan_encapsulated_proto;
3364 } else {
3365 protocol = skb->protocol;
3366 }
3367
3368 switch (protocol) {
09640e63 3369 case cpu_to_be16(ETH_P_IP):
9d5c8243 3370 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3371 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3372 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3373 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3374 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3375 break;
09640e63 3376 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3377 /* XXX what about other V6 headers?? */
3378 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3379 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3380 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3381 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3382 break;
3383 default:
3384 if (unlikely(net_ratelimit()))
80785298 3385 dev_warn(&pdev->dev,
44b0cda3
MW		 3386 "partial checksum but proto=%x!\n",
3387 skb->protocol);
3388 break;
3389 }
9d5c8243
AK		 3390 }
3391
3392 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3393 context_desc->seqnum_seed = 0;
85ad76b2 3394 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3395 context_desc->mss_l4len_idx =
85ad76b2 3396 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3397
3398 buffer_info->time_stamp = jiffies;
0e014cb1 3399 buffer_info->next_to_watch = i;
9d5c8243
AK		 3400 buffer_info->dma = 0;
3401
3402 i++;
3403 if (i == tx_ring->count)
3404 i = 0;
3405 tx_ring->next_to_use = i;
3406
3407 return true;
3408 }
9d5c8243
AK		 3409 return false;
3410}
3411
3412#define IGB_MAX_TXD_PWR 16
3413#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3414
80785298 3415static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3416 unsigned int first)
9d5c8243
AK		 3417{
3418 struct igb_buffer *buffer_info;
80785298 3419 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3420 unsigned int len = skb_headlen(skb);
3421 unsigned int count = 0, i;
3422 unsigned int f;
65689fef 3423 dma_addr_t *map;
9d5c8243
AK		 3424
3425 i = tx_ring->next_to_use;
3426
80785298
AD		 3427 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3428 dev_err(&pdev->dev, "TX DMA map failed\n");
65689fef
AD		 3429 return 0;
3430 }
3431
3432 map = skb_shinfo(skb)->dma_maps;
3433
9d5c8243
AK		 3434 buffer_info = &tx_ring->buffer_info[i];
3435 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3436 buffer_info->length = len;
3437 /* set time_stamp *before* dma to help avoid a possible race */
3438 buffer_info->time_stamp = jiffies;
0e014cb1 3439 buffer_info->next_to_watch = i;
042a53a9 3440 buffer_info->dma = skb_shinfo(skb)->dma_head;
9d5c8243
AK		 3441
3442 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3443 struct skb_frag_struct *frag;
3444
65689fef
AD		 3445 i++;
3446 if (i == tx_ring->count)
3447 i = 0;
3448
9d5c8243
AK		 3449 frag = &skb_shinfo(skb)->frags[f];
3450 len = frag->size;
3451
3452 buffer_info = &tx_ring->buffer_info[i];
3453 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3454 buffer_info->length = len;
3455 buffer_info->time_stamp = jiffies;
0e014cb1 3456 buffer_info->next_to_watch = i;
65689fef 3457 buffer_info->dma = map[count];
9d5c8243 3458 count++;
9d5c8243
AK		 3459 }
3460
9d5c8243 3461 tx_ring->buffer_info[i].skb = skb;
0e014cb1 3462 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3463
cdfd01fc 3464 return ++count;
9d5c8243
AK		 3465}
3466
85ad76b2 3467static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3468 int tx_flags, int count, u32 paylen,
3469 u8 hdr_len)
3470{
cdfd01fc 3471 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3472 struct igb_buffer *buffer_info;
3473 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3474 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3475
3476 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3477 E1000_ADVTXD_DCMD_DEXT);
3478
3479 if (tx_flags & IGB_TX_FLAGS_VLAN)
3480 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3481
33af6bcc
PO		 3482 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3483 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3484
9d5c8243
AK		 3485 if (tx_flags & IGB_TX_FLAGS_TSO) {
3486 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3487
3488 /* insert tcp checksum */
3489 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3490
3491 /* insert ip checksum */
3492 if (tx_flags & IGB_TX_FLAGS_IPV4)
3493 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3494
3495 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3496 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3497 }
3498
85ad76b2
AD		 3499 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3500 (tx_flags & (IGB_TX_FLAGS_CSUM |
3501 IGB_TX_FLAGS_TSO |
7dfc16fa 3502 IGB_TX_FLAGS_VLAN)))
85ad76b2 3503 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3504
3505 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3506
cdfd01fc 3507 do {
9d5c8243
AK		 3508 buffer_info = &tx_ring->buffer_info[i];
3509 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3510 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3511 tx_desc->read.cmd_type_len =
3512 cpu_to_le32(cmd_type_len | buffer_info->length);
3513 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3514 count--;
9d5c8243
AK		 3515 i++;
3516 if (i == tx_ring->count)
3517 i = 0;
cdfd01fc 3518 } while (count > 0);
9d5c8243 3519
85ad76b2 3520 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3521 /* Force memory writes to complete before letting h/w
3522 * know there are new descriptors to fetch. (Only
3523 * applicable for weak-ordered memory model archs,
3524 * such as IA-64). */
3525 wmb();
3526
3527 tx_ring->next_to_use = i;
fce99e34 3528 writel(i, tx_ring->tail);
9d5c8243
AK		 3529 /* we need this if more than one processor can write to our tail
3530 * at a time, it syncronizes IO on IA64/Altix systems */
3531 mmiowb();
3532}
3533
e694e964 3534static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3535{
e694e964
AD		 3536 struct net_device *netdev = tx_ring->netdev;
3537
661086df 3538 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3539
9d5c8243
AK		 3540 /* Herbert's original patch had:
3541 * smp_mb__after_netif_stop_queue();
3542 * but since that doesn't exist yet, just open code it. */
3543 smp_mb();
3544
3545 /* We need to check again in a case another CPU has just
3546 * made room available. */
c493ea45 3547 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3548 return -EBUSY;
3549
3550 /* A reprieve! */
661086df 3551 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3552 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3553 return 0;
3554}
3555
e694e964 3556static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3557{
c493ea45 3558 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3559 return 0;
e694e964 3560 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3561}
3562
b1a436c3
AD		 3563netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3564 struct igb_ring *tx_ring)
9d5c8243 3565{
e694e964 3566 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
0e014cb1 3567 unsigned int first;
9d5c8243 3568 unsigned int tx_flags = 0;
9d5c8243 3569 u8 hdr_len = 0;
cdfd01fc 3570 int tso = 0, count;
c5b9bd5e 3571 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3572
9d5c8243
AK		 3573 /* need: 1 descriptor per page,
3574 * + 2 desc gap to keep tail from touching head,
3575 * + 1 desc for skb->data,
3576 * + 1 desc for context descriptor,
3577 * otherwise try next time */
e694e964 3578 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3579 /* this is a hard error */
9d5c8243
AK		 3580 return NETDEV_TX_BUSY;
3581 }
33af6bcc 3582
33af6bcc
PO		 3583 if (unlikely(shtx->hardware)) {
3584 shtx->in_progress = 1;
3585 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3586 }
9d5c8243 3587
cdfd01fc 3588 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3589 tx_flags |= IGB_TX_FLAGS_VLAN;
3590 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3591 }
3592
661086df
PWJ		 3593 if (skb->protocol == htons(ETH_P_IP))
3594 tx_flags |= IGB_TX_FLAGS_IPV4;
3595
0e014cb1 3596 first = tx_ring->next_to_use;
85ad76b2
AD		 3597 if (skb_is_gso(skb)) {
3598 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3599
85ad76b2
AD		 3600 if (tso < 0) {
3601 dev_kfree_skb_any(skb);
3602 return NETDEV_TX_OK;
3603 }
9d5c8243
AK		 3604 }
3605
3606 if (tso)
3607 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3608 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3609 (skb->ip_summed == CHECKSUM_PARTIAL))
3610 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3611
65689fef 3612 /*
cdfd01fc 3613 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3614 * has occured and we need to rewind the descriptor queue
3615 */
80785298 3616 count = igb_tx_map_adv(tx_ring, skb, first);
cdfd01fc 3617 if (count <= 0) {
65689fef
AD		 3618 dev_kfree_skb_any(skb);
3619 tx_ring->buffer_info[first].time_stamp = 0;
3620 tx_ring->next_to_use = first;
85ad76b2 3621 return NETDEV_TX_OK;
65689fef 3622 }
9d5c8243 3623
85ad76b2
AD		 3624 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3625
3626 /* Make sure there is space in the ring for the next send. */
e694e964 3627 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3628
9d5c8243
AK		 3629 return NETDEV_TX_OK;
3630}
3631
3b29a56d
SH		 3632static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3633 struct net_device *netdev)
9d5c8243
AK		 3634{
3635 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3636 struct igb_ring *tx_ring;
661086df 3637 int r_idx = 0;
b1a436c3
AD		 3638
3639 if (test_bit(__IGB_DOWN, &adapter->state)) {
3640 dev_kfree_skb_any(skb);
3641 return NETDEV_TX_OK;
3642 }
3643
3644 if (skb->len <= 0) {
3645 dev_kfree_skb_any(skb);
3646 return NETDEV_TX_OK;
3647 }
3648
1bfaf07b 3649 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3650 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3651
3652 /* This goes back to the question of how to logically map a tx queue
3653 * to a flow. Right now, performance is impacted slightly negatively
3654 * if using multiple tx queues. If the stack breaks away from a
3655 * single qdisc implementation, we can look at this again. */
e694e964 3656 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3657}
3658
3659/**
3660 * igb_tx_timeout - Respond to a Tx Hang
3661 * @netdev: network interface device structure
3662 **/
3663static void igb_tx_timeout(struct net_device *netdev)
3664{
3665 struct igb_adapter *adapter = netdev_priv(netdev);
3666 struct e1000_hw *hw = &adapter->hw;
3667
3668 /* Do the reset outside of interrupt context */
3669 adapter->tx_timeout_count++;
f7ba205e 3670
9d5c8243 3671 schedule_work(&adapter->reset_task);
265de409
AD		 3672 wr32(E1000_EICS,
3673 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3674}
3675
3676static void igb_reset_task(struct work_struct *work)
3677{
3678 struct igb_adapter *adapter;
3679 adapter = container_of(work, struct igb_adapter, reset_task);
3680
3681 igb_reinit_locked(adapter);
3682}
3683
3684/**
3685 * igb_get_stats - Get System Network Statistics
3686 * @netdev: network interface device structure
3687 *
3688 * Returns the address of the device statistics structure.
3689 * The statistics are actually updated from the timer callback.
3690 **/
73cd78f1 3691static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3692{
9d5c8243 3693 /* only return the current stats */
8d24e933 3694 return &netdev->stats;
9d5c8243
AK		 3695}
3696
3697/**
3698 * igb_change_mtu - Change the Maximum Transfer Unit
3699 * @netdev: network interface device structure
3700 * @new_mtu: new value for maximum frame size
3701 *
3702 * Returns 0 on success, negative on failure
3703 **/
3704static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3705{
3706 struct igb_adapter *adapter = netdev_priv(netdev);
3707 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3708 u32 rx_buffer_len, i;
9d5c8243
AK		 3709
3710 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3711 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3712 dev_err(&adapter->pdev->dev, "Invalid MTU setting\n");
3713 return -EINVAL;
3714 }
3715
9d5c8243
AK		 3716 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3717 dev_err(&adapter->pdev->dev, "MTU > 9216 not supported.\n");
3718 return -EINVAL;
3719 }
3720
3721 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3722 msleep(1);
73cd78f1 3723
9d5c8243
AK		 3724 /* igb_down has a dependency on max_frame_size */
3725 adapter->max_frame_size = max_frame;
9d5c8243
AK		 3726 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3727 * means we reserve 2 more, this pushes us to allocate from the next
3728 * larger slab size.
3729 * i.e. RXBUFFER_2048 --> size-4096 slab
3730 */
3731
7d95b717 3732 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3733 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3734 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3735 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3736 else
4c844851
AD		 3737 rx_buffer_len = IGB_RXBUFFER_128;
3738
3739 if (netif_running(netdev))
3740 igb_down(adapter);
9d5c8243
AK		 3741
3742 dev_info(&adapter->pdev->dev, "changing MTU from %d to %d\n",
3743 netdev->mtu, new_mtu);
3744 netdev->mtu = new_mtu;
3745
4c844851
AD		 3746 for (i = 0; i < adapter->num_rx_queues; i++)
3747 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3748
9d5c8243
AK		 3749 if (netif_running(netdev))
3750 igb_up(adapter);
3751 else
3752 igb_reset(adapter);
3753
3754 clear_bit(__IGB_RESETTING, &adapter->state);
3755
3756 return 0;
3757}
3758
3759/**
3760 * igb_update_stats - Update the board statistics counters
3761 * @adapter: board private structure
3762 **/
3763
3764void igb_update_stats(struct igb_adapter *adapter)
3765{
8d24e933 3766 struct net_device *netdev = adapter->netdev;
9d5c8243
AK		 3767 struct e1000_hw *hw = &adapter->hw;
3768 struct pci_dev *pdev = adapter->pdev;
3f9c0164 3769 u32 rnbc;
9d5c8243 3770 u16 phy_tmp;
3f9c0164
AD		 3771 int i;
3772 u64 bytes, packets;
9d5c8243
AK		 3773
3774#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3775
3776 /*
3777 * Prevent stats update while adapter is being reset, or if the pci
3778 * connection is down.
3779 */
3780 if (adapter->link_speed == 0)
3781 return;
3782 if (pci_channel_offline(pdev))
3783 return;
3784
3f9c0164
AD		 3785 bytes = 0;
3786 packets = 0;
3787 for (i = 0; i < adapter->num_rx_queues; i++) {
3788 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3789 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3790 netdev->stats.rx_fifo_errors += rqdpc_tmp;
3791 bytes += adapter->rx_ring[i].rx_stats.bytes;
3792 packets += adapter->rx_ring[i].rx_stats.packets;
3793 }
3794
3795 netdev->stats.rx_bytes = bytes;
3796 netdev->stats.rx_packets = packets;
3797
3798 bytes = 0;
3799 packets = 0;
3800 for (i = 0; i < adapter->num_tx_queues; i++) {
3801 bytes += adapter->tx_ring[i].tx_stats.bytes;
3802 packets += adapter->tx_ring[i].tx_stats.packets;
3803 }
3804 netdev->stats.tx_bytes = bytes;
3805 netdev->stats.tx_packets = packets;
3806
3807 /* read stats registers */
9d5c8243
AK		 3808 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3809 adapter->stats.gprc += rd32(E1000_GPRC);
3810 adapter->stats.gorc += rd32(E1000_GORCL);
3811 rd32(E1000_GORCH); /* clear GORCL */
3812 adapter->stats.bprc += rd32(E1000_BPRC);
3813 adapter->stats.mprc += rd32(E1000_MPRC);
3814 adapter->stats.roc += rd32(E1000_ROC);
3815
3816 adapter->stats.prc64 += rd32(E1000_PRC64);
3817 adapter->stats.prc127 += rd32(E1000_PRC127);
3818 adapter->stats.prc255 += rd32(E1000_PRC255);
3819 adapter->stats.prc511 += rd32(E1000_PRC511);
3820 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3821 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3822 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3823 adapter->stats.sec += rd32(E1000_SEC);
3824
3825 adapter->stats.mpc += rd32(E1000_MPC);
3826 adapter->stats.scc += rd32(E1000_SCC);
3827 adapter->stats.ecol += rd32(E1000_ECOL);
3828 adapter->stats.mcc += rd32(E1000_MCC);
3829 adapter->stats.latecol += rd32(E1000_LATECOL);
3830 adapter->stats.dc += rd32(E1000_DC);
3831 adapter->stats.rlec += rd32(E1000_RLEC);
3832 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3833 adapter->stats.xontxc += rd32(E1000_XONTXC);
3834 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3835 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3836 adapter->stats.fcruc += rd32(E1000_FCRUC);
3837 adapter->stats.gptc += rd32(E1000_GPTC);
3838 adapter->stats.gotc += rd32(E1000_GOTCL);
3839 rd32(E1000_GOTCH); /* clear GOTCL */
3f9c0164
AD		 3840 rnbc = rd32(E1000_RNBC);
3841 adapter->stats.rnbc += rnbc;
3842 netdev->stats.rx_fifo_errors += rnbc;
9d5c8243
AK		 3843 adapter->stats.ruc += rd32(E1000_RUC);
3844 adapter->stats.rfc += rd32(E1000_RFC);
3845 adapter->stats.rjc += rd32(E1000_RJC);
3846 adapter->stats.tor += rd32(E1000_TORH);
3847 adapter->stats.tot += rd32(E1000_TOTH);
3848 adapter->stats.tpr += rd32(E1000_TPR);
3849
3850 adapter->stats.ptc64 += rd32(E1000_PTC64);
3851 adapter->stats.ptc127 += rd32(E1000_PTC127);
3852 adapter->stats.ptc255 += rd32(E1000_PTC255);
3853 adapter->stats.ptc511 += rd32(E1000_PTC511);
3854 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3855 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3856
3857 adapter->stats.mptc += rd32(E1000_MPTC);
3858 adapter->stats.bptc += rd32(E1000_BPTC);
3859
3860 /* used for adaptive IFS */
9d5c8243
AK		 3861 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3862 adapter->stats.tpt += hw->mac.tx_packet_delta;
3863 hw->mac.collision_delta = rd32(E1000_COLC);
3864 adapter->stats.colc += hw->mac.collision_delta;
3865
3866 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3867 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3868 adapter->stats.tncrs += rd32(E1000_TNCRS);
3869 adapter->stats.tsctc += rd32(E1000_TSCTC);
3870 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3871
3872 adapter->stats.iac += rd32(E1000_IAC);
3873 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3874 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3875 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3876 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3877 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3878 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3879 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3880 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3881
3882 /* Fill out the OS statistics structure */
8d24e933
AK		 3883 netdev->stats.multicast = adapter->stats.mprc;
3884 netdev->stats.collisions = adapter->stats.colc;
9d5c8243
AK		 3885
3886 /* Rx Errors */
3887
3888 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 3889 * our own version based on RUC and ROC */
8d24e933 3890 netdev->stats.rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 3891 adapter->stats.crcerrs + adapter->stats.algnerrc +
3892 adapter->stats.ruc + adapter->stats.roc +
3893 adapter->stats.cexterr;
8d24e933 3894 netdev->stats.rx_length_errors = adapter->stats.ruc +
9d5c8243 3895 adapter->stats.roc;
8d24e933
AK		 3896 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3897 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3898 netdev->stats.rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 3899
3900 /* Tx Errors */
8d24e933 3901 netdev->stats.tx_errors = adapter->stats.ecol +
9d5c8243 3902 adapter->stats.latecol;
8d24e933
AK		 3903 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3904 netdev->stats.tx_window_errors = adapter->stats.latecol;
3905 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 3906
3907 /* Tx Dropped needs to be maintained elsewhere */
3908
3909 /* Phy Stats */
3910 if (hw->phy.media_type == e1000_media_type_copper) {
3911 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 3912 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 3913 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3914 adapter->phy_stats.idle_errors += phy_tmp;
3915 }
3916 }
3917
3918 /* Management Stats */
3919 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3920 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3921 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3922}
3923
9d5c8243
AK		 3924static irqreturn_t igb_msix_other(int irq, void *data)
3925{
047e0030 3926 struct igb_adapter *adapter = data;
9d5c8243 3927 struct e1000_hw *hw = &adapter->hw;
844290e5 3928 u32 icr = rd32(E1000_ICR);
844290e5 3929 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 3930
047e0030 3931 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 3932 /* HW is reporting DMA is out of sync */
3933 adapter->stats.doosync++;
3934 }
eebbbdba 3935
4ae196df
AD		 3936 /* Check for a mailbox event */
3937 if (icr & E1000_ICR_VMMB)
3938 igb_msg_task(adapter);
3939
3940 if (icr & E1000_ICR_LSC) {
3941 hw->mac.get_link_status = 1;
3942 /* guard against interrupt when we're going down */
3943 if (!test_bit(__IGB_DOWN, &adapter->state))
3944 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3945 }
3946
25568a53
AD		 3947 if (adapter->vfs_allocated_count)
3948 wr32(E1000_IMS, E1000_IMS_LSC |
3949 E1000_IMS_VMMB |
3950 E1000_IMS_DOUTSYNC);
3951 else
3952 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 3953 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 3954
3955 return IRQ_HANDLED;
3956}
3957
047e0030 3958static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 3959{
047e0030 3960 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 3961
047e0030
AD		 3962 if (!q_vector->set_itr)
3963 return;
73cd78f1 3964
047e0030
AD		 3965 if (!itr_val)
3966 itr_val = 0x4;
661086df 3967
047e0030
AD		 3968 if (q_vector->itr_shift)
3969 itr_val |= itr_val << q_vector->itr_shift;
661086df 3970 else
047e0030 3971 itr_val |= 0x8000000;
661086df 3972
047e0030
AD		 3973 writel(itr_val, q_vector->itr_register);
3974 q_vector->set_itr = 0;
6eb5a7f1
AD		 3975}
3976
047e0030 3977static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 3978{
047e0030 3979 struct igb_q_vector *q_vector = data;
9d5c8243 3980
047e0030
AD		 3981 /* Write the ITR value calculated from the previous interrupt. */
3982 igb_write_itr(q_vector);
9d5c8243 3983
047e0030 3984 napi_schedule(&q_vector->napi);
844290e5 3985
047e0030 3986 return IRQ_HANDLED;
fe4506b6
JC		 3987}
3988
421e02f0 3989#ifdef CONFIG_IGB_DCA
047e0030 3990static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 3991{
047e0030 3992 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 3993 struct e1000_hw *hw = &adapter->hw;
3994 int cpu = get_cpu();
fe4506b6 3995
047e0030
AD		 3996 if (q_vector->cpu == cpu)
3997 goto out_no_update;
3998
3999 if (q_vector->tx_ring) {
4000 int q = q_vector->tx_ring->reg_idx;
4001 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4002 if (hw->mac.type == e1000_82575) {
4003 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4004 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4005 } else {
047e0030
AD		 4006 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4007 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4008 E1000_DCA_TXCTRL_CPUID_SHIFT;
4009 }
4010 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4011 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4012 }
4013 if (q_vector->rx_ring) {
4014 int q = q_vector->rx_ring->reg_idx;
4015 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4016 if (hw->mac.type == e1000_82575) {
2d064c06 4017 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4018 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4019 } else {
4020 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4021 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4022 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4023 }
fe4506b6
JC		 4024 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4025 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4026 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4027 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4028 }
047e0030
AD		 4029 q_vector->cpu = cpu;
4030out_no_update:
fe4506b6
JC		 4031 put_cpu();
4032}
4033
4034static void igb_setup_dca(struct igb_adapter *adapter)
4035{
7e0e99ef 4036 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4037 int i;
4038
7dfc16fa 4039 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4040 return;
4041
7e0e99ef
AD		 4042 /* Always use CB2 mode, difference is masked in the CB driver. */
4043 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4044
047e0030
AD		 4045 for (i = 0; i < adapter->num_q_vectors; i++) {
4046 struct igb_q_vector *q_vector = adapter->q_vector[i];
4047 q_vector->cpu = -1;
4048 igb_update_dca(q_vector);
fe4506b6
JC		 4049 }
4050}
4051
4052static int __igb_notify_dca(struct device *dev, void *data)
4053{
4054 struct net_device *netdev = dev_get_drvdata(dev);
4055 struct igb_adapter *adapter = netdev_priv(netdev);
4056 struct e1000_hw *hw = &adapter->hw;
4057 unsigned long event = *(unsigned long *)data;
4058
4059 switch (event) {
4060 case DCA_PROVIDER_ADD:
4061 /* if already enabled, don't do it again */
7dfc16fa 4062 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4063 break;
fe4506b6
JC		 4064 /* Always use CB2 mode, difference is masked
4065 * in the CB driver. */
cbd347ad 4066 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
fe4506b6 4067 if (dca_add_requester(dev) == 0) {
bbd98fe4 4068 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6
JC		 4069 dev_info(&adapter->pdev->dev, "DCA enabled\n");
4070 igb_setup_dca(adapter);
4071 break;
4072 }
4073 /* Fall Through since DCA is disabled. */
4074 case DCA_PROVIDER_REMOVE:
7dfc16fa 4075 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4076 /* without this a class_device is left
047e0030 4077 * hanging around in the sysfs model */
fe4506b6
JC		 4078 dca_remove_requester(dev);
4079 dev_info(&adapter->pdev->dev, "DCA disabled\n");
7dfc16fa 4080 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4081 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4082 }
4083 break;
4084 }
bbd98fe4 4085
fe4506b6 4086 return 0;
9d5c8243
AK		 4087}
4088
fe4506b6
JC		 4089static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4090 void *p)
4091{
4092 int ret_val;
4093
4094 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4095 __igb_notify_dca);
4096
4097 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4098}
421e02f0 4099#endif /* CONFIG_IGB_DCA */
9d5c8243 4100
4ae196df
AD		 4101static void igb_ping_all_vfs(struct igb_adapter *adapter)
4102{
4103 struct e1000_hw *hw = &adapter->hw;
4104 u32 ping;
4105 int i;
4106
4107 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4108 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4109 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4110 ping |= E1000_VT_MSGTYPE_CTS;
4111 igb_write_mbx(hw, &ping, 1, i);
4112 }
4113}
4114
7d5753f0
AD		 4115static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4116{
4117 struct e1000_hw *hw = &adapter->hw;
4118 u32 vmolr = rd32(E1000_VMOLR(vf));
4119 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4120
4121 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4122 IGB_VF_FLAG_MULTI_PROMISC);
4123 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4124
4125 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4126 vmolr |= E1000_VMOLR_MPME;
4127 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4128 } else {
4129 /*
4130 * if we have hashes and we are clearing a multicast promisc
4131 * flag we need to write the hashes to the MTA as this step
4132 * was previously skipped
4133 */
4134 if (vf_data->num_vf_mc_hashes > 30) {
4135 vmolr |= E1000_VMOLR_MPME;
4136 } else if (vf_data->num_vf_mc_hashes) {
4137 int j;
4138 vmolr |= E1000_VMOLR_ROMPE;
4139 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4140 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4141 }
4142 }
4143
4144 wr32(E1000_VMOLR(vf), vmolr);
4145
4146 /* there are flags left unprocessed, likely not supported */
4147 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4148 return -EINVAL;
4149
4150 return 0;
4151
4152}
4153
4ae196df
AD		 4154static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4155 u32 *msgbuf, u32 vf)
4156{
4157 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4158 u16 *hash_list = (u16 *)&msgbuf[1];
4159 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4160 int i;
4161
7d5753f0 4162 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4163 * to this VF for later use to restore when the PF multi cast
4164 * list changes
4165 */
4166 vf_data->num_vf_mc_hashes = n;
4167
7d5753f0
AD		 4168 /* only up to 30 hash values supported */
4169 if (n > 30)
4170 n = 30;
4171
4172 /* store the hashes for later use */
4ae196df 4173 for (i = 0; i < n; i++)
a419aef8 4174 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4175
4176 /* Flush and reset the mta with the new values */
ff41f8dc 4177 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4178
4179 return 0;
4180}
4181
4182static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4183{
4184 struct e1000_hw *hw = &adapter->hw;
4185 struct vf_data_storage *vf_data;
4186 int i, j;
4187
4188 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4189 u32 vmolr = rd32(E1000_VMOLR(i));
4190 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4191
4ae196df 4192 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4193
4194 if ((vf_data->num_vf_mc_hashes > 30) ||
4195 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4196 vmolr |= E1000_VMOLR_MPME;
4197 } else if (vf_data->num_vf_mc_hashes) {
4198 vmolr |= E1000_VMOLR_ROMPE;
4199 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4200 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4201 }
4202 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4203 }
4204}
4205
4206static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4207{
4208 struct e1000_hw *hw = &adapter->hw;
4209 u32 pool_mask, reg, vid;
4210 int i;
4211
4212 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4213
4214 /* Find the vlan filter for this id */
4215 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4216 reg = rd32(E1000_VLVF(i));
4217
4218 /* remove the vf from the pool */
4219 reg &= ~pool_mask;
4220
4221 /* if pool is empty then remove entry from vfta */
4222 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4223 (reg & E1000_VLVF_VLANID_ENABLE)) {
4224 reg = 0;
4225 vid = reg & E1000_VLVF_VLANID_MASK;
4226 igb_vfta_set(hw, vid, false);
4227 }
4228
4229 wr32(E1000_VLVF(i), reg);
4230 }
ae641bdc
AD		 4231
4232 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4233}
4234
4235static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4236{
4237 struct e1000_hw *hw = &adapter->hw;
4238 u32 reg, i;
4239
51466239
AD		 4240 /* The vlvf table only exists on 82576 hardware and newer */
4241 if (hw->mac.type < e1000_82576)
4242 return -1;
4243
4244 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4245 if (!adapter->vfs_allocated_count)
4246 return -1;
4247
4248 /* Find the vlan filter for this id */
4249 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4250 reg = rd32(E1000_VLVF(i));
4251 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4252 vid == (reg & E1000_VLVF_VLANID_MASK))
4253 break;
4254 }
4255
4256 if (add) {
4257 if (i == E1000_VLVF_ARRAY_SIZE) {
4258 /* Did not find a matching VLAN ID entry that was
4259 * enabled. Search for a free filter entry, i.e.
4260 * one without the enable bit set
4261 */
4262 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4263 reg = rd32(E1000_VLVF(i));
4264 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4265 break;
4266 }
4267 }
4268 if (i < E1000_VLVF_ARRAY_SIZE) {
4269 /* Found an enabled/available entry */
4270 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4271
4272 /* if !enabled we need to set this up in vfta */
4273 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4274 /* add VID to filter table */
4275 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4276 reg |= E1000_VLVF_VLANID_ENABLE;
4277 }
cad6d05f
AD		 4278 reg &= ~E1000_VLVF_VLANID_MASK;
4279 reg |= vid;
4ae196df 4280 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4281
4282 /* do not modify RLPML for PF devices */
4283 if (vf >= adapter->vfs_allocated_count)
4284 return 0;
4285
4286 if (!adapter->vf_data[vf].vlans_enabled) {
4287 u32 size;
4288 reg = rd32(E1000_VMOLR(vf));
4289 size = reg & E1000_VMOLR_RLPML_MASK;
4290 size += 4;
4291 reg &= ~E1000_VMOLR_RLPML_MASK;
4292 reg |= size;
4293 wr32(E1000_VMOLR(vf), reg);
4294 }
ae641bdc 4295
51466239 4296 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4297 return 0;
4298 }
4299 } else {
4300 if (i < E1000_VLVF_ARRAY_SIZE) {
4301 /* remove vf from the pool */
4302 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4303 /* if pool is empty then remove entry from vfta */
4304 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4305 reg = 0;
4306 igb_vfta_set(hw, vid, false);
4307 }
4308 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4309
4310 /* do not modify RLPML for PF devices */
4311 if (vf >= adapter->vfs_allocated_count)
4312 return 0;
4313
4314 adapter->vf_data[vf].vlans_enabled--;
4315 if (!adapter->vf_data[vf].vlans_enabled) {
4316 u32 size;
4317 reg = rd32(E1000_VMOLR(vf));
4318 size = reg & E1000_VMOLR_RLPML_MASK;
4319 size -= 4;
4320 reg &= ~E1000_VMOLR_RLPML_MASK;
4321 reg |= size;
4322 wr32(E1000_VMOLR(vf), reg);
4323 }
4ae196df
AD		 4324 return 0;
4325 }
4326 }
4327 return -1;
4328}
4329
4330static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4331{
4332 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4333 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4334
4335 return igb_vlvf_set(adapter, vid, add, vf);
4336}
4337
f2ca0dbe 4338static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4339{
f2ca0dbe
AD		 4340 /* clear all flags */
4341 adapter->vf_data[vf].flags = 0;
4342 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4343
4344 /* reset offloads to defaults */
7d5753f0 4345 igb_set_vmolr(adapter, vf);
4ae196df
AD		 4346
4347 /* reset vlans for device */
4348 igb_clear_vf_vfta(adapter, vf);
4349
4350 /* reset multicast table array for vf */
4351 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4352
4353 /* Flush and reset the mta with the new values */
ff41f8dc 4354 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4355}
4356
f2ca0dbe
AD		 4357static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4358{
4359 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4360
4361 /* generate a new mac address as we were hotplug removed/added */
4362 random_ether_addr(vf_mac);
4363
4364 /* process remaining reset events */
4365 igb_vf_reset(adapter, vf);
4366}
4367
4368static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4369{
4370 struct e1000_hw *hw = &adapter->hw;
4371 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4372 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4373 u32 reg, msgbuf[3];
4374 u8 *addr = (u8 *)(&msgbuf[1]);
4375
4376 /* process all the same items cleared in a function level reset */
f2ca0dbe 4377 igb_vf_reset(adapter, vf);
4ae196df
AD		 4378
4379 /* set vf mac address */
26ad9178 4380 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4381
4382 /* enable transmit and receive for vf */
4383 reg = rd32(E1000_VFTE);
4384 wr32(E1000_VFTE, reg | (1 << vf));
4385 reg = rd32(E1000_VFRE);
4386 wr32(E1000_VFRE, reg | (1 << vf));
4387
f2ca0dbe 4388 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4389
4390 /* reply to reset with ack and vf mac address */
4391 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4392 memcpy(addr, vf_mac, 6);
4393 igb_write_mbx(hw, msgbuf, 3, vf);
4394}
4395
4396static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4397{
f2ca0dbe
AD		 4398 unsigned char *addr = (char *)&msg[1];
4399 int err = -1;
4ae196df 4400
f2ca0dbe
AD		 4401 if (is_valid_ether_addr(addr))
4402 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4403
f2ca0dbe 4404 return err;
4ae196df
AD		 4405}
4406
4407static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4408{
4409 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4410 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4411 u32 msg = E1000_VT_MSGTYPE_NACK;
4412
4413 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4414 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4415 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4416 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4417 vf_data->last_nack = jiffies;
4ae196df
AD		 4418 }
4419}
4420
f2ca0dbe 4421static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4422{
f2ca0dbe
AD		 4423 struct pci_dev *pdev = adapter->pdev;
4424 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4425 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4426 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4427 s32 retval;
4428
f2ca0dbe 4429 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df
AD		 4430
4431 if (retval)
f2ca0dbe 4432 dev_err(&pdev->dev, "Error receiving message from VF\n");
4ae196df
AD		 4433
4434 /* this is a message we already processed, do nothing */
4435 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4436 return;
4ae196df
AD		 4437
4438 /*
4439 * until the vf completes a reset it should not be
4440 * allowed to start any configuration.
4441 */
4442
4443 if (msgbuf[0] == E1000_VF_RESET) {
4444 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4445 return;
4ae196df
AD		 4446 }
4447
f2ca0dbe
AD		 4448 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4449 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
4450 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4451 igb_write_mbx(hw, msgbuf, 1, vf);
4452 vf_data->last_nack = jiffies;
4453 }
4454 return;
4ae196df
AD		 4455 }
4456
4457 switch ((msgbuf[0] & 0xFFFF)) {
4458 case E1000_VF_SET_MAC_ADDR:
4459 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4460 break;
7d5753f0
AD		 4461 case E1000_VF_SET_PROMISC:
4462 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4463 break;
4ae196df
AD		 4464 case E1000_VF_SET_MULTICAST:
4465 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4466 break;
4467 case E1000_VF_SET_LPE:
4468 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4469 break;
4470 case E1000_VF_SET_VLAN:
4471 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4472 break;
4473 default:
4474 dev_err(&adapter->pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4475 retval = -1;
4476 break;
4477 }
4478
4479 /* notify the VF of the results of what it sent us */
4480 if (retval)
4481 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4482 else
4483 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4484
4485 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4486
4487 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4488}
4ae196df 4489
f2ca0dbe
AD		 4490static void igb_msg_task(struct igb_adapter *adapter)
4491{
4492 struct e1000_hw *hw = &adapter->hw;
4493 u32 vf;
4494
4495 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4496 /* process any reset requests */
4497 if (!igb_check_for_rst(hw, vf))
4498 igb_vf_reset_event(adapter, vf);
4499
4500 /* process any messages pending */
4501 if (!igb_check_for_msg(hw, vf))
4502 igb_rcv_msg_from_vf(adapter, vf);
4503
4504 /* process any acks */
4505 if (!igb_check_for_ack(hw, vf))
4506 igb_rcv_ack_from_vf(adapter, vf);
4507 }
4ae196df
AD		 4508}
4509
68d480c4
AD		 4510/**
4511 * igb_set_uta - Set unicast filter table address
4512 * @adapter: board private structure
4513 *
4514 * The unicast table address is a register array of 32-bit registers.
4515 * The table is meant to be used in a way similar to how the MTA is used
4516 * however due to certain limitations in the hardware it is necessary to
4517 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4518 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4519 **/
4520static void igb_set_uta(struct igb_adapter *adapter)
4521{
4522 struct e1000_hw *hw = &adapter->hw;
4523 int i;
4524
4525 /* The UTA table only exists on 82576 hardware and newer */
4526 if (hw->mac.type < e1000_82576)
4527 return;
4528
4529 /* we only need to do this if VMDq is enabled */
4530 if (!adapter->vfs_allocated_count)
4531 return;
4532
4533 for (i = 0; i < hw->mac.uta_reg_count; i++)
4534 array_wr32(E1000_UTA, i, ~0);
4535}
4536
9d5c8243
AK		 4537/**
4538 * igb_intr_msi - Interrupt Handler
4539 * @irq: interrupt number
4540 * @data: pointer to a network interface device structure
4541 **/
4542static irqreturn_t igb_intr_msi(int irq, void *data)
4543{
047e0030
AD		 4544 struct igb_adapter *adapter = data;
4545 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4546 struct e1000_hw *hw = &adapter->hw;
4547 /* read ICR disables interrupts using IAM */
4548 u32 icr = rd32(E1000_ICR);
4549
047e0030 4550 igb_write_itr(q_vector);
9d5c8243 4551
047e0030 4552 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4553 /* HW is reporting DMA is out of sync */
4554 adapter->stats.doosync++;
4555 }
4556
9d5c8243
AK		 4557 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4558 hw->mac.get_link_status = 1;
4559 if (!test_bit(__IGB_DOWN, &adapter->state))
4560 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4561 }
4562
047e0030 4563 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4564
4565 return IRQ_HANDLED;
4566}
4567
4568/**
4a3c6433 4569 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4570 * @irq: interrupt number
4571 * @data: pointer to a network interface device structure
4572 **/
4573static irqreturn_t igb_intr(int irq, void *data)
4574{
047e0030
AD		 4575 struct igb_adapter *adapter = data;
4576 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4577 struct e1000_hw *hw = &adapter->hw;
4578 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4579 * need for the IMC write */
4580 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4581 if (!icr)
4582 return IRQ_NONE; /* Not our interrupt */
4583
047e0030 4584 igb_write_itr(q_vector);
9d5c8243
AK		 4585
4586 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4587 * not set, then the adapter didn't send an interrupt */
4588 if (!(icr & E1000_ICR_INT_ASSERTED))
4589 return IRQ_NONE;
4590
047e0030 4591 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4592 /* HW is reporting DMA is out of sync */
4593 adapter->stats.doosync++;
4594 }
4595
9d5c8243
AK		 4596 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4597 hw->mac.get_link_status = 1;
4598 /* guard against interrupt when we're going down */
4599 if (!test_bit(__IGB_DOWN, &adapter->state))
4600 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4601 }
4602
047e0030 4603 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4604
4605 return IRQ_HANDLED;
4606}
4607
047e0030 4608static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4609{
047e0030 4610 struct igb_adapter *adapter = q_vector->adapter;
46544258 4611 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4612
4fc82adf
AD		 4613 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4614 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4615 if (!adapter->msix_entries)
6eb5a7f1 4616 igb_set_itr(adapter);
46544258 4617 else
047e0030 4618 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4619 }
4620
46544258
AD		 4621 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4622 if (adapter->msix_entries)
047e0030 4623 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4624 else
4625 igb_irq_enable(adapter);
4626 }
9d5c8243
AK		 4627}
4628
46544258
AD		 4629/**
4630 * igb_poll - NAPI Rx polling callback
4631 * @napi: napi polling structure
4632 * @budget: count of how many packets we should handle
4633 **/
4634static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4635{
047e0030
AD		 4636 struct igb_q_vector *q_vector = container_of(napi,
4637 struct igb_q_vector,
4638 napi);
4639 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4640
421e02f0 4641#ifdef CONFIG_IGB_DCA
047e0030
AD		 4642 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4643 igb_update_dca(q_vector);
fe4506b6 4644#endif
047e0030
AD		 4645 if (q_vector->tx_ring)
4646 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4647
047e0030
AD		 4648 if (q_vector->rx_ring)
4649 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4650
4651 if (!tx_clean_complete)
4652 work_done = budget;
46544258 4653
9d5c8243 4654 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4655 if (work_done < budget) {
288379f0 4656 napi_complete(napi);
047e0030 4657 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4658 }
4659
46544258 4660 return work_done;
9d5c8243 4661}
6d8126f9 4662
33af6bcc 4663/**
c5b9bd5e 4664 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4665 * @adapter: board private structure
c5b9bd5e
AD		 4666 * @shhwtstamps: timestamp structure to update
4667 * @regval: unsigned 64bit system time value.
4668 *
4669 * We need to convert the system time value stored in the RX/TXSTMP registers
4670 * into a hwtstamp which can be used by the upper level timestamping functions
4671 */
4672static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4673 struct skb_shared_hwtstamps *shhwtstamps,
4674 u64 regval)
4675{
4676 u64 ns;
4677
4678 ns = timecounter_cyc2time(&adapter->clock, regval);
4679 timecompare_update(&adapter->compare, ns);
4680 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4681 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4682 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4683}
4684
4685/**
4686 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4687 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4688 * @skb: packet that was just sent
4689 *
4690 * If we were asked to do hardware stamping and such a time stamp is
4691 * available, then it must have been for this skb here because we only
4692 * allow only one such packet into the queue.
4693 */
c5b9bd5e 4694static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4695{
c5b9bd5e 4696 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4697 union skb_shared_tx *shtx = skb_tx(skb);
4698 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4699 struct skb_shared_hwtstamps shhwtstamps;
4700 u64 regval;
33af6bcc 4701
c5b9bd5e
AD		 4702 /* if skb does not support hw timestamp or TX stamp not valid exit */
4703 if (likely(!shtx->hardware) ||
4704 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4705 return;
4706
4707 regval = rd32(E1000_TXSTMPL);
4708 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4709
4710 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4711 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4712}
4713
9d5c8243
AK		 4714/**
4715 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4716 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4717 * returns true if ring is completely cleaned
4718 **/
047e0030 4719static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4720{
047e0030
AD		 4721 struct igb_adapter *adapter = q_vector->adapter;
4722 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4723 struct net_device *netdev = tx_ring->netdev;
0e014cb1 4724 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 4725 struct igb_buffer *buffer_info;
4726 struct sk_buff *skb;
0e014cb1 4727 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 4728 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 4729 unsigned int i, eop, count = 0;
4730 bool cleaned = false;
9d5c8243 4731
9d5c8243 4732 i = tx_ring->next_to_clean;
0e014cb1
AD		 4733 eop = tx_ring->buffer_info[i].next_to_watch;
4734 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4735
4736 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4737 (count < tx_ring->count)) {
4738 for (cleaned = false; !cleaned; count++) {
4739 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 4740 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 4741 cleaned = (i == eop);
9d5c8243
AK		 4742 skb = buffer_info->skb;
4743
4744 if (skb) {
4745 unsigned int segs, bytecount;
4746 /* gso_segs is currently only valid for tcp */
4747 segs = skb_shinfo(skb)->gso_segs ?: 1;
4748 /* multiply data chunks by size of headers */
4749 bytecount = ((segs - 1) * skb_headlen(skb)) +
4750 skb->len;
4751 total_packets += segs;
4752 total_bytes += bytecount;
33af6bcc 4753
c5b9bd5e 4754 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 4755 }
4756
80785298 4757 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 4758 tx_desc->wb.status = 0;
9d5c8243
AK		 4759
4760 i++;
4761 if (i == tx_ring->count)
4762 i = 0;
9d5c8243 4763 }
0e014cb1
AD		 4764 eop = tx_ring->buffer_info[i].next_to_watch;
4765 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4766 }
4767
9d5c8243
AK		 4768 tx_ring->next_to_clean = i;
4769
fc7d345d 4770 if (unlikely(count &&
9d5c8243 4771 netif_carrier_ok(netdev) &&
c493ea45 4772 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 4773 /* Make sure that anybody stopping the queue after this
4774 * sees the new next_to_clean.
4775 */
4776 smp_mb();
661086df
PWJ		 4777 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4778 !(test_bit(__IGB_DOWN, &adapter->state))) {
4779 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4780 tx_ring->tx_stats.restart_queue++;
661086df 4781 }
9d5c8243
AK		 4782 }
4783
4784 if (tx_ring->detect_tx_hung) {
4785 /* Detect a transmit hang in hardware, this serializes the
4786 * check with the clearing of time_stamp and movement of i */
4787 tx_ring->detect_tx_hung = false;
4788 if (tx_ring->buffer_info[i].time_stamp &&
4789 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4790 (adapter->tx_timeout_factor * HZ))
4791 && !(rd32(E1000_STATUS) &
4792 E1000_STATUS_TXOFF)) {
4793
9d5c8243 4794 /* detected Tx unit hang */
80785298 4795 dev_err(&tx_ring->pdev->dev,
9d5c8243 4796 "Detected Tx Unit Hang\n"
2d064c06 4797 " Tx Queue <%d>\n"
9d5c8243
AK		 4798 " TDH <%x>\n"
4799 " TDT <%x>\n"
4800 " next_to_use <%x>\n"
4801 " next_to_clean <%x>\n"
9d5c8243
AK		 4802 "buffer_info[next_to_clean]\n"
4803 " time_stamp <%lx>\n"
0e014cb1 4804 " next_to_watch <%x>\n"
9d5c8243
AK		 4805 " jiffies <%lx>\n"
4806 " desc.status <%x>\n",
2d064c06 4807 tx_ring->queue_index,
fce99e34
AD		 4808 readl(tx_ring->head),
4809 readl(tx_ring->tail),
9d5c8243
AK		 4810 tx_ring->next_to_use,
4811 tx_ring->next_to_clean,
f7ba205e 4812 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 4813 eop,
9d5c8243 4814 jiffies,
0e014cb1 4815 eop_desc->wb.status);
661086df 4816 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 4817 }
4818 }
4819 tx_ring->total_bytes += total_bytes;
4820 tx_ring->total_packets += total_packets;
e21ed353
AD		 4821 tx_ring->tx_stats.bytes += total_bytes;
4822 tx_ring->tx_stats.packets += total_packets;
0e014cb1 4823 return (count < tx_ring->count);
9d5c8243
AK		 4824}
4825
9d5c8243
AK		 4826/**
4827 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 4828 * @q_vector: structure containing interrupt and ring information
4829 * @skb: packet to send up
4830 * @vlan_tag: vlan tag for packet
9d5c8243 4831 **/
047e0030
AD		 4832static void igb_receive_skb(struct igb_q_vector *q_vector,
4833 struct sk_buff *skb,
4834 u16 vlan_tag)
4835{
4836 struct igb_adapter *adapter = q_vector->adapter;
4837
4838 if (vlan_tag)
4839 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4840 vlan_tag, skb);
182ff8df 4841 else
047e0030 4842 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 4843}
4844
04a5fcaa 4845static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 4846 u32 status_err, struct sk_buff *skb)
4847{
4848 skb->ip_summed = CHECKSUM_NONE;
4849
4850 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 4851 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4852 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 4853 return;
85ad76b2 4854
9d5c8243
AK		 4855 /* TCP/UDP checksum error bit is set */
4856 if (status_err &
4857 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 4858 /*
4859 * work around errata with sctp packets where the TCPE aka
4860 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4861 * packets, (aka let the stack check the crc32c)
4862 */
85ad76b2
AD		 4863 if ((skb->len == 60) &&
4864 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 4865 ring->rx_stats.csum_err++;
85ad76b2 4866
9d5c8243 4867 /* let the stack verify checksum errors */
9d5c8243
AK		 4868 return;
4869 }
4870 /* It must be a TCP or UDP packet with a valid checksum */
4871 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4872 skb->ip_summed = CHECKSUM_UNNECESSARY;
4873
85ad76b2 4874 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 4875}
4876
c5b9bd5e
AD		 4877static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
4878 struct sk_buff *skb)
4879{
4880 struct igb_adapter *adapter = q_vector->adapter;
4881 struct e1000_hw *hw = &adapter->hw;
4882 u64 regval;
4883
4884 /*
4885 * If this bit is set, then the RX registers contain the time stamp. No
4886 * other packet will be time stamped until we read these registers, so
4887 * read the registers to make them available again. Because only one
4888 * packet can be time stamped at a time, we know that the register
4889 * values must belong to this one here and therefore we don't need to
4890 * compare any of the additional attributes stored for it.
4891 *
4892 * If nothing went wrong, then it should have a skb_shared_tx that we
4893 * can turn into a skb_shared_hwtstamps.
4894 */
4895 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
4896 return;
4897 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
4898 return;
4899
4900 regval = rd32(E1000_RXSTMPL);
4901 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4902
4903 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
4904}
4c844851 4905static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 4906 union e1000_adv_rx_desc *rx_desc)
4907{
4908 /* HW will not DMA in data larger than the given buffer, even if it
4909 * parses the (NFS, of course) header to be larger. In that case, it
4910 * fills the header buffer and spills the rest into the page.
4911 */
4912 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4913 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 4914 if (hlen > rx_ring->rx_buffer_len)
4915 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 4916 return hlen;
4917}
4918
047e0030
AD		 4919static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4920 int *work_done, int budget)
9d5c8243 4921{
047e0030 4922 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 4923 struct net_device *netdev = rx_ring->netdev;
80785298 4924 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 4925 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4926 struct igb_buffer *buffer_info , *next_buffer;
4927 struct sk_buff *skb;
9d5c8243
AK		 4928 bool cleaned = false;
4929 int cleaned_count = 0;
4930 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 4931 unsigned int i;
2d94d8ab
AD		 4932 u32 staterr;
4933 u16 length;
047e0030 4934 u16 vlan_tag;
9d5c8243
AK		 4935
4936 i = rx_ring->next_to_clean;
69d3ca53 4937 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 4938 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4939 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4940
4941 while (staterr & E1000_RXD_STAT_DD) {
4942 if (*work_done >= budget)
4943 break;
4944 (*work_done)++;
9d5c8243 4945
69d3ca53
AD		 4946 skb = buffer_info->skb;
4947 prefetch(skb->data - NET_IP_ALIGN);
4948 buffer_info->skb = NULL;
4949
4950 i++;
4951 if (i == rx_ring->count)
4952 i = 0;
42d0781a 4953
69d3ca53
AD		 4954 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4955 prefetch(next_rxd);
4956 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 4957
4958 length = le16_to_cpu(rx_desc->wb.upper.length);
4959 cleaned = true;
4960 cleaned_count++;
4961
2d94d8ab 4962 if (buffer_info->dma) {
bf36c1a0 4963 pci_unmap_single(pdev, buffer_info->dma,
4c844851 4964 rx_ring->rx_buffer_len,
bf36c1a0 4965 PCI_DMA_FROMDEVICE);
91615f76 4966 buffer_info->dma = 0;
4c844851 4967 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 4968 skb_put(skb, length);
4969 goto send_up;
4970 }
4c844851 4971 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 4972 }
4973
4974 if (length) {
9d5c8243 4975 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 4976 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 4977 buffer_info->page_dma = 0;
bf36c1a0
AD		 4978
4979 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4980 buffer_info->page,
4981 buffer_info->page_offset,
4982 length);
4983
6ec43fe6 4984 if (page_count(buffer_info->page) != 1)
bf36c1a0
AD		 4985 buffer_info->page = NULL;
4986 else
4987 get_page(buffer_info->page);
9d5c8243
AK		 4988
4989 skb->len += length;
4990 skb->data_len += length;
bf36c1a0 4991 skb->truesize += length;
9d5c8243 4992 }
9d5c8243 4993
bf36c1a0 4994 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 4995 buffer_info->skb = next_buffer->skb;
4996 buffer_info->dma = next_buffer->dma;
4997 next_buffer->skb = skb;
4998 next_buffer->dma = 0;
bf36c1a0
AD		 4999 goto next_desc;
5000 }
69d3ca53 5001send_up:
9d5c8243
AK		 5002 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5003 dev_kfree_skb_irq(skb);
5004 goto next_desc;
5005 }
9d5c8243 5006
c5b9bd5e 5007 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5008 total_bytes += skb->len;
5009 total_packets++;
5010
85ad76b2 5011 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5012
5013 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5014 skb_record_rx_queue(skb, rx_ring->queue_index);
5015
5016 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5017 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5018
047e0030 5019 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5020
9d5c8243
AK		 5021next_desc:
5022 rx_desc->wb.upper.status_error = 0;
5023
5024 /* return some buffers to hardware, one at a time is too slow */
5025 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5026 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5027 cleaned_count = 0;
5028 }
5029
5030 /* use prefetched values */
5031 rx_desc = next_rxd;
5032 buffer_info = next_buffer;
9d5c8243
AK		 5033 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5034 }
bf36c1a0 5035
9d5c8243 5036 rx_ring->next_to_clean = i;
c493ea45 5037 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5038
5039 if (cleaned_count)
3b644cf6 5040 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5041
5042 rx_ring->total_packets += total_packets;
5043 rx_ring->total_bytes += total_bytes;
5044 rx_ring->rx_stats.packets += total_packets;
5045 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5046 return cleaned;
5047}
5048
9d5c8243
AK		 5049/**
5050 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5051 * @adapter: address of board private structure
5052 **/
d7ee5b3a 5053void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5054{
e694e964 5055 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5056 union e1000_adv_rx_desc *rx_desc;
5057 struct igb_buffer *buffer_info;
5058 struct sk_buff *skb;
5059 unsigned int i;
db761762 5060 int bufsz;
9d5c8243
AK		 5061
5062 i = rx_ring->next_to_use;
5063 buffer_info = &rx_ring->buffer_info[i];
5064
4c844851 5065 bufsz = rx_ring->rx_buffer_len;
db761762 5066
9d5c8243
AK		 5067 while (cleaned_count--) {
5068 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5069
6ec43fe6 5070 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5071 if (!buffer_info->page) {
42d0781a 5072 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5073 if (!buffer_info->page) {
04a5fcaa 5074 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5075 goto no_buffers;
5076 }
5077 buffer_info->page_offset = 0;
5078 } else {
5079 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5080 }
5081 buffer_info->page_dma =
80785298 5082 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5083 buffer_info->page_offset,
5084 PAGE_SIZE / 2,
9d5c8243 5085 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5086 if (pci_dma_mapping_error(rx_ring->pdev,
5087 buffer_info->page_dma)) {
5088 buffer_info->page_dma = 0;
5089 rx_ring->rx_stats.alloc_failed++;
5090 goto no_buffers;
5091 }
9d5c8243
AK		 5092 }
5093
42d0781a
AD		 5094 skb = buffer_info->skb;
5095 if (!skb) {
89d71a66 5096 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5097 if (!skb) {
04a5fcaa 5098 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5099 goto no_buffers;
5100 }
5101
9d5c8243 5102 buffer_info->skb = skb;
42d0781a
AD		 5103 }
5104 if (!buffer_info->dma) {
80785298
AD		 5105 buffer_info->dma = pci_map_single(rx_ring->pdev,
5106 skb->data,
9d5c8243
AK		 5107 bufsz,
5108 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5109 if (pci_dma_mapping_error(rx_ring->pdev,
5110 buffer_info->dma)) {
5111 buffer_info->dma = 0;
5112 rx_ring->rx_stats.alloc_failed++;
5113 goto no_buffers;
5114 }
9d5c8243
AK		 5115 }
5116 /* Refresh the desc even if buffer_addrs didn't change because
5117 * each write-back erases this info. */
6ec43fe6 5118 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5119 rx_desc->read.pkt_addr =
5120 cpu_to_le64(buffer_info->page_dma);
5121 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5122 } else {
42d0781a 5123 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5124 rx_desc->read.hdr_addr = 0;
5125 }
5126
5127 i++;
5128 if (i == rx_ring->count)
5129 i = 0;
5130 buffer_info = &rx_ring->buffer_info[i];
5131 }
5132
5133no_buffers:
5134 if (rx_ring->next_to_use != i) {
5135 rx_ring->next_to_use = i;
5136 if (i == 0)
5137 i = (rx_ring->count - 1);
5138 else
5139 i--;
5140
5141 /* Force memory writes to complete before letting h/w
5142 * know there are new descriptors to fetch. (Only
5143 * applicable for weak-ordered memory model archs,
5144 * such as IA-64). */
5145 wmb();
fce99e34 5146 writel(i, rx_ring->tail);
9d5c8243
AK		 5147 }
5148}
5149
5150/**
5151 * igb_mii_ioctl -
5152 * @netdev:
5153 * @ifreq:
5154 * @cmd:
5155 **/
5156static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5157{
5158 struct igb_adapter *adapter = netdev_priv(netdev);
5159 struct mii_ioctl_data *data = if_mii(ifr);
5160
5161 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5162 return -EOPNOTSUPP;
5163
5164 switch (cmd) {
5165 case SIOCGMIIPHY:
5166 data->phy_id = adapter->hw.phy.addr;
5167 break;
5168 case SIOCGMIIREG:
f5f4cf08
AD		 5169 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5170 &data->val_out))
9d5c8243
AK		 5171 return -EIO;
5172 break;
5173 case SIOCSMIIREG:
5174 default:
5175 return -EOPNOTSUPP;
5176 }
5177 return 0;
5178}
5179
c6cb090b
PO		 5180/**
5181 * igb_hwtstamp_ioctl - control hardware time stamping
5182 * @netdev:
5183 * @ifreq:
5184 * @cmd:
5185 *
33af6bcc
PO		 5186 * Outgoing time stamping can be enabled and disabled. Play nice and
5187 * disable it when requested, although it shouldn't case any overhead
5188 * when no packet needs it. At most one packet in the queue may be
5189 * marked for time stamping, otherwise it would be impossible to tell
5190 * for sure to which packet the hardware time stamp belongs.
5191 *
5192 * Incoming time stamping has to be configured via the hardware
5193 * filters. Not all combinations are supported, in particular event
5194 * type has to be specified. Matching the kind of event packet is
5195 * not supported, with the exception of "all V2 events regardless of
5196 * level 2 or 4".
5197 *
c6cb090b
PO		 5198 **/
5199static int igb_hwtstamp_ioctl(struct net_device *netdev,
5200 struct ifreq *ifr, int cmd)
5201{
33af6bcc
PO		 5202 struct igb_adapter *adapter = netdev_priv(netdev);
5203 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5204 struct hwtstamp_config config;
c5b9bd5e
AD		 5205 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5206 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5207 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5208 bool is_l4 = false;
5209 bool is_l2 = false;
33af6bcc 5210 u32 regval;
c6cb090b
PO		 5211
5212 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5213 return -EFAULT;
5214
5215 /* reserved for future extensions */
5216 if (config.flags)
5217 return -EINVAL;
5218
33af6bcc
PO		 5219 switch (config.tx_type) {
5220 case HWTSTAMP_TX_OFF:
c5b9bd5e 5221 tsync_tx_ctl = 0;
33af6bcc 5222 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5223 break;
5224 default:
5225 return -ERANGE;
5226 }
5227
5228 switch (config.rx_filter) {
5229 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5230 tsync_rx_ctl = 0;
33af6bcc
PO		 5231 break;
5232 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5233 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5234 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5235 case HWTSTAMP_FILTER_ALL:
5236 /*
5237 * register TSYNCRXCFG must be set, therefore it is not
5238 * possible to time stamp both Sync and Delay_Req messages
5239 * => fall back to time stamping all packets
5240 */
c5b9bd5e 5241 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5242 config.rx_filter = HWTSTAMP_FILTER_ALL;
5243 break;
5244 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5245 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5246 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5247 is_l4 = true;
33af6bcc
PO		 5248 break;
5249 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5250 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5251 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5252 is_l4 = true;
33af6bcc
PO		 5253 break;
5254 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5255 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5256 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5257 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5258 is_l2 = true;
5259 is_l4 = true;
33af6bcc
PO		 5260 config.rx_filter = HWTSTAMP_FILTER_SOME;
5261 break;
5262 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5263 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5264 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5265 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5266 is_l2 = true;
5267 is_l4 = true;
33af6bcc
PO		 5268 config.rx_filter = HWTSTAMP_FILTER_SOME;
5269 break;
5270 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5271 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5272 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5273 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5274 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5275 is_l2 = true;
33af6bcc
PO		 5276 break;
5277 default:
5278 return -ERANGE;
5279 }
5280
c5b9bd5e
AD		 5281 if (hw->mac.type == e1000_82575) {
5282 if (tsync_rx_ctl | tsync_tx_ctl)
5283 return -EINVAL;
5284 return 0;
5285 }
5286
33af6bcc
PO		 5287 /* enable/disable TX */
5288 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5289 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5290 regval |= tsync_tx_ctl;
33af6bcc
PO		 5291 wr32(E1000_TSYNCTXCTL, regval);
5292
c5b9bd5e 5293 /* enable/disable RX */
33af6bcc 5294 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5295 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5296 regval |= tsync_rx_ctl;
33af6bcc 5297 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5298
c5b9bd5e
AD		 5299 /* define which PTP packets are time stamped */
5300 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5301
c5b9bd5e
AD		 5302 /* define ethertype filter for timestamped packets */
5303 if (is_l2)
5304 wr32(E1000_ETQF(3),
5305 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5306 E1000_ETQF_1588 | /* enable timestamping */
5307 ETH_P_1588)); /* 1588 eth protocol type */
5308 else
5309 wr32(E1000_ETQF(3), 0);
5310
5311#define PTP_PORT 319
5312 /* L4 Queue Filter[3]: filter by destination port and protocol */
5313 if (is_l4) {
5314 u32 ftqf = (IPPROTO_UDP /* UDP */
5315 | E1000_FTQF_VF_BP /* VF not compared */
5316 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5317 | E1000_FTQF_MASK); /* mask all inputs */
5318 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5319
5320 wr32(E1000_IMIR(3), htons(PTP_PORT));
5321 wr32(E1000_IMIREXT(3),
5322 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5323 if (hw->mac.type == e1000_82576) {
5324 /* enable source port check */
5325 wr32(E1000_SPQF(3), htons(PTP_PORT));
5326 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5327 }
5328 wr32(E1000_FTQF(3), ftqf);
5329 } else {
5330 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5331 }
33af6bcc
PO		 5332 wrfl();
5333
5334 adapter->hwtstamp_config = config;
5335
5336 /* clear TX/RX time stamp registers, just to be sure */
5337 regval = rd32(E1000_TXSTMPH);
5338 regval = rd32(E1000_RXSTMPH);
c6cb090b 5339
33af6bcc
PO		 5340 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5341 -EFAULT : 0;
c6cb090b
PO		 5342}
5343
9d5c8243
AK		 5344/**
5345 * igb_ioctl -
5346 * @netdev:
5347 * @ifreq:
5348 * @cmd:
5349 **/
5350static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5351{
5352 switch (cmd) {
5353 case SIOCGMIIPHY:
5354 case SIOCGMIIREG:
5355 case SIOCSMIIREG:
5356 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5357 case SIOCSHWTSTAMP:
5358 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5359 default:
5360 return -EOPNOTSUPP;
5361 }
5362}
5363
009bc06e
AD		 5364s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5365{
5366 struct igb_adapter *adapter = hw->back;
5367 u16 cap_offset;
5368
5369 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5370 if (!cap_offset)
5371 return -E1000_ERR_CONFIG;
5372
5373 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5374
5375 return 0;
5376}
5377
5378s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5379{
5380 struct igb_adapter *adapter = hw->back;
5381 u16 cap_offset;
5382
5383 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5384 if (!cap_offset)
5385 return -E1000_ERR_CONFIG;
5386
5387 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5388
5389 return 0;
5390}
5391
9d5c8243
AK		 5392static void igb_vlan_rx_register(struct net_device *netdev,
5393 struct vlan_group *grp)
5394{
5395 struct igb_adapter *adapter = netdev_priv(netdev);
5396 struct e1000_hw *hw = &adapter->hw;
5397 u32 ctrl, rctl;
5398
5399 igb_irq_disable(adapter);
5400 adapter->vlgrp = grp;
5401
5402 if (grp) {
5403 /* enable VLAN tag insert/strip */
5404 ctrl = rd32(E1000_CTRL);
5405 ctrl |= E1000_CTRL_VME;
5406 wr32(E1000_CTRL, ctrl);
5407
51466239 5408 /* Disable CFI check */
9d5c8243 5409 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5410 rctl &= ~E1000_RCTL_CFIEN;
5411 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5412 } else {
5413 /* disable VLAN tag insert/strip */
5414 ctrl = rd32(E1000_CTRL);
5415 ctrl &= ~E1000_CTRL_VME;
5416 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5417 }
5418
e1739522
AD		 5419 igb_rlpml_set(adapter);
5420
9d5c8243
AK		 5421 if (!test_bit(__IGB_DOWN, &adapter->state))
5422 igb_irq_enable(adapter);
5423}
5424
5425static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5426{
5427 struct igb_adapter *adapter = netdev_priv(netdev);
5428 struct e1000_hw *hw = &adapter->hw;
4ae196df 5429 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5430
51466239
AD		 5431 /* attempt to add filter to vlvf array */
5432 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5433
51466239
AD		 5434 /* add the filter since PF can receive vlans w/o entry in vlvf */
5435 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5436}
5437
5438static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5439{
5440 struct igb_adapter *adapter = netdev_priv(netdev);
5441 struct e1000_hw *hw = &adapter->hw;
4ae196df 5442 int pf_id = adapter->vfs_allocated_count;
51466239 5443 s32 err;
9d5c8243
AK		 5444
5445 igb_irq_disable(adapter);
5446 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5447
5448 if (!test_bit(__IGB_DOWN, &adapter->state))
5449 igb_irq_enable(adapter);
5450
51466239
AD		 5451 /* remove vlan from VLVF table array */
5452 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5453
51466239
AD		 5454 /* if vid was not present in VLVF just remove it from table */
5455 if (err)
4ae196df 5456 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5457}
5458
5459static void igb_restore_vlan(struct igb_adapter *adapter)
5460{
5461 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5462
5463 if (adapter->vlgrp) {
5464 u16 vid;
5465 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5466 if (!vlan_group_get_device(adapter->vlgrp, vid))
5467 continue;
5468 igb_vlan_rx_add_vid(adapter->netdev, vid);
5469 }
5470 }
5471}
5472
5473int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5474{
5475 struct e1000_mac_info *mac = &adapter->hw.mac;
5476
5477 mac->autoneg = 0;
5478
9d5c8243
AK		 5479 switch (spddplx) {
5480 case SPEED_10 + DUPLEX_HALF:
5481 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5482 break;
5483 case SPEED_10 + DUPLEX_FULL:
5484 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5485 break;
5486 case SPEED_100 + DUPLEX_HALF:
5487 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5488 break;
5489 case SPEED_100 + DUPLEX_FULL:
5490 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5491 break;
5492 case SPEED_1000 + DUPLEX_FULL:
5493 mac->autoneg = 1;
5494 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5495 break;
5496 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5497 default:
5498 dev_err(&adapter->pdev->dev,
5499 "Unsupported Speed/Duplex configuration\n");
5500 return -EINVAL;
5501 }
5502 return 0;
5503}
5504
3fe7c4c9 5505static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5506{
5507 struct net_device *netdev = pci_get_drvdata(pdev);
5508 struct igb_adapter *adapter = netdev_priv(netdev);
5509 struct e1000_hw *hw = &adapter->hw;
2d064c06 5510 u32 ctrl, rctl, status;
9d5c8243
AK		 5511 u32 wufc = adapter->wol;
5512#ifdef CONFIG_PM
5513 int retval = 0;
5514#endif
5515
5516 netif_device_detach(netdev);
5517
a88f10ec
AD		 5518 if (netif_running(netdev))
5519 igb_close(netdev);
5520
047e0030 5521 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5522
5523#ifdef CONFIG_PM
5524 retval = pci_save_state(pdev);
5525 if (retval)
5526 return retval;
5527#endif
5528
5529 status = rd32(E1000_STATUS);
5530 if (status & E1000_STATUS_LU)
5531 wufc &= ~E1000_WUFC_LNKC;
5532
5533 if (wufc) {
5534 igb_setup_rctl(adapter);
ff41f8dc 5535 igb_set_rx_mode(netdev);
9d5c8243
AK		 5536
5537 /* turn on all-multi mode if wake on multicast is enabled */
5538 if (wufc & E1000_WUFC_MC) {
5539 rctl = rd32(E1000_RCTL);
5540 rctl |= E1000_RCTL_MPE;
5541 wr32(E1000_RCTL, rctl);
5542 }
5543
5544 ctrl = rd32(E1000_CTRL);
5545 /* advertise wake from D3Cold */
5546 #define E1000_CTRL_ADVD3WUC 0x00100000
5547 /* phy power management enable */
5548 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5549 ctrl |= E1000_CTRL_ADVD3WUC;
5550 wr32(E1000_CTRL, ctrl);
5551
9d5c8243 5552 /* Allow time for pending master requests to run */
330a6d6a 5553 igb_disable_pcie_master(hw);
9d5c8243
AK		 5554
5555 wr32(E1000_WUC, E1000_WUC_PME_EN);
5556 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5557 } else {
5558 wr32(E1000_WUC, 0);
5559 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5560 }
5561
3fe7c4c9
RW		 5562 *enable_wake = wufc || adapter->en_mng_pt;
5563 if (!*enable_wake)
2fb02a26 5564 igb_shutdown_serdes_link_82575(hw);
9d5c8243
AK		 5565
5566 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5567 * would have already happened in close and is redundant. */
5568 igb_release_hw_control(adapter);
5569
5570 pci_disable_device(pdev);
5571
9d5c8243
AK		 5572 return 0;
5573}
5574
5575#ifdef CONFIG_PM
3fe7c4c9
RW		 5576static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5577{
5578 int retval;
5579 bool wake;
5580
5581 retval = __igb_shutdown(pdev, &wake);
5582 if (retval)
5583 return retval;
5584
5585 if (wake) {
5586 pci_prepare_to_sleep(pdev);
5587 } else {
5588 pci_wake_from_d3(pdev, false);
5589 pci_set_power_state(pdev, PCI_D3hot);
5590 }
5591
5592 return 0;
5593}
5594
9d5c8243
AK		 5595static int igb_resume(struct pci_dev *pdev)
5596{
5597 struct net_device *netdev = pci_get_drvdata(pdev);
5598 struct igb_adapter *adapter = netdev_priv(netdev);
5599 struct e1000_hw *hw = &adapter->hw;
5600 u32 err;
5601
5602 pci_set_power_state(pdev, PCI_D0);
5603 pci_restore_state(pdev);
42bfd33a 5604
aed5dec3 5605 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5606 if (err) {
5607 dev_err(&pdev->dev,
5608 "igb: Cannot enable PCI device from suspend\n");
5609 return err;
5610 }
5611 pci_set_master(pdev);
5612
5613 pci_enable_wake(pdev, PCI_D3hot, 0);
5614 pci_enable_wake(pdev, PCI_D3cold, 0);
5615
047e0030 5616 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5617 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5618 return -ENOMEM;
9d5c8243
AK		 5619 }
5620
5621 /* e1000_power_up_phy(adapter); */
5622
5623 igb_reset(adapter);
a8564f03
AD		 5624
5625 /* let the f/w know that the h/w is now under the control of the
5626 * driver. */
5627 igb_get_hw_control(adapter);
5628
9d5c8243
AK		 5629 wr32(E1000_WUS, ~0);
5630
a88f10ec
AD		 5631 if (netif_running(netdev)) {
5632 err = igb_open(netdev);
5633 if (err)
5634 return err;
5635 }
9d5c8243
AK		 5636
5637 netif_device_attach(netdev);
5638
9d5c8243
AK		 5639 return 0;
5640}
5641#endif
5642
5643static void igb_shutdown(struct pci_dev *pdev)
5644{
3fe7c4c9
RW		 5645 bool wake;
5646
5647 __igb_shutdown(pdev, &wake);
5648
5649 if (system_state == SYSTEM_POWER_OFF) {
5650 pci_wake_from_d3(pdev, wake);
5651 pci_set_power_state(pdev, PCI_D3hot);
5652 }
9d5c8243
AK		 5653}
5654
5655#ifdef CONFIG_NET_POLL_CONTROLLER
5656/*
5657 * Polling 'interrupt' - used by things like netconsole to send skbs
5658 * without having to re-enable interrupts. It's not called while
5659 * the interrupt routine is executing.
5660 */
5661static void igb_netpoll(struct net_device *netdev)
5662{
5663 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5664 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5665 int i;
9d5c8243 5666
eebbbdba 5667 if (!adapter->msix_entries) {
047e0030 5668 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5669 igb_irq_disable(adapter);
047e0030 5670 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5671 return;
5672 }
9d5c8243 5673
047e0030
AD		 5674 for (i = 0; i < adapter->num_q_vectors; i++) {
5675 struct igb_q_vector *q_vector = adapter->q_vector[i];
5676 wr32(E1000_EIMC, q_vector->eims_value);
5677 napi_schedule(&q_vector->napi);
eebbbdba 5678 }
9d5c8243
AK		 5679}
5680#endif /* CONFIG_NET_POLL_CONTROLLER */
5681
5682/**
5683 * igb_io_error_detected - called when PCI error is detected
5684 * @pdev: Pointer to PCI device
5685 * @state: The current pci connection state
5686 *
5687 * This function is called after a PCI bus error affecting
5688 * this device has been detected.
5689 */
5690static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5691 pci_channel_state_t state)
5692{
5693 struct net_device *netdev = pci_get_drvdata(pdev);
5694 struct igb_adapter *adapter = netdev_priv(netdev);
5695
5696 netif_device_detach(netdev);
5697
59ed6eec
AD		 5698 if (state == pci_channel_io_perm_failure)
5699 return PCI_ERS_RESULT_DISCONNECT;
5700
9d5c8243
AK		 5701 if (netif_running(netdev))
5702 igb_down(adapter);
5703 pci_disable_device(pdev);
5704
5705 /* Request a slot slot reset. */
5706 return PCI_ERS_RESULT_NEED_RESET;
5707}
5708
5709/**
5710 * igb_io_slot_reset - called after the pci bus has been reset.
5711 * @pdev: Pointer to PCI device
5712 *
5713 * Restart the card from scratch, as if from a cold-boot. Implementation
5714 * resembles the first-half of the igb_resume routine.
5715 */
5716static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5717{
5718 struct net_device *netdev = pci_get_drvdata(pdev);
5719 struct igb_adapter *adapter = netdev_priv(netdev);
5720 struct e1000_hw *hw = &adapter->hw;
40a914fa 5721 pci_ers_result_t result;
42bfd33a 5722 int err;
9d5c8243 5723
aed5dec3 5724 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 5725 dev_err(&pdev->dev,
5726 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 5727 result = PCI_ERS_RESULT_DISCONNECT;
5728 } else {
5729 pci_set_master(pdev);
5730 pci_restore_state(pdev);
9d5c8243 5731
40a914fa
AD		 5732 pci_enable_wake(pdev, PCI_D3hot, 0);
5733 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 5734
40a914fa
AD		 5735 igb_reset(adapter);
5736 wr32(E1000_WUS, ~0);
5737 result = PCI_ERS_RESULT_RECOVERED;
5738 }
9d5c8243 5739
ea943d41
JK		 5740 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5741 if (err) {
5742 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5743 "failed 0x%0x\n", err);
5744 /* non-fatal, continue */
5745 }
40a914fa
AD		 5746
5747 return result;
9d5c8243
AK		 5748}
5749
5750/**
5751 * igb_io_resume - called when traffic can start flowing again.
5752 * @pdev: Pointer to PCI device
5753 *
5754 * This callback is called when the error recovery driver tells us that
5755 * its OK to resume normal operation. Implementation resembles the
5756 * second-half of the igb_resume routine.
5757 */
5758static void igb_io_resume(struct pci_dev *pdev)
5759{
5760 struct net_device *netdev = pci_get_drvdata(pdev);
5761 struct igb_adapter *adapter = netdev_priv(netdev);
5762
9d5c8243
AK		 5763 if (netif_running(netdev)) {
5764 if (igb_up(adapter)) {
5765 dev_err(&pdev->dev, "igb_up failed after reset\n");
5766 return;
5767 }
5768 }
5769
5770 netif_device_attach(netdev);
5771
5772 /* let the f/w know that the h/w is now under the control of the
5773 * driver. */
5774 igb_get_hw_control(adapter);
9d5c8243
AK		 5775}
5776
26ad9178
AD		 5777static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5778 u8 qsel)
5779{
5780 u32 rar_low, rar_high;
5781 struct e1000_hw *hw = &adapter->hw;
5782
5783 /* HW expects these in little endian so we reverse the byte order
5784 * from network order (big endian) to little endian
5785 */
5786 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5787 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5788 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5789
5790 /* Indicate to hardware the Address is Valid. */
5791 rar_high |= E1000_RAH_AV;
5792
5793 if (hw->mac.type == e1000_82575)
5794 rar_high |= E1000_RAH_POOL_1 * qsel;
5795 else
5796 rar_high |= E1000_RAH_POOL_1 << qsel;
5797
5798 wr32(E1000_RAL(index), rar_low);
5799 wrfl();
5800 wr32(E1000_RAH(index), rar_high);
5801 wrfl();
5802}
5803
4ae196df
AD		 5804static int igb_set_vf_mac(struct igb_adapter *adapter,
5805 int vf, unsigned char *mac_addr)
5806{
5807 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 5808 /* VF MAC addresses start at end of receive addresses and moves
5809 * torwards the first, as a result a collision should not be possible */
5810 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 5811
37680117 5812 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 5813
26ad9178 5814 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 5815
5816 return 0;
5817}
5818
5819static void igb_vmm_control(struct igb_adapter *adapter)
5820{
5821 struct e1000_hw *hw = &adapter->hw;
10d8e907 5822 u32 reg;
4ae196df 5823
d4960307
AD		 5824 /* replication is not supported for 82575 */
5825 if (hw->mac.type == e1000_82575)
4ae196df
AD		 5826 return;
5827
10d8e907
AD		 5828 /* enable replication vlan tag stripping */
5829 reg = rd32(E1000_RPLOLR);
5830 reg |= E1000_RPLOLR_STRVLAN;
5831 wr32(E1000_RPLOLR, reg);
5832
5833 /* notify HW that the MAC is adding vlan tags */
5834 reg = rd32(E1000_DTXCTL);
5835 reg |= E1000_DTXCTL_VLAN_ADDED;
5836 wr32(E1000_DTXCTL, reg);
5837
d4960307
AD		 5838 if (adapter->vfs_allocated_count) {
5839 igb_vmdq_set_loopback_pf(hw, true);
5840 igb_vmdq_set_replication_pf(hw, true);
5841 } else {
5842 igb_vmdq_set_loopback_pf(hw, false);
5843 igb_vmdq_set_replication_pf(hw, false);
5844 }
4ae196df
AD		 5845}
5846
9d5c8243 5847/* igb_main.c */
kernel source for telekom puls (alcatel/mediatek)