projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: change descriptor control thresholds
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
55cac248 52#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
a3aa1884 63static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
55cac248
AD		 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
79 /* required last entry */
80 {0, }
81};
82
83MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
84
85void igb_reset(struct igb_adapter *);
86static int igb_setup_all_tx_resources(struct igb_adapter *);
87static int igb_setup_all_rx_resources(struct igb_adapter *);
88static void igb_free_all_tx_resources(struct igb_adapter *);
89static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 90static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 91void igb_update_stats(struct igb_adapter *);
92static int igb_probe(struct pci_dev *, const struct pci_device_id *);
93static void __devexit igb_remove(struct pci_dev *pdev);
94static int igb_sw_init(struct igb_adapter *);
95static int igb_open(struct net_device *);
96static int igb_close(struct net_device *);
97static void igb_configure_tx(struct igb_adapter *);
98static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 99static void igb_clean_all_tx_rings(struct igb_adapter *);
100static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 101static void igb_clean_tx_ring(struct igb_ring *);
102static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 103static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 104static void igb_update_phy_info(unsigned long);
105static void igb_watchdog(unsigned long);
106static void igb_watchdog_task(struct work_struct *);
b1a436c3 107static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 108static struct net_device_stats *igb_get_stats(struct net_device *);
109static int igb_change_mtu(struct net_device *, int);
110static int igb_set_mac(struct net_device *, void *);
68d480c4 111static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 112static irqreturn_t igb_intr(int irq, void *);
113static irqreturn_t igb_intr_msi(int irq, void *);
114static irqreturn_t igb_msix_other(int irq, void *);
047e0030 115static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 116#ifdef CONFIG_IGB_DCA
047e0030 117static void igb_update_dca(struct igb_q_vector *);
fe4506b6 118static void igb_setup_dca(struct igb_adapter *);
421e02f0 119#endif /* CONFIG_IGB_DCA */
047e0030 120static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 121static int igb_poll(struct napi_struct *, int);
047e0030 122static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 123static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
124static void igb_tx_timeout(struct net_device *);
125static void igb_reset_task(struct work_struct *);
126static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
127static void igb_vlan_rx_add_vid(struct net_device *, u16);
128static void igb_vlan_rx_kill_vid(struct net_device *, u16);
129static void igb_restore_vlan(struct igb_adapter *);
26ad9178 130static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 131static void igb_ping_all_vfs(struct igb_adapter *);
132static void igb_msg_task(struct igb_adapter *);
4ae196df 133static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 134static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 135static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 136static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
137static int igb_ndo_set_vf_vlan(struct net_device *netdev,
138 int vf, u16 vlan, u8 qos);
139static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
140static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
141 struct ifla_vf_info *ivi);
9d5c8243 142
9d5c8243 143#ifdef CONFIG_PM
3fe7c4c9 144static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 145static int igb_resume(struct pci_dev *);
146#endif
147static void igb_shutdown(struct pci_dev *);
421e02f0 148#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 149static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
150static struct notifier_block dca_notifier = {
151 .notifier_call = igb_notify_dca,
152 .next = NULL,
153 .priority = 0
154};
155#endif
9d5c8243
AK		 156#ifdef CONFIG_NET_POLL_CONTROLLER
157/* for netdump / net console */
158static void igb_netpoll(struct net_device *);
159#endif
37680117 160#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 161static unsigned int max_vfs = 0;
162module_param(max_vfs, uint, 0);
163MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
164 "per physical function");
165#endif /* CONFIG_PCI_IOV */
166
9d5c8243
AK		 167static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
168 pci_channel_state_t);
169static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
170static void igb_io_resume(struct pci_dev *);
171
172static struct pci_error_handlers igb_err_handler = {
173 .error_detected = igb_io_error_detected,
174 .slot_reset = igb_io_slot_reset,
175 .resume = igb_io_resume,
176};
177
178
179static struct pci_driver igb_driver = {
180 .name = igb_driver_name,
181 .id_table = igb_pci_tbl,
182 .probe = igb_probe,
183 .remove = __devexit_p(igb_remove),
184#ifdef CONFIG_PM
185 /* Power Managment Hooks */
186 .suspend = igb_suspend,
187 .resume = igb_resume,
188#endif
189 .shutdown = igb_shutdown,
190 .err_handler = &igb_err_handler
191};
192
193MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
194MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
195MODULE_LICENSE("GPL");
196MODULE_VERSION(DRV_VERSION);
197
38c845c7
PO		 198/**
199 * igb_read_clock - read raw cycle counter (to be used by time counter)
200 */
201static cycle_t igb_read_clock(const struct cyclecounter *tc)
202{
203 struct igb_adapter *adapter =
204 container_of(tc, struct igb_adapter, cycles);
205 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 206 u64 stamp = 0;
207 int shift = 0;
38c845c7 208
55cac248
AD		 209 /*
210 * The timestamp latches on lowest register read. For the 82580
211 * the lowest register is SYSTIMR instead of SYSTIML. However we never
212 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
213 */
214 if (hw->mac.type == e1000_82580) {
215 stamp = rd32(E1000_SYSTIMR) >> 8;
216 shift = IGB_82580_TSYNC_SHIFT;
217 }
218
c5b9bd5e
AD		 219 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
220 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 221 return stamp;
222}
223
9d5c8243
AK		 224#ifdef DEBUG
225/**
226 * igb_get_hw_dev_name - return device name string
227 * used by hardware layer to print debugging information
228 **/
229char *igb_get_hw_dev_name(struct e1000_hw *hw)
230{
231 struct igb_adapter *adapter = hw->back;
232 return adapter->netdev->name;
233}
38c845c7
PO		 234
235/**
236 * igb_get_time_str - format current NIC and system time as string
237 */
238static char *igb_get_time_str(struct igb_adapter *adapter,
239 char buffer[160])
240{
241 cycle_t hw = adapter->cycles.read(&adapter->cycles);
242 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
243 struct timespec sys;
244 struct timespec delta;
245 getnstimeofday(&sys);
246
247 delta = timespec_sub(nic, sys);
248
249 sprintf(buffer,
33af6bcc
PO		 250 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
251 hw,
38c845c7
PO		 252 (long)nic.tv_sec, nic.tv_nsec,
253 (long)sys.tv_sec, sys.tv_nsec,
254 (long)delta.tv_sec, delta.tv_nsec);
255
256 return buffer;
257}
9d5c8243
AK		 258#endif
259
260/**
261 * igb_init_module - Driver Registration Routine
262 *
263 * igb_init_module is the first routine called when the driver is
264 * loaded. All it does is register with the PCI subsystem.
265 **/
266static int __init igb_init_module(void)
267{
268 int ret;
269 printk(KERN_INFO "%s - version %s\n",
270 igb_driver_string, igb_driver_version);
271
272 printk(KERN_INFO "%s\n", igb_copyright);
273
421e02f0 274#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 275 dca_register_notify(&dca_notifier);
276#endif
bbd98fe4 277 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 278 return ret;
279}
280
281module_init(igb_init_module);
282
283/**
284 * igb_exit_module - Driver Exit Cleanup Routine
285 *
286 * igb_exit_module is called just before the driver is removed
287 * from memory.
288 **/
289static void __exit igb_exit_module(void)
290{
421e02f0 291#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 292 dca_unregister_notify(&dca_notifier);
293#endif
9d5c8243
AK		 294 pci_unregister_driver(&igb_driver);
295}
296
297module_exit(igb_exit_module);
298
26bc19ec
AD		 299#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
300/**
301 * igb_cache_ring_register - Descriptor ring to register mapping
302 * @adapter: board private structure to initialize
303 *
304 * Once we know the feature-set enabled for the device, we'll cache
305 * the register offset the descriptor ring is assigned to.
306 **/
307static void igb_cache_ring_register(struct igb_adapter *adapter)
308{
ee1b9f06 309 int i = 0, j = 0;
047e0030 310 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 311
312 switch (adapter->hw.mac.type) {
313 case e1000_82576:
314 /* The queues are allocated for virtualization such that VF 0
315 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
316 * In order to avoid collision we start at the first free queue
317 * and continue consuming queues in the same sequence
318 */
ee1b9f06 319 if (adapter->vfs_allocated_count) {
a99955fc 320 for (; i < adapter->rss_queues; i++)
3025a446
AD		 321 adapter->rx_ring[i]->reg_idx = rbase_offset +
322 Q_IDX_82576(i);
a99955fc 323 for (; j < adapter->rss_queues; j++)
3025a446
AD		 324 adapter->tx_ring[j]->reg_idx = rbase_offset +
325 Q_IDX_82576(j);
ee1b9f06 326 }
26bc19ec 327 case e1000_82575:
55cac248 328 case e1000_82580:
26bc19ec 329 default:
ee1b9f06 330 for (; i < adapter->num_rx_queues; i++)
3025a446 331 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 332 for (; j < adapter->num_tx_queues; j++)
3025a446 333 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 334 break;
335 }
336}
337
047e0030
AD		 338static void igb_free_queues(struct igb_adapter *adapter)
339{
3025a446 340 int i;
047e0030 341
3025a446
AD		 342 for (i = 0; i < adapter->num_tx_queues; i++) {
343 kfree(adapter->tx_ring[i]);
344 adapter->tx_ring[i] = NULL;
345 }
346 for (i = 0; i < adapter->num_rx_queues; i++) {
347 kfree(adapter->rx_ring[i]);
348 adapter->rx_ring[i] = NULL;
349 }
047e0030
AD		 350 adapter->num_rx_queues = 0;
351 adapter->num_tx_queues = 0;
352}
353
9d5c8243
AK		 354/**
355 * igb_alloc_queues - Allocate memory for all rings
356 * @adapter: board private structure to initialize
357 *
358 * We allocate one ring per queue at run-time since we don't know the
359 * number of queues at compile-time.
360 **/
361static int igb_alloc_queues(struct igb_adapter *adapter)
362{
3025a446 363 struct igb_ring *ring;
9d5c8243
AK		 364 int i;
365
661086df 366 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 367 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
368 if (!ring)
369 goto err;
68fd9910 370 ring->count = adapter->tx_ring_count;
661086df 371 ring->queue_index = i;
80785298 372 ring->pdev = adapter->pdev;
e694e964 373 ring->netdev = adapter->netdev;
85ad76b2
AD		 374 /* For 82575, context index must be unique per ring. */
375 if (adapter->hw.mac.type == e1000_82575)
376 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 377 adapter->tx_ring[i] = ring;
661086df 378 }
85ad76b2 379
9d5c8243 380 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 381 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
382 if (!ring)
383 goto err;
68fd9910 384 ring->count = adapter->rx_ring_count;
844290e5 385 ring->queue_index = i;
80785298 386 ring->pdev = adapter->pdev;
e694e964 387 ring->netdev = adapter->netdev;
4c844851 388 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 389 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
390 /* set flag indicating ring supports SCTP checksum offload */
391 if (adapter->hw.mac.type >= e1000_82576)
392 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 393 adapter->rx_ring[i] = ring;
9d5c8243 394 }
26bc19ec
AD		 395
396 igb_cache_ring_register(adapter);
9d5c8243 397
047e0030 398 return 0;
a88f10ec 399
047e0030
AD		 400err:
401 igb_free_queues(adapter);
d1a8c9e1 402
047e0030 403 return -ENOMEM;
a88f10ec
AD		 404}
405
9d5c8243 406#define IGB_N0_QUEUE -1
047e0030 407static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 408{
409 u32 msixbm = 0;
047e0030 410 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 411 struct e1000_hw *hw = &adapter->hw;
2d064c06 412 u32 ivar, index;
047e0030
AD		 413 int rx_queue = IGB_N0_QUEUE;
414 int tx_queue = IGB_N0_QUEUE;
415
416 if (q_vector->rx_ring)
417 rx_queue = q_vector->rx_ring->reg_idx;
418 if (q_vector->tx_ring)
419 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 420
421 switch (hw->mac.type) {
422 case e1000_82575:
9d5c8243
AK		 423 /* The 82575 assigns vectors using a bitmask, which matches the
424 bitmask for the EICR/EIMS/EIMC registers. To assign one
425 or more queues to a vector, we write the appropriate bits
426 into the MSIXBM register for that vector. */
047e0030 427 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 428 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 429 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 430 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 431 if (!adapter->msix_entries && msix_vector == 0)
432 msixbm |= E1000_EIMS_OTHER;
9d5c8243 433 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 434 q_vector->eims_value = msixbm;
2d064c06
AD		 435 break;
436 case e1000_82576:
26bc19ec 437 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 438 Each queue has a single entry in the table to which we write
439 a vector number along with a "valid" bit. Sadly, the layout
440 of the table is somewhat counterintuitive. */
441 if (rx_queue > IGB_N0_QUEUE) {
047e0030 442 index = (rx_queue & 0x7);
2d064c06 443 ivar = array_rd32(E1000_IVAR0, index);
047e0030 444 if (rx_queue < 8) {
26bc19ec
AD		 445 /* vector goes into low byte of register */
446 ivar = ivar & 0xFFFFFF00;
447 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 448 } else {
449 /* vector goes into third byte of register */
450 ivar = ivar & 0xFF00FFFF;
451 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 452 }
2d064c06
AD		 453 array_wr32(E1000_IVAR0, index, ivar);
454 }
455 if (tx_queue > IGB_N0_QUEUE) {
047e0030 456 index = (tx_queue & 0x7);
2d064c06 457 ivar = array_rd32(E1000_IVAR0, index);
047e0030 458 if (tx_queue < 8) {
26bc19ec
AD		 459 /* vector goes into second byte of register */
460 ivar = ivar & 0xFFFF00FF;
461 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 462 } else {
463 /* vector goes into high byte of register */
464 ivar = ivar & 0x00FFFFFF;
465 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 466 }
2d064c06
AD		 467 array_wr32(E1000_IVAR0, index, ivar);
468 }
047e0030 469 q_vector->eims_value = 1 << msix_vector;
2d064c06 470 break;
55cac248
AD		 471 case e1000_82580:
472 /* 82580 uses the same table-based approach as 82576 but has fewer
473 entries as a result we carry over for queues greater than 4. */
474 if (rx_queue > IGB_N0_QUEUE) {
475 index = (rx_queue >> 1);
476 ivar = array_rd32(E1000_IVAR0, index);
477 if (rx_queue & 0x1) {
478 /* vector goes into third byte of register */
479 ivar = ivar & 0xFF00FFFF;
480 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
481 } else {
482 /* vector goes into low byte of register */
483 ivar = ivar & 0xFFFFFF00;
484 ivar |= msix_vector | E1000_IVAR_VALID;
485 }
486 array_wr32(E1000_IVAR0, index, ivar);
487 }
488 if (tx_queue > IGB_N0_QUEUE) {
489 index = (tx_queue >> 1);
490 ivar = array_rd32(E1000_IVAR0, index);
491 if (tx_queue & 0x1) {
492 /* vector goes into high byte of register */
493 ivar = ivar & 0x00FFFFFF;
494 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
495 } else {
496 /* vector goes into second byte of register */
497 ivar = ivar & 0xFFFF00FF;
498 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
499 }
500 array_wr32(E1000_IVAR0, index, ivar);
501 }
502 q_vector->eims_value = 1 << msix_vector;
503 break;
2d064c06
AD		 504 default:
505 BUG();
506 break;
507 }
26b39276
AD		 508
509 /* add q_vector eims value to global eims_enable_mask */
510 adapter->eims_enable_mask |= q_vector->eims_value;
511
512 /* configure q_vector to set itr on first interrupt */
513 q_vector->set_itr = 1;
9d5c8243
AK		 514}
515
516/**
517 * igb_configure_msix - Configure MSI-X hardware
518 *
519 * igb_configure_msix sets up the hardware to properly
520 * generate MSI-X interrupts.
521 **/
522static void igb_configure_msix(struct igb_adapter *adapter)
523{
524 u32 tmp;
525 int i, vector = 0;
526 struct e1000_hw *hw = &adapter->hw;
527
528 adapter->eims_enable_mask = 0;
9d5c8243
AK		 529
530 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 531 switch (hw->mac.type) {
532 case e1000_82575:
9d5c8243
AK		 533 tmp = rd32(E1000_CTRL_EXT);
534 /* enable MSI-X PBA support*/
535 tmp |= E1000_CTRL_EXT_PBA_CLR;
536
537 /* Auto-Mask interrupts upon ICR read. */
538 tmp |= E1000_CTRL_EXT_EIAME;
539 tmp |= E1000_CTRL_EXT_IRCA;
540
541 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 542
543 /* enable msix_other interrupt */
544 array_wr32(E1000_MSIXBM(0), vector++,
545 E1000_EIMS_OTHER);
844290e5 546 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 547
2d064c06
AD		 548 break;
549
550 case e1000_82576:
55cac248 551 case e1000_82580:
047e0030
AD		 552 /* Turn on MSI-X capability first, or our settings
553 * won't stick. And it will take days to debug. */
554 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
555 E1000_GPIE_PBA | E1000_GPIE_EIAME |
556 E1000_GPIE_NSICR);
557
558 /* enable msix_other interrupt */
559 adapter->eims_other = 1 << vector;
2d064c06 560 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 561
047e0030 562 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 563 break;
564 default:
565 /* do nothing, since nothing else supports MSI-X */
566 break;
567 } /* switch (hw->mac.type) */
047e0030
AD		 568
569 adapter->eims_enable_mask |= adapter->eims_other;
570
26b39276
AD		 571 for (i = 0; i < adapter->num_q_vectors; i++)
572 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 573
9d5c8243
AK		 574 wrfl();
575}
576
577/**
578 * igb_request_msix - Initialize MSI-X interrupts
579 *
580 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
581 * kernel.
582 **/
583static int igb_request_msix(struct igb_adapter *adapter)
584{
585 struct net_device *netdev = adapter->netdev;
047e0030 586 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 587 int i, err = 0, vector = 0;
588
047e0030 589 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 590 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 591 if (err)
592 goto out;
593 vector++;
594
595 for (i = 0; i < adapter->num_q_vectors; i++) {
596 struct igb_q_vector *q_vector = adapter->q_vector[i];
597
598 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
599
600 if (q_vector->rx_ring && q_vector->tx_ring)
601 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
602 q_vector->rx_ring->queue_index);
603 else if (q_vector->tx_ring)
604 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
605 q_vector->tx_ring->queue_index);
606 else if (q_vector->rx_ring)
607 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
608 q_vector->rx_ring->queue_index);
9d5c8243 609 else
047e0030
AD		 610 sprintf(q_vector->name, "%s-unused", netdev->name);
611
9d5c8243 612 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 613 igb_msix_ring, 0, q_vector->name,
047e0030 614 q_vector);
9d5c8243
AK		 615 if (err)
616 goto out;
9d5c8243
AK		 617 vector++;
618 }
619
9d5c8243
AK		 620 igb_configure_msix(adapter);
621 return 0;
622out:
623 return err;
624}
625
626static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
627{
628 if (adapter->msix_entries) {
629 pci_disable_msix(adapter->pdev);
630 kfree(adapter->msix_entries);
631 adapter->msix_entries = NULL;
047e0030 632 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 633 pci_disable_msi(adapter->pdev);
047e0030 634 }
9d5c8243
AK		 635}
636
047e0030
AD		 637/**
638 * igb_free_q_vectors - Free memory allocated for interrupt vectors
639 * @adapter: board private structure to initialize
640 *
641 * This function frees the memory allocated to the q_vectors. In addition if
642 * NAPI is enabled it will delete any references to the NAPI struct prior
643 * to freeing the q_vector.
644 **/
645static void igb_free_q_vectors(struct igb_adapter *adapter)
646{
647 int v_idx;
648
649 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
650 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
651 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 652 if (!q_vector)
653 continue;
047e0030
AD		 654 netif_napi_del(&q_vector->napi);
655 kfree(q_vector);
656 }
657 adapter->num_q_vectors = 0;
658}
659
660/**
661 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
662 *
663 * This function resets the device so that it has 0 rx queues, tx queues, and
664 * MSI-X interrupts allocated.
665 */
666static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
667{
668 igb_free_queues(adapter);
669 igb_free_q_vectors(adapter);
670 igb_reset_interrupt_capability(adapter);
671}
9d5c8243
AK		 672
673/**
674 * igb_set_interrupt_capability - set MSI or MSI-X if supported
675 *
676 * Attempt to configure interrupts using the best available
677 * capabilities of the hardware and kernel.
678 **/
679static void igb_set_interrupt_capability(struct igb_adapter *adapter)
680{
681 int err;
682 int numvecs, i;
683
83b7180d 684 /* Number of supported queues. */
a99955fc
AD		 685 adapter->num_rx_queues = adapter->rss_queues;
686 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 687
047e0030
AD		 688 /* start with one vector for every rx queue */
689 numvecs = adapter->num_rx_queues;
690
691 /* if tx handler is seperate add 1 for every tx queue */
a99955fc
AD		 692 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
693 numvecs += adapter->num_tx_queues;
047e0030
AD		 694
695 /* store the number of vectors reserved for queues */
696 adapter->num_q_vectors = numvecs;
697
698 /* add 1 vector for link status interrupts */
699 numvecs++;
9d5c8243
AK		 700 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
701 GFP_KERNEL);
702 if (!adapter->msix_entries)
703 goto msi_only;
704
705 for (i = 0; i < numvecs; i++)
706 adapter->msix_entries[i].entry = i;
707
708 err = pci_enable_msix(adapter->pdev,
709 adapter->msix_entries,
710 numvecs);
711 if (err == 0)
34a20e89 712 goto out;
9d5c8243
AK		 713
714 igb_reset_interrupt_capability(adapter);
715
716 /* If we can't do MSI-X, try MSI */
717msi_only:
2a3abf6d
AD		 718#ifdef CONFIG_PCI_IOV
719 /* disable SR-IOV for non MSI-X configurations */
720 if (adapter->vf_data) {
721 struct e1000_hw *hw = &adapter->hw;
722 /* disable iov and allow time for transactions to clear */
723 pci_disable_sriov(adapter->pdev);
724 msleep(500);
725
726 kfree(adapter->vf_data);
727 adapter->vf_data = NULL;
728 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
729 msleep(100);
730 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
731 }
732#endif
4fc82adf 733 adapter->vfs_allocated_count = 0;
a99955fc 734 adapter->rss_queues = 1;
4fc82adf 735 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 736 adapter->num_rx_queues = 1;
661086df 737 adapter->num_tx_queues = 1;
047e0030 738 adapter->num_q_vectors = 1;
9d5c8243 739 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 740 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 741out:
661086df 742 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 743 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 744 return;
745}
746
047e0030
AD		 747/**
748 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
749 * @adapter: board private structure to initialize
750 *
751 * We allocate one q_vector per queue interrupt. If allocation fails we
752 * return -ENOMEM.
753 **/
754static int igb_alloc_q_vectors(struct igb_adapter *adapter)
755{
756 struct igb_q_vector *q_vector;
757 struct e1000_hw *hw = &adapter->hw;
758 int v_idx;
759
760 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
761 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
762 if (!q_vector)
763 goto err_out;
764 q_vector->adapter = adapter;
047e0030
AD		 765 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
766 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 767 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
768 adapter->q_vector[v_idx] = q_vector;
769 }
770 return 0;
771
772err_out:
fe0592b4 773 igb_free_q_vectors(adapter);
047e0030
AD		 774 return -ENOMEM;
775}
776
777static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
778 int ring_idx, int v_idx)
779{
3025a446 780 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 781
3025a446 782 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 783 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 784 q_vector->itr_val = adapter->rx_itr_setting;
785 if (q_vector->itr_val && q_vector->itr_val <= 3)
786 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 787}
788
789static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
790 int ring_idx, int v_idx)
791{
3025a446 792 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 793
3025a446 794 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 795 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 796 q_vector->itr_val = adapter->tx_itr_setting;
797 if (q_vector->itr_val && q_vector->itr_val <= 3)
798 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 799}
800
801/**
802 * igb_map_ring_to_vector - maps allocated queues to vectors
803 *
804 * This function maps the recently allocated queues to vectors.
805 **/
806static int igb_map_ring_to_vector(struct igb_adapter *adapter)
807{
808 int i;
809 int v_idx = 0;
810
811 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
812 (adapter->num_q_vectors < adapter->num_tx_queues))
813 return -ENOMEM;
814
815 if (adapter->num_q_vectors >=
816 (adapter->num_rx_queues + adapter->num_tx_queues)) {
817 for (i = 0; i < adapter->num_rx_queues; i++)
818 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
819 for (i = 0; i < adapter->num_tx_queues; i++)
820 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
821 } else {
822 for (i = 0; i < adapter->num_rx_queues; i++) {
823 if (i < adapter->num_tx_queues)
824 igb_map_tx_ring_to_vector(adapter, i, v_idx);
825 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
826 }
827 for (; i < adapter->num_tx_queues; i++)
828 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
829 }
830 return 0;
831}
832
833/**
834 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
835 *
836 * This function initializes the interrupts and allocates all of the queues.
837 **/
838static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
839{
840 struct pci_dev *pdev = adapter->pdev;
841 int err;
842
843 igb_set_interrupt_capability(adapter);
844
845 err = igb_alloc_q_vectors(adapter);
846 if (err) {
847 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
848 goto err_alloc_q_vectors;
849 }
850
851 err = igb_alloc_queues(adapter);
852 if (err) {
853 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
854 goto err_alloc_queues;
855 }
856
857 err = igb_map_ring_to_vector(adapter);
858 if (err) {
859 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
860 goto err_map_queues;
861 }
862
863
864 return 0;
865err_map_queues:
866 igb_free_queues(adapter);
867err_alloc_queues:
868 igb_free_q_vectors(adapter);
869err_alloc_q_vectors:
870 igb_reset_interrupt_capability(adapter);
871 return err;
872}
873
9d5c8243
AK		 874/**
875 * igb_request_irq - initialize interrupts
876 *
877 * Attempts to configure interrupts using the best available
878 * capabilities of the hardware and kernel.
879 **/
880static int igb_request_irq(struct igb_adapter *adapter)
881{
882 struct net_device *netdev = adapter->netdev;
047e0030 883 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 884 int err = 0;
885
886 if (adapter->msix_entries) {
887 err = igb_request_msix(adapter);
844290e5 888 if (!err)
9d5c8243 889 goto request_done;
9d5c8243 890 /* fall back to MSI */
047e0030 891 igb_clear_interrupt_scheme(adapter);
9d5c8243 892 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 893 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 894 igb_free_all_tx_resources(adapter);
895 igb_free_all_rx_resources(adapter);
047e0030 896 adapter->num_tx_queues = 1;
9d5c8243 897 adapter->num_rx_queues = 1;
047e0030
AD		 898 adapter->num_q_vectors = 1;
899 err = igb_alloc_q_vectors(adapter);
900 if (err) {
901 dev_err(&pdev->dev,
902 "Unable to allocate memory for vectors\n");
903 goto request_done;
904 }
905 err = igb_alloc_queues(adapter);
906 if (err) {
907 dev_err(&pdev->dev,
908 "Unable to allocate memory for queues\n");
909 igb_free_q_vectors(adapter);
910 goto request_done;
911 }
912 igb_setup_all_tx_resources(adapter);
913 igb_setup_all_rx_resources(adapter);
844290e5 914 } else {
feeb2721 915 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 916 }
844290e5 917
7dfc16fa 918 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 919 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 920 netdev->name, adapter);
9d5c8243
AK		 921 if (!err)
922 goto request_done;
047e0030 923
9d5c8243
AK		 924 /* fall back to legacy interrupts */
925 igb_reset_interrupt_capability(adapter);
7dfc16fa 926 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 927 }
928
a0607fd3 929 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 930 netdev->name, adapter);
9d5c8243 931
6cb5e577 932 if (err)
9d5c8243
AK		 933 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
934 err);
9d5c8243
AK		 935
936request_done:
937 return err;
938}
939
940static void igb_free_irq(struct igb_adapter *adapter)
941{
9d5c8243
AK		 942 if (adapter->msix_entries) {
943 int vector = 0, i;
944
047e0030 945 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 946
047e0030
AD		 947 for (i = 0; i < adapter->num_q_vectors; i++) {
948 struct igb_q_vector *q_vector = adapter->q_vector[i];
949 free_irq(adapter->msix_entries[vector++].vector,
950 q_vector);
951 }
952 } else {
953 free_irq(adapter->pdev->irq, adapter);
9d5c8243 954 }
9d5c8243
AK		 955}
956
957/**
958 * igb_irq_disable - Mask off interrupt generation on the NIC
959 * @adapter: board private structure
960 **/
961static void igb_irq_disable(struct igb_adapter *adapter)
962{
963 struct e1000_hw *hw = &adapter->hw;
964
25568a53
AD		 965 /*
966 * we need to be careful when disabling interrupts. The VFs are also
967 * mapped into these registers and so clearing the bits can cause
968 * issues on the VF drivers so we only need to clear what we set
969 */
9d5c8243 970 if (adapter->msix_entries) {
2dfd1212
AD		 971 u32 regval = rd32(E1000_EIAM);
972 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
973 wr32(E1000_EIMC, adapter->eims_enable_mask);
974 regval = rd32(E1000_EIAC);
975 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 976 }
844290e5
PW		 977
978 wr32(E1000_IAM, 0);
9d5c8243
AK		 979 wr32(E1000_IMC, ~0);
980 wrfl();
981 synchronize_irq(adapter->pdev->irq);
982}
983
984/**
985 * igb_irq_enable - Enable default interrupt generation settings
986 * @adapter: board private structure
987 **/
988static void igb_irq_enable(struct igb_adapter *adapter)
989{
990 struct e1000_hw *hw = &adapter->hw;
991
992 if (adapter->msix_entries) {
25568a53 993 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 994 u32 regval = rd32(E1000_EIAC);
995 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
996 regval = rd32(E1000_EIAM);
997 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 998 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 999 if (adapter->vfs_allocated_count) {
4ae196df 1000 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1001 ims |= E1000_IMS_VMMB;
1002 }
55cac248
AD		 1003 if (adapter->hw.mac.type == e1000_82580)
1004 ims |= E1000_IMS_DRSTA;
1005
25568a53 1006 wr32(E1000_IMS, ims);
844290e5 1007 } else {
55cac248
AD		 1008 wr32(E1000_IMS, IMS_ENABLE_MASK |
1009 E1000_IMS_DRSTA);
1010 wr32(E1000_IAM, IMS_ENABLE_MASK |
1011 E1000_IMS_DRSTA);
844290e5 1012 }
9d5c8243
AK		 1013}
1014
1015static void igb_update_mng_vlan(struct igb_adapter *adapter)
1016{
51466239 1017 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1018 u16 vid = adapter->hw.mng_cookie.vlan_id;
1019 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1020
1021 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1022 /* add VID to filter table */
1023 igb_vfta_set(hw, vid, true);
1024 adapter->mng_vlan_id = vid;
1025 } else {
1026 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1027 }
1028
1029 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1030 (vid != old_vid) &&
1031 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1032 /* remove VID from filter table */
1033 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1034 }
1035}
1036
1037/**
1038 * igb_release_hw_control - release control of the h/w to f/w
1039 * @adapter: address of board private structure
1040 *
1041 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1042 * For ASF and Pass Through versions of f/w this means that the
1043 * driver is no longer loaded.
1044 *
1045 **/
1046static void igb_release_hw_control(struct igb_adapter *adapter)
1047{
1048 struct e1000_hw *hw = &adapter->hw;
1049 u32 ctrl_ext;
1050
1051 /* Let firmware take over control of h/w */
1052 ctrl_ext = rd32(E1000_CTRL_EXT);
1053 wr32(E1000_CTRL_EXT,
1054 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1055}
1056
9d5c8243
AK		 1057/**
1058 * igb_get_hw_control - get control of the h/w from f/w
1059 * @adapter: address of board private structure
1060 *
1061 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1062 * For ASF and Pass Through versions of f/w this means that
1063 * the driver is loaded.
1064 *
1065 **/
1066static void igb_get_hw_control(struct igb_adapter *adapter)
1067{
1068 struct e1000_hw *hw = &adapter->hw;
1069 u32 ctrl_ext;
1070
1071 /* Let firmware know the driver has taken over */
1072 ctrl_ext = rd32(E1000_CTRL_EXT);
1073 wr32(E1000_CTRL_EXT,
1074 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1075}
1076
9d5c8243
AK		 1077/**
1078 * igb_configure - configure the hardware for RX and TX
1079 * @adapter: private board structure
1080 **/
1081static void igb_configure(struct igb_adapter *adapter)
1082{
1083 struct net_device *netdev = adapter->netdev;
1084 int i;
1085
1086 igb_get_hw_control(adapter);
ff41f8dc 1087 igb_set_rx_mode(netdev);
9d5c8243
AK		 1088
1089 igb_restore_vlan(adapter);
9d5c8243 1090
85b430b4 1091 igb_setup_tctl(adapter);
06cf2666 1092 igb_setup_mrqc(adapter);
9d5c8243 1093 igb_setup_rctl(adapter);
85b430b4
AD		 1094
1095 igb_configure_tx(adapter);
9d5c8243 1096 igb_configure_rx(adapter);
662d7205
AD		 1097
1098 igb_rx_fifo_flush_82575(&adapter->hw);
1099
c493ea45 1100 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1101 * at least 1 descriptor unused to make sure
1102 * next_to_use != next_to_clean */
1103 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1104 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1105 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1106 }
1107
1108
1109 adapter->tx_queue_len = netdev->tx_queue_len;
1110}
1111
88a268c1
NN		 1112/**
1113 * igb_power_up_link - Power up the phy/serdes link
1114 * @adapter: address of board private structure
1115 **/
1116void igb_power_up_link(struct igb_adapter *adapter)
1117{
1118 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1119 igb_power_up_phy_copper(&adapter->hw);
1120 else
1121 igb_power_up_serdes_link_82575(&adapter->hw);
1122}
1123
1124/**
1125 * igb_power_down_link - Power down the phy/serdes link
1126 * @adapter: address of board private structure
1127 */
1128static void igb_power_down_link(struct igb_adapter *adapter)
1129{
1130 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1131 igb_power_down_phy_copper_82575(&adapter->hw);
1132 else
1133 igb_shutdown_serdes_link_82575(&adapter->hw);
1134}
9d5c8243
AK		 1135
1136/**
1137 * igb_up - Open the interface and prepare it to handle traffic
1138 * @adapter: board private structure
1139 **/
9d5c8243
AK		 1140int igb_up(struct igb_adapter *adapter)
1141{
1142 struct e1000_hw *hw = &adapter->hw;
1143 int i;
1144
1145 /* hardware has been reset, we need to reload some things */
1146 igb_configure(adapter);
1147
1148 clear_bit(__IGB_DOWN, &adapter->state);
1149
047e0030
AD		 1150 for (i = 0; i < adapter->num_q_vectors; i++) {
1151 struct igb_q_vector *q_vector = adapter->q_vector[i];
1152 napi_enable(&q_vector->napi);
1153 }
844290e5 1154 if (adapter->msix_entries)
9d5c8243 1155 igb_configure_msix(adapter);
feeb2721
AD		 1156 else
1157 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1158
1159 /* Clear any pending interrupts. */
1160 rd32(E1000_ICR);
1161 igb_irq_enable(adapter);
1162
d4960307
AD		 1163 /* notify VFs that reset has been completed */
1164 if (adapter->vfs_allocated_count) {
1165 u32 reg_data = rd32(E1000_CTRL_EXT);
1166 reg_data |= E1000_CTRL_EXT_PFRSTD;
1167 wr32(E1000_CTRL_EXT, reg_data);
1168 }
1169
4cb9be7a
JB		 1170 netif_tx_start_all_queues(adapter->netdev);
1171
25568a53
AD		 1172 /* start the watchdog. */
1173 hw->mac.get_link_status = 1;
1174 schedule_work(&adapter->watchdog_task);
1175
9d5c8243
AK		 1176 return 0;
1177}
1178
1179void igb_down(struct igb_adapter *adapter)
1180{
9d5c8243 1181 struct net_device *netdev = adapter->netdev;
330a6d6a 1182 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1183 u32 tctl, rctl;
1184 int i;
1185
1186 /* signal that we're down so the interrupt handler does not
1187 * reschedule our watchdog timer */
1188 set_bit(__IGB_DOWN, &adapter->state);
1189
1190 /* disable receives in the hardware */
1191 rctl = rd32(E1000_RCTL);
1192 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1193 /* flush and sleep below */
1194
fd2ea0a7 1195 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1196
1197 /* disable transmits in the hardware */
1198 tctl = rd32(E1000_TCTL);
1199 tctl &= ~E1000_TCTL_EN;
1200 wr32(E1000_TCTL, tctl);
1201 /* flush both disables and wait for them to finish */
1202 wrfl();
1203 msleep(10);
1204
047e0030
AD		 1205 for (i = 0; i < adapter->num_q_vectors; i++) {
1206 struct igb_q_vector *q_vector = adapter->q_vector[i];
1207 napi_disable(&q_vector->napi);
1208 }
9d5c8243 1209
9d5c8243
AK		 1210 igb_irq_disable(adapter);
1211
1212 del_timer_sync(&adapter->watchdog_timer);
1213 del_timer_sync(&adapter->phy_info_timer);
1214
1215 netdev->tx_queue_len = adapter->tx_queue_len;
1216 netif_carrier_off(netdev);
04fe6358
AD		 1217
1218 /* record the stats before reset*/
1219 igb_update_stats(adapter);
1220
9d5c8243
AK		 1221 adapter->link_speed = 0;
1222 adapter->link_duplex = 0;
1223
3023682e
JK		 1224 if (!pci_channel_offline(adapter->pdev))
1225 igb_reset(adapter);
9d5c8243
AK		 1226 igb_clean_all_tx_rings(adapter);
1227 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1228#ifdef CONFIG_IGB_DCA
1229
1230 /* since we reset the hardware DCA settings were cleared */
1231 igb_setup_dca(adapter);
1232#endif
9d5c8243
AK		 1233}
1234
1235void igb_reinit_locked(struct igb_adapter *adapter)
1236{
1237 WARN_ON(in_interrupt());
1238 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1239 msleep(1);
1240 igb_down(adapter);
1241 igb_up(adapter);
1242 clear_bit(__IGB_RESETTING, &adapter->state);
1243}
1244
1245void igb_reset(struct igb_adapter *adapter)
1246{
090b1795 1247 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1248 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1249 struct e1000_mac_info *mac = &hw->mac;
1250 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1251 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1252 u16 hwm;
1253
1254 /* Repartition Pba for greater than 9k mtu
1255 * To take effect CTRL.RST is required.
1256 */
fa4dfae0 1257 switch (mac->type) {
55cac248
AD		 1258 case e1000_82580:
1259 pba = rd32(E1000_RXPBS);
1260 pba = igb_rxpbs_adjust_82580(pba);
1261 break;
fa4dfae0 1262 case e1000_82576:
d249be54
AD		 1263 pba = rd32(E1000_RXPBS);
1264 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1265 break;
1266 case e1000_82575:
1267 default:
1268 pba = E1000_PBA_34K;
1269 break;
2d064c06 1270 }
9d5c8243 1271
2d064c06
AD		 1272 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1273 (mac->type < e1000_82576)) {
9d5c8243
AK		 1274 /* adjust PBA for jumbo frames */
1275 wr32(E1000_PBA, pba);
1276
1277 /* To maintain wire speed transmits, the Tx FIFO should be
1278 * large enough to accommodate two full transmit packets,
1279 * rounded up to the next 1KB and expressed in KB. Likewise,
1280 * the Rx FIFO should be large enough to accommodate at least
1281 * one full receive packet and is similarly rounded up and
1282 * expressed in KB. */
1283 pba = rd32(E1000_PBA);
1284 /* upper 16 bits has Tx packet buffer allocation size in KB */
1285 tx_space = pba >> 16;
1286 /* lower 16 bits has Rx packet buffer allocation size in KB */
1287 pba &= 0xffff;
1288 /* the tx fifo also stores 16 bytes of information about the tx
1289 * but don't include ethernet FCS because hardware appends it */
1290 min_tx_space = (adapter->max_frame_size +
85e8d004 1291 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1292 ETH_FCS_LEN) * 2;
1293 min_tx_space = ALIGN(min_tx_space, 1024);
1294 min_tx_space >>= 10;
1295 /* software strips receive CRC, so leave room for it */
1296 min_rx_space = adapter->max_frame_size;
1297 min_rx_space = ALIGN(min_rx_space, 1024);
1298 min_rx_space >>= 10;
1299
1300 /* If current Tx allocation is less than the min Tx FIFO size,
1301 * and the min Tx FIFO size is less than the current Rx FIFO
1302 * allocation, take space away from current Rx allocation */
1303 if (tx_space < min_tx_space &&
1304 ((min_tx_space - tx_space) < pba)) {
1305 pba = pba - (min_tx_space - tx_space);
1306
1307 /* if short on rx space, rx wins and must trump tx
1308 * adjustment */
1309 if (pba < min_rx_space)
1310 pba = min_rx_space;
1311 }
2d064c06 1312 wr32(E1000_PBA, pba);
9d5c8243 1313 }
9d5c8243
AK		 1314
1315 /* flow control settings */
1316 /* The high water mark must be low enough to fit one full frame
1317 * (or the size used for early receive) above it in the Rx FIFO.
1318 * Set it to the lower of:
1319 * - 90% of the Rx FIFO size, or
1320 * - the full Rx FIFO size minus one full frame */
1321 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1322 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1323
d405ea3e
AD		 1324 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1325 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1326 fc->pause_time = 0xFFFF;
1327 fc->send_xon = 1;
0cce119a 1328 fc->current_mode = fc->requested_mode;
9d5c8243 1329
4ae196df
AD		 1330 /* disable receive for all VFs and wait one second */
1331 if (adapter->vfs_allocated_count) {
1332 int i;
1333 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1334 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1335
1336 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1337 igb_ping_all_vfs(adapter);
4ae196df
AD		 1338
1339 /* disable transmits and receives */
1340 wr32(E1000_VFRE, 0);
1341 wr32(E1000_VFTE, 0);
1342 }
1343
9d5c8243 1344 /* Allow time for pending master requests to run */
330a6d6a 1345 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1346 wr32(E1000_WUC, 0);
1347
330a6d6a 1348 if (hw->mac.ops.init_hw(hw))
090b1795 1349 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1350
55cac248
AD		 1351 if (hw->mac.type == e1000_82580) {
1352 u32 reg = rd32(E1000_PCIEMISC);
1353 wr32(E1000_PCIEMISC,
1354 reg & ~E1000_PCIEMISC_LX_DECISION);
1355 }
88a268c1
NN		 1356 if (!netif_running(adapter->netdev))
1357 igb_power_down_link(adapter);
1358
9d5c8243
AK		 1359 igb_update_mng_vlan(adapter);
1360
1361 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1362 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1363
330a6d6a 1364 igb_get_phy_info(hw);
9d5c8243
AK		 1365}
1366
2e5c6922 1367static const struct net_device_ops igb_netdev_ops = {
559e9c49 1368 .ndo_open = igb_open,
2e5c6922 1369 .ndo_stop = igb_close,
00829823 1370 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1371 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1372 .ndo_set_rx_mode = igb_set_rx_mode,
1373 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1374 .ndo_set_mac_address = igb_set_mac,
1375 .ndo_change_mtu = igb_change_mtu,
1376 .ndo_do_ioctl = igb_ioctl,
1377 .ndo_tx_timeout = igb_tx_timeout,
1378 .ndo_validate_addr = eth_validate_addr,
1379 .ndo_vlan_rx_register = igb_vlan_rx_register,
1380 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1381 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1382 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1383 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1384 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1385 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1386#ifdef CONFIG_NET_POLL_CONTROLLER
1387 .ndo_poll_controller = igb_netpoll,
1388#endif
1389};
1390
9d5c8243
AK		 1391/**
1392 * igb_probe - Device Initialization Routine
1393 * @pdev: PCI device information struct
1394 * @ent: entry in igb_pci_tbl
1395 *
1396 * Returns 0 on success, negative on failure
1397 *
1398 * igb_probe initializes an adapter identified by a pci_dev structure.
1399 * The OS initialization, configuring of the adapter private structure,
1400 * and a hardware reset occur.
1401 **/
1402static int __devinit igb_probe(struct pci_dev *pdev,
1403 const struct pci_device_id *ent)
1404{
1405 struct net_device *netdev;
1406 struct igb_adapter *adapter;
1407 struct e1000_hw *hw;
4337e993
AD		 1408 u16 eeprom_data = 0;
1409 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1410 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1411 unsigned long mmio_start, mmio_len;
2d6a5e95 1412 int err, pci_using_dac;
9d5c8243
AK		 1413 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1414 u32 part_num;
1415
aed5dec3 1416 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1417 if (err)
1418 return err;
1419
1420 pci_using_dac = 0;
6a35528a 1421 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1422 if (!err) {
6a35528a 1423 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1424 if (!err)
1425 pci_using_dac = 1;
1426 } else {
284901a9 1427 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1428 if (err) {
284901a9 1429 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1430 if (err) {
1431 dev_err(&pdev->dev, "No usable DMA "
1432 "configuration, aborting\n");
1433 goto err_dma;
1434 }
1435 }
1436 }
1437
aed5dec3
AD		 1438 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1439 IORESOURCE_MEM),
1440 igb_driver_name);
9d5c8243
AK		 1441 if (err)
1442 goto err_pci_reg;
1443
19d5afd4 1444 pci_enable_pcie_error_reporting(pdev);
40a914fa 1445
9d5c8243 1446 pci_set_master(pdev);
c682fc23 1447 pci_save_state(pdev);
9d5c8243
AK		 1448
1449 err = -ENOMEM;
1bfaf07b
AD		 1450 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1451 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1452 if (!netdev)
1453 goto err_alloc_etherdev;
1454
1455 SET_NETDEV_DEV(netdev, &pdev->dev);
1456
1457 pci_set_drvdata(pdev, netdev);
1458 adapter = netdev_priv(netdev);
1459 adapter->netdev = netdev;
1460 adapter->pdev = pdev;
1461 hw = &adapter->hw;
1462 hw->back = adapter;
1463 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1464
1465 mmio_start = pci_resource_start(pdev, 0);
1466 mmio_len = pci_resource_len(pdev, 0);
1467
1468 err = -EIO;
28b0759c
AD		 1469 hw->hw_addr = ioremap(mmio_start, mmio_len);
1470 if (!hw->hw_addr)
9d5c8243
AK		 1471 goto err_ioremap;
1472
2e5c6922 1473 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1474 igb_set_ethtool_ops(netdev);
9d5c8243 1475 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1476
1477 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1478
1479 netdev->mem_start = mmio_start;
1480 netdev->mem_end = mmio_start + mmio_len;
1481
9d5c8243
AK		 1482 /* PCI config space info */
1483 hw->vendor_id = pdev->vendor;
1484 hw->device_id = pdev->device;
1485 hw->revision_id = pdev->revision;
1486 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1487 hw->subsystem_device_id = pdev->subsystem_device;
1488
9d5c8243
AK		 1489 /* Copy the default MAC, PHY and NVM function pointers */
1490 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1491 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1492 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1493 /* Initialize skew-specific constants */
1494 err = ei->get_invariants(hw);
1495 if (err)
450c87c8 1496 goto err_sw_init;
9d5c8243 1497
450c87c8 1498 /* setup the private structure */
9d5c8243
AK		 1499 err = igb_sw_init(adapter);
1500 if (err)
1501 goto err_sw_init;
1502
1503 igb_get_bus_info_pcie(hw);
1504
1505 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1506
1507 /* Copper options */
1508 if (hw->phy.media_type == e1000_media_type_copper) {
1509 hw->phy.mdix = AUTO_ALL_MODES;
1510 hw->phy.disable_polarity_correction = false;
1511 hw->phy.ms_type = e1000_ms_hw_default;
1512 }
1513
1514 if (igb_check_reset_block(hw))
1515 dev_info(&pdev->dev,
1516 "PHY reset is blocked due to SOL/IDER session.\n");
1517
1518 netdev->features = NETIF_F_SG |
7d8eb29e 1519 NETIF_F_IP_CSUM |
9d5c8243
AK		 1520 NETIF_F_HW_VLAN_TX |
1521 NETIF_F_HW_VLAN_RX |
1522 NETIF_F_HW_VLAN_FILTER;
1523
7d8eb29e 1524 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1525 netdev->features |= NETIF_F_TSO;
9d5c8243 1526 netdev->features |= NETIF_F_TSO6;
5c0999b7 1527 netdev->features |= NETIF_F_GRO;
d3352520 1528
48f29ffc
JK		 1529 netdev->vlan_features |= NETIF_F_TSO;
1530 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1531 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1532 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1533 netdev->vlan_features |= NETIF_F_SG;
1534
9d5c8243
AK		 1535 if (pci_using_dac)
1536 netdev->features |= NETIF_F_HIGHDMA;
1537
5b043fb0 1538 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1539 netdev->features |= NETIF_F_SCTP_CSUM;
1540
330a6d6a 1541 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1542
1543 /* before reading the NVM, reset the controller to put the device in a
1544 * known good starting state */
1545 hw->mac.ops.reset_hw(hw);
1546
1547 /* make sure the NVM is good */
1548 if (igb_validate_nvm_checksum(hw) < 0) {
1549 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1550 err = -EIO;
1551 goto err_eeprom;
1552 }
1553
1554 /* copy the MAC address out of the NVM */
1555 if (hw->mac.ops.read_mac_addr(hw))
1556 dev_err(&pdev->dev, "NVM Read Error\n");
1557
1558 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1559 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1560
1561 if (!is_valid_ether_addr(netdev->perm_addr)) {
1562 dev_err(&pdev->dev, "Invalid MAC Address\n");
1563 err = -EIO;
1564 goto err_eeprom;
1565 }
1566
0e340485
AD		 1567 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1568 (unsigned long) adapter);
1569 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1570 (unsigned long) adapter);
9d5c8243
AK		 1571
1572 INIT_WORK(&adapter->reset_task, igb_reset_task);
1573 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1574
450c87c8 1575 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1576 adapter->fc_autoneg = true;
1577 hw->mac.autoneg = true;
1578 hw->phy.autoneg_advertised = 0x2f;
1579
0cce119a
AD		 1580 hw->fc.requested_mode = e1000_fc_default;
1581 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1582
9d5c8243
AK		 1583 igb_validate_mdi_setting(hw);
1584
9d5c8243
AK		 1585 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1586 * enable the ACPI Magic Packet filter
1587 */
1588
a2cf8b6c 1589 if (hw->bus.func == 0)
312c75ae 1590 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1591 else if (hw->mac.type == e1000_82580)
1592 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1593 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1594 &eeprom_data);
a2cf8b6c
AD		 1595 else if (hw->bus.func == 1)
1596 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1597
1598 if (eeprom_data & eeprom_apme_mask)
1599 adapter->eeprom_wol |= E1000_WUFC_MAG;
1600
1601 /* now that we have the eeprom settings, apply the special cases where
1602 * the eeprom may be wrong or the board simply won't support wake on
1603 * lan on a particular port */
1604 switch (pdev->device) {
1605 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1606 adapter->eeprom_wol = 0;
1607 break;
1608 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1609 case E1000_DEV_ID_82576_FIBER:
1610 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1611 /* Wake events only supported on port A for dual fiber
1612 * regardless of eeprom setting */
1613 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1614 adapter->eeprom_wol = 0;
1615 break;
c8ea5ea9
AD		 1616 case E1000_DEV_ID_82576_QUAD_COPPER:
1617 /* if quad port adapter, disable WoL on all but port A */
1618 if (global_quad_port_a != 0)
1619 adapter->eeprom_wol = 0;
1620 else
1621 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1622 /* Reset for multiple quad port adapters */
1623 if (++global_quad_port_a == 4)
1624 global_quad_port_a = 0;
1625 break;
9d5c8243
AK		 1626 }
1627
1628 /* initialize the wol settings based on the eeprom settings */
1629 adapter->wol = adapter->eeprom_wol;
e1b86d84 1630 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1631
1632 /* reset the hardware with the new settings */
1633 igb_reset(adapter);
1634
1635 /* let the f/w know that the h/w is now under the control of the
1636 * driver. */
1637 igb_get_hw_control(adapter);
1638
9d5c8243
AK		 1639 strcpy(netdev->name, "eth%d");
1640 err = register_netdev(netdev);
1641 if (err)
1642 goto err_register;
1643
b168dfc5
JB		 1644 /* carrier off reporting is important to ethtool even BEFORE open */
1645 netif_carrier_off(netdev);
1646
421e02f0 1647#ifdef CONFIG_IGB_DCA
bbd98fe4 1648 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1649 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1650 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1651 igb_setup_dca(adapter);
1652 }
fe4506b6 1653
38c845c7 1654#endif
9d5c8243
AK		 1655 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1656 /* print bus type/speed/width info */
7c510e4b 1657 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1658 netdev->name,
559e9c49
AD		 1659 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1660 "unknown"),
59c3de89
AD		 1661 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1662 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1663 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1664 "unknown"),
7c510e4b 1665 netdev->dev_addr);
9d5c8243
AK		 1666
1667 igb_read_part_num(hw, &part_num);
1668 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1669 (part_num >> 8), (part_num & 0xff));
1670
1671 dev_info(&pdev->dev,
1672 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1673 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1674 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1675 adapter->num_rx_queues, adapter->num_tx_queues);
1676
9d5c8243
AK		 1677 return 0;
1678
1679err_register:
1680 igb_release_hw_control(adapter);
1681err_eeprom:
1682 if (!igb_check_reset_block(hw))
f5f4cf08 1683 igb_reset_phy(hw);
9d5c8243
AK		 1684
1685 if (hw->flash_address)
1686 iounmap(hw->flash_address);
9d5c8243 1687err_sw_init:
047e0030 1688 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1689 iounmap(hw->hw_addr);
1690err_ioremap:
1691 free_netdev(netdev);
1692err_alloc_etherdev:
559e9c49
AD		 1693 pci_release_selected_regions(pdev,
1694 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1695err_pci_reg:
1696err_dma:
1697 pci_disable_device(pdev);
1698 return err;
1699}
1700
1701/**
1702 * igb_remove - Device Removal Routine
1703 * @pdev: PCI device information struct
1704 *
1705 * igb_remove is called by the PCI subsystem to alert the driver
1706 * that it should release a PCI device. The could be caused by a
1707 * Hot-Plug event, or because the driver is going to be removed from
1708 * memory.
1709 **/
1710static void __devexit igb_remove(struct pci_dev *pdev)
1711{
1712 struct net_device *netdev = pci_get_drvdata(pdev);
1713 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1714 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1715
1716 /* flush_scheduled work may reschedule our watchdog task, so
1717 * explicitly disable watchdog tasks from being rescheduled */
1718 set_bit(__IGB_DOWN, &adapter->state);
1719 del_timer_sync(&adapter->watchdog_timer);
1720 del_timer_sync(&adapter->phy_info_timer);
1721
1722 flush_scheduled_work();
1723
421e02f0 1724#ifdef CONFIG_IGB_DCA
7dfc16fa 1725 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1726 dev_info(&pdev->dev, "DCA disabled\n");
1727 dca_remove_requester(&pdev->dev);
7dfc16fa 1728 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1729 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1730 }
1731#endif
1732
9d5c8243
AK		 1733 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1734 * would have already happened in close and is redundant. */
1735 igb_release_hw_control(adapter);
1736
1737 unregister_netdev(netdev);
1738
047e0030 1739 igb_clear_interrupt_scheme(adapter);
9d5c8243 1740
37680117
AD		 1741#ifdef CONFIG_PCI_IOV
1742 /* reclaim resources allocated to VFs */
1743 if (adapter->vf_data) {
1744 /* disable iov and allow time for transactions to clear */
1745 pci_disable_sriov(pdev);
1746 msleep(500);
1747
1748 kfree(adapter->vf_data);
1749 adapter->vf_data = NULL;
1750 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1751 msleep(100);
1752 dev_info(&pdev->dev, "IOV Disabled\n");
1753 }
1754#endif
559e9c49 1755
28b0759c
AD		 1756 iounmap(hw->hw_addr);
1757 if (hw->flash_address)
1758 iounmap(hw->flash_address);
559e9c49
AD		 1759 pci_release_selected_regions(pdev,
1760 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1761
1762 free_netdev(netdev);
1763
19d5afd4 1764 pci_disable_pcie_error_reporting(pdev);
40a914fa 1765
9d5c8243
AK		 1766 pci_disable_device(pdev);
1767}
1768
a6b623e0
AD		 1769/**
1770 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1771 * @adapter: board private structure to initialize
1772 *
1773 * This function initializes the vf specific data storage and then attempts to
1774 * allocate the VFs. The reason for ordering it this way is because it is much
1775 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1776 * the memory for the VFs.
1777 **/
1778static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1779{
1780#ifdef CONFIG_PCI_IOV
1781 struct pci_dev *pdev = adapter->pdev;
1782
1783 if (adapter->vfs_allocated_count > 7)
1784 adapter->vfs_allocated_count = 7;
1785
1786 if (adapter->vfs_allocated_count) {
1787 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1788 sizeof(struct vf_data_storage),
1789 GFP_KERNEL);
1790 /* if allocation failed then we do not support SR-IOV */
1791 if (!adapter->vf_data) {
1792 adapter->vfs_allocated_count = 0;
1793 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1794 "Data Storage\n");
1795 }
1796 }
1797
1798 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1799 kfree(adapter->vf_data);
1800 adapter->vf_data = NULL;
1801#endif /* CONFIG_PCI_IOV */
1802 adapter->vfs_allocated_count = 0;
1803#ifdef CONFIG_PCI_IOV
1804 } else {
1805 unsigned char mac_addr[ETH_ALEN];
1806 int i;
1807 dev_info(&pdev->dev, "%d vfs allocated\n",
1808 adapter->vfs_allocated_count);
1809 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1810 random_ether_addr(mac_addr);
1811 igb_set_vf_mac(adapter, i, mac_addr);
1812 }
1813 }
1814#endif /* CONFIG_PCI_IOV */
1815}
1816
115f459a
AD		 1817
1818/**
1819 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1820 * @adapter: board private structure to initialize
1821 *
1822 * igb_init_hw_timer initializes the function pointer and values for the hw
1823 * timer found in hardware.
1824 **/
1825static void igb_init_hw_timer(struct igb_adapter *adapter)
1826{
1827 struct e1000_hw *hw = &adapter->hw;
1828
1829 switch (hw->mac.type) {
55cac248
AD		 1830 case e1000_82580:
1831 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1832 adapter->cycles.read = igb_read_clock;
1833 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1834 adapter->cycles.mult = 1;
1835 /*
1836 * The 82580 timesync updates the system timer every 8ns by 8ns
1837 * and the value cannot be shifted. Instead we need to shift
1838 * the registers to generate a 64bit timer value. As a result
1839 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1840 * 24 in order to generate a larger value for synchronization.
1841 */
1842 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1843 /* disable system timer temporarily by setting bit 31 */
1844 wr32(E1000_TSAUXC, 0x80000000);
1845 wrfl();
1846
1847 /* Set registers so that rollover occurs soon to test this. */
1848 wr32(E1000_SYSTIMR, 0x00000000);
1849 wr32(E1000_SYSTIML, 0x80000000);
1850 wr32(E1000_SYSTIMH, 0x000000FF);
1851 wrfl();
1852
1853 /* enable system timer by clearing bit 31 */
1854 wr32(E1000_TSAUXC, 0x0);
1855 wrfl();
1856
1857 timecounter_init(&adapter->clock,
1858 &adapter->cycles,
1859 ktime_to_ns(ktime_get_real()));
1860 /*
1861 * Synchronize our NIC clock against system wall clock. NIC
1862 * time stamp reading requires ~3us per sample, each sample
1863 * was pretty stable even under load => only require 10
1864 * samples for each offset comparison.
1865 */
1866 memset(&adapter->compare, 0, sizeof(adapter->compare));
1867 adapter->compare.source = &adapter->clock;
1868 adapter->compare.target = ktime_get_real;
1869 adapter->compare.num_samples = 10;
1870 timecompare_update(&adapter->compare, 0);
1871 break;
115f459a
AD		 1872 case e1000_82576:
1873 /*
1874 * Initialize hardware timer: we keep it running just in case
1875 * that some program needs it later on.
1876 */
1877 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1878 adapter->cycles.read = igb_read_clock;
1879 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1880 adapter->cycles.mult = 1;
1881 /**
1882 * Scale the NIC clock cycle by a large factor so that
1883 * relatively small clock corrections can be added or
1884 * substracted at each clock tick. The drawbacks of a large
1885 * factor are a) that the clock register overflows more quickly
1886 * (not such a big deal) and b) that the increment per tick has
1887 * to fit into 24 bits. As a result we need to use a shift of
1888 * 19 so we can fit a value of 16 into the TIMINCA register.
1889 */
1890 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1891 wr32(E1000_TIMINCA,
1892 (1 << E1000_TIMINCA_16NS_SHIFT) |
1893 (16 << IGB_82576_TSYNC_SHIFT));
1894
1895 /* Set registers so that rollover occurs soon to test this. */
1896 wr32(E1000_SYSTIML, 0x00000000);
1897 wr32(E1000_SYSTIMH, 0xFF800000);
1898 wrfl();
1899
1900 timecounter_init(&adapter->clock,
1901 &adapter->cycles,
1902 ktime_to_ns(ktime_get_real()));
1903 /*
1904 * Synchronize our NIC clock against system wall clock. NIC
1905 * time stamp reading requires ~3us per sample, each sample
1906 * was pretty stable even under load => only require 10
1907 * samples for each offset comparison.
1908 */
1909 memset(&adapter->compare, 0, sizeof(adapter->compare));
1910 adapter->compare.source = &adapter->clock;
1911 adapter->compare.target = ktime_get_real;
1912 adapter->compare.num_samples = 10;
1913 timecompare_update(&adapter->compare, 0);
1914 break;
1915 case e1000_82575:
1916 /* 82575 does not support timesync */
1917 default:
1918 break;
1919 }
1920
1921}
1922
9d5c8243
AK		 1923/**
1924 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1925 * @adapter: board private structure to initialize
1926 *
1927 * igb_sw_init initializes the Adapter private data structure.
1928 * Fields are initialized based on PCI device information and
1929 * OS network device settings (MTU size).
1930 **/
1931static int __devinit igb_sw_init(struct igb_adapter *adapter)
1932{
1933 struct e1000_hw *hw = &adapter->hw;
1934 struct net_device *netdev = adapter->netdev;
1935 struct pci_dev *pdev = adapter->pdev;
1936
1937 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1938
68fd9910
AD		 1939 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1940 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1941 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1942 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1943
9d5c8243
AK		 1944 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1945 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1946
a6b623e0
AD		 1947#ifdef CONFIG_PCI_IOV
1948 if (hw->mac.type == e1000_82576)
1949 adapter->vfs_allocated_count = max_vfs;
1950
1951#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 1952 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1953
1954 /*
1955 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1956 * then we should combine the queues into a queue pair in order to
1957 * conserve interrupts due to limited supply
1958 */
1959 if ((adapter->rss_queues > 4) ||
1960 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1961 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1962
a6b623e0 1963 /* This call may decrease the number of queues */
047e0030 1964 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1965 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1966 return -ENOMEM;
1967 }
1968
115f459a 1969 igb_init_hw_timer(adapter);
a6b623e0
AD		 1970 igb_probe_vfs(adapter);
1971
9d5c8243
AK		 1972 /* Explicitly disable IRQ since the NIC can be in any state. */
1973 igb_irq_disable(adapter);
1974
1975 set_bit(__IGB_DOWN, &adapter->state);
1976 return 0;
1977}
1978
1979/**
1980 * igb_open - Called when a network interface is made active
1981 * @netdev: network interface device structure
1982 *
1983 * Returns 0 on success, negative value on failure
1984 *
1985 * The open entry point is called when a network interface is made
1986 * active by the system (IFF_UP). At this point all resources needed
1987 * for transmit and receive operations are allocated, the interrupt
1988 * handler is registered with the OS, the watchdog timer is started,
1989 * and the stack is notified that the interface is ready.
1990 **/
1991static int igb_open(struct net_device *netdev)
1992{
1993 struct igb_adapter *adapter = netdev_priv(netdev);
1994 struct e1000_hw *hw = &adapter->hw;
1995 int err;
1996 int i;
1997
1998 /* disallow open during test */
1999 if (test_bit(__IGB_TESTING, &adapter->state))
2000 return -EBUSY;
2001
b168dfc5
JB		 2002 netif_carrier_off(netdev);
2003
9d5c8243
AK		 2004 /* allocate transmit descriptors */
2005 err = igb_setup_all_tx_resources(adapter);
2006 if (err)
2007 goto err_setup_tx;
2008
2009 /* allocate receive descriptors */
2010 err = igb_setup_all_rx_resources(adapter);
2011 if (err)
2012 goto err_setup_rx;
2013
88a268c1 2014 igb_power_up_link(adapter);
9d5c8243 2015
9d5c8243
AK		 2016 /* before we allocate an interrupt, we must be ready to handle it.
2017 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2018 * as soon as we call pci_request_irq, so we have to setup our
2019 * clean_rx handler before we do so. */
2020 igb_configure(adapter);
2021
2022 err = igb_request_irq(adapter);
2023 if (err)
2024 goto err_req_irq;
2025
2026 /* From here on the code is the same as igb_up() */
2027 clear_bit(__IGB_DOWN, &adapter->state);
2028
047e0030
AD		 2029 for (i = 0; i < adapter->num_q_vectors; i++) {
2030 struct igb_q_vector *q_vector = adapter->q_vector[i];
2031 napi_enable(&q_vector->napi);
2032 }
9d5c8243
AK		 2033
2034 /* Clear any pending interrupts. */
2035 rd32(E1000_ICR);
844290e5
PW		 2036
2037 igb_irq_enable(adapter);
2038
d4960307
AD		 2039 /* notify VFs that reset has been completed */
2040 if (adapter->vfs_allocated_count) {
2041 u32 reg_data = rd32(E1000_CTRL_EXT);
2042 reg_data |= E1000_CTRL_EXT_PFRSTD;
2043 wr32(E1000_CTRL_EXT, reg_data);
2044 }
2045
d55b53ff
JK		 2046 netif_tx_start_all_queues(netdev);
2047
25568a53
AD		 2048 /* start the watchdog. */
2049 hw->mac.get_link_status = 1;
2050 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2051
2052 return 0;
2053
2054err_req_irq:
2055 igb_release_hw_control(adapter);
88a268c1 2056 igb_power_down_link(adapter);
9d5c8243
AK		 2057 igb_free_all_rx_resources(adapter);
2058err_setup_rx:
2059 igb_free_all_tx_resources(adapter);
2060err_setup_tx:
2061 igb_reset(adapter);
2062
2063 return err;
2064}
2065
2066/**
2067 * igb_close - Disables a network interface
2068 * @netdev: network interface device structure
2069 *
2070 * Returns 0, this is not allowed to fail
2071 *
2072 * The close entry point is called when an interface is de-activated
2073 * by the OS. The hardware is still under the driver's control, but
2074 * needs to be disabled. A global MAC reset is issued to stop the
2075 * hardware, and all transmit and receive resources are freed.
2076 **/
2077static int igb_close(struct net_device *netdev)
2078{
2079 struct igb_adapter *adapter = netdev_priv(netdev);
2080
2081 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2082 igb_down(adapter);
2083
2084 igb_free_irq(adapter);
2085
2086 igb_free_all_tx_resources(adapter);
2087 igb_free_all_rx_resources(adapter);
2088
9d5c8243
AK		 2089 return 0;
2090}
2091
2092/**
2093 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2094 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2095 *
2096 * Return 0 on success, negative on failure
2097 **/
80785298 2098int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2099{
80785298 2100 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 2101 int size;
2102
2103 size = sizeof(struct igb_buffer) * tx_ring->count;
2104 tx_ring->buffer_info = vmalloc(size);
2105 if (!tx_ring->buffer_info)
2106 goto err;
2107 memset(tx_ring->buffer_info, 0, size);
2108
2109 /* round up to nearest 4K */
85e8d004 2110 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2111 tx_ring->size = ALIGN(tx_ring->size, 4096);
2112
439705e1
AD		 2113 tx_ring->desc = pci_alloc_consistent(pdev,
2114 tx_ring->size,
9d5c8243
AK		 2115 &tx_ring->dma);
2116
2117 if (!tx_ring->desc)
2118 goto err;
2119
9d5c8243
AK		 2120 tx_ring->next_to_use = 0;
2121 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2122 return 0;
2123
2124err:
2125 vfree(tx_ring->buffer_info);
047e0030 2126 dev_err(&pdev->dev,
9d5c8243
AK		 2127 "Unable to allocate memory for the transmit descriptor ring\n");
2128 return -ENOMEM;
2129}
2130
2131/**
2132 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2133 * (Descriptors) for all queues
2134 * @adapter: board private structure
2135 *
2136 * Return 0 on success, negative on failure
2137 **/
2138static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2139{
439705e1 2140 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2141 int i, err = 0;
2142
2143 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2144 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2145 if (err) {
439705e1 2146 dev_err(&pdev->dev,
9d5c8243
AK		 2147 "Allocation for Tx Queue %u failed\n", i);
2148 for (i--; i >= 0; i--)
3025a446 2149 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2150 break;
2151 }
2152 }
2153
a99955fc 2154 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2155 int r_idx = i % adapter->num_tx_queues;
3025a446 2156 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2157 }
9d5c8243
AK		 2158 return err;
2159}
2160
2161/**
85b430b4
AD		 2162 * igb_setup_tctl - configure the transmit control registers
2163 * @adapter: Board private structure
9d5c8243 2164 **/
d7ee5b3a 2165void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2166{
9d5c8243
AK		 2167 struct e1000_hw *hw = &adapter->hw;
2168 u32 tctl;
9d5c8243 2169
85b430b4
AD		 2170 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2171 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2172
2173 /* Program the Transmit Control Register */
9d5c8243
AK		 2174 tctl = rd32(E1000_TCTL);
2175 tctl &= ~E1000_TCTL_CT;
2176 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2177 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2178
2179 igb_config_collision_dist(hw);
2180
9d5c8243
AK		 2181 /* Enable transmits */
2182 tctl |= E1000_TCTL_EN;
2183
2184 wr32(E1000_TCTL, tctl);
2185}
2186
85b430b4
AD		 2187/**
2188 * igb_configure_tx_ring - Configure transmit ring after Reset
2189 * @adapter: board private structure
2190 * @ring: tx ring to configure
2191 *
2192 * Configure a transmit ring after a reset.
2193 **/
d7ee5b3a
AD		 2194void igb_configure_tx_ring(struct igb_adapter *adapter,
2195 struct igb_ring *ring)
85b430b4
AD		 2196{
2197 struct e1000_hw *hw = &adapter->hw;
2198 u32 txdctl;
2199 u64 tdba = ring->dma;
2200 int reg_idx = ring->reg_idx;
2201
2202 /* disable the queue */
2203 txdctl = rd32(E1000_TXDCTL(reg_idx));
2204 wr32(E1000_TXDCTL(reg_idx),
2205 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2206 wrfl();
2207 mdelay(10);
2208
2209 wr32(E1000_TDLEN(reg_idx),
2210 ring->count * sizeof(union e1000_adv_tx_desc));
2211 wr32(E1000_TDBAL(reg_idx),
2212 tdba & 0x00000000ffffffffULL);
2213 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2214
fce99e34
AD		 2215 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2216 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2217 writel(0, ring->head);
2218 writel(0, ring->tail);
85b430b4
AD		 2219
2220 txdctl |= IGB_TX_PTHRESH;
2221 txdctl |= IGB_TX_HTHRESH << 8;
2222 txdctl |= IGB_TX_WTHRESH << 16;
2223
2224 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2225 wr32(E1000_TXDCTL(reg_idx), txdctl);
2226}
2227
2228/**
2229 * igb_configure_tx - Configure transmit Unit after Reset
2230 * @adapter: board private structure
2231 *
2232 * Configure the Tx unit of the MAC after a reset.
2233 **/
2234static void igb_configure_tx(struct igb_adapter *adapter)
2235{
2236 int i;
2237
2238 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2239 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2240}
2241
9d5c8243
AK		 2242/**
2243 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2244 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2245 *
2246 * Returns 0 on success, negative on failure
2247 **/
80785298 2248int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2249{
80785298 2250 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2251 int size, desc_len;
2252
2253 size = sizeof(struct igb_buffer) * rx_ring->count;
2254 rx_ring->buffer_info = vmalloc(size);
2255 if (!rx_ring->buffer_info)
2256 goto err;
2257 memset(rx_ring->buffer_info, 0, size);
2258
2259 desc_len = sizeof(union e1000_adv_rx_desc);
2260
2261 /* Round up to nearest 4K */
2262 rx_ring->size = rx_ring->count * desc_len;
2263 rx_ring->size = ALIGN(rx_ring->size, 4096);
2264
2265 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2266 &rx_ring->dma);
2267
2268 if (!rx_ring->desc)
2269 goto err;
2270
2271 rx_ring->next_to_clean = 0;
2272 rx_ring->next_to_use = 0;
9d5c8243 2273
9d5c8243
AK		 2274 return 0;
2275
2276err:
2277 vfree(rx_ring->buffer_info);
439705e1 2278 rx_ring->buffer_info = NULL;
80785298 2279 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2280 "the receive descriptor ring\n");
2281 return -ENOMEM;
2282}
2283
2284/**
2285 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2286 * (Descriptors) for all queues
2287 * @adapter: board private structure
2288 *
2289 * Return 0 on success, negative on failure
2290 **/
2291static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2292{
439705e1 2293 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2294 int i, err = 0;
2295
2296 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2297 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2298 if (err) {
439705e1 2299 dev_err(&pdev->dev,
9d5c8243
AK		 2300 "Allocation for Rx Queue %u failed\n", i);
2301 for (i--; i >= 0; i--)
3025a446 2302 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2303 break;
2304 }
2305 }
2306
2307 return err;
2308}
2309
06cf2666
AD		 2310/**
2311 * igb_setup_mrqc - configure the multiple receive queue control registers
2312 * @adapter: Board private structure
2313 **/
2314static void igb_setup_mrqc(struct igb_adapter *adapter)
2315{
2316 struct e1000_hw *hw = &adapter->hw;
2317 u32 mrqc, rxcsum;
2318 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2319 union e1000_reta {
2320 u32 dword;
2321 u8 bytes[4];
2322 } reta;
2323 static const u8 rsshash[40] = {
2324 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2325 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2326 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2327 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2328
2329 /* Fill out hash function seeds */
2330 for (j = 0; j < 10; j++) {
2331 u32 rsskey = rsshash[(j * 4)];
2332 rsskey |= rsshash[(j * 4) + 1] << 8;
2333 rsskey |= rsshash[(j * 4) + 2] << 16;
2334 rsskey |= rsshash[(j * 4) + 3] << 24;
2335 array_wr32(E1000_RSSRK(0), j, rsskey);
2336 }
2337
a99955fc 2338 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2339
2340 if (adapter->vfs_allocated_count) {
2341 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2342 switch (hw->mac.type) {
55cac248
AD		 2343 case e1000_82580:
2344 num_rx_queues = 1;
2345 shift = 0;
2346 break;
06cf2666
AD		 2347 case e1000_82576:
2348 shift = 3;
2349 num_rx_queues = 2;
2350 break;
2351 case e1000_82575:
2352 shift = 2;
2353 shift2 = 6;
2354 default:
2355 break;
2356 }
2357 } else {
2358 if (hw->mac.type == e1000_82575)
2359 shift = 6;
2360 }
2361
2362 for (j = 0; j < (32 * 4); j++) {
2363 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2364 if (shift2)
2365 reta.bytes[j & 3] |= num_rx_queues << shift2;
2366 if ((j & 3) == 3)
2367 wr32(E1000_RETA(j >> 2), reta.dword);
2368 }
2369
2370 /*
2371 * Disable raw packet checksumming so that RSS hash is placed in
2372 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2373 * offloads as they are enabled by default
2374 */
2375 rxcsum = rd32(E1000_RXCSUM);
2376 rxcsum |= E1000_RXCSUM_PCSD;
2377
2378 if (adapter->hw.mac.type >= e1000_82576)
2379 /* Enable Receive Checksum Offload for SCTP */
2380 rxcsum |= E1000_RXCSUM_CRCOFL;
2381
2382 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2383 wr32(E1000_RXCSUM, rxcsum);
2384
2385 /* If VMDq is enabled then we set the appropriate mode for that, else
2386 * we default to RSS so that an RSS hash is calculated per packet even
2387 * if we are only using one queue */
2388 if (adapter->vfs_allocated_count) {
2389 if (hw->mac.type > e1000_82575) {
2390 /* Set the default pool for the PF's first queue */
2391 u32 vtctl = rd32(E1000_VT_CTL);
2392 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2393 E1000_VT_CTL_DISABLE_DEF_POOL);
2394 vtctl |= adapter->vfs_allocated_count <<
2395 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2396 wr32(E1000_VT_CTL, vtctl);
2397 }
a99955fc 2398 if (adapter->rss_queues > 1)
06cf2666
AD		 2399 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2400 else
2401 mrqc = E1000_MRQC_ENABLE_VMDQ;
2402 } else {
2403 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2404 }
2405 igb_vmm_control(adapter);
2406
2407 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2408 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2409 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2410 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2411 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2412 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2413 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2414 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2415
2416 wr32(E1000_MRQC, mrqc);
2417}
2418
9d5c8243
AK		 2419/**
2420 * igb_setup_rctl - configure the receive control registers
2421 * @adapter: Board private structure
2422 **/
d7ee5b3a 2423void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2424{
2425 struct e1000_hw *hw = &adapter->hw;
2426 u32 rctl;
9d5c8243
AK		 2427
2428 rctl = rd32(E1000_RCTL);
2429
2430 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2431 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2432
69d728ba 2433 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2434 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2435
87cb7e8c
AK		 2436 /*
2437 * enable stripping of CRC. It's unlikely this will break BMC
2438 * redirection as it did with e1000. Newer features require
2439 * that the HW strips the CRC.
73cd78f1 2440 */
87cb7e8c 2441 rctl |= E1000_RCTL_SECRC;
9d5c8243 2442
559e9c49 2443 /* disable store bad packets and clear size bits. */
ec54d7d6 2444 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2445
6ec43fe6
AD		 2446 /* enable LPE to prevent packets larger than max_frame_size */
2447 rctl |= E1000_RCTL_LPE;
9d5c8243 2448
952f72a8
AD		 2449 /* disable queue 0 to prevent tail write w/o re-config */
2450 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2451
e1739522
AD		 2452 /* Attention!!! For SR-IOV PF driver operations you must enable
2453 * queue drop for all VF and PF queues to prevent head of line blocking
2454 * if an un-trusted VF does not provide descriptors to hardware.
2455 */
2456 if (adapter->vfs_allocated_count) {
e1739522
AD		 2457 /* set all queue drop enable bits */
2458 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2459 }
2460
9d5c8243
AK		 2461 wr32(E1000_RCTL, rctl);
2462}
2463
7d5753f0
AD		 2464static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2465 int vfn)
2466{
2467 struct e1000_hw *hw = &adapter->hw;
2468 u32 vmolr;
2469
2470 /* if it isn't the PF check to see if VFs are enabled and
2471 * increase the size to support vlan tags */
2472 if (vfn < adapter->vfs_allocated_count &&
2473 adapter->vf_data[vfn].vlans_enabled)
2474 size += VLAN_TAG_SIZE;
2475
2476 vmolr = rd32(E1000_VMOLR(vfn));
2477 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2478 vmolr |= size | E1000_VMOLR_LPE;
2479 wr32(E1000_VMOLR(vfn), vmolr);
2480
2481 return 0;
2482}
2483
e1739522
AD		 2484/**
2485 * igb_rlpml_set - set maximum receive packet size
2486 * @adapter: board private structure
2487 *
2488 * Configure maximum receivable packet size.
2489 **/
2490static void igb_rlpml_set(struct igb_adapter *adapter)
2491{
2492 u32 max_frame_size = adapter->max_frame_size;
2493 struct e1000_hw *hw = &adapter->hw;
2494 u16 pf_id = adapter->vfs_allocated_count;
2495
2496 if (adapter->vlgrp)
2497 max_frame_size += VLAN_TAG_SIZE;
2498
2499 /* if vfs are enabled we set RLPML to the largest possible request
2500 * size and set the VMOLR RLPML to the size we need */
2501 if (pf_id) {
2502 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2503 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2504 }
2505
2506 wr32(E1000_RLPML, max_frame_size);
2507}
2508
8151d294
WM		 2509static inline void igb_set_vmolr(struct igb_adapter *adapter,
2510 int vfn, bool aupe)
7d5753f0
AD		 2511{
2512 struct e1000_hw *hw = &adapter->hw;
2513 u32 vmolr;
2514
2515 /*
2516 * This register exists only on 82576 and newer so if we are older then
2517 * we should exit and do nothing
2518 */
2519 if (hw->mac.type < e1000_82576)
2520 return;
2521
2522 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2523 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2524 if (aupe)
2525 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2526 else
2527 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2528
2529 /* clear all bits that might not be set */
2530 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2531
a99955fc 2532 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2533 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2534 /*
2535 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2536 * multicast packets
2537 */
2538 if (vfn <= adapter->vfs_allocated_count)
2539 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2540
2541 wr32(E1000_VMOLR(vfn), vmolr);
2542}
2543
85b430b4
AD		 2544/**
2545 * igb_configure_rx_ring - Configure a receive ring after Reset
2546 * @adapter: board private structure
2547 * @ring: receive ring to be configured
2548 *
2549 * Configure the Rx unit of the MAC after a reset.
2550 **/
d7ee5b3a
AD		 2551void igb_configure_rx_ring(struct igb_adapter *adapter,
2552 struct igb_ring *ring)
85b430b4
AD		 2553{
2554 struct e1000_hw *hw = &adapter->hw;
2555 u64 rdba = ring->dma;
2556 int reg_idx = ring->reg_idx;
952f72a8 2557 u32 srrctl, rxdctl;
85b430b4
AD		 2558
2559 /* disable the queue */
2560 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2561 wr32(E1000_RXDCTL(reg_idx),
2562 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2563
2564 /* Set DMA base address registers */
2565 wr32(E1000_RDBAL(reg_idx),
2566 rdba & 0x00000000ffffffffULL);
2567 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2568 wr32(E1000_RDLEN(reg_idx),
2569 ring->count * sizeof(union e1000_adv_rx_desc));
2570
2571 /* initialize head and tail */
fce99e34
AD		 2572 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2573 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2574 writel(0, ring->head);
2575 writel(0, ring->tail);
85b430b4 2576
952f72a8 2577 /* set descriptor configuration */
4c844851
AD		 2578 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2579 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2580 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2581#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2582 srrctl |= IGB_RXBUFFER_16384 >>
2583 E1000_SRRCTL_BSIZEPKT_SHIFT;
2584#else
2585 srrctl |= (PAGE_SIZE / 2) >>
2586 E1000_SRRCTL_BSIZEPKT_SHIFT;
2587#endif
2588 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2589 } else {
4c844851 2590 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2591 E1000_SRRCTL_BSIZEPKT_SHIFT;
2592 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2593 }
e6bdb6fe
NN		 2594 /* Only set Drop Enable if we are supporting multiple queues */
2595 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2596 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2597
2598 wr32(E1000_SRRCTL(reg_idx), srrctl);
2599
7d5753f0 2600 /* set filtering for VMDQ pools */
8151d294 2601 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2602
85b430b4
AD		 2603 /* enable receive descriptor fetching */
2604 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2605 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2606 rxdctl &= 0xFFF00000;
2607 rxdctl |= IGB_RX_PTHRESH;
2608 rxdctl |= IGB_RX_HTHRESH << 8;
2609 rxdctl |= IGB_RX_WTHRESH << 16;
2610 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2611}
2612
9d5c8243
AK		 2613/**
2614 * igb_configure_rx - Configure receive Unit after Reset
2615 * @adapter: board private structure
2616 *
2617 * Configure the Rx unit of the MAC after a reset.
2618 **/
2619static void igb_configure_rx(struct igb_adapter *adapter)
2620{
9107584e 2621 int i;
9d5c8243 2622
68d480c4
AD		 2623 /* set UTA to appropriate mode */
2624 igb_set_uta(adapter);
2625
26ad9178
AD		 2626 /* set the correct pool for the PF default MAC address in entry 0 */
2627 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2628 adapter->vfs_allocated_count);
2629
06cf2666
AD		 2630 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2631 * the Base and Length of the Rx Descriptor Ring */
2632 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2633 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2634}
2635
2636/**
2637 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2638 * @tx_ring: Tx descriptor ring for a specific queue
2639 *
2640 * Free all transmit software resources
2641 **/
68fd9910 2642void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2643{
3b644cf6 2644 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2645
2646 vfree(tx_ring->buffer_info);
2647 tx_ring->buffer_info = NULL;
2648
439705e1
AD		 2649 /* if not set, then don't free */
2650 if (!tx_ring->desc)
2651 return;
2652
80785298
AD		 2653 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2654 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2655
2656 tx_ring->desc = NULL;
2657}
2658
2659/**
2660 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2661 * @adapter: board private structure
2662 *
2663 * Free all transmit software resources
2664 **/
2665static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2666{
2667 int i;
2668
2669 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2670 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2671}
2672
b1a436c3
AD		 2673void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2674 struct igb_buffer *buffer_info)
9d5c8243 2675{
6366ad33
AD		 2676 if (buffer_info->dma) {
2677 if (buffer_info->mapped_as_page)
2678 pci_unmap_page(tx_ring->pdev,
2679 buffer_info->dma,
2680 buffer_info->length,
2681 PCI_DMA_TODEVICE);
2682 else
2683 pci_unmap_single(tx_ring->pdev,
2684 buffer_info->dma,
2685 buffer_info->length,
2686 PCI_DMA_TODEVICE);
2687 buffer_info->dma = 0;
2688 }
9d5c8243
AK		 2689 if (buffer_info->skb) {
2690 dev_kfree_skb_any(buffer_info->skb);
2691 buffer_info->skb = NULL;
2692 }
2693 buffer_info->time_stamp = 0;
6366ad33
AD		 2694 buffer_info->length = 0;
2695 buffer_info->next_to_watch = 0;
2696 buffer_info->mapped_as_page = false;
9d5c8243
AK		 2697}
2698
2699/**
2700 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2701 * @tx_ring: ring to be cleaned
2702 **/
3b644cf6 2703static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2704{
2705 struct igb_buffer *buffer_info;
2706 unsigned long size;
2707 unsigned int i;
2708
2709 if (!tx_ring->buffer_info)
2710 return;
2711 /* Free all the Tx ring sk_buffs */
2712
2713 for (i = 0; i < tx_ring->count; i++) {
2714 buffer_info = &tx_ring->buffer_info[i];
80785298 2715 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2716 }
2717
2718 size = sizeof(struct igb_buffer) * tx_ring->count;
2719 memset(tx_ring->buffer_info, 0, size);
2720
2721 /* Zero out the descriptor ring */
9d5c8243
AK		 2722 memset(tx_ring->desc, 0, tx_ring->size);
2723
2724 tx_ring->next_to_use = 0;
2725 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2726}
2727
2728/**
2729 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2730 * @adapter: board private structure
2731 **/
2732static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2733{
2734 int i;
2735
2736 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2737 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 2738}
2739
2740/**
2741 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2742 * @rx_ring: ring to clean the resources from
2743 *
2744 * Free all receive software resources
2745 **/
68fd9910 2746void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2747{
3b644cf6 2748 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2749
2750 vfree(rx_ring->buffer_info);
2751 rx_ring->buffer_info = NULL;
2752
439705e1
AD		 2753 /* if not set, then don't free */
2754 if (!rx_ring->desc)
2755 return;
2756
80785298
AD		 2757 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2758 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2759
2760 rx_ring->desc = NULL;
2761}
2762
2763/**
2764 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2765 * @adapter: board private structure
2766 *
2767 * Free all receive software resources
2768 **/
2769static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2770{
2771 int i;
2772
2773 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2774 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2775}
2776
2777/**
2778 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2779 * @rx_ring: ring to free buffers from
2780 **/
3b644cf6 2781static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2782{
2783 struct igb_buffer *buffer_info;
9d5c8243
AK		 2784 unsigned long size;
2785 unsigned int i;
2786
2787 if (!rx_ring->buffer_info)
2788 return;
439705e1 2789
9d5c8243
AK		 2790 /* Free all the Rx ring sk_buffs */
2791 for (i = 0; i < rx_ring->count; i++) {
2792 buffer_info = &rx_ring->buffer_info[i];
2793 if (buffer_info->dma) {
80785298
AD		 2794 pci_unmap_single(rx_ring->pdev,
2795 buffer_info->dma,
4c844851 2796 rx_ring->rx_buffer_len,
6ec43fe6 2797 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2798 buffer_info->dma = 0;
2799 }
2800
2801 if (buffer_info->skb) {
2802 dev_kfree_skb(buffer_info->skb);
2803 buffer_info->skb = NULL;
2804 }
6ec43fe6 2805 if (buffer_info->page_dma) {
80785298
AD		 2806 pci_unmap_page(rx_ring->pdev,
2807 buffer_info->page_dma,
6ec43fe6
AD		 2808 PAGE_SIZE / 2,
2809 PCI_DMA_FROMDEVICE);
2810 buffer_info->page_dma = 0;
2811 }
9d5c8243 2812 if (buffer_info->page) {
9d5c8243
AK		 2813 put_page(buffer_info->page);
2814 buffer_info->page = NULL;
bf36c1a0 2815 buffer_info->page_offset = 0;
9d5c8243
AK		 2816 }
2817 }
2818
9d5c8243
AK		 2819 size = sizeof(struct igb_buffer) * rx_ring->count;
2820 memset(rx_ring->buffer_info, 0, size);
2821
2822 /* Zero out the descriptor ring */
2823 memset(rx_ring->desc, 0, rx_ring->size);
2824
2825 rx_ring->next_to_clean = 0;
2826 rx_ring->next_to_use = 0;
9d5c8243
AK		 2827}
2828
2829/**
2830 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2831 * @adapter: board private structure
2832 **/
2833static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2834{
2835 int i;
2836
2837 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2838 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 2839}
2840
2841/**
2842 * igb_set_mac - Change the Ethernet Address of the NIC
2843 * @netdev: network interface device structure
2844 * @p: pointer to an address structure
2845 *
2846 * Returns 0 on success, negative on failure
2847 **/
2848static int igb_set_mac(struct net_device *netdev, void *p)
2849{
2850 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2851 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2852 struct sockaddr *addr = p;
2853
2854 if (!is_valid_ether_addr(addr->sa_data))
2855 return -EADDRNOTAVAIL;
2856
2857 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2858 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2859
26ad9178
AD		 2860 /* set the correct pool for the new PF MAC address in entry 0 */
2861 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2862 adapter->vfs_allocated_count);
e1739522 2863
9d5c8243
AK		 2864 return 0;
2865}
2866
2867/**
68d480c4 2868 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2869 * @netdev: network interface device structure
2870 *
68d480c4
AD		 2871 * Writes multicast address list to the MTA hash table.
2872 * Returns: -ENOMEM on failure
2873 * 0 on no addresses written
2874 * X on writing X addresses to MTA
9d5c8243 2875 **/
68d480c4 2876static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2877{
2878 struct igb_adapter *adapter = netdev_priv(netdev);
2879 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2880 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4 2881 u8 *mta_list;
9d5c8243
AK		 2882 int i;
2883
4cd24eaf 2884 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 2885 /* nothing to program, so clear mc list */
2886 igb_update_mc_addr_list(hw, NULL, 0);
2887 igb_restore_vf_multicasts(adapter);
2888 return 0;
2889 }
9d5c8243 2890
4cd24eaf 2891 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 2892 if (!mta_list)
2893 return -ENOMEM;
ff41f8dc 2894
68d480c4
AD		 2895 /* The shared function expects a packed array of only addresses. */
2896 mc_ptr = netdev->mc_list;
2897
4cd24eaf 2898 for (i = 0; i < netdev_mc_count(netdev); i++) {
68d480c4
AD		 2899 if (!mc_ptr)
2900 break;
2901 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2902 mc_ptr = mc_ptr->next;
746b9f02 2903 }
68d480c4
AD		 2904 igb_update_mc_addr_list(hw, mta_list, i);
2905 kfree(mta_list);
2906
4cd24eaf 2907 return netdev_mc_count(netdev);
68d480c4
AD		 2908}
2909
2910/**
2911 * igb_write_uc_addr_list - write unicast addresses to RAR table
2912 * @netdev: network interface device structure
2913 *
2914 * Writes unicast address list to the RAR table.
2915 * Returns: -ENOMEM on failure/insufficient address space
2916 * 0 on no addresses written
2917 * X on writing X addresses to the RAR table
2918 **/
2919static int igb_write_uc_addr_list(struct net_device *netdev)
2920{
2921 struct igb_adapter *adapter = netdev_priv(netdev);
2922 struct e1000_hw *hw = &adapter->hw;
2923 unsigned int vfn = adapter->vfs_allocated_count;
2924 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2925 int count = 0;
2926
2927 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 2928 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 2929 return -ENOMEM;
9d5c8243 2930
32e7bfc4 2931 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 2932 struct netdev_hw_addr *ha;
32e7bfc4
JP		 2933
2934 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 2935 if (!rar_entries)
2936 break;
26ad9178
AD		 2937 igb_rar_set_qsel(adapter, ha->addr,
2938 rar_entries--,
68d480c4
AD		 2939 vfn);
2940 count++;
ff41f8dc
AD		 2941 }
2942 }
2943 /* write the addresses in reverse order to avoid write combining */
2944 for (; rar_entries > 0 ; rar_entries--) {
2945 wr32(E1000_RAH(rar_entries), 0);
2946 wr32(E1000_RAL(rar_entries), 0);
2947 }
2948 wrfl();
2949
68d480c4
AD		 2950 return count;
2951}
2952
2953/**
2954 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2955 * @netdev: network interface device structure
2956 *
2957 * The set_rx_mode entry point is called whenever the unicast or multicast
2958 * address lists or the network interface flags are updated. This routine is
2959 * responsible for configuring the hardware for proper unicast, multicast,
2960 * promiscuous mode, and all-multi behavior.
2961 **/
2962static void igb_set_rx_mode(struct net_device *netdev)
2963{
2964 struct igb_adapter *adapter = netdev_priv(netdev);
2965 struct e1000_hw *hw = &adapter->hw;
2966 unsigned int vfn = adapter->vfs_allocated_count;
2967 u32 rctl, vmolr = 0;
2968 int count;
2969
2970 /* Check for Promiscuous and All Multicast modes */
2971 rctl = rd32(E1000_RCTL);
2972
2973 /* clear the effected bits */
2974 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2975
2976 if (netdev->flags & IFF_PROMISC) {
2977 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2978 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2979 } else {
2980 if (netdev->flags & IFF_ALLMULTI) {
2981 rctl |= E1000_RCTL_MPE;
2982 vmolr |= E1000_VMOLR_MPME;
2983 } else {
2984 /*
2985 * Write addresses to the MTA, if the attempt fails
2986 * then we should just turn on promiscous mode so
2987 * that we can at least receive multicast traffic
2988 */
2989 count = igb_write_mc_addr_list(netdev);
2990 if (count < 0) {
2991 rctl |= E1000_RCTL_MPE;
2992 vmolr |= E1000_VMOLR_MPME;
2993 } else if (count) {
2994 vmolr |= E1000_VMOLR_ROMPE;
2995 }
2996 }
2997 /*
2998 * Write addresses to available RAR registers, if there is not
2999 * sufficient space to store all the addresses then enable
3000 * unicast promiscous mode
3001 */
3002 count = igb_write_uc_addr_list(netdev);
3003 if (count < 0) {
3004 rctl |= E1000_RCTL_UPE;
3005 vmolr |= E1000_VMOLR_ROPE;
3006 }
3007 rctl |= E1000_RCTL_VFE;
28fc06f5 3008 }
68d480c4 3009 wr32(E1000_RCTL, rctl);
28fc06f5 3010
68d480c4
AD		 3011 /*
3012 * In order to support SR-IOV and eventually VMDq it is necessary to set
3013 * the VMOLR to enable the appropriate modes. Without this workaround
3014 * we will have issues with VLAN tag stripping not being done for frames
3015 * that are only arriving because we are the default pool
3016 */
3017 if (hw->mac.type < e1000_82576)
28fc06f5 3018 return;
9d5c8243 3019
68d480c4
AD		 3020 vmolr |= rd32(E1000_VMOLR(vfn)) &
3021 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3022 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3023 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3024}
3025
3026/* Need to wait a few seconds after link up to get diagnostic information from
3027 * the phy */
3028static void igb_update_phy_info(unsigned long data)
3029{
3030 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3031 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3032}
3033
4d6b725e
AD		 3034/**
3035 * igb_has_link - check shared code for link and determine up/down
3036 * @adapter: pointer to driver private info
3037 **/
3145535a 3038bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3039{
3040 struct e1000_hw *hw = &adapter->hw;
3041 bool link_active = false;
3042 s32 ret_val = 0;
3043
3044 /* get_link_status is set on LSC (link status) interrupt or
3045 * rx sequence error interrupt. get_link_status will stay
3046 * false until the e1000_check_for_link establishes link
3047 * for copper adapters ONLY
3048 */
3049 switch (hw->phy.media_type) {
3050 case e1000_media_type_copper:
3051 if (hw->mac.get_link_status) {
3052 ret_val = hw->mac.ops.check_for_link(hw);
3053 link_active = !hw->mac.get_link_status;
3054 } else {
3055 link_active = true;
3056 }
3057 break;
4d6b725e
AD		 3058 case e1000_media_type_internal_serdes:
3059 ret_val = hw->mac.ops.check_for_link(hw);
3060 link_active = hw->mac.serdes_has_link;
3061 break;
3062 default:
3063 case e1000_media_type_unknown:
3064 break;
3065 }
3066
3067 return link_active;
3068}
3069
9d5c8243
AK		 3070/**
3071 * igb_watchdog - Timer Call-back
3072 * @data: pointer to adapter cast into an unsigned long
3073 **/
3074static void igb_watchdog(unsigned long data)
3075{
3076 struct igb_adapter *adapter = (struct igb_adapter *)data;
3077 /* Do the rest outside of interrupt context */
3078 schedule_work(&adapter->watchdog_task);
3079}
3080
3081static void igb_watchdog_task(struct work_struct *work)
3082{
3083 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3084 struct igb_adapter,
3085 watchdog_task);
9d5c8243 3086 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3087 struct net_device *netdev = adapter->netdev;
9d5c8243 3088 u32 link;
7a6ea550 3089 int i;
9d5c8243 3090
4d6b725e 3091 link = igb_has_link(adapter);
9d5c8243
AK		 3092 if (link) {
3093 if (!netif_carrier_ok(netdev)) {
3094 u32 ctrl;
330a6d6a
AD		 3095 hw->mac.ops.get_speed_and_duplex(hw,
3096 &adapter->link_speed,
3097 &adapter->link_duplex);
9d5c8243
AK		 3098
3099 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3100 /* Links status message must follow this format */
3101 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3102 "Flow Control: %s\n",
559e9c49
AD		 3103 netdev->name,
3104 adapter->link_speed,
3105 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3106 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3107 ((ctrl & E1000_CTRL_TFCE) &&
3108 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3109 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3110 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243
AK		 3111
3112 /* tweak tx_queue_len according to speed/duplex and
3113 * adjust the timeout factor */
3114 netdev->tx_queue_len = adapter->tx_queue_len;
3115 adapter->tx_timeout_factor = 1;
3116 switch (adapter->link_speed) {
3117 case SPEED_10:
3118 netdev->tx_queue_len = 10;
3119 adapter->tx_timeout_factor = 14;
3120 break;
3121 case SPEED_100:
3122 netdev->tx_queue_len = 100;
3123 /* maybe add some timeout factor ? */
3124 break;
3125 }
3126
3127 netif_carrier_on(netdev);
9d5c8243 3128
4ae196df
AD		 3129 igb_ping_all_vfs(adapter);
3130
4b1a9877 3131 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3132 if (!test_bit(__IGB_DOWN, &adapter->state))
3133 mod_timer(&adapter->phy_info_timer,
3134 round_jiffies(jiffies + 2 * HZ));
3135 }
3136 } else {
3137 if (netif_carrier_ok(netdev)) {
3138 adapter->link_speed = 0;
3139 adapter->link_duplex = 0;
527d47c1
AD		 3140 /* Links status message must follow this format */
3141 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3142 netdev->name);
9d5c8243 3143 netif_carrier_off(netdev);
4b1a9877 3144
4ae196df
AD		 3145 igb_ping_all_vfs(adapter);
3146
4b1a9877 3147 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3148 if (!test_bit(__IGB_DOWN, &adapter->state))
3149 mod_timer(&adapter->phy_info_timer,
3150 round_jiffies(jiffies + 2 * HZ));
3151 }
3152 }
3153
9d5c8243 3154 igb_update_stats(adapter);
9d5c8243 3155
dbabb065 3156 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3157 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3158 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3159 /* We've lost link, so the controller stops DMA,
3160 * but we've got queued Tx work that's never going
3161 * to get done, so reset controller to flush Tx.
3162 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3163 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3164 adapter->tx_timeout_count++;
3165 schedule_work(&adapter->reset_task);
3166 /* return immediately since reset is imminent */
3167 return;
3168 }
9d5c8243 3169 }
9d5c8243 3170
dbabb065
AD		 3171 /* Force detection of hung controller every watchdog period */
3172 tx_ring->detect_tx_hung = true;
3173 }
f7ba205e 3174
9d5c8243 3175 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3176 if (adapter->msix_entries) {
047e0030
AD		 3177 u32 eics = 0;
3178 for (i = 0; i < adapter->num_q_vectors; i++) {
3179 struct igb_q_vector *q_vector = adapter->q_vector[i];
3180 eics |= q_vector->eims_value;
3181 }
7a6ea550
AD		 3182 wr32(E1000_EICS, eics);
3183 } else {
3184 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3185 }
9d5c8243 3186
9d5c8243
AK		 3187 /* Reset the timer */
3188 if (!test_bit(__IGB_DOWN, &adapter->state))
3189 mod_timer(&adapter->watchdog_timer,
3190 round_jiffies(jiffies + 2 * HZ));
3191}
3192
3193enum latency_range {
3194 lowest_latency = 0,
3195 low_latency = 1,
3196 bulk_latency = 2,
3197 latency_invalid = 255
3198};
3199
6eb5a7f1
AD		 3200/**
3201 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3202 *
3203 * Stores a new ITR value based on strictly on packet size. This
3204 * algorithm is less sophisticated than that used in igb_update_itr,
3205 * due to the difficulty of synchronizing statistics across multiple
3206 * receive rings. The divisors and thresholds used by this fuction
3207 * were determined based on theoretical maximum wire speed and testing
3208 * data, in order to minimize response time while increasing bulk
3209 * throughput.
3210 * This functionality is controlled by the InterruptThrottleRate module
3211 * parameter (see igb_param.c)
3212 * NOTE: This function is called only when operating in a multiqueue
3213 * receive environment.
047e0030 3214 * @q_vector: pointer to q_vector
6eb5a7f1 3215 **/
047e0030 3216static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3217{
047e0030 3218 int new_val = q_vector->itr_val;
6eb5a7f1 3219 int avg_wire_size = 0;
047e0030 3220 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3221
6eb5a7f1
AD		 3222 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3223 * ints/sec - ITR timer value of 120 ticks.
3224 */
3225 if (adapter->link_speed != SPEED_1000) {
047e0030 3226 new_val = 976;
6eb5a7f1 3227 goto set_itr_val;
9d5c8243 3228 }
047e0030
AD		 3229
3230 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3231 struct igb_ring *ring = q_vector->rx_ring;
3232 avg_wire_size = ring->total_bytes / ring->total_packets;
3233 }
3234
3235 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3236 struct igb_ring *ring = q_vector->tx_ring;
3237 avg_wire_size = max_t(u32, avg_wire_size,
3238 (ring->total_bytes /
3239 ring->total_packets));
3240 }
3241
3242 /* if avg_wire_size isn't set no work was done */
3243 if (!avg_wire_size)
3244 goto clear_counts;
9d5c8243 3245
6eb5a7f1
AD		 3246 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3247 avg_wire_size += 24;
3248
3249 /* Don't starve jumbo frames */
3250 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3251
6eb5a7f1
AD		 3252 /* Give a little boost to mid-size frames */
3253 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3254 new_val = avg_wire_size / 3;
3255 else
3256 new_val = avg_wire_size / 2;
9d5c8243 3257
abe1c363
NN		 3258 /* when in itr mode 3 do not exceed 20K ints/sec */
3259 if (adapter->rx_itr_setting == 3 && new_val < 196)
3260 new_val = 196;
3261
6eb5a7f1 3262set_itr_val:
047e0030
AD		 3263 if (new_val != q_vector->itr_val) {
3264 q_vector->itr_val = new_val;
3265 q_vector->set_itr = 1;
9d5c8243 3266 }
6eb5a7f1 3267clear_counts:
047e0030
AD		 3268 if (q_vector->rx_ring) {
3269 q_vector->rx_ring->total_bytes = 0;
3270 q_vector->rx_ring->total_packets = 0;
3271 }
3272 if (q_vector->tx_ring) {
3273 q_vector->tx_ring->total_bytes = 0;
3274 q_vector->tx_ring->total_packets = 0;
3275 }
9d5c8243
AK		 3276}
3277
3278/**
3279 * igb_update_itr - update the dynamic ITR value based on statistics
3280 * Stores a new ITR value based on packets and byte
3281 * counts during the last interrupt. The advantage of per interrupt
3282 * computation is faster updates and more accurate ITR for the current
3283 * traffic pattern. Constants in this function were computed
3284 * based on theoretical maximum wire speed and thresholds were set based
3285 * on testing data as well as attempting to minimize response time
3286 * while increasing bulk throughput.
3287 * this functionality is controlled by the InterruptThrottleRate module
3288 * parameter (see igb_param.c)
3289 * NOTE: These calculations are only valid when operating in a single-
3290 * queue environment.
3291 * @adapter: pointer to adapter
047e0030 3292 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3293 * @packets: the number of packets during this measurement interval
3294 * @bytes: the number of bytes during this measurement interval
3295 **/
3296static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3297 int packets, int bytes)
3298{
3299 unsigned int retval = itr_setting;
3300
3301 if (packets == 0)
3302 goto update_itr_done;
3303
3304 switch (itr_setting) {
3305 case lowest_latency:
3306 /* handle TSO and jumbo frames */
3307 if (bytes/packets > 8000)
3308 retval = bulk_latency;
3309 else if ((packets < 5) && (bytes > 512))
3310 retval = low_latency;
3311 break;
3312 case low_latency: /* 50 usec aka 20000 ints/s */
3313 if (bytes > 10000) {
3314 /* this if handles the TSO accounting */
3315 if (bytes/packets > 8000) {
3316 retval = bulk_latency;
3317 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3318 retval = bulk_latency;
3319 } else if ((packets > 35)) {
3320 retval = lowest_latency;
3321 }
3322 } else if (bytes/packets > 2000) {
3323 retval = bulk_latency;
3324 } else if (packets <= 2 && bytes < 512) {
3325 retval = lowest_latency;
3326 }
3327 break;
3328 case bulk_latency: /* 250 usec aka 4000 ints/s */
3329 if (bytes > 25000) {
3330 if (packets > 35)
3331 retval = low_latency;
1e5c3d21 3332 } else if (bytes < 1500) {
9d5c8243
AK		 3333 retval = low_latency;
3334 }
3335 break;
3336 }
3337
3338update_itr_done:
3339 return retval;
3340}
3341
6eb5a7f1 3342static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3343{
047e0030 3344 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3345 u16 current_itr;
047e0030 3346 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3347
3348 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3349 if (adapter->link_speed != SPEED_1000) {
3350 current_itr = 0;
3351 new_itr = 4000;
3352 goto set_itr_now;
3353 }
3354
3355 adapter->rx_itr = igb_update_itr(adapter,
3356 adapter->rx_itr,
3025a446
AD		 3357 q_vector->rx_ring->total_packets,
3358 q_vector->rx_ring->total_bytes);
9d5c8243 3359
047e0030
AD		 3360 adapter->tx_itr = igb_update_itr(adapter,
3361 adapter->tx_itr,
3025a446
AD		 3362 q_vector->tx_ring->total_packets,
3363 q_vector->tx_ring->total_bytes);
047e0030 3364 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3365
6eb5a7f1 3366 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3367 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3368 current_itr = low_latency;
3369
9d5c8243
AK		 3370 switch (current_itr) {
3371 /* counts and packets in update_itr are dependent on these numbers */
3372 case lowest_latency:
78b1f607 3373 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3374 break;
3375 case low_latency:
78b1f607 3376 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3377 break;
3378 case bulk_latency:
78b1f607 3379 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3380 break;
3381 default:
3382 break;
3383 }
3384
3385set_itr_now:
3025a446
AD		 3386 q_vector->rx_ring->total_bytes = 0;
3387 q_vector->rx_ring->total_packets = 0;
3388 q_vector->tx_ring->total_bytes = 0;
3389 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3390
047e0030 3391 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3392 /* this attempts to bias the interrupt rate towards Bulk
3393 * by adding intermediate steps when interrupt rate is
3394 * increasing */
047e0030
AD		 3395 new_itr = new_itr > q_vector->itr_val ?
3396 max((new_itr * q_vector->itr_val) /
3397 (new_itr + (q_vector->itr_val >> 2)),
3398 new_itr) :
9d5c8243
AK		 3399 new_itr;
3400 /* Don't write the value here; it resets the adapter's
3401 * internal timer, and causes us to delay far longer than
3402 * we should between interrupts. Instead, we write the ITR
3403 * value at the beginning of the next interrupt so the timing
3404 * ends up being correct.
3405 */
047e0030
AD		 3406 q_vector->itr_val = new_itr;
3407 q_vector->set_itr = 1;
9d5c8243
AK		 3408 }
3409
3410 return;
3411}
3412
9d5c8243
AK		 3413#define IGB_TX_FLAGS_CSUM 0x00000001
3414#define IGB_TX_FLAGS_VLAN 0x00000002
3415#define IGB_TX_FLAGS_TSO 0x00000004
3416#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3417#define IGB_TX_FLAGS_TSTAMP 0x00000010
3418#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3419#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3420
85ad76b2 3421static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3422 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3423{
3424 struct e1000_adv_tx_context_desc *context_desc;
3425 unsigned int i;
3426 int err;
3427 struct igb_buffer *buffer_info;
3428 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3429 u32 mss_l4len_idx;
3430 u8 l4len;
9d5c8243
AK		 3431
3432 if (skb_header_cloned(skb)) {
3433 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3434 if (err)
3435 return err;
3436 }
3437
3438 l4len = tcp_hdrlen(skb);
3439 *hdr_len += l4len;
3440
3441 if (skb->protocol == htons(ETH_P_IP)) {
3442 struct iphdr *iph = ip_hdr(skb);
3443 iph->tot_len = 0;
3444 iph->check = 0;
3445 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3446 iph->daddr, 0,
3447 IPPROTO_TCP,
3448 0);
8e1e8a47 3449 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3450 ipv6_hdr(skb)->payload_len = 0;
3451 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3452 &ipv6_hdr(skb)->daddr,
3453 0, IPPROTO_TCP, 0);
3454 }
3455
3456 i = tx_ring->next_to_use;
3457
3458 buffer_info = &tx_ring->buffer_info[i];
3459 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3460 /* VLAN MACLEN IPLEN */
3461 if (tx_flags & IGB_TX_FLAGS_VLAN)
3462 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3463 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3464 *hdr_len += skb_network_offset(skb);
3465 info |= skb_network_header_len(skb);
3466 *hdr_len += skb_network_header_len(skb);
3467 context_desc->vlan_macip_lens = cpu_to_le32(info);
3468
3469 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3470 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3471
3472 if (skb->protocol == htons(ETH_P_IP))
3473 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3474 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3475
3476 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3477
3478 /* MSS L4LEN IDX */
3479 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3480 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3481
73cd78f1 3482 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3483 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3484 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3485
3486 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3487 context_desc->seqnum_seed = 0;
3488
3489 buffer_info->time_stamp = jiffies;
0e014cb1 3490 buffer_info->next_to_watch = i;
9d5c8243
AK		 3491 buffer_info->dma = 0;
3492 i++;
3493 if (i == tx_ring->count)
3494 i = 0;
3495
3496 tx_ring->next_to_use = i;
3497
3498 return true;
3499}
3500
85ad76b2
AD		 3501static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3502 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3503{
3504 struct e1000_adv_tx_context_desc *context_desc;
80785298 3505 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3506 struct igb_buffer *buffer_info;
3507 u32 info = 0, tu_cmd = 0;
80785298 3508 unsigned int i;
9d5c8243
AK		 3509
3510 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3511 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3512 i = tx_ring->next_to_use;
3513 buffer_info = &tx_ring->buffer_info[i];
3514 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3515
3516 if (tx_flags & IGB_TX_FLAGS_VLAN)
3517 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3518
9d5c8243
AK		 3519 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3520 if (skb->ip_summed == CHECKSUM_PARTIAL)
3521 info |= skb_network_header_len(skb);
3522
3523 context_desc->vlan_macip_lens = cpu_to_le32(info);
3524
3525 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3526
3527 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3528 __be16 protocol;
3529
3530 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3531 const struct vlan_ethhdr *vhdr =
3532 (const struct vlan_ethhdr*)skb->data;
3533
3534 protocol = vhdr->h_vlan_encapsulated_proto;
3535 } else {
3536 protocol = skb->protocol;
3537 }
3538
3539 switch (protocol) {
09640e63 3540 case cpu_to_be16(ETH_P_IP):
9d5c8243 3541 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3542 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3543 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3544 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3545 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3546 break;
09640e63 3547 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3548 /* XXX what about other V6 headers?? */
3549 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3550 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3551 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3552 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3553 break;
3554 default:
3555 if (unlikely(net_ratelimit()))
80785298 3556 dev_warn(&pdev->dev,
44b0cda3
MW		 3557 "partial checksum but proto=%x!\n",
3558 skb->protocol);
3559 break;
3560 }
9d5c8243
AK		 3561 }
3562
3563 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3564 context_desc->seqnum_seed = 0;
85ad76b2 3565 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3566 context_desc->mss_l4len_idx =
85ad76b2 3567 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3568
3569 buffer_info->time_stamp = jiffies;
0e014cb1 3570 buffer_info->next_to_watch = i;
9d5c8243
AK		 3571 buffer_info->dma = 0;
3572
3573 i++;
3574 if (i == tx_ring->count)
3575 i = 0;
3576 tx_ring->next_to_use = i;
3577
3578 return true;
3579 }
9d5c8243
AK		 3580 return false;
3581}
3582
3583#define IGB_MAX_TXD_PWR 16
3584#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3585
80785298 3586static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3587 unsigned int first)
9d5c8243
AK		 3588{
3589 struct igb_buffer *buffer_info;
80785298 3590 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3591 unsigned int len = skb_headlen(skb);
3592 unsigned int count = 0, i;
3593 unsigned int f;
3594
3595 i = tx_ring->next_to_use;
3596
3597 buffer_info = &tx_ring->buffer_info[i];
3598 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3599 buffer_info->length = len;
3600 /* set time_stamp *before* dma to help avoid a possible race */
3601 buffer_info->time_stamp = jiffies;
0e014cb1 3602 buffer_info->next_to_watch = i;
6366ad33
AD		 3603 buffer_info->dma = pci_map_single(pdev, skb->data, len,
3604 PCI_DMA_TODEVICE);
3605 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3606 goto dma_error;
9d5c8243
AK		 3607
3608 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3609 struct skb_frag_struct *frag;
3610
8581145f 3611 count++;
65689fef
AD		 3612 i++;
3613 if (i == tx_ring->count)
3614 i = 0;
3615
9d5c8243
AK		 3616 frag = &skb_shinfo(skb)->frags[f];
3617 len = frag->size;
3618
3619 buffer_info = &tx_ring->buffer_info[i];
3620 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3621 buffer_info->length = len;
3622 buffer_info->time_stamp = jiffies;
0e014cb1 3623 buffer_info->next_to_watch = i;
6366ad33
AD		 3624 buffer_info->mapped_as_page = true;
3625 buffer_info->dma = pci_map_page(pdev,
3626 frag->page,
3627 frag->page_offset,
3628 len,
3629 PCI_DMA_TODEVICE);
3630 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3631 goto dma_error;
3632
9d5c8243
AK		 3633 }
3634
9d5c8243 3635 tx_ring->buffer_info[i].skb = skb;
40e90c26 3636 tx_ring->buffer_info[i].gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
0e014cb1 3637 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3638
cdfd01fc 3639 return ++count;
6366ad33
AD		 3640
3641dma_error:
3642 dev_err(&pdev->dev, "TX DMA map failed\n");
3643
3644 /* clear timestamp and dma mappings for failed buffer_info mapping */
3645 buffer_info->dma = 0;
3646 buffer_info->time_stamp = 0;
3647 buffer_info->length = 0;
3648 buffer_info->next_to_watch = 0;
3649 buffer_info->mapped_as_page = false;
3650 count--;
3651
3652 /* clear timestamp and dma mappings for remaining portion of packet */
3653 while (count >= 0) {
3654 count--;
3655 i--;
3656 if (i < 0)
3657 i += tx_ring->count;
3658 buffer_info = &tx_ring->buffer_info[i];
3659 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3660 }
3661
3662 return 0;
9d5c8243
AK		 3663}
3664
85ad76b2 3665static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3666 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3667 u8 hdr_len)
3668{
cdfd01fc 3669 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3670 struct igb_buffer *buffer_info;
3671 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3672 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3673
3674 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3675 E1000_ADVTXD_DCMD_DEXT);
3676
3677 if (tx_flags & IGB_TX_FLAGS_VLAN)
3678 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3679
33af6bcc
PO		 3680 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3681 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3682
9d5c8243
AK		 3683 if (tx_flags & IGB_TX_FLAGS_TSO) {
3684 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3685
3686 /* insert tcp checksum */
3687 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3688
3689 /* insert ip checksum */
3690 if (tx_flags & IGB_TX_FLAGS_IPV4)
3691 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3692
3693 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3694 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3695 }
3696
85ad76b2
AD		 3697 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3698 (tx_flags & (IGB_TX_FLAGS_CSUM |
3699 IGB_TX_FLAGS_TSO |
7dfc16fa 3700 IGB_TX_FLAGS_VLAN)))
85ad76b2 3701 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3702
3703 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3704
cdfd01fc 3705 do {
9d5c8243
AK		 3706 buffer_info = &tx_ring->buffer_info[i];
3707 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3708 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3709 tx_desc->read.cmd_type_len =
3710 cpu_to_le32(cmd_type_len | buffer_info->length);
3711 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3712 count--;
9d5c8243
AK		 3713 i++;
3714 if (i == tx_ring->count)
3715 i = 0;
cdfd01fc 3716 } while (count > 0);
9d5c8243 3717
85ad76b2 3718 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3719 /* Force memory writes to complete before letting h/w
3720 * know there are new descriptors to fetch. (Only
3721 * applicable for weak-ordered memory model archs,
3722 * such as IA-64). */
3723 wmb();
3724
3725 tx_ring->next_to_use = i;
fce99e34 3726 writel(i, tx_ring->tail);
9d5c8243
AK		 3727 /* we need this if more than one processor can write to our tail
3728 * at a time, it syncronizes IO on IA64/Altix systems */
3729 mmiowb();
3730}
3731
e694e964 3732static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3733{
e694e964
AD		 3734 struct net_device *netdev = tx_ring->netdev;
3735
661086df 3736 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3737
9d5c8243
AK		 3738 /* Herbert's original patch had:
3739 * smp_mb__after_netif_stop_queue();
3740 * but since that doesn't exist yet, just open code it. */
3741 smp_mb();
3742
3743 /* We need to check again in a case another CPU has just
3744 * made room available. */
c493ea45 3745 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3746 return -EBUSY;
3747
3748 /* A reprieve! */
661086df 3749 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3750 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3751 return 0;
3752}
3753
717ba089 3754static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3755{
c493ea45 3756 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3757 return 0;
e694e964 3758 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3759}
3760
b1a436c3
AD		 3761netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3762 struct igb_ring *tx_ring)
9d5c8243 3763{
e694e964 3764 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 3765 int tso = 0, count;
91d4ee33
NN		 3766 u32 tx_flags = 0;
3767 u16 first;
3768 u8 hdr_len = 0;
c5b9bd5e 3769 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3770
9d5c8243
AK		 3771 /* need: 1 descriptor per page,
3772 * + 2 desc gap to keep tail from touching head,
3773 * + 1 desc for skb->data,
3774 * + 1 desc for context descriptor,
3775 * otherwise try next time */
e694e964 3776 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3777 /* this is a hard error */
9d5c8243
AK		 3778 return NETDEV_TX_BUSY;
3779 }
33af6bcc 3780
33af6bcc
PO		 3781 if (unlikely(shtx->hardware)) {
3782 shtx->in_progress = 1;
3783 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3784 }
9d5c8243 3785
cdfd01fc 3786 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3787 tx_flags |= IGB_TX_FLAGS_VLAN;
3788 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3789 }
3790
661086df
PWJ		 3791 if (skb->protocol == htons(ETH_P_IP))
3792 tx_flags |= IGB_TX_FLAGS_IPV4;
3793
0e014cb1 3794 first = tx_ring->next_to_use;
85ad76b2
AD		 3795 if (skb_is_gso(skb)) {
3796 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3797
85ad76b2
AD		 3798 if (tso < 0) {
3799 dev_kfree_skb_any(skb);
3800 return NETDEV_TX_OK;
3801 }
9d5c8243
AK		 3802 }
3803
3804 if (tso)
3805 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3806 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3807 (skb->ip_summed == CHECKSUM_PARTIAL))
3808 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3809
65689fef 3810 /*
cdfd01fc 3811 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3812 * has occured and we need to rewind the descriptor queue
3813 */
80785298 3814 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 3815 if (!count) {
65689fef
AD		 3816 dev_kfree_skb_any(skb);
3817 tx_ring->buffer_info[first].time_stamp = 0;
3818 tx_ring->next_to_use = first;
85ad76b2 3819 return NETDEV_TX_OK;
65689fef 3820 }
9d5c8243 3821
85ad76b2
AD		 3822 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3823
3824 /* Make sure there is space in the ring for the next send. */
e694e964 3825 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3826
9d5c8243
AK		 3827 return NETDEV_TX_OK;
3828}
3829
3b29a56d
SH		 3830static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3831 struct net_device *netdev)
9d5c8243
AK		 3832{
3833 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3834 struct igb_ring *tx_ring;
661086df 3835 int r_idx = 0;
b1a436c3
AD		 3836
3837 if (test_bit(__IGB_DOWN, &adapter->state)) {
3838 dev_kfree_skb_any(skb);
3839 return NETDEV_TX_OK;
3840 }
3841
3842 if (skb->len <= 0) {
3843 dev_kfree_skb_any(skb);
3844 return NETDEV_TX_OK;
3845 }
3846
1bfaf07b 3847 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3848 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3849
3850 /* This goes back to the question of how to logically map a tx queue
3851 * to a flow. Right now, performance is impacted slightly negatively
3852 * if using multiple tx queues. If the stack breaks away from a
3853 * single qdisc implementation, we can look at this again. */
e694e964 3854 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3855}
3856
3857/**
3858 * igb_tx_timeout - Respond to a Tx Hang
3859 * @netdev: network interface device structure
3860 **/
3861static void igb_tx_timeout(struct net_device *netdev)
3862{
3863 struct igb_adapter *adapter = netdev_priv(netdev);
3864 struct e1000_hw *hw = &adapter->hw;
3865
3866 /* Do the reset outside of interrupt context */
3867 adapter->tx_timeout_count++;
f7ba205e 3868
55cac248
AD		 3869 if (hw->mac.type == e1000_82580)
3870 hw->dev_spec._82575.global_device_reset = true;
3871
9d5c8243 3872 schedule_work(&adapter->reset_task);
265de409
AD		 3873 wr32(E1000_EICS,
3874 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3875}
3876
3877static void igb_reset_task(struct work_struct *work)
3878{
3879 struct igb_adapter *adapter;
3880 adapter = container_of(work, struct igb_adapter, reset_task);
3881
3882 igb_reinit_locked(adapter);
3883}
3884
3885/**
3886 * igb_get_stats - Get System Network Statistics
3887 * @netdev: network interface device structure
3888 *
3889 * Returns the address of the device statistics structure.
3890 * The statistics are actually updated from the timer callback.
3891 **/
73cd78f1 3892static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3893{
9d5c8243 3894 /* only return the current stats */
8d24e933 3895 return &netdev->stats;
9d5c8243
AK		 3896}
3897
3898/**
3899 * igb_change_mtu - Change the Maximum Transfer Unit
3900 * @netdev: network interface device structure
3901 * @new_mtu: new value for maximum frame size
3902 *
3903 * Returns 0 on success, negative on failure
3904 **/
3905static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3906{
3907 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 3908 struct pci_dev *pdev = adapter->pdev;
9d5c8243 3909 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3910 u32 rx_buffer_len, i;
9d5c8243 3911
c809d227 3912 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 3913 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 3914 return -EINVAL;
3915 }
3916
9d5c8243 3917 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 3918 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 3919 return -EINVAL;
3920 }
3921
3922 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3923 msleep(1);
73cd78f1 3924
9d5c8243
AK		 3925 /* igb_down has a dependency on max_frame_size */
3926 adapter->max_frame_size = max_frame;
559e9c49 3927
9d5c8243
AK		 3928 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3929 * means we reserve 2 more, this pushes us to allocate from the next
3930 * larger slab size.
3931 * i.e. RXBUFFER_2048 --> size-4096 slab
3932 */
3933
7d95b717 3934 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3935 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3936 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3937 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3938 else
4c844851
AD		 3939 rx_buffer_len = IGB_RXBUFFER_128;
3940
3941 if (netif_running(netdev))
3942 igb_down(adapter);
9d5c8243 3943
090b1795 3944 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 3945 netdev->mtu, new_mtu);
3946 netdev->mtu = new_mtu;
3947
4c844851 3948 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3949 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 3950
9d5c8243
AK		 3951 if (netif_running(netdev))
3952 igb_up(adapter);
3953 else
3954 igb_reset(adapter);
3955
3956 clear_bit(__IGB_RESETTING, &adapter->state);
3957
3958 return 0;
3959}
3960
3961/**
3962 * igb_update_stats - Update the board statistics counters
3963 * @adapter: board private structure
3964 **/
3965
3966void igb_update_stats(struct igb_adapter *adapter)
3967{
128e45eb 3968 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 3969 struct e1000_hw *hw = &adapter->hw;
3970 struct pci_dev *pdev = adapter->pdev;
43915c7c 3971 u32 rnbc, reg;
9d5c8243 3972 u16 phy_tmp;
3f9c0164
AD		 3973 int i;
3974 u64 bytes, packets;
9d5c8243
AK		 3975
3976#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3977
3978 /*
3979 * Prevent stats update while adapter is being reset, or if the pci
3980 * connection is down.
3981 */
3982 if (adapter->link_speed == 0)
3983 return;
3984 if (pci_channel_offline(pdev))
3985 return;
3986
3f9c0164
AD		 3987 bytes = 0;
3988 packets = 0;
3989 for (i = 0; i < adapter->num_rx_queues; i++) {
3990 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 3991 struct igb_ring *ring = adapter->rx_ring[i];
3992 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 3993 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 3994 bytes += ring->rx_stats.bytes;
3995 packets += ring->rx_stats.packets;
3f9c0164
AD		 3996 }
3997
128e45eb
AD		 3998 net_stats->rx_bytes = bytes;
3999 net_stats->rx_packets = packets;
3f9c0164
AD		 4000
4001 bytes = 0;
4002 packets = 0;
4003 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 4004 struct igb_ring *ring = adapter->tx_ring[i];
4005 bytes += ring->tx_stats.bytes;
4006 packets += ring->tx_stats.packets;
3f9c0164 4007 }
128e45eb
AD		 4008 net_stats->tx_bytes = bytes;
4009 net_stats->tx_packets = packets;
3f9c0164
AD		 4010
4011 /* read stats registers */
9d5c8243
AK		 4012 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4013 adapter->stats.gprc += rd32(E1000_GPRC);
4014 adapter->stats.gorc += rd32(E1000_GORCL);
4015 rd32(E1000_GORCH); /* clear GORCL */
4016 adapter->stats.bprc += rd32(E1000_BPRC);
4017 adapter->stats.mprc += rd32(E1000_MPRC);
4018 adapter->stats.roc += rd32(E1000_ROC);
4019
4020 adapter->stats.prc64 += rd32(E1000_PRC64);
4021 adapter->stats.prc127 += rd32(E1000_PRC127);
4022 adapter->stats.prc255 += rd32(E1000_PRC255);
4023 adapter->stats.prc511 += rd32(E1000_PRC511);
4024 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4025 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4026 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4027 adapter->stats.sec += rd32(E1000_SEC);
4028
4029 adapter->stats.mpc += rd32(E1000_MPC);
4030 adapter->stats.scc += rd32(E1000_SCC);
4031 adapter->stats.ecol += rd32(E1000_ECOL);
4032 adapter->stats.mcc += rd32(E1000_MCC);
4033 adapter->stats.latecol += rd32(E1000_LATECOL);
4034 adapter->stats.dc += rd32(E1000_DC);
4035 adapter->stats.rlec += rd32(E1000_RLEC);
4036 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4037 adapter->stats.xontxc += rd32(E1000_XONTXC);
4038 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4039 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4040 adapter->stats.fcruc += rd32(E1000_FCRUC);
4041 adapter->stats.gptc += rd32(E1000_GPTC);
4042 adapter->stats.gotc += rd32(E1000_GOTCL);
4043 rd32(E1000_GOTCH); /* clear GOTCL */
3f9c0164
AD		 4044 rnbc = rd32(E1000_RNBC);
4045 adapter->stats.rnbc += rnbc;
128e45eb 4046 net_stats->rx_fifo_errors += rnbc;
9d5c8243
AK		 4047 adapter->stats.ruc += rd32(E1000_RUC);
4048 adapter->stats.rfc += rd32(E1000_RFC);
4049 adapter->stats.rjc += rd32(E1000_RJC);
4050 adapter->stats.tor += rd32(E1000_TORH);
4051 adapter->stats.tot += rd32(E1000_TOTH);
4052 adapter->stats.tpr += rd32(E1000_TPR);
4053
4054 adapter->stats.ptc64 += rd32(E1000_PTC64);
4055 adapter->stats.ptc127 += rd32(E1000_PTC127);
4056 adapter->stats.ptc255 += rd32(E1000_PTC255);
4057 adapter->stats.ptc511 += rd32(E1000_PTC511);
4058 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4059 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4060
4061 adapter->stats.mptc += rd32(E1000_MPTC);
4062 adapter->stats.bptc += rd32(E1000_BPTC);
4063
2d0b0f69
NN		 4064 adapter->stats.tpt += rd32(E1000_TPT);
4065 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4066
4067 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4068 /* read internal phy specific stats */
4069 reg = rd32(E1000_CTRL_EXT);
4070 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4071 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4072 adapter->stats.tncrs += rd32(E1000_TNCRS);
4073 }
4074
9d5c8243
AK		 4075 adapter->stats.tsctc += rd32(E1000_TSCTC);
4076 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4077
4078 adapter->stats.iac += rd32(E1000_IAC);
4079 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4080 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4081 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4082 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4083 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4084 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4085 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4086 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4087
4088 /* Fill out the OS statistics structure */
128e45eb
AD		 4089 net_stats->multicast = adapter->stats.mprc;
4090 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4091
4092 /* Rx Errors */
4093
4094 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4095 * our own version based on RUC and ROC */
128e45eb 4096 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4097 adapter->stats.crcerrs + adapter->stats.algnerrc +
4098 adapter->stats.ruc + adapter->stats.roc +
4099 adapter->stats.cexterr;
128e45eb
AD		 4100 net_stats->rx_length_errors = adapter->stats.ruc +
4101 adapter->stats.roc;
4102 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4103 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4104 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4105
4106 /* Tx Errors */
128e45eb
AD		 4107 net_stats->tx_errors = adapter->stats.ecol +
4108 adapter->stats.latecol;
4109 net_stats->tx_aborted_errors = adapter->stats.ecol;
4110 net_stats->tx_window_errors = adapter->stats.latecol;
4111 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4112
4113 /* Tx Dropped needs to be maintained elsewhere */
4114
4115 /* Phy Stats */
4116 if (hw->phy.media_type == e1000_media_type_copper) {
4117 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4118 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4119 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4120 adapter->phy_stats.idle_errors += phy_tmp;
4121 }
4122 }
4123
4124 /* Management Stats */
4125 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4126 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4127 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4128}
4129
9d5c8243
AK		 4130static irqreturn_t igb_msix_other(int irq, void *data)
4131{
047e0030 4132 struct igb_adapter *adapter = data;
9d5c8243 4133 struct e1000_hw *hw = &adapter->hw;
844290e5 4134 u32 icr = rd32(E1000_ICR);
844290e5 4135 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4136
7f081d40
AD		 4137 if (icr & E1000_ICR_DRSTA)
4138 schedule_work(&adapter->reset_task);
4139
047e0030 4140 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4141 /* HW is reporting DMA is out of sync */
4142 adapter->stats.doosync++;
4143 }
eebbbdba 4144
4ae196df
AD		 4145 /* Check for a mailbox event */
4146 if (icr & E1000_ICR_VMMB)
4147 igb_msg_task(adapter);
4148
4149 if (icr & E1000_ICR_LSC) {
4150 hw->mac.get_link_status = 1;
4151 /* guard against interrupt when we're going down */
4152 if (!test_bit(__IGB_DOWN, &adapter->state))
4153 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4154 }
4155
25568a53
AD		 4156 if (adapter->vfs_allocated_count)
4157 wr32(E1000_IMS, E1000_IMS_LSC |
4158 E1000_IMS_VMMB |
4159 E1000_IMS_DOUTSYNC);
4160 else
4161 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4162 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4163
4164 return IRQ_HANDLED;
4165}
4166
047e0030 4167static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4168{
26b39276 4169 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4170 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4171
047e0030
AD		 4172 if (!q_vector->set_itr)
4173 return;
73cd78f1 4174
047e0030
AD		 4175 if (!itr_val)
4176 itr_val = 0x4;
661086df 4177
26b39276
AD		 4178 if (adapter->hw.mac.type == e1000_82575)
4179 itr_val |= itr_val << 16;
661086df 4180 else
047e0030 4181 itr_val |= 0x8000000;
661086df 4182
047e0030
AD		 4183 writel(itr_val, q_vector->itr_register);
4184 q_vector->set_itr = 0;
6eb5a7f1
AD		 4185}
4186
047e0030 4187static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4188{
047e0030 4189 struct igb_q_vector *q_vector = data;
9d5c8243 4190
047e0030
AD		 4191 /* Write the ITR value calculated from the previous interrupt. */
4192 igb_write_itr(q_vector);
9d5c8243 4193
047e0030 4194 napi_schedule(&q_vector->napi);
844290e5 4195
047e0030 4196 return IRQ_HANDLED;
fe4506b6
JC		 4197}
4198
421e02f0 4199#ifdef CONFIG_IGB_DCA
047e0030 4200static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4201{
047e0030 4202 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4203 struct e1000_hw *hw = &adapter->hw;
4204 int cpu = get_cpu();
fe4506b6 4205
047e0030
AD		 4206 if (q_vector->cpu == cpu)
4207 goto out_no_update;
4208
4209 if (q_vector->tx_ring) {
4210 int q = q_vector->tx_ring->reg_idx;
4211 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4212 if (hw->mac.type == e1000_82575) {
4213 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4214 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4215 } else {
047e0030
AD		 4216 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4217 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4218 E1000_DCA_TXCTRL_CPUID_SHIFT;
4219 }
4220 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4221 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4222 }
4223 if (q_vector->rx_ring) {
4224 int q = q_vector->rx_ring->reg_idx;
4225 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4226 if (hw->mac.type == e1000_82575) {
2d064c06 4227 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4228 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4229 } else {
4230 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4231 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4232 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4233 }
fe4506b6
JC		 4234 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4235 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4236 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4237 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4238 }
047e0030
AD		 4239 q_vector->cpu = cpu;
4240out_no_update:
fe4506b6
JC		 4241 put_cpu();
4242}
4243
4244static void igb_setup_dca(struct igb_adapter *adapter)
4245{
7e0e99ef 4246 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4247 int i;
4248
7dfc16fa 4249 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4250 return;
4251
7e0e99ef
AD		 4252 /* Always use CB2 mode, difference is masked in the CB driver. */
4253 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4254
047e0030 4255 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4256 adapter->q_vector[i]->cpu = -1;
4257 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4258 }
4259}
4260
4261static int __igb_notify_dca(struct device *dev, void *data)
4262{
4263 struct net_device *netdev = dev_get_drvdata(dev);
4264 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4265 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4266 struct e1000_hw *hw = &adapter->hw;
4267 unsigned long event = *(unsigned long *)data;
4268
4269 switch (event) {
4270 case DCA_PROVIDER_ADD:
4271 /* if already enabled, don't do it again */
7dfc16fa 4272 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4273 break;
fe4506b6 4274 if (dca_add_requester(dev) == 0) {
bbd98fe4 4275 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4276 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4277 igb_setup_dca(adapter);
4278 break;
4279 }
4280 /* Fall Through since DCA is disabled. */
4281 case DCA_PROVIDER_REMOVE:
7dfc16fa 4282 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4283 /* without this a class_device is left
047e0030 4284 * hanging around in the sysfs model */
fe4506b6 4285 dca_remove_requester(dev);
090b1795 4286 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4287 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4288 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4289 }
4290 break;
4291 }
bbd98fe4 4292
fe4506b6 4293 return 0;
9d5c8243
AK		 4294}
4295
fe4506b6
JC		 4296static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4297 void *p)
4298{
4299 int ret_val;
4300
4301 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4302 __igb_notify_dca);
4303
4304 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4305}
421e02f0 4306#endif /* CONFIG_IGB_DCA */
9d5c8243 4307
4ae196df
AD		 4308static void igb_ping_all_vfs(struct igb_adapter *adapter)
4309{
4310 struct e1000_hw *hw = &adapter->hw;
4311 u32 ping;
4312 int i;
4313
4314 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4315 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4316 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4317 ping |= E1000_VT_MSGTYPE_CTS;
4318 igb_write_mbx(hw, &ping, 1, i);
4319 }
4320}
4321
7d5753f0
AD		 4322static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4323{
4324 struct e1000_hw *hw = &adapter->hw;
4325 u32 vmolr = rd32(E1000_VMOLR(vf));
4326 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4327
4328 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4329 IGB_VF_FLAG_MULTI_PROMISC);
4330 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4331
4332 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4333 vmolr |= E1000_VMOLR_MPME;
4334 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4335 } else {
4336 /*
4337 * if we have hashes and we are clearing a multicast promisc
4338 * flag we need to write the hashes to the MTA as this step
4339 * was previously skipped
4340 */
4341 if (vf_data->num_vf_mc_hashes > 30) {
4342 vmolr |= E1000_VMOLR_MPME;
4343 } else if (vf_data->num_vf_mc_hashes) {
4344 int j;
4345 vmolr |= E1000_VMOLR_ROMPE;
4346 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4347 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4348 }
4349 }
4350
4351 wr32(E1000_VMOLR(vf), vmolr);
4352
4353 /* there are flags left unprocessed, likely not supported */
4354 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4355 return -EINVAL;
4356
4357 return 0;
4358
4359}
4360
4ae196df
AD		 4361static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4362 u32 *msgbuf, u32 vf)
4363{
4364 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4365 u16 *hash_list = (u16 *)&msgbuf[1];
4366 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4367 int i;
4368
7d5753f0 4369 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4370 * to this VF for later use to restore when the PF multi cast
4371 * list changes
4372 */
4373 vf_data->num_vf_mc_hashes = n;
4374
7d5753f0
AD		 4375 /* only up to 30 hash values supported */
4376 if (n > 30)
4377 n = 30;
4378
4379 /* store the hashes for later use */
4ae196df 4380 for (i = 0; i < n; i++)
a419aef8 4381 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4382
4383 /* Flush and reset the mta with the new values */
ff41f8dc 4384 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4385
4386 return 0;
4387}
4388
4389static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4390{
4391 struct e1000_hw *hw = &adapter->hw;
4392 struct vf_data_storage *vf_data;
4393 int i, j;
4394
4395 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4396 u32 vmolr = rd32(E1000_VMOLR(i));
4397 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4398
4ae196df 4399 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4400
4401 if ((vf_data->num_vf_mc_hashes > 30) ||
4402 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4403 vmolr |= E1000_VMOLR_MPME;
4404 } else if (vf_data->num_vf_mc_hashes) {
4405 vmolr |= E1000_VMOLR_ROMPE;
4406 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4407 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4408 }
4409 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4410 }
4411}
4412
4413static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4414{
4415 struct e1000_hw *hw = &adapter->hw;
4416 u32 pool_mask, reg, vid;
4417 int i;
4418
4419 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4420
4421 /* Find the vlan filter for this id */
4422 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4423 reg = rd32(E1000_VLVF(i));
4424
4425 /* remove the vf from the pool */
4426 reg &= ~pool_mask;
4427
4428 /* if pool is empty then remove entry from vfta */
4429 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4430 (reg & E1000_VLVF_VLANID_ENABLE)) {
4431 reg = 0;
4432 vid = reg & E1000_VLVF_VLANID_MASK;
4433 igb_vfta_set(hw, vid, false);
4434 }
4435
4436 wr32(E1000_VLVF(i), reg);
4437 }
ae641bdc
AD		 4438
4439 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4440}
4441
4442static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4443{
4444 struct e1000_hw *hw = &adapter->hw;
4445 u32 reg, i;
4446
51466239
AD		 4447 /* The vlvf table only exists on 82576 hardware and newer */
4448 if (hw->mac.type < e1000_82576)
4449 return -1;
4450
4451 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4452 if (!adapter->vfs_allocated_count)
4453 return -1;
4454
4455 /* Find the vlan filter for this id */
4456 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4457 reg = rd32(E1000_VLVF(i));
4458 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4459 vid == (reg & E1000_VLVF_VLANID_MASK))
4460 break;
4461 }
4462
4463 if (add) {
4464 if (i == E1000_VLVF_ARRAY_SIZE) {
4465 /* Did not find a matching VLAN ID entry that was
4466 * enabled. Search for a free filter entry, i.e.
4467 * one without the enable bit set
4468 */
4469 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4470 reg = rd32(E1000_VLVF(i));
4471 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4472 break;
4473 }
4474 }
4475 if (i < E1000_VLVF_ARRAY_SIZE) {
4476 /* Found an enabled/available entry */
4477 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4478
4479 /* if !enabled we need to set this up in vfta */
4480 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4481 /* add VID to filter table */
4482 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4483 reg |= E1000_VLVF_VLANID_ENABLE;
4484 }
cad6d05f
AD		 4485 reg &= ~E1000_VLVF_VLANID_MASK;
4486 reg |= vid;
4ae196df 4487 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4488
4489 /* do not modify RLPML for PF devices */
4490 if (vf >= adapter->vfs_allocated_count)
4491 return 0;
4492
4493 if (!adapter->vf_data[vf].vlans_enabled) {
4494 u32 size;
4495 reg = rd32(E1000_VMOLR(vf));
4496 size = reg & E1000_VMOLR_RLPML_MASK;
4497 size += 4;
4498 reg &= ~E1000_VMOLR_RLPML_MASK;
4499 reg |= size;
4500 wr32(E1000_VMOLR(vf), reg);
4501 }
ae641bdc 4502
51466239 4503 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4504 return 0;
4505 }
4506 } else {
4507 if (i < E1000_VLVF_ARRAY_SIZE) {
4508 /* remove vf from the pool */
4509 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4510 /* if pool is empty then remove entry from vfta */
4511 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4512 reg = 0;
4513 igb_vfta_set(hw, vid, false);
4514 }
4515 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4516
4517 /* do not modify RLPML for PF devices */
4518 if (vf >= adapter->vfs_allocated_count)
4519 return 0;
4520
4521 adapter->vf_data[vf].vlans_enabled--;
4522 if (!adapter->vf_data[vf].vlans_enabled) {
4523 u32 size;
4524 reg = rd32(E1000_VMOLR(vf));
4525 size = reg & E1000_VMOLR_RLPML_MASK;
4526 size -= 4;
4527 reg &= ~E1000_VMOLR_RLPML_MASK;
4528 reg |= size;
4529 wr32(E1000_VMOLR(vf), reg);
4530 }
4ae196df
AD		 4531 }
4532 }
8151d294
WM		 4533 return 0;
4534}
4535
4536static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4537{
4538 struct e1000_hw *hw = &adapter->hw;
4539
4540 if (vid)
4541 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4542 else
4543 wr32(E1000_VMVIR(vf), 0);
4544}
4545
4546static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4547 int vf, u16 vlan, u8 qos)
4548{
4549 int err = 0;
4550 struct igb_adapter *adapter = netdev_priv(netdev);
4551
4552 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4553 return -EINVAL;
4554 if (vlan || qos) {
4555 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4556 if (err)
4557 goto out;
4558 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4559 igb_set_vmolr(adapter, vf, !vlan);
4560 adapter->vf_data[vf].pf_vlan = vlan;
4561 adapter->vf_data[vf].pf_qos = qos;
4562 dev_info(&adapter->pdev->dev,
4563 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4564 if (test_bit(__IGB_DOWN, &adapter->state)) {
4565 dev_warn(&adapter->pdev->dev,
4566 "The VF VLAN has been set,"
4567 " but the PF device is not up.\n");
4568 dev_warn(&adapter->pdev->dev,
4569 "Bring the PF device up before"
4570 " attempting to use the VF device.\n");
4571 }
4572 } else {
4573 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4574 false, vf);
4575 igb_set_vmvir(adapter, vlan, vf);
4576 igb_set_vmolr(adapter, vf, true);
4577 adapter->vf_data[vf].pf_vlan = 0;
4578 adapter->vf_data[vf].pf_qos = 0;
4579 }
4580out:
4581 return err;
4ae196df
AD		 4582}
4583
4584static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4585{
4586 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4587 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4588
4589 return igb_vlvf_set(adapter, vid, add, vf);
4590}
4591
f2ca0dbe 4592static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4593{
8151d294
WM		 4594 /* clear flags */
4595 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4596 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4597
4598 /* reset offloads to defaults */
8151d294 4599 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4600
4601 /* reset vlans for device */
4602 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4603 if (adapter->vf_data[vf].pf_vlan)
4604 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4605 adapter->vf_data[vf].pf_vlan,
4606 adapter->vf_data[vf].pf_qos);
4607 else
4608 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4609
4610 /* reset multicast table array for vf */
4611 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4612
4613 /* Flush and reset the mta with the new values */
ff41f8dc 4614 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4615}
4616
f2ca0dbe
AD		 4617static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4618{
4619 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4620
4621 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4622 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4623 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4624
4625 /* process remaining reset events */
4626 igb_vf_reset(adapter, vf);
4627}
4628
4629static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4630{
4631 struct e1000_hw *hw = &adapter->hw;
4632 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4633 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4634 u32 reg, msgbuf[3];
4635 u8 *addr = (u8 *)(&msgbuf[1]);
4636
4637 /* process all the same items cleared in a function level reset */
f2ca0dbe 4638 igb_vf_reset(adapter, vf);
4ae196df
AD		 4639
4640 /* set vf mac address */
26ad9178 4641 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4642
4643 /* enable transmit and receive for vf */
4644 reg = rd32(E1000_VFTE);
4645 wr32(E1000_VFTE, reg | (1 << vf));
4646 reg = rd32(E1000_VFRE);
4647 wr32(E1000_VFRE, reg | (1 << vf));
4648
f2ca0dbe 4649 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4650
4651 /* reply to reset with ack and vf mac address */
4652 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4653 memcpy(addr, vf_mac, 6);
4654 igb_write_mbx(hw, msgbuf, 3, vf);
4655}
4656
4657static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4658{
f2ca0dbe
AD		 4659 unsigned char *addr = (char *)&msg[1];
4660 int err = -1;
4ae196df 4661
f2ca0dbe
AD		 4662 if (is_valid_ether_addr(addr))
4663 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4664
f2ca0dbe 4665 return err;
4ae196df
AD		 4666}
4667
4668static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4669{
4670 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4671 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4672 u32 msg = E1000_VT_MSGTYPE_NACK;
4673
4674 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4675 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4676 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4677 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4678 vf_data->last_nack = jiffies;
4ae196df
AD		 4679 }
4680}
4681
f2ca0dbe 4682static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4683{
f2ca0dbe
AD		 4684 struct pci_dev *pdev = adapter->pdev;
4685 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4686 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4687 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4688 s32 retval;
4689
f2ca0dbe 4690 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 4691
fef45f4c
AD		 4692 if (retval) {
4693 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 4694 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 4695 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4696 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4697 return;
4698 goto out;
4699 }
4ae196df
AD		 4700
4701 /* this is a message we already processed, do nothing */
4702 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4703 return;
4ae196df
AD		 4704
4705 /*
4706 * until the vf completes a reset it should not be
4707 * allowed to start any configuration.
4708 */
4709
4710 if (msgbuf[0] == E1000_VF_RESET) {
4711 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4712 return;
4ae196df
AD		 4713 }
4714
f2ca0dbe 4715 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 4716 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4717 return;
4718 retval = -1;
4719 goto out;
4ae196df
AD		 4720 }
4721
4722 switch ((msgbuf[0] & 0xFFFF)) {
4723 case E1000_VF_SET_MAC_ADDR:
4724 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4725 break;
7d5753f0
AD		 4726 case E1000_VF_SET_PROMISC:
4727 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4728 break;
4ae196df
AD		 4729 case E1000_VF_SET_MULTICAST:
4730 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4731 break;
4732 case E1000_VF_SET_LPE:
4733 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4734 break;
4735 case E1000_VF_SET_VLAN:
8151d294
WM		 4736 if (adapter->vf_data[vf].pf_vlan)
4737 retval = -1;
4738 else
4739 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 4740 break;
4741 default:
090b1795 4742 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 4743 retval = -1;
4744 break;
4745 }
4746
fef45f4c
AD		 4747 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4748out:
4ae196df
AD		 4749 /* notify the VF of the results of what it sent us */
4750 if (retval)
4751 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4752 else
4753 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4754
4ae196df 4755 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4756}
4ae196df 4757
f2ca0dbe
AD		 4758static void igb_msg_task(struct igb_adapter *adapter)
4759{
4760 struct e1000_hw *hw = &adapter->hw;
4761 u32 vf;
4762
4763 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4764 /* process any reset requests */
4765 if (!igb_check_for_rst(hw, vf))
4766 igb_vf_reset_event(adapter, vf);
4767
4768 /* process any messages pending */
4769 if (!igb_check_for_msg(hw, vf))
4770 igb_rcv_msg_from_vf(adapter, vf);
4771
4772 /* process any acks */
4773 if (!igb_check_for_ack(hw, vf))
4774 igb_rcv_ack_from_vf(adapter, vf);
4775 }
4ae196df
AD		 4776}
4777
68d480c4
AD		 4778/**
4779 * igb_set_uta - Set unicast filter table address
4780 * @adapter: board private structure
4781 *
4782 * The unicast table address is a register array of 32-bit registers.
4783 * The table is meant to be used in a way similar to how the MTA is used
4784 * however due to certain limitations in the hardware it is necessary to
4785 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4786 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4787 **/
4788static void igb_set_uta(struct igb_adapter *adapter)
4789{
4790 struct e1000_hw *hw = &adapter->hw;
4791 int i;
4792
4793 /* The UTA table only exists on 82576 hardware and newer */
4794 if (hw->mac.type < e1000_82576)
4795 return;
4796
4797 /* we only need to do this if VMDq is enabled */
4798 if (!adapter->vfs_allocated_count)
4799 return;
4800
4801 for (i = 0; i < hw->mac.uta_reg_count; i++)
4802 array_wr32(E1000_UTA, i, ~0);
4803}
4804
9d5c8243
AK		 4805/**
4806 * igb_intr_msi - Interrupt Handler
4807 * @irq: interrupt number
4808 * @data: pointer to a network interface device structure
4809 **/
4810static irqreturn_t igb_intr_msi(int irq, void *data)
4811{
047e0030
AD		 4812 struct igb_adapter *adapter = data;
4813 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4814 struct e1000_hw *hw = &adapter->hw;
4815 /* read ICR disables interrupts using IAM */
4816 u32 icr = rd32(E1000_ICR);
4817
047e0030 4818 igb_write_itr(q_vector);
9d5c8243 4819
7f081d40
AD		 4820 if (icr & E1000_ICR_DRSTA)
4821 schedule_work(&adapter->reset_task);
4822
047e0030 4823 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4824 /* HW is reporting DMA is out of sync */
4825 adapter->stats.doosync++;
4826 }
4827
9d5c8243
AK		 4828 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4829 hw->mac.get_link_status = 1;
4830 if (!test_bit(__IGB_DOWN, &adapter->state))
4831 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4832 }
4833
047e0030 4834 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4835
4836 return IRQ_HANDLED;
4837}
4838
4839/**
4a3c6433 4840 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4841 * @irq: interrupt number
4842 * @data: pointer to a network interface device structure
4843 **/
4844static irqreturn_t igb_intr(int irq, void *data)
4845{
047e0030
AD		 4846 struct igb_adapter *adapter = data;
4847 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4848 struct e1000_hw *hw = &adapter->hw;
4849 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4850 * need for the IMC write */
4851 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4852 if (!icr)
4853 return IRQ_NONE; /* Not our interrupt */
4854
047e0030 4855 igb_write_itr(q_vector);
9d5c8243
AK		 4856
4857 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4858 * not set, then the adapter didn't send an interrupt */
4859 if (!(icr & E1000_ICR_INT_ASSERTED))
4860 return IRQ_NONE;
4861
7f081d40
AD		 4862 if (icr & E1000_ICR_DRSTA)
4863 schedule_work(&adapter->reset_task);
4864
047e0030 4865 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4866 /* HW is reporting DMA is out of sync */
4867 adapter->stats.doosync++;
4868 }
4869
9d5c8243
AK		 4870 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4871 hw->mac.get_link_status = 1;
4872 /* guard against interrupt when we're going down */
4873 if (!test_bit(__IGB_DOWN, &adapter->state))
4874 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4875 }
4876
047e0030 4877 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4878
4879 return IRQ_HANDLED;
4880}
4881
047e0030 4882static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4883{
047e0030 4884 struct igb_adapter *adapter = q_vector->adapter;
46544258 4885 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4886
4fc82adf
AD		 4887 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4888 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4889 if (!adapter->msix_entries)
6eb5a7f1 4890 igb_set_itr(adapter);
46544258 4891 else
047e0030 4892 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4893 }
4894
46544258
AD		 4895 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4896 if (adapter->msix_entries)
047e0030 4897 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4898 else
4899 igb_irq_enable(adapter);
4900 }
9d5c8243
AK		 4901}
4902
46544258
AD		 4903/**
4904 * igb_poll - NAPI Rx polling callback
4905 * @napi: napi polling structure
4906 * @budget: count of how many packets we should handle
4907 **/
4908static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4909{
047e0030
AD		 4910 struct igb_q_vector *q_vector = container_of(napi,
4911 struct igb_q_vector,
4912 napi);
4913 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4914
421e02f0 4915#ifdef CONFIG_IGB_DCA
047e0030
AD		 4916 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4917 igb_update_dca(q_vector);
fe4506b6 4918#endif
047e0030
AD		 4919 if (q_vector->tx_ring)
4920 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4921
047e0030
AD		 4922 if (q_vector->rx_ring)
4923 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4924
4925 if (!tx_clean_complete)
4926 work_done = budget;
46544258 4927
9d5c8243 4928 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4929 if (work_done < budget) {
288379f0 4930 napi_complete(napi);
047e0030 4931 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4932 }
4933
46544258 4934 return work_done;
9d5c8243 4935}
6d8126f9 4936
33af6bcc 4937/**
c5b9bd5e 4938 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4939 * @adapter: board private structure
c5b9bd5e
AD		 4940 * @shhwtstamps: timestamp structure to update
4941 * @regval: unsigned 64bit system time value.
4942 *
4943 * We need to convert the system time value stored in the RX/TXSTMP registers
4944 * into a hwtstamp which can be used by the upper level timestamping functions
4945 */
4946static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4947 struct skb_shared_hwtstamps *shhwtstamps,
4948 u64 regval)
4949{
4950 u64 ns;
4951
55cac248
AD		 4952 /*
4953 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4954 * 24 to match clock shift we setup earlier.
4955 */
4956 if (adapter->hw.mac.type == e1000_82580)
4957 regval <<= IGB_82580_TSYNC_SHIFT;
4958
c5b9bd5e
AD		 4959 ns = timecounter_cyc2time(&adapter->clock, regval);
4960 timecompare_update(&adapter->compare, ns);
4961 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4962 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4963 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4964}
4965
4966/**
4967 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4968 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4969 * @skb: packet that was just sent
4970 *
4971 * If we were asked to do hardware stamping and such a time stamp is
4972 * available, then it must have been for this skb here because we only
4973 * allow only one such packet into the queue.
4974 */
c5b9bd5e 4975static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4976{
c5b9bd5e 4977 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4978 union skb_shared_tx *shtx = skb_tx(skb);
4979 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4980 struct skb_shared_hwtstamps shhwtstamps;
4981 u64 regval;
33af6bcc 4982
c5b9bd5e
AD		 4983 /* if skb does not support hw timestamp or TX stamp not valid exit */
4984 if (likely(!shtx->hardware) ||
4985 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4986 return;
4987
4988 regval = rd32(E1000_TXSTMPL);
4989 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4990
4991 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4992 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4993}
4994
9d5c8243
AK		 4995/**
4996 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4997 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4998 * returns true if ring is completely cleaned
4999 **/
047e0030 5000static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5001{
047e0030
AD		 5002 struct igb_adapter *adapter = q_vector->adapter;
5003 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5004 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5005 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 5006 struct igb_buffer *buffer_info;
5007 struct sk_buff *skb;
0e014cb1 5008 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5009 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5010 unsigned int i, eop, count = 0;
5011 bool cleaned = false;
9d5c8243 5012
9d5c8243 5013 i = tx_ring->next_to_clean;
0e014cb1
AD		 5014 eop = tx_ring->buffer_info[i].next_to_watch;
5015 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5016
5017 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5018 (count < tx_ring->count)) {
5019 for (cleaned = false; !cleaned; count++) {
5020 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5021 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5022 cleaned = (i == eop);
9d5c8243
AK		 5023 skb = buffer_info->skb;
5024
5025 if (skb) {
5026 unsigned int segs, bytecount;
5027 /* gso_segs is currently only valid for tcp */
40e90c26 5028 segs = buffer_info->gso_segs;
9d5c8243
AK		 5029 /* multiply data chunks by size of headers */
5030 bytecount = ((segs - 1) * skb_headlen(skb)) +
5031 skb->len;
5032 total_packets += segs;
5033 total_bytes += bytecount;
33af6bcc 5034
c5b9bd5e 5035 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 5036 }
5037
80785298 5038 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5039 tx_desc->wb.status = 0;
9d5c8243
AK		 5040
5041 i++;
5042 if (i == tx_ring->count)
5043 i = 0;
9d5c8243 5044 }
0e014cb1
AD		 5045 eop = tx_ring->buffer_info[i].next_to_watch;
5046 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5047 }
5048
9d5c8243
AK		 5049 tx_ring->next_to_clean = i;
5050
fc7d345d 5051 if (unlikely(count &&
9d5c8243 5052 netif_carrier_ok(netdev) &&
c493ea45 5053 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5054 /* Make sure that anybody stopping the queue after this
5055 * sees the new next_to_clean.
5056 */
5057 smp_mb();
661086df
PWJ		 5058 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5059 !(test_bit(__IGB_DOWN, &adapter->state))) {
5060 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5061 tx_ring->tx_stats.restart_queue++;
661086df 5062 }
9d5c8243
AK		 5063 }
5064
5065 if (tx_ring->detect_tx_hung) {
5066 /* Detect a transmit hang in hardware, this serializes the
5067 * check with the clearing of time_stamp and movement of i */
5068 tx_ring->detect_tx_hung = false;
5069 if (tx_ring->buffer_info[i].time_stamp &&
5070 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5071 (adapter->tx_timeout_factor * HZ)) &&
5072 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5073
9d5c8243 5074 /* detected Tx unit hang */
80785298 5075 dev_err(&tx_ring->pdev->dev,
9d5c8243 5076 "Detected Tx Unit Hang\n"
2d064c06 5077 " Tx Queue <%d>\n"
9d5c8243
AK		 5078 " TDH <%x>\n"
5079 " TDT <%x>\n"
5080 " next_to_use <%x>\n"
5081 " next_to_clean <%x>\n"
9d5c8243
AK		 5082 "buffer_info[next_to_clean]\n"
5083 " time_stamp <%lx>\n"
0e014cb1 5084 " next_to_watch <%x>\n"
9d5c8243
AK		 5085 " jiffies <%lx>\n"
5086 " desc.status <%x>\n",
2d064c06 5087 tx_ring->queue_index,
fce99e34
AD		 5088 readl(tx_ring->head),
5089 readl(tx_ring->tail),
9d5c8243
AK		 5090 tx_ring->next_to_use,
5091 tx_ring->next_to_clean,
f7ba205e 5092 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5093 eop,
9d5c8243 5094 jiffies,
0e014cb1 5095 eop_desc->wb.status);
661086df 5096 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5097 }
5098 }
5099 tx_ring->total_bytes += total_bytes;
5100 tx_ring->total_packets += total_packets;
e21ed353
AD		 5101 tx_ring->tx_stats.bytes += total_bytes;
5102 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5103 return (count < tx_ring->count);
9d5c8243
AK		 5104}
5105
9d5c8243
AK		 5106/**
5107 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5108 * @q_vector: structure containing interrupt and ring information
5109 * @skb: packet to send up
5110 * @vlan_tag: vlan tag for packet
9d5c8243 5111 **/
047e0030
AD		 5112static void igb_receive_skb(struct igb_q_vector *q_vector,
5113 struct sk_buff *skb,
5114 u16 vlan_tag)
5115{
5116 struct igb_adapter *adapter = q_vector->adapter;
5117
5118 if (vlan_tag)
5119 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5120 vlan_tag, skb);
182ff8df 5121 else
047e0030 5122 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5123}
5124
04a5fcaa 5125static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5126 u32 status_err, struct sk_buff *skb)
5127{
5128 skb->ip_summed = CHECKSUM_NONE;
5129
5130 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5131 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5132 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5133 return;
85ad76b2 5134
9d5c8243
AK		 5135 /* TCP/UDP checksum error bit is set */
5136 if (status_err &
5137 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5138 /*
5139 * work around errata with sctp packets where the TCPE aka
5140 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5141 * packets, (aka let the stack check the crc32c)
5142 */
85ad76b2
AD		 5143 if ((skb->len == 60) &&
5144 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5145 ring->rx_stats.csum_err++;
85ad76b2 5146
9d5c8243 5147 /* let the stack verify checksum errors */
9d5c8243
AK		 5148 return;
5149 }
5150 /* It must be a TCP or UDP packet with a valid checksum */
5151 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5152 skb->ip_summed = CHECKSUM_UNNECESSARY;
5153
85ad76b2 5154 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5155}
5156
c5b9bd5e
AD		 5157static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5158 struct sk_buff *skb)
5159{
5160 struct igb_adapter *adapter = q_vector->adapter;
5161 struct e1000_hw *hw = &adapter->hw;
5162 u64 regval;
5163
5164 /*
5165 * If this bit is set, then the RX registers contain the time stamp. No
5166 * other packet will be time stamped until we read these registers, so
5167 * read the registers to make them available again. Because only one
5168 * packet can be time stamped at a time, we know that the register
5169 * values must belong to this one here and therefore we don't need to
5170 * compare any of the additional attributes stored for it.
5171 *
5172 * If nothing went wrong, then it should have a skb_shared_tx that we
5173 * can turn into a skb_shared_hwtstamps.
5174 */
5175 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
5176 return;
5177 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5178 return;
5179
5180 regval = rd32(E1000_RXSTMPL);
5181 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5182
5183 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5184}
4c844851 5185static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5186 union e1000_adv_rx_desc *rx_desc)
5187{
5188 /* HW will not DMA in data larger than the given buffer, even if it
5189 * parses the (NFS, of course) header to be larger. In that case, it
5190 * fills the header buffer and spills the rest into the page.
5191 */
5192 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5193 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5194 if (hlen > rx_ring->rx_buffer_len)
5195 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5196 return hlen;
5197}
5198
047e0030
AD		 5199static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5200 int *work_done, int budget)
9d5c8243 5201{
047e0030 5202 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5203 struct net_device *netdev = rx_ring->netdev;
80785298 5204 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 5205 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5206 struct igb_buffer *buffer_info , *next_buffer;
5207 struct sk_buff *skb;
9d5c8243
AK		 5208 bool cleaned = false;
5209 int cleaned_count = 0;
d1eff350 5210 int current_node = numa_node_id();
9d5c8243 5211 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5212 unsigned int i;
2d94d8ab
AD		 5213 u32 staterr;
5214 u16 length;
047e0030 5215 u16 vlan_tag;
9d5c8243
AK		 5216
5217 i = rx_ring->next_to_clean;
69d3ca53 5218 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5219 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5220 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5221
5222 while (staterr & E1000_RXD_STAT_DD) {
5223 if (*work_done >= budget)
5224 break;
5225 (*work_done)++;
9d5c8243 5226
69d3ca53
AD		 5227 skb = buffer_info->skb;
5228 prefetch(skb->data - NET_IP_ALIGN);
5229 buffer_info->skb = NULL;
5230
5231 i++;
5232 if (i == rx_ring->count)
5233 i = 0;
42d0781a 5234
69d3ca53
AD		 5235 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5236 prefetch(next_rxd);
5237 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5238
5239 length = le16_to_cpu(rx_desc->wb.upper.length);
5240 cleaned = true;
5241 cleaned_count++;
5242
2d94d8ab 5243 if (buffer_info->dma) {
bf36c1a0 5244 pci_unmap_single(pdev, buffer_info->dma,
4c844851 5245 rx_ring->rx_buffer_len,
bf36c1a0 5246 PCI_DMA_FROMDEVICE);
91615f76 5247 buffer_info->dma = 0;
4c844851 5248 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5249 skb_put(skb, length);
5250 goto send_up;
5251 }
4c844851 5252 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5253 }
5254
5255 if (length) {
9d5c8243 5256 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 5257 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 5258 buffer_info->page_dma = 0;
bf36c1a0
AD		 5259
5260 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
5261 buffer_info->page,
5262 buffer_info->page_offset,
5263 length);
5264
d1eff350
AD		 5265 if ((page_count(buffer_info->page) != 1) ||
5266 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5267 buffer_info->page = NULL;
5268 else
5269 get_page(buffer_info->page);
9d5c8243
AK		 5270
5271 skb->len += length;
5272 skb->data_len += length;
bf36c1a0 5273 skb->truesize += length;
9d5c8243 5274 }
9d5c8243 5275
bf36c1a0 5276 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5277 buffer_info->skb = next_buffer->skb;
5278 buffer_info->dma = next_buffer->dma;
5279 next_buffer->skb = skb;
5280 next_buffer->dma = 0;
bf36c1a0
AD		 5281 goto next_desc;
5282 }
69d3ca53 5283send_up:
9d5c8243
AK		 5284 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5285 dev_kfree_skb_irq(skb);
5286 goto next_desc;
5287 }
9d5c8243 5288
c5b9bd5e 5289 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5290 total_bytes += skb->len;
5291 total_packets++;
5292
85ad76b2 5293 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5294
5295 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5296 skb_record_rx_queue(skb, rx_ring->queue_index);
5297
5298 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5299 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5300
047e0030 5301 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5302
9d5c8243
AK		 5303next_desc:
5304 rx_desc->wb.upper.status_error = 0;
5305
5306 /* return some buffers to hardware, one at a time is too slow */
5307 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5308 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5309 cleaned_count = 0;
5310 }
5311
5312 /* use prefetched values */
5313 rx_desc = next_rxd;
5314 buffer_info = next_buffer;
9d5c8243
AK		 5315 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5316 }
bf36c1a0 5317
9d5c8243 5318 rx_ring->next_to_clean = i;
c493ea45 5319 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5320
5321 if (cleaned_count)
3b644cf6 5322 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5323
5324 rx_ring->total_packets += total_packets;
5325 rx_ring->total_bytes += total_bytes;
5326 rx_ring->rx_stats.packets += total_packets;
5327 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5328 return cleaned;
5329}
5330
9d5c8243
AK		 5331/**
5332 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5333 * @adapter: address of board private structure
5334 **/
d7ee5b3a 5335void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5336{
e694e964 5337 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5338 union e1000_adv_rx_desc *rx_desc;
5339 struct igb_buffer *buffer_info;
5340 struct sk_buff *skb;
5341 unsigned int i;
db761762 5342 int bufsz;
9d5c8243
AK		 5343
5344 i = rx_ring->next_to_use;
5345 buffer_info = &rx_ring->buffer_info[i];
5346
4c844851 5347 bufsz = rx_ring->rx_buffer_len;
db761762 5348
9d5c8243
AK		 5349 while (cleaned_count--) {
5350 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5351
6ec43fe6 5352 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5353 if (!buffer_info->page) {
42d0781a 5354 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5355 if (!buffer_info->page) {
04a5fcaa 5356 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5357 goto no_buffers;
5358 }
5359 buffer_info->page_offset = 0;
5360 } else {
5361 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5362 }
5363 buffer_info->page_dma =
80785298 5364 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5365 buffer_info->page_offset,
5366 PAGE_SIZE / 2,
9d5c8243 5367 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5368 if (pci_dma_mapping_error(rx_ring->pdev,
5369 buffer_info->page_dma)) {
5370 buffer_info->page_dma = 0;
5371 rx_ring->rx_stats.alloc_failed++;
5372 goto no_buffers;
5373 }
9d5c8243
AK		 5374 }
5375
42d0781a
AD		 5376 skb = buffer_info->skb;
5377 if (!skb) {
89d71a66 5378 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5379 if (!skb) {
04a5fcaa 5380 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5381 goto no_buffers;
5382 }
5383
9d5c8243 5384 buffer_info->skb = skb;
42d0781a
AD		 5385 }
5386 if (!buffer_info->dma) {
80785298
AD		 5387 buffer_info->dma = pci_map_single(rx_ring->pdev,
5388 skb->data,
9d5c8243
AK		 5389 bufsz,
5390 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5391 if (pci_dma_mapping_error(rx_ring->pdev,
5392 buffer_info->dma)) {
5393 buffer_info->dma = 0;
5394 rx_ring->rx_stats.alloc_failed++;
5395 goto no_buffers;
5396 }
9d5c8243
AK		 5397 }
5398 /* Refresh the desc even if buffer_addrs didn't change because
5399 * each write-back erases this info. */
6ec43fe6 5400 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5401 rx_desc->read.pkt_addr =
5402 cpu_to_le64(buffer_info->page_dma);
5403 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5404 } else {
42d0781a 5405 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5406 rx_desc->read.hdr_addr = 0;
5407 }
5408
5409 i++;
5410 if (i == rx_ring->count)
5411 i = 0;
5412 buffer_info = &rx_ring->buffer_info[i];
5413 }
5414
5415no_buffers:
5416 if (rx_ring->next_to_use != i) {
5417 rx_ring->next_to_use = i;
5418 if (i == 0)
5419 i = (rx_ring->count - 1);
5420 else
5421 i--;
5422
5423 /* Force memory writes to complete before letting h/w
5424 * know there are new descriptors to fetch. (Only
5425 * applicable for weak-ordered memory model archs,
5426 * such as IA-64). */
5427 wmb();
fce99e34 5428 writel(i, rx_ring->tail);
9d5c8243
AK		 5429 }
5430}
5431
5432/**
5433 * igb_mii_ioctl -
5434 * @netdev:
5435 * @ifreq:
5436 * @cmd:
5437 **/
5438static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5439{
5440 struct igb_adapter *adapter = netdev_priv(netdev);
5441 struct mii_ioctl_data *data = if_mii(ifr);
5442
5443 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5444 return -EOPNOTSUPP;
5445
5446 switch (cmd) {
5447 case SIOCGMIIPHY:
5448 data->phy_id = adapter->hw.phy.addr;
5449 break;
5450 case SIOCGMIIREG:
f5f4cf08
AD		 5451 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5452 &data->val_out))
9d5c8243
AK		 5453 return -EIO;
5454 break;
5455 case SIOCSMIIREG:
5456 default:
5457 return -EOPNOTSUPP;
5458 }
5459 return 0;
5460}
5461
c6cb090b
PO		 5462/**
5463 * igb_hwtstamp_ioctl - control hardware time stamping
5464 * @netdev:
5465 * @ifreq:
5466 * @cmd:
5467 *
33af6bcc
PO		 5468 * Outgoing time stamping can be enabled and disabled. Play nice and
5469 * disable it when requested, although it shouldn't case any overhead
5470 * when no packet needs it. At most one packet in the queue may be
5471 * marked for time stamping, otherwise it would be impossible to tell
5472 * for sure to which packet the hardware time stamp belongs.
5473 *
5474 * Incoming time stamping has to be configured via the hardware
5475 * filters. Not all combinations are supported, in particular event
5476 * type has to be specified. Matching the kind of event packet is
5477 * not supported, with the exception of "all V2 events regardless of
5478 * level 2 or 4".
5479 *
c6cb090b
PO		 5480 **/
5481static int igb_hwtstamp_ioctl(struct net_device *netdev,
5482 struct ifreq *ifr, int cmd)
5483{
33af6bcc
PO		 5484 struct igb_adapter *adapter = netdev_priv(netdev);
5485 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5486 struct hwtstamp_config config;
c5b9bd5e
AD		 5487 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5488 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5489 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5490 bool is_l4 = false;
5491 bool is_l2 = false;
33af6bcc 5492 u32 regval;
c6cb090b
PO		 5493
5494 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5495 return -EFAULT;
5496
5497 /* reserved for future extensions */
5498 if (config.flags)
5499 return -EINVAL;
5500
33af6bcc
PO		 5501 switch (config.tx_type) {
5502 case HWTSTAMP_TX_OFF:
c5b9bd5e 5503 tsync_tx_ctl = 0;
33af6bcc 5504 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5505 break;
5506 default:
5507 return -ERANGE;
5508 }
5509
5510 switch (config.rx_filter) {
5511 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5512 tsync_rx_ctl = 0;
33af6bcc
PO		 5513 break;
5514 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5515 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5516 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5517 case HWTSTAMP_FILTER_ALL:
5518 /*
5519 * register TSYNCRXCFG must be set, therefore it is not
5520 * possible to time stamp both Sync and Delay_Req messages
5521 * => fall back to time stamping all packets
5522 */
c5b9bd5e 5523 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5524 config.rx_filter = HWTSTAMP_FILTER_ALL;
5525 break;
5526 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5527 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5528 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5529 is_l4 = true;
33af6bcc
PO		 5530 break;
5531 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5532 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5533 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5534 is_l4 = true;
33af6bcc
PO		 5535 break;
5536 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5537 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5538 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5539 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5540 is_l2 = true;
5541 is_l4 = true;
33af6bcc
PO		 5542 config.rx_filter = HWTSTAMP_FILTER_SOME;
5543 break;
5544 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5545 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5546 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5547 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5548 is_l2 = true;
5549 is_l4 = true;
33af6bcc
PO		 5550 config.rx_filter = HWTSTAMP_FILTER_SOME;
5551 break;
5552 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5553 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5554 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5555 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5556 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5557 is_l2 = true;
33af6bcc
PO		 5558 break;
5559 default:
5560 return -ERANGE;
5561 }
5562
c5b9bd5e
AD		 5563 if (hw->mac.type == e1000_82575) {
5564 if (tsync_rx_ctl | tsync_tx_ctl)
5565 return -EINVAL;
5566 return 0;
5567 }
5568
33af6bcc
PO		 5569 /* enable/disable TX */
5570 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5571 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5572 regval |= tsync_tx_ctl;
33af6bcc
PO		 5573 wr32(E1000_TSYNCTXCTL, regval);
5574
c5b9bd5e 5575 /* enable/disable RX */
33af6bcc 5576 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5577 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5578 regval |= tsync_rx_ctl;
33af6bcc 5579 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5580
c5b9bd5e
AD		 5581 /* define which PTP packets are time stamped */
5582 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5583
c5b9bd5e
AD		 5584 /* define ethertype filter for timestamped packets */
5585 if (is_l2)
5586 wr32(E1000_ETQF(3),
5587 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5588 E1000_ETQF_1588 | /* enable timestamping */
5589 ETH_P_1588)); /* 1588 eth protocol type */
5590 else
5591 wr32(E1000_ETQF(3), 0);
5592
5593#define PTP_PORT 319
5594 /* L4 Queue Filter[3]: filter by destination port and protocol */
5595 if (is_l4) {
5596 u32 ftqf = (IPPROTO_UDP /* UDP */
5597 | E1000_FTQF_VF_BP /* VF not compared */
5598 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5599 | E1000_FTQF_MASK); /* mask all inputs */
5600 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5601
5602 wr32(E1000_IMIR(3), htons(PTP_PORT));
5603 wr32(E1000_IMIREXT(3),
5604 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5605 if (hw->mac.type == e1000_82576) {
5606 /* enable source port check */
5607 wr32(E1000_SPQF(3), htons(PTP_PORT));
5608 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5609 }
5610 wr32(E1000_FTQF(3), ftqf);
5611 } else {
5612 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5613 }
33af6bcc
PO		 5614 wrfl();
5615
5616 adapter->hwtstamp_config = config;
5617
5618 /* clear TX/RX time stamp registers, just to be sure */
5619 regval = rd32(E1000_TXSTMPH);
5620 regval = rd32(E1000_RXSTMPH);
c6cb090b 5621
33af6bcc
PO		 5622 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5623 -EFAULT : 0;
c6cb090b
PO		 5624}
5625
9d5c8243
AK		 5626/**
5627 * igb_ioctl -
5628 * @netdev:
5629 * @ifreq:
5630 * @cmd:
5631 **/
5632static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5633{
5634 switch (cmd) {
5635 case SIOCGMIIPHY:
5636 case SIOCGMIIREG:
5637 case SIOCSMIIREG:
5638 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5639 case SIOCSHWTSTAMP:
5640 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5641 default:
5642 return -EOPNOTSUPP;
5643 }
5644}
5645
009bc06e
AD		 5646s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5647{
5648 struct igb_adapter *adapter = hw->back;
5649 u16 cap_offset;
5650
5651 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5652 if (!cap_offset)
5653 return -E1000_ERR_CONFIG;
5654
5655 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5656
5657 return 0;
5658}
5659
5660s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5661{
5662 struct igb_adapter *adapter = hw->back;
5663 u16 cap_offset;
5664
5665 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5666 if (!cap_offset)
5667 return -E1000_ERR_CONFIG;
5668
5669 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5670
5671 return 0;
5672}
5673
9d5c8243
AK		 5674static void igb_vlan_rx_register(struct net_device *netdev,
5675 struct vlan_group *grp)
5676{
5677 struct igb_adapter *adapter = netdev_priv(netdev);
5678 struct e1000_hw *hw = &adapter->hw;
5679 u32 ctrl, rctl;
5680
5681 igb_irq_disable(adapter);
5682 adapter->vlgrp = grp;
5683
5684 if (grp) {
5685 /* enable VLAN tag insert/strip */
5686 ctrl = rd32(E1000_CTRL);
5687 ctrl |= E1000_CTRL_VME;
5688 wr32(E1000_CTRL, ctrl);
5689
51466239 5690 /* Disable CFI check */
9d5c8243 5691 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5692 rctl &= ~E1000_RCTL_CFIEN;
5693 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5694 } else {
5695 /* disable VLAN tag insert/strip */
5696 ctrl = rd32(E1000_CTRL);
5697 ctrl &= ~E1000_CTRL_VME;
5698 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5699 }
5700
e1739522
AD		 5701 igb_rlpml_set(adapter);
5702
9d5c8243
AK		 5703 if (!test_bit(__IGB_DOWN, &adapter->state))
5704 igb_irq_enable(adapter);
5705}
5706
5707static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5708{
5709 struct igb_adapter *adapter = netdev_priv(netdev);
5710 struct e1000_hw *hw = &adapter->hw;
4ae196df 5711 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5712
51466239
AD		 5713 /* attempt to add filter to vlvf array */
5714 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5715
51466239
AD		 5716 /* add the filter since PF can receive vlans w/o entry in vlvf */
5717 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5718}
5719
5720static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5721{
5722 struct igb_adapter *adapter = netdev_priv(netdev);
5723 struct e1000_hw *hw = &adapter->hw;
4ae196df 5724 int pf_id = adapter->vfs_allocated_count;
51466239 5725 s32 err;
9d5c8243
AK		 5726
5727 igb_irq_disable(adapter);
5728 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5729
5730 if (!test_bit(__IGB_DOWN, &adapter->state))
5731 igb_irq_enable(adapter);
5732
51466239
AD		 5733 /* remove vlan from VLVF table array */
5734 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5735
51466239
AD		 5736 /* if vid was not present in VLVF just remove it from table */
5737 if (err)
4ae196df 5738 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5739}
5740
5741static void igb_restore_vlan(struct igb_adapter *adapter)
5742{
5743 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5744
5745 if (adapter->vlgrp) {
5746 u16 vid;
5747 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5748 if (!vlan_group_get_device(adapter->vlgrp, vid))
5749 continue;
5750 igb_vlan_rx_add_vid(adapter->netdev, vid);
5751 }
5752 }
5753}
5754
5755int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5756{
090b1795 5757 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 5758 struct e1000_mac_info *mac = &adapter->hw.mac;
5759
5760 mac->autoneg = 0;
5761
9d5c8243
AK		 5762 switch (spddplx) {
5763 case SPEED_10 + DUPLEX_HALF:
5764 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5765 break;
5766 case SPEED_10 + DUPLEX_FULL:
5767 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5768 break;
5769 case SPEED_100 + DUPLEX_HALF:
5770 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5771 break;
5772 case SPEED_100 + DUPLEX_FULL:
5773 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5774 break;
5775 case SPEED_1000 + DUPLEX_FULL:
5776 mac->autoneg = 1;
5777 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5778 break;
5779 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5780 default:
090b1795 5781 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 5782 return -EINVAL;
5783 }
5784 return 0;
5785}
5786
3fe7c4c9 5787static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5788{
5789 struct net_device *netdev = pci_get_drvdata(pdev);
5790 struct igb_adapter *adapter = netdev_priv(netdev);
5791 struct e1000_hw *hw = &adapter->hw;
2d064c06 5792 u32 ctrl, rctl, status;
9d5c8243
AK		 5793 u32 wufc = adapter->wol;
5794#ifdef CONFIG_PM
5795 int retval = 0;
5796#endif
5797
5798 netif_device_detach(netdev);
5799
a88f10ec
AD		 5800 if (netif_running(netdev))
5801 igb_close(netdev);
5802
047e0030 5803 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5804
5805#ifdef CONFIG_PM
5806 retval = pci_save_state(pdev);
5807 if (retval)
5808 return retval;
5809#endif
5810
5811 status = rd32(E1000_STATUS);
5812 if (status & E1000_STATUS_LU)
5813 wufc &= ~E1000_WUFC_LNKC;
5814
5815 if (wufc) {
5816 igb_setup_rctl(adapter);
ff41f8dc 5817 igb_set_rx_mode(netdev);
9d5c8243
AK		 5818
5819 /* turn on all-multi mode if wake on multicast is enabled */
5820 if (wufc & E1000_WUFC_MC) {
5821 rctl = rd32(E1000_RCTL);
5822 rctl |= E1000_RCTL_MPE;
5823 wr32(E1000_RCTL, rctl);
5824 }
5825
5826 ctrl = rd32(E1000_CTRL);
5827 /* advertise wake from D3Cold */
5828 #define E1000_CTRL_ADVD3WUC 0x00100000
5829 /* phy power management enable */
5830 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5831 ctrl |= E1000_CTRL_ADVD3WUC;
5832 wr32(E1000_CTRL, ctrl);
5833
9d5c8243 5834 /* Allow time for pending master requests to run */
330a6d6a 5835 igb_disable_pcie_master(hw);
9d5c8243
AK		 5836
5837 wr32(E1000_WUC, E1000_WUC_PME_EN);
5838 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5839 } else {
5840 wr32(E1000_WUC, 0);
5841 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5842 }
5843
3fe7c4c9
RW		 5844 *enable_wake = wufc || adapter->en_mng_pt;
5845 if (!*enable_wake)
88a268c1
NN		 5846 igb_power_down_link(adapter);
5847 else
5848 igb_power_up_link(adapter);
9d5c8243
AK		 5849
5850 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5851 * would have already happened in close and is redundant. */
5852 igb_release_hw_control(adapter);
5853
5854 pci_disable_device(pdev);
5855
9d5c8243
AK		 5856 return 0;
5857}
5858
5859#ifdef CONFIG_PM
3fe7c4c9
RW		 5860static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5861{
5862 int retval;
5863 bool wake;
5864
5865 retval = __igb_shutdown(pdev, &wake);
5866 if (retval)
5867 return retval;
5868
5869 if (wake) {
5870 pci_prepare_to_sleep(pdev);
5871 } else {
5872 pci_wake_from_d3(pdev, false);
5873 pci_set_power_state(pdev, PCI_D3hot);
5874 }
5875
5876 return 0;
5877}
5878
9d5c8243
AK		 5879static int igb_resume(struct pci_dev *pdev)
5880{
5881 struct net_device *netdev = pci_get_drvdata(pdev);
5882 struct igb_adapter *adapter = netdev_priv(netdev);
5883 struct e1000_hw *hw = &adapter->hw;
5884 u32 err;
5885
5886 pci_set_power_state(pdev, PCI_D0);
5887 pci_restore_state(pdev);
b94f2d77 5888 pci_save_state(pdev);
42bfd33a 5889
aed5dec3 5890 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5891 if (err) {
5892 dev_err(&pdev->dev,
5893 "igb: Cannot enable PCI device from suspend\n");
5894 return err;
5895 }
5896 pci_set_master(pdev);
5897
5898 pci_enable_wake(pdev, PCI_D3hot, 0);
5899 pci_enable_wake(pdev, PCI_D3cold, 0);
5900
047e0030 5901 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5902 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5903 return -ENOMEM;
9d5c8243
AK		 5904 }
5905
9d5c8243 5906 igb_reset(adapter);
a8564f03
AD		 5907
5908 /* let the f/w know that the h/w is now under the control of the
5909 * driver. */
5910 igb_get_hw_control(adapter);
5911
9d5c8243
AK		 5912 wr32(E1000_WUS, ~0);
5913
a88f10ec
AD		 5914 if (netif_running(netdev)) {
5915 err = igb_open(netdev);
5916 if (err)
5917 return err;
5918 }
9d5c8243
AK		 5919
5920 netif_device_attach(netdev);
5921
9d5c8243
AK		 5922 return 0;
5923}
5924#endif
5925
5926static void igb_shutdown(struct pci_dev *pdev)
5927{
3fe7c4c9
RW		 5928 bool wake;
5929
5930 __igb_shutdown(pdev, &wake);
5931
5932 if (system_state == SYSTEM_POWER_OFF) {
5933 pci_wake_from_d3(pdev, wake);
5934 pci_set_power_state(pdev, PCI_D3hot);
5935 }
9d5c8243
AK		 5936}
5937
5938#ifdef CONFIG_NET_POLL_CONTROLLER
5939/*
5940 * Polling 'interrupt' - used by things like netconsole to send skbs
5941 * without having to re-enable interrupts. It's not called while
5942 * the interrupt routine is executing.
5943 */
5944static void igb_netpoll(struct net_device *netdev)
5945{
5946 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5947 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5948 int i;
9d5c8243 5949
eebbbdba 5950 if (!adapter->msix_entries) {
047e0030 5951 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5952 igb_irq_disable(adapter);
047e0030 5953 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5954 return;
5955 }
9d5c8243 5956
047e0030
AD		 5957 for (i = 0; i < adapter->num_q_vectors; i++) {
5958 struct igb_q_vector *q_vector = adapter->q_vector[i];
5959 wr32(E1000_EIMC, q_vector->eims_value);
5960 napi_schedule(&q_vector->napi);
eebbbdba 5961 }
9d5c8243
AK		 5962}
5963#endif /* CONFIG_NET_POLL_CONTROLLER */
5964
5965/**
5966 * igb_io_error_detected - called when PCI error is detected
5967 * @pdev: Pointer to PCI device
5968 * @state: The current pci connection state
5969 *
5970 * This function is called after a PCI bus error affecting
5971 * this device has been detected.
5972 */
5973static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5974 pci_channel_state_t state)
5975{
5976 struct net_device *netdev = pci_get_drvdata(pdev);
5977 struct igb_adapter *adapter = netdev_priv(netdev);
5978
5979 netif_device_detach(netdev);
5980
59ed6eec
AD		 5981 if (state == pci_channel_io_perm_failure)
5982 return PCI_ERS_RESULT_DISCONNECT;
5983
9d5c8243
AK		 5984 if (netif_running(netdev))
5985 igb_down(adapter);
5986 pci_disable_device(pdev);
5987
5988 /* Request a slot slot reset. */
5989 return PCI_ERS_RESULT_NEED_RESET;
5990}
5991
5992/**
5993 * igb_io_slot_reset - called after the pci bus has been reset.
5994 * @pdev: Pointer to PCI device
5995 *
5996 * Restart the card from scratch, as if from a cold-boot. Implementation
5997 * resembles the first-half of the igb_resume routine.
5998 */
5999static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6000{
6001 struct net_device *netdev = pci_get_drvdata(pdev);
6002 struct igb_adapter *adapter = netdev_priv(netdev);
6003 struct e1000_hw *hw = &adapter->hw;
40a914fa 6004 pci_ers_result_t result;
42bfd33a 6005 int err;
9d5c8243 6006
aed5dec3 6007 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6008 dev_err(&pdev->dev,
6009 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6010 result = PCI_ERS_RESULT_DISCONNECT;
6011 } else {
6012 pci_set_master(pdev);
6013 pci_restore_state(pdev);
b94f2d77 6014 pci_save_state(pdev);
9d5c8243 6015
40a914fa
AD		 6016 pci_enable_wake(pdev, PCI_D3hot, 0);
6017 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6018
40a914fa
AD		 6019 igb_reset(adapter);
6020 wr32(E1000_WUS, ~0);
6021 result = PCI_ERS_RESULT_RECOVERED;
6022 }
9d5c8243 6023
ea943d41
JK		 6024 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6025 if (err) {
6026 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6027 "failed 0x%0x\n", err);
6028 /* non-fatal, continue */
6029 }
40a914fa
AD		 6030
6031 return result;
9d5c8243
AK		 6032}
6033
6034/**
6035 * igb_io_resume - called when traffic can start flowing again.
6036 * @pdev: Pointer to PCI device
6037 *
6038 * This callback is called when the error recovery driver tells us that
6039 * its OK to resume normal operation. Implementation resembles the
6040 * second-half of the igb_resume routine.
6041 */
6042static void igb_io_resume(struct pci_dev *pdev)
6043{
6044 struct net_device *netdev = pci_get_drvdata(pdev);
6045 struct igb_adapter *adapter = netdev_priv(netdev);
6046
9d5c8243
AK		 6047 if (netif_running(netdev)) {
6048 if (igb_up(adapter)) {
6049 dev_err(&pdev->dev, "igb_up failed after reset\n");
6050 return;
6051 }
6052 }
6053
6054 netif_device_attach(netdev);
6055
6056 /* let the f/w know that the h/w is now under the control of the
6057 * driver. */
6058 igb_get_hw_control(adapter);
9d5c8243
AK		 6059}
6060
26ad9178
AD		 6061static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6062 u8 qsel)
6063{
6064 u32 rar_low, rar_high;
6065 struct e1000_hw *hw = &adapter->hw;
6066
6067 /* HW expects these in little endian so we reverse the byte order
6068 * from network order (big endian) to little endian
6069 */
6070 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6071 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6072 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6073
6074 /* Indicate to hardware the Address is Valid. */
6075 rar_high |= E1000_RAH_AV;
6076
6077 if (hw->mac.type == e1000_82575)
6078 rar_high |= E1000_RAH_POOL_1 * qsel;
6079 else
6080 rar_high |= E1000_RAH_POOL_1 << qsel;
6081
6082 wr32(E1000_RAL(index), rar_low);
6083 wrfl();
6084 wr32(E1000_RAH(index), rar_high);
6085 wrfl();
6086}
6087
4ae196df
AD		 6088static int igb_set_vf_mac(struct igb_adapter *adapter,
6089 int vf, unsigned char *mac_addr)
6090{
6091 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6092 /* VF MAC addresses start at end of receive addresses and moves
6093 * torwards the first, as a result a collision should not be possible */
6094 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6095
37680117 6096 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6097
26ad9178 6098 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6099
6100 return 0;
6101}
6102
8151d294
WM		 6103static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6104{
6105 struct igb_adapter *adapter = netdev_priv(netdev);
6106 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6107 return -EINVAL;
6108 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6109 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6110 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6111 " change effective.");
6112 if (test_bit(__IGB_DOWN, &adapter->state)) {
6113 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6114 " but the PF device is not up.\n");
6115 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6116 " attempting to use the VF device.\n");
6117 }
6118 return igb_set_vf_mac(adapter, vf, mac);
6119}
6120
6121static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6122{
6123 return -EOPNOTSUPP;
6124}
6125
6126static int igb_ndo_get_vf_config(struct net_device *netdev,
6127 int vf, struct ifla_vf_info *ivi)
6128{
6129 struct igb_adapter *adapter = netdev_priv(netdev);
6130 if (vf >= adapter->vfs_allocated_count)
6131 return -EINVAL;
6132 ivi->vf = vf;
6133 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6134 ivi->tx_rate = 0;
6135 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6136 ivi->qos = adapter->vf_data[vf].pf_qos;
6137 return 0;
6138}
6139
4ae196df
AD		 6140static void igb_vmm_control(struct igb_adapter *adapter)
6141{
6142 struct e1000_hw *hw = &adapter->hw;
10d8e907 6143 u32 reg;
4ae196df 6144
d4960307
AD		 6145 /* replication is not supported for 82575 */
6146 if (hw->mac.type == e1000_82575)
4ae196df
AD		 6147 return;
6148
10d8e907
AD		 6149 /* enable replication vlan tag stripping */
6150 reg = rd32(E1000_RPLOLR);
6151 reg |= E1000_RPLOLR_STRVLAN;
6152 wr32(E1000_RPLOLR, reg);
6153
6154 /* notify HW that the MAC is adding vlan tags */
6155 reg = rd32(E1000_DTXCTL);
6156 reg |= E1000_DTXCTL_VLAN_ADDED;
6157 wr32(E1000_DTXCTL, reg);
6158
d4960307
AD		 6159 if (adapter->vfs_allocated_count) {
6160 igb_vmdq_set_loopback_pf(hw, true);
6161 igb_vmdq_set_replication_pf(hw, true);
6162 } else {
6163 igb_vmdq_set_loopback_pf(hw, false);
6164 igb_vmdq_set_replication_pf(hw, false);
6165 }
4ae196df
AD		 6166}
6167
9d5c8243 6168/* igb_main.c */
kernel source for telekom puls (alcatel/mediatek)