projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'idle-release' of git://git.kernel.org/pub/scm/linux/kernel/git/lenb...
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2011 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18 #include <linux/etherdevice.h>
19 #include <linux/if_vlan.h>
20 #include <linux/interrupt.h>
21 #include <linux/ip.h>
22 #include <net/ipv6.h>
23 #include <net/ip6_checksum.h>
24 #include <linux/firmware.h>
25 #include <linux/prefetch.h>
26 #include "bnx2x_cmn.h"
27 #include "bnx2x_init.h"
28 #include "bnx2x_sp.h"
29
30
31
32 /**
33 * bnx2x_bz_fp - zero content of the fastpath structure.
34 *
35 * @bp: driver handle
36 * @index: fastpath index to be zeroed
37 *
38 * Makes sure the contents of the bp->fp[index].napi is kept
39 * intact.
40 */
41 static inline void bnx2x_bz_fp(struct bnx2x *bp, int index)
42 {
43 struct bnx2x_fastpath *fp = &bp->fp[index];
44 struct napi_struct orig_napi = fp->napi;
45 /* bzero bnx2x_fastpath contents */
46 memset(fp, 0, sizeof(*fp));
47
48 /* Restore the NAPI object as it has been already initialized */
49 fp->napi = orig_napi;
50
51 fp->bp = bp;
52 fp->index = index;
53 if (IS_ETH_FP(fp))
54 fp->max_cos = bp->max_cos;
55 else
56 /* Special queues support only one CoS */
57 fp->max_cos = 1;
58
59 /*
60 * set the tpa flag for each queue. The tpa flag determines the queue
61 * minimal size so it must be set prior to queue memory allocation
62 */
63 fp->disable_tpa = ((bp->flags & TPA_ENABLE_FLAG) == 0);
64
65 #ifdef BCM_CNIC
66 /* We don't want TPA on FCoE, FWD and OOO L2 rings */
67 bnx2x_fcoe(bp, disable_tpa) = 1;
68 #endif
69 }
70
71 /**
72 * bnx2x_move_fp - move content of the fastpath structure.
73 *
74 * @bp: driver handle
75 * @from: source FP index
76 * @to: destination FP index
77 *
78 * Makes sure the contents of the bp->fp[to].napi is kept
79 * intact.
80 */
81 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
82 {
83 struct bnx2x_fastpath *from_fp = &bp->fp[from];
84 struct bnx2x_fastpath *to_fp = &bp->fp[to];
85 struct napi_struct orig_napi = to_fp->napi;
86 /* Move bnx2x_fastpath contents */
87 memcpy(to_fp, from_fp, sizeof(*to_fp));
88 to_fp->index = to;
89
90 /* Restore the NAPI object as it has been already initialized */
91 to_fp->napi = orig_napi;
92 }
93
94 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
95
96 /* free skb in the packet ring at pos idx
97 * return idx of last bd freed
98 */
99 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
100 u16 idx)
101 {
102 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
103 struct eth_tx_start_bd *tx_start_bd;
104 struct eth_tx_bd *tx_data_bd;
105 struct sk_buff *skb = tx_buf->skb;
106 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
107 int nbd;
108
109 /* prefetch skb end pointer to speedup dev_kfree_skb() */
110 prefetch(&skb->end);
111
112 DP(BNX2X_MSG_FP, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
113 txdata->txq_index, idx, tx_buf, skb);
114
115 /* unmap first bd */
116 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
117 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
118 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
119 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
120
121
122 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
123 #ifdef BNX2X_STOP_ON_ERROR
124 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
125 BNX2X_ERR("BAD nbd!\n");
126 bnx2x_panic();
127 }
128 #endif
129 new_cons = nbd + tx_buf->first_bd;
130
131 /* Get the next bd */
132 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
133
134 /* Skip a parse bd... */
135 --nbd;
136 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
137
138 /* ...and the TSO split header bd since they have no mapping */
139 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
140 --nbd;
141 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
142 }
143
144 /* now free frags */
145 while (nbd > 0) {
146
147 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
148 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
149 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
150 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
151 if (--nbd)
152 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
153 }
154
155 /* release skb */
156 WARN_ON(!skb);
157 dev_kfree_skb_any(skb);
158 tx_buf->first_bd = 0;
159 tx_buf->skb = NULL;
160
161 return new_cons;
162 }
163
164 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
165 {
166 struct netdev_queue *txq;
167 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
168
169 #ifdef BNX2X_STOP_ON_ERROR
170 if (unlikely(bp->panic))
171 return -1;
172 #endif
173
174 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
175 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
176 sw_cons = txdata->tx_pkt_cons;
177
178 while (sw_cons != hw_cons) {
179 u16 pkt_cons;
180
181 pkt_cons = TX_BD(sw_cons);
182
183 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u sw_cons %u "
184 " pkt_cons %u\n",
185 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
186
187 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons);
188 sw_cons++;
189 }
190
191 txdata->tx_pkt_cons = sw_cons;
192 txdata->tx_bd_cons = bd_cons;
193
194 /* Need to make the tx_bd_cons update visible to start_xmit()
195 * before checking for netif_tx_queue_stopped(). Without the
196 * memory barrier, there is a small possibility that
197 * start_xmit() will miss it and cause the queue to be stopped
198 * forever.
199 * On the other hand we need an rmb() here to ensure the proper
200 * ordering of bit testing in the following
201 * netif_tx_queue_stopped(txq) call.
202 */
203 smp_mb();
204
205 if (unlikely(netif_tx_queue_stopped(txq))) {
206 /* Taking tx_lock() is needed to prevent reenabling the queue
207 * while it's empty. This could have happen if rx_action() gets
208 * suspended in bnx2x_tx_int() after the condition before
209 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
210 *
211 * stops the queue->sees fresh tx_bd_cons->releases the queue->
212 * sends some packets consuming the whole queue again->
213 * stops the queue
214 */
215
216 __netif_tx_lock(txq, smp_processor_id());
217
218 if ((netif_tx_queue_stopped(txq)) &&
219 (bp->state == BNX2X_STATE_OPEN) &&
220 (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
221 netif_tx_wake_queue(txq);
222
223 __netif_tx_unlock(txq);
224 }
225 return 0;
226 }
227
228 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
229 u16 idx)
230 {
231 u16 last_max = fp->last_max_sge;
232
233 if (SUB_S16(idx, last_max) > 0)
234 fp->last_max_sge = idx;
235 }
236
237 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
238 struct eth_fast_path_rx_cqe *fp_cqe)
239 {
240 struct bnx2x *bp = fp->bp;
241 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
242 le16_to_cpu(fp_cqe->len_on_bd)) >>
243 SGE_PAGE_SHIFT;
244 u16 last_max, last_elem, first_elem;
245 u16 delta = 0;
246 u16 i;
247
248 if (!sge_len)
249 return;
250
251 /* First mark all used pages */
252 for (i = 0; i < sge_len; i++)
253 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
254 RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
255
256 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
257 sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
258
259 /* Here we assume that the last SGE index is the biggest */
260 prefetch((void *)(fp->sge_mask));
261 bnx2x_update_last_max_sge(fp,
262 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
263
264 last_max = RX_SGE(fp->last_max_sge);
265 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
266 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
267
268 /* If ring is not full */
269 if (last_elem + 1 != first_elem)
270 last_elem++;
271
272 /* Now update the prod */
273 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
274 if (likely(fp->sge_mask[i]))
275 break;
276
277 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
278 delta += BIT_VEC64_ELEM_SZ;
279 }
280
281 if (delta > 0) {
282 fp->rx_sge_prod += delta;
283 /* clear page-end entries */
284 bnx2x_clear_sge_mask_next_elems(fp);
285 }
286
287 DP(NETIF_MSG_RX_STATUS,
288 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
289 fp->last_max_sge, fp->rx_sge_prod);
290 }
291
292 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
293 struct sk_buff *skb, u16 cons, u16 prod,
294 struct eth_fast_path_rx_cqe *cqe)
295 {
296 struct bnx2x *bp = fp->bp;
297 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
298 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
299 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
300 dma_addr_t mapping;
301 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
302 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
303
304 /* print error if current state != stop */
305 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
306 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
307
308 /* Try to map an empty skb from the aggregation info */
309 mapping = dma_map_single(&bp->pdev->dev,
310 first_buf->skb->data,
311 fp->rx_buf_size, DMA_FROM_DEVICE);
312 /*
313 * ...if it fails - move the skb from the consumer to the producer
314 * and set the current aggregation state as ERROR to drop it
315 * when TPA_STOP arrives.
316 */
317
318 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
319 /* Move the BD from the consumer to the producer */
320 bnx2x_reuse_rx_skb(fp, cons, prod);
321 tpa_info->tpa_state = BNX2X_TPA_ERROR;
322 return;
323 }
324
325 /* move empty skb from pool to prod */
326 prod_rx_buf->skb = first_buf->skb;
327 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
328 /* point prod_bd to new skb */
329 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
330 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
331
332 /* move partial skb from cons to pool (don't unmap yet) */
333 *first_buf = *cons_rx_buf;
334
335 /* mark bin state as START */
336 tpa_info->parsing_flags =
337 le16_to_cpu(cqe->pars_flags.flags);
338 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
339 tpa_info->tpa_state = BNX2X_TPA_START;
340 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
341 tpa_info->placement_offset = cqe->placement_offset;
342
343 #ifdef BNX2X_STOP_ON_ERROR
344 fp->tpa_queue_used |= (1 << queue);
345 #ifdef _ASM_GENERIC_INT_L64_H
346 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
347 #else
348 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
349 #endif
350 fp->tpa_queue_used);
351 #endif
352 }
353
354 /* Timestamp option length allowed for TPA aggregation:
355 *
356 * nop nop kind length echo val
357 */
358 #define TPA_TSTAMP_OPT_LEN 12
359 /**
360 * bnx2x_set_lro_mss - calculate the approximate value of the MSS
361 *
362 * @bp: driver handle
363 * @parsing_flags: parsing flags from the START CQE
364 * @len_on_bd: total length of the first packet for the
365 * aggregation.
366 *
367 * Approximate value of the MSS for this aggregation calculated using
368 * the first packet of it.
369 */
370 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
371 u16 len_on_bd)
372 {
373 /*
374 * TPA arrgregation won't have either IP options or TCP options
375 * other than timestamp or IPv6 extension headers.
376 */
377 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
378
379 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
380 PRS_FLAG_OVERETH_IPV6)
381 hdrs_len += sizeof(struct ipv6hdr);
382 else /* IPv4 */
383 hdrs_len += sizeof(struct iphdr);
384
385
386 /* Check if there was a TCP timestamp, if there is it's will
387 * always be 12 bytes length: nop nop kind length echo val.
388 *
389 * Otherwise FW would close the aggregation.
390 */
391 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
392 hdrs_len += TPA_TSTAMP_OPT_LEN;
393
394 return len_on_bd - hdrs_len;
395 }
396
397 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
398 u16 queue, struct sk_buff *skb,
399 struct eth_end_agg_rx_cqe *cqe,
400 u16 cqe_idx)
401 {
402 struct sw_rx_page *rx_pg, old_rx_pg;
403 u32 i, frag_len, frag_size, pages;
404 int err;
405 int j;
406 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
407 u16 len_on_bd = tpa_info->len_on_bd;
408
409 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
410 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
411
412 /* This is needed in order to enable forwarding support */
413 if (frag_size)
414 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
415 tpa_info->parsing_flags, len_on_bd);
416
417 #ifdef BNX2X_STOP_ON_ERROR
418 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
419 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
420 pages, cqe_idx);
421 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
422 bnx2x_panic();
423 return -EINVAL;
424 }
425 #endif
426
427 /* Run through the SGL and compose the fragmented skb */
428 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
429 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
430
431 /* FW gives the indices of the SGE as if the ring is an array
432 (meaning that "next" element will consume 2 indices) */
433 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
434 rx_pg = &fp->rx_page_ring[sge_idx];
435 old_rx_pg = *rx_pg;
436
437 /* If we fail to allocate a substitute page, we simply stop
438 where we are and drop the whole packet */
439 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
440 if (unlikely(err)) {
441 fp->eth_q_stats.rx_skb_alloc_failed++;
442 return err;
443 }
444
445 /* Unmap the page as we r going to pass it to the stack */
446 dma_unmap_page(&bp->pdev->dev,
447 dma_unmap_addr(&old_rx_pg, mapping),
448 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
449
450 /* Add one frag and update the appropriate fields in the skb */
451 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
452
453 skb->data_len += frag_len;
454 skb->truesize += frag_len;
455 skb->len += frag_len;
456
457 frag_size -= frag_len;
458 }
459
460 return 0;
461 }
462
463 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
464 u16 queue, struct eth_end_agg_rx_cqe *cqe,
465 u16 cqe_idx)
466 {
467 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
468 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
469 u8 pad = tpa_info->placement_offset;
470 u16 len = tpa_info->len_on_bd;
471 struct sk_buff *skb = rx_buf->skb;
472 /* alloc new skb */
473 struct sk_buff *new_skb;
474 u8 old_tpa_state = tpa_info->tpa_state;
475
476 tpa_info->tpa_state = BNX2X_TPA_STOP;
477
478 /* If we there was an error during the handling of the TPA_START -
479 * drop this aggregation.
480 */
481 if (old_tpa_state == BNX2X_TPA_ERROR)
482 goto drop;
483
484 /* Try to allocate the new skb */
485 new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
486
487 /* Unmap skb in the pool anyway, as we are going to change
488 pool entry status to BNX2X_TPA_STOP even if new skb allocation
489 fails. */
490 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
491 fp->rx_buf_size, DMA_FROM_DEVICE);
492
493 if (likely(new_skb)) {
494 prefetch(skb);
495 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
496
497 #ifdef BNX2X_STOP_ON_ERROR
498 if (pad + len > fp->rx_buf_size) {
499 BNX2X_ERR("skb_put is about to fail... "
500 "pad %d len %d rx_buf_size %d\n",
501 pad, len, fp->rx_buf_size);
502 bnx2x_panic();
503 return;
504 }
505 #endif
506
507 skb_reserve(skb, pad);
508 skb_put(skb, len);
509
510 skb->protocol = eth_type_trans(skb, bp->dev);
511 skb->ip_summed = CHECKSUM_UNNECESSARY;
512
513 if (!bnx2x_fill_frag_skb(bp, fp, queue, skb, cqe, cqe_idx)) {
514 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
515 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
516 napi_gro_receive(&fp->napi, skb);
517 } else {
518 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
519 " - dropping packet!\n");
520 dev_kfree_skb_any(skb);
521 }
522
523
524 /* put new skb in bin */
525 rx_buf->skb = new_skb;
526
527 return;
528 }
529
530 drop:
531 /* drop the packet and keep the buffer in the bin */
532 DP(NETIF_MSG_RX_STATUS,
533 "Failed to allocate or map a new skb - dropping packet!\n");
534 fp->eth_q_stats.rx_skb_alloc_failed++;
535 }
536
537 /* Set Toeplitz hash value in the skb using the value from the
538 * CQE (calculated by HW).
539 */
540 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
541 struct sk_buff *skb)
542 {
543 /* Set Toeplitz hash from CQE */
544 if ((bp->dev->features & NETIF_F_RXHASH) &&
545 (cqe->fast_path_cqe.status_flags &
546 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
547 skb->rxhash =
548 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
549 }
550
551 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
552 {
553 struct bnx2x *bp = fp->bp;
554 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
555 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
556 int rx_pkt = 0;
557
558 #ifdef BNX2X_STOP_ON_ERROR
559 if (unlikely(bp->panic))
560 return 0;
561 #endif
562
563 /* CQ "next element" is of the size of the regular element,
564 that's why it's ok here */
565 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
566 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
567 hw_comp_cons++;
568
569 bd_cons = fp->rx_bd_cons;
570 bd_prod = fp->rx_bd_prod;
571 bd_prod_fw = bd_prod;
572 sw_comp_cons = fp->rx_comp_cons;
573 sw_comp_prod = fp->rx_comp_prod;
574
575 /* Memory barrier necessary as speculative reads of the rx
576 * buffer can be ahead of the index in the status block
577 */
578 rmb();
579
580 DP(NETIF_MSG_RX_STATUS,
581 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
582 fp->index, hw_comp_cons, sw_comp_cons);
583
584 while (sw_comp_cons != hw_comp_cons) {
585 struct sw_rx_bd *rx_buf = NULL;
586 struct sk_buff *skb;
587 union eth_rx_cqe *cqe;
588 struct eth_fast_path_rx_cqe *cqe_fp;
589 u8 cqe_fp_flags;
590 enum eth_rx_cqe_type cqe_fp_type;
591 u16 len, pad;
592
593 #ifdef BNX2X_STOP_ON_ERROR
594 if (unlikely(bp->panic))
595 return 0;
596 #endif
597
598 comp_ring_cons = RCQ_BD(sw_comp_cons);
599 bd_prod = RX_BD(bd_prod);
600 bd_cons = RX_BD(bd_cons);
601
602 /* Prefetch the page containing the BD descriptor
603 at producer's index. It will be needed when new skb is
604 allocated */
605 prefetch((void *)(PAGE_ALIGN((unsigned long)
606 (&fp->rx_desc_ring[bd_prod])) -
607 PAGE_SIZE + 1));
608
609 cqe = &fp->rx_comp_ring[comp_ring_cons];
610 cqe_fp = &cqe->fast_path_cqe;
611 cqe_fp_flags = cqe_fp->type_error_flags;
612 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
613
614 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x"
615 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags),
616 cqe_fp_flags, cqe_fp->status_flags,
617 le32_to_cpu(cqe_fp->rss_hash_result),
618 le16_to_cpu(cqe_fp->vlan_tag), le16_to_cpu(cqe_fp->pkt_len));
619
620 /* is this a slowpath msg? */
621 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
622 bnx2x_sp_event(fp, cqe);
623 goto next_cqe;
624
625 /* this is an rx packet */
626 } else {
627 rx_buf = &fp->rx_buf_ring[bd_cons];
628 skb = rx_buf->skb;
629 prefetch(skb);
630
631 if (!CQE_TYPE_FAST(cqe_fp_type)) {
632 #ifdef BNX2X_STOP_ON_ERROR
633 /* sanity check */
634 if (fp->disable_tpa &&
635 (CQE_TYPE_START(cqe_fp_type) ||
636 CQE_TYPE_STOP(cqe_fp_type)))
637 BNX2X_ERR("START/STOP packet while "
638 "disable_tpa type %x\n",
639 CQE_TYPE(cqe_fp_type));
640 #endif
641
642 if (CQE_TYPE_START(cqe_fp_type)) {
643 u16 queue = cqe_fp->queue_index;
644 DP(NETIF_MSG_RX_STATUS,
645 "calling tpa_start on queue %d\n",
646 queue);
647
648 bnx2x_tpa_start(fp, queue, skb,
649 bd_cons, bd_prod,
650 cqe_fp);
651
652 /* Set Toeplitz hash for LRO skb */
653 bnx2x_set_skb_rxhash(bp, cqe, skb);
654
655 goto next_rx;
656
657 } else {
658 u16 queue =
659 cqe->end_agg_cqe.queue_index;
660 DP(NETIF_MSG_RX_STATUS,
661 "calling tpa_stop on queue %d\n",
662 queue);
663
664 bnx2x_tpa_stop(bp, fp, queue,
665 &cqe->end_agg_cqe,
666 comp_ring_cons);
667 #ifdef BNX2X_STOP_ON_ERROR
668 if (bp->panic)
669 return 0;
670 #endif
671
672 bnx2x_update_sge_prod(fp, cqe_fp);
673 goto next_cqe;
674 }
675 }
676 /* non TPA */
677 len = le16_to_cpu(cqe_fp->pkt_len);
678 pad = cqe_fp->placement_offset;
679 dma_sync_single_for_cpu(&bp->pdev->dev,
680 dma_unmap_addr(rx_buf, mapping),
681 pad + RX_COPY_THRESH,
682 DMA_FROM_DEVICE);
683 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
684
685 /* is this an error packet? */
686 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
687 DP(NETIF_MSG_RX_ERR,
688 "ERROR flags %x rx packet %u\n",
689 cqe_fp_flags, sw_comp_cons);
690 fp->eth_q_stats.rx_err_discard_pkt++;
691 goto reuse_rx;
692 }
693
694 /* Since we don't have a jumbo ring
695 * copy small packets if mtu > 1500
696 */
697 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
698 (len <= RX_COPY_THRESH)) {
699 struct sk_buff *new_skb;
700
701 new_skb = netdev_alloc_skb(bp->dev, len + pad);
702 if (new_skb == NULL) {
703 DP(NETIF_MSG_RX_ERR,
704 "ERROR packet dropped "
705 "because of alloc failure\n");
706 fp->eth_q_stats.rx_skb_alloc_failed++;
707 goto reuse_rx;
708 }
709
710 /* aligned copy */
711 skb_copy_from_linear_data_offset(skb, pad,
712 new_skb->data + pad, len);
713 skb_reserve(new_skb, pad);
714 skb_put(new_skb, len);
715
716 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
717
718 skb = new_skb;
719
720 } else
721 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
722 dma_unmap_single(&bp->pdev->dev,
723 dma_unmap_addr(rx_buf, mapping),
724 fp->rx_buf_size,
725 DMA_FROM_DEVICE);
726 skb_reserve(skb, pad);
727 skb_put(skb, len);
728
729 } else {
730 DP(NETIF_MSG_RX_ERR,
731 "ERROR packet dropped because "
732 "of alloc failure\n");
733 fp->eth_q_stats.rx_skb_alloc_failed++;
734 reuse_rx:
735 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
736 goto next_rx;
737 }
738
739 skb->protocol = eth_type_trans(skb, bp->dev);
740
741 /* Set Toeplitz hash for a none-LRO skb */
742 bnx2x_set_skb_rxhash(bp, cqe, skb);
743
744 skb_checksum_none_assert(skb);
745
746 if (bp->dev->features & NETIF_F_RXCSUM) {
747
748 if (likely(BNX2X_RX_CSUM_OK(cqe)))
749 skb->ip_summed = CHECKSUM_UNNECESSARY;
750 else
751 fp->eth_q_stats.hw_csum_err++;
752 }
753 }
754
755 skb_record_rx_queue(skb, fp->index);
756
757 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
758 PARSING_FLAGS_VLAN)
759 __vlan_hwaccel_put_tag(skb,
760 le16_to_cpu(cqe_fp->vlan_tag));
761 napi_gro_receive(&fp->napi, skb);
762
763
764 next_rx:
765 rx_buf->skb = NULL;
766
767 bd_cons = NEXT_RX_IDX(bd_cons);
768 bd_prod = NEXT_RX_IDX(bd_prod);
769 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
770 rx_pkt++;
771 next_cqe:
772 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
773 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
774
775 if (rx_pkt == budget)
776 break;
777 } /* while */
778
779 fp->rx_bd_cons = bd_cons;
780 fp->rx_bd_prod = bd_prod_fw;
781 fp->rx_comp_cons = sw_comp_cons;
782 fp->rx_comp_prod = sw_comp_prod;
783
784 /* Update producers */
785 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
786 fp->rx_sge_prod);
787
788 fp->rx_pkt += rx_pkt;
789 fp->rx_calls++;
790
791 return rx_pkt;
792 }
793
794 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
795 {
796 struct bnx2x_fastpath *fp = fp_cookie;
797 struct bnx2x *bp = fp->bp;
798 u8 cos;
799
800 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
801 "[fp %d fw_sd %d igusb %d]\n",
802 fp->index, fp->fw_sb_id, fp->igu_sb_id);
803 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
804
805 #ifdef BNX2X_STOP_ON_ERROR
806 if (unlikely(bp->panic))
807 return IRQ_HANDLED;
808 #endif
809
810 /* Handle Rx and Tx according to MSI-X vector */
811 prefetch(fp->rx_cons_sb);
812
813 for_each_cos_in_tx_queue(fp, cos)
814 prefetch(fp->txdata[cos].tx_cons_sb);
815
816 prefetch(&fp->sb_running_index[SM_RX_ID]);
817 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
818
819 return IRQ_HANDLED;
820 }
821
822 /* HW Lock for shared dual port PHYs */
823 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
824 {
825 mutex_lock(&bp->port.phy_mutex);
826
827 if (bp->port.need_hw_lock)
828 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
829 }
830
831 void bnx2x_release_phy_lock(struct bnx2x *bp)
832 {
833 if (bp->port.need_hw_lock)
834 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
835
836 mutex_unlock(&bp->port.phy_mutex);
837 }
838
839 /* calculates MF speed according to current linespeed and MF configuration */
840 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
841 {
842 u16 line_speed = bp->link_vars.line_speed;
843 if (IS_MF(bp)) {
844 u16 maxCfg = bnx2x_extract_max_cfg(bp,
845 bp->mf_config[BP_VN(bp)]);
846
847 /* Calculate the current MAX line speed limit for the MF
848 * devices
849 */
850 if (IS_MF_SI(bp))
851 line_speed = (line_speed * maxCfg) / 100;
852 else { /* SD mode */
853 u16 vn_max_rate = maxCfg * 100;
854
855 if (vn_max_rate < line_speed)
856 line_speed = vn_max_rate;
857 }
858 }
859
860 return line_speed;
861 }
862
863 /**
864 * bnx2x_fill_report_data - fill link report data to report
865 *
866 * @bp: driver handle
867 * @data: link state to update
868 *
869 * It uses a none-atomic bit operations because is called under the mutex.
870 */
871 static inline void bnx2x_fill_report_data(struct bnx2x *bp,
872 struct bnx2x_link_report_data *data)
873 {
874 u16 line_speed = bnx2x_get_mf_speed(bp);
875
876 memset(data, 0, sizeof(*data));
877
878 /* Fill the report data: efective line speed */
879 data->line_speed = line_speed;
880
881 /* Link is down */
882 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
883 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
884 &data->link_report_flags);
885
886 /* Full DUPLEX */
887 if (bp->link_vars.duplex == DUPLEX_FULL)
888 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
889
890 /* Rx Flow Control is ON */
891 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
892 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
893
894 /* Tx Flow Control is ON */
895 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
896 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
897 }
898
899 /**
900 * bnx2x_link_report - report link status to OS.
901 *
902 * @bp: driver handle
903 *
904 * Calls the __bnx2x_link_report() under the same locking scheme
905 * as a link/PHY state managing code to ensure a consistent link
906 * reporting.
907 */
908
909 void bnx2x_link_report(struct bnx2x *bp)
910 {
911 bnx2x_acquire_phy_lock(bp);
912 __bnx2x_link_report(bp);
913 bnx2x_release_phy_lock(bp);
914 }
915
916 /**
917 * __bnx2x_link_report - report link status to OS.
918 *
919 * @bp: driver handle
920 *
921 * None atomic inmlementation.
922 * Should be called under the phy_lock.
923 */
924 void __bnx2x_link_report(struct bnx2x *bp)
925 {
926 struct bnx2x_link_report_data cur_data;
927
928 /* reread mf_cfg */
929 if (!CHIP_IS_E1(bp))
930 bnx2x_read_mf_cfg(bp);
931
932 /* Read the current link report info */
933 bnx2x_fill_report_data(bp, &cur_data);
934
935 /* Don't report link down or exactly the same link status twice */
936 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
937 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
938 &bp->last_reported_link.link_report_flags) &&
939 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
940 &cur_data.link_report_flags)))
941 return;
942
943 bp->link_cnt++;
944
945 /* We are going to report a new link parameters now -
946 * remember the current data for the next time.
947 */
948 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
949
950 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
951 &cur_data.link_report_flags)) {
952 netif_carrier_off(bp->dev);
953 netdev_err(bp->dev, "NIC Link is Down\n");
954 return;
955 } else {
956 netif_carrier_on(bp->dev);
957 netdev_info(bp->dev, "NIC Link is Up, ");
958 pr_cont("%d Mbps ", cur_data.line_speed);
959
960 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
961 &cur_data.link_report_flags))
962 pr_cont("full duplex");
963 else
964 pr_cont("half duplex");
965
966 /* Handle the FC at the end so that only these flags would be
967 * possibly set. This way we may easily check if there is no FC
968 * enabled.
969 */
970 if (cur_data.link_report_flags) {
971 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
972 &cur_data.link_report_flags)) {
973 pr_cont(", receive ");
974 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
975 &cur_data.link_report_flags))
976 pr_cont("& transmit ");
977 } else {
978 pr_cont(", transmit ");
979 }
980 pr_cont("flow control ON");
981 }
982 pr_cont("\n");
983 }
984 }
985
986 void bnx2x_init_rx_rings(struct bnx2x *bp)
987 {
988 int func = BP_FUNC(bp);
989 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
990 ETH_MAX_AGGREGATION_QUEUES_E1H_E2;
991 u16 ring_prod;
992 int i, j;
993
994 /* Allocate TPA resources */
995 for_each_rx_queue(bp, j) {
996 struct bnx2x_fastpath *fp = &bp->fp[j];
997
998 DP(NETIF_MSG_IFUP,
999 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1000
1001 if (!fp->disable_tpa) {
1002 /* Fill the per-aggregtion pool */
1003 for (i = 0; i < max_agg_queues; i++) {
1004 struct bnx2x_agg_info *tpa_info =
1005 &fp->tpa_info[i];
1006 struct sw_rx_bd *first_buf =
1007 &tpa_info->first_buf;
1008
1009 first_buf->skb = netdev_alloc_skb(bp->dev,
1010 fp->rx_buf_size);
1011 if (!first_buf->skb) {
1012 BNX2X_ERR("Failed to allocate TPA "
1013 "skb pool for queue[%d] - "
1014 "disabling TPA on this "
1015 "queue!\n", j);
1016 bnx2x_free_tpa_pool(bp, fp, i);
1017 fp->disable_tpa = 1;
1018 break;
1019 }
1020 dma_unmap_addr_set(first_buf, mapping, 0);
1021 tpa_info->tpa_state = BNX2X_TPA_STOP;
1022 }
1023
1024 /* "next page" elements initialization */
1025 bnx2x_set_next_page_sgl(fp);
1026
1027 /* set SGEs bit mask */
1028 bnx2x_init_sge_ring_bit_mask(fp);
1029
1030 /* Allocate SGEs and initialize the ring elements */
1031 for (i = 0, ring_prod = 0;
1032 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1033
1034 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1035 BNX2X_ERR("was only able to allocate "
1036 "%d rx sges\n", i);
1037 BNX2X_ERR("disabling TPA for "
1038 "queue[%d]\n", j);
1039 /* Cleanup already allocated elements */
1040 bnx2x_free_rx_sge_range(bp, fp,
1041 ring_prod);
1042 bnx2x_free_tpa_pool(bp, fp,
1043 max_agg_queues);
1044 fp->disable_tpa = 1;
1045 ring_prod = 0;
1046 break;
1047 }
1048 ring_prod = NEXT_SGE_IDX(ring_prod);
1049 }
1050
1051 fp->rx_sge_prod = ring_prod;
1052 }
1053 }
1054
1055 for_each_rx_queue(bp, j) {
1056 struct bnx2x_fastpath *fp = &bp->fp[j];
1057
1058 fp->rx_bd_cons = 0;
1059
1060 /* Activate BD ring */
1061 /* Warning!
1062 * this will generate an interrupt (to the TSTORM)
1063 * must only be done after chip is initialized
1064 */
1065 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1066 fp->rx_sge_prod);
1067
1068 if (j != 0)
1069 continue;
1070
1071 if (CHIP_IS_E1(bp)) {
1072 REG_WR(bp, BAR_USTRORM_INTMEM +
1073 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1074 U64_LO(fp->rx_comp_mapping));
1075 REG_WR(bp, BAR_USTRORM_INTMEM +
1076 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1077 U64_HI(fp->rx_comp_mapping));
1078 }
1079 }
1080 }
1081
1082 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1083 {
1084 int i;
1085 u8 cos;
1086
1087 for_each_tx_queue(bp, i) {
1088 struct bnx2x_fastpath *fp = &bp->fp[i];
1089 for_each_cos_in_tx_queue(fp, cos) {
1090 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
1091
1092 u16 bd_cons = txdata->tx_bd_cons;
1093 u16 sw_prod = txdata->tx_pkt_prod;
1094 u16 sw_cons = txdata->tx_pkt_cons;
1095
1096 while (sw_cons != sw_prod) {
1097 bd_cons = bnx2x_free_tx_pkt(bp, txdata,
1098 TX_BD(sw_cons));
1099 sw_cons++;
1100 }
1101 }
1102 }
1103 }
1104
1105 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1106 {
1107 struct bnx2x *bp = fp->bp;
1108 int i;
1109
1110 /* ring wasn't allocated */
1111 if (fp->rx_buf_ring == NULL)
1112 return;
1113
1114 for (i = 0; i < NUM_RX_BD; i++) {
1115 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1116 struct sk_buff *skb = rx_buf->skb;
1117
1118 if (skb == NULL)
1119 continue;
1120 dma_unmap_single(&bp->pdev->dev,
1121 dma_unmap_addr(rx_buf, mapping),
1122 fp->rx_buf_size, DMA_FROM_DEVICE);
1123
1124 rx_buf->skb = NULL;
1125 dev_kfree_skb(skb);
1126 }
1127 }
1128
1129 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1130 {
1131 int j;
1132
1133 for_each_rx_queue(bp, j) {
1134 struct bnx2x_fastpath *fp = &bp->fp[j];
1135
1136 bnx2x_free_rx_bds(fp);
1137
1138 if (!fp->disable_tpa)
1139 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
1140 ETH_MAX_AGGREGATION_QUEUES_E1 :
1141 ETH_MAX_AGGREGATION_QUEUES_E1H_E2);
1142 }
1143 }
1144
1145 void bnx2x_free_skbs(struct bnx2x *bp)
1146 {
1147 bnx2x_free_tx_skbs(bp);
1148 bnx2x_free_rx_skbs(bp);
1149 }
1150
1151 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1152 {
1153 /* load old values */
1154 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1155
1156 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1157 /* leave all but MAX value */
1158 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1159
1160 /* set new MAX value */
1161 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1162 & FUNC_MF_CFG_MAX_BW_MASK;
1163
1164 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1165 }
1166 }
1167
1168 /**
1169 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1170 *
1171 * @bp: driver handle
1172 * @nvecs: number of vectors to be released
1173 */
1174 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1175 {
1176 int i, offset = 0;
1177
1178 if (nvecs == offset)
1179 return;
1180 free_irq(bp->msix_table[offset].vector, bp->dev);
1181 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1182 bp->msix_table[offset].vector);
1183 offset++;
1184 #ifdef BCM_CNIC
1185 if (nvecs == offset)
1186 return;
1187 offset++;
1188 #endif
1189
1190 for_each_eth_queue(bp, i) {
1191 if (nvecs == offset)
1192 return;
1193 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d "
1194 "irq\n", i, bp->msix_table[offset].vector);
1195
1196 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1197 }
1198 }
1199
1200 void bnx2x_free_irq(struct bnx2x *bp)
1201 {
1202 if (bp->flags & USING_MSIX_FLAG)
1203 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1204 CNIC_PRESENT + 1);
1205 else if (bp->flags & USING_MSI_FLAG)
1206 free_irq(bp->pdev->irq, bp->dev);
1207 else
1208 free_irq(bp->pdev->irq, bp->dev);
1209 }
1210
1211 int bnx2x_enable_msix(struct bnx2x *bp)
1212 {
1213 int msix_vec = 0, i, rc, req_cnt;
1214
1215 bp->msix_table[msix_vec].entry = msix_vec;
1216 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1217 bp->msix_table[0].entry);
1218 msix_vec++;
1219
1220 #ifdef BCM_CNIC
1221 bp->msix_table[msix_vec].entry = msix_vec;
1222 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1223 bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1224 msix_vec++;
1225 #endif
1226 /* We need separate vectors for ETH queues only (not FCoE) */
1227 for_each_eth_queue(bp, i) {
1228 bp->msix_table[msix_vec].entry = msix_vec;
1229 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1230 "(fastpath #%u)\n", msix_vec, msix_vec, i);
1231 msix_vec++;
1232 }
1233
1234 req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1235
1236 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1237
1238 /*
1239 * reconfigure number of tx/rx queues according to available
1240 * MSI-X vectors
1241 */
1242 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1243 /* how less vectors we will have? */
1244 int diff = req_cnt - rc;
1245
1246 DP(NETIF_MSG_IFUP,
1247 "Trying to use less MSI-X vectors: %d\n", rc);
1248
1249 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1250
1251 if (rc) {
1252 DP(NETIF_MSG_IFUP,
1253 "MSI-X is not attainable rc %d\n", rc);
1254 return rc;
1255 }
1256 /*
1257 * decrease number of queues by number of unallocated entries
1258 */
1259 bp->num_queues -= diff;
1260
1261 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1262 bp->num_queues);
1263 } else if (rc) {
1264 /* fall to INTx if not enough memory */
1265 if (rc == -ENOMEM)
1266 bp->flags |= DISABLE_MSI_FLAG;
1267 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc);
1268 return rc;
1269 }
1270
1271 bp->flags |= USING_MSIX_FLAG;
1272
1273 return 0;
1274 }
1275
1276 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1277 {
1278 int i, rc, offset = 0;
1279
1280 rc = request_irq(bp->msix_table[offset++].vector,
1281 bnx2x_msix_sp_int, 0,
1282 bp->dev->name, bp->dev);
1283 if (rc) {
1284 BNX2X_ERR("request sp irq failed\n");
1285 return -EBUSY;
1286 }
1287
1288 #ifdef BCM_CNIC
1289 offset++;
1290 #endif
1291 for_each_eth_queue(bp, i) {
1292 struct bnx2x_fastpath *fp = &bp->fp[i];
1293 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1294 bp->dev->name, i);
1295
1296 rc = request_irq(bp->msix_table[offset].vector,
1297 bnx2x_msix_fp_int, 0, fp->name, fp);
1298 if (rc) {
1299 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1300 bp->msix_table[offset].vector, rc);
1301 bnx2x_free_msix_irqs(bp, offset);
1302 return -EBUSY;
1303 }
1304
1305 offset++;
1306 }
1307
1308 i = BNX2X_NUM_ETH_QUEUES(bp);
1309 offset = 1 + CNIC_PRESENT;
1310 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d"
1311 " ... fp[%d] %d\n",
1312 bp->msix_table[0].vector,
1313 0, bp->msix_table[offset].vector,
1314 i - 1, bp->msix_table[offset + i - 1].vector);
1315
1316 return 0;
1317 }
1318
1319 int bnx2x_enable_msi(struct bnx2x *bp)
1320 {
1321 int rc;
1322
1323 rc = pci_enable_msi(bp->pdev);
1324 if (rc) {
1325 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1326 return -1;
1327 }
1328 bp->flags |= USING_MSI_FLAG;
1329
1330 return 0;
1331 }
1332
1333 static int bnx2x_req_irq(struct bnx2x *bp)
1334 {
1335 unsigned long flags;
1336 int rc;
1337
1338 if (bp->flags & USING_MSI_FLAG)
1339 flags = 0;
1340 else
1341 flags = IRQF_SHARED;
1342
1343 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1344 bp->dev->name, bp->dev);
1345 return rc;
1346 }
1347
1348 static inline int bnx2x_setup_irqs(struct bnx2x *bp)
1349 {
1350 int rc = 0;
1351 if (bp->flags & USING_MSIX_FLAG) {
1352 rc = bnx2x_req_msix_irqs(bp);
1353 if (rc)
1354 return rc;
1355 } else {
1356 bnx2x_ack_int(bp);
1357 rc = bnx2x_req_irq(bp);
1358 if (rc) {
1359 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1360 return rc;
1361 }
1362 if (bp->flags & USING_MSI_FLAG) {
1363 bp->dev->irq = bp->pdev->irq;
1364 netdev_info(bp->dev, "using MSI IRQ %d\n",
1365 bp->pdev->irq);
1366 }
1367 }
1368
1369 return 0;
1370 }
1371
1372 static inline void bnx2x_napi_enable(struct bnx2x *bp)
1373 {
1374 int i;
1375
1376 for_each_rx_queue(bp, i)
1377 napi_enable(&bnx2x_fp(bp, i, napi));
1378 }
1379
1380 static inline void bnx2x_napi_disable(struct bnx2x *bp)
1381 {
1382 int i;
1383
1384 for_each_rx_queue(bp, i)
1385 napi_disable(&bnx2x_fp(bp, i, napi));
1386 }
1387
1388 void bnx2x_netif_start(struct bnx2x *bp)
1389 {
1390 if (netif_running(bp->dev)) {
1391 bnx2x_napi_enable(bp);
1392 bnx2x_int_enable(bp);
1393 if (bp->state == BNX2X_STATE_OPEN)
1394 netif_tx_wake_all_queues(bp->dev);
1395 }
1396 }
1397
1398 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1399 {
1400 bnx2x_int_disable_sync(bp, disable_hw);
1401 bnx2x_napi_disable(bp);
1402 }
1403
1404 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1405 {
1406 struct bnx2x *bp = netdev_priv(dev);
1407 #ifdef BCM_CNIC
1408 if (NO_FCOE(bp))
1409 return skb_tx_hash(dev, skb);
1410 else {
1411 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1412 u16 ether_type = ntohs(hdr->h_proto);
1413
1414 /* Skip VLAN tag if present */
1415 if (ether_type == ETH_P_8021Q) {
1416 struct vlan_ethhdr *vhdr =
1417 (struct vlan_ethhdr *)skb->data;
1418
1419 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1420 }
1421
1422 /* If ethertype is FCoE or FIP - use FCoE ring */
1423 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1424 return bnx2x_fcoe_tx(bp, txq_index);
1425 }
1426 #endif
1427 /* Select a none-FCoE queue: if FCoE is enabled, exclude FCoE L2 ring
1428 */
1429 return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1430 }
1431
1432 void bnx2x_set_num_queues(struct bnx2x *bp)
1433 {
1434 switch (bp->multi_mode) {
1435 case ETH_RSS_MODE_DISABLED:
1436 bp->num_queues = 1;
1437 break;
1438 case ETH_RSS_MODE_REGULAR:
1439 bp->num_queues = bnx2x_calc_num_queues(bp);
1440 break;
1441
1442 default:
1443 bp->num_queues = 1;
1444 break;
1445 }
1446
1447 /* Add special queues */
1448 bp->num_queues += NON_ETH_CONTEXT_USE;
1449 }
1450
1451 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1452 {
1453 int rc, tx, rx;
1454
1455 tx = MAX_TXQS_PER_COS * bp->max_cos;
1456 rx = BNX2X_NUM_ETH_QUEUES(bp);
1457
1458 /* account for fcoe queue */
1459 #ifdef BCM_CNIC
1460 if (!NO_FCOE(bp)) {
1461 rx += FCOE_PRESENT;
1462 tx += FCOE_PRESENT;
1463 }
1464 #endif
1465
1466 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1467 if (rc) {
1468 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1469 return rc;
1470 }
1471 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1472 if (rc) {
1473 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1474 return rc;
1475 }
1476
1477 DP(NETIF_MSG_DRV, "Setting real num queues to (tx, rx) (%d, %d)\n",
1478 tx, rx);
1479
1480 return rc;
1481 }
1482
1483 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1484 {
1485 int i;
1486
1487 for_each_queue(bp, i) {
1488 struct bnx2x_fastpath *fp = &bp->fp[i];
1489
1490 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1491 if (IS_FCOE_IDX(i))
1492 /*
1493 * Although there are no IP frames expected to arrive to
1494 * this ring we still want to add an
1495 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1496 * overrun attack.
1497 */
1498 fp->rx_buf_size =
1499 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
1500 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1501 else
1502 fp->rx_buf_size =
1503 bp->dev->mtu + ETH_OVREHEAD +
1504 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1505 }
1506 }
1507
1508 static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
1509 {
1510 int i;
1511 u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
1512 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1513
1514 /*
1515 * Prepare the inital contents fo the indirection table if RSS is
1516 * enabled
1517 */
1518 if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1519 for (i = 0; i < sizeof(ind_table); i++)
1520 ind_table[i] =
1521 bp->fp->cl_id + (i % num_eth_queues);
1522 }
1523
1524 /*
1525 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1526 * per-port, so if explicit configuration is needed , do it only
1527 * for a PMF.
1528 *
1529 * For 57712 and newer on the other hand it's a per-function
1530 * configuration.
1531 */
1532 return bnx2x_config_rss_pf(bp, ind_table,
1533 bp->port.pmf || !CHIP_IS_E1x(bp));
1534 }
1535
1536 int bnx2x_config_rss_pf(struct bnx2x *bp, u8 *ind_table, bool config_hash)
1537 {
1538 struct bnx2x_config_rss_params params = {0};
1539 int i;
1540
1541 /* Although RSS is meaningless when there is a single HW queue we
1542 * still need it enabled in order to have HW Rx hash generated.
1543 *
1544 * if (!is_eth_multi(bp))
1545 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1546 */
1547
1548 params.rss_obj = &bp->rss_conf_obj;
1549
1550 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1551
1552 /* RSS mode */
1553 switch (bp->multi_mode) {
1554 case ETH_RSS_MODE_DISABLED:
1555 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
1556 break;
1557 case ETH_RSS_MODE_REGULAR:
1558 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1559 break;
1560 case ETH_RSS_MODE_VLAN_PRI:
1561 __set_bit(BNX2X_RSS_MODE_VLAN_PRI, &params.rss_flags);
1562 break;
1563 case ETH_RSS_MODE_E1HOV_PRI:
1564 __set_bit(BNX2X_RSS_MODE_E1HOV_PRI, &params.rss_flags);
1565 break;
1566 case ETH_RSS_MODE_IP_DSCP:
1567 __set_bit(BNX2X_RSS_MODE_IP_DSCP, &params.rss_flags);
1568 break;
1569 default:
1570 BNX2X_ERR("Unknown multi_mode: %d\n", bp->multi_mode);
1571 return -EINVAL;
1572 }
1573
1574 /* If RSS is enabled */
1575 if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1576 /* RSS configuration */
1577 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1578 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1579 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1580 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1581
1582 /* Hash bits */
1583 params.rss_result_mask = MULTI_MASK;
1584
1585 memcpy(params.ind_table, ind_table, sizeof(params.ind_table));
1586
1587 if (config_hash) {
1588 /* RSS keys */
1589 for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1590 params.rss_key[i] = random32();
1591
1592 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1593 }
1594 }
1595
1596 return bnx2x_config_rss(bp, &params);
1597 }
1598
1599 static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1600 {
1601 struct bnx2x_func_state_params func_params = {0};
1602
1603 /* Prepare parameters for function state transitions */
1604 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1605
1606 func_params.f_obj = &bp->func_obj;
1607 func_params.cmd = BNX2X_F_CMD_HW_INIT;
1608
1609 func_params.params.hw_init.load_phase = load_code;
1610
1611 return bnx2x_func_state_change(bp, &func_params);
1612 }
1613
1614 /*
1615 * Cleans the object that have internal lists without sending
1616 * ramrods. Should be run when interrutps are disabled.
1617 */
1618 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1619 {
1620 int rc;
1621 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1622 struct bnx2x_mcast_ramrod_params rparam = {0};
1623 struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;
1624
1625 /***************** Cleanup MACs' object first *************************/
1626
1627 /* Wait for completion of requested */
1628 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1629 /* Perform a dry cleanup */
1630 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1631
1632 /* Clean ETH primary MAC */
1633 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1634 rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
1635 &ramrod_flags);
1636 if (rc != 0)
1637 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1638
1639 /* Cleanup UC list */
1640 vlan_mac_flags = 0;
1641 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1642 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1643 &ramrod_flags);
1644 if (rc != 0)
1645 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1646
1647 /***************** Now clean mcast object *****************************/
1648 rparam.mcast_obj = &bp->mcast_obj;
1649 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1650
1651 /* Add a DEL command... */
1652 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1653 if (rc < 0)
1654 BNX2X_ERR("Failed to add a new DEL command to a multi-cast "
1655 "object: %d\n", rc);
1656
1657 /* ...and wait until all pending commands are cleared */
1658 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1659 while (rc != 0) {
1660 if (rc < 0) {
1661 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1662 rc);
1663 return;
1664 }
1665
1666 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1667 }
1668 }
1669
1670 #ifndef BNX2X_STOP_ON_ERROR
1671 #define LOAD_ERROR_EXIT(bp, label) \
1672 do { \
1673 (bp)->state = BNX2X_STATE_ERROR; \
1674 goto label; \
1675 } while (0)
1676 #else
1677 #define LOAD_ERROR_EXIT(bp, label) \
1678 do { \
1679 (bp)->state = BNX2X_STATE_ERROR; \
1680 (bp)->panic = 1; \
1681 return -EBUSY; \
1682 } while (0)
1683 #endif
1684
1685 /* must be called with rtnl_lock */
1686 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1687 {
1688 int port = BP_PORT(bp);
1689 u32 load_code;
1690 int i, rc;
1691
1692 #ifdef BNX2X_STOP_ON_ERROR
1693 if (unlikely(bp->panic))
1694 return -EPERM;
1695 #endif
1696
1697 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1698
1699 /* Set the initial link reported state to link down */
1700 bnx2x_acquire_phy_lock(bp);
1701 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1702 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1703 &bp->last_reported_link.link_report_flags);
1704 bnx2x_release_phy_lock(bp);
1705
1706 /* must be called before memory allocation and HW init */
1707 bnx2x_ilt_set_info(bp);
1708
1709 /*
1710 * Zero fastpath structures preserving invariants like napi, which are
1711 * allocated only once, fp index, max_cos, bp pointer.
1712 * Also set fp->disable_tpa.
1713 */
1714 for_each_queue(bp, i)
1715 bnx2x_bz_fp(bp, i);
1716
1717
1718 /* Set the receive queues buffer size */
1719 bnx2x_set_rx_buf_size(bp);
1720
1721 if (bnx2x_alloc_mem(bp))
1722 return -ENOMEM;
1723
1724 /* As long as bnx2x_alloc_mem() may possibly update
1725 * bp->num_queues, bnx2x_set_real_num_queues() should always
1726 * come after it.
1727 */
1728 rc = bnx2x_set_real_num_queues(bp);
1729 if (rc) {
1730 BNX2X_ERR("Unable to set real_num_queues\n");
1731 LOAD_ERROR_EXIT(bp, load_error0);
1732 }
1733
1734 /* configure multi cos mappings in kernel.
1735 * this configuration may be overriden by a multi class queue discipline
1736 * or by a dcbx negotiation result.
1737 */
1738 bnx2x_setup_tc(bp->dev, bp->max_cos);
1739
1740 bnx2x_napi_enable(bp);
1741
1742 /* Send LOAD_REQUEST command to MCP
1743 * Returns the type of LOAD command:
1744 * if it is the first port to be initialized
1745 * common blocks should be initialized, otherwise - not
1746 */
1747 if (!BP_NOMCP(bp)) {
1748 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1749 if (!load_code) {
1750 BNX2X_ERR("MCP response failure, aborting\n");
1751 rc = -EBUSY;
1752 LOAD_ERROR_EXIT(bp, load_error1);
1753 }
1754 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1755 rc = -EBUSY; /* other port in diagnostic mode */
1756 LOAD_ERROR_EXIT(bp, load_error1);
1757 }
1758
1759 } else {
1760 int path = BP_PATH(bp);
1761
1762 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
1763 path, load_count[path][0], load_count[path][1],
1764 load_count[path][2]);
1765 load_count[path][0]++;
1766 load_count[path][1 + port]++;
1767 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
1768 path, load_count[path][0], load_count[path][1],
1769 load_count[path][2]);
1770 if (load_count[path][0] == 1)
1771 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1772 else if (load_count[path][1 + port] == 1)
1773 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1774 else
1775 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1776 }
1777
1778 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1779 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1780 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
1781 bp->port.pmf = 1;
1782 /*
1783 * We need the barrier to ensure the ordering between the
1784 * writing to bp->port.pmf here and reading it from the
1785 * bnx2x_periodic_task().
1786 */
1787 smp_mb();
1788 queue_delayed_work(bnx2x_wq, &bp->period_task, 0);
1789 } else
1790 bp->port.pmf = 0;
1791
1792 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1793
1794 /* Init Function state controlling object */
1795 bnx2x__init_func_obj(bp);
1796
1797 /* Initialize HW */
1798 rc = bnx2x_init_hw(bp, load_code);
1799 if (rc) {
1800 BNX2X_ERR("HW init failed, aborting\n");
1801 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1802 LOAD_ERROR_EXIT(bp, load_error2);
1803 }
1804
1805 /* Connect to IRQs */
1806 rc = bnx2x_setup_irqs(bp);
1807 if (rc) {
1808 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1809 LOAD_ERROR_EXIT(bp, load_error2);
1810 }
1811
1812 /* Setup NIC internals and enable interrupts */
1813 bnx2x_nic_init(bp, load_code);
1814
1815 /* Init per-function objects */
1816 bnx2x_init_bp_objs(bp);
1817
1818 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1819 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1820 (bp->common.shmem2_base)) {
1821 if (SHMEM2_HAS(bp, dcc_support))
1822 SHMEM2_WR(bp, dcc_support,
1823 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1824 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1825 }
1826
1827 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1828 rc = bnx2x_func_start(bp);
1829 if (rc) {
1830 BNX2X_ERR("Function start failed!\n");
1831 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1832 LOAD_ERROR_EXIT(bp, load_error3);
1833 }
1834
1835 /* Send LOAD_DONE command to MCP */
1836 if (!BP_NOMCP(bp)) {
1837 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1838 if (!load_code) {
1839 BNX2X_ERR("MCP response failure, aborting\n");
1840 rc = -EBUSY;
1841 LOAD_ERROR_EXIT(bp, load_error3);
1842 }
1843 }
1844
1845 rc = bnx2x_setup_leading(bp);
1846 if (rc) {
1847 BNX2X_ERR("Setup leading failed!\n");
1848 LOAD_ERROR_EXIT(bp, load_error3);
1849 }
1850
1851 #ifdef BCM_CNIC
1852 /* Enable Timer scan */
1853 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
1854 #endif
1855
1856 for_each_nondefault_queue(bp, i) {
1857 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
1858 if (rc)
1859 LOAD_ERROR_EXIT(bp, load_error4);
1860 }
1861
1862 rc = bnx2x_init_rss_pf(bp);
1863 if (rc)
1864 LOAD_ERROR_EXIT(bp, load_error4);
1865
1866 /* Now when Clients are configured we are ready to work */
1867 bp->state = BNX2X_STATE_OPEN;
1868
1869 /* Configure a ucast MAC */
1870 rc = bnx2x_set_eth_mac(bp, true);
1871 if (rc)
1872 LOAD_ERROR_EXIT(bp, load_error4);
1873
1874 if (bp->pending_max) {
1875 bnx2x_update_max_mf_config(bp, bp->pending_max);
1876 bp->pending_max = 0;
1877 }
1878
1879 if (bp->port.pmf)
1880 bnx2x_initial_phy_init(bp, load_mode);
1881
1882 /* Start fast path */
1883
1884 /* Initialize Rx filter. */
1885 netif_addr_lock_bh(bp->dev);
1886 bnx2x_set_rx_mode(bp->dev);
1887 netif_addr_unlock_bh(bp->dev);
1888
1889 /* Start the Tx */
1890 switch (load_mode) {
1891 case LOAD_NORMAL:
1892 /* Tx queue should be only reenabled */
1893 netif_tx_wake_all_queues(bp->dev);
1894 break;
1895
1896 case LOAD_OPEN:
1897 netif_tx_start_all_queues(bp->dev);
1898 smp_mb__after_clear_bit();
1899 break;
1900
1901 case LOAD_DIAG:
1902 bp->state = BNX2X_STATE_DIAG;
1903 break;
1904
1905 default:
1906 break;
1907 }
1908
1909 if (!bp->port.pmf)
1910 bnx2x__link_status_update(bp);
1911
1912 /* start the timer */
1913 mod_timer(&bp->timer, jiffies + bp->current_interval);
1914
1915 #ifdef BCM_CNIC
1916 bnx2x_setup_cnic_irq_info(bp);
1917 if (bp->state == BNX2X_STATE_OPEN)
1918 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1919 #endif
1920 bnx2x_inc_load_cnt(bp);
1921
1922 /* Wait for all pending SP commands to complete */
1923 if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
1924 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
1925 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
1926 return -EBUSY;
1927 }
1928
1929 bnx2x_dcbx_init(bp);
1930 return 0;
1931
1932 #ifndef BNX2X_STOP_ON_ERROR
1933 load_error4:
1934 #ifdef BCM_CNIC
1935 /* Disable Timer scan */
1936 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
1937 #endif
1938 load_error3:
1939 bnx2x_int_disable_sync(bp, 1);
1940
1941 /* Clean queueable objects */
1942 bnx2x_squeeze_objects(bp);
1943
1944 /* Free SKBs, SGEs, TPA pool and driver internals */
1945 bnx2x_free_skbs(bp);
1946 for_each_rx_queue(bp, i)
1947 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1948
1949 /* Release IRQs */
1950 bnx2x_free_irq(bp);
1951 load_error2:
1952 if (!BP_NOMCP(bp)) {
1953 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1954 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1955 }
1956
1957 bp->port.pmf = 0;
1958 load_error1:
1959 bnx2x_napi_disable(bp);
1960 load_error0:
1961 bnx2x_free_mem(bp);
1962
1963 return rc;
1964 #endif /* ! BNX2X_STOP_ON_ERROR */
1965 }
1966
1967 /* must be called with rtnl_lock */
1968 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1969 {
1970 int i;
1971 bool global = false;
1972
1973 if ((bp->state == BNX2X_STATE_CLOSED) ||
1974 (bp->state == BNX2X_STATE_ERROR)) {
1975 /* We can get here if the driver has been unloaded
1976 * during parity error recovery and is either waiting for a
1977 * leader to complete or for other functions to unload and
1978 * then ifdown has been issued. In this case we want to
1979 * unload and let other functions to complete a recovery
1980 * process.
1981 */
1982 bp->recovery_state = BNX2X_RECOVERY_DONE;
1983 bp->is_leader = 0;
1984 bnx2x_release_leader_lock(bp);
1985 smp_mb();
1986
1987 DP(NETIF_MSG_HW, "Releasing a leadership...\n");
1988
1989 return -EINVAL;
1990 }
1991
1992 /* Stop Tx */
1993 bnx2x_tx_disable(bp);
1994
1995 #ifdef BCM_CNIC
1996 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1997 #endif
1998 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1999 smp_mb();
2000
2001 bp->rx_mode = BNX2X_RX_MODE_NONE;
2002
2003 del_timer_sync(&bp->timer);
2004
2005 /* Set ALWAYS_ALIVE bit in shmem */
2006 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2007
2008 bnx2x_drv_pulse(bp);
2009
2010 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2011
2012 /* Cleanup the chip if needed */
2013 if (unload_mode != UNLOAD_RECOVERY)
2014 bnx2x_chip_cleanup(bp, unload_mode);
2015 else {
2016 /* Send the UNLOAD_REQUEST to the MCP */
2017 bnx2x_send_unload_req(bp, unload_mode);
2018
2019 /*
2020 * Prevent transactions to host from the functions on the
2021 * engine that doesn't reset global blocks in case of global
2022 * attention once gloabl blocks are reset and gates are opened
2023 * (the engine which leader will perform the recovery
2024 * last).
2025 */
2026 if (!CHIP_IS_E1x(bp))
2027 bnx2x_pf_disable(bp);
2028
2029 /* Disable HW interrupts, NAPI */
2030 bnx2x_netif_stop(bp, 1);
2031
2032 /* Release IRQs */
2033 bnx2x_free_irq(bp);
2034
2035 /* Report UNLOAD_DONE to MCP */
2036 bnx2x_send_unload_done(bp);
2037 }
2038
2039 /*
2040 * At this stage no more interrupts will arrive so we may safly clean
2041 * the queueable objects here in case they failed to get cleaned so far.
2042 */
2043 bnx2x_squeeze_objects(bp);
2044
2045 /* There should be no more pending SP commands at this stage */
2046 bp->sp_state = 0;
2047
2048 bp->port.pmf = 0;
2049
2050 /* Free SKBs, SGEs, TPA pool and driver internals */
2051 bnx2x_free_skbs(bp);
2052 for_each_rx_queue(bp, i)
2053 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2054
2055 bnx2x_free_mem(bp);
2056
2057 bp->state = BNX2X_STATE_CLOSED;
2058
2059 /* Check if there are pending parity attentions. If there are - set
2060 * RECOVERY_IN_PROGRESS.
2061 */
2062 if (bnx2x_chk_parity_attn(bp, &global, false)) {
2063 bnx2x_set_reset_in_progress(bp);
2064
2065 /* Set RESET_IS_GLOBAL if needed */
2066 if (global)
2067 bnx2x_set_reset_global(bp);
2068 }
2069
2070
2071 /* The last driver must disable a "close the gate" if there is no
2072 * parity attention or "process kill" pending.
2073 */
2074 if (!bnx2x_dec_load_cnt(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2075 bnx2x_disable_close_the_gate(bp);
2076
2077 return 0;
2078 }
2079
2080 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2081 {
2082 u16 pmcsr;
2083
2084 /* If there is no power capability, silently succeed */
2085 if (!bp->pm_cap) {
2086 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
2087 return 0;
2088 }
2089
2090 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2091
2092 switch (state) {
2093 case PCI_D0:
2094 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2095 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2096 PCI_PM_CTRL_PME_STATUS));
2097
2098 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2099 /* delay required during transition out of D3hot */
2100 msleep(20);
2101 break;
2102
2103 case PCI_D3hot:
2104 /* If there are other clients above don't
2105 shut down the power */
2106 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2107 return 0;
2108 /* Don't shut down the power for emulation and FPGA */
2109 if (CHIP_REV_IS_SLOW(bp))
2110 return 0;
2111
2112 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2113 pmcsr |= 3;
2114
2115 if (bp->wol)
2116 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2117
2118 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2119 pmcsr);
2120
2121 /* No more memory access after this point until
2122 * device is brought back to D0.
2123 */
2124 break;
2125
2126 default:
2127 return -EINVAL;
2128 }
2129 return 0;
2130 }
2131
2132 /*
2133 * net_device service functions
2134 */
2135 int bnx2x_poll(struct napi_struct *napi, int budget)
2136 {
2137 int work_done = 0;
2138 u8 cos;
2139 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2140 napi);
2141 struct bnx2x *bp = fp->bp;
2142
2143 while (1) {
2144 #ifdef BNX2X_STOP_ON_ERROR
2145 if (unlikely(bp->panic)) {
2146 napi_complete(napi);
2147 return 0;
2148 }
2149 #endif
2150
2151 for_each_cos_in_tx_queue(fp, cos)
2152 if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
2153 bnx2x_tx_int(bp, &fp->txdata[cos]);
2154
2155
2156 if (bnx2x_has_rx_work(fp)) {
2157 work_done += bnx2x_rx_int(fp, budget - work_done);
2158
2159 /* must not complete if we consumed full budget */
2160 if (work_done >= budget)
2161 break;
2162 }
2163
2164 /* Fall out from the NAPI loop if needed */
2165 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2166 #ifdef BCM_CNIC
2167 /* No need to update SB for FCoE L2 ring as long as
2168 * it's connected to the default SB and the SB
2169 * has been updated when NAPI was scheduled.
2170 */
2171 if (IS_FCOE_FP(fp)) {
2172 napi_complete(napi);
2173 break;
2174 }
2175 #endif
2176
2177 bnx2x_update_fpsb_idx(fp);
2178 /* bnx2x_has_rx_work() reads the status block,
2179 * thus we need to ensure that status block indices
2180 * have been actually read (bnx2x_update_fpsb_idx)
2181 * prior to this check (bnx2x_has_rx_work) so that
2182 * we won't write the "newer" value of the status block
2183 * to IGU (if there was a DMA right after
2184 * bnx2x_has_rx_work and if there is no rmb, the memory
2185 * reading (bnx2x_update_fpsb_idx) may be postponed
2186 * to right before bnx2x_ack_sb). In this case there
2187 * will never be another interrupt until there is
2188 * another update of the status block, while there
2189 * is still unhandled work.
2190 */
2191 rmb();
2192
2193 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2194 napi_complete(napi);
2195 /* Re-enable interrupts */
2196 DP(NETIF_MSG_HW,
2197 "Update index to %d\n", fp->fp_hc_idx);
2198 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2199 le16_to_cpu(fp->fp_hc_idx),
2200 IGU_INT_ENABLE, 1);
2201 break;
2202 }
2203 }
2204 }
2205
2206 return work_done;
2207 }
2208
2209 /* we split the first BD into headers and data BDs
2210 * to ease the pain of our fellow microcode engineers
2211 * we use one mapping for both BDs
2212 * So far this has only been observed to happen
2213 * in Other Operating Systems(TM)
2214 */
2215 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2216 struct bnx2x_fp_txdata *txdata,
2217 struct sw_tx_bd *tx_buf,
2218 struct eth_tx_start_bd **tx_bd, u16 hlen,
2219 u16 bd_prod, int nbd)
2220 {
2221 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2222 struct eth_tx_bd *d_tx_bd;
2223 dma_addr_t mapping;
2224 int old_len = le16_to_cpu(h_tx_bd->nbytes);
2225
2226 /* first fix first BD */
2227 h_tx_bd->nbd = cpu_to_le16(nbd);
2228 h_tx_bd->nbytes = cpu_to_le16(hlen);
2229
2230 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
2231 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
2232 h_tx_bd->addr_lo, h_tx_bd->nbd);
2233
2234 /* now get a new data BD
2235 * (after the pbd) and fill it */
2236 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2237 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2238
2239 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2240 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2241
2242 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2243 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2244 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2245
2246 /* this marks the BD as one that has no individual mapping */
2247 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2248
2249 DP(NETIF_MSG_TX_QUEUED,
2250 "TSO split data size is %d (%x:%x)\n",
2251 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2252
2253 /* update tx_bd */
2254 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2255
2256 return bd_prod;
2257 }
2258
2259 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2260 {
2261 if (fix > 0)
2262 csum = (u16) ~csum_fold(csum_sub(csum,
2263 csum_partial(t_header - fix, fix, 0)));
2264
2265 else if (fix < 0)
2266 csum = (u16) ~csum_fold(csum_add(csum,
2267 csum_partial(t_header, -fix, 0)));
2268
2269 return swab16(csum);
2270 }
2271
2272 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2273 {
2274 u32 rc;
2275
2276 if (skb->ip_summed != CHECKSUM_PARTIAL)
2277 rc = XMIT_PLAIN;
2278
2279 else {
2280 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2281 rc = XMIT_CSUM_V6;
2282 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2283 rc |= XMIT_CSUM_TCP;
2284
2285 } else {
2286 rc = XMIT_CSUM_V4;
2287 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2288 rc |= XMIT_CSUM_TCP;
2289 }
2290 }
2291
2292 if (skb_is_gso_v6(skb))
2293 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2294 else if (skb_is_gso(skb))
2295 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2296
2297 return rc;
2298 }
2299
2300 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2301 /* check if packet requires linearization (packet is too fragmented)
2302 no need to check fragmentation if page size > 8K (there will be no
2303 violation to FW restrictions) */
2304 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2305 u32 xmit_type)
2306 {
2307 int to_copy = 0;
2308 int hlen = 0;
2309 int first_bd_sz = 0;
2310
2311 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2312 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2313
2314 if (xmit_type & XMIT_GSO) {
2315 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2316 /* Check if LSO packet needs to be copied:
2317 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2318 int wnd_size = MAX_FETCH_BD - 3;
2319 /* Number of windows to check */
2320 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2321 int wnd_idx = 0;
2322 int frag_idx = 0;
2323 u32 wnd_sum = 0;
2324
2325 /* Headers length */
2326 hlen = (int)(skb_transport_header(skb) - skb->data) +
2327 tcp_hdrlen(skb);
2328
2329 /* Amount of data (w/o headers) on linear part of SKB*/
2330 first_bd_sz = skb_headlen(skb) - hlen;
2331
2332 wnd_sum = first_bd_sz;
2333
2334 /* Calculate the first sum - it's special */
2335 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2336 wnd_sum +=
2337 skb_shinfo(skb)->frags[frag_idx].size;
2338
2339 /* If there was data on linear skb data - check it */
2340 if (first_bd_sz > 0) {
2341 if (unlikely(wnd_sum < lso_mss)) {
2342 to_copy = 1;
2343 goto exit_lbl;
2344 }
2345
2346 wnd_sum -= first_bd_sz;
2347 }
2348
2349 /* Others are easier: run through the frag list and
2350 check all windows */
2351 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2352 wnd_sum +=
2353 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
2354
2355 if (unlikely(wnd_sum < lso_mss)) {
2356 to_copy = 1;
2357 break;
2358 }
2359 wnd_sum -=
2360 skb_shinfo(skb)->frags[wnd_idx].size;
2361 }
2362 } else {
2363 /* in non-LSO too fragmented packet should always
2364 be linearized */
2365 to_copy = 1;
2366 }
2367 }
2368
2369 exit_lbl:
2370 if (unlikely(to_copy))
2371 DP(NETIF_MSG_TX_QUEUED,
2372 "Linearization IS REQUIRED for %s packet. "
2373 "num_frags %d hlen %d first_bd_sz %d\n",
2374 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2375 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2376
2377 return to_copy;
2378 }
2379 #endif
2380
2381 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2382 u32 xmit_type)
2383 {
2384 *parsing_data |= (skb_shinfo(skb)->gso_size <<
2385 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2386 ETH_TX_PARSE_BD_E2_LSO_MSS;
2387 if ((xmit_type & XMIT_GSO_V6) &&
2388 (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2389 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2390 }
2391
2392 /**
2393 * bnx2x_set_pbd_gso - update PBD in GSO case.
2394 *
2395 * @skb: packet skb
2396 * @pbd: parse BD
2397 * @xmit_type: xmit flags
2398 */
2399 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2400 struct eth_tx_parse_bd_e1x *pbd,
2401 u32 xmit_type)
2402 {
2403 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2404 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2405 pbd->tcp_flags = pbd_tcp_flags(skb);
2406
2407 if (xmit_type & XMIT_GSO_V4) {
2408 pbd->ip_id = swab16(ip_hdr(skb)->id);
2409 pbd->tcp_pseudo_csum =
2410 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2411 ip_hdr(skb)->daddr,
2412 0, IPPROTO_TCP, 0));
2413
2414 } else
2415 pbd->tcp_pseudo_csum =
2416 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2417 &ipv6_hdr(skb)->daddr,
2418 0, IPPROTO_TCP, 0));
2419
2420 pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2421 }
2422
2423 /**
2424 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2425 *
2426 * @bp: driver handle
2427 * @skb: packet skb
2428 * @parsing_data: data to be updated
2429 * @xmit_type: xmit flags
2430 *
2431 * 57712 related
2432 */
2433 static inline u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2434 u32 *parsing_data, u32 xmit_type)
2435 {
2436 *parsing_data |=
2437 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2438 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2439 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2440
2441 if (xmit_type & XMIT_CSUM_TCP) {
2442 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2443 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2444 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2445
2446 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2447 } else
2448 /* We support checksum offload for TCP and UDP only.
2449 * No need to pass the UDP header length - it's a constant.
2450 */
2451 return skb_transport_header(skb) +
2452 sizeof(struct udphdr) - skb->data;
2453 }
2454
2455 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2456 struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2457 {
2458 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2459
2460 if (xmit_type & XMIT_CSUM_V4)
2461 tx_start_bd->bd_flags.as_bitfield |=
2462 ETH_TX_BD_FLAGS_IP_CSUM;
2463 else
2464 tx_start_bd->bd_flags.as_bitfield |=
2465 ETH_TX_BD_FLAGS_IPV6;
2466
2467 if (!(xmit_type & XMIT_CSUM_TCP))
2468 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
2469 }
2470
2471 /**
2472 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2473 *
2474 * @bp: driver handle
2475 * @skb: packet skb
2476 * @pbd: parse BD to be updated
2477 * @xmit_type: xmit flags
2478 */
2479 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2480 struct eth_tx_parse_bd_e1x *pbd,
2481 u32 xmit_type)
2482 {
2483 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2484
2485 /* for now NS flag is not used in Linux */
2486 pbd->global_data =
2487 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2488 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2489
2490 pbd->ip_hlen_w = (skb_transport_header(skb) -
2491 skb_network_header(skb)) >> 1;
2492
2493 hlen += pbd->ip_hlen_w;
2494
2495 /* We support checksum offload for TCP and UDP only */
2496 if (xmit_type & XMIT_CSUM_TCP)
2497 hlen += tcp_hdrlen(skb) / 2;
2498 else
2499 hlen += sizeof(struct udphdr) / 2;
2500
2501 pbd->total_hlen_w = cpu_to_le16(hlen);
2502 hlen = hlen*2;
2503
2504 if (xmit_type & XMIT_CSUM_TCP) {
2505 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2506
2507 } else {
2508 s8 fix = SKB_CS_OFF(skb); /* signed! */
2509
2510 DP(NETIF_MSG_TX_QUEUED,
2511 "hlen %d fix %d csum before fix %x\n",
2512 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2513
2514 /* HW bug: fixup the CSUM */
2515 pbd->tcp_pseudo_csum =
2516 bnx2x_csum_fix(skb_transport_header(skb),
2517 SKB_CS(skb), fix);
2518
2519 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2520 pbd->tcp_pseudo_csum);
2521 }
2522
2523 return hlen;
2524 }
2525
2526 /* called with netif_tx_lock
2527 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2528 * netif_wake_queue()
2529 */
2530 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2531 {
2532 struct bnx2x *bp = netdev_priv(dev);
2533
2534 struct bnx2x_fastpath *fp;
2535 struct netdev_queue *txq;
2536 struct bnx2x_fp_txdata *txdata;
2537 struct sw_tx_bd *tx_buf;
2538 struct eth_tx_start_bd *tx_start_bd, *first_bd;
2539 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2540 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2541 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2542 u32 pbd_e2_parsing_data = 0;
2543 u16 pkt_prod, bd_prod;
2544 int nbd, txq_index, fp_index, txdata_index;
2545 dma_addr_t mapping;
2546 u32 xmit_type = bnx2x_xmit_type(bp, skb);
2547 int i;
2548 u8 hlen = 0;
2549 __le16 pkt_size = 0;
2550 struct ethhdr *eth;
2551 u8 mac_type = UNICAST_ADDRESS;
2552
2553 #ifdef BNX2X_STOP_ON_ERROR
2554 if (unlikely(bp->panic))
2555 return NETDEV_TX_BUSY;
2556 #endif
2557
2558 txq_index = skb_get_queue_mapping(skb);
2559 txq = netdev_get_tx_queue(dev, txq_index);
2560
2561 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2562
2563 /* decode the fastpath index and the cos index from the txq */
2564 fp_index = TXQ_TO_FP(txq_index);
2565 txdata_index = TXQ_TO_COS(txq_index);
2566
2567 #ifdef BCM_CNIC
2568 /*
2569 * Override the above for the FCoE queue:
2570 * - FCoE fp entry is right after the ETH entries.
2571 * - FCoE L2 queue uses bp->txdata[0] only.
2572 */
2573 if (unlikely(!NO_FCOE(bp) && (txq_index ==
2574 bnx2x_fcoe_tx(bp, txq_index)))) {
2575 fp_index = FCOE_IDX;
2576 txdata_index = 0;
2577 }
2578 #endif
2579
2580 /* enable this debug print to view the transmission queue being used
2581 DP(BNX2X_MSG_FP, "indices: txq %d, fp %d, txdata %d",
2582 txq_index, fp_index, txdata_index); */
2583
2584 /* locate the fastpath and the txdata */
2585 fp = &bp->fp[fp_index];
2586 txdata = &fp->txdata[txdata_index];
2587
2588 /* enable this debug print to view the tranmission details
2589 DP(BNX2X_MSG_FP,"transmitting packet cid %d fp index %d txdata_index %d"
2590 " tx_data ptr %p fp pointer %p",
2591 txdata->cid, fp_index, txdata_index, txdata, fp); */
2592
2593 if (unlikely(bnx2x_tx_avail(bp, txdata) <
2594 (skb_shinfo(skb)->nr_frags + 3))) {
2595 fp->eth_q_stats.driver_xoff++;
2596 netif_tx_stop_queue(txq);
2597 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2598 return NETDEV_TX_BUSY;
2599 }
2600
2601 DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x protocol %x "
2602 "protocol(%x,%x) gso type %x xmit_type %x\n",
2603 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2604 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2605
2606 eth = (struct ethhdr *)skb->data;
2607
2608 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2609 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2610 if (is_broadcast_ether_addr(eth->h_dest))
2611 mac_type = BROADCAST_ADDRESS;
2612 else
2613 mac_type = MULTICAST_ADDRESS;
2614 }
2615
2616 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2617 /* First, check if we need to linearize the skb (due to FW
2618 restrictions). No need to check fragmentation if page size > 8K
2619 (there will be no violation to FW restrictions) */
2620 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2621 /* Statistics of linearization */
2622 bp->lin_cnt++;
2623 if (skb_linearize(skb) != 0) {
2624 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2625 "silently dropping this SKB\n");
2626 dev_kfree_skb_any(skb);
2627 return NETDEV_TX_OK;
2628 }
2629 }
2630 #endif
2631 /* Map skb linear data for DMA */
2632 mapping = dma_map_single(&bp->pdev->dev, skb->data,
2633 skb_headlen(skb), DMA_TO_DEVICE);
2634 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2635 DP(NETIF_MSG_TX_QUEUED, "SKB mapping failed - "
2636 "silently dropping this SKB\n");
2637 dev_kfree_skb_any(skb);
2638 return NETDEV_TX_OK;
2639 }
2640 /*
2641 Please read carefully. First we use one BD which we mark as start,
2642 then we have a parsing info BD (used for TSO or xsum),
2643 and only then we have the rest of the TSO BDs.
2644 (don't forget to mark the last one as last,
2645 and to unmap only AFTER you write to the BD ...)
2646 And above all, all pdb sizes are in words - NOT DWORDS!
2647 */
2648
2649 /* get current pkt produced now - advance it just before sending packet
2650 * since mapping of pages may fail and cause packet to be dropped
2651 */
2652 pkt_prod = txdata->tx_pkt_prod;
2653 bd_prod = TX_BD(txdata->tx_bd_prod);
2654
2655 /* get a tx_buf and first BD
2656 * tx_start_bd may be changed during SPLIT,
2657 * but first_bd will always stay first
2658 */
2659 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
2660 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
2661 first_bd = tx_start_bd;
2662
2663 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2664 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2665 mac_type);
2666
2667 /* header nbd */
2668 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2669
2670 /* remember the first BD of the packet */
2671 tx_buf->first_bd = txdata->tx_bd_prod;
2672 tx_buf->skb = skb;
2673 tx_buf->flags = 0;
2674
2675 DP(NETIF_MSG_TX_QUEUED,
2676 "sending pkt %u @%p next_idx %u bd %u @%p\n",
2677 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
2678
2679 if (vlan_tx_tag_present(skb)) {
2680 tx_start_bd->vlan_or_ethertype =
2681 cpu_to_le16(vlan_tx_tag_get(skb));
2682 tx_start_bd->bd_flags.as_bitfield |=
2683 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2684 } else
2685 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2686
2687 /* turn on parsing and get a BD */
2688 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2689
2690 if (xmit_type & XMIT_CSUM)
2691 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
2692
2693 if (!CHIP_IS_E1x(bp)) {
2694 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
2695 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2696 /* Set PBD in checksum offload case */
2697 if (xmit_type & XMIT_CSUM)
2698 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2699 &pbd_e2_parsing_data,
2700 xmit_type);
2701 if (IS_MF_SI(bp)) {
2702 /*
2703 * fill in the MAC addresses in the PBD - for local
2704 * switching
2705 */
2706 bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
2707 &pbd_e2->src_mac_addr_mid,
2708 &pbd_e2->src_mac_addr_lo,
2709 eth->h_source);
2710 bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
2711 &pbd_e2->dst_mac_addr_mid,
2712 &pbd_e2->dst_mac_addr_lo,
2713 eth->h_dest);
2714 }
2715 } else {
2716 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
2717 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2718 /* Set PBD in checksum offload case */
2719 if (xmit_type & XMIT_CSUM)
2720 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2721
2722 }
2723
2724 /* Setup the data pointer of the first BD of the packet */
2725 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2726 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2727 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
2728 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2729 pkt_size = tx_start_bd->nbytes;
2730
2731 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d"
2732 " nbytes %d flags %x vlan %x\n",
2733 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2734 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2735 tx_start_bd->bd_flags.as_bitfield,
2736 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2737
2738 if (xmit_type & XMIT_GSO) {
2739
2740 DP(NETIF_MSG_TX_QUEUED,
2741 "TSO packet len %d hlen %d total len %d tso size %d\n",
2742 skb->len, hlen, skb_headlen(skb),
2743 skb_shinfo(skb)->gso_size);
2744
2745 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2746
2747 if (unlikely(skb_headlen(skb) > hlen))
2748 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
2749 &tx_start_bd, hlen,
2750 bd_prod, ++nbd);
2751 if (!CHIP_IS_E1x(bp))
2752 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2753 xmit_type);
2754 else
2755 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2756 }
2757
2758 /* Set the PBD's parsing_data field if not zero
2759 * (for the chips newer than 57711).
2760 */
2761 if (pbd_e2_parsing_data)
2762 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2763
2764 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2765
2766 /* Handle fragmented skb */
2767 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2768 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2769
2770 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2771 frag->page_offset, frag->size,
2772 DMA_TO_DEVICE);
2773 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2774
2775 DP(NETIF_MSG_TX_QUEUED, "Unable to map page - "
2776 "dropping packet...\n");
2777
2778 /* we need unmap all buffers already mapped
2779 * for this SKB;
2780 * first_bd->nbd need to be properly updated
2781 * before call to bnx2x_free_tx_pkt
2782 */
2783 first_bd->nbd = cpu_to_le16(nbd);
2784 bnx2x_free_tx_pkt(bp, txdata,
2785 TX_BD(txdata->tx_pkt_prod));
2786 return NETDEV_TX_OK;
2787 }
2788
2789 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2790 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2791 if (total_pkt_bd == NULL)
2792 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2793
2794 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2795 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2796 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2797 le16_add_cpu(&pkt_size, frag->size);
2798 nbd++;
2799
2800 DP(NETIF_MSG_TX_QUEUED,
2801 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2802 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2803 le16_to_cpu(tx_data_bd->nbytes));
2804 }
2805
2806 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2807
2808 /* update with actual num BDs */
2809 first_bd->nbd = cpu_to_le16(nbd);
2810
2811 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2812
2813 /* now send a tx doorbell, counting the next BD
2814 * if the packet contains or ends with it
2815 */
2816 if (TX_BD_POFF(bd_prod) < nbd)
2817 nbd++;
2818
2819 /* total_pkt_bytes should be set on the first data BD if
2820 * it's not an LSO packet and there is more than one
2821 * data BD. In this case pkt_size is limited by an MTU value.
2822 * However we prefer to set it for an LSO packet (while we don't
2823 * have to) in order to save some CPU cycles in a none-LSO
2824 * case, when we much more care about them.
2825 */
2826 if (total_pkt_bd != NULL)
2827 total_pkt_bd->total_pkt_bytes = pkt_size;
2828
2829 if (pbd_e1x)
2830 DP(NETIF_MSG_TX_QUEUED,
2831 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
2832 " tcp_flags %x xsum %x seq %u hlen %u\n",
2833 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2834 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2835 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2836 le16_to_cpu(pbd_e1x->total_hlen_w));
2837 if (pbd_e2)
2838 DP(NETIF_MSG_TX_QUEUED,
2839 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
2840 pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2841 pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2842 pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2843 pbd_e2->parsing_data);
2844 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
2845
2846 txdata->tx_pkt_prod++;
2847 /*
2848 * Make sure that the BD data is updated before updating the producer
2849 * since FW might read the BD right after the producer is updated.
2850 * This is only applicable for weak-ordered memory model archs such
2851 * as IA-64. The following barrier is also mandatory since FW will
2852 * assumes packets must have BDs.
2853 */
2854 wmb();
2855
2856 txdata->tx_db.data.prod += nbd;
2857 barrier();
2858
2859 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
2860
2861 mmiowb();
2862
2863 txdata->tx_bd_prod += nbd;
2864
2865 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
2866 netif_tx_stop_queue(txq);
2867
2868 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2869 * ordering of set_bit() in netif_tx_stop_queue() and read of
2870 * fp->bd_tx_cons */
2871 smp_mb();
2872
2873 fp->eth_q_stats.driver_xoff++;
2874 if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
2875 netif_tx_wake_queue(txq);
2876 }
2877 txdata->tx_pkt++;
2878
2879 return NETDEV_TX_OK;
2880 }
2881
2882 /**
2883 * bnx2x_setup_tc - routine to configure net_device for multi tc
2884 *
2885 * @netdev: net device to configure
2886 * @tc: number of traffic classes to enable
2887 *
2888 * callback connected to the ndo_setup_tc function pointer
2889 */
2890 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
2891 {
2892 int cos, prio, count, offset;
2893 struct bnx2x *bp = netdev_priv(dev);
2894
2895 /* setup tc must be called under rtnl lock */
2896 ASSERT_RTNL();
2897
2898 /* no traffic classes requested. aborting */
2899 if (!num_tc) {
2900 netdev_reset_tc(dev);
2901 return 0;
2902 }
2903
2904 /* requested to support too many traffic classes */
2905 if (num_tc > bp->max_cos) {
2906 DP(NETIF_MSG_TX_ERR, "support for too many traffic classes"
2907 " requested: %d. max supported is %d",
2908 num_tc, bp->max_cos);
2909 return -EINVAL;
2910 }
2911
2912 /* declare amount of supported traffic classes */
2913 if (netdev_set_num_tc(dev, num_tc)) {
2914 DP(NETIF_MSG_TX_ERR, "failed to declare %d traffic classes",
2915 num_tc);
2916 return -EINVAL;
2917 }
2918
2919 /* configure priority to traffic class mapping */
2920 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
2921 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
2922 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d",
2923 prio, bp->prio_to_cos[prio]);
2924 }
2925
2926
2927 /* Use this configuration to diffrentiate tc0 from other COSes
2928 This can be used for ets or pfc, and save the effort of setting
2929 up a multio class queue disc or negotiating DCBX with a switch
2930 netdev_set_prio_tc_map(dev, 0, 0);
2931 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d", 0, 0);
2932 for (prio = 1; prio < 16; prio++) {
2933 netdev_set_prio_tc_map(dev, prio, 1);
2934 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d", prio, 1);
2935 } */
2936
2937 /* configure traffic class to transmission queue mapping */
2938 for (cos = 0; cos < bp->max_cos; cos++) {
2939 count = BNX2X_NUM_ETH_QUEUES(bp);
2940 offset = cos * MAX_TXQS_PER_COS;
2941 netdev_set_tc_queue(dev, cos, count, offset);
2942 DP(BNX2X_MSG_SP, "mapping tc %d to offset %d count %d",
2943 cos, offset, count);
2944 }
2945
2946 return 0;
2947 }
2948
2949 /* called with rtnl_lock */
2950 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2951 {
2952 struct sockaddr *addr = p;
2953 struct bnx2x *bp = netdev_priv(dev);
2954 int rc = 0;
2955
2956 if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2957 return -EINVAL;
2958
2959 if (netif_running(dev)) {
2960 rc = bnx2x_set_eth_mac(bp, false);
2961 if (rc)
2962 return rc;
2963 }
2964
2965 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2966
2967 if (netif_running(dev))
2968 rc = bnx2x_set_eth_mac(bp, true);
2969
2970 return rc;
2971 }
2972
2973 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
2974 {
2975 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
2976 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
2977 u8 cos;
2978
2979 /* Common */
2980 #ifdef BCM_CNIC
2981 if (IS_FCOE_IDX(fp_index)) {
2982 memset(sb, 0, sizeof(union host_hc_status_block));
2983 fp->status_blk_mapping = 0;
2984
2985 } else {
2986 #endif
2987 /* status blocks */
2988 if (!CHIP_IS_E1x(bp))
2989 BNX2X_PCI_FREE(sb->e2_sb,
2990 bnx2x_fp(bp, fp_index,
2991 status_blk_mapping),
2992 sizeof(struct host_hc_status_block_e2));
2993 else
2994 BNX2X_PCI_FREE(sb->e1x_sb,
2995 bnx2x_fp(bp, fp_index,
2996 status_blk_mapping),
2997 sizeof(struct host_hc_status_block_e1x));
2998 #ifdef BCM_CNIC
2999 }
3000 #endif
3001 /* Rx */
3002 if (!skip_rx_queue(bp, fp_index)) {
3003 bnx2x_free_rx_bds(fp);
3004
3005 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3006 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3007 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3008 bnx2x_fp(bp, fp_index, rx_desc_mapping),
3009 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3010
3011 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3012 bnx2x_fp(bp, fp_index, rx_comp_mapping),
3013 sizeof(struct eth_fast_path_rx_cqe) *
3014 NUM_RCQ_BD);
3015
3016 /* SGE ring */
3017 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3018 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3019 bnx2x_fp(bp, fp_index, rx_sge_mapping),
3020 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3021 }
3022
3023 /* Tx */
3024 if (!skip_tx_queue(bp, fp_index)) {
3025 /* fastpath tx rings: tx_buf tx_desc */
3026 for_each_cos_in_tx_queue(fp, cos) {
3027 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3028
3029 DP(BNX2X_MSG_SP,
3030 "freeing tx memory of fp %d cos %d cid %d",
3031 fp_index, cos, txdata->cid);
3032
3033 BNX2X_FREE(txdata->tx_buf_ring);
3034 BNX2X_PCI_FREE(txdata->tx_desc_ring,
3035 txdata->tx_desc_mapping,
3036 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3037 }
3038 }
3039 /* end of fastpath */
3040 }
3041
3042 void bnx2x_free_fp_mem(struct bnx2x *bp)
3043 {
3044 int i;
3045 for_each_queue(bp, i)
3046 bnx2x_free_fp_mem_at(bp, i);
3047 }
3048
3049 static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
3050 {
3051 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3052 if (!CHIP_IS_E1x(bp)) {
3053 bnx2x_fp(bp, index, sb_index_values) =
3054 (__le16 *)status_blk.e2_sb->sb.index_values;
3055 bnx2x_fp(bp, index, sb_running_index) =
3056 (__le16 *)status_blk.e2_sb->sb.running_index;
3057 } else {
3058 bnx2x_fp(bp, index, sb_index_values) =
3059 (__le16 *)status_blk.e1x_sb->sb.index_values;
3060 bnx2x_fp(bp, index, sb_running_index) =
3061 (__le16 *)status_blk.e1x_sb->sb.running_index;
3062 }
3063 }
3064
3065 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3066 {
3067 union host_hc_status_block *sb;
3068 struct bnx2x_fastpath *fp = &bp->fp[index];
3069 int ring_size = 0;
3070 u8 cos;
3071
3072 /* if rx_ring_size specified - use it */
3073 int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
3074 MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3075
3076 /* allocate at least number of buffers required by FW */
3077 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3078 MIN_RX_SIZE_TPA,
3079 rx_ring_size);
3080
3081 /* Common */
3082 sb = &bnx2x_fp(bp, index, status_blk);
3083 #ifdef BCM_CNIC
3084 if (!IS_FCOE_IDX(index)) {
3085 #endif
3086 /* status blocks */
3087 if (!CHIP_IS_E1x(bp))
3088 BNX2X_PCI_ALLOC(sb->e2_sb,
3089 &bnx2x_fp(bp, index, status_blk_mapping),
3090 sizeof(struct host_hc_status_block_e2));
3091 else
3092 BNX2X_PCI_ALLOC(sb->e1x_sb,
3093 &bnx2x_fp(bp, index, status_blk_mapping),
3094 sizeof(struct host_hc_status_block_e1x));
3095 #ifdef BCM_CNIC
3096 }
3097 #endif
3098
3099 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3100 * set shortcuts for it.
3101 */
3102 if (!IS_FCOE_IDX(index))
3103 set_sb_shortcuts(bp, index);
3104
3105 /* Tx */
3106 if (!skip_tx_queue(bp, index)) {
3107 /* fastpath tx rings: tx_buf tx_desc */
3108 for_each_cos_in_tx_queue(fp, cos) {
3109 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3110
3111 DP(BNX2X_MSG_SP, "allocating tx memory of "
3112 "fp %d cos %d",
3113 index, cos);
3114
3115 BNX2X_ALLOC(txdata->tx_buf_ring,
3116 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3117 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3118 &txdata->tx_desc_mapping,
3119 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3120 }
3121 }
3122
3123 /* Rx */
3124 if (!skip_rx_queue(bp, index)) {
3125 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3126 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3127 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3128 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3129 &bnx2x_fp(bp, index, rx_desc_mapping),
3130 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3131
3132 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3133 &bnx2x_fp(bp, index, rx_comp_mapping),
3134 sizeof(struct eth_fast_path_rx_cqe) *
3135 NUM_RCQ_BD);
3136
3137 /* SGE ring */
3138 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3139 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3140 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3141 &bnx2x_fp(bp, index, rx_sge_mapping),
3142 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3143 /* RX BD ring */
3144 bnx2x_set_next_page_rx_bd(fp);
3145
3146 /* CQ ring */
3147 bnx2x_set_next_page_rx_cq(fp);
3148
3149 /* BDs */
3150 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3151 if (ring_size < rx_ring_size)
3152 goto alloc_mem_err;
3153 }
3154
3155 return 0;
3156
3157 /* handles low memory cases */
3158 alloc_mem_err:
3159 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3160 index, ring_size);
3161 /* FW will drop all packets if queue is not big enough,
3162 * In these cases we disable the queue
3163 * Min size is different for OOO, TPA and non-TPA queues
3164 */
3165 if (ring_size < (fp->disable_tpa ?
3166 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3167 /* release memory allocated for this queue */
3168 bnx2x_free_fp_mem_at(bp, index);
3169 return -ENOMEM;
3170 }
3171 return 0;
3172 }
3173
3174 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3175 {
3176 int i;
3177
3178 /**
3179 * 1. Allocate FP for leading - fatal if error
3180 * 2. {CNIC} Allocate FCoE FP - fatal if error
3181 * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3182 * 4. Allocate RSS - fix number of queues if error
3183 */
3184
3185 /* leading */
3186 if (bnx2x_alloc_fp_mem_at(bp, 0))
3187 return -ENOMEM;
3188
3189 #ifdef BCM_CNIC
3190 if (!NO_FCOE(bp))
3191 /* FCoE */
3192 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
3193 /* we will fail load process instead of mark
3194 * NO_FCOE_FLAG
3195 */
3196 return -ENOMEM;
3197 #endif
3198
3199 /* RSS */
3200 for_each_nondefault_eth_queue(bp, i)
3201 if (bnx2x_alloc_fp_mem_at(bp, i))
3202 break;
3203
3204 /* handle memory failures */
3205 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3206 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3207
3208 WARN_ON(delta < 0);
3209 #ifdef BCM_CNIC
3210 /**
3211 * move non eth FPs next to last eth FP
3212 * must be done in that order
3213 * FCOE_IDX < FWD_IDX < OOO_IDX
3214 */
3215
3216 /* move FCoE fp even NO_FCOE_FLAG is on */
3217 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
3218 #endif
3219 bp->num_queues -= delta;
3220 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3221 bp->num_queues + delta, bp->num_queues);
3222 }
3223
3224 return 0;
3225 }
3226
3227 void bnx2x_free_mem_bp(struct bnx2x *bp)
3228 {
3229 kfree(bp->fp);
3230 kfree(bp->msix_table);
3231 kfree(bp->ilt);
3232 }
3233
3234 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3235 {
3236 struct bnx2x_fastpath *fp;
3237 struct msix_entry *tbl;
3238 struct bnx2x_ilt *ilt;
3239 int msix_table_size = 0;
3240
3241 /*
3242 * The biggest MSI-X table we might need is as a maximum number of fast
3243 * path IGU SBs plus default SB (for PF).
3244 */
3245 msix_table_size = bp->igu_sb_cnt + 1;
3246
3247 /* fp array: RSS plus CNIC related L2 queues */
3248 fp = kzalloc((BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE) *
3249 sizeof(*fp), GFP_KERNEL);
3250 if (!fp)
3251 goto alloc_err;
3252 bp->fp = fp;
3253
3254 /* msix table */
3255 tbl = kzalloc(msix_table_size * sizeof(*tbl), GFP_KERNEL);
3256 if (!tbl)
3257 goto alloc_err;
3258 bp->msix_table = tbl;
3259
3260 /* ilt */
3261 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3262 if (!ilt)
3263 goto alloc_err;
3264 bp->ilt = ilt;
3265
3266 return 0;
3267 alloc_err:
3268 bnx2x_free_mem_bp(bp);
3269 return -ENOMEM;
3270
3271 }
3272
3273 int bnx2x_reload_if_running(struct net_device *dev)
3274 {
3275 struct bnx2x *bp = netdev_priv(dev);
3276
3277 if (unlikely(!netif_running(dev)))
3278 return 0;
3279
3280 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3281 return bnx2x_nic_load(bp, LOAD_NORMAL);
3282 }
3283
3284 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3285 {
3286 u32 sel_phy_idx = 0;
3287 if (bp->link_params.num_phys <= 1)
3288 return INT_PHY;
3289
3290 if (bp->link_vars.link_up) {
3291 sel_phy_idx = EXT_PHY1;
3292 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3293 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3294 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3295 sel_phy_idx = EXT_PHY2;
3296 } else {
3297
3298 switch (bnx2x_phy_selection(&bp->link_params)) {
3299 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3300 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3301 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3302 sel_phy_idx = EXT_PHY1;
3303 break;
3304 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3305 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3306 sel_phy_idx = EXT_PHY2;
3307 break;
3308 }
3309 }
3310
3311 return sel_phy_idx;
3312
3313 }
3314 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3315 {
3316 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3317 /*
3318 * The selected actived PHY is always after swapping (in case PHY
3319 * swapping is enabled). So when swapping is enabled, we need to reverse
3320 * the configuration
3321 */
3322
3323 if (bp->link_params.multi_phy_config &
3324 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3325 if (sel_phy_idx == EXT_PHY1)
3326 sel_phy_idx = EXT_PHY2;
3327 else if (sel_phy_idx == EXT_PHY2)
3328 sel_phy_idx = EXT_PHY1;
3329 }
3330 return LINK_CONFIG_IDX(sel_phy_idx);
3331 }
3332
3333 #if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3334 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3335 {
3336 struct bnx2x *bp = netdev_priv(dev);
3337 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3338
3339 switch (type) {
3340 case NETDEV_FCOE_WWNN:
3341 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3342 cp->fcoe_wwn_node_name_lo);
3343 break;
3344 case NETDEV_FCOE_WWPN:
3345 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3346 cp->fcoe_wwn_port_name_lo);
3347 break;
3348 default:
3349 return -EINVAL;
3350 }
3351
3352 return 0;
3353 }
3354 #endif
3355
3356 /* called with rtnl_lock */
3357 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3358 {
3359 struct bnx2x *bp = netdev_priv(dev);
3360
3361 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3362 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
3363 return -EAGAIN;
3364 }
3365
3366 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3367 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
3368 return -EINVAL;
3369
3370 /* This does not race with packet allocation
3371 * because the actual alloc size is
3372 * only updated as part of load
3373 */
3374 dev->mtu = new_mtu;
3375
3376 return bnx2x_reload_if_running(dev);
3377 }
3378
3379 u32 bnx2x_fix_features(struct net_device *dev, u32 features)
3380 {
3381 struct bnx2x *bp = netdev_priv(dev);
3382
3383 /* TPA requires Rx CSUM offloading */
3384 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa)
3385 features &= ~NETIF_F_LRO;
3386
3387 return features;
3388 }
3389
3390 int bnx2x_set_features(struct net_device *dev, u32 features)
3391 {
3392 struct bnx2x *bp = netdev_priv(dev);
3393 u32 flags = bp->flags;
3394 bool bnx2x_reload = false;
3395
3396 if (features & NETIF_F_LRO)
3397 flags |= TPA_ENABLE_FLAG;
3398 else
3399 flags &= ~TPA_ENABLE_FLAG;
3400
3401 if (features & NETIF_F_LOOPBACK) {
3402 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3403 bp->link_params.loopback_mode = LOOPBACK_BMAC;
3404 bnx2x_reload = true;
3405 }
3406 } else {
3407 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3408 bp->link_params.loopback_mode = LOOPBACK_NONE;
3409 bnx2x_reload = true;
3410 }
3411 }
3412
3413 if (flags ^ bp->flags) {
3414 bp->flags = flags;
3415 bnx2x_reload = true;
3416 }
3417
3418 if (bnx2x_reload) {
3419 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3420 return bnx2x_reload_if_running(dev);
3421 /* else: bnx2x_nic_load() will be called at end of recovery */
3422 }
3423
3424 return 0;
3425 }
3426
3427 void bnx2x_tx_timeout(struct net_device *dev)
3428 {
3429 struct bnx2x *bp = netdev_priv(dev);
3430
3431 #ifdef BNX2X_STOP_ON_ERROR
3432 if (!bp->panic)
3433 bnx2x_panic();
3434 #endif
3435
3436 smp_mb__before_clear_bit();
3437 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3438 smp_mb__after_clear_bit();
3439
3440 /* This allows the netif to be shutdown gracefully before resetting */
3441 schedule_delayed_work(&bp->sp_rtnl_task, 0);
3442 }
3443
3444 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3445 {
3446 struct net_device *dev = pci_get_drvdata(pdev);
3447 struct bnx2x *bp;
3448
3449 if (!dev) {
3450 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3451 return -ENODEV;
3452 }
3453 bp = netdev_priv(dev);
3454
3455 rtnl_lock();
3456
3457 pci_save_state(pdev);
3458
3459 if (!netif_running(dev)) {
3460 rtnl_unlock();
3461 return 0;
3462 }
3463
3464 netif_device_detach(dev);
3465
3466 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3467
3468 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3469
3470 rtnl_unlock();
3471
3472 return 0;
3473 }
3474
3475 int bnx2x_resume(struct pci_dev *pdev)
3476 {
3477 struct net_device *dev = pci_get_drvdata(pdev);
3478 struct bnx2x *bp;
3479 int rc;
3480
3481 if (!dev) {
3482 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3483 return -ENODEV;
3484 }
3485 bp = netdev_priv(dev);
3486
3487 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3488 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
3489 return -EAGAIN;
3490 }
3491
3492 rtnl_lock();
3493
3494 pci_restore_state(pdev);
3495
3496 if (!netif_running(dev)) {
3497 rtnl_unlock();
3498 return 0;
3499 }
3500
3501 bnx2x_set_power_state(bp, PCI_D0);
3502 netif_device_attach(dev);
3503
3504 /* Since the chip was reset, clear the FW sequence number */
3505 bp->fw_seq = 0;
3506 rc = bnx2x_nic_load(bp, LOAD_OPEN);
3507
3508 rtnl_unlock();
3509
3510 return rc;
3511 }
3512
3513
3514 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
3515 u32 cid)
3516 {
3517 /* ustorm cxt validation */
3518 cxt->ustorm_ag_context.cdu_usage =
3519 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3520 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
3521 /* xcontext validation */
3522 cxt->xstorm_ag_context.cdu_reserved =
3523 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3524 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
3525 }
3526
3527 static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
3528 u8 fw_sb_id, u8 sb_index,
3529 u8 ticks)
3530 {
3531
3532 u32 addr = BAR_CSTRORM_INTMEM +
3533 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
3534 REG_WR8(bp, addr, ticks);
3535 DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d ticks %d\n",
3536 port, fw_sb_id, sb_index, ticks);
3537 }
3538
3539 static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
3540 u16 fw_sb_id, u8 sb_index,
3541 u8 disable)
3542 {
3543 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
3544 u32 addr = BAR_CSTRORM_INTMEM +
3545 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
3546 u16 flags = REG_RD16(bp, addr);
3547 /* clear and set */
3548 flags &= ~HC_INDEX_DATA_HC_ENABLED;
3549 flags |= enable_flag;
3550 REG_WR16(bp, addr, flags);
3551 DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d disable %d\n",
3552 port, fw_sb_id, sb_index, disable);
3553 }
3554
3555 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
3556 u8 sb_index, u8 disable, u16 usec)
3557 {
3558 int port = BP_PORT(bp);
3559 u8 ticks = usec / BNX2X_BTR;
3560
3561 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
3562
3563 disable = disable ? 1 : (usec ? 0 : 1);
3564 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
3565 }
kernel source for telekom puls (alcatel/mediatek)